feat: Workspace 1049

Pipfile

@@ -35,7 +35,7 @@ codeforlife-portal = "==8.9.9" # TODO: remove
 rapid-router = "==7.6.10" # TODO: remove
 phonenumbers = "==8.12.12" # TODO: remove
 google-auth = "==2.40.3"
-google-cloud-storage = "==3.4.0"
+google-cloud-bigquery = "==3.38.0"
 
 [dev-packages]
 celery-types = "==0.23.0"

codeforlife/auth.py

@@ -0,0 +1,90 @@
+"""
+© Ocado Group
+Created on 04/12/2025 at 18:58:44(+00:00).
+
+Authentication credentials.
+"""
+
+import typing as t
+
+from boto3 import Session as AwsSession
+from django.conf import settings
+from google.auth.aws import (
+    AwsSecurityCredentials,
+    AwsSecurityCredentialsSupplier,
+)
+from google.auth.aws import Credentials as AwsCredentials
+from google.auth.credentials import Credentials
+from google.oauth2.service_account import (
+    Credentials as GcpServiceAccountCredentials,
+)
+
+
+class AwsSessionSecurityCredentialsSupplier(AwsSecurityCredentialsSupplier):
+    """Supplies AWS security credentials from the current boto3 session."""
+
+    def get_aws_region(self, _, __):
+        return settings.AWS_REGION
+
+    def get_aws_security_credentials(self, _, __) -> AwsSecurityCredentials:
+        aws_credentials = AwsSession().get_credentials()
+        assert aws_credentials
+
+        aws_read_only_credentials = aws_credentials.get_frozen_credentials()
+        assert aws_read_only_credentials.access_key
+        assert aws_read_only_credentials.secret_key
+        assert aws_read_only_credentials.token
+
+        return AwsSecurityCredentials(
+            access_key_id=aws_read_only_credentials.access_key,
+            secret_access_key=aws_read_only_credentials.secret_key,
+            session_token=aws_read_only_credentials.token,
+        )
+
+
+# pylint: disable-next=abstract-method,too-many-ancestors
+class GcpWifCredentials(AwsCredentials):
+    """Workload Identity Federation credentials for GCP using AWS IAM roles."""
+
+    def __init__(self, token_lifetime_seconds: int = 600):
+        super().__init__(
+            subject_token_type="urn:ietf:params:aws:token-type:aws4_request",
+            audience=settings.GCP_WIF_AUDIENCE,
+            universe_domain="googleapis.com",
+            token_url="https://sts.googleapis.com/v1/token",
+            service_account_impersonation_url=(
+                "https://iamcredentials.googleapis.com/v1/projects/-/"
+                f"serviceAccounts/{settings.GCP_WIF_SERVICE_ACCOUNT}"
+                ":generateAccessToken"
+            ),
+            service_account_impersonation_options={
+                "token_lifetime_seconds": token_lifetime_seconds
+            },
+            aws_security_credentials_supplier=(
+                AwsSessionSecurityCredentialsSupplier()
+            ),
+        )
+
+
+def get_gcp_service_account_credentials(
+    token_lifetime_seconds: int = 600,
+    service_account_json: t.Optional[str] = None,
+) -> Credentials:
+    """Get GCP service account credentials.
+
+    Args:
+        token_lifetime_seconds: The lifetime of the token in seconds.
+        service_account_json: The path to the service account JSON file.
+
+    Returns:
+        The GCP service account credentials.
+    """
+    if settings.ENV != "local":
+        return GcpWifCredentials(token_lifetime_seconds=token_lifetime_seconds)
+    assert (
+        service_account_json
+    ), "Service account JSON file path must be provided in local environment."
+
+    return GcpServiceAccountCredentials.from_service_account_file(
+        service_account_json
+    )

codeforlife/settings/custom.py

@@ -140,12 +140,7 @@ def get_redis_url():
     "GOOGLE_CLOUD_PROJECT_ID", "decent-digit-629"
 )
 
-# The bucket on Google Cloud Storage used to export data to BigQuery and the
-# service account to impersonate which has access to the bucket.
-GOOGLE_CLOUD_STORAGE_BUCKET_NAME = os.getenv(
-    "GOOGLE_CLOUD_STORAGE_BUCKET_NAME", "REPLACE_ME"
-)
-GOOGLE_CLOUD_STORAGE_SERVICE_ACCOUNT_NAME = (
-    os.getenv("GOOGLE_CLOUD_STORAGE_SERVICE_ACCOUNT_NAME", "REPLACE_ME")
-    + f"@{GOOGLE_CLOUD_PROJECT_ID}.iam.gserviceaccount.com"
+# The ID of our BigQuery dataset.
+GOOGLE_CLOUD_BIGQUERY_DATASET_ID = os.getenv(
+    "GOOGLE_CLOUD_BIGQUERY_DATASET_ID", "REPLACE_ME"
 )

codeforlife/settings/otp.py

@@ -60,6 +60,11 @@
     f"{AWS_S3_APP_FOLDER}/elasticacheMetadata/{CACHE_CLUSTER_ID}.dbdata"
 )
 
+# GCP
+
+GCP_WIF_AUDIENCE = os.getenv("GCP_WIF_AUDIENCE")
+GCP_WIF_SERVICE_ACCOUNT = os.getenv("GCP_WIF_SERVICE_ACCOUNT")
+
 # SQS
 
 SQS_URL = os.getenv("SQS_URL")

codeforlife/tasks/__init__.py

@@ -3,5 +3,5 @@
 Created on 06/10/2025 at 17:14:31(+01:00).
 """
 
-from .data_warehouse import DataWarehouseTask
+from .bigquery import BigQueryTask
 from .utils import get_local_sqs_url, get_task_name, shared_task