kubernetes · upodroid · Nov 30, 2025
diff --git a/infra/gcp/bash/ensure-prod-storage.sh b/infra/gcp/bash/ensure-prod-storage.sh
@@ -89,109 +89,6 @@ readonly PROD_PROJECT_DISABLED_SERVICES=(
     containerscanning.googleapis.com
 )
 
-# Regions for prod GCR.
-GCR_PROD_REGIONS=(us eu asia)
-# Regions for prod AR. gcloud artifacts locations list --format json | jq '.[] | select(.name!="europe" and .name!="asia" and .name!="us") | .name' -r | xargs
-AR_PROD_REGIONS=(asia-east1 asia-east2 asia-south1 asia-northeast1 asia-northeast2 australia-southeast1 europe-north1 europe-southwest1 europe-west1 europe-west10 europe-west12 europe-west2 europe-west3 europe-west4 europe-west8 europe-west9 southamerica-east1 southamerica-west1 us-central1 us-east1 us-east4 us-east5 us-south1 us-west1 us-west2 us-west3 us-west4)
-
-# Minimum time we expect to keep prod GCS artifacts.
-PROD_RETENTION="10y"
-
-# Make a prod GCR repository and grant access to it.
-#
-# $1: The GCP project name (GCR names == project names)
-function ensure_prod_gcr() {
-    if [ $# != 1 ] || [ -z "$1" ]; then
-        echo "ensure_prod_gcr(project) requires 1 argument" >&2
-        return 1
-    fi
-    local project="${1}"
-
-    color 6 "Ensuring prod GCR for regions: ${GCR_PROD_REGIONS[*]}"
-    for region in "${GCR_PROD_REGIONS[@]}"; do
-        local gcr_bucket="gs://${region}.artifacts.${project}.appspot.com"
-
-        color 3 "region: ${region}"
-        color 6 "Ensuring a GCR repo exists in region: ${region} for project: ${project}"
-        ensure_gcr_repo "${project}" "${region}"
-
-        color 6 "Ensuring GCR admins can admin GCR in region: ${region} for project: ${project}"
-        empower_gcr_admins "${project}" "${region}"
-
-        color 6 "Empowering image promoter for region: ${region} in project: ${project}"
-        empower_image_promoter "${project}" "${region}"
-
-        color 6 "Ensuring GCS access logs enabled for GCR bucket in region: ${region} in project: ${project}"
-        ensure_gcs_bucket_logging "${gcr_bucket}"
-    done 2>&1 | indent
-}
-
-# Make a prod AR repository and grant access to it.
-#
-# $1: The GCP project name (GCR names == project names)
-function ensure_prod_ar() {
-    if [ $# != 1 ] || [ -z "$1" ]; then
-        echo "ensure_prod_ar(project) requires 1 argument" >&2
-        return 1
-    fi
-    local project="${1}"
-    local serviceaccount
-
-    color 6 "Ensuring prod AR registry for locations: ${AR_PROD_REGIONS[*]}"
-    for region in "${AR_PROD_REGIONS[@]}"; do
-
-        color 3 "region: ${region}"
-        color 6 "Ensuring an AR repo exists in location: ${region} for project: ${project}"
-        ensure_ar_repo "${project}" "${region}"
-
-        color 6 "Ensuring GCR admins can admin AR in location: ${region} for project: ${project}"
-        empower_ar_admins "${project}" "${region}"
-
-        color 6 "Empowering image promoter with roles/artifactregistry.repoAdmin in project: ${project}"
-        serviceaccount=$(svc_acct_email "${project}" "${IMAGE_PROMOTER_SVCACCT}")
-        ensure_project_role_binding "${project}" "serviceAccount:$serviceaccount" "roles/artifactregistry.repoAdmin"
-    done 2>&1 | indent
-}
-
-# Make a prod GCS bucket and grant access to it.  We need whole buckets for
-# this because we want to grant minimal permissions, but there's no concept of
-# permissions on a "subdirectory" of a bucket.  If we had a GCS promoter akin
-# to the GCR promoter, we might have used a single bucket, but we don't have
-# that yet.
-#
-# $1: The GCP project to make the bucket
-# $2: The bucket, including gs:// prefix
-# $3: The group email to empower (optional)
-function ensure_prod_gcs_bucket() {
-    if [ $# -lt 2 ] || [ $# -gt 3 ] || [ -z "$1" ] || [ -z "$2" ]; then
-        echo "ensure_prod_gcs_bucket(project, bucket, [group]) requires 2 or 3 arguments" >&2
-        return 1
-    fi
-    local project="${1}"
-    local bucket="${2}"
-    local group="${3:-}"
-
-    color 6 "Ensuring the GCS bucket exists and is readable"
-    ensure_public_gcs_bucket "${project}" "${bucket}"
-
-    color 6 "Ensuring the bucket retention policy is set"
-    ensure_gcs_bucket_retention "${bucket}" "${PROD_RETENTION}"
-
-    color 6 "Empowering GCS admins"
-    empower_gcs_admins "${project}" "${bucket}"
-
-    color 6 "Empowering file promoter in project: ${project}"
-    empower_file_promoter "${project}" "${bucket}"
-
-    color 6 "Ensuring GCS access logs enabled for ${bucket} in project: ${project}"
-    ensure_gcs_bucket_logging "${bucket}"
-
-    if [ -n "${group}" ]; then
-        color 6 "Empowering ${group} to write to the bucket"
-        empower_group_to_write_gcs_bucket "${group}" "${bucket}"
-    fi
-}
-
 # Grant access to "fake prod" projects for tol testing
 # $1: The GCP project
 # $2: The googlegroups group

diff --git a/infra/gcp/terraform/k8s-artifacts-prod/buckets.tf b/infra/gcp/terraform/k8s-artifacts-prod/buckets.tf
@@ -0,0 +1,41 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+# module "gcb_bucket" {
+#   source  = "terraform-google-modules/cloud-storage/google//modules/simple_bucket"
+#   version = "~> 8.0"
+
+#   name       = "k8s-staging-images-gcb"
+#   project_id = module.project.project_id
+#   location   = "us"
+
+#   lifecycle_rules = [{
+#     action = {
+#       type = "Delete"
+#     }
+#     condition = {
+#       age        = 90 # 90d
+#       with_state = "ANY"
+#     }
+#   }]
+
+#   iam_members = [
+#     {
+#       role   = "roles/storage.admin"
+#       member = "serviceAccount:[email protected]"
+#     }
+#   ]
+# }
diff --git a/infra/gcp/terraform/k8s-artifacts-prod/iam.tf b/infra/gcp/terraform/k8s-artifacts-prod/iam.tf
@@ -0,0 +1,62 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+module "iam" {
+  source  = "terraform-google-modules/iam/google//modules/projects_iam"
+  version = "~> 8.2"
+
+  projects = [module.project.project_id]
+
+  mode = "authoritative"
+
+  bindings = {
+    "roles/artifactregistry.admin" = [
+      "group:[email protected]",
+    ]
+    "roles/artifactregistry.repoAdmin" = [
+      "serviceAccount:k8s-infra-gcr-promoter@k8s-artifacts-prod.iam.gserviceaccount.com"
+    ]
+    "roles/errorreporting.user" = [
+      "group:[email protected]",
+    ]
+    "roles/serviceusage.serviceUsageConsumer" = [
+      "group:[email protected]",
+    ]
+    "roles/viewer" = [
+      "group:[email protected]",
+      "group:[email protected]",
+    ]
+  }
+}
+
+module "audit_logs" {
+  source  = "terraform-google-modules/iam/google//modules/audit_config"
+  version = "~> 8.2"
+
+  project = module.project.project_id
+  audit_log_config = [
+    {
+      service          = "artifactregistry.googleapis.com"
+      log_type         = "DATA_READ"
+      exempted_members = null
+    },
+    {
+      service          = "artifactregistry.googleapis.com"
+      log_type         = "DATA_WRITE"
+      exempted_members = null
+    },
+  ]
+}
diff --git a/infra/gcp/terraform/k8s-artifacts-prod/main.tf b/infra/gcp/terraform/k8s-artifacts-prod/main.tf
@@ -0,0 +1,38 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+module "project" {
+  source  = "terraform-google-modules/project-factory/google"
+  version = "~> 18.2"
+
+  name            = "k8s-artifacts-prod"
+  project_id      = "k8s-artifacts-prod"
+  org_id          = "758905017065"
+  billing_account = "018801-93540E-22A20E"
+
+  # Sane project defaults
+  default_service_account     = "keep"
+  disable_services_on_destroy = false
+  create_project_sa           = false
+  random_project_id           = false
+  auto_create_network         = false
+
+
+  activate_apis = [
+    "artifactregistry.googleapis.com",
+    "run.googleapis.com",
+  ]
+}
diff --git a/infra/gcp/terraform/k8s-artifacts-prod/provider.tf b/infra/gcp/terraform/k8s-artifacts-prod/provider.tf
@@ -0,0 +1,35 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+terraform {
+  required_version = "1.13.5"
+
+  backend "gcs" {
+    bucket = "k8s-infra-terraform"
+    prefix = "k8s-artifacts-prod"
+  }
+
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 7.12"
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "~> 7.12"
+    }
+  }
+}
diff --git a/infra/gcp/terraform/k8s-artifacts-prod/registries.tf b/infra/gcp/terraform/k8s-artifacts-prod/registries.tf
@@ -0,0 +1,89 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+locals {
+  // We want to have a registry in every location except the multi region ones
+  // gcloud artifacts locations list --format json | jq '.[] | select(.name!="europe" and .name!="asia" and .name!="us") | .name' | awk '{print $0","}' | --version-sort
+  registries = [
+    "africa-south1",
+    # "asia-east1",
+    "asia-east2",
+    # "asia-northeast1",
+    # "asia-northeast2",
+    "asia-northeast3",
+    # "asia-south1",
+    "asia-south2",
+    "asia-southeast1",
+    "asia-southeast2",
+    # "australia-southeast1",
+    "australia-southeast2",
+    "europe-central2",
+    # "europe-north1",
+    "europe-north2",
+    # "europe-southwest1",
+    # "europe-west1",
+    # "europe-west2",
+    # "europe-west3",
+    # "europe-west4",
+    "europe-west6",
+    # "europe-west8",
+    # "europe-west9",
+    # "europe-west10",
+    "europe-west12",
+    "me-central1",
+    # "me-central2", # THIS REGION REQUIRES SUPPORT APPROVAL WHICH I STARTED
+    "me-west1",
+    "northamerica-northeast1",
+    "northamerica-northeast2",
+    "northamerica-south1",
+    "southamerica-east1",
+    # "southamerica-west1",
+    # "us-central1",
+    # "us-east1",
+    # "us-east4",
+    # "us-east5",
+    # "us-south1",
+    # "us-west1",
+    # "us-west2",
+    "us-west3",
+    "us-west4",
+  ]
+
+}
+
+module "artifact_registry" {
+  for_each = toset(local.registries)
+  source   = "GoogleCloudPlatform/artifact-registry/google"
+  version  = "~> 0.2"
+
+  project_id    = module.project.project_id
+  location      = each.key
+  format        = "DOCKER"
+  repository_id = "images"
+  # docker_config = {
+  #   immutable_tags = true // APPLY THIS SOON
+  # }
+  members = {
+    readers = ["allUsers"],
+  }
+}
+
+# # DELETE THIS AFTER ALL THE REGISTRIES ARE IMPORTED
+# import {
+#   for_each = toset(local.registries)
+#   to       = module.artifact_registry[each.key].google_artifact_registry_repository.repo
+#   id       = "k8s-artifacts-prod/${each.key}/images"
+# }