[WIP] expand to all S3 regions for registry.k8s.io

infra/aws/terraform/registry.k8s.io/README.md

@@ -1,41 +1,17 @@
-# infra/aws/terraform/registry.k8s.io
+# AWS Infrastructure for registry.k8s.io
 
-Legacy Terraform to provision AWS infra for supporting registry.k8s.io
-This has been replaced by https://github.com/kubernetes/k8s.io/tree/main/infra/gcp/terraform/k8s-infra-oci-proxy-prod
+The resources created here are used by [archeio](https://github.com/kubernetes/registry.k8s.io/tree/main/cmd/archeio)
 
 ## Goals
 
 - create publicly-readable regional S3 buckets for serving Kubernetes container images
+- use S3 replication rules to fan out the blob replication to all the S3 buckets
 - create an IAM role to provide write access into each bucket
-- provide a way for testing the buckets
-
-## Applying the Terraform
-
-In order to apply the Terraform in production, the variable of _prefix_ must be set to `prod-`, i.e
-
-```bash
-terraform apply -var prefix=prod-
-```
-
-## Testing
-
-The variable _prefix_ can be set to prefix all variables with a value
-
-```bash
-terraform apply -var prefix=test-
-```
-
-## State management
-
-The state of the Terraform was currently stored in a private bucket in the CNCF account
-
-## Notes
-
-- All provider blocks assumed the role of registry.k8s.io_s3admin inside of the registry.k8s.io account (513428760722)
 
 ## Extending regions
 
 Supporting more AWS regions is simple, the steps are:
 
-1. add a new _aws_ provider with a region and alias in the [providers.tf](./providers.tf) file
-2. add a new module block for the region in the [main.tf](./main.tf) file, including the correct region name for all three values (module name, provider+alias and region variable)
+1. run the command `aws ec2 describe-regions --all-regions --query "Regions[].RegionName" --output json | jq .[] | awk '{print $0","}' | sort --version-sort` to get full set of AWS regions and update the main.tf file
+1. Login to the management account and enable the new region for this account(513428760722) using AWS Organizations
+1. The closest region to the AWS China regions `cn-northwest-1` & `cn-north-1` is `ap-east-1` (Hong Kong), we don't have access to the aws-cn partition.

infra/aws/terraform/registry.k8s.io/atlantis.config

@@ -0,0 +1,3 @@
+assume_role = {
+  role_arn = "arn:aws:iam::513428760722:role/OrganizationAccountAccessRole"
+}

infra/aws/terraform/registry.k8s.io/atlantis.tfvars

@@ -0,0 +1 @@
+atlantis_role_arn = "arn:aws:iam::513428760722:role/OrganizationAccountAccessRole"

infra/aws/terraform/registry.k8s.io/main.tf

@@ -14,168 +14,70 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// prefix prefixes every resource so that the resources
-// can be created without using the same names. Useful
-// for testing and staging
-
-variable "prefix" {
-  type        = string
-  default     = "test-"
-  description = "The prefix for all resources"
-
-  validation {
-    condition     = can(regex(".*-$|^$", var.prefix))
-    error_message = "The string must end with a hyphen or be empty."
-  }
-}
-
-module "us-west-1" {
-  source = "./s3"
-
-  providers = {
-    aws = aws.us-west-1
-  }
-
-  prefix = var.prefix
-}
-
-module "us-west-2" {
-  source = "./s3"
-
-  providers = {
-    aws = aws.us-west-2
-  }
-
-  prefix = var.prefix
-}
-
-module "us-east-1" {
-  source = "./s3"
-
-  providers = {
-    aws = aws.us-east-1
-  }
-
-  prefix = var.prefix
+locals {
+  // aws ec2 describe-regions --all-regions --query "Regions[].RegionName" --output json | jq .[] | awk '{print $0","}' | sort --version-sort
+  regions = [
+    "af-south-1",
+    # "ap-east-1",
+    # "ap-east-2",
+    "ap-northeast-1",
+    # "ap-northeast-2",
+    # "ap-northeast-3",
+    "ap-southeast-1",
+    "ap-southeast-2",
+    # "ap-southeast-3",
+    # "ap-southeast-4",
+    # "ap-southeast-5",
+    # "ap-southeast-6",
+    # "ap-southeast-7",
+    "ap-south-1",
+    # "ap-south-2",
+    # "ca-central-1",
+    # "ca-west-1",
+    "eu-central-1",
+    # "eu-central-2",
+    # "eu-north-1",
+    "eu-south-1",
+    # "eu-south-2",
+    "eu-west-1",
+    # "eu-west-2",
+    "eu-west-3",
+    # "il-central-1",
+    # "me-central-1",
+    # "me-south-1",
+    # "mx-central-1",
+    # "sa-east-1",
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2",
+  ]
 }
 
 module "us-east-2" {
-  source = "./s3"
-
-  providers = {
-    aws = aws.us-east-2
-  }
-
-  prefix = var.prefix
-
+  // This is the authoritative bucket where objects are initially uploaded to
+  source                      = "./s3"
+  region                      = "us-east-2"
+  prefix                      = var.prefix
   s3_replication_iam_role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3writer"
-
   s3_replication_rules = [
+    for idx, region in local.regions :
     {
-      id                               = "registry-k8s-io-us-east-2-to-registry-k8s-io-us-west-1"
-      status                           = "Enabled"
-      priority                         = 1
-      destination_bucket_arn           = "arn:aws:s3:::${var.prefix}registry-k8s-io-us-west-1"
-      destination_bucket_storage_class = "STANDARD"
-    },
-    {
-      id                               = "registry-k8s-io-us-east-2-to-registry-k8s-io-us-west-2"
-      status                           = "Enabled"
-      priority                         = 2
-      destination_bucket_arn           = "arn:aws:s3:::${var.prefix}registry-k8s-io-us-west-2"
-      destination_bucket_storage_class = "STANDARD"
-    },
-    {
-      id                               = "registry-k8s-io-us-east-2-to-registry-k8s-io-us-east-1"
+      id                               = "registry-k8s-io-us-east-2-to-registry-k8s-io-${region}"
       status                           = "Enabled"
-      priority                         = 3
-      destination_bucket_arn           = "arn:aws:s3:::${var.prefix}registry-k8s-io-us-east-1"
+      priority                         = idx + 1 # priorities start at 1
+      destination_bucket_arn           = module.s3_buckets[region].bucket_arn
       destination_bucket_storage_class = "STANDARD"
-    },
-    {
-      id                               = "registry-k8s-io-us-east-2-to-registry-k8s-io-eu-west-1"
-      status                           = "Enabled"
-      priority                         = 4
-      destination_bucket_arn           = "arn:aws:s3:::${var.prefix}registry-k8s-io-eu-west-1"
-      destination_bucket_storage_class = "STANDARD"
-    },
-    {
-      id                               = "registry-k8s-io-us-east-2-to-registry-k8s-io-eu-central-1"
-      status                           = "Enabled"
-      priority                         = 5
-      destination_bucket_arn           = "arn:aws:s3:::${var.prefix}registry-k8s-io-eu-central-1"
-      destination_bucket_storage_class = "STANDARD"
-    },
-    {
-      id                               = "registry-k8s-io-us-east-2-to-registry-k8s-io-ap-southeast-1"
-      status                           = "Enabled"
-      priority                         = 6
-      destination_bucket_arn           = "arn:aws:s3:::${var.prefix}registry-k8s-io-ap-southeast-1"
-      destination_bucket_storage_class = "STANDARD"
-    },
-    {
-      id                               = "registry-k8s-io-us-east-2-to-registry-k8s-io-ap-northeast-1"
-      status                           = "Enabled"
-      priority                         = 7
-      destination_bucket_arn           = "arn:aws:s3:::${var.prefix}registry-k8s-io-ap-northeast-1"
-      destination_bucket_storage_class = "STANDARD"
-    },
-    {
-      id                               = "registry-k8s-io-us-east-2-to-registry-k8s-io-ap-south-1"
-      status                           = "Enabled"
-      priority                         = 8
-      destination_bucket_arn           = "arn:aws:s3:::${var.prefix}registry-k8s-io-ap-south-1"
-      destination_bucket_storage_class = "STANDARD"
-    },
+    }
+    // exclude the source region itself
+    if region != "us-east-2"
   ]
 }
 
-module "eu-west-1" {
-  source = "./s3"
-
-  providers = {
-    aws = aws.eu-west-1
-  }
-
-  prefix = var.prefix
-}
-
-module "eu-central-1" {
-  source = "./s3"
-
-  providers = {
-    aws = aws.eu-central-1
-  }
-
-  prefix = var.prefix
-}
-
-module "ap-southeast-1" {
-  source = "./s3"
-
-  providers = {
-    aws = aws.ap-southeast-1
-  }
-
-  prefix = var.prefix
-}
-
-module "ap-northeast-1" {
-  source = "./s3"
-
-  providers = {
-    aws = aws.ap-northeast-1
-  }
-
-  prefix = var.prefix
-}
-
-module "ap-south-1" {
-  source = "./s3"
-
-  providers = {
-    aws = aws.ap-south-1
-  }
+module "s3_buckets" {
+  for_each = setsubtract(toset(local.regions), ["us-east-2"])
+  source   = "./s3"
+  region   = each.key
 
   prefix = var.prefix
 }

infra/aws/terraform/registry.k8s.io/providers.tf

@@ -16,110 +16,24 @@ limitations under the License.
 
 terraform {
   backend "s3" {
-    bucket = "registry-k8s-io-tfstate"
+    bucket = "k8s-infra-registry-k8s-io-tfstate"
     key    = "terraform.tfstate"
     region = "us-east-2"
   }
 
-  required_version = ">= 1.0.0"
+  required_version = ">= 1.13.0"
 
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 4.0"
+      version = "~> 6.22"
     }
   }
 }
 
-# The role_arn (arn:aws:iam::513428760722:role/registry.k8s.io_s3admin)
-# used in each provider block is managed in
-# https://github.com/cncf-infra/aws-infra/blob/2ac2e63c162134a9e6036d84beee2d5adf6b4ff2/terraform/iam/main.tf
-
-provider "aws" {
-  region = "us-west-1"
-
-  assume_role {
-    role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin"
-  }
-}
-
-provider "aws" {
-  alias  = "us-west-1"
-  region = "us-west-1"
-
-  assume_role {
-    role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin"
-  }
-}
-
-provider "aws" {
-  alias  = "us-west-2"
-  region = "us-west-2"
-
-  assume_role {
-    role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin"
-  }
-}
-
-provider "aws" {
-  alias  = "us-east-1"
-  region = "us-east-1"
-
-  assume_role {
-    role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin"
-  }
-}
-
 provider "aws" {
-  alias  = "us-east-2"
   region = "us-east-2"
-
-  assume_role {
-    role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin"
-  }
-}
-
-provider "aws" {
-  alias  = "eu-west-1"
-  region = "eu-west-1"
-
-  assume_role {
-    role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin"
-  }
-}
-
-provider "aws" {
-  alias  = "eu-central-1"
-  region = "eu-central-1"
-
-  assume_role {
-    role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin"
-  }
-}
-
-provider "aws" {
-  alias  = "ap-southeast-1"
-  region = "ap-southeast-1"
-
-  assume_role {
-    role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin"
-  }
-}
-
-provider "aws" {
-  alias  = "ap-northeast-1"
-  region = "ap-northeast-1"
-
-  assume_role {
-    role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin"
-  }
-}
-
-provider "aws" {
-  alias  = "ap-south-1"
-  region = "ap-south-1"
-
   assume_role {
-    role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin"
+    role_arn = var.atlantis_role_arn
   }
 }