From c5896a1c44b9e32c4882909b69bb868a9c53b0ba Mon Sep 17 00:00:00 2001 From: upodroid Date: Mon, 1 Dec 2025 01:53:12 +0300 Subject: [PATCH] expand to all S3 regions for registry.k8s.io --- infra/aws/terraform/registry.k8s.io/README.md | 36 +--- .../terraform/registry.k8s.io/atlantis.config | 3 + .../terraform/registry.k8s.io/atlantis.tfvars | 1 + infra/aws/terraform/registry.k8s.io/main.tf | 204 +++++------------- .../terraform/registry.k8s.io/providers.tf | 94 +------- .../terraform/registry.k8s.io/s3/bucket.tf | 23 +- .../terraform/registry.k8s.io/s3/output.tf | 3 + .../terraform/registry.k8s.io/s3/variables.tf | 8 +- .../{s3/data.tf => variables.tf} | 18 +- 9 files changed, 108 insertions(+), 282 deletions(-) create mode 100644 infra/aws/terraform/registry.k8s.io/atlantis.config create mode 100644 infra/aws/terraform/registry.k8s.io/atlantis.tfvars create mode 100644 infra/aws/terraform/registry.k8s.io/s3/output.tf rename infra/aws/terraform/registry.k8s.io/{s3/data.tf => variables.tf} (51%) diff --git a/infra/aws/terraform/registry.k8s.io/README.md b/infra/aws/terraform/registry.k8s.io/README.md index 75aad136ad4..ce700e37abf 100644 --- a/infra/aws/terraform/registry.k8s.io/README.md +++ b/infra/aws/terraform/registry.k8s.io/README.md @@ -1,41 +1,17 @@ -# infra/aws/terraform/registry.k8s.io +# AWS Infrastructure for registry.k8s.io -Legacy Terraform to provision AWS infra for supporting registry.k8s.io -This has been replaced by https://github.com/kubernetes/k8s.io/tree/main/infra/gcp/terraform/k8s-infra-oci-proxy-prod +The resources created here are used by [archeio](https://github.com/kubernetes/registry.k8s.io/tree/main/cmd/archeio) ## Goals - create publicly-readable regional S3 buckets for serving Kubernetes container images +- use S3 replication rules to fan out the blob replication to all the S3 buckets - create an IAM role to provide write access into each bucket -- provide a way for testing the buckets - -## Applying the Terraform - -In order to apply the Terraform in production, the variable of _prefix_ must be set to `prod-`, i.e - -```bash -terraform apply -var prefix=prod- -``` - -## Testing - -The variable _prefix_ can be set to prefix all variables with a value - -```bash -terraform apply -var prefix=test- -``` - -## State management - -The state of the Terraform was currently stored in a private bucket in the CNCF account - -## Notes - -- All provider blocks assumed the role of registry.k8s.io_s3admin inside of the registry.k8s.io account (513428760722) ## Extending regions Supporting more AWS regions is simple, the steps are: -1. add a new _aws_ provider with a region and alias in the [providers.tf](./providers.tf) file -2. add a new module block for the region in the [main.tf](./main.tf) file, including the correct region name for all three values (module name, provider+alias and region variable) +1. run the command `aws ec2 describe-regions --all-regions --query "Regions[].RegionName" --output json | jq .[] | awk '{print $0","}' | sort --version-sort` to get full set of AWS regions and update the main.tf file +1. Login to the management account and enable the new region for this account(513428760722) using AWS Organizations +1. The closest region to the AWS China regions `cn-northwest-1` & `cn-north-1` is `ap-east-1` (Hong Kong), we don't have access to the aws-cn partition. diff --git a/infra/aws/terraform/registry.k8s.io/atlantis.config b/infra/aws/terraform/registry.k8s.io/atlantis.config new file mode 100644 index 00000000000..4519548d3ed --- /dev/null +++ b/infra/aws/terraform/registry.k8s.io/atlantis.config @@ -0,0 +1,3 @@ +assume_role = { + role_arn = "arn:aws:iam::513428760722:role/OrganizationAccountAccessRole" +} diff --git a/infra/aws/terraform/registry.k8s.io/atlantis.tfvars b/infra/aws/terraform/registry.k8s.io/atlantis.tfvars new file mode 100644 index 00000000000..bde1ce78833 --- /dev/null +++ b/infra/aws/terraform/registry.k8s.io/atlantis.tfvars @@ -0,0 +1 @@ +atlantis_role_arn = "arn:aws:iam::513428760722:role/OrganizationAccountAccessRole" diff --git a/infra/aws/terraform/registry.k8s.io/main.tf b/infra/aws/terraform/registry.k8s.io/main.tf index 4e7b964ea11..b99d2841322 100644 --- a/infra/aws/terraform/registry.k8s.io/main.tf +++ b/infra/aws/terraform/registry.k8s.io/main.tf @@ -14,168 +14,70 @@ See the License for the specific language governing permissions and limitations under the License. */ -// prefix prefixes every resource so that the resources -// can be created without using the same names. Useful -// for testing and staging - -variable "prefix" { - type = string - default = "test-" - description = "The prefix for all resources" - - validation { - condition = can(regex(".*-$|^$", var.prefix)) - error_message = "The string must end with a hyphen or be empty." - } -} - -module "us-west-1" { - source = "./s3" - - providers = { - aws = aws.us-west-1 - } - - prefix = var.prefix -} - -module "us-west-2" { - source = "./s3" - - providers = { - aws = aws.us-west-2 - } - - prefix = var.prefix -} - -module "us-east-1" { - source = "./s3" - - providers = { - aws = aws.us-east-1 - } - - prefix = var.prefix +locals { + // aws ec2 describe-regions --all-regions --query "Regions[].RegionName" --output json | jq .[] | awk '{print $0","}' | sort --version-sort + regions = [ + "af-south-1", + # "ap-east-1", + # "ap-east-2", + "ap-northeast-1", + # "ap-northeast-2", + # "ap-northeast-3", + "ap-southeast-1", + "ap-southeast-2", + # "ap-southeast-3", + # "ap-southeast-4", + # "ap-southeast-5", + # "ap-southeast-6", + # "ap-southeast-7", + "ap-south-1", + # "ap-south-2", + # "ca-central-1", + # "ca-west-1", + "eu-central-1", + # "eu-central-2", + # "eu-north-1", + "eu-south-1", + # "eu-south-2", + "eu-west-1", + # "eu-west-2", + "eu-west-3", + # "il-central-1", + # "me-central-1", + # "me-south-1", + # "mx-central-1", + # "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + ] } module "us-east-2" { - source = "./s3" - - providers = { - aws = aws.us-east-2 - } - - prefix = var.prefix - + // This is the authoritative bucket where objects are initially uploaded to + source = "./s3" + region = "us-east-2" + prefix = var.prefix s3_replication_iam_role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3writer" - s3_replication_rules = [ + for idx, region in local.regions : { - id = "registry-k8s-io-us-east-2-to-registry-k8s-io-us-west-1" - status = "Enabled" - priority = 1 - destination_bucket_arn = "arn:aws:s3:::${var.prefix}registry-k8s-io-us-west-1" - destination_bucket_storage_class = "STANDARD" - }, - { - id = "registry-k8s-io-us-east-2-to-registry-k8s-io-us-west-2" - status = "Enabled" - priority = 2 - destination_bucket_arn = "arn:aws:s3:::${var.prefix}registry-k8s-io-us-west-2" - destination_bucket_storage_class = "STANDARD" - }, - { - id = "registry-k8s-io-us-east-2-to-registry-k8s-io-us-east-1" + id = "registry-k8s-io-us-east-2-to-registry-k8s-io-${region}" status = "Enabled" - priority = 3 - destination_bucket_arn = "arn:aws:s3:::${var.prefix}registry-k8s-io-us-east-1" + priority = idx + 1 # priorities start at 1 + destination_bucket_arn = module.s3_buckets[region].bucket_arn destination_bucket_storage_class = "STANDARD" - }, - { - id = "registry-k8s-io-us-east-2-to-registry-k8s-io-eu-west-1" - status = "Enabled" - priority = 4 - destination_bucket_arn = "arn:aws:s3:::${var.prefix}registry-k8s-io-eu-west-1" - destination_bucket_storage_class = "STANDARD" - }, - { - id = "registry-k8s-io-us-east-2-to-registry-k8s-io-eu-central-1" - status = "Enabled" - priority = 5 - destination_bucket_arn = "arn:aws:s3:::${var.prefix}registry-k8s-io-eu-central-1" - destination_bucket_storage_class = "STANDARD" - }, - { - id = "registry-k8s-io-us-east-2-to-registry-k8s-io-ap-southeast-1" - status = "Enabled" - priority = 6 - destination_bucket_arn = "arn:aws:s3:::${var.prefix}registry-k8s-io-ap-southeast-1" - destination_bucket_storage_class = "STANDARD" - }, - { - id = "registry-k8s-io-us-east-2-to-registry-k8s-io-ap-northeast-1" - status = "Enabled" - priority = 7 - destination_bucket_arn = "arn:aws:s3:::${var.prefix}registry-k8s-io-ap-northeast-1" - destination_bucket_storage_class = "STANDARD" - }, - { - id = "registry-k8s-io-us-east-2-to-registry-k8s-io-ap-south-1" - status = "Enabled" - priority = 8 - destination_bucket_arn = "arn:aws:s3:::${var.prefix}registry-k8s-io-ap-south-1" - destination_bucket_storage_class = "STANDARD" - }, + } + // exclude the source region itself + if region != "us-east-2" ] } -module "eu-west-1" { - source = "./s3" - - providers = { - aws = aws.eu-west-1 - } - - prefix = var.prefix -} - -module "eu-central-1" { - source = "./s3" - - providers = { - aws = aws.eu-central-1 - } - - prefix = var.prefix -} - -module "ap-southeast-1" { - source = "./s3" - - providers = { - aws = aws.ap-southeast-1 - } - - prefix = var.prefix -} - -module "ap-northeast-1" { - source = "./s3" - - providers = { - aws = aws.ap-northeast-1 - } - - prefix = var.prefix -} - -module "ap-south-1" { - source = "./s3" - - providers = { - aws = aws.ap-south-1 - } +module "s3_buckets" { + for_each = setsubtract(toset(local.regions), ["us-east-2"]) + source = "./s3" + region = each.key prefix = var.prefix } diff --git a/infra/aws/terraform/registry.k8s.io/providers.tf b/infra/aws/terraform/registry.k8s.io/providers.tf index 6a8712e686b..a9a0d2547cc 100644 --- a/infra/aws/terraform/registry.k8s.io/providers.tf +++ b/infra/aws/terraform/registry.k8s.io/providers.tf @@ -16,110 +16,24 @@ limitations under the License. terraform { backend "s3" { - bucket = "registry-k8s-io-tfstate" + bucket = "k8s-infra-registry-k8s-io-tfstate" key = "terraform.tfstate" region = "us-east-2" } - required_version = ">= 1.0.0" + required_version = ">= 1.13.0" required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.0" + version = "~> 6.22" } } } -# The role_arn (arn:aws:iam::513428760722:role/registry.k8s.io_s3admin) -# used in each provider block is managed in -# https://github.com/cncf-infra/aws-infra/blob/2ac2e63c162134a9e6036d84beee2d5adf6b4ff2/terraform/iam/main.tf - -provider "aws" { - region = "us-west-1" - - assume_role { - role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin" - } -} - -provider "aws" { - alias = "us-west-1" - region = "us-west-1" - - assume_role { - role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin" - } -} - -provider "aws" { - alias = "us-west-2" - region = "us-west-2" - - assume_role { - role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin" - } -} - -provider "aws" { - alias = "us-east-1" - region = "us-east-1" - - assume_role { - role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin" - } -} - provider "aws" { - alias = "us-east-2" region = "us-east-2" - - assume_role { - role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin" - } -} - -provider "aws" { - alias = "eu-west-1" - region = "eu-west-1" - - assume_role { - role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin" - } -} - -provider "aws" { - alias = "eu-central-1" - region = "eu-central-1" - - assume_role { - role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin" - } -} - -provider "aws" { - alias = "ap-southeast-1" - region = "ap-southeast-1" - - assume_role { - role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin" - } -} - -provider "aws" { - alias = "ap-northeast-1" - region = "ap-northeast-1" - - assume_role { - role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin" - } -} - -provider "aws" { - alias = "ap-south-1" - region = "ap-south-1" - assume_role { - role_arn = "arn:aws:iam::513428760722:role/registry.k8s.io_s3admin" + role_arn = var.atlantis_role_arn } } diff --git a/infra/aws/terraform/registry.k8s.io/s3/bucket.tf b/infra/aws/terraform/registry.k8s.io/s3/bucket.tf index 4c322ff9aed..fe6d9a037e6 100644 --- a/infra/aws/terraform/registry.k8s.io/s3/bucket.tf +++ b/infra/aws/terraform/registry.k8s.io/s3/bucket.tf @@ -15,16 +15,23 @@ limitations under the License. */ resource "aws_s3_bucket" "registry-k8s-io" { - bucket = "${var.prefix}registry-k8s-io-${data.aws_region.current.name}" + bucket = "${var.prefix}registry-k8s-io-${var.region}" + region = var.region } -resource "aws_s3_bucket_acl" "registry-k8s-io" { +resource "aws_s3_bucket_public_access_block" "public" { bucket = aws_s3_bucket.registry-k8s-io.bucket - acl = "public-read" + region = var.region + + block_public_acls = false + block_public_policy = false + ignore_public_acls = false + restrict_public_buckets = false } resource "aws_s3_bucket_policy" "registry-k8s-io-public-read" { - bucket = aws_s3_bucket.registry-k8s-io.bucket + bucket = aws_s3_bucket_public_access_block.public.id + region = var.region policy = jsonencode({ "Id" : "Public-Access", @@ -60,13 +67,13 @@ resource "aws_s3_bucket_policy" "registry-k8s-io-public-read" { resource "aws_s3_bucket_ownership_controls" "registry-k8s-io" { bucket = aws_s3_bucket.registry-k8s-io.bucket + region = var.region rule { object_ownership = "BucketOwnerEnforced" } depends_on = [ aws_s3_bucket.registry-k8s-io, - aws_s3_bucket_acl.registry-k8s-io, aws_s3_bucket_policy.registry-k8s-io-public-read ] } @@ -74,13 +81,15 @@ resource "aws_s3_bucket_ownership_controls" "registry-k8s-io" { # Versioning must be enabled for S3 replication resource "aws_s3_bucket_versioning" "registry-k8s-io" { bucket = aws_s3_bucket.registry-k8s-io.id + region = var.region versioning_configuration { status = "Enabled" } } resource "aws_s3_bucket_replication_configuration" "registry-k8s-io" { - count = length(var.s3_replication_rules) > 0 ? 1 : 0 + count = length(var.s3_replication_rules) > 0 ? 1 : 0 + region = var.region # Must have bucket versioning enabled first depends_on = [aws_s3_bucket_versioning.registry-k8s-io] @@ -107,8 +116,6 @@ resource "aws_s3_bucket_replication_configuration" "registry-k8s-io" { status = "Enabled" } - - destination { bucket = rule.value.destination_bucket_arn storage_class = rule.value.destination_bucket_storage_class diff --git a/infra/aws/terraform/registry.k8s.io/s3/output.tf b/infra/aws/terraform/registry.k8s.io/s3/output.tf new file mode 100644 index 00000000000..d8ee40e136d --- /dev/null +++ b/infra/aws/terraform/registry.k8s.io/s3/output.tf @@ -0,0 +1,3 @@ +output "bucket_arn" { + value = aws_s3_bucket.registry-k8s-io.arn +} diff --git a/infra/aws/terraform/registry.k8s.io/s3/variables.tf b/infra/aws/terraform/registry.k8s.io/s3/variables.tf index 88a9ad3cdad..866daeb020a 100644 --- a/infra/aws/terraform/registry.k8s.io/s3/variables.tf +++ b/infra/aws/terraform/registry.k8s.io/s3/variables.tf @@ -16,7 +16,7 @@ limitations under the License. variable "prefix" { type = string - default = "" + default = "prod-" } variable "s3_replication_iam_role_arn" { @@ -30,3 +30,9 @@ variable "s3_replication_rules" { type = list(map(string)) default = [] } + +variable "region" { + description = "AWS region" + type = string +} + diff --git a/infra/aws/terraform/registry.k8s.io/s3/data.tf b/infra/aws/terraform/registry.k8s.io/variables.tf similarity index 51% rename from infra/aws/terraform/registry.k8s.io/s3/data.tf rename to infra/aws/terraform/registry.k8s.io/variables.tf index 6b9e3a3dff6..ef6a8513f54 100644 --- a/infra/aws/terraform/registry.k8s.io/s3/data.tf +++ b/infra/aws/terraform/registry.k8s.io/variables.tf @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,4 +14,18 @@ See the License for the specific language governing permissions and limitations under the License. */ -data "aws_region" "current" {} +variable "prefix" { + description = "Prefix for every resource so that the resources can be created without using the same names. Useful for testing and staging" + type = string + default = "prod-" + + validation { + condition = can(regex(".*-$|^$", var.prefix)) + error_message = "The string must end with a hyphen or be empty." + } +} + +variable "atlantis_role_arn" { + description = "The ARN of the Atlantis IAM role" + default = null +}