diff --git a/eks.tf b/eks.tf index 1029fb3..f4cf249 100644 --- a/eks.tf +++ b/eks.tf @@ -1,35 +1,65 @@ module "eks" { source = "terraform-aws-modules/eks/aws" - version = "20.31.6" + version = "21.23.0" - cluster_name = var.eks_cluster_name - cluster_version = var.eks_cluster_version - cluster_endpoint_private_access = true - cluster_endpoint_public_access = true - cluster_endpoint_public_access_cidrs = var.cluster_endpoint_public_access_cidrs - cluster_security_group_name = "${var.eks_cluster_name}-sg" - enable_irsa = true + iam_role_use_name_prefix = !var.allow_long_names + node_iam_role_use_name_prefix = !var.allow_long_names + + name = var.eks_cluster_name + kubernetes_version = var.eks_cluster_version + endpoint_private_access = true + endpoint_public_access = true + endpoint_public_access_cidrs = var.cluster_endpoint_public_access_cidrs + security_group_name = "${var.eks_cluster_name}-sg" + enable_irsa = true access_entries = local.merged_access_entries + iam_role_additional_policies = var.enable_eks_auto_mode ? {} : { + AmazonEKSVPCResourceController = "arn:aws:iam::aws:policy/AmazonEKSVPCResourceController" + } + ## Control plane logging create_cloudwatch_log_group = true - cluster_enabled_log_types = var.cluster_enabled_log_types + enabled_log_types = var.cluster_enabled_log_types cloudwatch_log_group_retention_in_days = var.cluster_log_retention_in_days - cluster_addons = { - coredns = { - addon_version = var.addons_versions.coredns - } - kube-proxy = { - addon_version = var.addons_versions.kube_proxy - } - vpc-cni = { - service_account_role_arn = module.vpc_cni_irsa.iam_role_arn - } - } + addons = merge( + var.enable_eks_auto_mode ? {} : { + coredns = { + addon_version = var.addons_versions.coredns + + resolve_conflicts_on_create = var.addons_versions.resolve_conflicts_on_create + } + kube-proxy = { + addon_version = var.addons_versions.kube_proxy - cluster_security_group_additional_rules = { + resolve_conflicts_on_create = var.addons_versions.resolve_conflicts_on_create + } + vpc-cni = { + addon_version = var.addons_versions.vpc_cni + before_compute = true + service_account_role_arn = module.vpc_cni_irsa.arn + + resolve_conflicts_on_create = var.addons_versions.resolve_conflicts_on_create + } + }, + var.enable_efs_csi ? { + aws-efs-csi-driver = { + addon_version = var.addons_versions.efs_csi + service_account_role_arn = module.irsa-ebs-csi.arn + tags = tomap({ eks_addon = "efs_csi" }) + + resolve_conflicts_on_create = var.addons_versions.resolve_conflicts_on_create + } + } : {} + ) + + compute_config = var.enable_eks_auto_mode ? { + enabled = true + } : null + + security_group_additional_rules = { egress_nodes_ephemeral_ports_tcp = { description = "To node 1025-65535" protocol = "tcp" @@ -51,7 +81,7 @@ module "eks" { } } - cluster_ip_family = "ipv4" + ip_family = "ipv4" create_cni_ipv6_iam_policy = false vpc_id = var.vpc_id @@ -61,32 +91,19 @@ module "eks" { # EKS Managed Node Group(s) - eks_managed_node_group_defaults = { - ami_type = var.eks_ami_type - disk_size = var.eks_disk_size - instance_types = var.eks_instance_types - - iam_role_attach_cni_policy = true - - iam_role_additional_policies = var.eks_node_additional_policies - - block_device_mappings = { - xvda = { - device_name = "/dev/xvda" - ebs = { - volume_size = var.eks_disk_size - volume_type = var.eks_volume_type - iops = var.eks_volume_iops - throughput = 150 - encrypted = true - delete_on_termination = true - } - } - } - } - - eks_managed_node_groups = { + eks_managed_node_groups = var.enable_eks_auto_mode ? {} : { eks_workers = { + iam_role_use_name_prefix = !var.allow_long_names + + ami_type = var.eks_ami_type + disk_size = var.eks_disk_size + instance_types = var.eks_instance_types + + iam_role_attach_cni_policy = true + iam_role_additional_policies = var.eks_node_additional_policies + enable_monitoring = true + use_latest_ami_release_version = false + name = "${var.eks_cluster_name}-ng" min_size = var.eks_ng_min_size max_size = var.eks_ng_max_size @@ -94,6 +111,20 @@ module "eks" { ebs_optimized = true + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = var.eks_disk_size + volume_type = var.eks_volume_type + iops = var.eks_volume_iops + throughput = 150 + encrypted = true + delete_on_termination = true + } + } + } + metadata_options = { http_endpoint = "enabled" http_tokens = "required" @@ -114,13 +145,29 @@ module "eks" { } +moved { + from = aws_eks_addon.ebs-csi + to = aws_eks_addon.ebs-csi[0] +} + +moved { + from = module.eks.aws_iam_role_policy_attachment.this["AmazonEKSVPCResourceController"] + to = module.eks.aws_iam_role_policy_attachment.additional["AmazonEKSVPCResourceController"] +} + resource "aws_eks_addon" "ebs-csi" { + count = var.enable_eks_auto_mode ? 0 : 1 + cluster_name = module.eks.cluster_name addon_name = "aws-ebs-csi-driver" addon_version = var.addons_versions.ebs_csi - service_account_role_arn = module.irsa-ebs-csi.iam_role_arn + service_account_role_arn = module.irsa-ebs-csi.arn tags = merge( var.eks_tags, tomap({ eks_addon = "ebs_csi" }) ) + + depends_on = [ + module.eks + ] } diff --git a/irsa.tf b/irsa.tf index 33a4a47..e50f01e 100644 --- a/irsa.tf +++ b/irsa.tf @@ -2,10 +2,14 @@ #IRSA for VPC-CNI addon for EKS # ################################# module "vpc_cni_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.34.0" + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" + version = "6.8.0" + + create = !var.enable_eks_auto_mode + name = "AmazonEKS-VPC-CNI-${var.eks_cluster_name}" + policy_name = "VPC_CNI_IPv4-${var.eks_cluster_name}" + use_name_prefix = false - role_name = "AmazonEKS-VPC-CNI-${var.eks_cluster_name}" attach_vpc_cni_policy = true vpc_cni_enable_ipv6 = false vpc_cni_enable_ipv4 = true @@ -22,12 +26,15 @@ module "vpc_cni_irsa" { #IRSA for EBS-CSI addon for EKS # ################################# module "irsa-ebs-csi" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.34.0" + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" + version = "6.8.0" + + create = !var.enable_eks_auto_mode + name = "AmazonEKS-EBS-CSI-${var.eks_cluster_name}" + policy_name = "EBS-CSI-${var.eks_cluster_name}" + use_name_prefix = false - create_role = true - role_name = "AmazonEKS-EBS-CSI-${var.eks_cluster_name}" - role_policy_arns = { + policies = { ebs_csi_policy = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy" } oidc_providers = { @@ -38,16 +45,42 @@ module "irsa-ebs-csi" { } } +################################# +#IRSA for EFS-CSI addon for EKS # +################################# +module "irsa-efs-csi" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" + version = "6.8.0" + + create = var.enable_efs_csi + name = "AmazonEKS-EFS-CSI-${var.eks_cluster_name}" + policy_name = "EFS-CSI-${var.eks_cluster_name}" + use_name_prefix = false + + policies = { + efs_csi_policy = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" + } + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:efs-csi-controller-sa"] + } + } +} + ##################################### #IRSA for External Secrets Operator # ##################################### module "iam_assumable_role_admin_secrets_operator" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.34.0" + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" + version = "6.8.0" + + create = true + name = "${var.eks_cluster_name}-secrets-operator" + policy_name = "${var.eks_cluster_name}-secrets-operator" + use_name_prefix = false - create_role = true - role_name = "${var.eks_cluster_name}-secrets-operator" - role_policy_arns = { + policies = { eso_policy = aws_iam_policy.secrets_operator.arn } oidc_providers = { @@ -62,12 +95,15 @@ module "iam_assumable_role_admin_secrets_operator" { #IRSA for External DNS # ######################## module "iam_assumable_role_external_dns" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.34.0" + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" + version = "6.8.0" - create_role = true - role_name = "${var.eks_cluster_name}-external-dns" - role_policy_arns = { + create = true + name = "${var.eks_cluster_name}-external-dns" + policy_name = "${var.eks_cluster_name}-external-dns" + use_name_prefix = false + + policies = { external_dns_policy = "arn:aws:iam::aws:policy/AmazonRoute53FullAccess" } oidc_providers = { @@ -114,12 +150,15 @@ EOT ########################## module "iam_assumable_role_admin_aws_load_balancer_controller" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.34.0" + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" + version = "6.8.0" + + create = true + name = "aws-load-balancer-controller-${var.eks_cluster_name}" - create_role = true - role_name = "aws-load-balancer-controller-${var.eks_cluster_name}" - role_policy_arns = { + use_name_prefix = false + + policies = { alb_controller_policy = aws_iam_policy.aws_load_balancer_controller.arn } oidc_providers = { @@ -166,6 +205,9 @@ resource "aws_iam_policy" "aws_load_balancer_controller" { "ec2:DescribeTags", "ec2:GetCoipPoolUsage", "ec2:DescribeCoipPools", + "ec2:GetSecurityGroupsForVpc", + "ec2:DescribeIpamPools", + "ec2:DescribeRouteTables", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancerAttributes", "elasticloadbalancing:DescribeListeners", @@ -175,7 +217,10 @@ resource "aws_iam_policy" "aws_load_balancer_controller" { "elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DescribeTargetGroupAttributes", "elasticloadbalancing:DescribeTargetHealth", - "elasticloadbalancing:DescribeTags" + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:DescribeTrustStores", + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation" ], "Resource": "*" }, @@ -324,7 +369,10 @@ resource "aws_iam_policy" "aws_load_balancer_controller" { "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", - "elasticloadbalancing:DeleteTargetGroup" + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation", + "elasticloadbalancing:ModifyIpPools" ], "Resource": "*", "Condition": { @@ -370,7 +418,8 @@ resource "aws_iam_policy" "aws_load_balancer_controller" { "elasticloadbalancing:ModifyListener", "elasticloadbalancing:AddListenerCertificates", "elasticloadbalancing:RemoveListenerCertificates", - "elasticloadbalancing:ModifyRule" + "elasticloadbalancing:ModifyRule", + "elasticloadbalancing:SetRulePriorities" ], "Resource": "*" } @@ -378,4 +427,3 @@ resource "aws_iam_policy" "aws_load_balancer_controller" { } EOT } - diff --git a/main.tf b/main.tf index 76600fe..a2858f9 100644 --- a/main.tf +++ b/main.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.3" + required_version = ">= 1.5.7" required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 6.42, != 6.57.0, < 7.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -22,5 +22,9 @@ terraform { source = "hashicorp/random" version = ">= 3" } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } } } diff --git a/managed-karpenter.tf b/managed-karpenter.tf new file mode 100644 index 0000000..22c0f68 --- /dev/null +++ b/managed-karpenter.tf @@ -0,0 +1,124 @@ +data "aws_availability_zones" "available" { + count = var.enable_eks_auto_mode ? 1 : 0 + + state = "available" +} + +moved { + from = kubectl_manifest.karpenter_nodeclass + to = kubectl_manifest.karpenter_nodeclass[0] +} + +resource "kubectl_manifest" "karpenter_nodeclass" { + count = var.enable_eks_auto_mode ? 1 : 0 + + lifecycle { + precondition { + condition = length(trimspace(var.project_name)) > 0 + error_message = "project_name must be set when enable_eks_auto_mode is true." + } + } + + yaml_body = < 0~} + - key: node.kubernetes.io/instance-type + operator: In + values: +%{for instance in var.karpenter_allowed_instance_types~} + - ${instance} +%{endfor~} +%{endif~} +YAML +} + +moved { + from = aws_eks_access_entry.karpenter + to = aws_eks_access_entry.karpenter[0] +} + +resource "aws_eks_access_entry" "karpenter" { + count = var.enable_eks_auto_mode ? 1 : 0 + + cluster_name = module.eks.cluster_name + principal_arn = module.eks.node_iam_role_arn + type = "EC2" +} + +moved { + from = aws_eks_access_policy_association.karpenter_auto_policy + to = aws_eks_access_policy_association.karpenter_auto_policy[0] +} + +resource "aws_eks_access_policy_association" "karpenter_auto_policy" { + count = var.enable_eks_auto_mode ? 1 : 0 + + cluster_name = module.eks.cluster_name + principal_arn = aws_eks_access_entry.karpenter[0].principal_arn + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAutoNodePolicy" + + access_scope { + type = "cluster" + } + + depends_on = [ + aws_eks_access_entry.karpenter + ] +} diff --git a/outputs.tf b/outputs.tf index 73b409d..1422b8d 100644 --- a/outputs.tf +++ b/outputs.tf @@ -19,7 +19,7 @@ output "eks_cluster_version" { } output "eks_irsa_external_dns_arn" { - value = module.iam_assumable_role_external_dns.iam_role_arn + value = module.iam_assumable_role_external_dns.arn } output "node_security_group_id" { @@ -28,11 +28,17 @@ output "node_security_group_id" { } output "node_iam_role_name" { - value = module.eks.eks_managed_node_groups["eks_workers"].iam_role_name + description = "Name of the IAM role used by EKS worker nodes" + value = var.enable_eks_auto_mode ? module.eks.node_iam_role_name : ( + module.eks.eks_managed_node_groups["eks_workers"].iam_role_name + ) } output "node_iam_role_arn" { - value = module.eks.eks_managed_node_groups["eks_workers"].iam_role_arn + description = "ARN of the IAM role used by EKS worker nodes" + value = var.enable_eks_auto_mode ? module.eks.node_iam_role_arn : ( + module.eks.eks_managed_node_groups["eks_workers"].iam_role_arn + ) } output "oidc_provider_arn" { diff --git a/variables.tf b/variables.tf index bc5dbbc..fd0271f 100644 --- a/variables.tf +++ b/variables.tf @@ -16,6 +16,18 @@ variable "environment" { description = "Environment in which resources are deployed" } +variable "project_name" { + type = string + description = "Name of the project, client, or product used to tag EKS Auto Mode nodes" + default = "" +} + +variable "allow_long_names" { + type = string + default = true + description = "Allows longer IAM role names without suffixes. Leave true for new clusters. Set to false for pre-existing clusters to avoid re-creation." +} + ################################################################################ # Networking variables ################################################################################ @@ -71,13 +83,51 @@ variable "cluster_log_retention_in_days" { default = 14 } +variable "enable_eks_auto_mode" { + type = bool + description = "Enable EKS Auto Mode instead of the managed node group and standard EKS add-ons" + default = false +} + +variable "enable_efs_csi" { + type = bool + description = "Enable EFS CSI EKS managed addon and its IRSA role" + default = false +} + variable "addons_versions" { + description = "Configuration of the standard EKS add-ons; versions are required when EKS Auto Mode is disabled" type = object({ - kube_proxy = string - vpc_cni = string - coredns = string - ebs_csi = string + kube_proxy = optional(string) + vpc_cni = optional(string) + coredns = optional(string) + ebs_csi = optional(string) + efs_csi = optional(string) + resolve_conflicts_on_create = optional(string, "OVERWRITE") }) + default = null + + validation { + condition = var.enable_eks_auto_mode || try(alltrue([ + for version in [ + var.addons_versions.kube_proxy, + var.addons_versions.vpc_cni, + var.addons_versions.coredns, + var.addons_versions.ebs_csi, + ] : length(trimspace(version)) > 0 + ]), false) + error_message = "Normal mode requires non-empty kube_proxy, vpc_cni, coredns, and ebs_csi versions." + } + + validation { + condition = !var.enable_efs_csi || try(length(trimspace(var.addons_versions.efs_csi)) > 0, false) + error_message = "enable_efs_csi requires a non-empty efs_csi version." + } + + validation { + condition = try(contains(["NONE", "OVERWRITE"], var.addons_versions.resolve_conflicts_on_create), true) + error_message = "addons_versions.resolve_conflicts_on_create must be one of: NONE, OVERWRITE." + } } variable "eks_tags" { @@ -108,7 +158,7 @@ variable "access_entries" { variable "eks_ami_type" { description = "Default AMI type for the EKS worker nodes" type = string - default = "AL2_x86_64" + default = "BOTTLEROCKET_x86_64" } variable "eks_disk_size" { @@ -167,6 +217,12 @@ variable "eks_ng_capacity_type" { default = "SPOT" } +variable "karpenter_allowed_instance_types" { + description = "Optional instance types allowed by the EKS Auto Mode NodePool; an empty list applies no instance type restriction" + type = list(string) + default = [] +} + variable "kms_key_enable_default_policy" { description = "Specifies whether to enable the default key policy. Defaults to `true`" type = bool