azure: Add support for single zone NAT gateway

Adding the option for the users to create a NAT gateway for the
compute nodes as an option to replace the traditional load balancer
setup. This is only for a single NAT gateway in the compute
subnet as CAPZ expects an outbound LB for control planes.

Author: rna-afk

data/data/install.openshift.io_installconfigs.yaml

@@ -4847,13 +4847,12 @@ spec:
                     type: string
                   outboundType:
                     default: Loadbalancer
-                    description: |-
-                      OutboundType is a strategy for how egress from cluster is achieved. When not specified default is "Loadbalancer".
-                      "NatGateway" is only available in TechPreview.
+                    description: OutboundType is a strategy for how egress from cluster
+                      is achieved. When not specified default is "Loadbalancer".
                     enum:
                     - ""
                     - Loadbalancer
-                    - NatGateway
+                    - NATGatewaySingleZone
                     - UserDefinedRouting
                     type: string
                   region:

pkg/asset/manifests/azure/cluster.go

@@ -83,10 +83,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 	// CAPZ enables NAT Gateways by default, so we are using this hack to disable
 	// nat gateways when we prefer to use load balancers for node egress.
 	nodeSubnetID := ""
-	if installConfig.Config.Platform.Azure.OutboundType != azure.NatGatewayOutboundType {
-		// Because the node subnet does not already exist, we are using an arbitrary value.
-		// We could populate this with the proper subnet ID in the case of BYO VNET, but
-		// the value currently has no practical effect.
+	switch installConfig.Config.Platform.Azure.OutboundType {
+	// Because the node subnet does not already exist, we are using an arbitrary value.
+	// We could populate this with the proper subnet ID in the case of BYO VNET, but
+	// the value currently has no practical effect.
+	case azure.LoadbalancerOutboundType:
+		fallthrough
+	case azure.UserDefinedRoutingOutboundType:
 		nodeSubnetID = "UNKNOWN"
 	}
 
@@ -104,6 +107,7 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 		Name:             clusterID.InfraID,
 		FrontendIPsCount: to.Ptr(int32(1)),
 	}
+
 	if installConfig.Config.Platform.Azure.OutboundType == azure.UserDefinedRoutingOutboundType {
 		controlPlaneOutboundLB = nil
 	}
@@ -175,6 +179,24 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 
 	azEnv := string(installConfig.Azure.CloudName)
 
+	computeSubnetSpec := capz.SubnetSpec{
+		ID: nodeSubnetID,
+		SubnetClassSpec: capz.SubnetClassSpec{
+			Name: computeSubnet,
+			Role: capz.SubnetNode,
+			CIDRBlocks: []string{
+				subnets[1].String(),
+			},
+		},
+		SecurityGroup: securityGroup,
+	}
+
+	if installConfig.Config.Azure.OutboundType == azure.NATGatewaySingleZoneOutboundType {
+		computeSubnetSpec.NatGateway = capz.NatGateway{
+			NatGatewayClassSpec: capz.NatGatewayClassSpec{Name: fmt.Sprintf("%s-natgw", clusterID.InfraID)},
+		}
+	}
+
 	azureCluster := &capz.AzureCluster{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      clusterID.InfraID,
@@ -225,17 +247,7 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 						},
 						SecurityGroup: securityGroup,
 					},
-					{
-						ID: nodeSubnetID,
-						SubnetClassSpec: capz.SubnetClassSpec{
-							Name: computeSubnet,
-							Role: capz.SubnetNode,
-							CIDRBlocks: []string{
-								subnets[1].String(),
-							},
-						},
-						SecurityGroup: securityGroup,
-					},
+					computeSubnetSpec,
 				},
 			},
 		},

pkg/explain/printer_test.go

@@ -332,9 +332,8 @@ platform configuration.
 
     outboundType <string>
       Default: "Loadbalancer"
-      Valid Values: "","Loadbalancer","NatGateway","UserDefinedRouting"
+      Valid Values: "","Loadbalancer","NATGatewaySingleZone","UserDefinedRouting"
       OutboundType is a strategy for how egress from cluster is achieved. When not specified default is "Loadbalancer".
-"NatGateway" is only available in TechPreview.
 
     region <string> -required-
       Region specifies the Azure region where the cluster will be created.

pkg/types/azure/platform.go

@@ -9,17 +9,17 @@ import (
 var aro bool
 
 // OutboundType is a strategy for how egress from cluster is achieved.
-// +kubebuilder:validation:Enum="";Loadbalancer;NatGateway;UserDefinedRouting
+// +kubebuilder:validation:Enum="";Loadbalancer;NATGatewaySingleZone;UserDefinedRouting
 type OutboundType string
 
 const (
 	// LoadbalancerOutboundType uses Standard loadbalancer for egress from the cluster.
 	// see https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#lb
 	LoadbalancerOutboundType OutboundType = "Loadbalancer"
 
-	// NatGatewayOutboundType uses NAT gateway for egress from the cluster
+	// NATGatewaySingleZoneOutboundType uses a single (non-zone-resilient) NAT Gateway for compute node outbound access.
 	// see https://learn.microsoft.com/en-us/azure/virtual-network/nat-gateway/nat-gateway-resource
-	NatGatewayOutboundType OutboundType = "NatGateway"
+	NATGatewaySingleZoneOutboundType OutboundType = "NATGatewaySingleZone"
 
 	// UserDefinedRoutingOutboundType uses user defined routing for egress from the cluster.
 	// see https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-udr-overview
@@ -76,7 +76,6 @@ type Platform struct {
 	CloudName CloudEnvironment `json:"cloudName,omitempty"`
 
 	// OutboundType is a strategy for how egress from cluster is achieved. When not specified default is "Loadbalancer".
-	// "NatGateway" is only available in TechPreview.
 	//
 	// +kubebuilder:default=Loadbalancer
 	// +optional

pkg/types/azure/validation/platform.go

@@ -8,7 +8,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
-	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/azure"
 )
@@ -98,13 +97,10 @@ func ValidatePlatform(p *azure.Platform, publish types.PublishingStrategy, fldPa
 	if p.OutboundType == azure.UserDefinedRoutingOutboundType && p.VirtualNetwork == "" {
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("outboundType"), p.OutboundType, fmt.Sprintf("%s is only allowed when installing to pre-existing network", azure.UserDefinedRoutingOutboundType)))
 	}
-	if p.OutboundType == azure.NatGatewayOutboundType {
-		if ic.FeatureSet != configv1.TechPreviewNoUpgrade {
-			allErrs = append(allErrs, field.Invalid(fldPath.Child("outboundType"), p.OutboundType, "not supported in this feature set"))
-		}
+	if p.OutboundType == azure.NATGatewaySingleZoneOutboundType {
 		if p.VirtualNetwork != "" {
 			// For now, BYO network and NAT gateways are not compatible
-			allErrs = append(allErrs, field.Invalid(fldPath.Child("outboundType"), p.OutboundType, fmt.Sprintf("%s is not allowed when installing to pre-existing network", azure.NatGatewayOutboundType)))
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("outboundType"), p.OutboundType, fmt.Sprintf("%s is not allowed when installing to pre-existing network", p.OutboundType)))
 		}
 	}
 
@@ -242,9 +238,9 @@ func findDuplicateTagKeys(tagSet map[string]string) error {
 
 var (
 	validOutboundTypes = map[azure.OutboundType]struct{}{
-		azure.LoadbalancerOutboundType:       {},
-		azure.NatGatewayOutboundType:         {},
-		azure.UserDefinedRoutingOutboundType: {},
+		azure.LoadbalancerOutboundType:         {},
+		azure.NATGatewaySingleZoneOutboundType: {},
+		azure.UserDefinedRoutingOutboundType:   {},
 	}
 
 	validOutboundTypeValues = func() []string {
@@ -262,10 +258,7 @@ func validateAzureStack(p *azure.Platform, fldPath *field.Path) field.ErrorList
 	if p.ARMEndpoint == "" {
 		allErrs = append(allErrs, field.Required(fldPath.Child("armEndpoint"), "ARM endpoint must be set when installing on Azure Stack"))
 	}
-	switch p.OutboundType {
-	case azure.UserDefinedRoutingOutboundType:
-		allErrs = append(allErrs, field.Invalid(fldPath.Child("outboundType"), p.OutboundType, "Azure Stack does not support user-defined routing"))
-	case azure.NatGatewayOutboundType:
+	if p.OutboundType != azure.LoadbalancerOutboundType {
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("outboundType"), p.OutboundType, "Azure Stack does not support NAT routing currently"))
 	}
 	return allErrs

pkg/types/azure/validation/platform_test.go

@@ -134,7 +134,7 @@ func TestValidatePlatform(t *testing.T) {
 				p.OutboundType = "random-egress"
 				return p
 			}(),
-			expected: `^test-path\.outboundType: Unsupported value: "random-egress": supported values: "Loadbalancer", "NatGateway", "UserDefinedRouting"$`,
+			expected: `^test-path\.outboundType: Unsupported value: "random-egress": supported values: "Loadbalancer", "NATGatewaySingleZone", "UserDefinedRouting"$`,
 		},
 		{
 			name: "invalid user defined type",
@@ -145,15 +145,6 @@ func TestValidatePlatform(t *testing.T) {
 			}(),
 			expected: `^test-path\.outboundType: Invalid value: "UserDefinedRouting": UserDefinedRouting is only allowed when installing to pre-existing network$`,
 		},
-		{
-			name: "invalid nat gateway",
-			platform: func() *azure.Platform {
-				p := validPlatform()
-				p.OutboundType = azure.NatGatewayOutboundType
-				return p
-			}(),
-			expected: `^test-path\.outboundType: Invalid value: "NatGateway": not supported in this feature set$`,
-		},
 		{
 			name: "missing key vault name",
 			platform: func() *azure.Platform {