[main] added azure overlay support

charts/aks-operator-crd/templates/crds.yaml

@@ -77,6 +77,9 @@ spec:
               networkPlugin:
                 nullable: true
                 type: string
+              networkPluginMode:
+                nullable: true
+                type: string
               networkPolicy:
                 nullable: true
                 type: string

controller/aks-cluster-config-handler.go

@@ -448,6 +448,20 @@ func (h *Handler) validateConfig(config *aksv1.AKSClusterConfig) error {
 		aks.String(config.Spec.NetworkPlugin) != string(armcontainerservice.NetworkPluginAzure) {
 		return fmt.Errorf("invalid network plugin value [%s] for [%s (id: %s)] cluster config", aks.String(config.Spec.NetworkPlugin), config.Spec.ClusterName, config.Name)
 	}
+	if config.Spec.NetworkPluginMode != nil {
+		mode := strings.ToLower(aks.String(config.Spec.NetworkPluginMode))
+		plugin := strings.ToLower(aks.String(config.Spec.NetworkPlugin))
+		// Only "overlay" is supported
+		if mode != strings.ToLower(string(armcontainerservice.NetworkPluginModeOverlay)) {
+			return fmt.Errorf("invalid networkPluginMode value [%s] for [%s (id: %s)] cluster config",
+				mode, config.Spec.ClusterName, config.Name)
+		}
+		// Overlay requires Azure CNI
+		if plugin != strings.ToLower(string(armcontainerservice.NetworkPluginAzure)) {
+			return fmt.Errorf("networkPluginMode 'overlay' can only be used when networkPlugin is 'azure' for [%s (id: %s)] cluster",
+				config.Spec.ClusterName, config.Name)
+		}
+	}
 	if config.Spec.NetworkPolicy != nil &&
 		aks.String(config.Spec.NetworkPolicy) != string(armcontainerservice.NetworkPolicyAzure) &&
 		aks.String(config.Spec.NetworkPolicy) != string(armcontainerservice.NetworkPolicyCalico) {
@@ -477,12 +491,21 @@ func (h *Handler) validateConfig(config *aksv1.AKSClusterConfig) error {
 
 	cannotBeNilErrorAzurePlugin := "field [%s] must be provided for cluster [%s (id: %s)] config when Azure CNI network plugin is used"
 	if aks.String(config.Spec.NetworkPlugin) == string(armcontainerservice.NetworkPluginAzure) {
-		if config.Spec.VirtualNetwork == nil {
-			return fmt.Errorf(cannotBeNilErrorAzurePlugin, "virtualNetwork", config.Spec.ClusterName, config.Name)
+		isOverlay := strings.EqualFold(aks.String(config.Spec.NetworkPluginMode), string(armcontainerservice.NetworkPluginModeOverlay))
+		if !isOverlay {
+			if config.Spec.VirtualNetwork == nil {
+				return fmt.Errorf(cannotBeNilErrorAzurePlugin, "virtualNetwork", config.Spec.ClusterName, config.Name)
+			}
+			if config.Spec.Subnet == nil {
+				return fmt.Errorf(cannotBeNilErrorAzurePlugin, "subnet", config.Spec.ClusterName, config.Name)
+			}
 		}
-		if config.Spec.Subnet == nil {
-			return fmt.Errorf(cannotBeNilErrorAzurePlugin, "subnet", config.Spec.ClusterName, config.Name)
+		if isOverlay {
+			if config.Spec.NetworkPodCIDR == nil {
+				return fmt.Errorf("field [podCidr] must be provided for cluster [%s (id: %s)] config when Azure CNI Overlay is used", config.Spec.ClusterName, config.Name)
+			}
 		}
+
 		if config.Spec.NetworkDNSServiceIP == nil {
 			return fmt.Errorf(cannotBeNilErrorAzurePlugin, "dnsServiceIp", config.Spec.ClusterName, config.Name)
 		}
@@ -671,6 +694,9 @@ func (h *Handler) buildUpstreamClusterState(ctx context.Context, credentials *ak
 	networkProfile := clusterState.Properties.NetworkProfile
 	if networkProfile != nil {
 		upstreamSpec.NetworkPlugin = to.Ptr(string(*networkProfile.NetworkPlugin))
+		if networkProfile.NetworkPluginMode != nil {
+			upstreamSpec.NetworkPluginMode = to.Ptr(string(*networkProfile.NetworkPluginMode))
+		}
 		upstreamSpec.NetworkDNSServiceIP = networkProfile.DNSServiceIP
 		upstreamSpec.NetworkServiceCIDR = networkProfile.ServiceCidr
 		upstreamSpec.NetworkPodCIDR = networkProfile.PodCidr

pkg/aks/create.go

@@ -105,10 +105,18 @@ func createManagedCluster(ctx context.Context, cred *Credentials, workplacesClie
 	switch strings.ToLower(String(spec.NetworkPlugin)) {
 	case string(armcontainerservice.NetworkPluginAzure):
 		networkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginAzure)
+		if strings.EqualFold(String(spec.NetworkPluginMode), string(armcontainerservice.NetworkPluginModeOverlay)) {
+			networkProfile.NetworkPluginMode = to.Ptr(armcontainerservice.NetworkPluginModeOverlay)
+		}
 	case string(armcontainerservice.NetworkPluginKubenet):
 		networkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginKubenet)
 	case "":
-		networkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginKubenet)
+		if strings.EqualFold(String(spec.NetworkPluginMode), string(armcontainerservice.NetworkPluginModeOverlay)) {
+			networkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginAzure)
+			networkProfile.NetworkPluginMode = to.Ptr(armcontainerservice.NetworkPluginModeOverlay)
+		} else {
+			networkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginKubenet)
+		}
 	default:
 		return nil, fmt.Errorf("networkPlugin '%s' is not supported", String(spec.NetworkPlugin))
 	}

pkg/apis/aks.cattle.io/v1/types.go

@@ -64,6 +64,13 @@ type AKSClusterConfigSpec struct {
 	NetworkPlugin *string `json:"networkPlugin" norman:"pointer"`
 	// VirualNetworkResourceGroup is the name of the Azure resource group for the VNet and Subnet.
 	// +optional
+	// NetworkPluginMode is the mode the network plugin should use.
+	// Allowed value is "overlay".
+	// Immutable.
+	// +kubebuilder:validation:Enum=overlay
+	// +optional
+	NetworkPluginMode *string `json:"networkPluginMode,omitempty" norman:"pointer"`
+	// -
 	VirtualNetworkResourceGroup *string `json:"virtualNetworkResourceGroup" norman:"pointer"`
 	// VirtualNetwork describes the vnet for the AKS cluster. Will be created if it does not exist.
 	// +optional