vmware-tanzu · DanielXiao · Mar 15, 2023 · Mar 9, 2023
diff --git a/packages/tkg-clusterclass-aws/bundle/config/upstream/base.yaml b/packages/tkg-clusterclass-aws/bundle/config/upstream/base.yaml
@@ -298,6 +298,23 @@ spec:
               required:
               - name
               - data
+  - name: additionalImageRegistries
+    required: false
+    schema:
+      openAPIV3Schema:
+        type: array
+        items:
+          type: object
+          required:
+          - host
+          properties:
+            host:
+              type: string
+            skipTlsVerify:
+              type: boolean
+              default: false
+            caCert:
+              type: string
   - name: auditLogging
     required: false
     schema:
@@ -1125,6 +1142,52 @@ spec:
             content: {{or $proxy $image}}
             encoding: base64
             permissions: "0444"
+  - name: additionalRegistryCACerts
+    enabledIf: '{{ not (empty .additionalImageRegistries) }}'
+    definitions:
+    - selector:
+        apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+        kind: KubeadmControlPlaneTemplate
+        matchResources:
+          controlPlane: true
+      jsonPatches:
+      - op: add
+        path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
+        value: /tmp/insert_registry_ca_certs.sh
+      - op: add
+        path: /spec/template/spec/kubeadmConfigSpec/files/-
+        valueFrom:
+          template: &insertCAScript |
+            path: /tmp/insert_registry_ca_certs.sh
+            content: |
+              #!/bin/bash
+              set -e
+              {{- range .additionalImageRegistries }}
+                {{- if eq .skipTlsVerify false }}
+              echo '{{ .caCert }}' | base64 -d > /etc/containerd/{{ .host }}.crt
+              echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .host }}".tls]' >> /etc/containerd/config.toml
+              echo '  ca_file = "/etc/containerd/{{ .host }}.crt"' >> /etc/containerd/config.toml
+                {{- else }}
+              echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .host }}".tls]' >> /etc/containerd/config.toml
+              echo "  insecure_skip_verify = true" >> /etc/containerd/config.toml
+                {{- end }}
+              {{- end }}
+            permissions: "0755"
+    - selector:
+        apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+        kind: KubeadmConfigTemplate
+        matchResources:
+          machineDeploymentClass:
+            names:
+            - tkg-worker
+      jsonPatches:
+      - op: add
+        path: /spec/template/spec/preKubeadmCommands/-
+        value: /tmp/insert_registry_ca_certs.sh
+      - op: add
+        path: /spec/template/spec/files/-
+        valueFrom:
+          template: *insertCAScript
   - name: auditLogging
     enabledIf: '{{ .auditLogging.enabled }}'
     definitions:

diff --git a/packages/tkg-clusterclass-aws/bundle/config/upstream/overlay.yaml b/packages/tkg-clusterclass-aws/bundle/config/upstream/overlay.yaml
@@ -53,7 +53,7 @@ spec:
     name: KCP_INIT_APISERVER_EMPTY_EXTRAVOLUMES_ARRAY
   #@overlay/append
   - name: restartContainerdService
-    enabledIf: '{{ not (and (empty .proxy) (empty .trust.additionalTrustedCAs) (empty .imageRepository.host)) }}'
+    enabledIf: '{{ not (and (empty .proxy) (empty .trust.additionalTrustedCAs) (empty .imageRepository.host) (empty .additionalImageRegistries)) }}'
     definitions:
     - selector:
         apiVersion: controlplane.cluster.x-k8s.io/v1beta1

diff --git a/packages/tkg-clusterclass-azure/bundle/config/upstream/base.yaml b/packages/tkg-clusterclass-azure/bundle/config/upstream/base.yaml
@@ -354,6 +354,23 @@ spec:
               required:
               - name
               - data
+  - name: additionalImageRegistries
+    required: false
+    schema:
+      openAPIV3Schema:
+        type: array
+        items:
+          type: object
+          required:
+          - host
+          properties:
+            host:
+              type: string
+            skipTlsVerify:
+              type: boolean
+              default: false
+            caCert:
+              type: string
   - name: auditLogging
     required: false
     schema:
@@ -1461,6 +1478,52 @@ spec:
             content: {{or $proxy $image}}
             encoding: base64
             permissions: "0444"
+  - name: additionalRegistryCACerts
+    enabledIf: '{{ not (empty .additionalImageRegistries) }}'
+    definitions:
+    - selector:
+        apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+        kind: KubeadmControlPlaneTemplate
+        matchResources:
+          controlPlane: true
+      jsonPatches:
+      - op: add
+        path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
+        value: /tmp/insert_registry_ca_certs.sh
+      - op: add
+        path: /spec/template/spec/kubeadmConfigSpec/files/-
+        valueFrom:
+          template: &insertCAScript |
+            path: /tmp/insert_registry_ca_certs.sh
+            content: |
+              #!/bin/bash
+              set -e
+              {{- range .additionalImageRegistries }}
+                {{- if eq .skipTlsVerify false }}
+              echo '{{ .caCert }}' | base64 -d > /etc/containerd/{{ .host }}.crt
+              echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .host }}".tls]' >> /etc/containerd/config.toml
+              echo '  ca_file = "/etc/containerd/{{ .host }}.crt"' >> /etc/containerd/config.toml
+                {{- else }}
+              echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .host }}".tls]' >> /etc/containerd/config.toml
+              echo "  insecure_skip_verify = true" >> /etc/containerd/config.toml
+                {{- end }}
+              {{- end }}
+            permissions: "0755"
+    - selector:
+        apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+        kind: KubeadmConfigTemplate
+        matchResources:
+          machineDeploymentClass:
+            names:
+            - tkg-worker
+      jsonPatches:
+      - op: add
+        path: /spec/template/spec/preKubeadmCommands/-
+        value: /tmp/insert_registry_ca_certs.sh
+      - op: add
+        path: /spec/template/spec/files/-
+        valueFrom:
+          template: *insertCAScript
   - name: auditLogging
     enabledIf: '{{ .auditLogging.enabled }}'
     definitions:

diff --git a/packages/tkg-clusterclass-azure/bundle/config/upstream/overlay.yaml b/packages/tkg-clusterclass-azure/bundle/config/upstream/overlay.yaml
@@ -53,7 +53,7 @@ spec:
     name: KCP_INIT_APISERVER_EMPTY_EXTRAVOLUMES_ARRAY
   #@overlay/append
   - name: restartContainerdService
-    enabledIf: '{{ not (and (empty .proxy) (empty .trust.additionalTrustedCAs) (empty .imageRepository.host)) }}'
+    enabledIf: '{{ not (and (empty .proxy) (empty .trust.additionalTrustedCAs) (empty .imageRepository.host) (empty .additionalImageRegistries)) }}'
     definitions:
     - selector:
         apiVersion: controlplane.cluster.x-k8s.io/v1beta1

diff --git a/packages/tkg-clusterclass-vsphere/bundle/config/upstream/base.yaml b/packages/tkg-clusterclass-vsphere/bundle/config/upstream/base.yaml
@@ -373,6 +373,23 @@ spec:
               required:
               - name
               - data
+  - name: additionalImageRegistries
+    required: false
+    schema:
+      openAPIV3Schema:
+        type: array
+        items:
+          type: object
+          required:
+          - host
+          properties:
+            host:
+              type: string
+            skipTlsVerify:
+              type: boolean
+              default: false
+            caCert:
+              type: string
   - name: auditLogging
     required: false
     schema:
@@ -1716,6 +1733,52 @@ spec:
             content: {{or $proxy $image}}
             encoding: base64
             permissions: "0444"
+  - name: additionalRegistryCACerts
+    enabledIf: '{{ not (empty .additionalImageRegistries) }}'
+    definitions:
+    - selector:
+        apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+        kind: KubeadmControlPlaneTemplate
+        matchResources:
+          controlPlane: true
+      jsonPatches:
+      - op: add
+        path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
+        value: /tmp/insert_registry_ca_certs.sh
+      - op: add
+        path: /spec/template/spec/kubeadmConfigSpec/files/-
+        valueFrom:
+          template: &insertCAScript |
+            path: /tmp/insert_registry_ca_certs.sh
+            content: |
+              #!/bin/bash
+              set -e
+              {{- range .additionalImageRegistries }}
+                {{- if eq .skipTlsVerify false }}
+              echo '{{ .caCert }}' | base64 -d > /etc/containerd/{{ .host }}.crt
+              echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .host }}".tls]' >> /etc/containerd/config.toml
+              echo '  ca_file = "/etc/containerd/{{ .host }}.crt"' >> /etc/containerd/config.toml
+                {{- else }}
+              echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .host }}".tls]' >> /etc/containerd/config.toml
+              echo "  insecure_skip_verify = true" >> /etc/containerd/config.toml
+                {{- end }}
+              {{- end }}
+            permissions: "0755"
+    - selector:
+        apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+        kind: KubeadmConfigTemplate
+        matchResources:
+          machineDeploymentClass:
+            names:
+            - tkg-worker
+      jsonPatches:
+      - op: add
+        path: /spec/template/spec/preKubeadmCommands/-
+        value: /tmp/insert_registry_ca_certs.sh
+      - op: add
+        path: /spec/template/spec/files/-
+        valueFrom:
+          template: *insertCAScript
   - name: auditLogging
     enabledIf: '{{ .auditLogging.enabled }}'
     definitions:

diff --git a/packages/tkg-clusterclass-vsphere/bundle/config/upstream/overlay.yaml b/packages/tkg-clusterclass-vsphere/bundle/config/upstream/overlay.yaml
@@ -71,7 +71,7 @@ spec:
     name: KCP_INIT_APISERVER_EMPTY_EXTRAVOLUMES_ARRAY
   #@overlay/append
   - name: restartContainerdService
-    enabledIf: '{{ not (and (empty .proxy) (empty .trust.additionalTrustedCAs) (empty .imageRepository.host)) }}'
+    enabledIf: '{{ not (and (empty .proxy) (empty .trust.additionalTrustedCAs) (empty .imageRepository.host) (empty .additionalImageRegistries)) }}'
     definitions:
     - selector:
         apiVersion: controlplane.cluster.x-k8s.io/v1beta1

diff --git a/providers/config_default.yaml b/providers/config_default.yaml
@@ -496,6 +496,25 @@ TKG_CUSTOM_IMAGE_REPOSITORY: ""
 TKG_CUSTOM_IMAGE_REPOSITORY_SKIP_TLS_VERIFY: false
 TKG_CUSTOM_IMAGE_REPOSITORY_CA_CERTIFICATE: ""
 
+#! When user's application images are from a registry with a self-signed CA
+#! set this value to configure the cluster to trust it. For a registry with
+#! a public CA nothing need to change. This is only for user's application
+#! images, to configure TKG image repository for cluster creation, use
+#! TKG_CUSTOM_IMAGE_REPOSITORY instead.
+#! Example image registry host name: 10.191.247.30:8443
+ADDITIONAL_IMAGE_REGISTRY_1: ""
+ADDITIONAL_IMAGE_REGISTRY_1_SKIP_TLS_VERIFY: false
+#! Base64 encoded self-signed CA certificate
+ADDITIONAL_IMAGE_REGISTRY_1_CA_CERTIFICATE: ""
+
+ADDITIONAL_IMAGE_REGISTRY_2: ""
+ADDITIONAL_IMAGE_REGISTRY_2_SKIP_TLS_VERIFY: false
+ADDITIONAL_IMAGE_REGISTRY_2_CA_CERTIFICATE: ""
+
+ADDITIONAL_IMAGE_REGISTRY_3: ""
+ADDITIONAL_IMAGE_REGISTRY_3_SKIP_TLS_VERIFY: false
+ADDITIONAL_IMAGE_REGISTRY_3_CA_CERTIFICATE: ""
+
 #! PROXY configuration
 #! If `TKG_PROXY_CA_CERT` is specified it gets used instead of
 #! `TKG_CUSTOM_IMAGE_REPOSITORY_CA_CERTIFICATE` while communicating

diff --git a/providers/infrastructure-aws/v2.0.2/cconly/base.yaml b/providers/infrastructure-aws/v2.0.2/cconly/base.yaml
@@ -298,6 +298,23 @@ spec:
               required:
               - name
               - data
+  - name: additionalImageRegistries
+    required: false
+    schema:
+      openAPIV3Schema:
+        type: array
+        items:
+          type: object
+          required:
+          - host
+          properties:
+            host:
+              type: string
+            skipTlsVerify:
+              type: boolean
+              default: false
+            caCert:
+              type: string
   - name: auditLogging
     required: false
     schema:
@@ -1125,6 +1142,52 @@ spec:
             content: {{or $proxy $image}}
             encoding: base64
             permissions: "0444"
+  - name: additionalRegistryCACerts
+    enabledIf: '{{ not (empty .additionalImageRegistries) }}'
+    definitions:
+    - selector:
+        apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+        kind: KubeadmControlPlaneTemplate
+        matchResources:
+          controlPlane: true
+      jsonPatches:
+      - op: add
+        path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
+        value: /tmp/insert_registry_ca_certs.sh
+      - op: add
+        path: /spec/template/spec/kubeadmConfigSpec/files/-
+        valueFrom:
+          template: &insertCAScript |
+            path: /tmp/insert_registry_ca_certs.sh
+            content: |
+              #!/bin/bash
+              set -e
+              {{- range .additionalImageRegistries }}
+                {{- if eq .skipTlsVerify false }}
+              echo '{{ .caCert }}' | base64 -d > /etc/containerd/{{ .host }}.crt
+              echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .host }}".tls]' >> /etc/containerd/config.toml
+              echo '  ca_file = "/etc/containerd/{{ .host }}.crt"' >> /etc/containerd/config.toml
+                {{- else }}
+              echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .host }}".tls]' >> /etc/containerd/config.toml
+              echo "  insecure_skip_verify = true" >> /etc/containerd/config.toml
+                {{- end }}
+              {{- end }}
+            permissions: "0755"
+    - selector:
+        apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+        kind: KubeadmConfigTemplate
+        matchResources:
+          machineDeploymentClass:
+            names:
+            - tkg-worker
+      jsonPatches:
+      - op: add
+        path: /spec/template/spec/preKubeadmCommands/-
+        value: /tmp/insert_registry_ca_certs.sh
+      - op: add
+        path: /spec/template/spec/files/-
+        valueFrom:
+          template: *insertCAScript
   - name: auditLogging
     enabledIf: '{{ .auditLogging.enabled }}'
     definitions:

diff --git a/providers/infrastructure-aws/v2.0.2/cconly/overlay.yaml b/providers/infrastructure-aws/v2.0.2/cconly/overlay.yaml
@@ -53,7 +53,7 @@ spec:
     name: KCP_INIT_APISERVER_EMPTY_EXTRAVOLUMES_ARRAY
   #@overlay/append
   - name: restartContainerdService
-    enabledIf: '{{ not (and (empty .proxy) (empty .trust.additionalTrustedCAs) (empty .imageRepository.host)) }}'
+    enabledIf: '{{ not (and (empty .proxy) (empty .trust.additionalTrustedCAs) (empty .imageRepository.host) (empty .additionalImageRegistries)) }}'
     definitions:
     - selector:
         apiVersion: controlplane.cluster.x-k8s.io/v1beta1

diff --git a/providers/infrastructure-aws/v2.0.2/yttcc/overlay.yaml b/providers/infrastructure-aws/v2.0.2/yttcc/overlay.yaml
@@ -109,7 +109,7 @@ spec:
     variables:
     #@ vars = get_aws_vars()
     #@ for configVariable in vars:
-    #@  if vars[configVariable] != None and configVariable in ["workerKubeletExtraArgs","controlPlaneKubeletExtraArgs","kubeControllerManagerExtraArgs","kubeSchedulerExtraArgs","apiServerExtraArgs","etcdExtraArgs", "region", "sshKeyName", "bastion", "network", "controlPlane", "worker", "loadBalancerSchemeInternal", "identityRef", "imageRepository", "trust", "auditLogging", "cni", "TKR_DATA", "proxy", "controlPlaneCertificateRotation", "podSecurityStandard", "eventRateLimitConf"]:
+    #@  if vars[configVariable] != None and configVariable in ["additionalImageRegistries","workerKubeletExtraArgs","controlPlaneKubeletExtraArgs","kubeControllerManagerExtraArgs","kubeSchedulerExtraArgs","apiServerExtraArgs","etcdExtraArgs", "region", "sshKeyName", "bastion", "network", "controlPlane", "worker", "loadBalancerSchemeInternal", "identityRef", "imageRepository", "trust", "auditLogging", "cni", "TKR_DATA", "proxy", "controlPlaneCertificateRotation", "podSecurityStandard", "eventRateLimitConf"]:
     - name: #@ configVariable
       value: #@ vars[configVariable]
     #@ end