Skip to content

Commit d7c3afe

Browse files
danielkzadanielkza
committed
Add Tolerations spec to chart configuration
Signed-off-by: Daniel Miranda <[email protected]>
1 parent f9103f6 commit d7c3afe

File tree

3 files changed

+286
-26
lines changed

3 files changed

+286
-26
lines changed

‎pkg/apis/helm.cattle.io/v1/types.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ type HelmChartSpec struct {
4242

4343
PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"`
4444
SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
45+
46+
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
4547
}
4648

4749
type HelmChartStatus struct {

‎pkg/controllers/chart/chart.go‎

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,35 @@ var (
7474
}
7575
)
7676

77+
var (
78+
bootstrapTolerations = []corev1.Toleration{
79+
{
80+
Key: corev1.TaintNodeNotReady,
81+
Effect: corev1.TaintEffectNoSchedule,
82+
},
83+
{
84+
Key: TaintExternalCloudProvider,
85+
Operator: corev1.TolerationOpEqual,
86+
Value: "true",
87+
Effect: corev1.TaintEffectNoSchedule,
88+
},
89+
{
90+
Key: "CriticalAddonsOnly",
91+
Operator: corev1.TolerationOpExists,
92+
},
93+
{
94+
Key: LabelNodeRolePrefix + LabelEtcdSuffix,
95+
Operator: corev1.TolerationOpExists,
96+
Effect: corev1.TaintEffectNoExecute,
97+
},
98+
{
99+
Key: LabelNodeRolePrefix + LabelControlPlaneSuffix,
100+
Operator: corev1.TolerationOpExists,
101+
Effect: corev1.TaintEffectNoSchedule,
102+
},
103+
}
104+
)
105+
77106
type Controller struct {
78107
systemNamespace string
79108
jobClusterRole string
@@ -525,32 +554,7 @@ func job(chart *v1.HelmChart, apiServerPort string) (*batch.Job, *corev1.Secret,
525554
if chart.Spec.Bootstrap {
526555
job.Spec.Template.Spec.NodeSelector[LabelNodeRolePrefix+LabelControlPlaneSuffix] = "true"
527556
job.Spec.Template.Spec.HostNetwork = true
528-
job.Spec.Template.Spec.Tolerations = []corev1.Toleration{
529-
{
530-
Key: corev1.TaintNodeNotReady,
531-
Effect: corev1.TaintEffectNoSchedule,
532-
},
533-
{
534-
Key: TaintExternalCloudProvider,
535-
Operator: corev1.TolerationOpEqual,
536-
Value: "true",
537-
Effect: corev1.TaintEffectNoSchedule,
538-
},
539-
{
540-
Key: "CriticalAddonsOnly",
541-
Operator: corev1.TolerationOpExists,
542-
},
543-
{
544-
Key: LabelNodeRolePrefix + LabelEtcdSuffix,
545-
Operator: corev1.TolerationOpExists,
546-
Effect: corev1.TaintEffectNoExecute,
547-
},
548-
{
549-
Key: LabelNodeRolePrefix + LabelControlPlaneSuffix,
550-
Operator: corev1.TolerationOpExists,
551-
Effect: corev1.TaintEffectNoSchedule,
552-
},
553-
}
557+
job.Spec.Template.Spec.Tolerations = bootstrapTolerations
554558
job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{
555559
{
556560
Name: "KUBERNETES_SERVICE_HOST",
@@ -563,6 +567,7 @@ func job(chart *v1.HelmChart, apiServerPort string) (*batch.Job, *corev1.Secret,
563567
Value: "true"},
564568
}...)
565569
}
570+
job.Spec.Template.Spec.Tolerations = append(job.Spec.Template.Spec.Tolerations, chart.Spec.Tolerations...)
566571

567572
setProxyEnv(job)
568573
setAuthSecret(job, chart)

‎test/suite/helm_test.go‎

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,33 @@ import (
1818
"k8s.io/utils/pointer"
1919
)
2020

21+
var bootstrapTolerations = &[]corev1.Toleration{
22+
{
23+
Key: "node.kubernetes.io/not-ready",
24+
Effect: "NoSchedule",
25+
},
26+
{
27+
Key: "node.cloudprovider.kubernetes.io/uninitialized",
28+
Operator: "Equal",
29+
Value: "true",
30+
Effect: "NoSchedule",
31+
},
32+
{
33+
Key: "CriticalAddonsOnly",
34+
Operator: "Exists",
35+
},
36+
{
37+
Key: "node-role.kubernetes.io/etcd",
38+
Operator: "Exists",
39+
Effect: "NoExecute",
40+
},
41+
{
42+
Key: "node-role.kubernetes.io/control-plane",
43+
Operator: "Exists",
44+
Effect: "NoSchedule",
45+
},
46+
}
47+
2148
var _ = Describe("Helm Tests", Ordered, func() {
2249
framework, _ := framework.New()
2350

@@ -755,4 +782,230 @@ var _ = Describe("Helm Tests", Ordered, func() {
755782
}, 120*time.Second, 5*time.Second).Should(BeTrue())
756783
})
757784
})
785+
786+
Context("When no tolerations are specified (non-boostrap)", func() {
787+
var (
788+
err error
789+
chart *v1.HelmChart
790+
job *batchv1.Job
791+
)
792+
BeforeEach(func() {
793+
chart = framework.NewHelmChart("traefik-example-no-tolerations",
794+
"stable/traefik",
795+
"1.86.1",
796+
"v3",
797+
map[string]intstr.IntOrString{
798+
"rbac.enabled": {
799+
Type: intstr.String,
800+
StrVal: "true",
801+
},
802+
"ssl.enabled": {
803+
Type: intstr.String,
804+
StrVal: "true",
805+
},
806+
})
807+
chart, err = framework.CreateHelmChart(chart, framework.Namespace)
808+
Expect(err).ToNot(HaveOccurred())
809+
810+
labelSelector := labels.SelectorFromSet(labels.Set{
811+
"owner": "helm",
812+
"name": chart.Name,
813+
})
814+
_, err = framework.WaitForRelease(chart, labelSelector, 120*time.Second, 1)
815+
Expect(err).ToNot(HaveOccurred())
816+
817+
chart, err = framework.GetHelmChart(chart.Name, chart.Namespace)
818+
Expect(err).ToNot(HaveOccurred())
819+
job, err = framework.GetJob(chart)
820+
Expect(err).ToNot(HaveOccurred())
821+
})
822+
FIt("Should have no tolerations set", func() {
823+
Expect(job.Spec.Template.Spec.Tolerations).To(BeEmpty())
824+
})
825+
AfterEach(func() {
826+
err = framework.DeleteHelmChart(chart.Name, framework.Namespace)
827+
Expect(err).ToNot(HaveOccurred())
828+
829+
Eventually(func() bool {
830+
_, err := framework.GetHelmChart(chart.Name, framework.Namespace)
831+
return err != nil && apierrors.IsNotFound(err)
832+
}, 120*time.Second, 5*time.Second).Should(BeTrue())
833+
})
834+
})
835+
836+
Context("When custom tolerations are specified (non-boostrap)", func() {
837+
var (
838+
err error
839+
chart *v1.HelmChart
840+
job *batchv1.Job
841+
tolerations = &[]corev1.Toleration{
842+
{
843+
Key: "test-1",
844+
Effect: corev1.TaintEffectNoSchedule,
845+
},
846+
{
847+
Key: "test-2",
848+
Effect: corev1.TaintEffectNoExecute,
849+
},
850+
}
851+
)
852+
BeforeEach(func() {
853+
chart = framework.NewHelmChart("traefik-example-custom-tolerations",
854+
"stable/traefik",
855+
"1.86.1",
856+
"v3",
857+
map[string]intstr.IntOrString{
858+
"rbac.enabled": {
859+
Type: intstr.String,
860+
StrVal: "true",
861+
},
862+
"ssl.enabled": {
863+
Type: intstr.String,
864+
StrVal: "true",
865+
},
866+
})
867+
chart.Spec.Tolerations = *tolerations
868+
chart, err = framework.CreateHelmChart(chart, framework.Namespace)
869+
Expect(err).ToNot(HaveOccurred())
870+
871+
labelSelector := labels.SelectorFromSet(labels.Set{
872+
"owner": "helm",
873+
"name": chart.Name,
874+
})
875+
_, err = framework.WaitForRelease(chart, labelSelector, 120*time.Second, 1)
876+
Expect(err).ToNot(HaveOccurred())
877+
878+
chart, err = framework.GetHelmChart(chart.Name, chart.Namespace)
879+
Expect(err).ToNot(HaveOccurred())
880+
job, err = framework.GetJob(chart)
881+
Expect(err).ToNot(HaveOccurred())
882+
})
883+
FIt("Should have tolerations set", func() {
884+
Expect(job.Spec.Template.Spec.Tolerations).To(Equal(*tolerations))
885+
})
886+
AfterEach(func() {
887+
err = framework.DeleteHelmChart(chart.Name, framework.Namespace)
888+
Expect(err).ToNot(HaveOccurred())
889+
890+
Eventually(func() bool {
891+
_, err := framework.GetHelmChart(chart.Name, framework.Namespace)
892+
return err != nil && apierrors.IsNotFound(err)
893+
}, 120*time.Second, 5*time.Second).Should(BeTrue())
894+
})
895+
})
896+
897+
Context("When no tolerations are specified (boostrap)", func() {
898+
var (
899+
err error
900+
chart *v1.HelmChart
901+
job *batchv1.Job
902+
)
903+
BeforeEach(func() {
904+
chart = framework.NewHelmChart("traefik-example-bootstrap-default-tolerations",
905+
"stable/traefik",
906+
"1.86.1",
907+
"v3",
908+
map[string]intstr.IntOrString{
909+
"rbac.enabled": {
910+
Type: intstr.String,
911+
StrVal: "true",
912+
},
913+
"ssl.enabled": {
914+
Type: intstr.String,
915+
StrVal: "true",
916+
},
917+
})
918+
chart.Spec.Bootstrap = true
919+
chart, err = framework.CreateHelmChart(chart, framework.Namespace)
920+
Expect(err).ToNot(HaveOccurred())
921+
922+
labelSelector := labels.SelectorFromSet(labels.Set{
923+
"owner": "helm",
924+
"name": chart.Name,
925+
})
926+
_, err = framework.WaitForRelease(chart, labelSelector, 120*time.Second, 1)
927+
Expect(err).ToNot(HaveOccurred())
928+
929+
chart, err = framework.GetHelmChart(chart.Name, chart.Namespace)
930+
Expect(err).ToNot(HaveOccurred())
931+
job, err = framework.GetJob(chart)
932+
Expect(err).ToNot(HaveOccurred())
933+
})
934+
FIt("Should have default tolerations set", func() {
935+
Expect(job.Spec.Template.Spec.Tolerations).To(Equal(*bootstrapTolerations))
936+
})
937+
AfterEach(func() {
938+
err = framework.DeleteHelmChart(chart.Name, framework.Namespace)
939+
Expect(err).ToNot(HaveOccurred())
940+
941+
Eventually(func() bool {
942+
_, err := framework.GetHelmChart(chart.Name, framework.Namespace)
943+
return err != nil && apierrors.IsNotFound(err)
944+
}, 120*time.Second, 5*time.Second).Should(BeTrue())
945+
})
946+
})
947+
948+
Context("When custom tolerations are specified (boostrap)", func() {
949+
var (
950+
err error
951+
chart *v1.HelmChart
952+
job *batchv1.Job
953+
specTolerations = &[]corev1.Toleration{
954+
{
955+
Key: "test-1",
956+
Effect: corev1.TaintEffectNoSchedule,
957+
},
958+
{
959+
Key: "test-2",
960+
Effect: corev1.TaintEffectNoExecute,
961+
},
962+
}
963+
finalTolerations = append(*bootstrapTolerations, *specTolerations...)
964+
)
965+
BeforeEach(func() {
966+
chart = framework.NewHelmChart("traefik-example-bootstrap-custom-tolerations",
967+
"stable/traefik",
968+
"1.86.1",
969+
"v3",
970+
map[string]intstr.IntOrString{
971+
"rbac.enabled": {
972+
Type: intstr.String,
973+
StrVal: "true",
974+
},
975+
"ssl.enabled": {
976+
Type: intstr.String,
977+
StrVal: "true",
978+
},
979+
})
980+
chart.Spec.Tolerations = *specTolerations
981+
chart.Spec.Bootstrap = true
982+
chart, err = framework.CreateHelmChart(chart, framework.Namespace)
983+
Expect(err).ToNot(HaveOccurred())
984+
985+
labelSelector := labels.SelectorFromSet(labels.Set{
986+
"owner": "helm",
987+
"name": chart.Name,
988+
})
989+
_, err = framework.WaitForRelease(chart, labelSelector, 120*time.Second, 1)
990+
Expect(err).ToNot(HaveOccurred())
991+
992+
chart, err = framework.GetHelmChart(chart.Name, chart.Namespace)
993+
Expect(err).ToNot(HaveOccurred())
994+
job, err = framework.GetJob(chart)
995+
Expect(err).ToNot(HaveOccurred())
996+
})
997+
FIt("Should have tolerations set", func() {
998+
Expect(job.Spec.Template.Spec.Tolerations).To(Equal(finalTolerations))
999+
})
1000+
AfterEach(func() {
1001+
err = framework.DeleteHelmChart(chart.Name, framework.Namespace)
1002+
Expect(err).ToNot(HaveOccurred())
1003+
1004+
Eventually(func() bool {
1005+
_, err := framework.GetHelmChart(chart.Name, framework.Namespace)
1006+
return err != nil && apierrors.IsNotFound(err)
1007+
}, 120*time.Second, 5*time.Second).Should(BeTrue())
1008+
})
1009+
})
1010+
7581011
})

0 commit comments

Comments
 (0)