Skip to content

[Feat] Volume mount service secrets on workloads #72

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 19 commits into
base: main
Choose a base branch
from
+1,813 −35
Open

[Feat] Volume mount service secrets on workloads #72

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
eef9c11
added secret volume mounts
anirudhprasad-sap Feb 12, 2024
f3da4ee
Added env check for volume mounts
anirudhprasad-sap Mar 7, 2024
3a5a9b2
Merge branch 'volumeMount' of https://github.com/SAP/cap-operator int…
anirudhprasad-sap Mar 8, 2024
bf3b034
added secret volume mounts
anirudhprasad-sap Feb 12, 2024
885298e
Added env check for volume mounts
anirudhprasad-sap Mar 7, 2024
bb3938d
Merge branch 'volumeMount' of https://github.com/SAP/cap-operator int…
anirudhprasad-sap Oct 2, 2024
57eb28b
use annotation check
anirudhprasad-sap Oct 8, 2024
645f0cb
fix env getting set on CAV
anirudhprasad-sap Oct 9, 2024
6639528
minor corrections
anirudhprasad-sap Oct 15, 2024
24653a8
Merge branch 'main' into volumeMount
anirudhprasad-sap Oct 15, 2024
2e4f5f8
Merge branch 'main' into volumeMount
anirudhprasad-sap Oct 15, 2024
422e152
adopt test case
anirudhprasad-sap Oct 15, 2024
460192d
added documentation
anirudhprasad-sap Oct 16, 2024
6a3e6a8
documentation updated
anirudhprasad-sap Oct 17, 2024
388b7d5
renamed method names to be more specific
anirudhprasad-sap Oct 17, 2024
89525e4
use service binding name instead of class name in the volume mounts
anirudhprasad-sap Oct 24, 2024
314a5a9
test cases adopted to use service name in volume mount path
anirudhprasad-sap Oct 24, 2024
e529752
Merge branch 'main' into volumeMount
anirudhprasad-sap Oct 24, 2024
ddff3c5
Merge branch 'main' into volumeMount
anirudhprasad-sap Dec 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 53 additions & 16 deletions internal/controller/reconcile-capapplicationversion.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,10 @@ type DeploymentParameters struct {
CAV *v1alpha1.CAPApplicationVersion
OwnerRef *metav1.OwnerReference
WorkloadDetails v1alpha1.WorkloadDetails
VCAPSecretName string
Env []corev1.EnvVar
EnvFrom []corev1.EnvFromSource
Volumes []corev1.Volume
VolumeMounts []corev1.VolumeMount
}

func (c *Controller) reconcileCAPApplicationVersion(ctx context.Context, item QueueItem, attempts int) (*ReconcileResult, error) {
Expand Down Expand Up @@ -260,10 +263,13 @@ func (c *Controller) handleContentDeployJob(ca *v1alpha1.CAPApplication, cav *v1
ownerRef := *metav1.NewControllerRef(cav, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPApplicationVersionKind))

// Get VCAP secret name
vcapSecretName, err = createVCAPSecret(jobName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient)
err = nil
if !useVolumeMountsForServiceCredentials(cav) {
vcapSecretName, err = createVCAPSecret(jobName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient)
}

if err == nil {
contentDeployJob, err = c.kubeClient.BatchV1().Jobs(cav.Namespace).Create(context.TODO(), newContentDeploymentJob(cav, workload, ownerRef, vcapSecretName), metav1.CreateOptions{})
contentDeployJob, err = c.kubeClient.BatchV1().Jobs(cav.Namespace).Create(context.TODO(), newContentDeploymentJob(ca, cav, workload, ownerRef, vcapSecretName), metav1.CreateOptions{})
if err == nil {
util.LogInfo("Content job created successfully", string(Processing), cav, contentDeployJob, "version", cav.Spec.Version)
}
Expand All @@ -274,7 +280,7 @@ func (c *Controller) handleContentDeployJob(ca *v1alpha1.CAPApplication, cav *v1
}

// newContentDeploymentJob creates a Content Deployment Job for the CAV resource. It also sets the appropriate OwnerReferences.
func newContentDeploymentJob(cav *v1alpha1.CAPApplicationVersion, workload *v1alpha1.WorkloadDetails, ownerRef metav1.OwnerReference, vcapSecretName string) *batchv1.Job {
func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVersion, workload *v1alpha1.WorkloadDetails, ownerRef metav1.OwnerReference, vcapSecretName string) *batchv1.Job {
labels := copyMaps(workload.Labels, map[string]string{
LabelDisableKarydia: "true",
})
Expand All @@ -287,6 +293,23 @@ func newContentDeploymentJob(cav *v1alpha1.CAPApplicationVersion, workload *v1al

util.LogInfo("Creating content job", string(Processing), cav, nil, "contentJobName", contentJobName, "version", cav.Spec.Version)

var envFrom []corev1.EnvFromSource
var serviceSecretVolumeMounts []corev1.VolumeMount
var serviceSecretVolumes []corev1.Volume

env := workload.JobDefinition.Env

if useVolumeMountsForServiceCredentials(cav) {
// Get ServiceInfos for consumed BTP services
consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services)

env = updateServiceBindingRootEnv(env)
serviceSecretVolumeMounts = getServiceCredentialVolumeMounts(consumedServiceInfos)
serviceSecretVolumes = getServiceCredentialVolumes(consumedServiceInfos)
} else {
envFrom = getEnvFrom(vcapSecretName)
}

return &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: contentJobName,
Expand Down Expand Up @@ -314,19 +337,19 @@ func newContentDeploymentJob(cav *v1alpha1.CAPApplicationVersion, workload *v1al
Args: workload.JobDefinition.Args,
Env: append([]corev1.EnvVar{
{Name: EnvCAPOpAppVersion, Value: cav.Spec.Version},
}, workload.JobDefinition.Env...),
EnvFrom: getEnvFrom(vcapSecretName),
VolumeMounts: workload.JobDefinition.VolumeMounts,
}, env...),
EnvFrom: envFrom,
VolumeMounts: append(workload.JobDefinition.VolumeMounts, serviceSecretVolumeMounts...),
Resources: workload.JobDefinition.Resources,
SecurityContext: workload.JobDefinition.SecurityContext,
},
},
InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, []corev1.EnvVar{
{Name: EnvCAPOpAppVersion, Value: cav.Spec.Version},
}, vcapSecretName),
}, serviceSecretVolumeMounts, envFrom),
SecurityContext: workload.JobDefinition.PodSecurityContext,
ServiceAccountName: workload.JobDefinition.ServiceAccountName,
Volumes: workload.JobDefinition.Volumes,
Volumes: append(workload.JobDefinition.Volumes, serviceSecretVolumes...),
ImagePullSecrets: convertToLocalObjectReferences(cav.Spec.RegistrySecrets),
RestartPolicy: getRestartPolicy(workload.JobDefinition.RestartPolicy, true),
NodeSelector: workload.JobDefinition.NodeSelector,
Expand Down Expand Up @@ -675,7 +698,10 @@ func (c *Controller) updateDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1
ownerRef := *metav1.NewControllerRef(cav, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPApplicationVersionKind))

// Get VCAP secret name
vcapSecretName, err = createVCAPSecret(deploymentName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient)
err = nil
if !useVolumeMountsForServiceCredentials(cav) {
vcapSecretName, err = createVCAPSecret(deploymentName, cav.Namespace, ownerRef, consumedServiceInfos, c.kubeClient)
}

if err == nil {
workloadDeployment, err = c.kubeClient.AppsV1().Deployments(cav.Namespace).Create(context.TODO(), newDeployment(ca, cav, workload, ownerRef, vcapSecretName), metav1.CreateOptions{})
Expand All @@ -695,7 +721,18 @@ func newDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVers
CAV: cav,
OwnerRef: &ownerRef,
WorkloadDetails: *workload,
VCAPSecretName: vcapSecretName,
Env: workload.DeploymentDefinition.Env,
}

if useVolumeMountsForServiceCredentials(cav) {
// Get ServiceInfos for consumed BTP services
consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services)

params.Env = updateServiceBindingRootEnv(params.Env)
params.VolumeMounts = getServiceCredentialVolumeMounts(consumedServiceInfos)
params.Volumes = getServiceCredentialVolumes(consumedServiceInfos)
} else {
params.EnvFrom = getEnvFrom(vcapSecretName)
}

return createDeployment(params)
Expand Down Expand Up @@ -732,10 +769,10 @@ func createDeployment(params *DeploymentParameters) *appsv1.Deployment {
ImagePullSecrets: convertToLocalObjectReferences(params.CAV.Spec.RegistrySecrets),
InitContainers: *updateInitContainers(params.WorkloadDetails.DeploymentDefinition.InitContainers, []corev1.EnvVar{
{Name: EnvCAPOpAppVersion, Value: params.CAV.Spec.Version},
}, params.VCAPSecretName),
}, params.VolumeMounts, params.EnvFrom),
Containers: getContainer(params),
ServiceAccountName: params.WorkloadDetails.DeploymentDefinition.ServiceAccountName,
Volumes: params.WorkloadDetails.DeploymentDefinition.Volumes,
Volumes: append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volumes...),
SecurityContext: params.WorkloadDetails.DeploymentDefinition.PodSecurityContext,
NodeSelector: params.WorkloadDetails.DeploymentDefinition.NodeSelector,
NodeName: params.WorkloadDetails.DeploymentDefinition.NodeName,
Expand All @@ -758,8 +795,8 @@ func getContainer(params *DeploymentParameters) []corev1.Container {
Command: params.WorkloadDetails.DeploymentDefinition.Command,
Args: params.WorkloadDetails.DeploymentDefinition.Args,
Env: getEnv(params),
EnvFrom: getEnvFrom(params.VCAPSecretName),
VolumeMounts: params.WorkloadDetails.DeploymentDefinition.VolumeMounts,
EnvFrom: params.EnvFrom,
VolumeMounts: append(params.WorkloadDetails.DeploymentDefinition.VolumeMounts, params.VolumeMounts...),
LivenessProbe: params.WorkloadDetails.DeploymentDefinition.LivenessProbe,
ReadinessProbe: params.WorkloadDetails.DeploymentDefinition.ReadinessProbe,
Resources: params.WorkloadDetails.DeploymentDefinition.Resources,
Expand All @@ -772,7 +809,7 @@ func getEnv(params *DeploymentParameters) []corev1.EnvVar {
env := []corev1.EnvVar{
{Name: EnvCAPOpAppVersion, Value: params.CAV.Spec.Version},
}
env = append(env, params.WorkloadDetails.DeploymentDefinition.Env...)
env = append(env, params.Env...)

if params.WorkloadDetails.DeploymentDefinition.Type == v1alpha1.DeploymentRouter {
// Add destinations env for `Router`
Expand Down
18 changes: 18 additions & 0 deletions internal/controller/reconcile-capapplicationversion_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -847,3 +847,21 @@ func TestCAV_InvalidMonitoringConfig(t *testing.T) {

}
}

func TestCAV_UseVolumeMount(t *testing.T) {
reconcileTestItem(
context.TODO(), t,
QueueItem{Key: ResourceCAPApplicationVersion, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-cav-v1"}},
TestData{
description: "capapplication version with initContainers",
initialResources: []string{
"testdata/common/capapplication.yaml",
"testdata/common/credential-secrets.yaml",
"testdata/capapplicationversion/cav-use-vol-mount.yaml",
},
expectedResources: "testdata/capapplicationversion/expected/cav-processing-use-vol-mount.yaml",
expectedRequeue: map[int][]NamespacedResourceKey{ResourceCAPApplicationVersion: {{Namespace: "default", Name: "test-cap-01-cav-v1"}}},
backlogItems: []string{},
},
)
}
56 changes: 40 additions & 16 deletions internal/controller/reconcile-captenantoperation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,11 +391,19 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c
return nil, fmt.Errorf("could not find workload %s in %s %s.%s", step.Name, v1alpha1.CAPApplicationVersionKind, relatedResources.CAPApplicationVersion.Namespace, relatedResources.CAPApplicationVersion.Name)
}

// create VCAP secret from consumed BTP services
consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), relatedResources.CAPApplication.Spec.BTP.Services)
vcapSecretName, err := createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient)
if err != nil {
return nil, err

// check volume mount annotation
useVolumeMountsForServiceCredentials := useVolumeMountsForServiceCredentials(relatedResources.CAPApplicationVersion)

// create VCAP secret from consumed BTP services
var vcapSecretName string
err = nil
if !useVolumeMountsForServiceCredentials {
vcapSecretName, err = createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient)
if err != nil {
return nil, err
}
}

annotations := copyMaps(workload.Annotations, map[string]string{
Expand All @@ -420,7 +428,6 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c
namePrefix: relatedResources.CAPTenant.Name + "-" + workload.Name + "-",
labels: labels,
annotations: annotations,
vcapSecretName: vcapSecretName,
imagePullSecrets: convertToLocalObjectReferences(relatedResources.CAPApplicationVersion.Spec.RegistrySecrets),
version: relatedResources.CAPApplicationVersion.Spec.Version,
appName: relatedResources.CAPApplication.Spec.BTPAppName,
Expand All @@ -430,6 +437,20 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c
tenantType: relatedResources.CAPTenant.Labels[LabelTenantType],
}

if workload.DeploymentDefinition == nil {
params.Env = workload.JobDefinition.Env
} else {
params.Env = workload.DeploymentDefinition.Env
}

if useVolumeMountsForServiceCredentials {
params.Env = updateServiceBindingRootEnv(params.Env)
params.volumeMounts = getServiceCredentialVolumeMounts(consumedServiceInfos)
params.volumes = getServiceCredentialVolumes(consumedServiceInfos)
} else {
params.EnvFrom = getEnvFrom(vcapSecretName)
}

var job *batchv1.Job
if ctop.Spec.Steps[*ctop.Status.CurrentStep-1].Type == v1alpha1.JobTenantOperation {
job, err = c.createTenantOperationJob(ctx, ctop, workload, params)
Expand All @@ -454,14 +475,17 @@ type jobCreateParams struct {
namePrefix string
labels map[string]string
annotations map[string]string
vcapSecretName string
imagePullSecrets []corev1.LocalObjectReference
version string
appName string
globalAccountId string
providerTenantId string
providerSubdomain string
tenantType string
Env []corev1.EnvVar
EnvFrom []corev1.EnvFromSource
volumes []corev1.Volume
volumeMounts []corev1.VolumeMount
}

func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha1.CAPTenantOperation, workload *v1alpha1.WorkloadDetails, params *jobCreateParams) (*batchv1.Job, error) {
Expand All @@ -488,8 +512,8 @@ func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha
RestartPolicy: getRestartPolicy(derivedWorkload.restartPolicy, true),
ImagePullSecrets: params.imagePullSecrets,
Containers: getContainers(ctop, derivedWorkload, workload, params),
InitContainers: *updateInitContainers(derivedWorkload.initContainers, getCTOPEnv(params, ctop, v1alpha1.JobTenantOperation), params.vcapSecretName),
Volumes: derivedWorkload.volumes,
InitContainers: *updateInitContainers(derivedWorkload.initContainers, getCTOPEnv(params, ctop, v1alpha1.JobTenantOperation), params.volumeMounts, params.EnvFrom),
Volumes: append(derivedWorkload.volumes, params.volumes...),
ServiceAccountName: derivedWorkload.serviceAccountName,
SecurityContext: derivedWorkload.podSecurityContext,
NodeSelector: derivedWorkload.nodeSelector,
Expand All @@ -512,9 +536,9 @@ func getContainers(ctop *v1alpha1.CAPTenantOperation, derivedWorkload tentantOpe
Name: workload.Name,
Image: derivedWorkload.image,
ImagePullPolicy: derivedWorkload.imagePullPolicy,
Env: append(getCTOPEnv(params, ctop, v1alpha1.JobTenantOperation), derivedWorkload.env...),
EnvFrom: getEnvFrom(params.vcapSecretName),
VolumeMounts: derivedWorkload.volumeMounts,
Env: append(getCTOPEnv(params, ctop, v1alpha1.JobTenantOperation), params.Env...),
EnvFrom: params.EnvFrom,
VolumeMounts: append(derivedWorkload.volumeMounts, params.volumeMounts...),
Resources: derivedWorkload.resources,
SecurityContext: derivedWorkload.securityContext,
}
Expand Down Expand Up @@ -607,7 +631,7 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v
Spec: corev1.PodSpec{
RestartPolicy: getRestartPolicy(workload.JobDefinition.RestartPolicy, true),
SecurityContext: workload.JobDefinition.PodSecurityContext,
Volumes: workload.JobDefinition.Volumes,
Volumes: append(workload.JobDefinition.Volumes, params.volumes...),
ServiceAccountName: workload.JobDefinition.ServiceAccountName,
NodeSelector: workload.JobDefinition.NodeSelector,
NodeName: workload.JobDefinition.NodeName,
Expand All @@ -621,16 +645,16 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v
Name: workload.Name,
Image: workload.JobDefinition.Image,
ImagePullPolicy: workload.JobDefinition.ImagePullPolicy,
Env: append(getCTOPEnv(params, ctop, v1alpha1.JobCustomTenantOperation), workload.JobDefinition.Env...),
EnvFrom: getEnvFrom(params.vcapSecretName),
VolumeMounts: workload.JobDefinition.VolumeMounts,
Env: append(getCTOPEnv(params, ctop, v1alpha1.JobCustomTenantOperation), params.Env...),
EnvFrom: params.EnvFrom,
VolumeMounts: append(workload.JobDefinition.VolumeMounts, params.volumeMounts...),
Command: workload.JobDefinition.Command,
Args: workload.JobDefinition.Args,
Resources: workload.JobDefinition.Resources,
SecurityContext: workload.JobDefinition.SecurityContext,
},
},
InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, getCTOPEnv(params, ctop, v1alpha1.JobCustomTenantOperation), params.vcapSecretName),
InitContainers: *updateInitContainers(workload.JobDefinition.InitContainers, getCTOPEnv(params, ctop, v1alpha1.JobCustomTenantOperation), params.volumeMounts, params.EnvFrom),
},
},
},
Expand Down
44 changes: 44 additions & 0 deletions internal/controller/reconcile-captenantoperation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,3 +731,47 @@ func TestProvisioningWithInitContainersCustom(t *testing.T) {
},
)
}

func TestProvisioningUsingVolMountForServices(t *testing.T) {
_ = reconcileTestItem(
context.TODO(), t,
QueueItem{Key: ResourceCAPTenantOperation, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider-abcd"}},
TestData{
backlogItems: []string{},
description: "Provisioning - Using service volume mount for TenantOperation",
initialResources: []string{
"testdata/common/capapplication.yaml",
"testdata/common/captenant-provider-ready.yaml",
"testdata/common/capapplicationversion-v1-use-vol-mount.yaml",
"testdata/common/credential-secrets.yaml",
"testdata/captenantoperation/ctop-use-vol-mount.initial.yaml",
},
expectedResources: "testdata/captenantoperation/ctop-use-vol-mount.expected.yaml",
expectedRequeue: map[int][]NamespacedResourceKey{
ResourceCAPTenantOperation: {{Namespace: "default", Name: "test-cap-01-provider-abcd"}},
},
},
)
}

func TestProvisioningUsingVolMountForServicesCustom(t *testing.T) {
_ = reconcileTestItem(
context.TODO(), t,
QueueItem{Key: ResourceCAPTenantOperation, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider-abcd"}},
TestData{
backlogItems: []string{},
description: "Provisioning - Using service volume mount for CustomTenantOperation",
initialResources: []string{
"testdata/common/capapplication.yaml",
"testdata/common/captenant-provider-ready.yaml",
"testdata/common/capapplicationversion-v1-use-vol-mount-custom.yaml",
"testdata/common/credential-secrets.yaml",
"testdata/captenantoperation/ctop-use-vol-mount-custom.initial.yaml",
},
expectedResources: "testdata/captenantoperation/ctop-use-vol-mount-custom.expected.yaml",
expectedRequeue: map[int][]NamespacedResourceKey{
ResourceCAPTenantOperation: {{Namespace: "default", Name: "test-cap-01-provider-abcd"}},
},
},
)
}
Loading
Loading