CCO-738, CCO-739: Set operator condition to Progressing when pod identity webhook pod are updating

pkg/operator/cleanup/status.go

@@ -1,6 +1,8 @@
 package cleanup
 
 import (
+	"context"
+
 	log "github.com/sirupsen/logrus"
 
 	configv1 "github.com/openshift/api/config/v1"
@@ -9,7 +11,7 @@ import (
 
 var _ status.Handler = &ReconcileStaleCredentialsRequest{}
 
-func (r *ReconcileStaleCredentialsRequest) GetConditions(logger log.FieldLogger) ([]configv1.ClusterOperatorStatusCondition, error) {
+func (r *ReconcileStaleCredentialsRequest) GetConditions(ctx context.Context, logger log.FieldLogger) ([]configv1.ClusterOperatorStatusCondition, error) {
 	return []configv1.ClusterOperatorStatusCondition{}, nil
 }
 

pkg/operator/credentialsrequest/credentialsrequest_controller.go

@@ -493,9 +493,9 @@ type ReconcileSecretMissingLabel struct {
 	mutatingClient corev1client.SecretsGetter
 }
 
-func (r *ReconcileSecretMissingLabel) GetConditions(logger log.FieldLogger) ([]configv1.ClusterOperatorStatusCondition, error) {
+func (r *ReconcileSecretMissingLabel) GetConditions(ctx context.Context, logger log.FieldLogger) ([]configv1.ClusterOperatorStatusCondition, error) {
 	var secrets corev1.SecretList
-	if err := r.cachedClient.List(context.TODO(), &secrets); err != nil {
+	if err := r.cachedClient.List(ctx, &secrets); err != nil {
 		return nil, err
 	}
 	var missing int

pkg/operator/credentialsrequest/credentialsrequest_controller_azure_test.go

@@ -253,7 +253,7 @@ func TestCredentialsRequestAzureReconcile(t *testing.T) {
 
 			if test.expectedCOConditions != nil {
 				logger := log.WithFields(log.Fields{"controller": controllerName})
-				currentConditions, err := rcr.GetConditions(logger)
+				currentConditions, err := rcr.GetConditions(context.TODO(), logger)
 				require.NoError(t, err, "failed getting conditions")
 
 				for _, expectedCondition := range test.expectedCOConditions {

pkg/operator/credentialsrequest/credentialsrequest_controller_gcp_test.go

@@ -974,7 +974,7 @@ func TestCredentialsRequestGCPReconcile(t *testing.T) {
 
 			if test.expectedCOConditions != nil {
 				logger := log.WithFields(log.Fields{"controller": controllerName})
-				currentConditions, err := rcr.GetConditions(logger)
+				currentConditions, err := rcr.GetConditions(context.TODO(), logger)
 				require.NoError(t, err, "failed getting conditions")
 
 				for _, expectedCondition := range test.expectedCOConditions {

pkg/operator/credentialsrequest/credentialsrequest_controller_label_test.go

@@ -251,7 +251,7 @@ func TestReconcileSecretMissingLabel_GetConditions(t *testing.T) {
 				mutatingClient: nil,
 			}
 
-			conditions, err := rcr.GetConditions(logrus.StandardLogger())
+			conditions, err := rcr.GetConditions(context.TODO(), logrus.StandardLogger())
 
 			if err != nil && !test.expectErr {
 				require.NoError(t, err, "Unexpected error: %v", err)

pkg/operator/credentialsrequest/credentialsrequest_controller_test.go

@@ -1535,7 +1535,7 @@ func TestCredentialsRequestReconcile(t *testing.T) {
 
 			if test.expectedCOConditions != nil {
 				logger := log.WithFields(log.Fields{"controller": controllerName})
-				currentConditions, err := rcr.GetConditions(logger)
+				currentConditions, err := rcr.GetConditions(context.TODO(), logger)
 				require.NoError(t, err, "failed getting conditions")
 
 				for _, expectedCondition := range test.expectedCOConditions {

pkg/operator/credentialsrequest/credentialsrequest_controller_vsphere_test.go

@@ -226,7 +226,7 @@ func TestCredentialsRequestVSphereReconcile(t *testing.T) {
 
 			if test.expectedCOConditions != nil {
 				logger := log.WithFields(log.Fields{"controller": controllerName})
-				currentConditions, err := rcr.GetConditions(logger)
+				currentConditions, err := rcr.GetConditions(context.TODO(), logger)
 				require.NoError(t, err, "failed getting conditions")
 
 				for _, expectedCondition := range test.expectedCOConditions {

pkg/operator/credentialsrequest/status.go

@@ -29,8 +29,8 @@ const (
 
 var _ status.Handler = &ReconcileCredentialsRequest{}
 
-func (r *ReconcileCredentialsRequest) GetConditions(logger log.FieldLogger) ([]configv1.ClusterOperatorStatusCondition, error) {
-	_, credRequests, mode, err := r.getOperatorState(logger)
+func (r *ReconcileCredentialsRequest) GetConditions(ctx context.Context, logger log.FieldLogger) ([]configv1.ClusterOperatorStatusCondition, error) {
+	_, credRequests, mode, err := r.getOperatorState(ctx, logger)
 	if err != nil {
 		return []configv1.ClusterOperatorStatusCondition{}, fmt.Errorf("failed to get operator state: %v", err)
 	}
@@ -43,7 +43,7 @@ func (r *ReconcileCredentialsRequest) GetConditions(logger log.FieldLogger) ([]c
 }
 
 func (r *ReconcileCredentialsRequest) GetRelatedObjects(logger log.FieldLogger) ([]configv1.ObjectReference, error) {
-	_, credRequests, _, err := r.getOperatorState(logger)
+	_, credRequests, _, err := r.getOperatorState(context.TODO(), logger)
 	if err != nil {
 		return []configv1.ObjectReference{}, fmt.Errorf("failed to get operator state: %v", err)
 	}
@@ -56,10 +56,10 @@ func (r *ReconcileCredentialsRequest) Name() string {
 
 // getOperatorState gets and returns the resources necessary to compute the
 // operator's current state.
-func (r *ReconcileCredentialsRequest) getOperatorState(logger log.FieldLogger) (*corev1.Namespace, []minterv1.CredentialsRequest, operatorv1.CloudCredentialsMode, error) {
+func (r *ReconcileCredentialsRequest) getOperatorState(ctx context.Context, logger log.FieldLogger) (*corev1.Namespace, []minterv1.CredentialsRequest, operatorv1.CloudCredentialsMode, error) {
 
 	ns := &corev1.Namespace{}
-	if err := r.Client.Get(context.TODO(), types.NamespacedName{Name: minterv1.CloudCredOperatorNamespace}, ns); err != nil {
+	if err := r.Client.Get(ctx, types.NamespacedName{Name: minterv1.CloudCredOperatorNamespace}, ns); err != nil {
 		if apierrors.IsNotFound(err) {
 			return nil, nil, operatorv1.CloudCredentialsModeDefault, nil
 		}
@@ -72,7 +72,7 @@ func (r *ReconcileCredentialsRequest) getOperatorState(logger log.FieldLogger) (
 	// central list to live. Other credentials requests in other namespaces will not affect status,
 	// but they will still work fine.
 	credRequestList := &minterv1.CredentialsRequestList{}
-	err := r.Client.List(context.TODO(), credRequestList, client.InNamespace(minterv1.CloudCredOperatorNamespace))
+	err := r.Client.List(ctx, credRequestList, client.InNamespace(minterv1.CloudCredOperatorNamespace))
 	if err != nil {
 		return nil, nil, operatorv1.CloudCredentialsModeDefault, fmt.Errorf(
 			"failed to list CredentialsRequests: %v", err)

pkg/operator/loglevel/status.go

@@ -1,6 +1,8 @@
 package loglevel
 
 import (
+	"context"
+
 	log "github.com/sirupsen/logrus"
 
 	configv1 "github.com/openshift/api/config/v1"
@@ -9,7 +11,7 @@ import (
 
 var _ status.Handler = &ReconcileCloudCredentialConfig{}
 
-func (r *ReconcileCloudCredentialConfig) GetConditions(logger log.FieldLogger) ([]configv1.ClusterOperatorStatusCondition, error) {
+func (r *ReconcileCloudCredentialConfig) GetConditions(ctx context.Context, logger log.FieldLogger) ([]configv1.ClusterOperatorStatusCondition, error) {
 	return []configv1.ClusterOperatorStatusCondition{}, nil
 }
 

pkg/operator/podidentity/podidentitywebhook_controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"sync"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -37,12 +38,14 @@ import (
 )
 
 const (
-	controllerName                      = "pod-identity"
-	deploymentName                      = "cloud-credential-operator"
-	operatorNamespace                   = "openshift-cloud-credential-operator"
-	retryInterval                       = 10 * time.Second
-	reasonStaticResourceReconcileFailed = "StaticResourceReconcileFailed"
-	pdb                                 = "v4.1.0/common/poddisruptionbudget.yaml"
+	controllerName                            = "pod-identity"
+	deploymentName                            = "cloud-credential-operator"
+	operatorNamespace                         = "openshift-cloud-credential-operator"
+	retryInterval                             = 10 * time.Second
+	reasonStaticResourceReconcileFailed       = "StaticResourceReconcileFailed"
+	reasonPodIdentityWebhookPodsStillUpdating = "PodIdentityWebhookPodsStillUpdating"
+	reasonPodIdentityWebhookPodsNotAvailable  = "PodIdentityWebhookPodsNotAvailable"
+	pdb                                       = "v4.1.0/common/poddisruptionbudget.yaml"
 )
 
 var (
@@ -198,6 +201,7 @@ func Add(mgr, rootCredentialManager manager.Manager, kubeconfig string) error {
 		logger:          logger,
 		eventRecorder:   eventRecorder,
 		imagePullSpec:   imagePullSpec,
+		conditionsMutex: &sync.RWMutex{},
 		conditions:      []configv1.ClusterOperatorStatusCondition{},
 		cache:           resourceapply.NewResourceCache(),
 		podIdentityType: podIdentityType,
@@ -282,6 +286,7 @@ type staticResourceReconciler struct {
 	eventRecorder        events.Recorder
 	deploymentGeneration int64
 	imagePullSpec        string
+	conditionsMutex      *sync.RWMutex
 	conditions           []configv1.ClusterOperatorStatusCondition
 	cache                resourceapply.ResourceCache
 	podIdentityType      PodIdentityManifestSource
@@ -291,6 +296,8 @@ var _ reconcile.Reconciler = &staticResourceReconciler{}
 
 func (r *staticResourceReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
 	r.logger.Debugf("reconciling after watch event %#v", request)
+	r.conditionsMutex.Lock()
+
 	err := r.ReconcileResources(ctx)
 	if err != nil {
 		r.logger.Errorf("reconciliation failed, retrying in %s", retryInterval.String())
@@ -302,9 +309,13 @@ func (r *staticResourceReconciler) Reconcile(ctx context.Context, request reconc
 				Message: fmt.Sprintf("static resource reconciliation failed: %v", err),
 			},
 		}
+
+		r.conditionsMutex.Unlock()
 		return reconcile.Result{RequeueAfter: retryInterval}, err
 	}
+
 	r.conditions = []configv1.ClusterOperatorStatusCondition{}
+	r.conditionsMutex.Unlock()
 	return reconcile.Result{}, nil
 }
 
@@ -384,10 +395,86 @@ func (r *staticResourceReconciler) ReconcileResources(ctx context.Context) error
 	return nil
 }
 
+type webhookPodStatus struct {
+	desiredReplicas   int32
+	availableReplicas int32
+	updatedReplicas   int32
+	totalReplicas     int32
+}
+
+func (wps *webhookPodStatus) Available() bool {
+	return wps.availableReplicas > 0
+}
+
+func (wps *webhookPodStatus) Progressing() bool {
+	return wps.updatedReplicas != wps.desiredReplicas || wps.totalReplicas != wps.desiredReplicas
+}
+
+func (r *staticResourceReconciler) CheckPodStatus(ctx context.Context) (*webhookPodStatus, error) {
+	deployment := &appsv1.Deployment{}
+	err := r.client.Get(ctx, client.ObjectKey{
+		Name:      "pod-identity-webhook",
+		Namespace: operatorNamespace,
+	}, deployment)
+	if err != nil {
+		return nil, err
+	}
+
+	desiredReplicas := int32(1) // kubernetes defaults to 1 if not specified
+	if deployment.Spec.Replicas != nil {
+		desiredReplicas = *deployment.Spec.Replicas
+	}
+
+	deploymentStatus := &webhookPodStatus{
+		desiredReplicas:   desiredReplicas,
+		availableReplicas: deployment.Status.AvailableReplicas,
+		updatedReplicas:   deployment.Status.UpdatedReplicas,
+		totalReplicas:     deployment.Status.Replicas,
+	}
+
+	return deploymentStatus, nil
+}
+
 var _ status.Handler = &staticResourceReconciler{}
 
-func (r *staticResourceReconciler) GetConditions(logger log.FieldLogger) ([]configv1.ClusterOperatorStatusCondition, error) {
-	return r.conditions, nil
+func (r *staticResourceReconciler) GetConditions(ctx context.Context, logger log.FieldLogger) ([]configv1.ClusterOperatorStatusCondition, error) {
+	r.conditionsMutex.RLock()
+	conditions := make([]configv1.ClusterOperatorStatusCondition, len(r.conditions))
+	copy(conditions, r.conditions)
+	r.conditionsMutex.RUnlock()
+
+	podStatus, err := r.CheckPodStatus(ctx)
+	if err != nil {
+		logger.WithError(err).Errorf("checking pod identity webhook status failed")
+		if len(conditions) > 0 {
+			// Ignore the pod status error for now, we have bigger conditions to report
+			return conditions, nil
+		}
+		return nil, err
+	}
+
+	if !podStatus.Available() {
+		conditions = append(conditions, configv1.ClusterOperatorStatusCondition{
+			Type:    configv1.OperatorAvailable,
+			Status:  configv1.ConditionFalse,
+			Reason:  reasonPodIdentityWebhookPodsNotAvailable,
+			Message: "No pod identity webhook pods are available",
+		})
+	}
+	if podStatus.Progressing() {
+		conditions = append(conditions, configv1.ClusterOperatorStatusCondition{
+			Type:   configv1.OperatorProgressing,
+			Status: configv1.ConditionTrue,
+			Reason: reasonPodIdentityWebhookPodsStillUpdating,
+			Message: fmt.Sprintf(
+				"Waiting for pod identity webhook deployment rollout to finish: %d out of %d new replica(s) have been updated...",
+				podStatus.updatedReplicas,
+				podStatus.desiredReplicas,
+			),
+		})
+	}
+
+	return conditions, nil
 }
 
 func (r *staticResourceReconciler) GetRelatedObjects(logger log.FieldLogger) ([]configv1.ObjectReference, error) {