Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/e2e-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -122,4 +122,18 @@ jobs:
echo "::group::warning events (all namespaces)"
kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp | tail -80
echo "::endgroup::"
# Broker/disk removal stalls are almost always a Cruise Control problem: a stuck CC
# Deployment rollout (two revisions racing) or a remove_broker task wedged InExecution.
# None of that is visible in the per-namespace pod dump above, so capture the koperator
# CRs and the CC rollout state explicitly.
echo "::group::kafka CRs + cruisecontrol rollout"
kubectl -n kafka get kafkacluster,cruisecontroloperation -o wide
echo "----- kafkacluster status.state -----"
kubectl -n kafka get kafkacluster -o jsonpath='{range .items[*]}{.metadata.name}{": "}{.status.state}{"\n"}{end}'
echo "----- cruisecontroloperation (yaml: currentTask.state / errorMessage) -----"
kubectl -n kafka get cruisecontroloperation -o yaml
echo "----- cruisecontrol deployment + replicasets (exposes stuck/duplicate revisions) -----"
kubectl -n kafka get deploy,rs -l app=cruisecontrol -o wide
kubectl -n kafka describe deploy -l app=cruisecontrol
echo "::endgroup::"
exit 0
4 changes: 2 additions & 2 deletions docs/examples/springboot-kafka-avro/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-schema-registry-client</artifactId> <!-- <1> -->
<version>8.3.0</version>
<version>8.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
Expand All @@ -56,7 +56,7 @@
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-avro-serializer</artifactId> <!-- <3> -->
<version>8.3.0</version>
<version>8.3.1</version>
</dependency>
</dependencies>

Expand Down
65 changes: 65 additions & 0 deletions tests/e2e/test_broker_removal.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package e2e

import (
"context"
"strconv"
"strings"
"time"

"github.com/gruntwork-io/terratest/modules/k8s"
Expand Down Expand Up @@ -60,6 +62,16 @@ func testBatchedBrokerRemoval() bool {
gomega.Eventually(context.Background(), func() (bool, error) {
return hasNoInFlightCruiseControlOperation(kubectlOptions)
}, batchedBrokerRemovalTimeout, batchedBrokerRemovalPollInterval).Should(gomega.BeTrue())

// The idle-operation gate above only checks CruiseControl's task queue, not the CC
// Deployment itself. If the operator is mid-rollout of a new CC revision, two CC pods
// race and the new one may never become Ready, resetting CC's metric-sampling window and
// stalling the remove_broker task. Gate on a settled CC Deployment so removal starts from
// a single, fully rolled-out CruiseControl pod.
ginkgo.By("Waiting until the Cruise Control Deployment is fully rolled out (single Ready replica)")
gomega.Eventually(context.Background(), func() (bool, error) {
return isCruiseControlDeploymentRolledOut(kubectlOptions)
}, batchedBrokerRemovalTimeout, batchedBrokerRemovalPollInterval).Should(gomega.BeTrue())
})

ginkgo.It("Applying 3-broker manifest to trigger removal of brokers 3 and 4", func() {
Expand Down Expand Up @@ -156,3 +168,56 @@ func hasExactlyNBrokerPods(kubectlOptions k8s.KubectlOptions, n int) (bool, erro
}
return len(pods) == n, nil
}

// isCruiseControlDeploymentRolledOut returns true when the Cruise Control Deployment has fully
// settled to a single Ready replica: the operator's latest spec has been observed
// (observedGeneration == generation) and the spec/total/updated/ready/available replica counts all
// agree, so no pod from a previous revision is lingering. Broker removal drives the operator to
// regenerate CruiseControl's capacity config; starting removal while CC is mid-rollout leaves two
// CC pods racing and resets CC's metric-sampling window, which can stall an in-flight remove_broker
// task. Gating on a settled CC Deployment avoids that race.
func isCruiseControlDeploymentRolledOut(kubectlOptions k8s.KubectlOptions) (bool, error) {
// One line per CC Deployment: generation/observedGeneration/spec.replicas/status.replicas/
// updatedReplicas/readyReplicas/availableReplicas. Absent status counts render as empty.
lines, err := getK8sResources(kubectlOptions,
[]string{"deployment"},
v1beta1.KafkaCRLabelKey+"="+kafkaClusterName+",app=cruisecontrol",
"",
"-o", "jsonpath={range .items[*]}{.metadata.generation}/{.status.observedGeneration}/{.spec.replicas}/{.status.replicas}/{.status.updatedReplicas}/{.status.readyReplicas}/{.status.availableReplicas}{'\\n'}{end}",
)
if err != nil {
return false, err
}
// Exactly one Cruise Control Deployment is expected; anything else is not a settled state.
if len(lines) != 1 {
return false, nil
}

fields := strings.Split(lines[0], "/")
if len(fields) != 7 {
return false, nil
}
nums := make([]int, len(fields))
for i, f := range fields {
if f == "" {
// A missing status replica count (e.g. readyReplicas before any pod is Ready) is 0.
nums[i] = 0
continue
}
n, convErr := strconv.Atoi(f)
if convErr != nil {
return false, nil
}
nums[i] = n
}

generation, observed := nums[0], nums[1]
specReplicas, statusReplicas, updated, ready, available := nums[2], nums[3], nums[4], nums[5], nums[6]

return observed == generation &&
specReplicas >= 1 &&
statusReplicas == specReplicas &&
updated == specReplicas &&
ready == specReplicas &&
available == specReplicas, nil
}
2 changes: 1 addition & 1 deletion tests/e2e/versions.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ const (
ContourVersion = "0.6.0" // renovate: datasource=helm depName=contour registryUrl=https://projectcontour.github.io/helm-charts

// PrometheusOperatorVersion is the version of kube-prometheus-stack Helm chart
PrometheusOperatorVersion = "88.1.5" // renovate: datasource=helm depName=kube-prometheus-stack registryUrl=https://prometheus-community.github.io/helm-charts
PrometheusOperatorVersion = "88.2.0" // renovate: datasource=helm depName=kube-prometheus-stack registryUrl=https://prometheus-community.github.io/helm-charts

// ZookeeperOperatorVersion is the version of zookeeper-operator
ZookeeperOperatorVersion = "0.2.15-adobe-20260423" // renovate: datasource=docker depName=ghcr.io/adobe/helm-charts/zookeeper-operator
Expand Down