feat(activate): gate cluster activation on failure-domain readiness

reconcileActivate fired POST /activate as soon as Spec.Action==Activate
and the cluster UUID resolved, with no check on node count, domain
count, or balance. If the backend correctly refused an under-provisioned
FD cluster (fewer than npcs+2 distinct domains, or unequal per-domain
host counts -- see simplyblock_core's fd_activation_domain_count_violation),
the CR got permanently stuck: failActivate sets ActionStatus.State=Failed,
and nothing in Reconcile/reconcileActivate ever resets ActionStatus away
from Failed back to Running, so the next reconcile falls straight through
to the GET-polling tail and loops forever without ever retrying the POST.

Added a readiness check at the very top of reconcileActivate, before any
ActionStatus mutation: for FD-enabled clusters, aggregate each host's
failure domain across every StorageNodeSet belonging to the cluster
(clusterFailureDomainHosts) and validate domain count/balance
(fdActivationDomainCountViolation, mirroring the Python-side check
exactly). On failure, just requeue -- ActionStatus is left completely
untouched, so there's nothing to get stuck once enough domains show up.
No-op for FD-disabled clusters, confirmed via the existing activate
tests passing unchanged.

Author: wmousa

operator/internal/controller/simplyblockstoragecluster_controller.go

@@ -521,13 +521,103 @@ func (r *StorageClusterReconciler) ensureFinalizer(
 	return true, r.Update(ctx, clusterCR)
 }
 
+// clusterFailureDomainHosts aggregates each host's (by management IP)
+// failure domain across every StorageNodeSet belonging to clusterName in
+// namespace. Hosts with no failure-domain assignment reported to status
+// yet are skipped, not treated as domain 0.
+func clusterFailureDomainHosts(
+	ctx context.Context, c client.Client, namespace, clusterName string,
+) (map[string]int32, error) {
+	var snList simplyblockv1alpha1.StorageNodeSetList
+	if err := c.List(ctx, &snList, client.InNamespace(namespace)); err != nil {
+		return nil, err
+	}
+	hostDomains := map[string]int32{}
+	for _, sn := range snList.Items {
+		if sn.Spec.ClusterName != clusterName {
+			continue
+		}
+		for _, ns := range sn.Status.Nodes {
+			if ns.FailureDomain == nil || ns.MgmtIp == "" {
+				continue
+			}
+			hostDomains[ns.MgmtIp] = *ns.FailureDomain
+		}
+	}
+	return hostDomains, nil
+}
+
+// fdActivationDomainCountViolation validates the number of distinct failure
+// domains and per-domain host balance for fresh activation, mirroring
+// simplyblock_core's fd_activation_domain_count_violation (Python side) so
+// the two stay in lockstep.
+//
+// A 2-FD layout can never absorb a second independent failure once one
+// domain is fully down, so fresh activation requires npcs+2 distinct
+// domains (3 for npcs=1, 4 for npcs=2) with an EQUAL host count in each --
+// below npcs+1 domains even the initial static role-placement rotation is
+// structurally wrong; at exactly npcs+1 it is correct but has zero spare
+// capacity for a later single add/remove. Returns a human-readable reason
+// when the cluster isn't ready yet, "" when it is.
+func fdActivationDomainCountViolation(npcs int, hostDomains map[string]int32) string {
+	if len(hostDomains) == 0 {
+		return "no storage nodes with a failure-domain assignment reported yet"
+	}
+	counts := map[int32]int{}
+	for _, fd := range hostDomains {
+		counts[fd]++
+	}
+	minDomains := npcs + 2
+	if len(counts) < minDomains {
+		return fmt.Sprintf(
+			"failure domains are enabled with npcs=%d, which requires at least "+
+				"%d distinct failure domains (2 domains is not supported at any "+
+				"npcs level); currently have %d",
+			npcs, minDomains, len(counts))
+	}
+	first := -1
+	for _, c := range counts {
+		if first == -1 {
+			first = c
+			continue
+		}
+		if c != first {
+			return fmt.Sprintf(
+				"failure domains must hold an equal number of hosts at "+
+					"activation; current split: %v", counts)
+		}
+	}
+	return ""
+}
+
 func (r *StorageClusterReconciler) reconcileActivate(
 	ctx context.Context,
 	clusterCR *simplyblockv1alpha1.StorageCluster,
 ) (ctrl.Result, error) {
 
 	log := logf.FromContext(ctx)
 
+	// Failure-domain readiness gate: checked BEFORE any ActionStatus
+	// transition below, and status is left untouched on failure. A failed
+	// activate call gets permanently stuck (nothing in this reconciler ever
+	// resets ActionStatus away from Failed back to Running), so a doomed
+	// activate attempt must never be allowed to fire in the first place.
+	// No-op for FD-disabled clusters -- falls straight through unchanged.
+	if ptr.BoolFromOrFalse(clusterCR.Spec.EnableFailureDomains) {
+		hostDomains, err := clusterFailureDomainHosts(ctx, r.Client, clusterCR.Namespace, clusterCR.Name)
+		if err != nil {
+			log.Error(err, "Failed to list StorageNodeSets for failure-domain readiness check",
+				"cluster", clusterCR.Name)
+			return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
+		}
+		npcs := stripeParityChunks(clusterCR.Spec.StripeSpec)
+		if reason := fdActivationDomainCountViolation(npcs, hostDomains); reason != "" {
+			log.Info("Cluster not ready for activation yet, waiting on failure-domain readiness",
+				"cluster", clusterCR.Name, "reason", reason)
+			return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
+		}
+	}
+
 	if clusterCR.Status.ActionStatus != nil &&
 		clusterCR.Status.ActionStatus.Action == utils.ClusterActionActivate &&
 		clusterCR.Status.ActionStatus.State == utils.ActionStateSuccess &&

operator/internal/controller/simplyblockstoragecluster_controller_unit_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/simplyblock/atlas/ptr"
 	simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1"
 	"github.com/simplyblock/simplyblock-operator/internal/utils"
 	"github.com/simplyblock/simplyblock-operator/internal/webapi"
@@ -146,6 +147,184 @@ func TestReconcileActivateInitializesObservedGeneration(t *testing.T) {
 	}
 }
 
+func TestFdActivationDomainCountViolation(t *testing.T) {
+	hosts := func(domainCounts ...int32) map[string]int32 {
+		m := map[string]int32{}
+		for i, fd := range domainCounts {
+			m[fmt.Sprintf("10.0.0.%d", i)] = fd
+		}
+		return m
+	}
+
+	cases := []struct {
+		name    string
+		npcs    int
+		hosts   map[string]int32
+		wantErr bool
+	}{
+		{"empty", 1, map[string]int32{}, true},
+		{"npcs1 two domains violates", 1, hosts(0, 0, 1, 1), true},
+		{"npcs1 three domains ok", 1, hosts(0, 1, 2), false},
+		{"npcs1 three domains unequal violates", 1, hosts(0, 1, 1, 2), true},
+		{"npcs2 two domains violates", 2, hosts(0, 0, 1, 1), true},
+		{"npcs2 three domains violates", 2, hosts(0, 1, 2), true},
+		{"npcs2 four domains ok", 2, hosts(0, 1, 2, 3), false},
+		{"npcs2 four domains unequal violates", 2, hosts(0, 0, 1, 2, 3), true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			reason := fdActivationDomainCountViolation(tc.npcs, tc.hosts)
+			if tc.wantErr && reason == "" {
+				t.Fatalf("expected a violation reason, got none")
+			}
+			if !tc.wantErr && reason != "" {
+				t.Fatalf("expected no violation, got: %s", reason)
+			}
+		})
+	}
+}
+
+func TestClusterFailureDomainHosts(t *testing.T) {
+	fd := func(v int32) *int32 { return &v }
+
+	nsA := &simplyblockv1alpha1.StorageNodeSet{
+		ObjectMeta: metav1.ObjectMeta{Name: "set-a", Namespace: "default"},
+		Spec:       simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: "cluster-x"},
+		Status: simplyblockv1alpha1.StorageNodeSetStatus{
+			Nodes: []simplyblockv1alpha1.NodeStatus{
+				{Hostname: "w0", MgmtIp: "10.0.0.1", FailureDomain: fd(0)},
+				{Hostname: "w1", MgmtIp: "10.0.0.2", FailureDomain: fd(1)},
+				{Hostname: "w2", MgmtIp: "10.0.0.3"}, // no domain yet, must be skipped
+			},
+		},
+	}
+	nsB := &simplyblockv1alpha1.StorageNodeSet{
+		ObjectMeta: metav1.ObjectMeta{Name: "set-b", Namespace: "default"},
+		Spec:       simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: "cluster-x"},
+		Status: simplyblockv1alpha1.StorageNodeSetStatus{
+			Nodes: []simplyblockv1alpha1.NodeStatus{
+				{Hostname: "w3", MgmtIp: "10.0.0.4", FailureDomain: fd(2)},
+			},
+		},
+	}
+	nsOther := &simplyblockv1alpha1.StorageNodeSet{
+		ObjectMeta: metav1.ObjectMeta{Name: "set-other", Namespace: "default"},
+		Spec:       simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: "cluster-y"},
+		Status: simplyblockv1alpha1.StorageNodeSetStatus{
+			Nodes: []simplyblockv1alpha1.NodeStatus{
+				{Hostname: "y0", MgmtIp: "10.0.0.9", FailureDomain: fd(0)},
+			},
+		},
+	}
+
+	r := newClusterStateTestReconciler(t, nsA, nsB, nsOther)
+
+	got, err := clusterFailureDomainHosts(context.Background(), r.Client, "default", "cluster-x")
+	if err != nil {
+		t.Fatalf("clusterFailureDomainHosts returned error: %v", err)
+	}
+	want := map[string]int32{"10.0.0.1": 0, "10.0.0.2": 1, "10.0.0.4": 2}
+	if len(got) != len(want) {
+		t.Fatalf("expected %v, got %v", want, got)
+	}
+	for ip, fdVal := range want {
+		if got[ip] != fdVal {
+			t.Fatalf("expected %s -> domain %d, got %v", ip, fdVal, got)
+		}
+	}
+}
+
+func TestReconcileActivateWaitsForFailureDomainReadiness(t *testing.T) {
+	fd := func(v int32) *int32 { return &v }
+
+	cluster := &simplyblockv1alpha1.StorageCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "cluster-fd-wait",
+			Namespace: "default",
+		},
+		Spec: simplyblockv1alpha1.StorageClusterSpec{
+			Action:               utils.ClusterActionActivate,
+			EnableFailureDomains: ptr.To(true),
+			StripeSpec: &simplyblockv1alpha1.StripeSpec{
+				ParityChunks: ptr.To(int32(2)),
+			},
+		},
+	}
+	// Only 2 distinct domains for npcs=2 -- must NOT be allowed through
+	// (requires npcs+2 = 4).
+	nodeSet := &simplyblockv1alpha1.StorageNodeSet{
+		ObjectMeta: metav1.ObjectMeta{Name: "set-fd-wait", Namespace: "default"},
+		Spec:       simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: "cluster-fd-wait"},
+		Status: simplyblockv1alpha1.StorageNodeSetStatus{
+			Nodes: []simplyblockv1alpha1.NodeStatus{
+				{Hostname: "w0", MgmtIp: "10.0.0.1", FailureDomain: fd(0)},
+				{Hostname: "w1", MgmtIp: "10.0.0.2", FailureDomain: fd(1)},
+			},
+		},
+	}
+
+	r := newClusterStateTestReconciler(t, cluster, nodeSet)
+
+	res, err := r.reconcileActivate(context.Background(), cluster)
+	if err != nil {
+		t.Fatalf("reconcileActivate returned error: %v", err)
+	}
+	if res.RequeueAfter == 0 {
+		t.Fatalf("expected a requeue while waiting on failure-domain readiness")
+	}
+	if cluster.Status.ActionStatus != nil {
+		t.Fatalf("expected ActionStatus to stay untouched while not ready, got %#v", cluster.Status.ActionStatus)
+	}
+}
+
+func TestReconcileActivateProceedsOnceFailureDomainsAreReady(t *testing.T) {
+	fd := func(v int32) *int32 { return &v }
+
+	cluster := &simplyblockv1alpha1.StorageCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "cluster-fd-ready",
+			Namespace: "default",
+		},
+		Spec: simplyblockv1alpha1.StorageClusterSpec{
+			Action:               utils.ClusterActionActivate,
+			EnableFailureDomains: ptr.To(true),
+			StripeSpec: &simplyblockv1alpha1.StripeSpec{
+				ParityChunks: ptr.To(int32(2)),
+			},
+		},
+	}
+	// 4 distinct, equally-sized domains for npcs=2 -- satisfies npcs+2 = 4,
+	// so the gate must let this through to the normal init-action path.
+	nodeSet := &simplyblockv1alpha1.StorageNodeSet{
+		ObjectMeta: metav1.ObjectMeta{Name: "set-fd-ready", Namespace: "default"},
+		Spec:       simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: "cluster-fd-ready"},
+		Status: simplyblockv1alpha1.StorageNodeSetStatus{
+			Nodes: []simplyblockv1alpha1.NodeStatus{
+				{Hostname: "w0", MgmtIp: "10.0.0.1", FailureDomain: fd(0)},
+				{Hostname: "w1", MgmtIp: "10.0.0.2", FailureDomain: fd(1)},
+				{Hostname: "w2", MgmtIp: "10.0.0.3", FailureDomain: fd(2)},
+				{Hostname: "w3", MgmtIp: "10.0.0.4", FailureDomain: fd(3)},
+			},
+		},
+	}
+
+	r := newClusterStateTestReconciler(t, cluster, nodeSet)
+
+	res, err := r.reconcileActivate(context.Background(), cluster)
+	if err != nil {
+		t.Fatalf("reconcileActivate returned error: %v", err)
+	}
+	if cluster.Status.ActionStatus == nil {
+		t.Fatalf("expected ActionStatus to be initialized once failure-domain readiness is satisfied")
+	}
+	if cluster.Status.ActionStatus.State != utils.ActionStateRunning {
+		t.Fatalf("expected Running state, got %#v", cluster.Status.ActionStatus)
+	}
+	if res.Requeue != true {
+		t.Fatalf("expected immediate requeue for the init-action step, got %+v", res)
+	}
+}
+
 func TestReconcileExpandTransitions(t *testing.T) {
 	t.Run("initializes running status for expand with observed generation", func(t *testing.T) {
 		cluster := &simplyblockv1alpha1.StorageCluster{