feat: Add selector and bootstrap observability metrics

docs/book/src/operations/monitoring.md

@@ -106,6 +106,24 @@ Number of nodes currently held or released by each `NodeReadinessRule`, collecte
 | `rule` | `NodeReadinessRule` name | Any non-dry-run rule name with a valid selector |
 | `state` | Whether matching nodes are still tainted by the rule or have had the taint removed | `held`, `released` |
 
+### `node_readiness_rule_matched_nodes`
+
+*Available starting from the v0.6.0 release.*
+
+Number of nodes matching each `NodeReadinessRule`'s `nodeSelector`, collected at scrape time.
+
+| Property | Value |
+| --- | --- |
+| Type | `gauge` |
+| Labels | `rule` |
+| Recorded when | Computed on each Prometheus scrape from the cached node list |
+
+#### Labels
+
+| Label | Description | Values |
+| --- | --- | --- |
+| `rule` | `NodeReadinessRule` name | Any rule name with a valid selector |
+
 ### `node_readiness_bootstrap_completed_total`
 
 Total number of nodes that have completed bootstrap.

internal/controller/collector_bench_test.go

@@ -99,3 +99,25 @@ func BenchmarkListRuleNodeStates(b *testing.B) {
 		}
 	}
 }
+
+func BenchmarkListRuleMatchedNodes(b *testing.B) {
+	nodeCounts := []int{100, 1000, 5000, 15000}
+	ruleCounts := []int{5, 20, 50}
+
+	for _, nodeCount := range nodeCounts {
+		for _, ruleCount := range ruleCounts {
+			b.Run(fmt.Sprintf("nodes=%d/rules=%d", nodeCount, ruleCount), func(b *testing.B) {
+				c := buildBenchController(b, nodeCount, ruleCount)
+				ctx := b.Context()
+
+				b.ResetTimer()
+				b.ReportAllocs()
+				for range b.N {
+					if _, err := c.ListRuleMatchedNodes(ctx); err != nil {
+						b.Fatalf("ListRuleMatchedNodes failed: %v", err)
+					}
+				}
+			})
+		}
+	}
+}

internal/controller/node_controller.go

@@ -156,7 +156,7 @@ func (r *RuleReadinessController) processNodeAgainstAllRules(ctx context.Context
 			log.Error(err, "Failed to evaluate rule for node",
 				"node", node.Name, "rule", rule.Name)
 			// Continue with other rules even if one fails
-			r.recordNodeFailure(rule, node.Name, "EvaluationError", err.Error())
+			r.recordNodeFailure(rule, node.Name, string(metrics.FailureReasonEvaluationError), err.Error())
 			errs = append(errs, err)
 			metrics.Failures.WithLabelValues(rule.Name, string(metrics.FailureReasonEvaluationError)).Inc()
 		}
@@ -483,6 +483,7 @@ func (r *RuleReadinessController) markBootstrapCompleted(ctx context.Context, no
 	switch {
 	case err != nil:
 		log.Error(err, "Failed to mark bootstrap completed", "node", nodeName, "rule", rule.Name, "uid", rule.GetUID())
+		metrics.Failures.WithLabelValues(rule.Name, string(metrics.FailureReasonAnnotationPatchFailed)).Inc()
 	case deferred:
 		log.Info("Deferring bootstrap completion - rule taint still present on node",
 			"node", nodeName, "rule", rule.Name, "taint", rule.Spec.Taint.Key)

internal/controller/nodereadinessrule_controller.go

@@ -290,16 +290,18 @@ func (r *RuleReadinessController) processAllNodesForRule(ctx context.Context, ru
 			log.Info("Processing node for rule", "rule", rule.Name, "node", node.Name)
 			if err := r.evaluateRuleForNode(ctx, rule, &node); err != nil {
 				log.Error(err, "Failed to evaluate node for rule", "rule", rule.Name, "node", node.Name)
-				r.recordNodeFailure(rule, node.Name, "EvaluationError", err.Error())
+				r.recordNodeFailure(rule, node.Name, string(metrics.FailureReasonEvaluationError), err.Error())
 				metrics.Failures.WithLabelValues(rule.Name, string(metrics.FailureReasonEvaluationError)).Inc()
 			} else {
 				appliedNodes = append(appliedNodes, node.Name)
+
 				var updatedFailedNodes []readinessv1alpha1.NodeFailure
 				for _, f := range rule.Status.FailedNodes {
 					if f.NodeName != node.Name {
 						updatedFailedNodes = append(updatedFailedNodes, f)
 					}
 				}
+
 				rule.Status.FailedNodes = updatedFailedNodes
 			}
 		}
@@ -577,6 +579,43 @@ func (r *RuleReadinessController) ListRuleNodeStates(ctx context.Context) (map[s
 	return counts, nil
 }
 
+// ListRuleMatchedNodes returns the number of nodes matching each rule's NodeSelector.
+func (r *RuleReadinessController) ListRuleMatchedNodes(ctx context.Context) (map[string]float64, error) {
+	ruleList := &readinessv1alpha1.NodeReadinessRuleList{}
+	if err := r.List(ctx, ruleList); err != nil {
+		return nil, err
+	}
+
+	nodeList := &corev1.NodeList{}
+	if err := r.List(ctx, nodeList); err != nil {
+		return nil, err
+	}
+
+	log := ctrl.LoggerFrom(ctx)
+
+	counts := make(map[string]float64, len(ruleList.Items))
+	for i := range ruleList.Items {
+		rule := &ruleList.Items[i]
+
+		// Parse the selector once per rule.
+		selector, err := metav1.LabelSelectorAsSelector(&rule.Spec.NodeSelector)
+		if err != nil {
+			log.V(2).Info("Invalid node selector for rule", "rule", rule.Name, "error", err)
+			continue
+		}
+
+		var matched float64
+		for i := range nodeList.Items {
+			if selector.Matches(labels.Set(nodeList.Items[i].Labels)) {
+				matched++
+			}
+		}
+		counts[rule.Name] = matched
+	}
+
+	return counts, nil
+}
+
 // ruleAppliesTo checks if a rule applies to a node.
 func (r *RuleReadinessController) ruleAppliesTo(ctx context.Context, rule *readinessv1alpha1.NodeReadinessRule, node *corev1.Node) bool {
 	log := ctrl.LoggerFrom(ctx)

internal/controller/nodereadinessrule_controller_test.go

@@ -2535,4 +2535,37 @@ var _ = Describe("NodeReadinessRule Controller", func() {
 			Expect(readinessController.hasTaintBySpec(anyOfNode, rule.Spec.Taint)).To(BeFalse())
 		})
 	})
+
+	Context("Metric: failures_total (reason=AnnotationPatchFailed)", func() {
+		It("should increment counter when annotation write fails", func() {
+			ruleName := "bce-error-rule"
+			rule := &nodereadinessiov1alpha1.NodeReadinessRule{
+				ObjectMeta: metav1.ObjectMeta{Name: ruleName, UID: types.UID("33333333-3333-3333-3333-333333333333")},
+			}
+
+			before := counterValue(metrics.Failures.WithLabelValues(ruleName, string(metrics.FailureReasonAnnotationPatchFailed)))
+
+			readinessController.markBootstrapCompleted(ctx, "nonexistent-node-for-bce-test", rule)
+
+			Expect(counterValue(metrics.Failures.WithLabelValues(ruleName, string(metrics.FailureReasonAnnotationPatchFailed)))).To(Equal(before + 1))
+		})
+
+		It("should not increment counter on successful annotation write", func() {
+			nodeName := "bce-success-node"
+			ruleName := "bce-success-rule"
+			rule := &nodereadinessiov1alpha1.NodeReadinessRule{
+				ObjectMeta: metav1.ObjectMeta{Name: ruleName, UID: types.UID("44444444-4444-4444-4444-444444444444")},
+			}
+
+			node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName}}
+			Expect(k8sClient.Create(ctx, node)).To(Succeed())
+			defer func() { _ = k8sClient.Delete(ctx, node) }()
+
+			before := counterValue(metrics.Failures.WithLabelValues(ruleName, string(metrics.FailureReasonAnnotationPatchFailed)))
+
+			readinessController.markBootstrapCompleted(ctx, nodeName, rule)
+
+			Expect(counterValue(metrics.Failures.WithLabelValues(ruleName, string(metrics.FailureReasonAnnotationPatchFailed)))).To(Equal(before))
+		})
+	})
 })

internal/controller/rule_matched_nodes_test.go

@@ -0,0 +1,143 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	readinessv1alpha1 "sigs.k8s.io/node-readiness-controller/api/v1alpha1"
+)
+
+func TestListRuleMatchedNodes_NoRules(t *testing.T) {
+	g := NewWithT(t)
+	fc := fakeclient.NewClientBuilder().WithScheme(newTestScheme(t)).Build()
+	c := &RuleReadinessController{
+		Client: fc,
+	}
+
+	counts, err := c.ListRuleMatchedNodes(t.Context())
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(counts).To(BeEmpty())
+}
+
+func TestListRuleMatchedNodes_ZeroMatches(t *testing.T) {
+	g := NewWithT(t)
+	fc := fakeclient.NewClientBuilder().WithScheme(newTestScheme(t)).WithObjects(
+		gpuRule(),
+		&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "cpu-node"}},
+	).Build()
+	c := &RuleReadinessController{
+		Client: fc,
+	}
+
+	counts, err := c.ListRuleMatchedNodes(t.Context())
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(counts).To(Equal(map[string]float64{"gpu-ready": 0}))
+}
+
+func TestListRuleMatchedNodes_MixedMatches(t *testing.T) {
+	g := NewWithT(t)
+	fc := fakeclient.NewClientBuilder().WithScheme(newTestScheme(t)).WithObjects(
+		gpuRule(),
+		gpuNode("held-1", true),
+		gpuNode("held-2", true),
+		gpuNode("released-1", false),
+		&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "non-matching"}},
+	).Build()
+	c := &RuleReadinessController{
+		Client: fc,
+	}
+
+	counts, err := c.ListRuleMatchedNodes(t.Context())
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(counts).To(Equal(map[string]float64{"gpu-ready": 3}))
+}
+
+func TestListRuleMatchedNodes_DryRunRuleIncluded(t *testing.T) {
+	g := NewWithT(t)
+	rule := gpuRule()
+	rule.Spec.DryRun = true
+	fc := fakeclient.NewClientBuilder().WithScheme(newTestScheme(t)).WithObjects(
+		rule,
+		gpuNode("held-1", true),
+	).Build()
+	c := &RuleReadinessController{
+		Client: fc,
+	}
+
+	counts, err := c.ListRuleMatchedNodes(t.Context())
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(counts).To(Equal(map[string]float64{"gpu-ready": 1}))
+}
+
+func TestListRuleMatchedNodes_DeletingRuleIncluded(t *testing.T) {
+	g := NewWithT(t)
+	rule := gpuRule()
+	now := metav1.Now()
+	rule.DeletionTimestamp = &now
+	rule.Finalizers = []string{"readiness.node.x-k8s.io/cleanup-taints"}
+	fc := fakeclient.NewClientBuilder().WithScheme(newTestScheme(t)).WithObjects(
+		rule,
+		gpuNode("held-1", true),
+		gpuNode("held-2", true),
+	).Build()
+	c := &RuleReadinessController{
+		Client: fc,
+	}
+
+	counts, err := c.ListRuleMatchedNodes(t.Context())
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(counts).To(Equal(map[string]float64{"gpu-ready": 2}))
+}
+
+func TestListRuleMatchedNodes_InvalidSelectorSkipped(t *testing.T) {
+	g := NewWithT(t)
+	validRule := gpuRule()
+	invalidRule := &readinessv1alpha1.NodeReadinessRule{
+		ObjectMeta: metav1.ObjectMeta{Name: "invalid-selector-rule"},
+		Spec: readinessv1alpha1.NodeReadinessRuleSpec{
+			NodeSelector: metav1.LabelSelector{
+				MatchExpressions: []metav1.LabelSelectorRequirement{
+					{Key: "gpu", Operator: "BogusOperator", Values: []string{"true"}},
+				},
+			},
+			Taint: corev1.Taint{
+				Key:    "readiness.k8s.io/invalid-selector",
+				Effect: corev1.TaintEffectNoSchedule,
+			},
+		},
+	}
+	fc := fakeclient.NewClientBuilder().WithScheme(newTestScheme(t)).WithObjects(
+		validRule,
+		invalidRule,
+		gpuNode("held-1", true),
+		gpuNode("released-1", false),
+	).Build()
+	c := &RuleReadinessController{
+		Client: fc,
+	}
+
+	counts, err := c.ListRuleMatchedNodes(t.Context())
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(counts).NotTo(HaveKey(invalidRule.Name))
+	g.Expect(counts).To(Equal(map[string]float64{"gpu-ready": 2}))
+}

internal/metrics/collector.go

@@ -38,25 +38,44 @@ type RuleNodeStateLister interface {
 	ListRuleNodeStates(ctx context.Context) (map[string]RuleNodeCounts, error)
 }
 
+// RuleMatchedNodesLister lists the number of nodes matching each rule's NodeSelector.
+type RuleMatchedNodesLister interface {
+	ListRuleMatchedNodes(ctx context.Context) (map[string]float64, error)
+}
+
+// ReadinessLister aggregates the scrape-time lookups the collector needs.
+type ReadinessLister interface {
+	RuleNodeStateLister
+	RuleMatchedNodesLister
+}
+
 var ruleNodesDesc = prometheus.NewDesc(
 	"node_readiness_rule_nodes",
 	"Number of nodes currently gated or released by the rule.",
 	[]string{"rule", "state"},
 	nil,
 )
 
+var ruleMatchedNodesDesc = prometheus.NewDesc(
+	"node_readiness_rule_matched_nodes",
+	"Number of nodes matched by a rule's NodeSelector.",
+	[]string{"rule"},
+	nil,
+)
+
 // ReadinessCollector is a prometheus.Collector that reads at scrape time.
 type ReadinessCollector struct {
-	lister RuleNodeStateLister
+	lister ReadinessLister
 }
 
-func NewReadinessCollector(lister RuleNodeStateLister) *ReadinessCollector {
+func NewReadinessCollector(lister ReadinessLister) *ReadinessCollector {
 	return &ReadinessCollector{lister: lister}
 }
 
 // Describe implements prometheus.Collector.
 func (c *ReadinessCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- ruleNodesDesc
+	ch <- ruleMatchedNodesDesc
 }
 
 // Collect implements prometheus.Collector.
@@ -67,11 +86,19 @@ func (c *ReadinessCollector) Collect(ch chan<- prometheus.Metric) {
 	counts, err := c.lister.ListRuleNodeStates(ctx)
 	if err != nil {
 		ctrl.Log.V(2).Info("Failed to list rule node states", "error", err)
-		return
+	} else {
+		for rule, rc := range counts {
+			ch <- prometheus.MustNewConstMetric(ruleNodesDesc, prometheus.GaugeValue, rc.Held, rule, string(RuleNodeStateHeld))
+			ch <- prometheus.MustNewConstMetric(ruleNodesDesc, prometheus.GaugeValue, rc.Released, rule, string(RuleNodeStateReleased))
+		}
 	}
 
-	for rule, rc := range counts {
-		ch <- prometheus.MustNewConstMetric(ruleNodesDesc, prometheus.GaugeValue, rc.Held, rule, string(RuleNodeStateHeld))
-		ch <- prometheus.MustNewConstMetric(ruleNodesDesc, prometheus.GaugeValue, rc.Released, rule, string(RuleNodeStateReleased))
+	matched, err := c.lister.ListRuleMatchedNodes(ctx)
+	if err != nil {
+		ctrl.Log.V(2).Info("Failed to list rule matched nodes", "error", err)
+		return
+	}
+	for rule, count := range matched {
+		ch <- prometheus.MustNewConstMetric(ruleMatchedNodesDesc, prometheus.GaugeValue, count, rule)
 	}
 }