Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions internal/controller/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,49 @@ limitations under the License.
package controller

import (
"encoding/json"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
)

const (
// bootstrapAnnotationPrefix is the common prefix for all bootstrap completion
// annotations on a Node. The suffix is the rule's metadata.uid (RFC 4122 UUID,
// ~36 chars), which is immutable for the object's lifetime and globally unique.
//
// Full key format: readiness.k8s.io/bootstrap-completed-<ruleUID>
// Value format: {"rule-name":"<ruleName>"} (for human readability)
bootstrapAnnotationPrefix = "readiness.k8s.io/bootstrap-completed-"

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not a blocker: Should we make this a generic rule annotation instead of being bootstrap specific? This would let us reuse it for other properties later, like showing a rule's dryRun status.

cc @ajaysundark

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point! Not opposed to making it generic. Wanted to flag one thing, if reconciles ever run concurrently, a single annotation could cause read-modify-write races. If we go generic, we'd need to be careful there. What do you think?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agree to not increase the scope for this change. We can handle this as a bug fix for now.

I think scoped annotation with readiness.k8s.io prefix will scale better than global/single metadata.. let us discuss the other use cases individually.

)

// bootstrapAnnotationKey returns the annotation key for a rule's bootstrap
// completion state, using the rule's UID as the suffix.
func bootstrapAnnotationKey(uid types.UID) string {
return bootstrapAnnotationPrefix + string(uid)
}

// bootstrapAnnotationValue returns the JSON-encoded value to store in the
// bootstrap annotation. It includes the rule name for human readability.
func bootstrapAnnotationValue(ruleName string) string {
v := struct {
RuleName string `json:"rule-name"`
}{RuleName: ruleName}
b, err := json.Marshal(v)
if err != nil {
return `{"rule-name":""}` // should never happen
}
return string(b)
}



// legacyBootstrapAnnotationKey returns the old-format annotation key used
// before the UID migration: readiness.k8s.io/bootstrap-completed-<ruleName>.
func legacyBootstrapAnnotationKey(ruleName string) string {
return bootstrapAnnotationPrefix + ruleName
}

// conditionsEqual checks if two condition slices are equal.
func conditionsEqual(a, b []corev1.NodeCondition) bool {
if len(a) != len(b) {
Expand Down
56 changes: 56 additions & 0 deletions internal/controller/helper_unit_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
"testing"

. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/types"
)

func TestBootstrapAnnotationKey(t *testing.T) {
g := NewWithT(t)

uid := types.UID("550e8400-e29b-41d4-a716-446655440000")
key := bootstrapAnnotationKey(uid)
g.Expect(key).To(Equal("readiness.k8s.io/bootstrap-completed-550e8400-e29b-41d4-a716-446655440000"))
}

func TestBootstrapAnnotationValue(t *testing.T) {
g := NewWithT(t)

t.Run("encodes rule name as JSON", func(t *testing.T) {
val := bootstrapAnnotationValue("my-rule")
g.Expect(val).To(Equal(`{"rule-name":"my-rule"}`))
})

t.Run("handles long rule names", func(t *testing.T) {
longName := "my-very-long-rule-name-that-exceeds-the-63-character-annotation-key-limit-strictly"
val := bootstrapAnnotationValue(longName)
g.Expect(val).To(ContainSubstring(longName))
})
}

func TestLegacyBootstrapAnnotationKey(t *testing.T) {
g := NewWithT(t)

key := legacyBootstrapAnnotationKey("my-rule")
g.Expect(key).To(Equal("readiness.k8s.io/bootstrap-completed-my-rule"))
}


37 changes: 16 additions & 21 deletions internal/controller/node_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
Expand Down Expand Up @@ -129,7 +130,7 @@ func (r *RuleReadinessController) processNodeAgainstAllRules(ctx context.Context
}

// Skip if bootstrap-only and already completed
if r.isBootstrapCompleted(ctx, node.Name, rule.Name) && rule.Spec.EnforcementMode == readinessv1alpha1.EnforcementModeBootstrapOnly {
if r.isBootstrapCompleted(ctx, node.Name, rule.Name, rule.GetUID()) && rule.Spec.EnforcementMode == readinessv1alpha1.EnforcementModeBootstrapOnly {
log.Info("Skipping bootstrap-only rule - already completed",
"node", node.Name, "rule", rule.Name)
continue
Expand Down Expand Up @@ -341,24 +342,20 @@ func (r *RuleReadinessController) removeTaintBySpec(ctx context.Context, node *c
})
}

// Bootstrap completion tracking.
func (r *RuleReadinessController) isBootstrapCompleted(ctx context.Context, nodeName, ruleName string) bool {
// Check node annotation
func (r *RuleReadinessController) isBootstrapCompleted(ctx context.Context, nodeName string, ruleName string, ruleUID types.UID) bool {
node := &corev1.Node{}
if err := r.Get(ctx, client.ObjectKey{Name: nodeName}, node); err != nil {
return false
}

annotationKey := fmt.Sprintf("readiness.k8s.io/bootstrap-completed-%s", ruleName)
_, exists := node.Annotations[annotationKey]
return exists
_, existsNew := node.Annotations[bootstrapAnnotationKey(ruleUID)]
_, existsLegacy := node.Annotations[legacyBootstrapAnnotationKey(ruleName)]
return existsNew || existsLegacy
}

func (r *RuleReadinessController) markBootstrapCompleted(ctx context.Context, nodeName, ruleName string) {
func (r *RuleReadinessController) markBootstrapCompleted(ctx context.Context, nodeName, ruleName string, ruleUID types.UID) {
log := ctrl.LoggerFrom(ctx)

annotationKey := fmt.Sprintf("readiness.k8s.io/bootstrap-completed-%s", ruleName)
marked := false
annotationKey := bootstrapAnnotationKey(ruleUID)

// retry to handle conflict with concurrent node updates
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
Expand All @@ -367,21 +364,19 @@ func (r *RuleReadinessController) markBootstrapCompleted(ctx context.Context, no
return err
}

// Check if already marked to avoid unnecessary updates
if node.Annotations != nil {
if _, exists := node.Annotations[annotationKey]; exists {
return nil
}
// Check if already marked to avoid unnecessary updates.
if _, exists := node.Annotations[annotationKey]; exists {
return nil
}

patch := client.MergeFrom(node.DeepCopy())

// Initialize annotations if nil
// Initialize annotations map if nil.
if node.Annotations == nil {
node.Annotations = make(map[string]string)
}

node.Annotations[annotationKey] = "true"
node.Annotations[annotationKey] = bootstrapAnnotationValue(ruleName)
if err := r.Patch(ctx, node, patch); err != nil {
return err
}
Expand All @@ -392,12 +387,12 @@ func (r *RuleReadinessController) markBootstrapCompleted(ctx context.Context, no

switch {
case err != nil:
log.Error(err, "Failed to mark bootstrap completed", "node", nodeName, "rule", ruleName)
log.Error(err, "Failed to mark bootstrap completed", "node", nodeName, "rule", ruleName, "uid", ruleUID)
case marked:
log.Info("Marked bootstrap completed", "node", nodeName, "rule", ruleName)
log.Info("Marked bootstrap completed", "node", nodeName, "rule", ruleName, "uid", ruleUID)
metrics.BootstrapCompleted.WithLabelValues(ruleName).Inc()
default:
log.V(4).Info("Bootstrap already completed", "node", nodeName, "rule", ruleName)
log.V(4).Info("Bootstrap already completed", "node", nodeName, "rule", ruleName, "uid", ruleUID)
}
}

Expand Down
134 changes: 134 additions & 0 deletions internal/controller/node_controller_reproduction_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
"context"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/reconcile"

nodereadinessiov1alpha1 "sigs.k8s.io/node-readiness-controller/api/v1alpha1"
)

var _ = Describe("Node Controller Reproduction", func() {
Context("when reconciling a node with a very long rule name", func() {
var (
ctx context.Context
readinessController *RuleReadinessController
nodeReconciler *NodeReconciler
fakeClientset *fake.Clientset
node *corev1.Node
rule *nodereadinessiov1alpha1.NodeReadinessRule
longRuleName = "my-very-long-rule-name-that-exceeds-the-annotation-key-limit-strictly"
)

BeforeEach(func() {
ctx = context.Background()

fakeClientset = fake.NewSimpleClientset()
readinessController = &RuleReadinessController{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
clientset: fakeClientset,
ruleCache: make(map[string]*nodereadinessiov1alpha1.NodeReadinessRule),
EventRecorder: record.NewFakeRecorder(10),
}

nodeReconciler = &NodeReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
Controller: readinessController,
}

node = &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "repro-node",
Labels: map[string]string{"env": "repro"},
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{Type: "ReproCondition", Status: corev1.ConditionTrue},
},
},
}

rule = &nodereadinessiov1alpha1.NodeReadinessRule{
ObjectMeta: metav1.ObjectMeta{
Name: longRuleName,
},
Spec: nodereadinessiov1alpha1.NodeReadinessRuleSpec{
Conditions: []nodereadinessiov1alpha1.ConditionRequirement{
{Type: "ReproCondition", RequiredStatus: corev1.ConditionTrue},
},
Taint: corev1.Taint{
Key: "readiness.k8s.io/repro-taint",
Effect: corev1.TaintEffectNoSchedule,
},
NodeSelector: metav1.LabelSelector{
MatchLabels: map[string]string{"env": "repro"},
},
EnforcementMode: nodereadinessiov1alpha1.EnforcementModeBootstrapOnly,
},
}
})

JustBeforeEach(func() {
Expect(k8sClient.Create(ctx, node)).To(Succeed())
Expect(k8sClient.Create(ctx, rule)).To(Succeed())
readinessController.updateRuleCache(ctx, rule)
})

AfterEach(func() {
_ = k8sClient.Delete(ctx, node)
updatedRule := &nodereadinessiov1alpha1.NodeReadinessRule{}
if err := k8sClient.Get(ctx, types.NamespacedName{Name: longRuleName}, updatedRule); err == nil {
updatedRule.Finalizers = nil
_ = k8sClient.Update(ctx, updatedRule)
_ = k8sClient.Delete(ctx, updatedRule)
}
readinessController.removeRuleFromCache(ctx, longRuleName)
})

It("should successfully mark bootstrap completed using the UID-based annotation key for long rule names", func() {
// Trigger reconciliation
_, err := nodeReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: "repro-node"}})
Expect(err).NotTo(HaveOccurred())

recheckedNode := &corev1.Node{}
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "repro-node"}, recheckedNode)).To(Succeed())

// The bootstrap annotation key uses the rule's UID as the suffix,
// so even very long rule names never violate the 63-char key limit.
uidKey := bootstrapAnnotationKey(rule.GetUID())
Expect(recheckedNode.Annotations).To(HaveKey(uidKey),
"UID-based bootstrap annotation should be present on the node")

// The annotation value should contain the full rule name for readability.
Expect(recheckedNode.Annotations[uidKey]).To(ContainSubstring(longRuleName),
"annotation value should contain the full rule name for human readability")
})
})


})
Loading