From ce1229c01edd8519a6af0d6abecf621d2328f15e Mon Sep 17 00:00:00 2001 From: Aritra Basu Date: Tue, 21 Oct 2025 13:46:43 -0700 Subject: [PATCH 1/2] added healthcheck support for agent Signed-off-by: Aritra Basu --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 75 ++++- calico-vpp-agent/health/health.go | 259 ++++++++++++++++++ calico-vpp-agent/watch_dog/watch_dog.go | 59 ---- config/config.go | 6 + test/healthcheck/test_healthcheck.sh | 101 +++++++ yaml/base/calico-vpp-daemonset.yaml | 27 ++ yaml/generated/calico-vpp-dpdk.yaml | 27 ++ .../calico-vpp-eks-dpdk-multinet.yaml | 27 ++ yaml/generated/calico-vpp-eks-dpdk.yaml | 27 ++ yaml/generated/calico-vpp-eks-multinet.yaml | 27 ++ yaml/generated/calico-vpp-eks.yaml | 27 ++ yaml/generated/calico-vpp-kind-multinet.yaml | 27 ++ yaml/generated/calico-vpp-kind.yaml | 27 ++ yaml/generated/calico-vpp-multinet.yaml | 27 ++ yaml/generated/calico-vpp-nohuge.yaml | 27 ++ yaml/generated/calico-vpp.yaml | 27 ++ 16 files changed, 728 insertions(+), 69 deletions(-) create mode 100644 calico-vpp-agent/health/health.go delete mode 100644 calico-vpp-agent/watch_dog/watch_dog.go create mode 100755 test/healthcheck/test_healthcheck.sh diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index dcc308a38..c17c23d17 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -38,13 +38,12 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/health" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/prometheus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/routing" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/services" - "github.com/projectcalico/vpp-dataplane/v3/config" - - watchdog "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watch_dog" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" + "github.com/projectcalico/vpp-dataplane/v3/config" ) /* @@ -83,6 +82,15 @@ func main() { log.Fatalf("Error writing pidfile: %v", err) } + /** + * Start health check server + */ + healthServer := health.NewHealthServer( + log.WithFields(logrus.Fields{"component": "health"}), + *config.GetCalicoVppInitialConfig().HealthCheckPort, + ) + Go(healthServer.ServeHealth) + /** * Connect to VPP & wait for it to be up */ @@ -90,12 +98,16 @@ func main() { if err != nil { log.Fatalf("Cannot create VPP client: %v", err) } + healthServer.SetComponentStatus(health.ComponentVPP, true, "VPP connection established") + // Once we have the api connection, we know vpp & vpp-manager are running and the // state is accurately reported. Wait for vpp-manager to finish the config. common.VppManagerInfo, err = common.WaitForVppManager() if err != nil { log.Fatalf("Vpp Manager not started: %v", err) } + healthServer.SetComponentStatus(health.ComponentVPPManager, true, "VPP Manager ready") + common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) /** @@ -168,15 +180,50 @@ func main() { routingServer.SetBGPConf(bgpConf) serviceServer.SetBGPConf(bgpConf) - watchDog := watchdog.NewWatchDog(log.WithFields(logrus.Fields{"component": "watchDog"}), &t) Go(felixServer.ServeFelix) - felixConfig := watchDog.Wait(felixServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") - ourBGPSpec := watchDog.Wait(felixServer.GotOurNodeBGPchan, "Waiting for bgp spec to be provided on node add") - // check if the watchDog timer has issued the t.Kill() which would mean we are dead - if !t.Alive() { - log.Fatal("WatchDog timed out waiting for config from felix. Exiting...") + + /* + * Mark as unhealthy while waiting for Felix config + * Kubernetes startup probe handles pod restart if needed + */ + healthServer.MarkAsUnhealthy("Waiting for Felix configuration") + log.Info("Waiting for Felix configuration...") + + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + var felixConfig interface{} + var ourBGPSpec interface{} + felixConfigReceived := false + bgpSpecReceived := false + + for !felixConfigReceived || !bgpSpecReceived { + select { + case value := <-felixServer.FelixConfigChan: + felixConfig = value + felixConfigReceived = true + log.Info("FelixConfig received from calico pod") + case value := <-felixServer.GotOurNodeBGPchan: + ourBGPSpec = value + bgpSpecReceived = true + log.Info("BGP spec received from node add") + case <-t.Dying(): + log.Error("Tomb dying while waiting for Felix config") + return + case <-ticker.C: + if !felixConfigReceived { + log.Info("Still waiting for FelixConfig from calico pod...") + } + if !bgpSpecReceived { + log.Info("Still waiting for BGP spec from node add...") + } + } } + healthServer.MarkAsHealthy("Felix configuration received") + healthServer.SetComponentStatus(health.ComponentFelix, true, "Felix config received") + log.Info("Felix configuration received") + if ourBGPSpec != nil { bgpSpec, ok := ourBGPSpec.(*common.LocalNodeSpec) if !ok { @@ -193,7 +240,14 @@ func main() { if *config.GetCalicoVppFeatureGates().MultinetEnabled { Go(netWatcher.WatchNetworks) - watchDog.Wait(netWatcher.InSync, "Waiting for networks to be listed and synced") + log.Info("Waiting for networks to be listed and synced...") + select { + case <-netWatcher.InSync: + log.Info("Networks synced") + case <-t.Dying(): + log.Error("Tomb dying while waiting for networks sync") + return + } } if felixConfig != nil { @@ -222,6 +276,7 @@ func main() { Go(localSIDWatcher.WatchLocalSID) } + healthServer.SetComponentStatus(health.ComponentAgent, true, "Agent ready") log.Infof("Agent started") interruptSignalChannel := make(chan os.Signal, 2) diff --git a/calico-vpp-agent/health/health.go b/calico-vpp-agent/health/health.go new file mode 100644 index 000000000..ed275ed64 --- /dev/null +++ b/calico-vpp-agent/health/health.go @@ -0,0 +1,259 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package health + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + "sync" + "time" + + "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" +) + +// HealthStatus represents the current health state +type HealthStatus struct { + Healthy bool `json:"healthy"` + Ready bool `json:"ready"` + Components map[string]ComponentStatus `json:"components"` + Message string `json:"message,omitempty"` + LastUpdate time.Time `json:"lastUpdate"` +} + +// ComponentStatus tracks the status of individual components +type ComponentStatus struct { + Initialized bool `json:"initialized"` + Message string `json:"message,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// HealthServer provides HTTP health check endpoints +type HealthServer struct { + log *logrus.Entry + port uint32 + status HealthStatus + statusMutex sync.RWMutex + server *http.Server +} + +const ( + ComponentVPP = "vpp" + ComponentVPPManager = "vpp-manager" + ComponentFelix = "felix" + ComponentAgent = "agent" +) + +// NewHealthServer creates a new health check server +func NewHealthServer(log *logrus.Entry, port uint32) *HealthServer { + return &HealthServer{ + log: log, + port: port, + status: HealthStatus{ + Healthy: true, + Ready: false, + Components: make(map[string]ComponentStatus), + LastUpdate: time.Now(), + }, + } +} + +// SetComponentStatus updates the status of a specific component +func (hs *HealthServer) SetComponentStatus(component string, initialized bool, message string) { + hs.statusMutex.Lock() + defer hs.statusMutex.Unlock() + + hs.status.Components[component] = ComponentStatus{ + Initialized: initialized, + Message: message, + UpdatedAt: time.Now(), + } + hs.status.LastUpdate = time.Now() + + // Update overall readiness + hs.updateReadiness() + + hs.log.WithFields(logrus.Fields{ + "component": component, + "initialized": initialized, + "message": message, + }).Debug("Component status updated") +} + +// updateReadiness determines overall readiness based on component status +func (hs *HealthServer) updateReadiness() { + // Required components for readiness + requiredComponents := []string{ + ComponentVPP, + ComponentVPPManager, + ComponentFelix, + ComponentAgent, + } + + allReady := true + for _, comp := range requiredComponents { + status, exists := hs.status.Components[comp] + if !exists || !status.Initialized { + allReady = false + break + } + } + + hs.status.Ready = allReady + + if allReady { + hs.status.Message = "All components initialized" + } else { + hs.status.Message = "Waiting for components to initialize" + } +} + +// MarkAsHealthy marks the agent as healthy (but not necessarily ready) +func (hs *HealthServer) MarkAsHealthy(message string) { + hs.statusMutex.Lock() + defer hs.statusMutex.Unlock() + + hs.status.Healthy = true + if message != "" { + hs.status.Message = message + } else { + hs.status.Message = "Agent is healthy" + } + hs.status.LastUpdate = time.Now() + + hs.log.WithField("message", message).Info("Agent marked as healthy") +} + +// MarkAsUnhealthy marks the agent as unhealthy +func (hs *HealthServer) MarkAsUnhealthy(reason string) { + hs.statusMutex.Lock() + defer hs.statusMutex.Unlock() + + hs.status.Healthy = false + hs.status.Ready = false + hs.status.Message = reason + hs.status.LastUpdate = time.Now() + + hs.log.WithField("reason", reason).Warn("Agent marked as unhealthy") +} + +// GetStatus returns the current health status (thread-safe) +func (hs *HealthServer) GetStatus() HealthStatus { + hs.statusMutex.RLock() + defer hs.statusMutex.RUnlock() + + // Create a copy to avoid race conditions + statusCopy := hs.status + statusCopy.Components = make(map[string]ComponentStatus) + for k, v := range hs.status.Components { + statusCopy.Components[k] = v + } + + return statusCopy +} + +// livenessHandler handles the /liveness endpoint +func (hs *HealthServer) livenessHandler(w http.ResponseWriter, r *http.Request) { + status := hs.GetStatus() + + if status.Healthy { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") + } else { + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprintf(w, "Unhealthy: %s", status.Message) + } +} + +// readinessHandler handles the /readiness endpoint +func (hs *HealthServer) readinessHandler(w http.ResponseWriter, r *http.Request) { + status := hs.GetStatus() + + if status.Ready { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "Ready") + } else { + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprintf(w, "Not ready: %s", status.Message) + } +} + +// statusHandler handles the /status endpoint (detailed JSON) +func (hs *HealthServer) statusHandler(w http.ResponseWriter, r *http.Request) { + status := hs.GetStatus() + + w.Header().Set("Content-Type", "application/json") + + httpStatus := http.StatusOK + if !status.Ready { + httpStatus = http.StatusServiceUnavailable + } + w.WriteHeader(httpStatus) + + if err := json.NewEncoder(w).Encode(status); err != nil { + hs.log.WithError(err).Error("Failed to encode status response") + } +} + +func (hs *HealthServer) ServeHealth(t *tomb.Tomb) error { + mux := http.NewServeMux() + mux.HandleFunc("/liveness", hs.livenessHandler) + mux.HandleFunc("/readiness", hs.readinessHandler) + mux.HandleFunc("/status", hs.statusHandler) + + // Create TCP listener for the health server + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", hs.port)) + if err != nil { + // Try with a retry mechanism + for i := 0; i < 3; i++ { + hs.log.Warnf("Failed to bind to port %d, retrying in 5 seconds...", hs.port) + time.Sleep(5 * time.Second) + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", hs.port)) + if err == nil { + break + } + } + if err != nil { + return fmt.Errorf("health server error: %w", err) + } + } + + hs.server = &http.Server{ + Addr: fmt.Sprintf(":%d", hs.port), + Handler: mux, + } + + hs.log.Infof("Starting health check server on port %d", hs.port) + + // Start server with our custom listener + errChan := make(chan error, 1) + go func() { + if err := hs.server.Serve(listener); err != nil && err != http.ErrServerClosed { + errChan <- err + } + }() + + // Wait for tomb to die or server error + select { + case <-t.Dying(): + hs.log.Info("Shutting down health check server") + return hs.server.Close() + case err := <-errChan: + return fmt.Errorf("health server error: %w", err) + } +} diff --git a/calico-vpp-agent/watch_dog/watch_dog.go b/calico-vpp-agent/watch_dog/watch_dog.go deleted file mode 100644 index 507d94f7b..000000000 --- a/calico-vpp-agent/watch_dog/watch_dog.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package watchdog - -import ( - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "gopkg.in/tomb.v2" -) - -type WatchDog struct { - log *logrus.Entry - t *tomb.Tomb -} - -func NewWatchDog(log *logrus.Entry, t *tomb.Tomb) *WatchDog { - return &WatchDog{ - log: log, - t: t, - } -} - -func (wd *WatchDog) Wait(myChan chan interface{}, msg string) interface{} { - ticker := time.NewTicker(time.Second * 5) - nbTicks := 0 - defer ticker.Stop() - for { - select { - case value := <-myChan: - return value - case <-wd.t.Dying(): - return nil - case <-ticker.C: - nbTicks++ - if nbTicks >= 30 { - wd.t.Kill(errors.Errorf("Timeout waiting for config from felix")) - } else if nbTicks >= 6 { // Start warning after 6 ticks, i.e. 30sec - wd.log.Warn(msg) - } else { - wd.log.Info(msg) - } - } - } -} diff --git a/config/config.go b/config/config.go index 9edfc8ee3..12f9004b9 100644 --- a/config/config.go +++ b/config/config.go @@ -469,6 +469,9 @@ type CalicoVppInitialConfigConfigType struct { //out of agent and vppmanager // PrometheusStatsPrefix is the prefix to use for Prometheus metrics // Defaults to "cni.projectcalico.vpp." PrometheusStatsPrefix string `json:"prometheusStatsPrefix"` + // HealthCheckPort is the port on which the health check HTTP server listens + // Defaults to 9090 + HealthCheckPort *uint32 `json:"healthCheckPort"` } func (cfg *CalicoVppInitialConfigConfigType) Validate() (err error) { @@ -494,6 +497,9 @@ func (cfg *CalicoVppInitialConfigConfigType) Validate() (err error) { if cfg.PrometheusStatsPrefix == "" { cfg.PrometheusStatsPrefix = "cni.projectcalico.vpp." } + cfg.HealthCheckPort = DefaultToPtr( + cfg.HealthCheckPort, 9090, + ) return nil } func (cfg *CalicoVppInitialConfigConfigType) GetDefaultGWs() (gws []net.IP, err error) { diff --git a/test/healthcheck/test_healthcheck.sh b/test/healthcheck/test_healthcheck.sh new file mode 100755 index 000000000..78c2ade7c --- /dev/null +++ b/test/healthcheck/test_healthcheck.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +# Test script for Calico VPP agent healthcheck endpoints +# This script can be used to verify the healthcheck implementation + +set -e + +HEALTHCHECK_PORT=${HEALTHCHECK_PORT:-9090} +POD_NAME=${1:-} +KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} +LOCAL_PORT=${LOCAL_PORT:-19090} + +# Function to check if kubectl is properly configured +check_kubectl_config() { + if ! kubectl --kubeconfig="$KUBECONFIG" get nodes &>/dev/null; then + echo "Error: Cannot connect to Kubernetes cluster. Please check your kubectl configuration." + echo "If running with sudo, try: sudo KUBECONFIG=$KUBECONFIG $0 $POD_NAME" + exit 1 + fi +} + +# Function to clean up port-forward process +cleanup() { + if [ -n "$PORT_FORWARD_PID" ]; then + echo "Cleaning up port-forward (PID: $PORT_FORWARD_PID)" + kill $PORT_FORWARD_PID 2>/dev/null || true + fi +} + +# Set up trap to clean up port-forward on exit +trap cleanup EXIT + +if [ -z "$POD_NAME" ]; then + echo "Usage: $0 " + echo "" + echo "Example:" + echo " $0 calico-vpp-node-xxxxx" + echo "" + echo "Options:" + echo " HEALTHCHECK_PORT= Set a different healthcheck port in the container (default: 9090)" + echo " LOCAL_PORT= Set a different local port for port-forwarding (default: 19090)" + echo " KUBECONFIG= Set a different kubeconfig path" + exit 1 +fi + +# Check kubectl configuration +check_kubectl_config + +# Verify pod exists +if ! kubectl --kubeconfig="$KUBECONFIG" get pod -n calico-vpp-dataplane "$POD_NAME" &>/dev/null; then + echo "Error: Pod $POD_NAME not found in namespace calico-vpp-dataplane" + echo "Available pods:" + kubectl --kubeconfig="$KUBECONFIG" get pods -n calico-vpp-dataplane + exit 1 +fi + +echo "Testing healthcheck endpoints for pod: $POD_NAME" +echo "Using healthcheck port: $HEALTHCHECK_PORT" +echo "Using kubeconfig: $KUBECONFIG" +echo "" + +# Port-forward approach (always use this since container has no curl/wget/nc) +echo "Setting up port-forward from localhost:$LOCAL_PORT to pod:$HEALTHCHECK_PORT" +kubectl --kubeconfig="$KUBECONFIG" port-forward -n calico-vpp-dataplane "$POD_NAME" $LOCAL_PORT:$HEALTHCHECK_PORT > /dev/null 2>&1 & +PORT_FORWARD_PID=$! + +# Wait for port-forward to establish +echo "Waiting for port-forward to establish..." +sleep 2 +echo "" + +# Test if port-forward is working +if ! curl -s "http://localhost:$LOCAL_PORT/liveness" &>/dev/null; then + echo "Error: Port-forward not working. Please check if port $LOCAL_PORT is available." + exit 1 +fi + +# Test liveness endpoint +echo "=== Testing /liveness endpoint ===" +curl -s -w "\nHTTP Status: %{http_code}\n" "http://localhost:$LOCAL_PORT/liveness" || true +echo "" + +# Test readiness endpoint +echo "=== Testing /readiness endpoint ===" +curl -s -w "\nHTTP Status: %{http_code}\n" "http://localhost:$LOCAL_PORT/readiness" || true +echo "" + +# Test status endpoint (detailed JSON) +echo "=== Testing /status endpoint (detailed) ===" +curl -s "http://localhost:$LOCAL_PORT/status" | python3 -m json.tool || \ + curl -s "http://localhost:$LOCAL_PORT/status" +echo "" + +# Check Kubernetes probe status +echo "=== Kubernetes Probe Status ===" +kubectl --kubeconfig="$KUBECONFIG" get pod -n calico-vpp-dataplane "$POD_NAME" -o jsonpath='{.status.conditions[?(@.type=="Ready")]}' | python3 -m json.tool || true +echo "" + +echo "=== Pod Status ===" +kubectl --kubeconfig="$KUBECONFIG" get pod -n calico-vpp-dataplane "$POD_NAME" -o wide +echo "" \ No newline at end of file diff --git a/yaml/base/calico-vpp-daemonset.yaml b/yaml/base/calico-vpp-daemonset.yaml index d8b497051..acccdae2d 100644 --- a/yaml/base/calico-vpp-daemonset.yaml +++ b/yaml/base/calico-vpp-daemonset.yaml @@ -288,6 +288,33 @@ spec: resources: requests: cpu: 250m + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico diff --git a/yaml/generated/calico-vpp-dpdk.yaml b/yaml/generated/calico-vpp-dpdk.yaml index 771107325..6bd36c02d 100644 --- a/yaml/generated/calico-vpp-dpdk.yaml +++ b/yaml/generated/calico-vpp-dpdk.yaml @@ -274,12 +274,39 @@ spec: name: calico-vpp-config image: docker.io/calicovpp/agent:v3.29.0 imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 name: agent + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 250m securityContext: privileged: true + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico diff --git a/yaml/generated/calico-vpp-eks-dpdk-multinet.yaml b/yaml/generated/calico-vpp-eks-dpdk-multinet.yaml index d7a52f5ab..b1983dd47 100644 --- a/yaml/generated/calico-vpp-eks-dpdk-multinet.yaml +++ b/yaml/generated/calico-vpp-eks-dpdk-multinet.yaml @@ -335,12 +335,39 @@ spec: name: calico-vpp-config image: docker.io/calicovpp/agent:v3.29.0 imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 name: agent + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 250m securityContext: privileged: true + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico diff --git a/yaml/generated/calico-vpp-eks-dpdk.yaml b/yaml/generated/calico-vpp-eks-dpdk.yaml index 9155e4e96..42c8545cf 100644 --- a/yaml/generated/calico-vpp-eks-dpdk.yaml +++ b/yaml/generated/calico-vpp-eks-dpdk.yaml @@ -285,12 +285,39 @@ spec: name: calico-vpp-config image: docker.io/calicovpp/agent:v3.29.0 imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 name: agent + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 250m securityContext: privileged: true + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico diff --git a/yaml/generated/calico-vpp-eks-multinet.yaml b/yaml/generated/calico-vpp-eks-multinet.yaml index 49fd7caee..a9acf357f 100644 --- a/yaml/generated/calico-vpp-eks-multinet.yaml +++ b/yaml/generated/calico-vpp-eks-multinet.yaml @@ -333,12 +333,39 @@ spec: name: calico-vpp-config image: docker.io/calicovpp/agent:v3.29.0 imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 name: agent + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 250m securityContext: privileged: true + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico diff --git a/yaml/generated/calico-vpp-eks.yaml b/yaml/generated/calico-vpp-eks.yaml index eb56b9f8b..4274b1ef5 100644 --- a/yaml/generated/calico-vpp-eks.yaml +++ b/yaml/generated/calico-vpp-eks.yaml @@ -283,12 +283,39 @@ spec: name: calico-vpp-config image: docker.io/calicovpp/agent:v3.29.0 imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 name: agent + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 250m securityContext: privileged: true + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico diff --git a/yaml/generated/calico-vpp-kind-multinet.yaml b/yaml/generated/calico-vpp-kind-multinet.yaml index bbbe59783..8512ca223 100644 --- a/yaml/generated/calico-vpp-kind-multinet.yaml +++ b/yaml/generated/calico-vpp-kind-multinet.yaml @@ -286,12 +286,39 @@ spec: name: calico-vpp-config image: docker.io/calicovpp/agent:v3.29.0 imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 name: agent + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 250m securityContext: privileged: true + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico diff --git a/yaml/generated/calico-vpp-kind.yaml b/yaml/generated/calico-vpp-kind.yaml index 3c95d5b27..3ed564513 100644 --- a/yaml/generated/calico-vpp-kind.yaml +++ b/yaml/generated/calico-vpp-kind.yaml @@ -236,12 +236,39 @@ spec: name: calico-vpp-config image: docker.io/calicovpp/agent:v3.29.0 imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 name: agent + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 250m securityContext: privileged: true + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico diff --git a/yaml/generated/calico-vpp-multinet.yaml b/yaml/generated/calico-vpp-multinet.yaml index c8d96b7f1..e4e65479d 100644 --- a/yaml/generated/calico-vpp-multinet.yaml +++ b/yaml/generated/calico-vpp-multinet.yaml @@ -283,12 +283,39 @@ spec: name: calico-vpp-config image: docker.io/calicovpp/agent:v3.29.0 imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 name: agent + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 250m securityContext: privileged: true + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico diff --git a/yaml/generated/calico-vpp-nohuge.yaml b/yaml/generated/calico-vpp-nohuge.yaml index e61b2f8ef..9e733a260 100644 --- a/yaml/generated/calico-vpp-nohuge.yaml +++ b/yaml/generated/calico-vpp-nohuge.yaml @@ -233,12 +233,39 @@ spec: name: calico-vpp-config image: docker.io/calicovpp/agent:v3.29.0 imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 name: agent + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 250m securityContext: privileged: true + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico diff --git a/yaml/generated/calico-vpp.yaml b/yaml/generated/calico-vpp.yaml index c70c349b6..500ee5087 100644 --- a/yaml/generated/calico-vpp.yaml +++ b/yaml/generated/calico-vpp.yaml @@ -274,12 +274,39 @@ spec: name: calico-vpp-config image: docker.io/calicovpp/agent:v3.29.0 imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 name: agent + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 9090 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 250m securityContext: privileged: true + startupProbe: + failureThreshold: 10 + httpGet: + path: /liveness + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/run/calico name: var-run-calico From af4e279b08df0ea5fc0112ec6420aee7b501d184 Mon Sep 17 00:00:00 2001 From: hedi bouattour Date: Fri, 27 Feb 2026 15:11:45 +0000 Subject: [PATCH 2/2] Mark agent unhealthy when uplink route watcher restarts If the interface is gone, the uplink route watcher may keep restarting. In this case, the agent should be marked as unhealthy, and healthagent should take care of restarting the pod. However, the watcher can occasionally restart for legitimate reasons, so we need to mark the agent healthy again once it starts properly. --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 16 +++++++-------- calico-vpp-agent/health/health.go | 2 ++ .../watchers/uplink_route_watcher.go | 20 +++++++++---------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index c17c23d17..2ed9db74b 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -85,11 +85,11 @@ func main() { /** * Start health check server */ - healthServer := health.NewHealthServer( + health.DefaultHealthServer = health.NewHealthServer( log.WithFields(logrus.Fields{"component": "health"}), *config.GetCalicoVppInitialConfig().HealthCheckPort, ) - Go(healthServer.ServeHealth) + Go(health.DefaultHealthServer.ServeHealth) /** * Connect to VPP & wait for it to be up @@ -98,7 +98,7 @@ func main() { if err != nil { log.Fatalf("Cannot create VPP client: %v", err) } - healthServer.SetComponentStatus(health.ComponentVPP, true, "VPP connection established") + health.DefaultHealthServer.SetComponentStatus(health.ComponentVPP, true, "VPP connection established") // Once we have the api connection, we know vpp & vpp-manager are running and the // state is accurately reported. Wait for vpp-manager to finish the config. @@ -106,7 +106,7 @@ func main() { if err != nil { log.Fatalf("Vpp Manager not started: %v", err) } - healthServer.SetComponentStatus(health.ComponentVPPManager, true, "VPP Manager ready") + health.DefaultHealthServer.SetComponentStatus(health.ComponentVPPManager, true, "VPP Manager ready") common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) @@ -186,7 +186,7 @@ func main() { * Mark as unhealthy while waiting for Felix config * Kubernetes startup probe handles pod restart if needed */ - healthServer.MarkAsUnhealthy("Waiting for Felix configuration") + health.DefaultHealthServer.MarkAsUnhealthy("Waiting for Felix configuration") log.Info("Waiting for Felix configuration...") ticker := time.NewTicker(10 * time.Second) @@ -220,8 +220,8 @@ func main() { } } - healthServer.MarkAsHealthy("Felix configuration received") - healthServer.SetComponentStatus(health.ComponentFelix, true, "Felix config received") + health.DefaultHealthServer.MarkAsHealthy("Felix configuration received") + health.DefaultHealthServer.SetComponentStatus(health.ComponentFelix, true, "Felix config received") log.Info("Felix configuration received") if ourBGPSpec != nil { @@ -276,7 +276,7 @@ func main() { Go(localSIDWatcher.WatchLocalSID) } - healthServer.SetComponentStatus(health.ComponentAgent, true, "Agent ready") + health.DefaultHealthServer.SetComponentStatus(health.ComponentAgent, true, "Agent ready") log.Infof("Agent started") interruptSignalChannel := make(chan os.Signal, 2) diff --git a/calico-vpp-agent/health/health.go b/calico-vpp-agent/health/health.go index ed275ed64..c33d374d9 100644 --- a/calico-vpp-agent/health/health.go +++ b/calico-vpp-agent/health/health.go @@ -59,6 +59,8 @@ const ( ComponentAgent = "agent" ) +var DefaultHealthServer *HealthServer + // NewHealthServer creates a new health check server func NewHealthServer(log *logrus.Entry, port uint32) *HealthServer { return &HealthServer{ diff --git a/calico-vpp-agent/watchers/uplink_route_watcher.go b/calico-vpp-agent/watchers/uplink_route_watcher.go index 8432cdacb..2b9ce3a6b 100644 --- a/calico-vpp-agent/watchers/uplink_route_watcher.go +++ b/calico-vpp-agent/watchers/uplink_route_watcher.go @@ -25,6 +25,7 @@ import ( "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/health" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -226,6 +227,7 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { r.log.Errorf("error adding routes %v", err) goto restart } + health.DefaultHealthServer.MarkAsHealthy("Uplink route watcher started") for { select { case <-t.Dying(): @@ -252,9 +254,8 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { goto restart } for _, route := range routes { - err = r.DelRoute(route) - if err != nil { - r.log.Errorf("Cannot add pool route %s through vpp tap: %v", key, err) + if err = r.DelRoute(route); err != nil { + r.log.Errorf("Cannot delete pool route %s through vpp tap: %v", key, err) goto restart } } @@ -271,8 +272,7 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { goto restart } for _, route := range routes { - err = r.AddRoute(route) - if err != nil { + if err = r.AddRoute(route); err != nil { r.log.Errorf("Cannot add pool route %s through vpp tap: %v", key, err) goto restart } @@ -291,8 +291,7 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { goto restart } for _, route := range routes { - err = r.DelRoute(route) - if err != nil { + if err = r.DelRoute(route); err != nil { r.log.Errorf("Cannot delete pool route %s through vpp tap: %v", old.Cidr, err) goto restart } @@ -309,8 +308,7 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { goto restart } for _, route := range routes { - err = r.AddRoute(route) - if err != nil { + if err = r.AddRoute(route); err != nil { r.log.Errorf("Cannot add pool route %s through vpp tap: %v", new.Cidr, err) goto restart } @@ -328,8 +326,7 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { // See if it is one of our routes if update.Dst != nil && update.Dst.String() == route.Dst.String() { r.log.Infof("Re-adding route %+v", route) - err = netlink.RouteReplace(&route) - if err != nil { + if err = netlink.RouteReplace(&route); err != nil { r.log.Errorf("error adding route %+v: %v", route, err) goto restart } @@ -346,6 +343,7 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { } } restart: + health.DefaultHealthServer.MarkAsUnhealthy("Uplink route watcher restarted") r.safeClose() time.Sleep(2 * time.Second) r.log.Info("Restarting route watcher")