Skip to content

Commit 79c2658

Browse files
Merge remote-tracking branch 'upstream/main' into 1488-nic-monitor-bugs
2 parents 72f6643 + 1fb0006 commit 79c2658

14 files changed

Lines changed: 374 additions & 21 deletions

File tree

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package server
16+
17+
import (
18+
"context"
19+
"fmt"
20+
"sync/atomic"
21+
"time"
22+
)
23+
24+
// epoch is the process-start reference point for monotonic elapsed time.
25+
// Using time.Since(epoch) preserves Go's monotonic clock reading and is
26+
// immune to wall-clock adjustments (NTP, etc.).
27+
var epoch = time.Now()
28+
29+
// PollingHealthChecker implements HealthChecker for monitors that run a
30+
// periodic polling loop. It reports unhealthy when the loop has not
31+
// completed an iteration within the configured staleness threshold.
32+
//
33+
// All timestamps use the monotonic clock to avoid false positives from
34+
// NTP adjustments.
35+
//
36+
// Usage:
37+
//
38+
// hc := server.NewPollingHealthChecker(5 * time.Minute)
39+
// srv := server.NewServer(
40+
// server.WithHealthCheck(hc),
41+
// server.WithPrometheusMetrics(),
42+
// )
43+
//
44+
// // In the polling loop:
45+
// for {
46+
// doWork()
47+
// hc.MarkAlive()
48+
// }
49+
type PollingHealthChecker struct {
50+
stalenessThreshold time.Duration
51+
lastMonoNano atomic.Int64
52+
// clock returns the current monotonic timestamp in nanoseconds.
53+
// Defaults to monoNow; overridden in tests for determinism.
54+
clock func() int64
55+
}
56+
57+
func monoNow() int64 {
58+
return time.Since(epoch).Nanoseconds()
59+
}
60+
61+
// NewPollingHealthChecker creates a PollingHealthChecker that reports
62+
// unhealthy if MarkAlive has not been called within the given threshold.
63+
// The initial state is alive to allow time for the first poll cycle.
64+
func NewPollingHealthChecker(stalenessThreshold time.Duration) *PollingHealthChecker {
65+
hc := &PollingHealthChecker{
66+
stalenessThreshold: stalenessThreshold,
67+
clock: monoNow,
68+
}
69+
hc.lastMonoNano.Store(hc.clock())
70+
71+
return hc
72+
}
73+
74+
// MarkAlive records that the polling loop is still running. This method
75+
// is safe to call from any goroutine.
76+
func (h *PollingHealthChecker) MarkAlive() {
77+
h.lastMonoNano.Store(h.clock())
78+
}
79+
80+
// Healthy implements HealthChecker. It returns nil if MarkAlive was
81+
// called within the staleness threshold, or an error otherwise.
82+
func (h *PollingHealthChecker) Healthy(_ context.Context) error {
83+
elapsed := time.Duration(h.clock()-h.lastMonoNano.Load()) * time.Nanosecond
84+
85+
if elapsed > h.stalenessThreshold {
86+
return fmt.Errorf(
87+
"polling loop stale: last iteration %s ago, threshold %s",
88+
elapsed.Truncate(time.Second), h.stalenessThreshold,
89+
)
90+
}
91+
92+
return nil
93+
}
Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package server
16+
17+
import (
18+
"context"
19+
"strings"
20+
"sync/atomic"
21+
"testing"
22+
"time"
23+
)
24+
25+
func TestPollingHealthChecker_InitiallyHealthy(t *testing.T) {
26+
hc := NewPollingHealthChecker(5 * time.Minute)
27+
28+
if err := hc.Healthy(context.Background()); err != nil {
29+
t.Fatalf("expected healthy immediately after creation, got: %v", err)
30+
}
31+
}
32+
33+
func TestPollingHealthChecker_MarkAliveResetsStale(t *testing.T) {
34+
var now atomic.Int64
35+
hc := &PollingHealthChecker{
36+
stalenessThreshold: 10 * time.Second,
37+
clock: func() int64 { return now.Load() },
38+
}
39+
now.Store(0)
40+
hc.lastMonoNano.Store(hc.clock())
41+
42+
// Advance past threshold.
43+
now.Store((15 * time.Second).Nanoseconds())
44+
if err := hc.Healthy(context.Background()); err == nil {
45+
t.Fatal("expected stale error after 15s with 10s threshold")
46+
}
47+
48+
// MarkAlive resets.
49+
hc.MarkAlive()
50+
if err := hc.Healthy(context.Background()); err != nil {
51+
t.Fatalf("expected healthy after MarkAlive, got: %v", err)
52+
}
53+
}
54+
55+
func TestPollingHealthChecker_StaleAfterThreshold(t *testing.T) {
56+
var now atomic.Int64
57+
hc := &PollingHealthChecker{
58+
stalenessThreshold: 30 * time.Second,
59+
clock: func() int64 { return now.Load() },
60+
}
61+
now.Store(0)
62+
hc.lastMonoNano.Store(hc.clock())
63+
64+
// At exactly threshold: not stale (> is the check, not >=).
65+
now.Store((30 * time.Second).Nanoseconds())
66+
if err := hc.Healthy(context.Background()); err != nil {
67+
t.Fatalf("expected healthy at exact threshold, got: %v", err)
68+
}
69+
70+
// One nanosecond past threshold: stale.
71+
now.Store((30 * time.Second).Nanoseconds() + 1)
72+
err := hc.Healthy(context.Background())
73+
if err == nil {
74+
t.Fatal("expected stale error just past threshold")
75+
}
76+
if !strings.Contains(err.Error(), "polling loop stale") {
77+
t.Fatalf("unexpected error message: %v", err)
78+
}
79+
}
80+
81+
func TestPollingHealthChecker_StaleErrorContainsDetails(t *testing.T) {
82+
var now atomic.Int64
83+
hc := &PollingHealthChecker{
84+
stalenessThreshold: 1 * time.Minute,
85+
clock: func() int64 { return now.Load() },
86+
}
87+
now.Store(0)
88+
hc.lastMonoNano.Store(hc.clock())
89+
90+
now.Store((90 * time.Second).Nanoseconds())
91+
err := hc.Healthy(context.Background())
92+
if err == nil {
93+
t.Fatal("expected error")
94+
}
95+
96+
msg := err.Error()
97+
if !strings.Contains(msg, "1m30s") {
98+
t.Errorf("expected elapsed duration in error, got: %s", msg)
99+
}
100+
if !strings.Contains(msg, "1m0s") {
101+
t.Errorf("expected threshold in error, got: %s", msg)
102+
}
103+
}

‎data-models/pkg/protos/external_remediation.pb.go‎

Lines changed: 6 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎data-models/protobufs/external_remediation.proto‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,12 @@ message ExternalRemediationRequest {
101101
// Cluster-scoped to match the other remediation CRDs (RebootNode,
102102
// TerminateNode, GPUReset) — an ExtRR represents a node-level claim and
103103
// operators interact with it by node name, not by namespace.
104-
scope: ST_CLUSTER
104+
scope: ST_CLUSTER,
105+
additional_columns: [
106+
{name: "Node", type: CT_STRING, json_path: ".spec.healthEvent.nodeName"},
107+
{name: "CheckName", type: CT_STRING, json_path: ".spec.healthEvent.checkName"},
108+
{name: "CompletionTime", type: CT_STRING, format: CF_DATE, json_path: ".status.completionTime"}
109+
]
105110
};
106111
ExternalRemediationRequestSpec spec = 1;
107112
ExternalRemediationRequestStatus status = 2;

‎distros/kubernetes/nvsentinel/charts/gpu-health-monitor/templates/daemonset-dcgm-3.x.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ spec:
7474
containerPort: {{ .Values.global.metricsPort }}
7575
livenessProbe:
7676
httpGet:
77-
path: /metrics
77+
path: /healthz
7878
port: {{ .Values.global.metricsPort }}
7979
initialDelaySeconds: 30
8080
periodSeconds: 30

‎distros/kubernetes/nvsentinel/charts/gpu-health-monitor/templates/daemonset-dcgm-4.x.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ spec:
7474
containerPort: {{ .Values.global.metricsPort }}
7575
livenessProbe:
7676
httpGet:
77-
path: /metrics
77+
path: /healthz
7878
port: {{ .Values.global.metricsPort }}
7979
initialDelaySeconds: 30
8080
periodSeconds: 30

‎distros/kubernetes/nvsentinel/charts/janitor/crds/external_remediation.crd.yaml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,4 +167,14 @@ spec:
167167
description: completionTime is set when the reconciler is done with this ExtRR — the apply terminated (success or NodeNotFound), the close path ran after Complete=True, or operator-initiated cleanup completed. Once set, the reconciler short-circuits to a CheckUnlock-only pass on subsequent reconciles. The TTL reconciler counts the cleanup window from this timestamp, matching the sibling janitor CRDs (RebootNode, TerminateNode, GPUReset).
168168
format: RFC3339
169169
description: ExternalRemediationRequest is the exit door from NVSentinel ownership. Created by fault-remediation when triage maps a fault to the CUSTOM recommended action (per ADR-036) with a customRecommendedAction configured to produce an ExtRR. See ADR-040 (docs/designs/040-external-remediation-request.md) for the full design including the asymmetric ExternalRemediationComplete=True vs False handling, the cleanup finalizer, the release taint, and the nvsentinel.dgxc.nvidia.com/managed label coordination mechanism.
170-
additionalPrinterColumns: []
170+
additionalPrinterColumns:
171+
- name: Node
172+
type: string
173+
jsonPath: .spec.healthEvent.nodeName
174+
- name: CheckName
175+
type: string
176+
jsonPath: .spec.healthEvent.checkName
177+
- name: CompletionTime
178+
type: string
179+
format: date
180+
jsonPath: .status.completionTime

‎distros/kubernetes/nvsentinel/charts/nic-health-monitor/templates/daemonset.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ spec:
7171
containerPort: {{ .Values.global.metricsPort }}
7272
livenessProbe:
7373
httpGet:
74-
path: /metrics
74+
path: /healthz
7575
port: {{ .Values.global.metricsPort }}
7676
initialDelaySeconds: 30
7777
periodSeconds: 30

‎distros/kubernetes/nvsentinel/charts/syslog-health-monitor/templates/_helpers.tpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ spec:
142142
containerPort: {{ $root.Values.global.metricsPort }}
143143
livenessProbe:
144144
httpGet:
145-
path: /metrics
145+
path: /healthz
146146
port: {{ $root.Values.global.metricsPort }}
147147
initialDelaySeconds: 30
148148
periodSeconds: 30

‎health-monitors/gpu-health-monitor/gpu_health_monitor/cli.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
import logging as log
1818
from importlib.metadata import version as get_package_version
1919
from threading import Event
20-
from prometheus_client import start_http_server
20+
from gpu_health_monitor.healthz import start_server as start_health_server
2121
import csv
2222
from .dcgm_watcher import dcgm
2323
from .platform_connector import platform_connector
@@ -193,7 +193,8 @@ def cli(
193193

194194
metadata_reader = MetadataReader(metadata_path) if thermal_margin_enabled else None
195195

196-
prom_server, t = start_http_server(port)
196+
poll_interval = int(dcgm_config["PollIntervalSeconds"])
197+
prom_server, t = start_health_server(port, staleness_seconds=poll_interval * 3)
197198

198199
def process_exit_signal(signum, frame):
199200
exit.set()
@@ -205,7 +206,7 @@ def process_exit_signal(signum, frame):
205206

206207
dcgm_watcher = dcgm.DCGMWatcher(
207208
addr=dcgm_addr,
208-
poll_interval_seconds=int(dcgm_config["PollIntervalSeconds"]),
209+
poll_interval_seconds=poll_interval,
209210
callbacks=enabled_event_processors,
210211
dcgm_k8s_service_enabled=dcgm_k8s_service_enabled,
211212
thermal_margin_enabled=thermal_margin_enabled,

0 commit comments

Comments
 (0)