Skip to content

feat(telemetry): surface per-integration health-check failure reasons #512

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mayankpande88 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat(telemetry): surface per-integration health-check failure reasons #512

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f2ad782
feat(telemetry): surface per-integration health-check failure reasons
mayankpande88 Jul 9, 2026
296a075
chore: update image tags for main release
github-actions[bot] Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion charts/nudgebee-agent/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ runnerServiceAccount:
runner:
image:
repository: ghcr.io/nudgebee/nudgebee-agent
tag: 2026-07-08T10-06-14_f2d9b536e5ebb3fa0b470241b64b01c6534c6bba
tag: 2026-07-08T12-07-17_760acd19456f642a6af7c95b2df8e071bad809fc
# Image template the pod_profiler action launches debugger pods from.
# The agent substitutes `{}` for the variant (bpf, jvm, python, perf, ruby).
# Surfaces as PROFILER_IMAGE; leave empty to fall back to the binary default.
Expand Down
66 changes: 46 additions & 20 deletions runner/cmd/agent/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"maps"
"net/http"
Expand Down Expand Up @@ -967,9 +968,10 @@ func run(ctx context.Context, logger *slog.Logger, cfg *config.Config) error {
probeCtx, probeCancel := context.WithTimeout(gctx, 30*time.Second)
defer probeCancel()
probeClient := &http.Client{Timeout: 5 * time.Second}
logsProvider, logsURL, logsOK, logCfg := probeLogsProvider(probeCtx, cfg)
logsProvider, logsURL, logsOK, logsErr, logCfg := probeLogsProvider(probeCtx, cfg)
as := telemetry.DetectAutoScaler(probeCtx, typedKube, providerInfo.Provider, logger)
clickhouseStatus := probeClickhouse(probeCtx, probeClient, clickhouseHost, clickhousePort)
promConnected, promErr := prometheusConnected(probeCtx, promClient, logger)
return telemetry.Datasources{
PrometheusURL: cfg.PrometheusURL,
AlertManagerURL: cfg.AlertManagerURL,
Expand All @@ -978,8 +980,10 @@ func run(ctx context.Context, logger *slog.Logger, cfg *config.Config) error {
LogsProvider: logsProvider,
LogsProviderURL: logsURL,
LogsProviderStatus: logsOK,
LogsProviderError: logsErr,
LogProviderConfig: logCfg,
PrometheusConnected: prometheusConnected(probeCtx, promClient, logger),
PrometheusConnected: promConnected,
PrometheusConnectedError: promErr,
NodeAgentCount: queryNodeAgentCount(probeCtx, promClient, logger),
PrometheusRetentionTime: telemetry.PrometheusRetention(probeCtx, promClient, logger),
PrometheusAdditionalLabels: promExtraLabels,
Expand Down Expand Up @@ -1137,24 +1141,27 @@ func (a *grafanaAdapter) HandlePrometheus(ctx context.Context, r *dispatch.Grafa
// auth — Chronosphere, Thanos Query, Grafana Mimir, Amazon Managed Prometheus —
// are reported Connected when metric queries work. Returns false on any error
// so a broken backend shows Disconnected rather than panicking the tick.
func prometheusConnected(ctx context.Context, c *prometheus.Client, logger *slog.Logger) bool {
func prometheusConnected(ctx context.Context, c *prometheus.Client, logger *slog.Logger) (ok bool, reason string) {
if c == nil || c.BaseURL == "" {
return false
return false, ""
}
cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
raw, err := c.Query(cctx, "vector(1)", "", "")
if err != nil {
logger.Debug("prometheus health query failed", "err", err)
return false
return false, err.Error()
}
var resp struct {
Status string `json:"status"`
}
if err := json.Unmarshal(raw, &resp); err != nil {
return false
return false, err.Error()
}
return resp.Status == "success"
if resp.Status == "success" {
return true, ""
}
return false, fmt.Sprintf("prometheus query returned status %q", resp.Status)
}

// queryNodeAgentCount
Expand Down Expand Up @@ -1199,37 +1206,37 @@ func queryNodeAgentCount(ctx context.Context, c *prometheus.Client, logger *slog
// configured provider's own client at action-handler time. Fail-closed: any
// non-2xx → status=false, URL stays in payload so the UI can show "URL
// configured but unhealthy".
func probeLogsProvider(ctx context.Context, cfg *config.Config) (provider, url string, ok bool, providerCfg map[string]any) {
func probeLogsProvider(ctx context.Context, cfg *config.Config) (provider, url string, ok bool, reason string, providerCfg map[string]any) {
httpClient := &http.Client{Timeout: 5 * time.Second}
switch {
case cfg.PinotURL != "":
ok = httpProbe(ctx, httpClient, cfg.PinotURL+"/health")
return "pinot", cfg.PinotURL, ok, map[string]any{}
ok, reason = httpProbeErr(ctx, httpClient, cfg.PinotURL+"/health")
return "pinot", cfg.PinotURL, ok, reason, map[string]any{}
case cfg.ElasticsearchEnabled && cfg.ElasticsearchURL != "":
// ES exposes a `_cluster/health` endpoint; we treat 200 as healthy.
// Probe with the configured credentials so the badge reflects whether
// queries will actually succeed — a secured OpenSearch/ES otherwise 401s
// on an unauthenticated probe even when the configured creds work fine.
ok = httpProbe(ctx, httpClient, cfg.ElasticsearchURL+"/_cluster/health", esAuthHeader(cfg))
ok, reason = httpProbeErr(ctx, httpClient, cfg.ElasticsearchURL+"/_cluster/health", esAuthHeader(cfg))
providerCfg = map[string]any{}
if v := os.Getenv("ELASTICSEARCH_LOG_INDEX"); v != "" {
providerCfg["default_index"] = v
}
return "ES", cfg.ElasticsearchURL, ok, providerCfg
return "ES", cfg.ElasticsearchURL, ok, reason, providerCfg
case cfg.SignozURL != "":
// Signoz health endpoint: /api/v1/health.
ok = httpProbe(ctx, httpClient, cfg.SignozURL+"/api/v1/health")
return "signoz", cfg.SignozURL, ok, map[string]any{}
ok, reason = httpProbeErr(ctx, httpClient, cfg.SignozURL+"/api/v1/health")
return "signoz", cfg.SignozURL, ok, reason, map[string]any{}
case cfg.LokiURL != "":
// LOKI_URL points at the loki gateway, whose nginx only proxies the
// `/loki/...` API paths — the backend `/ready` is not exposed there and
// 404s. Probe a gateway-served API endpoint instead so the badge
// reflects query reachability.
ok = httpProbe(ctx, httpClient, cfg.LokiURL+"/loki/api/v1/status/buildinfo")
ok, reason = httpProbeErr(ctx, httpClient, cfg.LokiURL+"/loki/api/v1/status/buildinfo")
providerCfg = map[string]any{"url": cfg.LokiURL}
return "loki", cfg.LokiURL, ok, providerCfg
return "loki", cfg.LokiURL, ok, reason, providerCfg
default:
return "", "", false, map[string]any{}
return "", "", false, "", map[string]any{}
}
}

Expand Down Expand Up @@ -1258,9 +1265,17 @@ func probeClickhouse(ctx context.Context, c *http.Client, host, port string) boo
// URL is sourced from operator-provided config (PROMETHEUS_URL, LOKI_URL,
// etc.), not request-derived — taint flow is operator → probe by design.
func httpProbe(ctx context.Context, c *http.Client, url string, headers ...map[string]string) bool {
ok, _ := httpProbeErr(ctx, c, url, headers...)
return ok
}

// httpProbeErr is httpProbe with the failure reason. Returns ok=true and an
// empty reason on 2xx; otherwise ok=false and a compact one-line reason
// (transport error or "HTTP <status>: <body-snippet>") for the health UI.
func httpProbeErr(ctx context.Context, c *http.Client, url string, headers ...map[string]string) (ok bool, reason string) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) //nolint:gosec // operator-provided URL
if err != nil {
return false
return false, err.Error()
}
for _, h := range headers {
for k, v := range h {
Expand All @@ -1269,10 +1284,21 @@ func httpProbe(ctx context.Context, c *http.Client, url string, headers ...map[s
}
resp, err := c.Do(req) //nolint:gosec // operator-provided URL
if err != nil {
return false
return false, err.Error()
}
defer func() { _ = resp.Body.Close() }()
return resp.StatusCode >= 200 && resp.StatusCode < 300
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return true, ""
}
Comment on lines +1290 to +1293

@gemini-code-assist gemini-code-assist Bot Jul 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Reading the response body on every successful health check probe is inefficient as it allocates memory unnecessarily. Additionally, if the response body of a successful probe is larger than 512 bytes, limiting the read to 512 bytes without fully draining the remaining body prevents the HTTP client from reusing the underlying TCP connection (keep-alive).

Checking the status code first allows us to fully drain the body to io.Discard on success, ensuring connection reuse and avoiding unnecessary allocations.

Suggested change
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return true, ""
}
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
_, _ = io.Copy(io.Discard, resp.Body)
return true, ""
}
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))

msg := strings.Join(strings.Fields(string(body)), " ")
if len(msg) > 200 {
msg = msg[:200] + "…"
}
if msg == "" {
return false, fmt.Sprintf("HTTP %d", resp.StatusCode)
}
return false, fmt.Sprintf("HTTP %d: %s", resp.StatusCode, msg)
}

// esHTTPClient returns the HTTP client the ES query client uses. When
Expand Down
16 changes: 9 additions & 7 deletions runner/cmd/agent/probe_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestProbeLogsProvider_ESDisabledFallsThroughToLoki(t *testing.T) {
ElasticsearchEnabled: false,
LokiURL: loki.URL,
}
provider, url, ok, _ := probeLogsProvider(context.Background(), cfg)
provider, url, ok, _, _ := probeLogsProvider(context.Background(), cfg)
if provider != "loki" {
t.Fatalf("provider = %q, want loki", provider)
}
Expand Down Expand Up @@ -64,7 +64,7 @@ func TestProbeLogsProvider_StrayESURLDoesNotMaskSignoz(t *testing.T) {
ElasticsearchEnabled: false,
SignozURL: signoz.URL,
}
provider, url, ok, _ := probeLogsProvider(context.Background(), cfg)
provider, url, ok, _, _ := probeLogsProvider(context.Background(), cfg)
if provider != "signoz" {
t.Fatalf("provider = %q, want signoz (stray ES URL must not mask SigNoz)", provider)
}
Expand Down Expand Up @@ -94,7 +94,7 @@ func TestProbeLogsProvider_ESProbeSendsAuth(t *testing.T) {
ElasticsearchUser: "admin",
ElasticsearchPassword: "pw",
}
provider, _, ok, _ := probeLogsProvider(context.Background(), cfg)
provider, _, ok, _, _ := probeLogsProvider(context.Background(), cfg)
if provider != "ES" {
t.Fatalf("provider = %q, want ES", provider)
}
Expand Down Expand Up @@ -131,7 +131,7 @@ func TestPrometheusConnected_ChronosphereStyleBackend(t *testing.T) {

c := prometheus.New(srv.URL, nil)
c.ExtraHeaders = config.ParseHeaders("Authorization: Bearer tok")
if !prometheusConnected(context.Background(), c, slog.Default()) {
if ok, _ := prometheusConnected(context.Background(), c, slog.Default()); !ok {
t.Error("expected connected=true for query-only backend serving /api/v1/query")
}
if !queried {
Expand All @@ -148,14 +148,16 @@ func TestPrometheusConnected_FailuresReportDisconnected(t *testing.T) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer down.Close()
if prometheusConnected(context.Background(), prometheus.New(down.URL, nil), slog.Default()) {
if ok, reason := prometheusConnected(context.Background(), prometheus.New(down.URL, nil), slog.Default()); ok {
t.Error("expected connected=false when backend returns 500")
} else if reason == "" {
t.Error("expected a non-empty failure reason when backend returns 500")
}
// Nil / unconfigured client → not connected, no panic.
if prometheusConnected(context.Background(), nil, slog.Default()) {
if ok, _ := prometheusConnected(context.Background(), nil, slog.Default()); ok {
t.Error("expected connected=false for nil client")
}
if prometheusConnected(context.Background(), prometheus.New("", nil), slog.Default()) {
if ok, _ := prometheusConnected(context.Background(), prometheus.New("", nil), slog.Default()); ok {
t.Error("expected connected=false for empty base URL")
}
}
Loading