|
| 1 | +// Unless explicitly stated otherwise all files in this repository are licensed |
| 2 | +// under the Apache License Version 2.0. |
| 3 | +// This product includes software developed at Datadog (https://www.datadoghq.com/). |
| 4 | +// Copyright 2026-present Datadog, Inc. |
| 5 | + |
| 6 | +//go:build kubeapiserver |
| 7 | + |
| 8 | +package workload |
| 9 | + |
| 10 | +import ( |
| 11 | + "embed" |
| 12 | + "io" |
| 13 | + "sort" |
| 14 | + "sync" |
| 15 | + "time" |
| 16 | + |
| 17 | + "github.com/DataDog/datadog-agent/comp/core/status" |
| 18 | + "github.com/DataDog/datadog-agent/pkg/config/remote/data" |
| 19 | + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" |
| 20 | + "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" |
| 21 | +) |
| 22 | + |
| 23 | +//go:embed status_templates |
| 24 | +var templatesFS embed.FS |
| 25 | + |
| 26 | +// statusStore is the live autoscaling store, registered by the provider at |
| 27 | +// startup. It is nil until workload autoscaling actually starts, which is how |
| 28 | +// the status distinguishes "not started" from "started with zero autoscalers". |
| 29 | +var statusStore struct { |
| 30 | + sync.RWMutex |
| 31 | + store *store |
| 32 | + isLeader func() bool |
| 33 | + rcInstance string |
| 34 | +} |
| 35 | + |
| 36 | +// InitStatus registers the live state used by the workload autoscaling status |
| 37 | +// section. rcInstance names the Remote Configuration client serving the |
| 38 | +// autoscaling products, so the status can point at the right RC instance when |
| 39 | +// extra clients are configured. |
| 40 | +func InitStatus(store *store, isLeader func() bool, rcInstance string) { |
| 41 | + statusStore.Lock() |
| 42 | + defer statusStore.Unlock() |
| 43 | + statusStore.store = store |
| 44 | + statusStore.isLeader = isLeader |
| 45 | + statusStore.rcInstance = rcInstance |
| 46 | +} |
| 47 | + |
| 48 | +// productStatus tracks what the last Remote Configuration update for a product |
| 49 | +// carried. Versions are per-config, so the highest one in an update is the most |
| 50 | +// useful single number to show. |
| 51 | +type productStatus struct { |
| 52 | + LastUpdate time.Time `json:"last_update"` |
| 53 | + LastVersion uint64 `json:"last_version"` |
| 54 | + ConfigCount int `json:"config_count"` |
| 55 | + UpdateCount uint64 `json:"update_count"` |
| 56 | + LastError string `json:"last_error,omitempty"` |
| 57 | + LastErrorTime time.Time `json:"last_error_time,omitempty"` |
| 58 | +} |
| 59 | + |
| 60 | +var rcTracker = struct { |
| 61 | + sync.RWMutex |
| 62 | + byProduct map[string]*productStatus |
| 63 | +}{byProduct: map[string]*productStatus{}} |
| 64 | + |
| 65 | +func trackedProduct(product string) *productStatus { |
| 66 | + if existing, found := rcTracker.byProduct[product]; found { |
| 67 | + return existing |
| 68 | + } |
| 69 | + created := &productStatus{} |
| 70 | + rcTracker.byProduct[product] = created |
| 71 | + return created |
| 72 | +} |
| 73 | + |
| 74 | +// recordRemoteConfigUpdate notes a Remote Configuration update for a product. |
| 75 | +func recordRemoteConfigUpdate(product string, timestamp time.Time, update map[string]state.RawConfig) { |
| 76 | + rcTracker.Lock() |
| 77 | + defer rcTracker.Unlock() |
| 78 | + |
| 79 | + tracked := trackedProduct(product) |
| 80 | + tracked.LastUpdate = timestamp |
| 81 | + tracked.ConfigCount = len(update) |
| 82 | + tracked.UpdateCount++ |
| 83 | + for _, rawConfig := range update { |
| 84 | + if rawConfig.Metadata.Version > tracked.LastVersion { |
| 85 | + tracked.LastVersion = rawConfig.Metadata.Version |
| 86 | + } |
| 87 | + } |
| 88 | +} |
| 89 | + |
| 90 | +// recordRemoteConfigError notes a config that failed to apply. |
| 91 | +func recordRemoteConfigError(product string, timestamp time.Time, err error) { |
| 92 | + rcTracker.Lock() |
| 93 | + defer rcTracker.Unlock() |
| 94 | + |
| 95 | + tracked := trackedProduct(product) |
| 96 | + tracked.LastError = err.Error() |
| 97 | + tracked.LastErrorTime = timestamp |
| 98 | +} |
| 99 | + |
| 100 | +// autoscalingProducts are the Remote Configuration products backing workload |
| 101 | +// autoscaling. Cluster autoscaling is deliberately out of scope here. |
| 102 | +var autoscalingProducts = []string{ |
| 103 | + data.ProductContainerAutoscalingSettings, |
| 104 | + data.ProductContainerAutoscalingValues, |
| 105 | +} |
| 106 | + |
| 107 | +// Provider populates the workload autoscaling status section. |
| 108 | +type Provider struct{} |
| 109 | + |
| 110 | +// Name returns the name |
| 111 | +func (Provider) Name() string { |
| 112 | + return "Workload Autoscaling" |
| 113 | +} |
| 114 | + |
| 115 | +// Section returns the section. |
| 116 | +// |
| 117 | +// Sections are rendered in alphabetical order (only "collector" is special |
| 118 | +// cased), so "Autoscaling" places this group directly after "Autodiscovery". |
| 119 | +// It is also the natural group for cluster autoscaling to join later. |
| 120 | +func (Provider) Section() string { |
| 121 | + return "Autoscaling" |
| 122 | +} |
| 123 | + |
| 124 | +// JSON populates the status map |
| 125 | +func (Provider) JSON(_ bool, stats map[string]interface{}) error { |
| 126 | + populateStatus(stats) |
| 127 | + return nil |
| 128 | +} |
| 129 | + |
| 130 | +// Text renders the text output |
| 131 | +func (Provider) Text(_ bool, buffer io.Writer) error { |
| 132 | + return status.RenderText(templatesFS, "workloadautoscaling.tmpl", buffer, getStatusInfo()) |
| 133 | +} |
| 134 | + |
| 135 | +// HTML renders the html output |
| 136 | +func (Provider) HTML(_ bool, buffer io.Writer) error { |
| 137 | + return status.RenderHTML(templatesFS, "workloadautoscalingHTML.tmpl", buffer, getStatusInfo()) |
| 138 | +} |
| 139 | + |
| 140 | +func getStatusInfo() map[string]interface{} { |
| 141 | + stats := make(map[string]interface{}) |
| 142 | + populateStatus(stats) |
| 143 | + return stats |
| 144 | +} |
| 145 | + |
| 146 | +func populateStatus(stats map[string]interface{}) { |
| 147 | + info := map[string]interface{}{} |
| 148 | + |
| 149 | + statusStore.RLock() |
| 150 | + liveStore, isLeader, rcInstance := statusStore.store, statusStore.isLeader, statusStore.rcInstance |
| 151 | + statusStore.RUnlock() |
| 152 | + |
| 153 | + if !pkgconfigsetup.Datadog().GetBool("autoscaling.workload.enabled") { |
| 154 | + info["Disabled"] = "Workload autoscaling is not enabled on the Cluster Agent" |
| 155 | + stats["workloadAutoscaling"] = info |
| 156 | + return |
| 157 | + } |
| 158 | + |
| 159 | + if liveStore == nil { |
| 160 | + // Enabled but the store is not registered yet: either still starting, or |
| 161 | + // StartWorkloadAutoscaling returned an error (which is logged, not fatal). |
| 162 | + info["Started"] = false |
| 163 | + stats["workloadAutoscaling"] = info |
| 164 | + return |
| 165 | + } |
| 166 | + |
| 167 | + info["Started"] = true |
| 168 | + info["PodAutoscalerCount"] = liveStore.Count() |
| 169 | + if isLeader != nil { |
| 170 | + info["IsLeader"] = isLeader() |
| 171 | + } |
| 172 | + if rcInstance != "" { |
| 173 | + info["RemoteConfigInstance"] = rcInstance |
| 174 | + } |
| 175 | + |
| 176 | + now := time.Now() |
| 177 | + products := make([]map[string]interface{}, 0, len(autoscalingProducts)) |
| 178 | + connected := false |
| 179 | + |
| 180 | + rcTracker.RLock() |
| 181 | + for _, product := range autoscalingProducts { |
| 182 | + entry := map[string]interface{}{"Product": product} |
| 183 | + tracked, found := rcTracker.byProduct[product] |
| 184 | + if !found || tracked.LastUpdate.IsZero() { |
| 185 | + // Subscribed, but the backend has not sent anything yet. This is the |
| 186 | + // normal state for an org with no autoscalers configured. |
| 187 | + entry["Received"] = false |
| 188 | + } else { |
| 189 | + connected = true |
| 190 | + entry["Received"] = true |
| 191 | + entry["LastUpdate"] = tracked.LastUpdate.UTC().Format(time.RFC3339) |
| 192 | + entry["LastUpdateAge"] = now.Sub(tracked.LastUpdate).Truncate(time.Second).String() |
| 193 | + entry["LastVersion"] = tracked.LastVersion |
| 194 | + entry["ConfigCount"] = tracked.ConfigCount |
| 195 | + entry["UpdateCount"] = tracked.UpdateCount |
| 196 | + if tracked.LastError != "" { |
| 197 | + entry["LastError"] = tracked.LastError |
| 198 | + entry["LastErrorTime"] = tracked.LastErrorTime.UTC().Format(time.RFC3339) |
| 199 | + } |
| 200 | + } |
| 201 | + products = append(products, entry) |
| 202 | + } |
| 203 | + rcTracker.RUnlock() |
| 204 | + |
| 205 | + sort.Slice(products, func(i, j int) bool { |
| 206 | + return products[i]["Product"].(string) < products[j]["Product"].(string) |
| 207 | + }) |
| 208 | + info["RemoteConfigProducts"] = products |
| 209 | + // "Connected" means at least one autoscaling product has delivered an update |
| 210 | + // to this process. It is subscription-level health, not TCP connectivity -- |
| 211 | + // see the Remote Configuration section for the transport/auth state. |
| 212 | + info["RemoteConfigConnected"] = connected |
| 213 | + |
| 214 | + stats["workloadAutoscaling"] = info |
| 215 | +} |
0 commit comments