fix(health): limit concurrent Redfish calls per BMC endpoint

crates/health/Cargo.toml

@@ -83,6 +83,7 @@ nv-redfish = { workspace = true, features = [
   "chassis",
   "computer-systems",
   "event-service",
+  "http-extras",
   "log-services",
   "managers",
   "memory",

crates/health/README.md

@@ -119,9 +119,10 @@ For a faster fault-injection loop, copy
 run hardware health with the copied file:
 
 ```toml
+bmc_request_concurrency = 4
+
 [collectors.sensors]
 sensor_fetch_interval = "5s"
-sensor_fetch_concurrency = 4
 include_sensor_thresholds = true
 ```
 

crates/health/example/config.example.toml

@@ -20,6 +20,10 @@ shard = 0
 shards_count = 1
 
 cache_size = 100
+# Maximum concurrent Redfish operations per BMC, including collector fan-out.
+# Must be greater than zero. Default: 4.
+# Changes take effect after restarting carbide-health.
+bmc_request_concurrency = 4
 bmc_proxy_url = "http://proxy.example.com:8080"
 
 endpoint_discovery_interval = "5m"
@@ -213,12 +217,10 @@ log_mode = "unreachable"
 
 [collectors.sensors]
 sensor_fetch_interval = "1m"
-sensor_fetch_concurrency = 10
 include_sensor_thresholds = true
 
 [collectors.metrics]
 fetch_interval = "2m"
-fetch_concurrency = 4
 
 # Redfish telemetry service: pre-aggregated metric reports, one GET per report
 # instead of one per resource. Published under the `hw_telemetry` series name so
@@ -278,9 +280,6 @@ exclude_services = ["Journal"]
 [collectors.logs.sse]
 initial_backoff = "1s"
 max_backoff = "30s"
-# Maximum number of concurrent Redfish GET requests used to fetch EventRecord
-# resources referenced by @odata.id in SSE notifications.
-event_record_fetch_concurrency = 4
 
 # required when
 # mode = "periodic"; must not be set when mode = "sse"

crates/health/src/api_client.rs

@@ -22,6 +22,7 @@
 use std::collections::{HashMap, HashSet};
 use std::convert::TryFrom;
 use std::net::IpAddr;
+use std::num::NonZeroUsize;
 use std::str::FromStr;
 use std::sync::{Arc, Mutex};
 
@@ -295,6 +296,7 @@ pub struct ApiEndpointSource {
     reqwest: ReqwestClient,
     proxy_url: Option<Url>,
     cache_size: usize,
+    bmc_request_concurrency: NonZeroUsize,
     bmc_latency_metrics: Option<Arc<BmcLatencyMetrics>>,
     bmc_client_cache: Mutex<HashMap<MacAddress, CachedBmcClient>>,
 }
@@ -313,12 +315,31 @@ impl ApiEndpointSource {
         proxy_url: Option<Url>,
         cache_size: usize,
         bmc_latency_metrics: Option<Arc<BmcLatencyMetrics>>,
+    ) -> Self {
+        Self::new_with_request_concurrency(
+            api,
+            reqwest,
+            proxy_url,
+            cache_size,
+            NonZeroUsize::MIN,
+            bmc_latency_metrics,
+        )
+    }
+
+    pub(crate) fn new_with_request_concurrency(
+        api: Arc<ApiClientWrapper>,
+        reqwest: ReqwestClient,
+        proxy_url: Option<Url>,
+        cache_size: usize,
+        bmc_request_concurrency: NonZeroUsize,
+        bmc_latency_metrics: Option<Arc<BmcLatencyMetrics>>,
     ) -> Self {
         Self {
             api,
             reqwest,
             proxy_url,
             cache_size,
+            bmc_request_concurrency,
             bmc_latency_metrics,
             bmc_client_cache: Mutex::new(HashMap::new()),
         }
@@ -621,6 +642,7 @@ impl ApiEndpointSource {
                     provider,
                     self.proxy_url.clone(),
                     self.cache_size,
+                    self.bmc_request_concurrency,
                     bmc_latency_instrumentation,
                 )?))
             })?
@@ -812,6 +834,7 @@ mod tests {
             provider,
             None,
             10,
+            NonZeroUsize::MIN,
             None,
         )?))
     }