fix(container-cache): size nginx workers from the CPU limit, not the host

worker_processes was `auto`, which makes nginx call
sysconf(_SC_NPROCESSORS_ONLN). That reports every core on the host and is blind
to the cgroup quota, so a pod with a CPU limit well below the node's core count
starts several times more workers than it can ever run. Observed in production:
about 128 worker processes against a 32 core limit on a 128 vCPU node.

It does not usually show up as sustained throttling, and did not here: the
measured throttled-period ratio was under 0.004 percent. The costs are
elsewhere.

- Memory. Each worker preallocates its own worker_connections structures, so
  that footprint is multiplied by the host core count rather than by the quota.
- Upstream keepalive. Pools are per worker process, so more workers than the
  pod can schedule fragments every pool and defeats connection reuse, including
  the peer-hop reuse added alongside this.
- Burst latency. Many runnable workers drain the quota well before the 100ms
  CFS period ends, so a burst becomes a multi-second stall rather than a small
  slowdown. One pod showed 431 seconds of throttled time across only 47
  throttled periods, which is about 9 seconds per event.

Derive it from resources.limits.cpu, with cache.workerProcesses as an override.
Both plain-integer and milliCPU limits are handled, sub-core limits floor to one
worker rather than zero, and no limit at all still falls back to auto, which is
the one case where the host count is the correct answer.

worker_connections is unchanged. It is a per-worker figure and its current value
was raised deliberately.

Tests: adds tests/chart-render/verify-worker-processes.sh covering the default
derivation, an explicit override, milliCPU conversion, the sub-core floor,
integer-typed limits, the no-limit fallback, and that worker_connections is left
alone. Verified the suite fails when the derivation is reverted to auto.

Relates to #1037

Co-Authored-By: Balaji Ganesan <bganesan@nvidia.com>

Author: balaji-g

deploy/helm/container-cache/deploy/files/nginx.conf

@@ -14,7 +14,33 @@
 # limitations under the License.
     user  nvs;
     env NODE_IP;
-    worker_processes  auto;
+{{- /*
+  worker_processes must track the container's CPU limit, not the node.
+  `auto` makes nginx call sysconf(_SC_NPROCESSORS_ONLN), which reports every
+  core on the host and is blind to the cgroup quota. On a large node with a
+  much smaller CPU limit that spawns several times more workers than the pod
+  can ever run, which costs memory (each worker preallocates its own
+  worker_connections structures), fragments the per-worker upstream keepalive
+  pools, and turns a burst into a multi-second CFS stall because many runnable
+  workers drain the quota well before the 100ms period ends.
+
+  Derived from resources.limits.cpu unless cache.workerProcesses overrides it.
+  Falls back to auto only when no limit is set, which is the one case where the
+  node count is the right answer.
+*/}}
+{{- $wp := ((($.Values.cache).workerProcesses) | default "" | toString) -}}
+{{- if eq $wp "" -}}
+{{-   $lim := (((($.Values.resources).limits).cpu) | default "" | toString) -}}
+{{-   if hasSuffix "m" $lim -}}
+{{-     $wp = (max 1 (div (atoi (trimSuffix "m" $lim)) 1000)) | toString -}}
+{{-   else if $lim -}}
+{{-     $n := atoi $lim -}}
+{{-     $wp = ternary "auto" ($n | toString) (eq $n 0) -}}
+{{-   else -}}
+{{-     $wp = "auto" -}}
+{{-   end -}}
+{{- end }}
+    worker_processes  {{ $wp }};
     worker_rlimit_nofile {{ $.Values.cache.workerRlimitNofile | default 65536 }};
     error_log  /var/log/nginx/error.log warn;
 

deploy/helm/container-cache/deploy/values.yaml

@@ -81,6 +81,14 @@ cache:
   # under heavy concurrent large transfers, which starved even /healthz and
   # tripped the liveness probe.
   workerConnection: 16384
+  # Number of nginx worker processes. Leave empty to derive it from
+  # resources.limits.cpu, which is almost always what you want: nginx's own
+  # `auto` reads the host's core count and cannot see the cgroup quota, so on a
+  # large node it starts several times more workers than the pod can run.
+  #
+  # This is per process, and worker_connections above is per worker, so total
+  # connection capacity is the product of the two.
+  workerProcesses: ""
 
 # Liveness/readiness probe tuning for the nginx-proxy container. /healthz is
 # served on a dedicated port but by the same worker processes, so under heavy

deploy/helm/container-cache/tests/chart-render/verify-worker-processes.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# worker_processes must track the container CPU limit, never the host core
+# count. nginx's own `auto` calls sysconf(_SC_NPROCESSORS_ONLN) and cannot see
+# the cgroup quota, so on a large node it starts several times more workers than
+# the pod can run. Run from the chart subtree:
+#   bash tests/chart-render/verify-worker-processes.sh
+set -euo pipefail
+CHART_DIR="$(cd "$(dirname "$0")/../.." && pwd)/deploy"
+fail() { echo "FAIL: $*" >&2; exit 1; }
+
+# Emits the rendered worker_processes value for the given --set arguments.
+wp() {
+  helm template t "$CHART_DIR" "$@" 2>/dev/null \
+    | grep -m1 'worker_processes' \
+    | sed -E 's/.*worker_processes +([^;]+);.*/\1/'
+}
+
+echo "1. default: derived from resources.limits.cpu, not from the host"
+got="$(wp)"
+[ "$got" = "32" ] || fail "expected 32 from the default cpu limit, got '$got'"
+
+echo "2. explicit cache.workerProcesses wins"
+got="$(wp --set cache.workerProcesses=8)"
+[ "$got" = "8" ] || fail "explicit override ignored, got '$got'"
+
+echo "3. milliCPU limits are converted, not parsed as a bare integer"
+got="$(wp --set resources.limits.cpu=16500m)"
+[ "$got" = "16" ] || fail "expected 16 from 16500m, got '$got'"
+
+echo "4. sub-core limits still yield at least one worker"
+# A naive integer division here yields 0, and `worker_processes 0` is a config
+# error that stops nginx from serving at all.
+got="$(wp --set resources.limits.cpu=500m)"
+[ "$got" = "1" ] || fail "expected 1 worker for a 500m limit, got '$got'"
+
+echo "5. integer-typed limits parse the same as string-typed"
+got="$(wp --set-json resources.limits.cpu=8)"
+[ "$got" = "8" ] || fail "expected 8 from an integer-typed limit, got '$got'"
+
+echo "6. no cpu limit falls back to auto"
+# With no quota the host core count is the correct answer, so auto is right.
+got="$(wp --set resources.limits.cpu=null)"
+[ "$got" = "auto" ] || fail "expected auto when no cpu limit is set, got '$got'"
+
+echo "7. worker_connections is per worker and is left alone"
+helm template t "$CHART_DIR" 2>/dev/null | grep -q 'worker_connections  16384' \
+  || fail "worker_connections must remain at its configured value"
+
+echo "PASS: worker_processes tracks the CPU limit in every value shape"