From 815ac1871e0c5e062f39dc023d6090ecfac77987 Mon Sep 17 00:00:00 2001 From: Nathan Hensley Date: Wed, 22 Apr 2026 21:27:28 +0000 Subject: [PATCH 1/4] feat(recipes/gb200-eks): self-fulfill NVreg_GrdmaPciTopoCheckOverride=1 Wire the existing kernel-module-params ConfigMap template into the GB200/EKS overlay and point gpu-operator ClusterPolicy at it via driver.kernelModuleConfig.name. The NVIDIA driver DaemonSet now mounts nvidia.conf at load time and the kernel comes up with the flag set, which is required on GB200+EFA for EFA dma-buf attach to the Grace PCI topology. Without the flag, NCCL silently falls back to the Socket transport. The existing NVreg preflight check stays in place as a belt-and-suspenders guard: it keeps its actionable error message for operators who disable the override at a higher layer or ship a cluster with a different module config. Scope: GB200/EKS only. The PCIe-topology issue is EKS+EFA specific; OKE, GKE, and AKS GB200 overlays are unaffected. Verified by bundling eks/gb200/ubuntu/training and inspecting gpu-operator/manifests/kernel-module-params.yaml + values.yaml; h100/eks bundle does NOT render the ConfigMap. --- recipes/overlays/gb200-eks-training.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/recipes/overlays/gb200-eks-training.yaml b/recipes/overlays/gb200-eks-training.yaml index bce387c14..456031c63 100644 --- a/recipes/overlays/gb200-eks-training.yaml +++ b/recipes/overlays/gb200-eks-training.yaml @@ -36,6 +36,13 @@ spec: # GB200-specific GPU Operator overrides (inherits valuesFile from eks-training) - name: gpu-operator type: Helm + manifestFiles: + # Ships a ConfigMap with options nvidia NVreg_GrdmaPciTopoCheckOverride=1. + # driver.kernelModuleConfig.name (below) points the ClusterPolicy at it, + # so the driver DaemonSet mounts nvidia.conf and the kernel loads with + # the flag set — required on GB200+EFA for EFA dma-buf attach to the + # Grace PCI topology. Without it, NCCL silently falls back to Socket. + - components/gpu-operator/manifests/kernel-module-params.yaml dependencyRefs: - nfd - cert-manager @@ -46,6 +53,9 @@ spec: enabled: true gdrcopy: enabled: true + driver: + kernelModuleConfig: + name: nvidia-kernel-module-params # GB200 uses nodewright no-op: the H100 tuning packages (nvidia-setup, # nvidia-tuned) are not compatible with GB200's ARM64 host CPU and From 36ce17496fb15d558ebd22064c80ccb76fe665d4 Mon Sep 17 00:00:00 2001 From: Nathan Hensley Date: Wed, 22 Apr 2026 21:28:17 +0000 Subject: [PATCH 2/4] feat(recipes/gb200-eks): bump driver to 580.126.20 (NVIDIA GB200+EFA floor) Override gpu-operator.driver.version at the GB200/EKS overlay layer so GB200+EFA recipes ship with the NVIDIA-recommended driver floor while H100/B200 and non-EKS GB200 stay on the global 580.105.08 default in components/gpu-operator/values.yaml. Narrower blast radius than a global bump: the version recommendation is specific to GB200+EFA dma-buf topology on EKS, and Skyhook compatibility already diverges between accelerators (see the GB200 no-op comment in this same overlay). Verified with aicr query --selector components.gpu-operator.values.driver.version: gb200/eks -> 580.126.20 h100/eks -> 580.105.08 (unchanged) gb200/oke -> 580.105.08 (unchanged) --- recipes/overlays/gb200-eks-training.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/recipes/overlays/gb200-eks-training.yaml b/recipes/overlays/gb200-eks-training.yaml index 456031c63..9223315c6 100644 --- a/recipes/overlays/gb200-eks-training.yaml +++ b/recipes/overlays/gb200-eks-training.yaml @@ -54,6 +54,10 @@ spec: gdrcopy: enabled: true driver: + # 580.126.20 is NVIDIA's recommended floor for GB200+EFA; the global + # default (580.105.08 in components/gpu-operator/values.yaml) stays + # unchanged for H100/B200 and non-EKS GB200 recipes. + version: 580.126.20 kernelModuleConfig: name: nvidia-kernel-module-params From 14c975dcc81eac67485f8f7768852c1d6d488a40 Mon Sep 17 00:00:00 2001 From: Nathan Hensley Date: Wed, 22 Apr 2026 21:30:46 +0000 Subject: [PATCH 3/4] feat(recipes/gb200-eks): adopt nccl-all-reduce-bw-net and -nvls constraints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Default GB200/EKS training recipes to the two transport-specific NCCL variants introduced earlier on this branch series. The validator Catalog entries already exist; no overlay referenced them until now. NET exercises EFA and NVLS exercises MNNVL across the NVL72 IMEX domain. Each variant asserts its transport actually carried traffic (via the verifyTransportFromLogs check in validators/performance), so a silent fallback to Socket or NET cannot masquerade as a pass — a failure mode the legacy nccl-all-reduce-bw check cannot distinguish. Thresholds are deliberately conservative (NET >= 40 GB/s, NVLS >= 500 GB/s), sized for a 2-node GB200 pair. They catch clear misconfigurations today and will be raised once production NVL72 data is available. Merge behavior: ValidationPhase replaces rather than merges, so this block replaces the inherited nccl-all-reduce-bw >= 720 from gb200-any-training on GB200/EKS recipes only. Non-EKS GB200 (OKE, etc.) and non-GB200 accelerators keep the legacy entry unchanged. Verified by resolving recipes for gb200/eks (NET+NVLS), gb200/oke (legacy 720), and h100/eks (legacy 300). --- recipes/overlays/gb200-eks-training.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/recipes/overlays/gb200-eks-training.yaml b/recipes/overlays/gb200-eks-training.yaml index 9223315c6..5b7ae8c76 100644 --- a/recipes/overlays/gb200-eks-training.yaml +++ b/recipes/overlays/gb200-eks-training.yaml @@ -79,6 +79,20 @@ spec: # Validation checks for GB200 on EKS training workloads. # Defined at the intent layer (not OS-specific) so all OS variants inherit them. validation: + performance: + # NET exercises EFA (the external interconnect) and NVLS exercises MNNVL + # across the NVL72 IMEX domain — each variant asserts its transport + # actually carried traffic, so a silent fallback to Socket or NET does + # not masquerade as a pass. Thresholds are sized for a 2-node GB200 pair + # and will be raised once production NVL72 data is available. + checks: + - nccl-all-reduce-bw-net + - nccl-all-reduce-bw-nvls + constraints: + - name: nccl-all-reduce-bw-net + value: ">= 40" + - name: nccl-all-reduce-bw-nvls + value: ">= 500" conformance: checks: - platform-health From 84d5746ee596d8985baa2d4213c0207be9bca93c Mon Sep 17 00:00:00 2001 From: Nathan Hensley Date: Thu, 23 Apr 2026 23:36:54 +0000 Subject: [PATCH 4/4] feat(recipes/gb200-eks): extend NCCL variants + NVreg fulfillment to inference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NCCL all-reduce-bw-net / -nvls measure fabric health (EFA inter-node + MNNVL intra-NVL72), not anything training-specific. Multi-node inference on GB200/EKS — tensor-parallel serving for large models, MoE expert parallelism — crosses the same fabrics as training all-reduce and has the same NVreg_GrdmaPciTopoCheckOverride=1 dma-buf attach requirement. Tried an extraction into a gb200-eks-gpuops mixin first, but the mixin system is strictly additive: a mixin can only introduce new componentRef names, not extend one already defined in the inheritance chain (and eks-training / eks-inference both declare gpu-operator with a valuesFile). Falling back to per-leaf duplication with "keep in sync" comments — 34 added lines on the inference side, 0 meaningful change on training. Changes: - gb200-eks-inference.yaml: gpu-operator componentRef gains the same kernel-module-params manifestFile + driver.kernelModuleConfig.name + driver.version:580.126.20 + cdi/gdrcopy overrides that landed for training in c162888f/3c32e9ed. Also adds the nccl-all-reduce-bw-net (>=40) and -nvls (>=500) performance constraints. - gb200-eks-training.yaml: comment updated to flag the training/inference sync relationship; content unchanged. - docs/user/validation.md: documents all three NCCL variants in a table with platform→variant selection rules, replacing the single-variant description. Closes the "docs/user/validation.md still only documents nccl-all-reduce-bw" follow-up now that an overlay adopts the variants. Verified via `aicr query`: - eks/gb200/training and eks/gb200/inference both hydrate driver.version=580.126.20 and kernelModuleConfig.name= nvidia-kernel-module-params. - Both carry nccl-all-reduce-bw-net/-nvls under validation.performance.constraints. - oke/gb200 and eks/h100 still hydrate driver.version=580.105.08 (the global default) — no collateral impact. --- docs/user/validation.md | 36 ++++++++++++++++------- recipes/overlays/gb200-eks-inference.yaml | 34 +++++++++++++++++++++ recipes/overlays/gb200-eks-training.yaml | 5 +++- 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/docs/user/validation.md b/docs/user/validation.md index 4619b7df0..02d203a02 100644 --- a/docs/user/validation.md +++ b/docs/user/validation.md @@ -40,9 +40,21 @@ any phase. If pre-flight fails, no validator Jobs are deployed. ## Training performance validation -Training performance runs the `nccl-all-reduce-bw` check — a Kubeflow `TrainJob` -that runs the canonical `all_reduce_perf` benchmark across all GPU nodes and -measures aggregate bus bandwidth. +Training performance runs an NCCL all-reduce benchmark — a Kubeflow `TrainJob` +that runs `all_reduce_perf` across GPU nodes and measures aggregate bus +bandwidth. Three check variants are available; the recipe picks the one (or +ones) that match the target fabric: + +| Check | Transport | When it's selected | +|---|---|---| +| `nccl-all-reduce-bw` | Auto-detect (whatever NCCL picks) | Default for H100 on EKS/GKE, and for GB200/B200 on non-EKS services. Preserves the pre-variant behavior. | +| `nccl-all-reduce-bw-net` | NET (EFA on EKS) | GB200 + EKS. Asserts EFA actually carried traffic — catches silent fallback to Socket when the NVIDIA driver is missing `NVreg_GrdmaPciTopoCheckOverride=1`. | +| `nccl-all-reduce-bw-nvls` | NVLS (MNNVL across an NVL72 IMEX domain) | GB200 + EKS. Asserts the NVLS communicator actually initialized — catches silent fallback to EFA when the IMEX domain is misconfigured. | + +GB200/EKS recipes (both `training` and `inference` intents) enable `-net` and +`-nvls` together rather than the auto-detect variant, because those nodes +expose two inter-node fabrics simultaneously and a single auto-detect test +would only exercise one of them. ```bash # Capture snapshot, generate training recipe, validate the performance phase. @@ -55,24 +67,26 @@ aicr recipe --service eks --accelerator h100 --os ubuntu \ aicr validate --recipe recipe.yaml --snapshot snapshot.yaml --phase performance ``` -The generated recipe lists `nccl-all-reduce-bw` under +The generated recipe lists the selected variant(s) under `validation.performance.checks` with a platform-tuned bandwidth constraint -(example: `>= 300 GB/s` for H100 + EFA). +(example: `>= 300 GB/s` for H100 + EFA; `>= 40 GB/s` NET and `>= 500 GB/s` +NVLS for GB200 + EFA, each sized for a 2-node pair). -Expected flow (~5–10 min): readiness pre-flight → deploy `TrainingRuntime` + -`TrainJob` in `aicr-validation` → worker pods reach `Running` → run -`all_reduce_perf` → parse peak bus bandwidth → compare to recipe constraint -(10 % tolerance) → cleanup. +Expected flow (~5–10 min per variant): readiness pre-flight → deploy +`TrainingRuntime` + `TrainJob` in `aicr-validation` → worker pods reach +`Running` → run `all_reduce_perf` → parse peak bus bandwidth → verify the +intended transport actually carried traffic (for `-net` / `-nvls`) → compare +to recipe constraint (10 % tolerance) → cleanup. A passing CTRF entry: ```json { - "name": "nccl-all-reduce-bw", + "name": "nccl-all-reduce-bw-net", "status": "passed", "suite": ["performance"], "stdout": [ - "NCCL All Reduce bandwidth: GB/s", + "NCCL All Reduce bandwidth (nccl-all-reduce-bw-net): GB/s", "Constraint: >= → true" ] } diff --git a/recipes/overlays/gb200-eks-inference.yaml b/recipes/overlays/gb200-eks-inference.yaml index 205fb951e..a6d9dac0d 100644 --- a/recipes/overlays/gb200-eks-inference.yaml +++ b/recipes/overlays/gb200-eks-inference.yaml @@ -33,13 +33,30 @@ spec: value: ">= 1.32.4" componentRefs: + # GB200+EKS GPU Operator prerequisites. Keep in sync with the same block + # in gb200-eks-training.yaml. The NVreg flag and cdi+gdrcopy are needed + # for multi-node inference that crosses EFA — tensor-parallel serving + # over the network has the same dma-buf attach requirement as training + # all-reduce. The mixin system can't extend componentRefs already + # declared in the inheritance chain, so this lives per-leaf for now. - name: gpu-operator type: Helm + manifestFiles: + - components/gpu-operator/manifests/kernel-module-params.yaml dependencyRefs: - nfd - cert-manager - kube-prometheus-stack - nodewright-customizations + overrides: + cdi: + enabled: true + gdrcopy: + enabled: true + driver: + version: 580.126.20 + kernelModuleConfig: + name: nvidia-kernel-module-params # GB200 uses nodewright no-op: the H100 tuning packages (nvidia-setup, # nvidia-tuned) are not compatible with GB200's ARM64 host CPU and @@ -55,3 +72,20 @@ spec: intent: inference dependencyRefs: - nodewright-operator + + # NCCL fabric health checks. NET exercises EFA (inter-node), NVLS exercises + # MNNVL (intra-NVL72). Both matter for multi-node inference that spans the + # fabric (tensor-parallel serving, MoE expert parallelism); single-node + # deployments hit the WorkerCount < 2 skip path gracefully. Thresholds sized + # for a 2-node GB200 pair — will be raised once production NVL72 data is + # available. + validation: + performance: + checks: + - nccl-all-reduce-bw-net + - nccl-all-reduce-bw-nvls + constraints: + - name: nccl-all-reduce-bw-net + value: ">= 40" + - name: nccl-all-reduce-bw-nvls + value: ">= 500" diff --git a/recipes/overlays/gb200-eks-training.yaml b/recipes/overlays/gb200-eks-training.yaml index 5b7ae8c76..00f201b96 100644 --- a/recipes/overlays/gb200-eks-training.yaml +++ b/recipes/overlays/gb200-eks-training.yaml @@ -33,7 +33,10 @@ spec: value: ">= 1.32.4" componentRefs: - # GB200-specific GPU Operator overrides (inherits valuesFile from eks-training) + # GB200+EKS GPU Operator prerequisites. Keep in sync with the same block + # in gb200-eks-inference.yaml — the mixin system can't extend componentRefs + # already declared in the inheritance chain (gpu-operator comes from + # eks-training / eks-inference), so this lives per-leaf for now. - name: gpu-operator type: Helm manifestFiles: