diff --git a/docs/user/validation.md b/docs/user/validation.md index 4619b7df0..02d203a02 100644 --- a/docs/user/validation.md +++ b/docs/user/validation.md @@ -40,9 +40,21 @@ any phase. If pre-flight fails, no validator Jobs are deployed. ## Training performance validation -Training performance runs the `nccl-all-reduce-bw` check — a Kubeflow `TrainJob` -that runs the canonical `all_reduce_perf` benchmark across all GPU nodes and -measures aggregate bus bandwidth. +Training performance runs an NCCL all-reduce benchmark — a Kubeflow `TrainJob` +that runs `all_reduce_perf` across GPU nodes and measures aggregate bus +bandwidth. Three check variants are available; the recipe picks the one (or +ones) that match the target fabric: + +| Check | Transport | When it's selected | +|---|---|---| +| `nccl-all-reduce-bw` | Auto-detect (whatever NCCL picks) | Default for H100 on EKS/GKE, and for GB200/B200 on non-EKS services. Preserves the pre-variant behavior. | +| `nccl-all-reduce-bw-net` | NET (EFA on EKS) | GB200 + EKS. Asserts EFA actually carried traffic — catches silent fallback to Socket when the NVIDIA driver is missing `NVreg_GrdmaPciTopoCheckOverride=1`. | +| `nccl-all-reduce-bw-nvls` | NVLS (MNNVL across an NVL72 IMEX domain) | GB200 + EKS. Asserts the NVLS communicator actually initialized — catches silent fallback to EFA when the IMEX domain is misconfigured. | + +GB200/EKS recipes (both `training` and `inference` intents) enable `-net` and +`-nvls` together rather than the auto-detect variant, because those nodes +expose two inter-node fabrics simultaneously and a single auto-detect test +would only exercise one of them. ```bash # Capture snapshot, generate training recipe, validate the performance phase. @@ -55,24 +67,26 @@ aicr recipe --service eks --accelerator h100 --os ubuntu \ aicr validate --recipe recipe.yaml --snapshot snapshot.yaml --phase performance ``` -The generated recipe lists `nccl-all-reduce-bw` under +The generated recipe lists the selected variant(s) under `validation.performance.checks` with a platform-tuned bandwidth constraint -(example: `>= 300 GB/s` for H100 + EFA). +(example: `>= 300 GB/s` for H100 + EFA; `>= 40 GB/s` NET and `>= 500 GB/s` +NVLS for GB200 + EFA, each sized for a 2-node pair). -Expected flow (~5–10 min): readiness pre-flight → deploy `TrainingRuntime` + -`TrainJob` in `aicr-validation` → worker pods reach `Running` → run -`all_reduce_perf` → parse peak bus bandwidth → compare to recipe constraint -(10 % tolerance) → cleanup. +Expected flow (~5–10 min per variant): readiness pre-flight → deploy +`TrainingRuntime` + `TrainJob` in `aicr-validation` → worker pods reach +`Running` → run `all_reduce_perf` → parse peak bus bandwidth → verify the +intended transport actually carried traffic (for `-net` / `-nvls`) → compare +to recipe constraint (10 % tolerance) → cleanup. A passing CTRF entry: ```json { - "name": "nccl-all-reduce-bw", + "name": "nccl-all-reduce-bw-net", "status": "passed", "suite": ["performance"], "stdout": [ - "NCCL All Reduce bandwidth: GB/s", + "NCCL All Reduce bandwidth (nccl-all-reduce-bw-net): GB/s", "Constraint: >= → true" ] } diff --git a/recipes/overlays/gb200-eks-inference.yaml b/recipes/overlays/gb200-eks-inference.yaml index 205fb951e..a6d9dac0d 100644 --- a/recipes/overlays/gb200-eks-inference.yaml +++ b/recipes/overlays/gb200-eks-inference.yaml @@ -33,13 +33,30 @@ spec: value: ">= 1.32.4" componentRefs: + # GB200+EKS GPU Operator prerequisites. Keep in sync with the same block + # in gb200-eks-training.yaml. The NVreg flag and cdi+gdrcopy are needed + # for multi-node inference that crosses EFA — tensor-parallel serving + # over the network has the same dma-buf attach requirement as training + # all-reduce. The mixin system can't extend componentRefs already + # declared in the inheritance chain, so this lives per-leaf for now. - name: gpu-operator type: Helm + manifestFiles: + - components/gpu-operator/manifests/kernel-module-params.yaml dependencyRefs: - nfd - cert-manager - kube-prometheus-stack - nodewright-customizations + overrides: + cdi: + enabled: true + gdrcopy: + enabled: true + driver: + version: 580.126.20 + kernelModuleConfig: + name: nvidia-kernel-module-params # GB200 uses nodewright no-op: the H100 tuning packages (nvidia-setup, # nvidia-tuned) are not compatible with GB200's ARM64 host CPU and @@ -55,3 +72,20 @@ spec: intent: inference dependencyRefs: - nodewright-operator + + # NCCL fabric health checks. NET exercises EFA (inter-node), NVLS exercises + # MNNVL (intra-NVL72). Both matter for multi-node inference that spans the + # fabric (tensor-parallel serving, MoE expert parallelism); single-node + # deployments hit the WorkerCount < 2 skip path gracefully. Thresholds sized + # for a 2-node GB200 pair — will be raised once production NVL72 data is + # available. + validation: + performance: + checks: + - nccl-all-reduce-bw-net + - nccl-all-reduce-bw-nvls + constraints: + - name: nccl-all-reduce-bw-net + value: ">= 40" + - name: nccl-all-reduce-bw-nvls + value: ">= 500" diff --git a/recipes/overlays/gb200-eks-training.yaml b/recipes/overlays/gb200-eks-training.yaml index bce387c14..00f201b96 100644 --- a/recipes/overlays/gb200-eks-training.yaml +++ b/recipes/overlays/gb200-eks-training.yaml @@ -33,9 +33,19 @@ spec: value: ">= 1.32.4" componentRefs: - # GB200-specific GPU Operator overrides (inherits valuesFile from eks-training) + # GB200+EKS GPU Operator prerequisites. Keep in sync with the same block + # in gb200-eks-inference.yaml — the mixin system can't extend componentRefs + # already declared in the inheritance chain (gpu-operator comes from + # eks-training / eks-inference), so this lives per-leaf for now. - name: gpu-operator type: Helm + manifestFiles: + # Ships a ConfigMap with options nvidia NVreg_GrdmaPciTopoCheckOverride=1. + # driver.kernelModuleConfig.name (below) points the ClusterPolicy at it, + # so the driver DaemonSet mounts nvidia.conf and the kernel loads with + # the flag set — required on GB200+EFA for EFA dma-buf attach to the + # Grace PCI topology. Without it, NCCL silently falls back to Socket. + - components/gpu-operator/manifests/kernel-module-params.yaml dependencyRefs: - nfd - cert-manager @@ -46,6 +56,13 @@ spec: enabled: true gdrcopy: enabled: true + driver: + # 580.126.20 is NVIDIA's recommended floor for GB200+EFA; the global + # default (580.105.08 in components/gpu-operator/values.yaml) stays + # unchanged for H100/B200 and non-EKS GB200 recipes. + version: 580.126.20 + kernelModuleConfig: + name: nvidia-kernel-module-params # GB200 uses nodewright no-op: the H100 tuning packages (nvidia-setup, # nvidia-tuned) are not compatible with GB200's ARM64 host CPU and @@ -65,6 +82,20 @@ spec: # Validation checks for GB200 on EKS training workloads. # Defined at the intent layer (not OS-specific) so all OS variants inherit them. validation: + performance: + # NET exercises EFA (the external interconnect) and NVLS exercises MNNVL + # across the NVL72 IMEX domain — each variant asserts its transport + # actually carried traffic, so a silent fallback to Socket or NET does + # not masquerade as a pass. Thresholds are sized for a 2-node GB200 pair + # and will be raised once production NVL72 data is available. + checks: + - nccl-all-reduce-bw-net + - nccl-all-reduce-bw-nvls + constraints: + - name: nccl-all-reduce-bw-net + value: ">= 40" + - name: nccl-all-reduce-bw-nvls + value: ">= 500" conformance: checks: - platform-health