Skip to content

Commit 74dfa0b

Browse files
cquil11cquil11
committed
perf(agentic): sweep GB200 prefill decode balance
1 parent 1e29f55 commit 74dfa0b

4 files changed

Lines changed: 524 additions & 4 deletions

File tree

‎.github/configs/nvidia-master.yaml‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12744,10 +12744,10 @@ dsv4-fp4-b300-sglang-agentic-hicache:
1274412744
- { tp: 8, ep: 8, dp-attn: true, offloading: none, conc-list: [52, 64, 72, 84, 100, 128, 196, 256, 512] }
1274512745

1274612746

12747-
# GB200 agentic disaggregation uses four TEP8 prefill replicas behind Dynamo's
12748-
# KV-aware router and one TP8 decode worker. TEP avoids DEP's per-rank request
12749-
# length imbalance, while four cache-affinitized prefill workers provide enough
12750-
# prefix-cache capacity for the long, high-reuse agentic trace.
12747+
# GB200 agentic disaggregation sweeps a constant 40-GPU budget from 4P/1D to
12748+
# 1P/4D. Every prefill uses TEP8 behind Dynamo's KV-aware router and every
12749+
# decode uses TP8. TEP avoids DEP's per-rank request-length imbalance while the
12750+
# topology axis measures the trace's cache-capacity versus decode-rate tradeoff.
1275112751
# Image matches the recipe's `model.container` (v0.23.0, required by the
1275212752
# current NVFP4 checkpoint; v0.20.0's NIXL connector also breaks TP8<->TP8
1275312753
# transfers, see the recipe header). The recipe also pins a Dynamo wheel new
@@ -12781,6 +12781,53 @@ dsv4-fp4-gb200-dynamo-vllm-agentic:
1278112781
ep: 1
1278212782
dp-attn: false
1278312783

12784+
# Constant 40-GPU topology sweep: shift one TP8 replica at a time from
12785+
# prefill to decode to locate the agentic trace's P/D rate-match point.
12786+
- spec-decoding: none
12787+
conc-list: [32, 64, 128, 192]
12788+
prefill:
12789+
num-worker: 3
12790+
tp: 8
12791+
ep: 8
12792+
dp-attn: false
12793+
additional-settings:
12794+
- "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml"
12795+
decode:
12796+
num-worker: 2
12797+
tp: 8
12798+
ep: 1
12799+
dp-attn: false
12800+
12801+
- spec-decoding: none
12802+
conc-list: [32, 64, 128, 192]
12803+
prefill:
12804+
num-worker: 2
12805+
tp: 8
12806+
ep: 8
12807+
dp-attn: false
12808+
additional-settings:
12809+
- "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p3d-tep8-tp8-agentic.yaml"
12810+
decode:
12811+
num-worker: 3
12812+
tp: 8
12813+
ep: 1
12814+
dp-attn: false
12815+
12816+
- spec-decoding: none
12817+
conc-list: [32, 64, 128, 192]
12818+
prefill:
12819+
num-worker: 1
12820+
tp: 8
12821+
ep: 8
12822+
dp-attn: false
12823+
additional-settings:
12824+
- "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p4d-tep8-tp8-agentic.yaml"
12825+
decode:
12826+
num-worker: 4
12827+
tp: 8
12828+
ep: 1
12829+
dp-attn: false
12830+
1278412831

1278512832
# Split from dsr1-fp4-b200-dynamo-trt: agentic-coding scenario only.
1278612833
dsr1-fp4-b200-dynamo-trt-agentic:

‎benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p4d-tep8-tp8-agentic.yaml‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
name: "svf-vllm-disagg-gb200-1p4d-tep8-tp8-agentic"
2+
3+
# Agentic-coding decode-max shape for GB200: one TEP8 prefill worker and four
4+
# TP8 decode workers. Every worker spans two four-GPU GB200 nodes. A dedicated
5+
# infra node brings the allocation to eleven nodes / 40 inference GPUs.
6+
#
7+
# TEP4 does not fit the DSv4 FP4 checkpoint in 186 GiB GB200 HBM: v0.23.0
8+
# reached 182.5 GiB/rank and needed another 1.97 GiB while constructing MoE
9+
# weights (run 27732316539). TP8 is the minimum viable model sharding here.
10+
# Dynamo KV routing plus explicit vLLM KV-event publication preserves cache
11+
# affinity across the prefill replicas. Unlike DEP, TEP does not put
12+
# unrelated, highly variable agentic requests behind one attention barrier.
13+
# The 2026-05-26 Dynamo wheel is required for vLLM v0.23 KV-event tuples. The
14+
# older 2026-04-26 wheel rejects every BlockStored event's trailing metadata,
15+
# which leaves the router trees empty and destroys prefix affinity.
16+
17+
model:
18+
path: "deepseek-v4-pro"
19+
container: "vllm/vllm-openai:v0.23.0"
20+
precision: "fp4"
21+
22+
dynamo:
23+
install: true
24+
wheel: "1.2.0.dev20260526"
25+
26+
setup_script: vllm-container-deps.sh
27+
28+
slurm:
29+
time_limit: "8:00:00"
30+
31+
health_check:
32+
max_attempts: 1440
33+
interval_seconds: 10
34+
35+
resources:
36+
gpu_type: "gb200"
37+
gpus_per_node: 4
38+
prefill_nodes: 2
39+
decode_nodes: 8
40+
prefill_workers: 1
41+
decode_workers: 4
42+
gpus_per_prefill: 8
43+
gpus_per_decode: 8
44+
45+
infra:
46+
etcd_nats_dedicated_node: true
47+
nats_max_payload_mb: 32
48+
49+
frontend:
50+
type: dynamo
51+
enable_multiple_frontends: false
52+
env:
53+
# srt-slurm defaults to NATS, whose bounded streaming responders rejected
54+
# c64 decode handoffs as "all workers are busy". Dynamo documents TCP as
55+
# the fastest request plane; event publication still uses NATS separately.
56+
DYN_REQUEST_PLANE: "tcp"
57+
args:
58+
router-mode: "kv"
59+
router-reset-states: true
60+
61+
backend:
62+
type: vllm
63+
connector: null
64+
prefill_environment:
65+
DYN_REQUEST_PLANE: "tcp"
66+
TILELANG_CLEANUP_TEMP_FILES: "1"
67+
VLLM_USE_NCCL_SYMM_MEM: "1"
68+
TORCH_SYMMMEM: "NVSHMEM"
69+
NCCL_CUMEM_ENABLE: "1"
70+
NCCL_MNNVL_ENABLE: "1"
71+
NCCL_NVLS_ENABLE: "1"
72+
VLLM_SERVER_DEV_MODE: "1"
73+
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
74+
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048"
75+
UCX_MEMTYPE_CACHE: "n"
76+
UCX_MEMTYPE_REG_WHOLE: "n"
77+
UCX_TLS: "cuda_copy,cuda_ipc,tcp"
78+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
79+
NCCL_P2P_LEVEL: NVL
80+
decode_environment:
81+
DYN_REQUEST_PLANE: "tcp"
82+
TILELANG_CLEANUP_TEMP_FILES: "1"
83+
VLLM_USE_NCCL_SYMM_MEM: "1"
84+
TORCH_SYMMMEM: "NVSHMEM"
85+
NCCL_CUMEM_ENABLE: "1"
86+
NCCL_MNNVL_ENABLE: "1"
87+
NCCL_NVLS_ENABLE: "1"
88+
VLLM_SERVER_DEV_MODE: "1"
89+
UCX_MEMTYPE_CACHE: "n"
90+
UCX_MEMTYPE_REG_WHOLE: "n"
91+
UCX_TLS: "cuda_copy,cuda_ipc,tcp"
92+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
93+
NCCL_P2P_LEVEL: NVL
94+
95+
vllm_config:
96+
prefill:
97+
# vLLM v0.23 broadcasts rank zero's generated engine ID across each TP
98+
# group. Leaving engine_id unset gives every prefill replica a distinct
99+
# ID while keeping both nodes of that replica consistent.
100+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
101+
kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
102+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
103+
kv-cache-dtype: "fp8"
104+
tensor-parallel-size: 8
105+
pipeline-parallel-size: 1
106+
enable-expert-parallel: true
107+
enable-ep-weight-filter: true
108+
attention-config: '{"use_fp4_indexer_cache": true}'
109+
enforce-eager: true
110+
max-num-seqs: 16
111+
max-num-batched-tokens: 16384
112+
trust-remote-code: true
113+
no-enable-flashinfer-autotune: true
114+
no-async-scheduling: true
115+
block-size: 256
116+
gpu-memory-utilization: 0.9
117+
no-disable-hybrid-kv-cache-manager: true
118+
tokenizer-mode: deepseek_v4
119+
decode:
120+
# Generated IDs are distinct per decode replica and synchronized across
121+
# each replica's two-node TP8 group by vLLM v0.23.
122+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
123+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
124+
kv-cache-dtype: "fp8"
125+
tensor-parallel-size: 8
126+
pipeline-parallel-size: 1
127+
attention-config: '{"use_fp4_indexer_cache": true}'
128+
max-num-seqs: 512
129+
max-cudagraph-capture-size: 512
130+
max-num-batched-tokens: 512
131+
trust-remote-code: true
132+
no-enable-flashinfer-autotune: true
133+
block-size: 256
134+
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
135+
gpu-memory-utilization: 0.9
136+
stream-interval: 50
137+
all2all-backend: "flashinfer_nvlink_one_sided"
138+
no-disable-hybrid-kv-cache-manager: true
139+
tokenizer-mode: deepseek_v4
140+
141+
sbatch_directives:
142+
cpus-per-task: "72"
143+
144+
srun_options:
145+
container-remap-root: ""
146+
147+
benchmark:
148+
type: custom
149+
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
150+
env:
151+
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
152+
RESULT_DIR: /logs/agentic
153+
PORT: "8000"
154+
IS_MULTINODE: "true"
155+
AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
156+
HF_HUB_CACHE: "/hf_hub_cache"
157+
WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_061526"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p3d-tep8-tp8-agentic.yaml‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
name: "svf-vllm-disagg-gb200-2p3d-tep8-tp8-agentic"
2+
3+
# Agentic-coding decode-heavy shape for GB200: two cache-affinitized TEP8
4+
# prefill workers and three TP8 decode workers. Every worker spans two four-GPU
5+
# GB200 nodes. A dedicated infra node brings the allocation to eleven nodes /
6+
# 40 inference GPUs.
7+
#
8+
# TEP4 does not fit the DSv4 FP4 checkpoint in 186 GiB GB200 HBM: v0.23.0
9+
# reached 182.5 GiB/rank and needed another 1.97 GiB while constructing MoE
10+
# weights (run 27732316539). TP8 is the minimum viable model sharding here.
11+
# Dynamo KV routing plus explicit vLLM KV-event publication preserves cache
12+
# affinity across the prefill replicas. Unlike DEP, TEP does not put
13+
# unrelated, highly variable agentic requests behind one attention barrier.
14+
# The 2026-05-26 Dynamo wheel is required for vLLM v0.23 KV-event tuples. The
15+
# older 2026-04-26 wheel rejects every BlockStored event's trailing metadata,
16+
# which leaves the router trees empty and destroys prefix affinity.
17+
18+
model:
19+
path: "deepseek-v4-pro"
20+
container: "vllm/vllm-openai:v0.23.0"
21+
precision: "fp4"
22+
23+
dynamo:
24+
install: true
25+
wheel: "1.2.0.dev20260526"
26+
27+
setup_script: vllm-container-deps.sh
28+
29+
slurm:
30+
time_limit: "8:00:00"
31+
32+
health_check:
33+
max_attempts: 1440
34+
interval_seconds: 10
35+
36+
resources:
37+
gpu_type: "gb200"
38+
gpus_per_node: 4
39+
prefill_nodes: 4
40+
decode_nodes: 6
41+
prefill_workers: 2
42+
decode_workers: 3
43+
gpus_per_prefill: 8
44+
gpus_per_decode: 8
45+
46+
infra:
47+
etcd_nats_dedicated_node: true
48+
nats_max_payload_mb: 32
49+
50+
frontend:
51+
type: dynamo
52+
enable_multiple_frontends: false
53+
env:
54+
# srt-slurm defaults to NATS, whose bounded streaming responders rejected
55+
# c64 decode handoffs as "all workers are busy". Dynamo documents TCP as
56+
# the fastest request plane; event publication still uses NATS separately.
57+
DYN_REQUEST_PLANE: "tcp"
58+
args:
59+
router-mode: "kv"
60+
router-reset-states: true
61+
62+
backend:
63+
type: vllm
64+
connector: null
65+
prefill_environment:
66+
DYN_REQUEST_PLANE: "tcp"
67+
TILELANG_CLEANUP_TEMP_FILES: "1"
68+
VLLM_USE_NCCL_SYMM_MEM: "1"
69+
TORCH_SYMMMEM: "NVSHMEM"
70+
NCCL_CUMEM_ENABLE: "1"
71+
NCCL_MNNVL_ENABLE: "1"
72+
NCCL_NVLS_ENABLE: "1"
73+
VLLM_SERVER_DEV_MODE: "1"
74+
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
75+
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048"
76+
UCX_MEMTYPE_CACHE: "n"
77+
UCX_MEMTYPE_REG_WHOLE: "n"
78+
UCX_TLS: "cuda_copy,cuda_ipc,tcp"
79+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
80+
NCCL_P2P_LEVEL: NVL
81+
decode_environment:
82+
DYN_REQUEST_PLANE: "tcp"
83+
TILELANG_CLEANUP_TEMP_FILES: "1"
84+
VLLM_USE_NCCL_SYMM_MEM: "1"
85+
TORCH_SYMMMEM: "NVSHMEM"
86+
NCCL_CUMEM_ENABLE: "1"
87+
NCCL_MNNVL_ENABLE: "1"
88+
NCCL_NVLS_ENABLE: "1"
89+
VLLM_SERVER_DEV_MODE: "1"
90+
UCX_MEMTYPE_CACHE: "n"
91+
UCX_MEMTYPE_REG_WHOLE: "n"
92+
UCX_TLS: "cuda_copy,cuda_ipc,tcp"
93+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
94+
NCCL_P2P_LEVEL: NVL
95+
96+
vllm_config:
97+
prefill:
98+
# vLLM v0.23 broadcasts rank zero's generated engine ID across each TP
99+
# group. Leaving engine_id unset gives every prefill replica a distinct
100+
# ID while keeping both nodes of that replica consistent.
101+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
102+
kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
103+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
104+
kv-cache-dtype: "fp8"
105+
tensor-parallel-size: 8
106+
pipeline-parallel-size: 1
107+
enable-expert-parallel: true
108+
enable-ep-weight-filter: true
109+
attention-config: '{"use_fp4_indexer_cache": true}'
110+
enforce-eager: true
111+
max-num-seqs: 16
112+
max-num-batched-tokens: 16384
113+
trust-remote-code: true
114+
no-enable-flashinfer-autotune: true
115+
no-async-scheduling: true
116+
block-size: 256
117+
gpu-memory-utilization: 0.9
118+
no-disable-hybrid-kv-cache-manager: true
119+
tokenizer-mode: deepseek_v4
120+
decode:
121+
# Generated IDs are distinct per decode replica and synchronized across
122+
# each replica's two-node TP8 group by vLLM v0.23.
123+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
124+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
125+
kv-cache-dtype: "fp8"
126+
tensor-parallel-size: 8
127+
pipeline-parallel-size: 1
128+
attention-config: '{"use_fp4_indexer_cache": true}'
129+
max-num-seqs: 512
130+
max-cudagraph-capture-size: 512
131+
max-num-batched-tokens: 512
132+
trust-remote-code: true
133+
no-enable-flashinfer-autotune: true
134+
block-size: 256
135+
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
136+
gpu-memory-utilization: 0.9
137+
stream-interval: 50
138+
all2all-backend: "flashinfer_nvlink_one_sided"
139+
no-disable-hybrid-kv-cache-manager: true
140+
tokenizer-mode: deepseek_v4
141+
142+
sbatch_directives:
143+
cpus-per-task: "72"
144+
145+
srun_options:
146+
container-remap-root: ""
147+
148+
benchmark:
149+
type: custom
150+
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
151+
env:
152+
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
153+
RESULT_DIR: /logs/agentic
154+
PORT: "8000"
155+
IS_MULTINODE: "true"
156+
AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
157+
HF_HUB_CACHE: "/hf_hub_cache"
158+
WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_061526"

0 commit comments

Comments
 (0)