Skip to content

Commit 3d55815

Browse files
cquil11cquil11
authored
[NV][AgentX] Add GB200 GLM-5.2 FP4 MTP sweep (#2620)
* Add GB200 GLM-5.2 FP4 AgentX sweep * perf: tune GB200 GLM-5.2 AgentX frontier
1 parent 384d0c4 commit 3d55815

5 files changed

Lines changed: 441 additions & 5 deletions

File tree

‎benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx-agg.yaml‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
name: gb200-fp4-glm5.2-agentx-agg
2+
model:
3+
path: glm-5.2-fp4
4+
container: dynamo-sglang
5+
precision: fp4
6+
identity:
7+
model:
8+
repo: nvidia/GLM-5.2-NVFP4
9+
revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa
10+
container:
11+
image: lmsysorg/sglang:v0.5.17-cu130
12+
frameworks:
13+
dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd
14+
sglang: 0.5.17
15+
resources:
16+
gpu_type: gb200
17+
gpus_per_node: 4
18+
agg_nodes: 2
19+
agg_workers: 1
20+
gpus_per_agg: 8
21+
frontend:
22+
type: dynamo
23+
env:
24+
PIP_BREAK_SYSTEM_PACKAGES: '1'
25+
args:
26+
router-mode: kv
27+
active-decode-blocks-threshold: None
28+
active-prefill-tokens-threshold: None
29+
active-prefill-tokens-threshold-frac: None
30+
router-session-affinity-ttl-secs: 3600
31+
dynamo:
32+
hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd
33+
install: true
34+
backend:
35+
type: sglang
36+
sglang_config:
37+
aggregated:
38+
served-model-name: nvidia/GLM-5.2-NVFP4
39+
trust-remote-code: true
40+
quantization: modelopt_fp4
41+
kv-cache-dtype: fp8_e4m3
42+
tensor-parallel-size: 8
43+
data-parallel-size: 1
44+
expert-parallel-size: 1
45+
enable-dp-attention: false
46+
enable-dp-lm-head: false
47+
max-running-requests: 10
48+
cuda-graph-max-bs: 10
49+
chunked-prefill-size: 8192
50+
max-prefill-tokens: 8192
51+
context-length: 1048576
52+
speculative-algorithm: EAGLE
53+
speculative-num-steps: 3
54+
speculative-eagle-topk: 1
55+
speculative-num-draft-tokens: 4
56+
nsa-prefill-backend: trtllm
57+
nsa-decode-backend: trtllm
58+
bf16-gemm-backend: cutedsl
59+
moe-runner-backend: flashinfer_trtllm
60+
fp4-gemm-backend: flashinfer_trtllm
61+
disable-shared-experts-fusion: true
62+
enable-flashinfer-allreduce-fusion: true
63+
weight-loader-prefetch-checkpoints: true
64+
model-loader-extra-config: '{"enable_multithread_load": true}'
65+
mem-fraction-static: 0.83
66+
enable-hierarchical-cache: true
67+
hicache-write-policy: write_back
68+
hicache-size: 135
69+
hicache-io-backend: direct
70+
hicache-mem-layout: page_first_direct
71+
watchdog-timeout: 1800
72+
enable-metrics: true
73+
enable-cache-report: true
74+
aggregated_environment:
75+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
76+
SGLANG_TIMEOUT_KEEP_ALIVE: '900'
77+
PYTHONUNBUFFERED: '1'
78+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
79+
MC_TE_METRIC: 'true'
80+
NCCL_CUMEM_ENABLE: '1'
81+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
82+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
83+
SGLANG_ENABLE_THINKING: '1'
84+
SGLANG_REASONING_EFFORT: max
85+
SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1'
86+
SGLANG_HICACHE_DEBUG_LOG: '1'
87+
SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384'
88+
SGLANG_MOE_NVFP4_DISPATCH: '1'
89+
SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1'
90+
SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8'
91+
SGLANG_SIMULATE_ACC_LEN: '2.99'
92+
SGLANG_SIMULATE_ACC_METHOD: match-expected
93+
SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
94+
PIP_BREAK_SYSTEM_PACKAGES: '1'
95+
SGLANG_DG_CACHE_DIR: /deepgemm_cache
96+
FLASHINFER_WORKSPACE_BASE: /flashinfer_cache
97+
MC_FORCE_MNNVL: '1'
98+
NCCL_MNNVL_ENABLE: '1'
99+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
100+
UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self
101+
NVSHMEM_REMOTE_TRANSPORT: none
102+
health_check:
103+
max_attempts: 1440
104+
interval_seconds: 10
105+
benchmark:
106+
type: custom
107+
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
108+
env:
109+
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
110+
RESULT_DIR: /logs/agentic
111+
PORT: '8000'
112+
IS_MULTINODE: 'true'
113+
AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
114+
AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
115+
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
116+
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
117+
AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
118+
HF_HUB_CACHE: /hf_hub_cache
119+
infra:
120+
etcd_nats_dedicated_node: false
121+
nats_max_payload_mb: 64
122+
sbatch_directives:
123+
mem: '0'
124+
cpus-per-task: '144'
125+
srun_options:
126+
mem: '0'
127+
container-remap-root: ''

‎benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx.yaml‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
base:
2+
name: gb200-fp4-glm5.2-agentx
3+
model:
4+
path: glm-5.2-fp4
5+
container: dynamo-sglang
6+
precision: fp4
7+
identity:
8+
model:
9+
repo: nvidia/GLM-5.2-NVFP4
10+
revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa
11+
container:
12+
image: lmsysorg/sglang:v0.5.17-cu130
13+
frameworks:
14+
dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd
15+
sglang: 0.5.17
16+
resources:
17+
gpu_type: gb200
18+
gpus_per_node: 4
19+
frontend:
20+
type: dynamo
21+
nginx_session_affinity: true
22+
nginx_session_affinity_header: X-Dynamo-Session-ID
23+
enable_multiple_frontends: true
24+
env:
25+
PIP_BREAK_SYSTEM_PACKAGES: '1'
26+
args:
27+
router-mode: kv
28+
router-kv-events: true
29+
router-temperature: 0
30+
kv-cache-block-size: 64
31+
active-decode-blocks-threshold: None
32+
active-prefill-tokens-threshold: None
33+
active-prefill-tokens-threshold-frac: None
34+
router-session-affinity-ttl-secs: 3600
35+
dynamo:
36+
hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd
37+
install: true
38+
backend:
39+
type: sglang
40+
kv_events_config: true
41+
prefill_environment:
42+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
43+
SGLANG_TIMEOUT_KEEP_ALIVE: '900'
44+
PYTHONUNBUFFERED: '1'
45+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
46+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
47+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
48+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
49+
MC_TE_METRIC: 'true'
50+
MC_FORCE_MNNVL: '1'
51+
NCCL_MNNVL_ENABLE: '1'
52+
NCCL_CUMEM_ENABLE: '1'
53+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
54+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
55+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
56+
SGLANG_ENABLE_THINKING: '1'
57+
SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1'
58+
SGLANG_HICACHE_DEBUG_LOG: '1'
59+
SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384'
60+
SGLANG_REASONING_EFFORT: max
61+
PIP_BREAK_SYSTEM_PACKAGES: '1'
62+
UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self
63+
NVSHMEM_REMOTE_TRANSPORT: none
64+
SGLANG_DG_CACHE_DIR: /deepgemm_cache
65+
FLASHINFER_WORKSPACE_BASE: /flashinfer_cache
66+
SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '2048'
67+
SGLANG_SIMULATE_ACC_LEN: '2.99'
68+
SGLANG_SIMULATE_ACC_METHOD: match-expected
69+
SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
70+
decode_environment:
71+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
72+
SGLANG_TIMEOUT_KEEP_ALIVE: '900'
73+
PYTHONUNBUFFERED: '1'
74+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
75+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
76+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
77+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
78+
MC_TE_METRIC: 'true'
79+
MC_FORCE_MNNVL: '1'
80+
NCCL_MNNVL_ENABLE: '1'
81+
NCCL_CUMEM_ENABLE: '1'
82+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
83+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
84+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
85+
SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512'
86+
SGLANG_MOE_NVFP4_DISPATCH: '1'
87+
SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1'
88+
SGLANG_ENABLE_THINKING: '1'
89+
SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8'
90+
SGLANG_REASONING_EFFORT: max
91+
PIP_BREAK_SYSTEM_PACKAGES: '1'
92+
UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self
93+
NVSHMEM_REMOTE_TRANSPORT: none
94+
SGLANG_DG_CACHE_DIR: /deepgemm_cache
95+
FLASHINFER_WORKSPACE_BASE: /flashinfer_cache
96+
SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '2048'
97+
SGLANG_SIMULATE_ACC_LEN: '2.99'
98+
SGLANG_SIMULATE_ACC_METHOD: match-expected
99+
SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
100+
sglang_config:
101+
prefill:
102+
served-model-name: nvidia/GLM-5.2-NVFP4
103+
trust-remote-code: true
104+
quantization: modelopt_fp4
105+
kv-cache-dtype: fp8_e4m3
106+
disaggregation-mode: prefill
107+
disaggregation-transfer-backend: nixl
108+
tensor-parallel-size: 4
109+
attn-cp-size: 1
110+
enable-prefill-cp: false
111+
enable-dsa-cache-layer-split: false
112+
data-parallel-size: 1
113+
expert-parallel-size: 1
114+
enable-dp-attention: false
115+
enable-dp-lm-head: false
116+
load-balance-method: total_tokens
117+
chunked-prefill-size: 32768
118+
max-prefill-tokens: 8192
119+
max-total-tokens: 1048576
120+
max-running-requests: 64
121+
cuda-graph-max-bs: 64
122+
disable-cuda-graph: true
123+
nsa-prefill-backend: trtllm
124+
nsa-decode-backend: trtllm
125+
bf16-gemm-backend: cutedsl
126+
moe-runner-backend: flashinfer_trtllm
127+
fp4-gemm-backend: flashinfer_trtllm
128+
disable-shared-experts-fusion: true
129+
enable-flashinfer-allreduce-fusion: true
130+
weight-loader-prefetch-checkpoints: true
131+
model-loader-extra-config: '{"enable_multithread_load": true}'
132+
mem-fraction-static: 0.83
133+
context-length: 1048576
134+
enable-hierarchical-cache: true
135+
hicache-write-policy: write_back
136+
hicache-size: 135
137+
hicache-io-backend: direct
138+
hicache-mem-layout: page_first_direct
139+
speculative-algorithm: EAGLE
140+
speculative-num-steps: 3
141+
speculative-eagle-topk: 1
142+
speculative-num-draft-tokens: 4
143+
watchdog-timeout: 1800
144+
enable-metrics: true
145+
enable-cache-report: true
146+
decode:
147+
served-model-name: nvidia/GLM-5.2-NVFP4
148+
trust-remote-code: true
149+
quantization: modelopt_fp4
150+
kv-cache-dtype: fp8_e4m3
151+
disaggregation-mode: decode
152+
disaggregation-transfer-backend: nixl
153+
disable-radix-cache: true
154+
speculative-algorithm: EAGLE
155+
speculative-num-steps: 3
156+
speculative-eagle-topk: 1
157+
speculative-num-draft-tokens: 4
158+
tensor-parallel-size: 4
159+
data-parallel-size: 1
160+
expert-parallel-size: 1
161+
enable-dp-attention: false
162+
enable-dp-lm-head: false
163+
max-running-requests: 64
164+
cuda-graph-max-bs: 64
165+
chunked-prefill-size: 64
166+
context-length: 1048576
167+
nsa-prefill-backend: trtllm
168+
nsa-decode-backend: trtllm
169+
bf16-gemm-backend: cutedsl
170+
moe-runner-backend: flashinfer_trtllm
171+
fp4-gemm-backend: flashinfer_trtllm
172+
skip-tokenizer-init: true
173+
stream-interval: 30
174+
enable-flashinfer-allreduce-fusion: true
175+
weight-loader-prefetch-checkpoints: true
176+
model-loader-extra-config: '{"enable_multithread_load": true}'
177+
mem-fraction-static: 0.9
178+
disaggregation-decode-extra-slots: 0
179+
watchdog-timeout: 1800
180+
enable-metrics: true
181+
enable-cache-report: true
182+
health_check:
183+
max_attempts: 1440
184+
interval_seconds: 10
185+
benchmark:
186+
type: custom
187+
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
188+
env:
189+
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
190+
RESULT_DIR: /logs/agentic
191+
PORT: '8000'
192+
IS_MULTINODE: 'true'
193+
AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
194+
AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
195+
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
196+
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
197+
AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
198+
HF_HUB_CACHE: /hf_hub_cache
199+
infra:
200+
etcd_nats_dedicated_node: false
201+
nats_max_payload_mb: 64
202+
sbatch_directives:
203+
mem: '0'
204+
cpus-per-task: '144'
205+
srun_options:
206+
mem: '0'
207+
container-remap-root: ''
208+
209+
zip_override_mtp_agentx_frontier:
210+
name: [agentx-1p1d-tp4-hicache]
211+
resources:
212+
decode_nodes: 1
213+
decode_workers: 1
214+
gpus_per_decode: 4
215+
gpus_per_prefill: 4
216+
prefill_nodes: 1
217+
prefill_workers: 1

0 commit comments

Comments
 (0)