diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 54e73a4b5c..8bf0091550 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -11641,6 +11641,230 @@ glm5-fp8-gb300-dynamo-sglang: ep: 1 dp-attn: false +glm5-fp8-gb300-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.11-cu130 + model: zai-org/GLM-5.1-FP8 + model-prefix: glm5 + runner: gb300-nv + precision: fp8 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + fixed-seq-len: + # ---------- 8k1k high-throughput (wide-EP decode, EAGLE MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [2800] + prefill: + num-worker: 14 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_0.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1700] + prefill: + num-worker: 12 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_1.yaml" + decode: + num-worker: 1 + tp: 24 + ep: 24 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1300] + prefill: + num-worker: 10 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_2.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [900] + prefill: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_3.yaml" + decode: + num-worker: 1 + tp: 40 + ep: 40 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers, EAGLE MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [150] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml" + decode: + num-worker: 9 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [128, 64, 32] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [24] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + # ---------- 1k1k high-throughput (wide-EP decode, EAGLE MTP) ---------- + - isl: 1024 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [8192] + prefill: + num-worker: 12 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_0.yaml" + decode: + num-worker: 1 + tp: 24 + ep: 24 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [7500] + prefill: + num-worker: 10 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_1.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [7300] + prefill: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_2.yaml" + decode: + num-worker: 1 + tp: 40 + ep: 40 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [6500] + prefill: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_3.yaml" + decode: + num-worker: 1 + tp: 48 + ep: 48 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [5700] + prefill: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_4.yaml" + decode: + num-worker: 1 + tp: 56 + ep: 56 + dp-attn: true + # ---------- 1k1k low-latency (per-node TP=4 decode workers, EAGLE MTP) ---------- + - isl: 1024 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [512, 256, 128, 64] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [32] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + # ============================================================================ # Net-new agentic recipes from chore/agentx-v0.3 (no overlap with main entries). # Recipes that ALREADY existed on main were intentionally left at main's version diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_0.yaml new file mode 100644 index 0000000000..e98343689a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_0.yaml @@ -0,0 +1,152 @@ +name: gb300-fp8-glm5-mtp_1k1k_hightpt_0 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 12 + prefill_workers: 12 + decode_nodes: 6 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 24 + expert-parallel-size: 24 + data-parallel-size: 24 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 8192 + cuda-graph-max-bs: 512 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '8192' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_1.yaml new file mode 100644 index 0000000000..bfdc191159 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_1.yaml @@ -0,0 +1,152 @@ +name: gb300-fp8-glm5-mtp_1k1k_hightpt_1 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 10 + prefill_workers: 10 + decode_nodes: 8 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 32 + expert-parallel-size: 32 + data-parallel-size: 32 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 8192 + cuda-graph-max-bs: 256 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '7500' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_2.yaml new file mode 100644 index 0000000000..37c6dd759b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_2.yaml @@ -0,0 +1,152 @@ +name: gb300-fp8-glm5-mtp_1k1k_hightpt_2 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 8 + prefill_workers: 8 + decode_nodes: 10 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 40 + expert-parallel-size: 40 + data-parallel-size: 40 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 24 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 7200 + cuda-graph-max-bs: 180 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '7300' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_3.yaml new file mode 100644 index 0000000000..e951e4871f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_3.yaml @@ -0,0 +1,152 @@ +name: gb300-fp8-glm5-mtp_1k1k_hightpt_3 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 6 + prefill_workers: 6 + decode_nodes: 12 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 48 + expert-parallel-size: 48 + data-parallel-size: 48 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 6144 + cuda-graph-max-bs: 128 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '6500' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_4.yaml new file mode 100644 index 0000000000..df6334f14a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_4.yaml @@ -0,0 +1,152 @@ +name: gb300-fp8-glm5-mtp_1k1k_hightpt_4 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 4 + decode_nodes: 14 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 56 + expert-parallel-size: 56 + data-parallel-size: 56 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 24 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 5600 + cuda-graph-max-bs: 100 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '5700' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml new file mode 100644 index 0000000000..d5a7abb0f9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml @@ -0,0 +1,144 @@ +name: gb300-fp8-glm5-mtp_1k1k_lowlat_0 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 17 + decode_workers: 17 +frontend: + type: dynamo +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 32 + cuda-graph-max-bs: 32 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: 512x256x128x64 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml new file mode 100644 index 0000000000..8c798b39b6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml @@ -0,0 +1,144 @@ +name: gb300-fp8-glm5-mtp_1k1k_lowlat_1 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 17 + decode_workers: 17 +frontend: + type: dynamo +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 1 + cuda-graph-max-bs: 1 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '32' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_0.yaml new file mode 100644 index 0000000000..650ec30c1f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_0.yaml @@ -0,0 +1,152 @@ +name: gb300-fp8-glm5-mtp_8k1k_hightpt_0 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 14 + prefill_workers: 14 + decode_nodes: 4 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 2800 + cuda-graph-max-bs: 175 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '2800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_1.yaml new file mode 100644 index 0000000000..f8d7532040 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_1.yaml @@ -0,0 +1,152 @@ +name: gb300-fp8-glm5-mtp_8k1k_hightpt_1 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 12 + prefill_workers: 12 + decode_nodes: 6 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 24 + expert-parallel-size: 24 + data-parallel-size: 24 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 1680 + cuda-graph-max-bs: 70 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '1700' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_2.yaml new file mode 100644 index 0000000000..e3b5c74cdd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_2.yaml @@ -0,0 +1,152 @@ +name: gb300-fp8-glm5-mtp_8k1k_hightpt_2 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 10 + prefill_workers: 10 + decode_nodes: 8 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 32 + expert-parallel-size: 32 + data-parallel-size: 32 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 1280 + cuda-graph-max-bs: 40 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '1300' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_3.yaml new file mode 100644 index 0000000000..5d19a961a3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_3.yaml @@ -0,0 +1,152 @@ +name: gb300-fp8-glm5-mtp_8k1k_hightpt_3 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 8 + prefill_workers: 8 + decode_nodes: 10 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 40 + expert-parallel-size: 40 + data-parallel-size: 40 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 24 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 880 + cuda-graph-max-bs: 22 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '900' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml new file mode 100644 index 0000000000..812168165e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml @@ -0,0 +1,144 @@ +name: gb300-fp8-glm5-mtp_8k1k_lowlat_0 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 9 + decode_workers: 9 +frontend: + type: dynamo +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 15 + cuda-graph-max-bs: 15 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '150' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml new file mode 100644 index 0000000000..5977fb91b7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml @@ -0,0 +1,144 @@ +name: gb300-fp8-glm5-mtp_8k1k_lowlat_1 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 17 + decode_workers: 17 +frontend: + type: dynamo +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 8 + cuda-graph-max-bs: 8 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: 128x64x32 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml new file mode 100644 index 0000000000..12589dd82a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml @@ -0,0 +1,144 @@ +name: gb300-fp8-glm5-mtp_8k1k_lowlat_2 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 17 + decode_workers: 17 +frontend: + type: dynamo +dynamo: + version: 1.2.1 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 1 + cuda-graph-max-bs: 1 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '24' diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 818092577f..a6454bd3bf 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4426,3 +4426,11 @@ - "Use 1k/1k TP4/EP1 c1-c32 and TP4/EP4 c64-c256; use 8k/1k TP4/EP1 c1-c32 and TP4/EP4 DP-attention c64-c256." - "Drop the TP8/EP8 single-concurrency points." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1975 + +- config-keys: + - glm5-fp8-gb300-dynamo-sglang-mtp + description: + - "Add GLM-5.1-FP8 GB300 multinode dynamo-sglang benchmark with EAGLE MTP speculative decoding" + - "Image: lmsysorg/sglang:v0.5.11-cu130" + - "14 topologies across 1k/1k and 8k/1k (prefill TP4 + decode wide-EP DEP16/24/32/40/48/56 high-throughput and per-node TP4 low-latency); MTP flags: speculative-algorithm EAGLE, num-steps 2, eagle-topk 1, num-draft-tokens 3" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1907 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 6766155001..34a4eae6a4 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -47,7 +47,7 @@ elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then export MODEL_PATH=/scratch/models/GLM-5-NVFP4 export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then - export MODEL_PATH=/scratch/models/GLM-5-FP8 + export MODEL_PATH=/scratch/models/GLM-5.1-FP8 export SRT_SLURM_MODEL_PREFIX="glm-5-fp8" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH=/data/models/MiniMax-M2.5-NVFP4