Skip to content

Commit e37fbc2

Browse files
ichbinblauichbinblau
committed
update dsv4 recipe
Signed-off-by: Theresa Shan <theresa.shan@amd.com>
1 parent c21ad06 commit e37fbc2

4 files changed

Lines changed: 67 additions & 54 deletions

File tree

‎.github/configs/amd-master.yaml‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2877,21 +2877,21 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache:
28772877
additional-settings:
28782878
- "DECODE_NODES=1"
28792879
- "DECODE_MTP_SIZE=0"
2880-
- spec-decoding: "none"
2881-
conc-list: [ 1 ]
2882-
offloading: hicache
2883-
prefill:
2884-
num-worker: 1
2885-
tp: 8
2886-
ep: 1
2887-
dp-attn: false
2888-
additional-settings:
2889-
- "PREFILL_NODES=1"
2890-
decode:
2891-
num-worker: 1
2892-
tp: 8
2893-
ep: 1
2894-
dp-attn: false
2895-
additional-settings:
2896-
- "DECODE_NODES=1"
2897-
- "DECODE_MTP_SIZE=0"
2880+
# - spec-decoding: "none"
2881+
# conc-list: [ 1 ]
2882+
# offloading: hicache
2883+
# prefill:
2884+
# num-worker: 1
2885+
# tp: 8
2886+
# ep: 1
2887+
# dp-attn: false
2888+
# additional-settings:
2889+
# - "PREFILL_NODES=1"
2890+
# decode:
2891+
# num-worker: 1
2892+
# tp: 8
2893+
# ep: 1
2894+
# dp-attn: false
2895+
# additional-settings:
2896+
# - "DECODE_NODES=1"
2897+
# - "DECODE_MTP_SIZE=0"

‎benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh‎

Lines changed: 35 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -71,39 +71,42 @@ export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}"
7171
# OFFLOADING=hicache | none (passed from YAML; default none for disagg).
7272
# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store.
7373
export OFFLOADING="${OFFLOADING:-none}"
74-
export HICACHE_TIER="${HICACHE_TIER:-L3}"
75-
export HICACHE_TOTAL_CPU_DRAM_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-64}"
76-
export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}"
77-
# DSV4 uses page-size 256 (set in models.yaml); HiCache must match.
78-
export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}"
79-
# Per-rank L2 host pool in GB.
80-
export HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-100}"
81-
82-
# ── HiCache layout/backend by tier ──
83-
# L3 (Mooncake): page_first + direct + write_through + storage=mooncake
84-
# L2 (CPU DRAM): layer_first + direct + write_through_selective + storage=none
85-
# NOTE: write_through_selective evicts only under GPU memory pressure, avoiding
86-
# the mori RDMA race that causes GPU memory access faults with write_through.
87-
if [[ "${HICACHE_TIER^^}" == "L3" ]]; then
88-
export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}"
89-
export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}"
90-
export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}"
91-
export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}"
92-
else
93-
export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-layer_first}"
94-
export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}"
95-
export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}"
96-
export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}"
74+
# HiCache/Mooncake tunables only matter when KV offloading is enabled.
75+
if [[ "$OFFLOADING" == "hicache" ]]; then
76+
export HICACHE_TIER="${HICACHE_TIER:-L2}"
77+
export HICACHE_TOTAL_CPU_DRAM_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-64}"
78+
export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}"
79+
# DSV4 uses page-size 256 (set in models.yaml); HiCache must match.
80+
export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}"
81+
# Per-rank L2 host pool in GB.
82+
export HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-100}"
83+
84+
# ── HiCache layout/backend by tier ──
85+
# L3 (Mooncake): page_first + direct + write_through + storage=mooncake
86+
# L2 (CPU DRAM): layer_first + direct + write_through_selective + storage=none
87+
# NOTE: write_through_selective evicts only under GPU memory pressure, avoiding
88+
# the mori RDMA race that causes GPU memory access faults with write_through.
89+
if [[ "${HICACHE_TIER^^}" == "L3" ]]; then
90+
export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}"
91+
export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}"
92+
export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}"
93+
export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}"
94+
else
95+
export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-layer_first}"
96+
export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}"
97+
export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}"
98+
export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}"
99+
fi
100+
export HICACHE_DECODE="${HICACHE_DECODE:-0}"
101+
# Shared nodes: use non-default Mooncake ports to avoid collisions.
102+
export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}"
103+
export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}"
104+
export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}"
105+
export MC_PROTOCOL="${MC_PROTOCOL:-tcp}"
106+
export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-30gb}"
107+
export MC_DEVICE="${MC_DEVICE:-rdma0}"
108+
export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}"
97109
fi
98-
export HICACHE_DECODE="${HICACHE_DECODE:-0}"
99-
# Shared nodes: use non-default Mooncake ports to avoid collisions.
100-
export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}"
101-
export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}"
102-
export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}"
103-
export MC_PROTOCOL="${MC_PROTOCOL:-tcp}"
104-
export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-30gb}"
105-
export MC_DEVICE="${MC_DEVICE:-rdma0}"
106-
export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}"
107110

108111
# ── MoRIIO RDMA Send Queue tuning ──
109112
export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}"

‎benchmarks/multi_node/amd_utils/models.yaml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -357,9 +357,8 @@ DeepSeek-R1-0528-MXFP4-v2:
357357
# thinking/reasoning-effort, dispatch dtypes, per-role PER_RANK dispatch tokens) is set
358358
# in env.sh's DeepSeek-V4-Pro block. The bench client uses --dsv4 framing (bench.sh).
359359
# prefill.disable_cuda_graph routes prefill to --disable-cuda-graph; decode keeps
360-
# --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md.
360+
# --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. hf_dir: "models--deepseek-ai--DeepSeek-V4-Pro"
361361
DeepSeek-V4-Pro:
362-
hf_dir: "models--deepseek-ai--DeepSeek-V4-Pro"
363362
base_flags: "--decode-log-interval 100 --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori"
364363
dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
365364
prefill:
@@ -369,12 +368,12 @@ DeepSeek-V4-Pro:
369368
dp:
370369
max_running_requests: 1024
371370
chunked_prefill_size: 131072
372-
context_length: 9217
371+
context_length: 1000000
373372
max_total_tokens: 262144
374373
no_dp:
375374
max_running_requests: 128
376375
chunked_prefill_size: 131072
377-
context_length: 9217
376+
context_length: 1000000
378377
max_total_tokens: 262144
379378
decode:
380379
mem_fraction_static: 0.85

‎benchmarks/multi_node/amd_utils/server_sglang.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,9 +231,14 @@ if [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "True" ]] || [[ "$PREFILL_DISABLE_CUDA_GR
231231
else
232232
PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --cuda-graph-bs ${prefill_cuda_graph_bs[*]} "
233233
fi
234+
234235
if [[ "$PREFILL_DISABLE_RADIX_CACHE" == "True" ]] || [[ "$PREFILL_DISABLE_RADIX_CACHE" == "true" ]]; then
235236
PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --disable-radix-cache"
236237
fi
238+
# Agentic runs: keep radix/prefix cache enabled by replacing --disable-radix-cache with empty.
239+
if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then
240+
PREFILL_MODE_FLAGS="${PREFILL_MODE_FLAGS//--disable-radix-cache/}"
241+
fi
237242
if [[ -n "$prefill_context_length" ]]; then
238243
PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --context-length ${prefill_context_length}"
239244
fi
@@ -367,6 +372,12 @@ build_server_config() {
367372
PREFILL_SERVER_CONFIG=$(build_server_config "prefill" "$MODEL_NAME" "$PREFILL_TP_SIZE" "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" "$DECODE_MTP_SIZE")
368373
DECODE_SERVER_CONFIG=$(build_server_config "decode" "$MODEL_NAME" "$DECODE_TP_SIZE" "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" "$DECODE_MTP_SIZE")
369374

375+
# Expose Prometheus /metrics on the servers when requested (ENABLE_METRICS=1).
376+
if [[ "${ENABLE_METRICS:-0}" == "1" ]]; then
377+
[[ "$PREFILL_SERVER_CONFIG" != *"--enable-metrics"* ]] && PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG --enable-metrics"
378+
[[ "$DECODE_SERVER_CONFIG" != *"--enable-metrics"* ]] && DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --enable-metrics"
379+
fi
380+
370381
if [[ -n "$MODEL_NAME" ]]; then
371382
echo "Using model-specific configuration for: $MODEL_NAME"
372383
fi

0 commit comments

Comments
 (0)