From ae81a2ce56a8c6db52ccc7906c46aec5a48540f3 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 15 Jul 2026 11:20:53 -0500
Subject: [PATCH 01/17] feat(agentic): split DeepSeek-V4 B300 vLLM recipe
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Carry the B300-only launcher, config search space, and changelog scope from PR #2202.

中文：拆分 DeepSeek-V4 B300 vLLM AgentX 配方，仅保留 PR #2202 中的 B300 启动器、配置搜索空间和变更日志范围。
---
 .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 218 +++++++++++-------
 configs/nvidia-master.yaml                    |  17 +-
 perf-changelog.yaml                           |  11 +-
 3 files changed, 145 insertions(+), 101 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index 3a5be7b57..72f2736fe 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -1,60 +1,46 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 set -x
 
 # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM.
-# Mirrors the fixed-seq-len parallelism options (pure TP and DEP) so the
-# agentic sweep can probe both interactivity and throughput regimes:
-#   pure TP (DP_ATTENTION=false, EP_SIZE=1):  attention TP-sharded across
-#       all $TP GPUs in a single engine. Lower TPOT, lower batch.
-#   TP+EP   (DP_ATTENTION=false, EP_SIZE>1):  attention TP-sharded, MoE
-#       experts EP-sharded within the TP group.
-#   DEP     (DP_ATTENTION=true, EP_SIZE>1):   per-DP-rank attention with
-#       experts EP-sharded across DP ranks (per the vLLM blog recipe).
-#       Highest aggregate throughput at large CONC.
+# v4pro-b300.yaml TP4 and DEP4 recipe. SimpleCPUOffload / MooncakeStore
 #
-# Image is vllm/vllm-openai:v0.20.0-cu130. block_size=256, kv-cache-dtype=fp8,
-# FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph capture with
-# custom_ops=all (per the vLLM blog recipe at https://vllm.ai/blog/deepseek-v4).
+# Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache,
+# sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer cache, mega-MoE,
+# and FULL_DECODE_ONLY CUDA graphs with every batch size captured explicitly.
 #
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake.
+# KV_OFFLOADING=none is used by TP4. DEP4 uses KV_OFFLOADING=dram with
+# KV_OFFLOAD_BACKEND=vllm-simple or mooncake.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
 check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
 
-DCP_SIZE="${DCP_SIZE:-1}"
-PCP_SIZE="${PCP_SIZE:-1}"
-VLLM_CP_ARGS=()
-if [ "$DCP_SIZE" -gt 1 ]; then
-    VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE")
-fi
-if [ "$PCP_SIZE" -gt 1 ]; then
-    VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE")
-fi
-
-GPU_COUNT="${GPU_COUNT:-$((TP * PCP_SIZE))}"
+GPU_COUNT=$TP
 if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then
     echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2
     exit 1
 fi
 export GPU_COUNT
 
-if declare -p SLURM_JOB_ID >/dev/null 2>&1 && [ -n "$SLURM_JOB_ID" ]; then
-    SLURM_NODE=unknown
-    if declare -p SLURMD_NODENAME >/dev/null 2>&1 && [ -n "$SLURMD_NODENAME" ]; then
-        SLURM_NODE="$SLURMD_NODENAME"
-    fi
-    echo "JOB $SLURM_JOB_ID running on $SLURM_NODE"
+# Under DP-attention the DP world size equals TP, and the DEP recipe sizes
+# per-rank batch as MAX_NUM_SEQS = 2*CONC/TP, which must be an integer.
+if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then
+    echo "Error: DEP requires 2*CONC divisible by TP, got CONC='$CONC' and TP='$TP'" >&2
+    exit 1
+fi
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
 fi
 
 # `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
+# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE.
 # Either way, MODEL_PATH is what the server is launched with.
-if declare -p MODEL_PATH >/dev/null 2>&1 && [ -n "$MODEL_PATH" ]; then
+if [[ -n "$MODEL_PATH" ]]; then
     if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
         hf download "$MODEL" --local-dir "$MODEL_PATH"
     fi
@@ -68,17 +54,9 @@ nvidia-smi
 resolve_trace_source
 install_agentic_deps
 
-# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings,
-# which fails DSV4 indexer compilation with mlir_global_dtors(..., data).
-# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved.
-agentic_pip_install --quiet --force-reinstall --no-deps \
-    'nvidia-cutlass-dsl-libs-cu13==4.5.2'
-
 # vllm-project/router expands the one HTTP backend into one logical worker per
-# DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's
-# X-Correlation-ID is stable for every turn of a conversation; alias it to the
-# router's preferred X-Session-ID header. This also keeps affinity correct when
-# testing older wheels that prioritize per-request X-Request-ID.
+# DP rank. Bind every turn of a conversation to the same rank by mapping
+# AIPerf's stable correlation ID to the router's X-Session-ID header.
 USE_VLLM_ROUTER=false
 VLLM_BACKEND_PORT="$PORT"
 if [ "$DP_ATTENTION" = "true" ]; then
@@ -91,13 +69,13 @@ if [ "$DP_ATTENTION" = "true" ]; then
     agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION"
 fi
 
-# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s.
+# Match the environment used by v4pro-b300.yaml.
+export VLLM_USE_V2_MODEL_RUNNER=1
 export VLLM_ENGINE_READY_TIMEOUT_S=3600
-
-# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while
-# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's
-# store mask. 32k matches the trace-replay tuning validated for this workload.
 export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
+export VLLM_DSV4_MEGA_FP8_COMBINE=1
+export NCCL_NVLS_ENABLE=1
+export VLLM_USE_RUST_FRONTEND=1
 
 # ---- Server config ----------------------------------------------------------
 SERVER_LOG="$RESULT_DIR/server.log"
@@ -109,13 +87,40 @@ SERVER_PID=""
 ROUTER_PID=""
 MOONCAKE_MASTER_PID=""
 
+# The generated TOTAL_CPU_DRAM_GB budget is proportional to allocated GPUs.
+# On cluster:b300-nv, dram-utilization=0.80 and DEP4 resolve to roughly the
+# source recipe's 280 GiB per DP rank. TP4 remains GPU-resident.
 OFFLOAD_ARGS=()
-if require_agentic_kv_offload_backend mooncake; then
-        # Mooncake embedded mode contributes one global segment per GPU rank to
-        # a shared distributed store. Pre-divide the aggregate host budget
-        # across those rank-contributed segments.
+case "$KV_OFFLOAD_BACKEND" in
+    "")
+        require_agentic_kv_offload_none
+        ;;
+    vllm-simple)
+        require_agentic_kv_offload_backend vllm-simple
+        CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT ))
+        # Identical prefixes must hash to identical block keys across DP ranks.
+        export PYTHONHASHSEED=42
+        OFFLOAD_CONFIG=$(cat <<EOF
+{
+  "kv_connector": "SimpleCPUOffloadConnector",
+  "kv_role": "kv_both",
+  "kv_connector_extra_config": {
+    "cpu_bytes_to_use": ${CPU_BYTES_PER_RANK},
+    "enable_cross_layers_blocks": "true"
+  }
+}
+EOF
+)
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "$OFFLOAD_CONFIG"
+        )
+        ;;
+    mooncake)
+        require_agentic_kv_offload_backend mooncake
+        # Embedded mode contributes one global segment per DP rank to the
+        # shared store, so divide the aggregate host budget across ranks.
         PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT))
-
         MOONCAKE_VERSION=0.3.11.post1
         agentic_pip_install --quiet --no-cache-dir --no-deps \
             --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
@@ -139,9 +144,7 @@ EOF
         export MC_ENABLE_DEST_DEVICE_AFFINITY=1
         # Identical prefixes must hash to identical store keys across DP ranks.
         export PYTHONHASHSEED=0
-        # Large agentic KV writes can exceed Mooncake Store's fixed 60-second
-        # transfer deadline at the default 64 KiB RDMA slice size. Reduce
-        # per-transfer bookkeeping and give the shared RNIC more workers.
+        export WITH_NVIDIA_PEERMEM=0
         export MC_SLICE_SIZE=1048576
         export MC_WORKERS_PER_CTX=4
 
@@ -165,54 +168,93 @@ EOF
         fi
 
         unset VLLM_USE_SIMPLE_KV_OFFLOAD
-        OFFLOAD_ARGS=(
-            --kv-transfer-config
-            '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
-        )
-fi
+        OFFLOAD_CONFIG='{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
+        OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG")
+        ;;
+    *)
+        echo "Error: unsupported B300 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2
+        exit 1
+        ;;
+esac
 
 PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
 if [ "$DP_ATTENTION" = "true" ]; then
     PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
 fi
 
-EP_ARGS=()
+TP_ARGS=()
+if [ "$DP_ATTENTION" = "true" ]; then
+    export PYTORCH_ALLOC_CONF=expandable_segments:True
+else
+    export VLLM_ALLREDUCE_USE_FLASHINFER=1
+    export VLLM_FLASHINFER_ALLREDUCE_BACKEND=auto
+    TP_ARGS+=(--disable-custom-all-reduce)
+fi
+
+MODE_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(--enable-expert-parallel)
+    MODE_ARGS+=(
+        --enable-expert-parallel
+        --enable-ep-weight-filter
+        --moe-backend deep_gemm_amxf4_mega_moe
+    )
+fi
+if [ "$DP_ATTENTION" = "true" ]; then
+    MODE_ARGS+=(
+        --prefill-schedule-interval 8
+        --max-num-batched-tokens 8192
+    )
 fi
 
-# AgentX concurrency counts live session trees, not individual requests.
-# Subagent fan-out can push instantaneous request concurrency above CONC, so
-# leave 2x headroom rather than clipping those bursts at the scheduler.
-MAX_NUM_SEQS=$((2 * CONC))
-if [ "$MAX_NUM_SEQS" -eq 128 ]; then
-    MAX_NUM_SEQS=136
+if [ "$DP_ATTENTION" = "true" ]; then
+    # The DEP source recipe enforces 2*CONC = DP_WORLD_SIZE*MAX_NUM_SEQS.
+    MAX_NUM_SEQS=$((2 * CONC / TP))
+else
+    # Preserve the previous TP4 scheduler headroom for agentic fan-out.
+    MAX_NUM_SEQS=$((2 * CONC))
 fi
+CUDA_GRAPH_CAPTURE_SIZES=""
+for ((capture_size = 1; capture_size <= MAX_NUM_SEQS; capture_size++)); do
+    if [ -n "$CUDA_GRAPH_CAPTURE_SIZES" ]; then
+        CUDA_GRAPH_CAPTURE_SIZES+=","
+    fi
+    CUDA_GRAPH_CAPTURE_SIZES+="$capture_size"
+done
+COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}],\"mode\":0}"
 
 echo "Starting vllm server..."
 export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
 export VLLM_FLOAT32_MATMUL_PRECISION=high
 
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
---host 0.0.0.0 \
---port "$VLLM_BACKEND_PORT" \
---trust-remote-code \
---kv-cache-dtype fp8 \
---block-size 256 \
-"${PARALLEL_ARGS[@]}" \
-"${VLLM_CP_ARGS[@]}" \
-"${EP_ARGS[@]}" \
---compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \
---attention_config.use_fp4_indexer_cache=True \
---tokenizer-mode deepseek_v4 \
---tool-call-parser deepseek_v4 \
---enable-auto-tool-choice \
---reasoning-parser deepseek_v4 \
---enable-prefix-caching \
---no-disable-hybrid-kv-cache-manager \
---max-num-seqs "$MAX_NUM_SEQS" \
-"${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 &
+{ set +x; } 2>/dev/null
+VLLM_CMD=(
+    vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$VLLM_BACKEND_PORT"
+    --gpu-memory-utilization 0.96
+    --trust-remote-code
+    --no-enable-flashinfer-autotune
+    --no-disable-hybrid-kv-cache-manager
+    --max-num-seqs "$MAX_NUM_SEQS"
+    --kv-cache-dtype fp8
+    --block-size 256
+    --max-model-len 1048576
+    --attention-config '{"use_fp4_indexer_cache":true,"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
+    --disable-uvicorn-access-log
+    --tokenizer-mode deepseek_v4
+    --tool-call-parser deepseek_v4
+    --enable-auto-tool-choice
+    --reasoning-parser deepseek_v4
+    --compilation-config "$COMPILATION_CONFIG"
+    "${PARALLEL_ARGS[@]}"
+    "${TP_ARGS[@]}"
+    "${MODE_ARGS[@]}"
+    "${OFFLOAD_ARGS[@]}"
+)
+printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
+"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
 SERVER_PID=$!
 echo "Server PID: $SERVER_PID"
 
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 33e9118b7..7e8899608 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3560,7 +3560,7 @@ dsv4-fp4-b300-vllm:
       - { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 }
 
 dsv4-fp4-b300-vllm-agentic:
-  image: vllm/vllm-openai:v0.23.0
+  image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:b300-nv
@@ -3571,16 +3571,11 @@ dsv4-fp4-b300-vllm-agentic:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      # Compare native GPU-cache and MooncakeStore CPU-offload cliffs.
-      - { tp: 4, kv-offloading: none,  conc-list: [1, 2, 4, 6, 8, 16] }
-      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 18, 20, 24] }
-      # TP8 remains cache-resident through conc 52.
-      - { tp: 8, kv-offloading: none,  conc-list: [1, 2, 4, 6, 8, 16, 32, 40, 48, 52] }
-      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [52] }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none,  conc-list: [8, 16], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [32], router: { name: vllm-router, version: "0.1.14" } }
-      # TP8 DEP retains representative low, peak, and transition points.
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,  conc-list: [52, 72, 100, 128, 144], router: { name: vllm-router, version: "0.1.14" } }
+      # Preserve the previous GPU-resident TP4 search space on the new image.
+      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4] }
+      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index fe3610186..8c321b3bb 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4807,7 +4807,7 @@
     - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)."
     - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2222
-  
+
 - config-keys:
     - minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1
     - minimaxm3-fp4-b300-dynamo-vllm-8k1k-legacy-max-tput
@@ -4865,5 +4865,12 @@
   description:
     - "Bump image to lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713"
     - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts"
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201  
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201
 
+- config-keys:
+    - dsv4-fp4-b300-vllm-agentic
+  description:
+    - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
+    - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
+    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2202

From 50ec3b78b65a678b491072e6cf4b37005462b5fe Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 15 Jul 2026 11:22:39 -0500
Subject: [PATCH 02/17] docs(changelog): link B300 replacement PR
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Point the split B300 changelog entry to PR #2225.

中文：将拆分后的 B300 变更日志条目链接到 PR #2225。
---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 8c321b3bb..0709faaca 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4873,4 +4873,4 @@
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
     - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2202
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225

From 3d578138db8121015fe2f6f3ec46d404a4b208e2 Mon Sep 17 00:00:00 2001
From: Yifan Qiao <yifanqiao@inferact.ai>
Date: Wed, 15 Jul 2026 19:38:02 +0000
Subject: [PATCH 03/17] chore: update conc

---
 configs/nvidia-master.yaml | 8 +++++---
 perf-changelog.yaml        | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 7e8899608..26e60ac1a 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3346,7 +3346,7 @@ dsr1-fp8-b200-trt:
       osl: 1024
       search-space:
       - { tp: 8, ep: 1, conc-start: 64, conc-end: 128 }
-      - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 } 
+      - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 }
       - { tp: 8, ep: 1, conc-start: 4, conc-end: 8 }
     - isl: 8192
       osl: 1024
@@ -3572,10 +3572,12 @@ dsv4-fp4-b300-vllm-agentic:
     - dram-utilization: 0.80
       search-space:
       # Preserve the previous GPU-resident TP4 search space on the new image.
-      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4] }
+      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
       - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 64, 96, 128, 160, 192, 224, 228], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [40, 72, 104, 136, 168, 200], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 0709faaca..1960515f4 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4872,5 +4872,5 @@
   description:
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
-    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
+    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72]; DEP8 at conc [32,64,96,128,160,192,224,228]; with both SimpleCPU and Mooncake 0.3.11.post1."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225

From 4acd3c4f17e067ef7a0f41634b04c7bb74fab181 Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Wed, 15 Jul 2026 17:54:42 -0700
Subject: [PATCH 04/17] update dep8 args

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
---
 .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 37 +++++++++++++++----
 configs/nvidia-master.yaml                    |  4 +-
 perf-changelog.yaml                           |  2 +-
 3 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index 72f2736fe..0f31a1881 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -3,7 +3,7 @@ set -eo pipefail
 set -x
 
 # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM.
-# v4pro-b300.yaml TP4 and DEP4 recipe. SimpleCPUOffload / MooncakeStore
+# v4pro-b300.yaml TP4, DEP4, and DEP8 recipe. SimpleCPUOffload / MooncakeStore
 #
 # Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache,
 # sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer cache, mega-MoE,
@@ -12,8 +12,8 @@ set -x
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# KV_OFFLOADING=none is used by TP4. DEP4 uses KV_OFFLOADING=dram with
-# KV_OFFLOAD_BACKEND=vllm-simple or mooncake.
+# TP4, TP8, and DEP8 (TP8 + DP-attention) are GPU-resident (KV_OFFLOADING=none).
+# DEP4 uses KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=vllm-simple or mooncake.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -33,6 +33,14 @@ if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then
     exit 1
 fi
 
+# DEP8 (TP8 + DP-attention) is a GPU-resident, high-concurrency arm that is
+# tuned separately from the smaller DEP4 arm (larger prefill token budget,
+# long-prefill chunking, and a lower GPU-memory-utilization headroom).
+IS_DEP8=false
+if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then
+    IS_DEP8=true
+fi
+
 if [[ -n "$SLURM_JOB_ID" ]]; then
     echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
 fi
@@ -200,10 +208,17 @@ if [ "$EP_SIZE" -gt 1 ]; then
     )
 fi
 if [ "$DP_ATTENTION" = "true" ]; then
-    MODE_ARGS+=(
-        --prefill-schedule-interval 8
-        --max-num-batched-tokens 8192
-    )
+    MODE_ARGS+=(--prefill-schedule-interval 8)
+    if [ "$IS_DEP8" = "true" ]; then
+        # GPU-resident DEP8 gets a larger prefill token budget and chunks long
+        # prefills so decode latency stays bounded at high concurrency.
+        MODE_ARGS+=(
+            --max-num-batched-tokens 16384
+            --long-prefill-token-threshold 4096
+        )
+    else
+        MODE_ARGS+=(--max-num-batched-tokens 8192)
+    fi
 fi
 
 if [ "$DP_ATTENTION" = "true" ]; then
@@ -227,12 +242,18 @@ export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
 export VLLM_FLOAT32_MATMUL_PRECISION=high
 
+# DEP8 leaves slightly more headroom for its larger prefill token budget.
+GPU_MEM_UTIL=0.96
+if [ "$IS_DEP8" = "true" ]; then
+    GPU_MEM_UTIL=0.95
+fi
+
 { set +x; } 2>/dev/null
 VLLM_CMD=(
     vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
     --host 0.0.0.0
     --port "$VLLM_BACKEND_PORT"
-    --gpu-memory-utilization 0.96
+    --gpu-memory-utilization "$GPU_MEM_UTIL"
     --trust-remote-code
     --no-enable-flashinfer-autotune
     --no-disable-hybrid-kv-cache-manager
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 26e60ac1a..550bae9c8 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3576,8 +3576,8 @@ dsv4-fp4-b300-vllm-agentic:
       - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 64, 96, 128, 160, 192, 224, 228], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [40, 72, 104, 136, 168, 200], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP8 is GPU-resident (no KV offload) with a DEP8-tuned server config.
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [32, 64, 96, 128, 160, 192, 196, 224, 228], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 1960515f4..dfea5f65e 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4872,5 +4872,5 @@
   description:
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
-    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72]; DEP8 at conc [32,64,96,128,160,192,224,228]; with both SimpleCPU and Mooncake 0.3.11.post1."
+    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.95); DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225

From 439248923f421bedb1ccbfa54f757b829fb8fca7 Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Thu, 16 Jul 2026 00:54:48 -0700
Subject: [PATCH 05/17] add more tp configs

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
---
 benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh | 9 ++++++++-
 configs/nvidia-master.yaml                           | 6 ++++--
 perf-changelog.yaml                                  | 2 +-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index 0f31a1881..bfae06d7c 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -108,13 +108,20 @@ case "$KV_OFFLOAD_BACKEND" in
         CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT ))
         # Identical prefixes must hash to identical block keys across DP ranks.
         export PYTHONHASHSEED=42
+        # The plain-TP (non-DP-attention) offload ladder uses lazy offload;
+        # DEP keeps eager offload for cross-rank block-hash stability.
+        SIMPLE_LAZY_OFFLOAD=false
+        if [ "$DP_ATTENTION" != "true" ]; then
+            SIMPLE_LAZY_OFFLOAD=true
+        fi
         OFFLOAD_CONFIG=$(cat <<EOF
 {
   "kv_connector": "SimpleCPUOffloadConnector",
   "kv_role": "kv_both",
   "kv_connector_extra_config": {
     "cpu_bytes_to_use": ${CPU_BYTES_PER_RANK},
-    "enable_cross_layers_blocks": "true"
+    "enable_cross_layers_blocks": "true",
+    "lazy_offload": ${SIMPLE_LAZY_OFFLOAD}
   }
 }
 EOF
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 550bae9c8..1091e4d5d 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3572,8 +3572,10 @@ dsv4-fp4-b300-vllm-agentic:
     - dram-utilization: 0.80
       search-space:
       # Preserve the previous GPU-resident TP4 search space on the new image.
-      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
-      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
+      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
+      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
+      # TP4 SimpleCPU lazy-offload ladder (extends the GPU-resident TP4 arm).
+      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [28, 32, 36, 40] }
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } }
       # DEP8 is GPU-resident (no KV offload) with a DEP8-tuned server config.
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index dfea5f65e..9414e69d2 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4872,5 +4872,5 @@
   description:
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
-    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.95); DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
+    - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.95); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225

From 739a6678e2d2ae18fcc20e65f7598bc4d4ddd662 Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Thu, 16 Jul 2026 00:58:39 -0700
Subject: [PATCH 06/17] dep8 reduce gpu mem util to 0.92

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
---
 benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh | 4 ++--
 perf-changelog.yaml                                  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index bfae06d7c..d4c0ca054 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -249,10 +249,10 @@ export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
 export VLLM_FLOAT32_MATMUL_PRECISION=high
 
-# DEP8 leaves slightly more headroom for its larger prefill token budget.
+# DEP8 leaves more headroom for its larger prefill token budget.
 GPU_MEM_UTIL=0.96
 if [ "$IS_DEP8" = "true" ]; then
-    GPU_MEM_UTIL=0.95
+    GPU_MEM_UTIL=0.92
 fi
 
 { set +x; } 2>/dev/null
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 9414e69d2..85be26a99 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4872,5 +4872,5 @@
   description:
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
-    - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.95); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
+    - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.92); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225

From 46a35669bbb47424566b826e1392bfd2285f6071 Mon Sep 17 00:00:00 2001
From: Yifan Qiao <yifanqiao@inferact.ai>
Date: Thu, 16 Jul 2026 08:43:49 +0000
Subject: [PATCH 07/17] chore: update conc list

---
 configs/nvidia-master.yaml | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 1091e4d5d..aa42c936b 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3571,15 +3571,14 @@ dsv4-fp4-b300-vllm-agentic:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      # Preserve the previous GPU-resident TP4 search space on the new image.
-      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
+      # TP4 GPU-resident
       - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
-      # TP4 SimpleCPU lazy-offload ladder (extends the GPU-resident TP4 arm).
+      # TP4 SimpleCPU
       - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [28, 32, 36, 40] }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } }
-      # DEP8 is GPU-resident (no KV offload) with a DEP8-tuned server config.
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [32, 64, 96, 128, 160, 192, 196, 224, 228], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP4 SimpleCPU
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP8 SimpleCPU
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066

From 8f023a555fc39e784745def9b1223dfaabb8e1ac Mon Sep 17 00:00:00 2001
From: Yifan Qiao <yifanqiao@inferact.ai>
Date: Thu, 16 Jul 2026 09:19:56 +0000
Subject: [PATCH 08/17] docs(changelog): point B300 entry to PR #2241

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LjHXfE1FRN9c5XQqgeTdez
---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 85be26a99..11cfb438e 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4873,4 +4873,4 @@
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
     - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.92); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2241

From 10f30edc560463f5530a3f31300e04b071245589 Mon Sep 17 00:00:00 2001
From: Yifan Qiao <yifanqiao@inferact.ai>
Date: Thu, 16 Jul 2026 09:23:20 +0000
Subject: [PATCH 09/17] fix(changelog): keep perf-changelog additions-only

Restore upstream lines whitespace-for-whitespace and append the B300
entry so the changelog diff contains no deletions.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LjHXfE1FRN9c5XQqgeTdez
---
 perf-changelog.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 11cfb438e..a9d497277 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4807,7 +4807,7 @@
     - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)."
     - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2222
-
+  
 - config-keys:
     - minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1
     - minimaxm3-fp4-b300-dynamo-vllm-8k1k-legacy-max-tput
@@ -4865,7 +4865,7 @@
   description:
     - "Bump image to lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713"
     - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts"
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201  
 
 - config-keys:
     - dsv4-fp4-b300-vllm-agentic

From 31f4ed1c495a7cd145f30e4374f52957d28d90c5 Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Thu, 16 Jul 2026 22:03:10 -0700
Subject: [PATCH 10/17] add mtp

---
 .../agentic/dsv4_fp4_b300_vllm_mtp.sh         | 326 ++++++++++++++++++
 configs/nvidia-master.yaml                    |   8 +-
 2 files changed, 332 insertions(+), 2 deletions(-)
 create mode 100755 benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
new file mode 100755
index 000000000..8a01e16c7
--- /dev/null
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
@@ -0,0 +1,326 @@
+#!/usr/bin/env bash
+set -eo pipefail
+set -x
+
+# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM,
+# with MTP speculative decoding (num_speculative_tokens=2).
+#
+# Identical to dsv4_fp4_b300_vllm.sh (same image, engine args, offload, GPU
+# topologies, and agentic aiperf rig) with exactly two MTP deltas:
+#   --speculative-config '{"method": "mtp", "num_speculative_tokens": 2}'
+#   cudagraph capture sizes expressed in TOKENS (see the capture block below).
+#
+# Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache,
+# sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer cache, mega-MoE,
+# and FULL_DECODE_ONLY CUDA graphs with every decode batch captured explicitly.
+#
+# Required env vars:
+#   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
+#
+# TP4, TP8, and DEP8 (TP8 + DP-attention) are GPU-resident (KV_OFFLOADING=none).
+# DEP4 uses KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=vllm-simple or mooncake.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
+
+GPU_COUNT=$TP
+if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then
+    echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2
+    exit 1
+fi
+export GPU_COUNT
+
+# Under DP-attention the DP world size equals TP, and the DEP recipe sizes
+# per-rank batch as MAX_NUM_SEQS = 2*CONC/TP, which must be an integer.
+if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then
+    echo "Error: DEP requires 2*CONC divisible by TP, got CONC='$CONC' and TP='$TP'" >&2
+    exit 1
+fi
+
+# DEP8 (TP8 + DP-attention) is a GPU-resident, high-concurrency arm that is
+# tuned separately from the smaller DEP4 arm (larger prefill token budget,
+# long-prefill chunking, and a lower GPU-memory-utilization headroom).
+IS_DEP8=false
+if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then
+    IS_DEP8=true
+fi
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+# `hf download` creates the target dir if missing and is itself idempotent.
+# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE.
+# Either way, MODEL_PATH is what the server is launched with.
+if [[ -n "$MODEL_PATH" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+else
+    hf download "$MODEL"
+    export MODEL_PATH="$MODEL"
+fi
+nvidia-smi
+
+# ---- Resolve traces and install deps ----------------------------------------
+resolve_trace_source
+install_agentic_deps
+
+# vllm-project/router expands the one HTTP backend into one logical worker per
+# DP rank. Bind every turn of a conversation to the same rank by mapping
+# AIPerf's stable correlation ID to the router's X-Session-ID header.
+USE_VLLM_ROUTER=false
+VLLM_BACKEND_PORT="$PORT"
+if [ "$DP_ATTENTION" = "true" ]; then
+    USE_VLLM_ROUTER=true
+    VLLM_BACKEND_PORT=$((PORT + 1))
+    VLLM_ROUTER_VERSION=0.1.14
+    VLLM_ROUTER_POLICY=consistent_hash
+    VLLM_ROUTER_METRICS_PORT=$((PORT + 10000))
+    export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1
+    agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION"
+fi
+
+# Match the environment used by v4pro-b300.yaml.
+export VLLM_USE_V2_MODEL_RUNNER=1
+export VLLM_ENGINE_READY_TIMEOUT_S=3600
+export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
+export VLLM_DSV4_MEGA_FP8_COMBINE=1
+export NCCL_NVLS_ENABLE=1
+export VLLM_USE_RUST_FRONTEND=1
+
+# ---- Server config ----------------------------------------------------------
+SERVER_LOG="$RESULT_DIR/server.log"
+ROUTER_LOG="$RESULT_DIR/router.log"
+MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
+mkdir -p "$RESULT_DIR"
+
+SERVER_PID=""
+ROUTER_PID=""
+MOONCAKE_MASTER_PID=""
+
+# The generated TOTAL_CPU_DRAM_GB budget is proportional to allocated GPUs.
+# On cluster:b300-nv, dram-utilization=0.80 and DEP4 resolve to roughly the
+# source recipe's 280 GiB per DP rank. TP4 remains GPU-resident.
+OFFLOAD_ARGS=()
+case "$KV_OFFLOAD_BACKEND" in
+    "")
+        require_agentic_kv_offload_none
+        ;;
+    vllm-simple)
+        require_agentic_kv_offload_backend vllm-simple
+        CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT ))
+        # Identical prefixes must hash to identical block keys across DP ranks.
+        export PYTHONHASHSEED=42
+        # The plain-TP (non-DP-attention) offload ladder uses lazy offload;
+        # DEP keeps eager offload for cross-rank block-hash stability.
+        SIMPLE_LAZY_OFFLOAD=false
+        if [ "$DP_ATTENTION" != "true" ]; then
+            SIMPLE_LAZY_OFFLOAD=true
+        fi
+        OFFLOAD_CONFIG=$(cat <<EOF
+{
+  "kv_connector": "SimpleCPUOffloadConnector",
+  "kv_role": "kv_both",
+  "kv_connector_extra_config": {
+    "cpu_bytes_to_use": ${CPU_BYTES_PER_RANK},
+    "enable_cross_layers_blocks": "true",
+    "lazy_offload": ${SIMPLE_LAZY_OFFLOAD}
+  }
+}
+EOF
+)
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "$OFFLOAD_CONFIG"
+        )
+        ;;
+    mooncake)
+        require_agentic_kv_offload_backend mooncake
+        # Embedded mode contributes one global segment per DP rank to the
+        # shared store, so divide the aggregate host budget across ranks.
+        PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT))
+        MOONCAKE_VERSION=0.3.11.post1
+        agentic_pip_install --quiet --no-cache-dir --no-deps \
+            --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
+        python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
+
+        MOONCAKE_MASTER_PORT=$((PORT + 12000))
+        MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
+        cat > "$MOONCAKE_CONFIG_PATH" <<EOF
+{
+  "mode": "embedded",
+  "metadata_server": "P2PHANDSHAKE",
+  "master_server_address": "127.0.0.1:$MOONCAKE_MASTER_PORT",
+  "global_segment_size": "${PER_RANK_GB}GB",
+  "local_buffer_size": "4GB",
+  "protocol": "rdma",
+  "device_name": "",
+  "enable_offload": false
+}
+EOF
+        export MOONCAKE_CONFIG_PATH
+        export MC_ENABLE_DEST_DEVICE_AFFINITY=1
+        # Identical prefixes must hash to identical store keys across DP ranks.
+        export PYTHONHASHSEED=0
+        export WITH_NVIDIA_PEERMEM=0
+        export MC_SLICE_SIZE=1048576
+        export MC_WORKERS_PER_CTX=4
+
+        # The store is shared, but each rank contributes a separate segment.
+        # Start eviction before an imbalanced rank exhausts its segment, and
+        # reclaim enough space for several concurrent multi-GB batch puts.
+        MOONCAKE_EVICTION_HIGH_WATERMARK_RATIO=0.80
+        MOONCAKE_EVICTION_RATIO=0.10
+
+        echo "Starting Mooncake master on port $MOONCAKE_MASTER_PORT..."
+        mooncake_master --port "$MOONCAKE_MASTER_PORT" \
+            --eviction_high_watermark_ratio="$MOONCAKE_EVICTION_HIGH_WATERMARK_RATIO" \
+            --eviction_ratio="$MOONCAKE_EVICTION_RATIO" \
+            > "$MOONCAKE_MASTER_LOG" 2>&1 &
+        MOONCAKE_MASTER_PID=$!
+        sleep 2
+        if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then
+            echo "Mooncake master died during startup." >&2
+            cat "$MOONCAKE_MASTER_LOG" >&2
+            exit 1
+        fi
+
+        unset VLLM_USE_SIMPLE_KV_OFFLOAD
+        OFFLOAD_CONFIG='{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
+        OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG")
+        ;;
+    *)
+        echo "Error: unsupported B300 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2
+        exit 1
+        ;;
+esac
+
+PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
+if [ "$DP_ATTENTION" = "true" ]; then
+    PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
+fi
+
+TP_ARGS=()
+if [ "$DP_ATTENTION" = "true" ]; then
+    export PYTORCH_ALLOC_CONF=expandable_segments:True
+else
+    export VLLM_ALLREDUCE_USE_FLASHINFER=1
+    export VLLM_FLASHINFER_ALLREDUCE_BACKEND=auto
+    TP_ARGS+=(--disable-custom-all-reduce)
+fi
+
+MODE_ARGS=()
+if [ "$EP_SIZE" -gt 1 ]; then
+    MODE_ARGS+=(
+        --enable-expert-parallel
+        --enable-ep-weight-filter
+        --moe-backend deep_gemm_amxf4_mega_moe
+    )
+fi
+if [ "$DP_ATTENTION" = "true" ]; then
+    MODE_ARGS+=(--prefill-schedule-interval 8)
+    if [ "$IS_DEP8" = "true" ]; then
+        # GPU-resident DEP8 gets a larger prefill token budget and chunks long
+        # prefills so decode latency stays bounded at high concurrency.
+        MODE_ARGS+=(
+            --max-num-batched-tokens 16384
+            --long-prefill-token-threshold 4096
+        )
+    else
+        MODE_ARGS+=(--max-num-batched-tokens 8192)
+    fi
+fi
+
+if [ "$DP_ATTENTION" = "true" ]; then
+    # The DEP source recipe enforces 2*CONC = DP_WORLD_SIZE*MAX_NUM_SEQS.
+    MAX_NUM_SEQS=$((2 * CONC / TP))
+else
+    # Preserve the previous TP4 scheduler headroom for agentic fan-out.
+    MAX_NUM_SEQS=$((2 * CONC))
+fi
+# MTP: cudagraph capture sizes are in TOKENS. With num_speculative_tokens=N,
+# every uniform decode batch of S seqs verifies S*(1+N) tokens, so capture the
+# explicit multiples (1+N), 2*(1+N), ..., MAX_NUM_SEQS*(1+N) -- one graph per
+# decode batch of 1..MAX_NUM_SEQS seqs. vLLM rounds configured sizes up to
+# multiples of (1+N) and dedups (adjust_cudagraph_sizes_for_spec_decode), so a
+# plain 1..MAX_NUM_SEQS list would collapse to coverage of only
+# MAX_NUM_SEQS/(1+N) seqs and drop the largest decode batches to eager.
+NUM_SPEC_TOKENS=2
+TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS))
+CUDA_GRAPH_CAPTURE_SIZES=""
+for ((num_seqs = 1; num_seqs <= MAX_NUM_SEQS; num_seqs++)); do
+    if [ -n "$CUDA_GRAPH_CAPTURE_SIZES" ]; then
+        CUDA_GRAPH_CAPTURE_SIZES+=","
+    fi
+    CUDA_GRAPH_CAPTURE_SIZES+="$((num_seqs * TOKENS_PER_SEQ))"
+done
+COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}],\"mode\":0}"
+
+echo "Starting vllm server..."
+export TORCH_CUDA_ARCH_LIST="10.0"
+export PYTHONNOUSERSITE=1
+export VLLM_FLOAT32_MATMUL_PRECISION=high
+
+# DEP8 leaves more headroom for its larger prefill token budget.
+GPU_MEM_UTIL=0.96
+if [ "$IS_DEP8" = "true" ]; then
+    GPU_MEM_UTIL=0.92
+fi
+
+{ set +x; } 2>/dev/null
+VLLM_CMD=(
+    vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$VLLM_BACKEND_PORT"
+    --gpu-memory-utilization "$GPU_MEM_UTIL"
+    --trust-remote-code
+    --no-enable-flashinfer-autotune
+    --no-disable-hybrid-kv-cache-manager
+    --max-num-seqs "$MAX_NUM_SEQS"
+    --kv-cache-dtype fp8
+    --block-size 256
+    --max-model-len 1048576
+    --attention-config '{"use_fp4_indexer_cache":true,"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
+    --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}"
+    --disable-uvicorn-access-log
+    --tokenizer-mode deepseek_v4
+    --tool-call-parser deepseek_v4
+    --enable-auto-tool-choice
+    --reasoning-parser deepseek_v4
+    --compilation-config "$COMPILATION_CONFIG"
+    "${PARALLEL_ARGS[@]}"
+    "${TP_ARGS[@]}"
+    "${MODE_ARGS[@]}"
+    "${OFFLOAD_ARGS[@]}"
+)
+printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
+"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+echo "Server PID: $SERVER_PID"
+
+wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [ "$USE_VLLM_ROUTER" = "true" ]; then
+    echo "Starting native vLLM router on port $PORT for $TP DP ranks..."
+    vllm-router \
+        --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \
+        --policy "$VLLM_ROUTER_POLICY" \
+        --intra-node-data-parallel-size "$TP" \
+        --host 0.0.0.0 \
+        --port "$PORT" \
+        --prometheus-host 127.0.0.1 \
+        --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \
+        --request-timeout-secs 14400 \
+        --disable-retries > "$ROUTER_LOG" 2>&1 &
+    ROUTER_PID=$!
+    echo "Router PID: $ROUTER_PID"
+    wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
+fi
+
+# ---- Run benchmark ----------------------------------------------------------
+build_replay_cmd "$RESULT_DIR"
+
+run_agentic_replay_and_write_outputs "$RESULT_DIR"
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index aa42c936b..11204baf9 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3571,10 +3571,14 @@ dsv4-fp4-b300-vllm-agentic:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
+      # TP8 GPU-resident
+      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8] }
       # TP4 GPU-resident
-      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
+      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20] }
+      # TP4 GPU-resident + MTP (num_speculative_tokens=2)
+      - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8, 12, 16, 20] }
       # TP4 SimpleCPU
-      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [28, 32, 36, 40] }
+      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [20, 24, 28, 32, 36, 40] }
       # DEP4 SimpleCPU
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
       # DEP8 SimpleCPU

From 9656e75bf48a876e1a4bccfab77005ccd7c5c19d Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Fri, 17 Jul 2026 00:04:40 -0700
Subject: [PATCH 11/17] [AgentX] DeepSeek-V4 B300 vLLM: MTP
 (num_speculative_tokens=3) across the recipe

Finalize the B300 AgentX vLLM search space (dsv4-fp4-b300-vllm-agentic):
- Add a TP8 GPU-resident arm at conc [1,2,4,6,8].
- Trim TP4 GPU-resident to conc [1,2,4,6,8,12,16,20]; extend TP4 SimpleCPU
  lazy-offload down to conc [20,24,28,32,36,40].
- Add an MTP speculative-decoding twin (num_speculative_tokens=3) for every
  topology (TP8/TP4 GPU-resident, TP4 SimpleCPU, DEP4, DEP8), each mirroring
  its non-MTP conc-list, routed via the launcher's spec-decoding=mtp suffix to
  dsv4_fp4_b300_vllm_mtp.sh.

MTP script: NUM_SPEC_TOKENS=3 -> TOKENS_PER_SEQ=4, so FULL_DECODE_ONLY
cudagraph capture sizes (num_seqs*TOKENS_PER_SEQ) scale to num_seqs*4. Sync the
tail with dsv4_fp4_b300_vllm.sh to restore the EVAL_ONLY branch (added upstream
by #1947) so MTP configs also run the SWE-bench Lite accuracy eval.

Add the perf-changelog entry for dsv4-fp4-b300-vllm-agentic (PR #2258).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../agentic/dsv4_fp4_b300_vllm_mtp.sh            | 16 +++++++++-------
 configs/nvidia-master.yaml                       | 10 +++++++++-
 perf-changelog.yaml                              |  8 ++++++++
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
index 8a01e16c7..abdacbb9e 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
@@ -3,11 +3,11 @@ set -eo pipefail
 set -x
 
 # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM,
-# with MTP speculative decoding (num_speculative_tokens=2).
+# with MTP speculative decoding (num_speculative_tokens=3).
 #
 # Identical to dsv4_fp4_b300_vllm.sh (same image, engine args, offload, GPU
 # topologies, and agentic aiperf rig) with exactly two MTP deltas:
-#   --speculative-config '{"method": "mtp", "num_speculative_tokens": 2}'
+#   --speculative-config '{"method": "mtp", "num_speculative_tokens": 3}'
 #   cudagraph capture sizes expressed in TOKENS (see the capture block below).
 #
 # Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache,
@@ -247,7 +247,7 @@ fi
 # multiples of (1+N) and dedups (adjust_cudagraph_sizes_for_spec_decode), so a
 # plain 1..MAX_NUM_SEQS list would collapse to coverage of only
 # MAX_NUM_SEQS/(1+N) seqs and drop the largest decode batches to eager.
-NUM_SPEC_TOKENS=2
+NUM_SPEC_TOKENS=3
 TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS))
 CUDA_GRAPH_CAPTURE_SIZES=""
 for ((num_seqs = 1; num_seqs <= MAX_NUM_SEQS; num_seqs++)); do
@@ -320,7 +320,9 @@ if [ "$USE_VLLM_ROUTER" = "true" ]; then
     wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
 fi
 
-# ---- Run benchmark ----------------------------------------------------------
-build_replay_cmd "$RESULT_DIR"
-
-run_agentic_replay_and_write_outputs "$RESULT_DIR"
+if [ "${EVAL_ONLY}" = "true" ]; then
+    run_eval --port "$PORT"
+else
+    build_replay_cmd "$RESULT_DIR"
+    run_agentic_replay_and_write_outputs "$RESULT_DIR"
+fi
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 1b18c33ca..e59444e43 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3956,16 +3956,24 @@ dsv4-fp4-b300-vllm-agentic:
       search-space:
       # TP8 GPU-resident
       - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8] }
+      # TP8 GPU-resident + MTP (num_speculative_tokens=3)
+      - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8] }
       # TP4 GPU-resident
       - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20] }
-      # TP4 GPU-resident + MTP (num_speculative_tokens=2)
+      # TP4 GPU-resident + MTP (num_speculative_tokens=3)
       - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8, 12, 16, 20] }
       # TP4 SimpleCPU
       - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [20, 24, 28, 32, 36, 40] }
+      # TP4 SimpleCPU + MTP (num_speculative_tokens=3)
+      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, spec-decoding: mtp, conc-list: [20, 24, 28, 32, 36, 40] }
       # DEP4 SimpleCPU
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP4 SimpleCPU + MTP (num_speculative_tokens=3)
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, spec-decoding: mtp, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
       # DEP8 SimpleCPU
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP8 SimpleCPU + MTP (num_speculative_tokens=3)
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index c620aceaa..e13e2ce3d 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4918,3 +4918,11 @@
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
     - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.92); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2241
+
+- config-keys:
+    - dsv4-fp4-b300-vllm-agentic
+  description:
+    - "Add an MTP speculative-decoding twin for every B300 AgentX topology (DeepSeek-V4 built-in MTP): --speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}, routed via the launcher's spec-decoding=mtp suffix to dsv4_fp4_b300_vllm_mtp.sh -- TP8/TP4 GPU-resident, TP4 SimpleCPU, DEP4 SimpleCPU, and DEP8, each mirroring its non-MTP conc-list."
+    - "MTP FULL_DECODE_ONLY cudagraph capture sizes are in TOKENS: captured as num_seqs*(1+num_speculative_tokens) (= num_seqs*4) so vLLM's round-up-and-dedup keeps one graph per decode batch of 1..MAX_NUM_SEQS seqs. MTP variants also run the SWE-bench Lite accuracy eval (EVAL_ONLY path synced with the base recipe)."
+    - "Non-MTP search-space tweaks: add TP8 GPU-resident at conc [1,2,4,6,8], trim TP4 GPU-resident to conc [1,2,4,6,8,12,16,20], and extend TP4 SimpleCPU lazy-offload down to conc [20,24,28,32,36,40]."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2258

From 5887e1a51c892857c199e98e89510603642caf7b Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Fri, 17 Jul 2026 00:56:07 -0700
Subject: [PATCH 12/17] fix(changelog): accept single-node agentic eval rows in
 matrix validation

#1947 made single-node agentic recipes generate a SWE-bench eval row
(run_eval/eval_only + agentic fields) but never widened the changelog
matrix schema, so ChangelogMatrixEntry.evals (typed list[SingleNodeMatrixEntry],
fixed-seq-len only) rejects every agentic eval row -- breaking check-changelog
for any single-node agentic PR.

Widen evals to the same Union single_node already uses, and give
SingleNodeAgenticMatrixEntry optional run_eval/eval_only (None-default, so
benchmark rows are unchanged).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 utils/matrix_logic/validation.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py
index 6b7c67128..7403c795e 100644
--- a/utils/matrix_logic/validation.py
+++ b/utils/matrix_logic/validation.py
@@ -291,6 +291,10 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
     duration: int = Field(alias=Fields.DURATION.value)
     exp_name: str = Field(alias=Fields.EXP_NAME.value)
     scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value)
+    # Agentic eval rows (SWE-bench) carry run_eval/eval_only; benchmark rows omit
+    # them. Optional so both shapes validate and benchmark output is unchanged.
+    run_eval: Optional[bool] = Field(default=None, alias=Fields.RUN_EVAL.value)
+    eval_only: Optional[bool] = Field(default=None, alias=Fields.EVAL_ONLY.value)
 
     @model_validator(mode='after')
     def validate_kv_offload_fields(self):
@@ -889,7 +893,7 @@ class ChangelogMatrixEntry(BaseModel):
                       ] = Field(default_factory=dict)
     multi_node: dict[str, list[Union[MultiNodeMatrixEntry, MultiNodeAgenticMatrixEntry]]
                      ] = Field(default_factory=dict)
-    evals: list[SingleNodeMatrixEntry] = Field(default_factory=list)
+    evals: list[Union[SingleNodeMatrixEntry, SingleNodeAgenticMatrixEntry]] = Field(default_factory=list)
     multinode_evals: list[MultiNodeMatrixEntry] = Field(default_factory=list)
     changelog_metadata: ChangelogMetadata
 

From d26aba5fdfab0154d5589d30ecf8ec10f8c839c0 Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Fri, 17 Jul 2026 01:24:53 -0700
Subject: [PATCH 13/17] [AgentX] B300 vLLM: split MTP + TP8 into separate
 config-keys

Restructure so the sweep runs only the new work, not the existing aggregate:
- Revert dsv4-fp4-b300-vllm-agentic to its main (#2241) search space (TP4
  [1..32], SimpleCPU [28..40], DEP4, DEP8) -- unchanged vs main, so it is not
  re-run.
- Add dsv4-fp4-b300-vllm-agentic-mtp: MTP twins of the aggregate arms
  (num_speculative_tokens=3), reshaped TP4 [1..20] / SimpleCPU [20..40].
- Add dsv4-fp4-b300-vllm-agentic-tp8: new TP8 GPU-resident topology at
  conc [1,2,4,6,8], both non-MTP and MTP.
- Point the perf-changelog entry at only the two new keys, so only they sweep.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 configs/nvidia-master.yaml | 48 +++++++++++++++++++++++++++++---------
 perf-changelog.yaml        |  9 +++----
 2 files changed, 42 insertions(+), 15 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index e59444e43..2fd8beb40 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3954,27 +3954,53 @@ dsv4-fp4-b300-vllm-agentic:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      # TP8 GPU-resident
-      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8] }
-      # TP8 GPU-resident + MTP (num_speculative_tokens=3)
-      - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8] }
       # TP4 GPU-resident
-      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20] }
+      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
+      # TP4 SimpleCPU
+      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [28, 32, 36, 40] }
+      # DEP4 SimpleCPU
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP8 SimpleCPU
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } }
+
+dsv4-fp4-b300-vllm-agentic-mtp:
+  image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: cluster:b300-nv
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
       # TP4 GPU-resident + MTP (num_speculative_tokens=3)
       - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8, 12, 16, 20] }
-      # TP4 SimpleCPU
-      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [20, 24, 28, 32, 36, 40] }
       # TP4 SimpleCPU + MTP (num_speculative_tokens=3)
       - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, spec-decoding: mtp, conc-list: [20, 24, 28, 32, 36, 40] }
-      # DEP4 SimpleCPU
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
       # DEP4 SimpleCPU + MTP (num_speculative_tokens=3)
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, spec-decoding: mtp, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
-      # DEP8 SimpleCPU
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } }
       # DEP8 SimpleCPU + MTP (num_speculative_tokens=3)
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } }
 
+dsv4-fp4-b300-vllm-agentic-tp8:
+  image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: cluster:b300-nv
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      # TP8 GPU-resident
+      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8] }
+      # TP8 GPU-resident + MTP (num_speculative_tokens=3)
+      - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8] }
+
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
   model: deepseek-ai/DeepSeek-V4-Pro
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index e13e2ce3d..4320299d1 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4920,9 +4920,10 @@
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2241
 
 - config-keys:
-    - dsv4-fp4-b300-vllm-agentic
+    - dsv4-fp4-b300-vllm-agentic-mtp
+    - dsv4-fp4-b300-vllm-agentic-tp8
   description:
-    - "Add an MTP speculative-decoding twin for every B300 AgentX topology (DeepSeek-V4 built-in MTP): --speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}, routed via the launcher's spec-decoding=mtp suffix to dsv4_fp4_b300_vllm_mtp.sh -- TP8/TP4 GPU-resident, TP4 SimpleCPU, DEP4 SimpleCPU, and DEP8, each mirroring its non-MTP conc-list."
-    - "MTP FULL_DECODE_ONLY cudagraph capture sizes are in TOKENS: captured as num_seqs*(1+num_speculative_tokens) (= num_seqs*4) so vLLM's round-up-and-dedup keeps one graph per decode batch of 1..MAX_NUM_SEQS seqs. MTP variants also run the SWE-bench Lite accuracy eval (EVAL_ONLY path synced with the base recipe)."
-    - "Non-MTP search-space tweaks: add TP8 GPU-resident at conc [1,2,4,6,8], trim TP4 GPU-resident to conc [1,2,4,6,8,12,16,20], and extend TP4 SimpleCPU lazy-offload down to conc [20,24,28,32,36,40]."
+    - "Add MTP speculative-decoding and a new TP8 GPU-resident arm for the B300 vLLM AgentX recipe as separate config-keys, so the existing dsv4-fp4-b300-vllm-agentic aggregate is not re-run."
+    - "dsv4-fp4-b300-vllm-agentic-mtp: MTP twins (--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}) of the aggregate arms -- TP4 GPU-resident [1,2,4,6,8,12,16,20], TP4 SimpleCPU [20,24,28,32,36,40], DEP4 [32,40,48,56,64,72], DEP8 [64,96,112,128,144,160,176,192,224]; routed via spec-decoding=mtp to dsv4_fp4_b300_vllm_mtp.sh (FULL_DECODE_ONLY cudagraph capture sizes in TOKENS = num_seqs*4)."
+    - "dsv4-fp4-b300-vllm-agentic-tp8: new TP8 GPU-resident topology at conc [1,2,4,6,8], both non-MTP and MTP."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2258

From 4c5b4a1b28df115c96634cb2f772842bf5ea3278 Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Fri, 17 Jul 2026 13:10:49 -0700
Subject: [PATCH 14/17] align synthetic acceptance length

Inject rejection_sample_method=synthetic + synthetic_acceptance_length=2.49
into the dsv4-fp4-b300-vllm MTP speculative-config, matching the dsv4-pro
golden AL (thinking_on, num_speculative_tokens=3) and the sglang gb300 MTP
recipe (SGLANG_SIMULATE_ACC_LEN=2.49).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
index abdacbb9e..4ed2dbe90 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
@@ -3,11 +3,11 @@ set -eo pipefail
 set -x
 
 # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM,
-# with MTP speculative decoding (num_speculative_tokens=3).
+# with MTP speculative decoding (num_speculative_tokens=3, synthetic acceptance length 2.49).
 #
 # Identical to dsv4_fp4_b300_vllm.sh (same image, engine args, offload, GPU
 # topologies, and agentic aiperf rig) with exactly two MTP deltas:
-#   --speculative-config '{"method": "mtp", "num_speculative_tokens": 3}'
+#   --speculative-config '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}'
 #   cudagraph capture sizes expressed in TOKENS (see the capture block below).
 #
 # Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache,
@@ -248,6 +248,9 @@ fi
 # plain 1..MAX_NUM_SEQS list would collapse to coverage of only
 # MAX_NUM_SEQS/(1+N) seqs and drop the largest decode batches to eager.
 NUM_SPEC_TOKENS=3
+# Standardize MTP acceptance to the dsv4-pro golden AL (thinking_on,
+# num_speculative_tokens=3) from golden_al_distribution/dsv4_mtp.yaml.
+SYNTHETIC_ACCEPT_LEN=2.49
 TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS))
 CUDA_GRAPH_CAPTURE_SIZES=""
 for ((num_seqs = 1; num_seqs <= MAX_NUM_SEQS; num_seqs++)); do
@@ -283,7 +286,7 @@ VLLM_CMD=(
     --block-size 256
     --max-model-len 1048576
     --attention-config '{"use_fp4_indexer_cache":true,"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
-    --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}"
+    --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}"
     --disable-uvicorn-access-log
     --tokenizer-mode deepseek_v4
     --tool-call-parser deepseek_v4

From ce0f1640131ff2c3f87b4d8676bcb821d604e891 Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Fri, 17 Jul 2026 14:27:40 -0700
Subject: [PATCH 15/17] reduce GPU memory utilization to 0.95 (except DEP8)

TP4, DEP4, and TP8 use gpu-memory-utilization 0.95 (down from 0.96); DEP8
stays at 0.92 for its larger prefill token budget.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
index 4ed2dbe90..082c5c79c 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh
@@ -266,8 +266,9 @@ export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
 export VLLM_FLOAT32_MATMUL_PRECISION=high
 
-# DEP8 leaves more headroom for its larger prefill token budget.
-GPU_MEM_UTIL=0.96
+# DEP8 leaves more headroom for its larger prefill token budget; all other
+# topologies (TP4/DEP4/TP8) use 0.95.
+GPU_MEM_UTIL=0.95
 if [ "$IS_DEP8" = "true" ]; then
     GPU_MEM_UTIL=0.92
 fi

From bc4fe4bba5f4cbf7fb7002924a61b885d21e4f81 Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Fri, 17 Jul 2026 19:18:14 -0700
Subject: [PATCH 16/17] trim B300 MTP search space: fold TP8 into -mtp, drop
 -tp8 key, trim DEP4

Remove the dsv4-fp4-b300-vllm-agentic-tp8 config-key entirely; its TP8
GPU-resident MTP arm moves into dsv4-fp4-b300-vllm-agentic-mtp (the non-MTP
TP8 baseline is dropped). Also drop conc 64 and 72 from the DEP4 arm
(now [32,40,48,56]).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 configs/nvidia-master.yaml | 21 +++------------------
 perf-changelog.yaml        |  6 ++----
 2 files changed, 5 insertions(+), 22 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index bbd9d91df..ed4809929 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3975,32 +3975,17 @@ dsv4-fp4-b300-vllm-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
+      # TP8 GPU-resident + MTP (num_speculative_tokens=3)
+      - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8] }
       # TP4 GPU-resident + MTP (num_speculative_tokens=3)
       - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8, 12, 16, 20] }
       # TP4 SimpleCPU + MTP (num_speculative_tokens=3)
       - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, spec-decoding: mtp, conc-list: [20, 24, 28, 32, 36, 40] }
       # DEP4 SimpleCPU + MTP (num_speculative_tokens=3)
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, spec-decoding: mtp, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, spec-decoding: mtp, conc-list: [32, 40, 48, 56], router: { name: vllm-router, version: "0.1.14" } }
       # DEP8 SimpleCPU + MTP (num_speculative_tokens=3)
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } }
 
-dsv4-fp4-b300-vllm-agentic-tp8:
-  image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec
-  model: deepseek-ai/DeepSeek-V4-Pro
-  model-prefix: dsv4
-  runner: cluster:b300-nv
-  precision: fp4
-  framework: vllm
-  multinode: false
-  scenarios:
-    agentic-coding:
-    - dram-utilization: 0.80
-      search-space:
-      # TP8 GPU-resident
-      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8] }
-      # TP8 GPU-resident + MTP (num_speculative_tokens=3)
-      - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8] }
-
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
   model: deepseek-ai/DeepSeek-V4-Pro
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 0658877c5..b388114df 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4921,11 +4921,9 @@
 
 - config-keys:
     - dsv4-fp4-b300-vllm-agentic-mtp
-    - dsv4-fp4-b300-vllm-agentic-tp8
   description:
-    - "Add MTP speculative-decoding and a new TP8 GPU-resident arm for the B300 vLLM AgentX recipe as separate config-keys, so the existing dsv4-fp4-b300-vllm-agentic aggregate is not re-run."
-    - "dsv4-fp4-b300-vllm-agentic-mtp: MTP twins (--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}) of the aggregate arms -- TP4 GPU-resident [1,2,4,6,8,12,16,20], TP4 SimpleCPU [20,24,28,32,36,40], DEP4 [32,40,48,56,64,72], DEP8 [64,96,112,128,144,160,176,192,224]; routed via spec-decoding=mtp to dsv4_fp4_b300_vllm_mtp.sh (FULL_DECODE_ONLY cudagraph capture sizes in TOKENS = num_seqs*4)."
-    - "dsv4-fp4-b300-vllm-agentic-tp8: new TP8 GPU-resident topology at conc [1,2,4,6,8], both non-MTP and MTP."
+    - "Add MTP speculative-decoding for the B300 vLLM AgentX recipe as a separate config-key, so the existing dsv4-fp4-b300-vllm-agentic aggregate is not re-run."
+    - "dsv4-fp4-b300-vllm-agentic-mtp: MTP twins (--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}) of the aggregate arms plus a new TP8 GPU-resident arm -- TP8 GPU-resident [1,2,4,6,8], TP4 GPU-resident [1,2,4,6,8,12,16,20], TP4 SimpleCPU [20,24,28,32,36,40], DEP4 [32,40,48,56], DEP8 [64,96,112,128,144,160,176,192,224]; routed via spec-decoding=mtp to dsv4_fp4_b300_vllm_mtp.sh (FULL_DECODE_ONLY cudagraph capture sizes in TOKENS = num_seqs*4)."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2258
 
 - config-keys:

From 9b7264a9442a796963b7a27b787ea648a241bcd3 Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Sat, 18 Jul 2026 02:03:33 -0700
Subject: [PATCH 17/17] Revert "fix(changelog): accept single-node agentic eval
 rows in matrix validation"

This reverts commit 5887e1a51c892857c199e98e89510603642caf7b.
---
 utils/matrix_logic/validation.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py
index 7403c795e..6b7c67128 100644
--- a/utils/matrix_logic/validation.py
+++ b/utils/matrix_logic/validation.py
@@ -291,10 +291,6 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
     duration: int = Field(alias=Fields.DURATION.value)
     exp_name: str = Field(alias=Fields.EXP_NAME.value)
     scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value)
-    # Agentic eval rows (SWE-bench) carry run_eval/eval_only; benchmark rows omit
-    # them. Optional so both shapes validate and benchmark output is unchanged.
-    run_eval: Optional[bool] = Field(default=None, alias=Fields.RUN_EVAL.value)
-    eval_only: Optional[bool] = Field(default=None, alias=Fields.EVAL_ONLY.value)
 
     @model_validator(mode='after')
     def validate_kv_offload_fields(self):
@@ -893,7 +889,7 @@ class ChangelogMatrixEntry(BaseModel):
                       ] = Field(default_factory=dict)
     multi_node: dict[str, list[Union[MultiNodeMatrixEntry, MultiNodeAgenticMatrixEntry]]
                      ] = Field(default_factory=dict)
-    evals: list[Union[SingleNodeMatrixEntry, SingleNodeAgenticMatrixEntry]] = Field(default_factory=list)
+    evals: list[SingleNodeMatrixEntry] = Field(default_factory=list)
     multinode_evals: list[MultiNodeMatrixEntry] = Field(default_factory=list)
     changelog_metadata: ChangelogMetadata