Skip to content

Commit e281cb7

Browse files
ZhanqiuHuZhanqiuHu
authored
[CI] Add MultiConnector (Nixl+Offloading) e2e edge case tests (vllm-project#39343)
Signed-off-by: ZhanqiuHu <zhu@redhat.com>
1 parent 51cfc0e commit e281cb7

3 files changed

Lines changed: 665 additions & 0 deletions

File tree

‎.buildkite/test_areas/distributed.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,20 @@ steps:
295295
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
296296
- bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
297297

298+
- label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs)
299+
timeout_in_minutes: 30
300+
working_dir: "/vllm-workspace/tests"
301+
num_devices: 2
302+
source_file_dependencies:
303+
- vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
304+
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
305+
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
306+
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
307+
- tests/v1/kv_connector/nixl_integration/
308+
commands:
309+
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
310+
- bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
311+
298312
- label: Pipeline + Context Parallelism (4 GPUs)
299313
timeout_in_minutes: 60
300314
working_dir: "/vllm-workspace/tests"

‎tests/v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
#!/bin/bash
2+
# SPDX-License-Identifier: Apache-2.0
3+
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
4+
#
5+
# Integration edge-case tests for MultiConnector (NixlConnector + OffloadingConnector).
6+
#
7+
# Launches a P/D setup where both prefill and decode instances use MultiConnector
8+
# wrapping NixlConnector and OffloadingConnector, then runs scenario-based edge
9+
# case tests including Prometheus metrics validation.
10+
#
11+
# Tests cover: block-size boundaries, decode-side cache-hit scenarios
12+
# (cold / full / partial), direct decode (control), and prefill-side CPU
13+
# offload recovery after GPU eviction.
14+
#
15+
# Usage:
16+
# bash tests/v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
17+
#
18+
# Environment variables:
19+
# MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B)
20+
# KV_CACHE_MEMORY_BYTES - GPU KV cache size in bytes (default: 268435456 = 256 MiB)
21+
# BLOCK_SIZE - KV cache block size (default: 128)
22+
# VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve
23+
set -xe
24+
25+
# ── Configuration ────────────────────────────────────────────────────────
26+
27+
MODEL_NAMES=${MODEL_NAMES:-}
28+
if [[ -n "$MODEL_NAMES" ]]; then
29+
MODELS=("$MODEL_NAMES")
30+
else
31+
MODELS=("Qwen/Qwen3-0.6B")
32+
fi
33+
34+
KV_CACHE_MEMORY_BYTES=${KV_CACHE_MEMORY_BYTES:-268435456} # 256 MiB
35+
MAX_MODEL_LEN=${MAX_MODEL_LEN:-2048}
36+
BLOCK_SIZE=${BLOCK_SIZE:-128}
37+
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
38+
39+
GIT_ROOT=$(git rev-parse --show-toplevel)
40+
41+
# ── KV transfer config ──────────────────────────────────────────────────
42+
43+
KV_CONFIG='{
44+
"kv_connector":"MultiConnector",
45+
"kv_role":"kv_both",
46+
"kv_connector_extra_config":{
47+
"connectors":[
48+
{"kv_connector":"NixlConnector","kv_role":"kv_both"},
49+
{"kv_connector":"OffloadingConnector","kv_role":"kv_both",
50+
"kv_connector_extra_config":{"cpu_bytes_to_use":2147483648}}
51+
]
52+
}
53+
}'
54+
KV_CONFIG=$(echo "$KV_CONFIG" | tr -d '[:space:]')
55+
56+
# ── Helpers ──────────────────────────────────────────────────────────────
57+
58+
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
59+
60+
wait_for_server() {
61+
local port=$1
62+
timeout 1200 bash -c "
63+
until curl -s localhost:${port}/v1/completions > /dev/null; do
64+
sleep 1
65+
done" && return 0 || return 1
66+
}
67+
68+
cleanup_instances() {
69+
echo "Cleaning up any running vLLM instances and proxy..."
70+
pkill -f "vllm serve" || true
71+
pkill -f "toy_proxy_server.py" || true
72+
sleep 2
73+
}
74+
75+
# ── Run tests for one model ──────────────────────────────────────────────
76+
77+
run_tests_for_model() {
78+
local model_name=$1
79+
80+
echo "================================================================"
81+
echo "Testing model: $model_name (MultiConnector edge cases)"
82+
echo "================================================================"
83+
84+
local PREFILL_PORT=8100
85+
local DECODE_PORT=8200
86+
local PROXY_PORT=8192
87+
local PREFILL_GPU=0
88+
local DECODE_GPU=1
89+
local PREFILL_SIDE_CHANNEL_PORT=5559
90+
local DECODE_SIDE_CHANNEL_PORT=5659
91+
92+
# ── Start prefill instance ──
93+
echo "Starting prefill instance on GPU $PREFILL_GPU, port $PREFILL_PORT"
94+
BASE_CMD="CUDA_VISIBLE_DEVICES=$PREFILL_GPU \
95+
VLLM_KV_CACHE_LAYOUT='HND' \
96+
UCX_NET_DEVICES=all \
97+
VLLM_NIXL_SIDE_CHANNEL_PORT=$PREFILL_SIDE_CHANNEL_PORT \
98+
vllm serve \"$model_name\" \
99+
--port $PREFILL_PORT \
100+
--enforce-eager \
101+
--block-size ${BLOCK_SIZE} \
102+
--max-model-len $MAX_MODEL_LEN \
103+
--kv-cache-memory-bytes $KV_CACHE_MEMORY_BYTES \
104+
--tensor-parallel-size 1 \
105+
--kv-transfer-config '$KV_CONFIG'"
106+
107+
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
108+
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
109+
for arg in "${extra_args[@]}"; do
110+
BASE_CMD="${BASE_CMD} $arg"
111+
done
112+
fi
113+
eval "$BASE_CMD &"
114+
115+
# ── Start decode instance ──
116+
echo "Starting decode instance on GPU $DECODE_GPU, port $DECODE_PORT"
117+
BASE_CMD="CUDA_VISIBLE_DEVICES=$DECODE_GPU \
118+
VLLM_KV_CACHE_LAYOUT='HND' \
119+
UCX_NET_DEVICES=all \
120+
VLLM_NIXL_SIDE_CHANNEL_PORT=$DECODE_SIDE_CHANNEL_PORT \
121+
vllm serve \"$model_name\" \
122+
--port $DECODE_PORT \
123+
--enforce-eager \
124+
--block-size ${BLOCK_SIZE} \
125+
--max-model-len $MAX_MODEL_LEN \
126+
--kv-cache-memory-bytes $KV_CACHE_MEMORY_BYTES \
127+
--tensor-parallel-size 1 \
128+
--kv-transfer-config '$KV_CONFIG'"
129+
130+
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
131+
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
132+
for arg in "${extra_args[@]}"; do
133+
BASE_CMD="${BASE_CMD} $arg"
134+
done
135+
fi
136+
eval "$BASE_CMD &"
137+
138+
# ── Wait for servers ──
139+
echo "Waiting for prefill instance on port $PREFILL_PORT to start..."
140+
wait_for_server "$PREFILL_PORT"
141+
echo "Waiting for decode instance on port $DECODE_PORT to start..."
142+
wait_for_server "$DECODE_PORT"
143+
144+
# ── Start proxy ──
145+
echo "Starting proxy server on port $PROXY_PORT"
146+
python3 "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py" \
147+
--port "$PROXY_PORT" \
148+
--prefiller-hosts localhost \
149+
--prefiller-ports "$PREFILL_PORT" \
150+
--decoder-hosts localhost \
151+
--decoder-ports "$DECODE_PORT" &
152+
sleep 5
153+
154+
# ── Run edge case tests ──
155+
echo "Running MultiConnector edge case tests for $model_name"
156+
PREFILL_PORT=$PREFILL_PORT \
157+
DECODE_PORT=$DECODE_PORT \
158+
PROXY_PORT=$PROXY_PORT \
159+
BLOCK_SIZE=$BLOCK_SIZE \
160+
python3 -m pytest -s -x \
161+
"${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_multi_connector_edge_cases.py"
162+
163+
# ── Cleanup ──
164+
cleanup_instances
165+
sleep 3
166+
}
167+
168+
# ── Main ─────────────────────────────────────────────────────────────────
169+
170+
for model in "${MODELS[@]}"; do
171+
run_tests_for_model "$model"
172+
done
173+
174+
echo "All MultiConnector edge case tests passed!"

0 commit comments

Comments
 (0)