ROCm · sbates130272 · Jul 17, 2026
@@ -2,9 +2,9 @@
 #
 # SPDX-License-Identifier: MIT
 #
-# Full Docker build for the AMD Infinity Context (AIC) stack (~45-60 min cold).
-# Triggered manually — the base image (rocm/dev-ubuntu-24.04:7.2.4-complete,
-# ~7 GB) plus vLLM wheel + LMCache + hipFile + NIXL source builds exceed
+# Full Docker build for the AMD Infinity Context (AIC) stack (~60-90 min cold).
+# Triggered manually — the base image (rocm/dev-ubuntu-24.04:7.14.0-full,
+# ~7 GB) plus vLLM source build + LMCache + NIXL source builds exceed
 # GitHub-hosted runner disk on cold runs without the GHA layer cache.
 #
 # GHA cache key: scope=aic

@@ -2,18 +2,20 @@
 #
 # SPDX-License-Identifier: MIT
 #
-# Nightly pip wheels for the AMD Infinity Context (AIC) patched LMCache + ROCm NIXL.
+# Nightly pip wheels for the AMD Infinity Context (AIC): source-built vLLM,
+# patched LMCache, and ROCm NIXL.
 #
 # Each night, if main changed since the last nightly, this builds the AIC
 # image's `wheels` stage on a GitHub-hosted runner (no GPU needed -- hipcc
 # host-compiles), extracts the .whl files, and publishes them to a single
 # rolling GitHub Release tagged `nightly`.  People then:
 #
 #   pip install \
+#     https://github.com/ROCm/rocm-aic/releases/download/nightly/vllm-<ver>-cp312-cp312-linux_x86_64.whl \
 #     https://github.com/ROCm/rocm-aic/releases/download/nightly/lmcache-<ver>-cp312-cp312-linux_x86_64.whl \
 #     https://github.com/ROCm/rocm-aic/releases/download/nightly/nixl_rocm-<ver>-cp312-cp312-linux_x86_64.whl
 #
-# COMPATIBILITY: the wheels are ROCm 7.2.x / Python 3.12 / x86_64 and are NOT
+# COMPATIBILITY: the wheels are ROCm 7.14 / Python 3.12 / x86_64 and are NOT
 # manylinux -- they only install-and-work on a matching ROCm host.  The NIXL
 # wheel carries libnixl + plugins/UCX but leaves the ROCm runtime (libamdhip64,
 # hipFile) as an external dependency.  See README.md.
@@ -163,7 +165,8 @@ jobs:
           wheels=(wheels/*.whl)
           echo "Extracted ${#wheels[@]} wheel(s):"
           for w in "${wheels[@]}"; do echo "  - $(basename "$w") ($(du -h "$w" | cut -f1))"; done
-          # Expect at least one lmcache and one nixl_rocm wheel.
+          # Expect vllm, lmcache, and nixl_rocm wheels.
+          ls wheels/vllm-*.whl      >/dev/null || { echo "ERROR: no vllm wheel"      >&2; exit 1; }
           ls wheels/lmcache-*.whl   >/dev/null || { echo "ERROR: no lmcache wheel"   >&2; exit 1; }
           ls wheels/nixl*rocm*.whl  >/dev/null || { echo "ERROR: no nixl_rocm wheel" >&2; exit 1; }
 
@@ -179,20 +182,22 @@ jobs:
             echo "- **Source:** main @ \`${{ needs.check.outputs.sha }}\`"
             echo "- **Built (UTC):** ${{ github.run_id }} / ${GITHUB_SHA}"
             echo "- **GPU arch set:** \`${ROCM_ARCH}\`"
-            echo "- **Base:** rocm/dev-ubuntu-24.04:7.2.4-complete (ROCm 7.2.4, Python 3.12, x86_64)"
-            echo "- **LMCache:** $(get LMCACHE_GIT_URL) @ \`$(get LMCACHE_SHA)\` + AIC patches"
-            echo "- **NIXL:** $(get NIXL_GIT_URL) @ \`$(get NIXL_SHA)\` + nixl-rocm-ais-mt patch"
+            echo "- **Base:** rocm/dev-ubuntu-24.04:7.14.0-full (ROCm 7.14, Python 3.12, x86_64)"
+            echo "- **vLLM:** $(get VLLM_GIT_URL) @ \`$(get VLLM_REF)\` (source build)"
+            echo "- **LMCache:** $(get LMCACHE_GIT_URL) @ \`$(get LMCACHE_REF)\` + AIC patches"
+            echo "- **NIXL:** $(get NIXL_GIT_URL) @ \`$(get NIXL_REF)\` + nixl-rocm-ais-mt patch"
             echo ""
             echo "### Install"
             echo '```bash'
             echo "pip install \\"
+            echo "  https://github.com/${GITHUB_REPOSITORY}/releases/download/${RELEASE_TAG}/<vllm-wheel> \\"
             echo "  https://github.com/${GITHUB_REPOSITORY}/releases/download/${RELEASE_TAG}/<lmcache-wheel> \\"
             echo "  https://github.com/${GITHUB_REPOSITORY}/releases/download/${RELEASE_TAG}/<nixl_rocm-wheel>"
             echo '```'
             echo ""
-            echo "> These wheels are **not** manylinux: ROCm 7.2.x + Python 3.12 + x86_64 only."
+            echo "> These wheels are **not** manylinux: ROCm 7.14 + Python 3.12 + x86_64 only."
             echo "> The \`nixl_rocm\` wheel needs the ROCm runtime (libamdhip64) and hipFile"
-            echo "> present on the host; see README.md."
+            echo "> present on the host (hipFile is GA in ROCm 7.14); see README.md."
           } > release-notes.md
           cat release-notes.md
 

@@ -160,7 +160,7 @@ AIC_DAY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
 # Infinity-Storage hardware -- if hipFile fails to build for them, narrow this
 # to the CDNA set "gfx90a;gfx942;gfx950".  Override via AIC_ROCM_ARCH.
 AIC_ROCM_ARCH="${AIC_ROCM_ARCH:-gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1150;gfx1151;gfx1200;gfx1201}"
-AIC_IMAGE="${AIC_IMAGE:-rocm-aic:latest}"
+AIC_IMAGE="${AIC_IMAGE:-${IMAGE_NAME:-rocm-aic}:${IMAGE_TAG:-7.14-latest}}"
 AIC_IMAGE_DIR="${AIC_IMAGE_DIR:-/scratch/${USER}/images}"
 AIC_SPUR_CLUSTER="${AIC_SPUR_CLUSTER:-0}"
 AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:-${SPUR_CONTROLLER_ADDR:?set SPUR_CONTROLLER_ADDR or AIC_SPUR_CONTROLLER before using AIC_SPUR_CLUSTER=1}}"
@@ -549,7 +549,9 @@ if [ "\${_rc[1]}" -ne 0 ]; then
     echo "[build] ERROR: compressor exited \${_rc[1]}; tarball may be corrupt" >&2; exit 1
 fi
 if [ "\${_rc[0]}" -ne 0 ]; then
-    echo "[build] WARN: docker buildx exited \${_rc[0]} (cache lock race?); tarball written, continuing"
+    echo "[build] ERROR: docker buildx exited \${_rc[0]}; build failed (patch apply error or Dockerfile issue)" >&2
+    rm -f "\${tmp}"
+    exit 1
 fi
 mv -f "\${tmp}" "${tarball}"
 echo "[build] saved \$(du -h "${tarball}" | cut -f1) -> ${tarball}"

@@ -160,6 +160,22 @@ fi
 # knob (built in the compose-stack setup block below).  Empty => no scaling.
 LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-20}"
 LMCACHE_NVME_POOL="${LMCACHE_NVME_POOL:-4096}"
+# POSIX-specific slot size and pool (separate from AIS_MT's 256 MiB staging buffer).
+# file_size for POSIX = actual KV chunk bytes (~9 MiB for gpt-oss-120b fp8 chunk_size=256);
+# 16 MiB gives headroom.  Pool must cover c×chunks: 250×71=17,750 -> default 32768.
+# Parallel POSIX writers: default 1 GPU worker serializes all POSIX writes on ZMQ.
+LMCACHE_NIXL_POSIX_SLOT_SIZE="${LMCACHE_NIXL_POSIX_SLOT_SIZE:-16777216}"
+# Post-warmup drain sleep: time in seconds to wait after the per-c warmup wave
+# before starting timed iters.  Must be long enough for async POSIX writes to
+# land on disk so iter 0 sees a warm, unsaturated cache.  Default 300 s is sized
+# for c=32 per-client with workers=1; reduce for shared prefix or more workers.
+POST_WARMUP_SLEEP_S="${POST_WARMUP_SLEEP_S:-300}"
+LMCACHE_NIXL_POSIX_POOL="${LMCACHE_NIXL_POSIX_POOL:-32768}"
+LMCACHE_MAX_GPU_WORKERS="${LMCACHE_MAX_GPU_WORKERS:-1}"
+# Cap concurrent NIXL POSIX load tasks.  At high concurrency the NIXL descriptor pool
+# exhausts and makeXferReq throws NIXL_ERR_INVALID_PARAM; limiting in-flight prefetches
+# to 4 keeps the pool from saturating while still overlapping L2 reads.
+LMCACHE_L2_PREFETCH_MAX_IN_FLIGHT="${LMCACHE_L2_PREFETCH_MAX_IN_FLIGHT:-4}"
 # CPU DRAM (L1) tier for the nvme arm.  Default 'false' = KV flows GPU VRAM
 # staging -> NVMe (no host DRAM tier).  Set AIC_LOCAL_CPU=true to add an LMCache
 # DRAM L1 in front of the NIXL NVMe L2 (a proper DRAM->NVMe tiered cache);
@@ -407,8 +423,21 @@ command -v docker >/dev/null 2>&1 || die "docker not found on $(hostname)"
 
 # Pre-create the compose log-mount dirs too, so docker doesn't auto-create them
 # as root under the per-job logs tree (the lmcache/vllm services bind-mount these).
-mkdir -p "${RESULTS}" "${PLOTS}" "${NVME_DATA}" "${NFS_DATA}" "${GDS_SLAB_DATA}" \
-         "${LOGDIR}/lmcache" "${LOGDIR}/vllm"
+# The per-job log dirs are always under REPO (writable).  The NVMe/slab paths may
+# live under a shared LVM (e.g. SPUR /mnt/m2m_nobackup) owned by root -- try plain
+# mkdir first; fall back to sudo -n (passwordless) if that fails so the first job
+# on a freshly imaged node doesn't abort when the caller-pinned path doesn't exist.
+_mkdir_shared() {
+    local d
+    for d in "$@"; do
+        mkdir -p "${d}" 2>/dev/null && continue
+        sudo -n mkdir -p "${d}" 2>/dev/null \
+            && { sudo -n chown "$(id -u):$(id -g)" "${d}" 2>/dev/null || true; continue; }
+        log "WARN: could not create ${d} (mkdir failed; no passwordless sudo?)"
+    done
+}
+mkdir -p "${RESULTS}" "${PLOTS}" "${LOGDIR}/lmcache" "${LOGDIR}/vllm"
+_mkdir_shared "${NVME_DATA}" "${NFS_DATA}" "${GDS_SLAB_DATA}"
 [[ -d "${HF_HOME}/hub" ]] || die "HF cache not found at ${HF_HOME}/hub"
 
 # --- Load images from shared tarballs when needed ----------------------------
@@ -478,6 +507,7 @@ fi
 HF_ENV=(-e HF_HOME=/hf -e HF_HUB_CACHE=/hf/hub -e HF_HUB_OFFLINE=1
         -e TRANSFORMERS_OFFLINE=1 -e VLLM_CACHE_ROOT=/hf/vllm
         -e ROCR_VISIBLE_DEVICES=0 -e PYTHONHASHSEED=0
+        -e VLLM_TARGET_DEVICE=rocm
         -e AMDGCN_USE_BUFFER_OPS=0 -e MIOPEN_FIND_MODE=FAST
         # AITER unified attention: required so a KV connector (LMCache) can unify
         # gpt-oss's hybrid (sliding-window + full) KV-cache specs.  Without
@@ -514,6 +544,8 @@ export AIC_ROCM_USE_AITER
 export LMCACHE_PORT LMCACHE_NVME_POOL LMCACHE_NFS_POOL
 export LMCACHE_NVME_SLOT_SIZE="${LMCACHE_NVME_SLOT_SIZE:-268435456}"
 export AIC_NIXL_USE_DIRECT_IO
+export AIC_NIXL_BACKEND
+export LMCACHE_NIXL_POSIX_SLOT_SIZE LMCACHE_NIXL_POSIX_POOL LMCACHE_MAX_GPU_WORKERS LMCACHE_L2_PREFETCH_MAX_IN_FLIGHT
 export NIXL_METRICS_PORT="${NIXL_METRICS_PORT:-19090}"
 # RoPE override (YaRN etc.) applied to ALL arms via vLLM --hf-overrides.  Single-
 # quote the JSON so compose's shlex splitting keeps the inner double quotes.
@@ -692,7 +724,11 @@ run_arm_kvd() {  # $1=mode(nvme|gds) $2=out_csv
             _l2desc="LMCache native LocalDiskBackend L2 (POSIX, $([[ "${AIC_L2_ODIRECT}" == true ]] && echo O_DIRECT || echo buffered), DRAM L1 ${LMCACHE_L1_SIZE_GB}GB)"
         else
             unset LMCACHE_CONFIG_FILE LMCACHE_CONFIG_FILE_HOST
-            _l2desc="NIXL AIS_MT NVMe L2 + POSIX L2b (DRAM L1 ${LMCACHE_L1_SIZE_GB}GB, pool ${LMCACHE_NVME_POOL})"
+            if [[ "${AIC_NIXL_BACKEND}" == "POSIX" ]]; then
+                _l2desc="NIXL POSIX NVMe L2 (DRAM L1 ${LMCACHE_L1_SIZE_GB}GB, pool ${LMCACHE_NIXL_POSIX_POOL}, slot $(( LMCACHE_NIXL_POSIX_SLOT_SIZE / 1048576 ))MiB, workers=${LMCACHE_MAX_GPU_WORKERS})"
+            else
+                _l2desc="NIXL AIS_MT NVMe L2 + POSIX L2b (DRAM L1 ${LMCACHE_L1_SIZE_GB}GB, pool ${LMCACHE_NVME_POOL})"
+            fi
         fi
     fi
     log "  stack: standalone lmcache server (${_l2desc}) + vLLM LMCacheMPConnector on :${VLLM_ENDPOINT_PORT}"
@@ -711,8 +747,27 @@ run_arm_kvd() {  # $1=mode(nvme|gds) $2=out_csv
         compose logs --tail 60 --no-color vllm    2>&1 | sed 's/^/  [vllm]    /'
         stop_stack; return 1
     fi
+    # Continuous container log streaming: capture lmcache + vllm stdout into
+    # per-run log files with timestamps throughout the arm, not just at teardown.
+    # The background tail is killed by stop_stack via the containers being removed.
+    log "streaming container logs -> ${LOGDIR}/lmcache/ and ${LOGDIR}/vllm/"
+    mkdir -p "${LOGDIR}/lmcache" "${LOGDIR}/vllm"
+    docker logs -f --timestamps aic-lmcache \
+        >> "${LOGDIR}/lmcache/lmcache-stream.log" 2>&1 &
+    _lmcache_log_pid=$!
+    docker logs -f --timestamps aic-vllm-gpu0 \
+        >> "${LOGDIR}/vllm/vllm-stream.log" 2>&1 &
+    _vllm_log_pid=$!
+
     # Per-c warmup + drain: required for per_client mode and async-save kvd.
-    run_bench kvd_v2 "${VLLM_ENDPOINT_PORT}" "${out}" --warmup-at-each-c --post-warmup-sleep-s 5
+    # post-warmup-sleep must cover the full POSIX write queue drain time for the
+    # current concurrency.  With workers=1 (serialised) and c=32 per-client,
+    # warmup writes take ~450 s; 300 s gives the queue time to mostly drain so
+    # iter 0 sees a warm, settled cache rather than write-queue back-pressure.
+    # Override via POST_WARMUP_SLEEP_S env if a different c or worker count is used.
+    run_bench kvd_v2 "${VLLM_ENDPOINT_PORT}" "${out}" --warmup-at-each-c \
+        --post-warmup-sleep-s "${POST_WARMUP_SLEEP_S:-300}"
+    kill "${_lmcache_log_pid}" "${_vllm_log_pid}" 2>/dev/null || true
     local rc=$?
     stop_stack
     return $rc
@@ -742,7 +797,12 @@ log "isl/prefix  : ISL=${BENCH_ISL} shared=${BENCH_SHARED_TOK} max_model_len=${V
 # collapse in jobs 67536798/67537066 — the LMCACHE_NVME_POOL override didn't land).
 # In MP mode the DRAM L1 is --l1-size-gb (nvme arm: max_local_cpu_size when
 # local_cpu=true, else the default) and the slab is --l1-size-gb (gds arm).
-log "tiers       : l2_backend=${AIC_L2_BACKEND} local_cpu=${AIC_LOCAL_CPU} dram_l1(nvme)=${LMCACHE_MAX_LOCAL_CPU_SIZE}GB nvme_pool=${LMCACHE_NVME_POOL} slots (~$(( LMCACHE_NVME_POOL * 45 / 10 / 1024 )) GiB) gds_slab=${AIC_GDS_SLAB_GB}GB local_disk=${AIC_LOCAL_DISK_SIZE_GB}GB(odirect=${AIC_L2_ODIRECT})"
+if [[ "${AIC_NIXL_BACKEND}" == "POSIX" ]]; then
+    _nvme_pool_log="${LMCACHE_NIXL_POSIX_POOL} slots slot_size=$(( LMCACHE_NIXL_POSIX_SLOT_SIZE / 1048576 ))MiB workers=${LMCACHE_MAX_GPU_WORKERS}"
+else
+    _nvme_pool_log="${LMCACHE_NVME_POOL} slots (~$(( LMCACHE_NVME_POOL * 45 / 10 / 1024 )) GiB)"
+fi
+log "tiers       : l2_backend=${AIC_L2_BACKEND} nixl_backend=${AIC_NIXL_BACKEND} local_cpu=${AIC_LOCAL_CPU} dram_l1(nvme)=${LMCACHE_MAX_LOCAL_CPU_SIZE}GB nvme_pool=${_nvme_pool_log} gds_slab=${AIC_GDS_SLAB_GB}GB local_disk=${AIC_LOCAL_DISK_SIZE_GB}GB(odirect=${AIC_L2_ODIRECT})"
 
 if _arm_on vram; then run_arm_vram && STATUS[vram_only]=ok || STATUS[vram_only]=FAIL; else STATUS[vram_only]=skip; fi
 if _arm_on nvme; then run_arm_kvd nvme "${KVD_NVME_CSV}" && STATUS[kvd_nvme]=ok || STATUS[kvd_nvme]=FAIL; else STATUS[kvd_nvme]=skip; fi

@@ -16,3 +16,45 @@ Add this before any `srun` invocation, or it will fail with
 - **Partition:** `amd-spur`
 - **No `--account` flag needed** for `srun`
 - Node naming convention: `crsuse2-m2m-NNN`
+
+## ROCm 7.14 torch wheel integration — known issues (2026-07-20)
+
+The Dockerfile was updated to use AMD's native ROCm 7.14 wheel index
+(`repo.amd.com/rocm/whl-multi-arch/`) for PyTorch. Testing on the SPUR
+cluster (MI355X / gfx950) revealed a **blocking incompatibility**:
+
+**`torch 2.12.0+rocm7.14.0` ships Triton 3.7.1, which breaks vLLM 0.25.1.**
+vLLM's engine core fails to start on gfx950 nodes — Triton kernel calls
+fail because Triton is disabled at startup due to circular import side-effects
+in vLLM's platform detection code.
+
+### Quick unblock options
+
+1. **Use gfx942 (MI300X) nodes** — the Triton issue manifests on gfx950
+   (MI355X) specifically; gfx942 nodes have not been fully tested but the
+   hipErrorInvalidImage issue is resolved with `AIC_ROCM_ARCH=gfx942`.
+
+2. **Revert torch to pytorch.org** — change the Dockerfile torch install back
+   to `torch 2.13.0+rocm7.2` from `download.pytorch.org/whl/rocm7.2`.
+   That wheel ships a Triton version compatible with vLLM 0.25.1.
+
+3. **Upgrade vLLM** — newer vLLM versions (0.28+) may support Triton 3.7.
+
+### SPUR cluster cliff submission quirks
+
+- Pass `--gres=` (empty) to override the embedded `#SBATCH --gres=gpu:1` —
+  SPUR's scheduler has no GPU GRES configured, so the directive causes
+  immediate job cancellation.
+- Do NOT use `--no-requeue` — SPUR sbatch doesn't support it.
+- Submit with `--chdir=/shared_nfs/$USER` when `/home` is at quota.
+- Use `SLURM_SUBMIT_DIR=/path/to/repo` to tell the cliff script where the repo is.
+- Pin to a node with no pre-existing containers: `--nodelist=crsuse2-m2m-042`
+  (as of 2026-07-20 crsuse2-m2m-036 has a long-running primus container
+  that blocks GPU access for new containers).
+- All SPUR nodes are **MI355X (gfx950)**, not MI300X (gfx942).
+
+### Full analysis
+
+See `.docs-remove/rocm-7.14-torch-wheel-integration.md` for the complete
+11-bug chain discovered during SPUR testing, including root causes and fix
+status for each.