ci: stage TokenSpeed serving for nightly perf gating, with an opt-in docker socket

.github/actions/rocm-ci-setup/action.yml

@@ -11,6 +11,16 @@ inputs:
       base image (public) is pulled anonymously.
     required: false
     default: ""
+  docker-socket:
+    description: >-
+      Mount the HOST docker daemon socket into the CI container, for workloads
+      that launch a container of their own (tokenspeed_serve). Off by default,
+      and it should stay off for every lane that does not need it: a container
+      that can reach the daemon can start a privileged one bind-mounting /, so
+      this is effectively root on the runner. Read the security note in
+      docker/docker-compose.docker-socket.yaml before setting it.
+    required: false
+    default: "false"
 
 runs:
   using: composite
@@ -37,9 +47,25 @@ runs:
           echo "::warning::Docker Hub login failed (ROCM_SHARED_KEY may be expired). Continuing with anonymous pulls."
         fi
 
+    # The socket override is a second `-f`, not an edit to the base compose, so
+    # a lane that does not ask for it gets a container with no route to the
+    # daemon -- the same posture as before this input existed. `up` needs the
+    # override; `build` does not, since it adds no build stage.
     - name: Build and start ROCm CI container
       shell: bash
       working-directory: docker
+      env:
+        # NOT `DOCKER_SOCKET`: the override reads that name as the socket's host
+        # PATH, so a true/false flag under it would be substituted as the bind
+        # source and compose would mount a directory called `true`.
+        MOUNT_DOCKER_SOCKET: ${{ inputs.docker-socket }}
       run: |
+        files="-f docker-compose.build.yaml"
+        if [ "${MOUNT_DOCKER_SOCKET}" = "true" ]; then
+          echo "::warning::Mounting the host docker socket into ${CONTAINER_NAME:-the CI container}: this grants effective root on the runner."
+          files="${files} -f docker-compose.docker-socket.yaml"
+        fi
+        # shellcheck disable=SC2086
         bash ../scripts/ci/docker_compose.sh --env-file .env.ci -f docker-compose.build.yaml build
-        bash ../scripts/ci/docker_compose.sh --env-file .env.ci -f docker-compose.build.yaml up -d
+        # shellcheck disable=SC2086
+        bash ../scripts/ci/docker_compose.sh --env-file .env.ci ${files} up -d

.github/workflows/bump-validate.yml

@@ -29,4 +29,11 @@ jobs:
     with:
       ref: ${{ github.event.pull_request.head.sha }}
       do_alert: false      # PRs must not file/close the nightly regression issue
+      # Explicitly off, not merely defaulted off. This lane runs PR head code on
+      # the self-hosted runner, so it must have no route to the host docker
+      # daemon -- a container that can reach the daemon can start a privileged
+      # one bind-mounting /. Stating it here means a future change to the
+      # input's default cannot silently arm the socket on a PR-triggered lane.
+      # The tokenspeed_serve_smoke entry reports `skip` without it, by design.
+      docker_socket: false
     secrets: inherit

.github/workflows/eval-reusable.yml

@@ -24,6 +24,21 @@ on:
         type: boolean
         required: false
         default: false
+      # MUST stay false for every PR-triggered lane (bump-validate.yml), and for
+      # any lane that does not run a workload needing its own container.
+      # Mounting the host daemon socket into the CI container is effectively root
+      # on the runner: a container that can reach the daemon can start a
+      # privileged one bind-mounting /. On a PR-triggered lane that is reachable
+      # by anyone who can open a PR that touches the trigger paths, so the socket
+      # is opt-in per caller and off by default -- deliberately NOT set here in
+      # the shared reusable workflow, where it would apply to every caller at
+      # once. Only nightly-eval.yml sets it true. See the security note in
+      # docker/docker-compose.docker-socket.yaml.
+      docker_socket:
+        description: "mount the host docker socket (nightly eval lane only)"
+        type: boolean
+        required: false
+        default: false
 
 env:
   CONTAINER_NAME: aorta-ci-gpu
@@ -72,6 +87,7 @@ jobs:
         uses: ./.github/actions/rocm-ci-setup
         with:
           rocm-shared-key: ${{ secrets.ROCM_SHARED_KEY }}
+          docker-socket: ${{ inputs.docker_socket }}
 
       - name: Install wheel and run evaluation
         env:

.github/workflows/nightly-eval.yml

@@ -35,6 +35,15 @@ jobs:
       ref: ${{ github.event.workflow_run.head_sha || github.sha }}
       wheel_run_id: ${{ github.event.workflow_run.id || '' }}
       do_alert: true
+      # The nightly is the ONLY lane that gets the host docker socket: the
+      # tokenspeed_serve_smoke entry launches the TokenSpeed container itself, so
+      # it needs a route to the daemon. This lane is not PR-triggered -- it runs
+      # on a published nightly wheel or an explicit dispatch -- which is what
+      # makes the grant acceptable. Do not copy this line into bump-validate.yml.
+      # Until the socket is signed off, leaving this false is also safe: the
+      # entry declares `needs_docker_daemon: true` and reports `skip` with the
+      # reason recorded rather than failing the nightly.
+      docker_socket: true
     secrets: inherit
 
   # Append today's results to the ci-results data branch. The dashboard itself is

.github/workflows/refresh-baselines.yml

@@ -21,6 +21,20 @@ on:
         type: boolean
         required: false
         default: false
+      # The scoping guard for a perf refresh. An unscoped `--perf-gate` rewrites
+      # the whole baseline file and arms step-time ceilings on every matrix
+      # entry -- gpu_smoke, inference_offline, training_ddp, training_fsdp, race
+      # and llm_determinism included -- each from a single observation with no
+      # variance data behind it. Naming the entry under test keeps the refresh
+      # correctness-only everywhere else, so perf gating can be rolled out one
+      # workload at a time. Unknown names are rejected by refresh_baselines.py
+      # rather than silently scoping the gate to nothing.
+      perf_gate_entry:
+        description: >-
+          Restrict perf_gate to these matrix entries (comma- or space-separated,
+          e.g. "tokenspeed_serve_smoke"). Empty gates EVERY entry.
+        required: false
+        default: ""
 
 permissions:
   contents: write
@@ -58,26 +72,81 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v5
 
+      # refresh_baselines.py rejects both of these too, but only after pip has
+      # installed the wheel inside the container -- minutes of GPU runner time to
+      # learn about a typo in a dispatch form. Same rules, checked in seconds.
+      - name: Validate perf-gate inputs
+        env:
+          PERF_GATE: ${{ inputs.perf_gate }}
+          PERF_GATE_ENTRY: ${{ inputs.perf_gate_entry }}
+        run: |
+          set -euo pipefail
+          if [ -n "${PERF_GATE_ENTRY}" ] && [ "${PERF_GATE}" != "true" ]; then
+            echo "::error::perf_gate_entry has no effect without perf_gate: true"
+            exit 1
+          fi
+          # Deliberately a warning, not an error: an unscoped perf refresh is a
+          # legitimate end-state operation (bless everything once every workload
+          # has variance data), and it is the behaviour every existing dispatch
+          # of this workflow has had. Loud, not blocked.
+          if [ "${PERF_GATE}" = "true" ] && [ -z "${PERF_GATE_ENTRY}" ]; then
+            echo "::warning::Unscoped perf refresh: this arms step-time ceilings on EVERY matrix entry from a single observation each. Set perf_gate_entry to scope it to the workload under test."
+          fi
+
       - name: Set up ROCm CI container
         uses: ./.github/actions/rocm-ci-setup
         with:
           rocm-shared-key: ${{ secrets.ROCM_SHARED_KEY }}
 
+      # Dispatch inputs reach the container as environment variables rather than
+      # being interpolated into this shell string. `perf_gate_entry` is free
+      # text, and `${{ }}` substitution happens before bash sees the script, so
+      # interpolating it would let a dispatch value close the quote and run
+      # arbitrary commands on the runner.
       - name: Install nightly wheel and regenerate baselines
+        env:
+          STEP_TIME_MARGIN: ${{ inputs.step_time_margin }}
+          THROUGHPUT_MARGIN: ${{ inputs.throughput_margin }}
+          PERF_GATE: ${{ inputs.perf_gate }}
+          PERF_GATE_ENTRY: ${{ inputs.perf_gate_entry }}
         run: |
-          bash scripts/ci/docker_cmd.sh exec "${{ env.CONTAINER_NAME }}" bash -lc '
+          bash scripts/ci/docker_cmd.sh exec \
+            -e STEP_TIME_MARGIN="${STEP_TIME_MARGIN}" \
+            -e THROUGHPUT_MARGIN="${THROUGHPUT_MARGIN}" \
+            -e PERF_GATE="${PERF_GATE}" \
+            -e PERF_GATE_ENTRY="${PERF_GATE_ENTRY}" \
+            -e DEV_WHEELS_INDEX="${{ env.DEV_WHEELS_INDEX }}" \
+            "${{ env.CONTAINER_NAME }}" bash -lc '
             set -euo pipefail
             cd /workspace/aorta
             python -m pip install --upgrade pip
             # Install under the same ci-constraints.txt as nightly-eval,
             # so baselines are generated on the exact dependency set they gate.
             constraint=""
             [ -f config/ci/ci-constraints.txt ] && constraint="-c config/ci/ci-constraints.txt"
-            pip install --pre "amd-aorta[hw-queue]" -f "${{ env.DEV_WHEELS_INDEX }}" $constraint
+            pip install --pre "amd-aorta[hw-queue]" -f "$DEV_WHEELS_INDEX" $constraint
+            # NOTE: this whole script is inside a single-quoted bash -lc, so it
+            # must contain no apostrophes.
+            args=()
+            if [ "${PERF_GATE}" = "true" ]; then
+              args+=(--perf-gate)
+            fi
+            # Comma- or space-separated list -> one --perf-gate-entry per name,
+            # which is what the repeatable flag expects. Empty fields (from a
+            # trailing comma) are dropped rather than passed as an empty string,
+            # which the known-entry check in refresh_baselines.py would reject.
+            if [ -n "${PERF_GATE_ENTRY}" ]; then
+              IFS=", " read -r -a _entries <<< "${PERF_GATE_ENTRY}"
+              for _e in "${_entries[@]}"; do
+                if [ -n "${_e}" ]; then
+                  args+=(--perf-gate-entry "${_e}")
+                fi
+              done
+            fi
             python scripts/ci/refresh_baselines.py \
-              --step-time-margin "${{ github.event.inputs.step_time_margin || 0.25 }}" \
-              --throughput-margin "${{ github.event.inputs.throughput_margin || 0.15 }}" \
-              ${{ github.event.inputs.perf_gate == 'true' && '--perf-gate' || '' }}
+              --step-time-margin "${STEP_TIME_MARGIN:-0.25}" \
+              --throughput-margin "${THROUGHPUT_MARGIN:-0.15}" \
+              ${args[@]+"${args[@]}"}
           '
 
       # The container wrote regression_baselines.yaml into the mounted workspace;

config/ci/nightly_eval_matrix.yaml

@@ -10,6 +10,21 @@
 #   nproc     - optional; launch under `torchrun --standalone --nproc_per_node=<n>`
 #   min_gpus  - optional; skip when torch.cuda.device_count() is lower (default 1;
 #               defaults to nproc when nproc is set)
+#   timeout_sec - optional; per-entry wall-clock budget (default 1800). Exceeding
+#               it kills the whole process group and FAILS the entry.
+#   needs_docker_daemon - optional; skip when no Docker daemon is reachable from
+#               wherever nightly_eval.py is running. For workloads that start
+#               their engine in a sibling container. Like min_gpus this states a
+#               runner CAPABILITY, so its absence is a skip, not a failure --
+#               and only entries that set it consult the probe, so a broken
+#               daemon cannot turn the rest of the nightly into skips.
+#
+# Record-only is NOT a field here: a cell is record-only exactly while
+# config/ci/regression_baselines.yaml has no key for it. Adding an entry to
+# `entries` therefore starts it record-only by construction. Note the limit of
+# that -- record-only defers PERFORMANCE gating only. A cell that errors or
+# fails is a `fail` with or without a baseline (nightly_eval.py is fail-closed),
+# so an entry may only be added here once it can actually pass on the runner.
 #
 # Phase 1 wires one config per registered workload (the smoke recipes). Per-workload
 # matrices (dtypes, larger GPU counts) are Phase 3 -- add rows here, no code change.
@@ -79,6 +94,61 @@ entries:
     nproc: 8
     min_gpus: 8
 
+  # TokenSpeed online serving. Chosen over the other four serving recipes as the
+  # cheapest cell that still answers something: Qwen3-0.6B is ~1.2 GB of weights
+  # against ~40 GB for the gpt-oss recipes, and two cells against four (models)
+  # or six (load). It is also the only serving recipe whose cells differ ONLY by
+  # mitigation, so the pair is a same-night control: a move in both cells is the
+  # stack, a move in one is the mitigation or noise. That is the triage lever the
+  # multi-model and load recipes cannot give, since their cells legitimately
+  # differ in speed.
+  #
+  # 3600s, not the 1800s default: two bring-ups at the observed startup spread
+  # (180-415 s measured over 13 cell-runs) plus the teardown VRAM drain is ~15
+  # min before the image pull and any cold-cache weight download, and a timeout
+  # is an unconditional entry FAILURE rather than a slow record.
+  #
+  # `needs_docker_daemon` is what makes this row safe to have here today. The
+  # nightly runs nightly_eval.py INSIDE aorta-ci-gpu, and `tokenspeed_serve`
+  # starts the TokenSpeed engine in its own container, so it needs a docker
+  # client in the image (present, and proven: the built image carries 29.7.2 and
+  # no daemon) plus a route to a daemon (a per-lane opt-in that no lane sets,
+  # because it grants effective root on the runner and that is the CI owner's
+  # call). Until a lane sets `docker-socket: true` this entry SKIPS, with the
+  # reason in the results, rather than failing on "cannot connect to the Docker
+  # daemon" -- the same contract min_gpus gives the 8-GPU rows.
+  #
+  # Proven on gfx950 on 2026-09-02: two sweeps driven from inside the CI
+  # container over a mounted socket, four cell-runs, all passed, twelve clean
+  # steps (1108-1149 ms), metrics within 5% of the host-side envelope.
+  #
+  # The recipe's `work_dir: /tmp/ts-work-serve` must be mounted at the SAME
+  # string on both sides and be root-owned -- see the work_dir section of
+  # docs/tokenspeed-gating-rollout.md, which the demonstration made a good deal
+  # more specific.
+  - name: tokenspeed_serve_smoke
+    recipe: recipes/tokenspeed/tokenspeed-serve-bench-smoke.yaml
+    min_gpus: 1
+    timeout_sec: 3600
+    needs_docker_daemon: true
+
+# Entries that are ready to run but blocked on a runner prerequisite. This key
+# is not read by anything: nightly_eval.py and refresh_baselines.py both iterate
+# `entries` only, so nothing below executes. It exists because the alternative
+# for a staged-but-blocked entry is a commented-out block (the convention the
+# two NOTEs above use), and a comment cannot be checked -- tests/ci/
+# test_nightly_eval.py validates these the same way it validates `entries`, so
+# the recipe path, the GPU requirement and the metric names are known-good on
+# the day the prerequisite lands rather than discovered by a red nightly.
+#
+# Promote by moving the entry (minus `blocked_on`) into `entries` above.
+# Empty as of 2026-09-02: tokenspeed_serve_smoke, the only entry that was ever
+# staged here, was promoted into `entries` once a cell was demonstrated running
+# from inside the CI container. Keep the key and this comment -- the convention
+# is worth more than the one row was, and the next blocked entry should land
+# here rather than in a comment block.
+pending_entries: []
+
 # dtype axis (fp32/fp16/bf16) is a follow-up: the smoke recipes pin one dtype
 # each, so per-dtype coverage needs dtype recipe variants (or workload_config
 # overrides). Add them as new rows here once those recipes exist -- no code change.