ci: stage TokenSpeed serving for nightly perf gating, with an opt-in docker socket #569

Workflow file for this run

.github/workflows/gpu-tests.yml at 765c9c0

	name: GPU Tests

	# Phase 2 GPU merge gate: run the GPU-runnable slice of the pytest suite on the
	# self-hosted MI350 runner inside a digest-pinned ROCm PyTorch container.
	# Complements cpu-tests.yml (`not gpu and not rocm`) with no overlap -- see
	# docs/ci-testing-plan.md and issue #268.
	#
	# `pytest (GPU, MI350)` is a required status check on `main`. Path filtering
	# therefore must NOT live on the `pull_request` trigger: a workflow skipped by
	# path filtering leaves its checks Pending forever rather than reporting, so any
	# PR outside those paths would be permanently unmergeable. A job skipped by a
	# conditional, by contrast, reports Success. So this workflow always starts and
	# the `changes` job below decides whether the GPU work is relevant.
	# See https://docs.github.com/en/pull-requests/how-tos/merge-and-close-pull-requests/troubleshooting-required-status-checks

	on:
	workflow_dispatch:
	schedule:
	# Nightly full GPU run (offset from nightly.yml at 11:00 UTC).
	- cron: "0 8 * * *"
	pull_request:

	# contents: read to check out; actions: read for artifact upload (matches the
	# other artifact-uploading workflows in this repo); pull-requests: read so the
	# `changes` gate can list the files a PR touches.
	permissions:
	contents: read
	actions: read
	pull-requests: read

	# Protect the single GPU runner: one workflow run at a time per ref; newer PR
	# pushes cancel the superseded run.
	concurrency:
	group: gpu-tests-${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	env:
	CONTAINER_NAME: aorta-ci-gpu

	jobs:
	# Cheap gate on a GitHub-hosted runner, replacing the trigger-level `paths:`
	# filter. Non-PR triggers always report true so scheduled and dispatched runs
	# exercise the full suite.
	changes:
	name: detect GPU-relevant changes
	runs-on: ubuntu-latest
	outputs:
	gpu: ${{ steps.filter.outputs.gpu }}
	steps:
	- name: Decide whether the GPU suite applies
	id: filter
	env:
	GH_TOKEN: ${{ github.token }}
	EVENT_NAME: ${{ github.event_name }}
	PR_NUMBER: ${{ github.event.pull_request.number }}
	run: \|
	set -euo pipefail

	if [ "$EVENT_NAME" != "pull_request" ]; then
	echo "$EVENT_NAME is not a pull request; running the full GPU suite."
	echo "gpu=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	# Fail open. This gate decides a required status check, so an
	# indeterminate answer must never be read as "no GPU work needed":
	# that would hand a GPU-touching PR a green check without testing it.
	# On any retrieval problem, claim relevance and run the suite.
	# Assigning inside `if !` keeps `set -e` from aborting on failure.
	if ! files="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \
	--paginate --jq '.[].filename' 2>&1)"; then
	echo "::warning::Could not list the PR's files, so running the GPU" \
	"suite rather than guessing. gh said: ${files}"
	echo "gpu=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	if [ -z "$files" ]; then
	echo "::warning::The PR file list came back empty, which should not" \
	"happen for a real PR; running the GPU suite."
	echo "gpu=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	# One-to-one with the `paths:` filter this replaced. Trailing `*` is a
	# `case` glob, which matches `/` too, so it covers whole subtrees.
	# Single-quoted and expanded via "${patterns[@]}" so the shell never
	# filename-expands them -- they must reach `case` as literal patterns.
	patterns=(
	'src/aorta/race/*'
	'src/aorta/ebpf/*'
	'src/aorta/hw_queue_eval/*'
	'src/aorta/workloads/*'
	'src/aorta/instrumentation/rocjitsu_sanitizers/*'
	'src/aorta/utils/gpu_control.py'
	'src/aorta/instrumentation/env_knobs.py'
	# The module that performs the capture. Its test file was listed
	# without it, so a PR touching only the probe ran neither the audit
	# nor the GPU-gated tests in that file.
	'src/aorta/instrumentation/environment.py'
	# The Proton collector attaches a profiler to a real GPU launch, so
	# its behaviour is only observable on hardware: the CPU suite can
	# assert the argv and the env bundle, but not that the capture came
	# back with kernels in it. Listed with its smoke test because a
	# backend pin that silently records nothing passes every CPU check.
	'src/aorta/instrumentation/proton/*'
	'scripts/audit_env_knobs.py'
	'tests/hw_queue_eval/*'
	'tests/instrumentation/test_environment.py'
	'tests/instrumentation/test_env_knob_audit.py'
	'tests/instrumentation/test_proton_smoke_gpu.py'
	# The Proton smoke test runs these payloads, so they are code the
	# GPU job covers: a PR that breaks pipeline.py or hotspot.py while
	# touching no collector or test file would otherwise skip the only
	# job that would have caught it.
	'examples/profiling/proton/*'
	'tests/instrumentation/rocjitsu_sanitizers/*'
	'tests/sanitizers/*'
	'tests/test_marker_partition.py'
	'config/ci/*'
	'recipes/ci/*'
	'recipes/sanitizers/*'
	'scripts/ci/*'
	'scripts/sanitizers/*'
	'docker/*'
	'.github/workflows/gpu-tests.yml'
	)

	gpu=false
	while read -r file; do
	[ -n "$file" ] \|\| continue
	for pattern in "${patterns[@]}"; do
	# shellcheck disable=SC2254 # unquoted glob match is intentional
	case "$file" in
	$pattern)
	echo "GPU-relevant change: $file (matched $pattern)"
	gpu=true
	break 2
	;;
	esac
	done
	done <<< "$files"

	if [ "$gpu" = false ]; then
	echo "No GPU-relevant paths changed; the GPU jobs will be skipped."
	echo "A skipped job reports Success, satisfying the required check."
	fi
	echo "gpu=$gpu" >> "$GITHUB_OUTPUT"

	gpu-tests:
	name: pytest (GPU, MI350)
	runs-on: [self-hosted, gpu]
	needs: changes
	# Skipped (and therefore reported as Success) when no GPU-relevant path
	# changed, which keeps the single MI350 runner free on docs-only PRs.
	#
	# Security: never run untrusted fork-PR code on the self-hosted GPU runner.
	# Only same-repo PRs (plus schedule / dispatch) reach the runner. Fork PRs
	# also lack the ROCM_SHARED_KEY secret, so they could not authenticate anyway.
	if: >-
	needs.changes.outputs.gpu == 'true' &&
	(github.event_name != 'pull_request' \|\|
	github.event.pull_request.head.repo.full_name == github.repository)
	timeout-minutes: 60

	steps:
	# The CI container runs as root and writes root-owned build artifacts
	# (.egg-info / __pycache__ / .pytest_cache) into the mounted checkout.
	# actions/checkout's `git clean` then runs as the runner user and cannot
	# delete them, failing the checkout. Reclaim ownership first via a
	# throwaway root container (repo scripts aren't checked out yet here).
	- name: Reclaim workspace ownership
	run: \|
	ws="${{ github.workspace }}"
	[ -d "$ws" ] \|\| exit 0
	uidgid="$(id -u):$(id -g)"
	# Pinned by digest for supply-chain safety / reproducibility on the
	# self-hosted runner (busybox:1.37).
	busybox="busybox:1.37@sha256:9532d8c39891ca2ecde4d30d7710e01fb739c87a8b9299685c63704296b16028"
	# Repo scripts aren't checked out yet, so resolve docker access inline.
	# Support runners with direct docker, sudo docker, or only host sudo.
	if docker info >/dev/null 2>&1; then
	docker run --rm -v "$ws":/ws "$busybox" chown -R "$uidgid" /ws \|\| true
	elif sudo -n docker info >/dev/null 2>&1; then
	sudo docker run --rm -v "$ws":/ws "$busybox" chown -R "$uidgid" /ws \|\| true
	else
	sudo chown -R "$uidgid" "$ws" \|\| true
	fi

	- name: Check out code
	uses: actions/checkout@v5

	- name: Set up ROCm CI container
	uses: ./.github/actions/rocm-ci-setup
	with:
	rocm-shared-key: ${{ secrets.ROCM_SHARED_KEY }}

	# -n 4 (not -n auto): the runner has one target GPU, so one xdist worker
	# per CPU core would spin up dozens of HIP contexts contending for a
	# single device (OOM / flakiness). A small bounded pool keeps parallelism
	# without oversubscribing the GPU.
	#
	# --timeout=300 (pytest-timeout, already in the `tests` extra) bounds a
	# wedged test rather than the job. A GPU test that hangs instead of
	# failing costs far more than one red test: the job runs to its
	# 60-minute cap, and because a GitHub-side cancel skips the junit upload,
	# the gate reports NO test results at all -- so any real failure in the
	# same run is destroyed along with it. Observed on run 33527313950, where
	# 105 of 108 tests reported in 54 s, a profiler payload then wedged for
	# 59 minutes, and the one genuine failure went out with the cancel.
	#
	# Sized from measurement: on run 33606698270 the whole suite was 104
	# tests in 51.6 s wall, and its slowest single test -- the rocprofv3
	# sweep -- took 20.2 s. 300 s is ~15x that, and more than the entire
	# suite has ever needed. Set here rather than in
	# pytest.ini because the opt-in operator payloads (AORTA_LN_GPU_SMOKE)
	# legitimately run for tens of minutes and must keep their own budgets;
	# only this job is being bounded.
	- name: Install package and run GPU test suite
	run: \|
	bash scripts/ci/docker_cmd.sh exec "${{ env.CONTAINER_NAME }}" bash -lc '
	set -euo pipefail
	cd /workspace/aorta
	python -m pip install --upgrade pip
	pip install -e ".[tests,hw-queue]"
	# The ROCm lib dirs on LD_LIBRARY_PATH, or the payloads these tests
	# compile with hipcc build and then fail to LAUNCH. Measured on the
	# ROCm 10 wheel base: hipcc output carries neither DT_RPATH nor
	# DT_RUNPATH and the image sets no LD_LIBRARY_PATH, so the loader has
	# nowhere to look and the payload dies before main with
	# error while loading shared libraries: libamdhip64.so.7
	# at exit 127 -- which is what test_payload_runs_and_self_checks
	# reported. Its sibling test runs the SAME binary under rocprofv3 and
	# passed, so this is the bare-launch loader path and nothing else.
	#
	# Byte-identical to both payloads in sanitizers-nightly.yml and to
	# _ROCM_LIB_PATH_EXPORT in gen_sanitizer_dashboard.py; the four are
	# asserted equal by test_rebuild_commands_export_the_rocm_library_path.
	# See the gate job in that workflow for the full rationale -- both lib
	# dirs with core first, derived from the resolver rather than a
	# literal site-packages path (issue #381), appended rather than
	# prepended so an inherited operator substitution keeps winning, and
	# written with no apostrophe because this is a single-quoted bash -lc
	# payload that any apostrophe would close early.
	export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}$(python -c "import os; from aorta.instrumentation.rocm_paths import resolve_rocm_roots as r; x = r(); print(os.pathsep.join(dict.fromkeys([str(x.core_lib_dir), str(x.lib_dir)])))")"
	pytest -m "gpu or rocm" -n 4 --timeout=300 --junitxml=gpu-results.xml
	'

	# Separate step, after pytest: under `set -euo pipefail` a knob the
	# registry omits (a base-image bump is enough) aborted the script before
	# pytest ran, so the GPU gate went red with zero test results and no
	# junit artifact. Coverage drift must fail the job without destroying the
	# test signal.
	- name: Audit env-knob registry coverage
	if: always()
	run: \|
	bash scripts/ci/docker_cmd.sh exec "${{ env.CONTAINER_NAME }}" bash -lc '
	set -euo pipefail
	cd /workspace/aorta
	# --rocm-lib deliberately OMITTED (issue #381). Omitting it IS the
	# resolver path: audit_env_knobs.py defaults to None and asks
	# default_rocm_lib() for the layout-appropriate directory. Pinning
	# /opt/rocm/lib here left that default with no CI coverage at all --
	# green today on the classic base, an immediate exit 2 at the
	# deferred wheel-layout flip (#383), and the failure would have read
	# as "the audit broke" rather than "the workflow pinned a path the
	# resolver was supposed to supply". This job is now the regression
	# test for default_rocm_lib(). Pass --rocm-lib only to audit a tree
	# that is NOT the resolved install.
	python scripts/audit_env_knobs.py --strict
	'

	- name: Upload GPU test results
	if: always()
	uses: actions/upload-artifact@v6
	with:
	name: gpu-test-results-${{ github.run_id }}
	path: gpu-results.xml
	if-no-files-found: ignore

	- name: Tear down ROCm CI container
	if: always()
	working-directory: docker
	run: bash ../scripts/ci/docker_compose.sh --env-file .env.ci -f docker-compose.build.yaml down -v

	gpu-regression:
	name: workload regression (GPU, MI350)
	runs-on: [self-hosted, gpu]
	needs: gpu-tests
	# Skipped along with gpu-tests when no GPU-relevant path changed, since a
	# dependency that did not succeed skips this job too. Otherwise PRs run the
	# fast, single-GPU "pr" tier and nightly / dispatch run the full manifest
	# (AORTA_CI_TIER below). Same fork-PR guard as gpu-tests: no untrusted fork
	# code on the self-hosted runner.
	if: >-
	github.event_name != 'pull_request' \|\|
	github.event.pull_request.head.repo.full_name == github.repository
	timeout-minutes: 120

	steps:
	# The CI container runs as root and writes root-owned build artifacts
	# (.egg-info / __pycache__ / .pytest_cache) into the mounted checkout.
	# actions/checkout's `git clean` then runs as the runner user and cannot
	# delete them, failing the checkout. Reclaim ownership first via a
	# throwaway root container (repo scripts aren't checked out yet here).
	- name: Reclaim workspace ownership
	run: \|
	ws="${{ github.workspace }}"
	[ -d "$ws" ] \|\| exit 0
	uidgid="$(id -u):$(id -g)"
	# Pinned by digest for supply-chain safety / reproducibility on the
	# self-hosted runner (busybox:1.37).
	busybox="busybox:1.37@sha256:9532d8c39891ca2ecde4d30d7710e01fb739c87a8b9299685c63704296b16028"
	# Repo scripts aren't checked out yet, so resolve docker access inline.
	# Support runners with direct docker, sudo docker, or only host sudo.
	if docker info >/dev/null 2>&1; then
	docker run --rm -v "$ws":/ws "$busybox" chown -R "$uidgid" /ws \|\| true
	elif sudo -n docker info >/dev/null 2>&1; then
	sudo docker run --rm -v "$ws":/ws "$busybox" chown -R "$uidgid" /ws \|\| true
	else
	sudo chown -R "$uidgid" "$ws" \|\| true
	fi

	- name: Check out code
	uses: actions/checkout@v5

	- name: Set up ROCm CI container
	uses: ./.github/actions/rocm-ci-setup
	with:
	rocm-shared-key: ${{ secrets.ROCM_SHARED_KEY }}

	# PR gate runs the fast single-GPU subset; nightly / dispatch run everything.
	- name: Select regression tier
	id: tier
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	echo "value=pr" >> "$GITHUB_OUTPUT"
	else
	echo "value=full" >> "$GITHUB_OUTPUT"
	fi

	- name: Install package and run workload regression smokes
	run: \|
	bash scripts/ci/docker_cmd.sh exec \
	-e AORTA_CI_TIER=${{ steps.tier.outputs.value }} \
	"${{ env.CONTAINER_NAME }}" bash -lc '
	set -euo pipefail
	cd /workspace/aorta
	python -m pip install --upgrade pip
	pip install -e ".[tests,hw-queue]"
	bash scripts/ci/run_gpu_regression_smokes.sh
	'

	- name: Tear down ROCm CI container
	if: always()
	working-directory: docker
	run: bash ../scripts/ci/docker_compose.sh --env-file .env.ci -f docker-compose.build.yaml down -v

Navigation Menu

Uh oh!

ci: stage TokenSpeed serving for nightly perf gating, with an opt-in docker socket #569

Workflow file

ci: stage TokenSpeed serving for nightly perf gating, with an opt-in docker socket #569

Uh oh!

Workflow file for this run