UAT - AWS #166
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: UAT - AWS | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| skip_delete: | |
| description: 'Skip cluster teardown (manual runs only; schedule always deletes)' | |
| type: boolean | |
| default: false | |
| skip_tests: | |
| description: 'Skip UAT test execution (manual runs only; schedule always tests)' | |
| type: boolean | |
| default: false | |
| schedule: | |
| - cron: '0 4 * * *' # daily at midnight PST, runs on default branch only | |
| permissions: | |
| contents: read | |
| # Long-running cloud provisioning — never cancel mid-teardown. | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: false | |
| jobs: | |
| uat-aws: | |
| if: github.repository == 'nvidia/aicr' | |
| runs-on: ubuntu-latest | |
| # Covers the uncapped non-UAT steps (build, validator image push, EKS | |
| # provisioning, evidence upload) + the UAT phase steps (prep 15 + install 45 | |
| # + validate 40 + train 25 + verify 5 = 130) + the always()-run Destroy | |
| # Cluster teardown. The teardown MUST fit in this budget: a job-level timeout | |
| # cancels pending always() steps, so an undersized cap would skip teardown | |
| # and leak the GPU node / capacity reservation. | |
| timeout-minutes: 180 | |
| permissions: | |
| contents: read | |
| actions: read | |
| id-token: write | |
| packages: write | |
| outputs: | |
| # Digest-pinned OCI ref of the signed evidence bundle, consumed by | |
| # the ingest-evidence job below. Empty when no bundle was produced. | |
| bundle_ref: ${{ steps.evidence_ref.outputs.ref }} | |
| env: | |
| AWS_ACCOUNT_ID: "615299774277" | |
| AWS_REGION: "us-east-1" | |
| GITHUB_ACTIONS_ROLE_NAME: "github-actions-role-aicr" | |
| DEPLOYMENT_ID: "aicr-uat-${{ github.run_id }}" | |
| CLUSTER_CONFIG: tests/uat/aws/cluster-config.yaml | |
| TEST_CONFIG: tests/uat/aws/tests/h100-training-config.yaml | |
| # v0.4.23 — required for the .eks-nested cluster-config schema; the | |
| # prior pin (v0.2.6) read .compute.nodeGroups.* directly and failed | |
| # Terraform parse with "object with 1 attribute eks". | |
| EKS_IMAGE: "ghcr.io/mchmarny/cluster/eks@sha256:57fd2bf77055f8e0090bd7b8e099e7f11fd874f0b5764057439d11f1cd32a8b1" | |
| VALIDATOR_IMAGE_PREFIX: "ghcr.io/nvidia/aicr-validators" | |
| VALIDATOR_TAG: "uat-${{ github.run_id }}" | |
| steps: | |
| # Checkout | |
| - name: Checkout | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| # Versions | |
| - name: Load versions | |
| id: versions | |
| uses: ./.github/actions/load-versions | |
| # Build from source | |
| - name: Setup Go | |
| uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version: '${{ steps.versions.outputs.go }}' | |
| cache: true | |
| cache-dependency-path: | | |
| go.sum | |
| vendor/modules.txt | |
| - name: Build aicr binary | |
| if: inputs.skip_tests != true | |
| env: | |
| GOFLAGS: -mod=vendor | |
| run: | | |
| go build -o ./aicr ./cmd/aicr | |
| ./aicr --version | |
| - name: Authenticate to GHCR | |
| uses: ./.github/actions/ghcr-login | |
| - name: Build and push validator images | |
| if: inputs.skip_tests != true | |
| run: | | |
| GO_VERSION="$(cat .go-version)" | |
| # CHAINSAW_* args dropped in #1236 — the deployment validator | |
| # no longer ships /usr/local/bin/chainsaw; it executes Chainsaw | |
| # Test format in-process via validators/chainsaw/inprocess.go. | |
| for phase in deployment performance conformance; do | |
| IMAGE="${VALIDATOR_IMAGE_PREFIX}/${phase}:${VALIDATOR_TAG}" | |
| docker build -f "validators/${phase}/Dockerfile" \ | |
| --build-arg "GO_VERSION=${GO_VERSION}" \ | |
| -t "${IMAGE}" . | |
| docker push "${IMAGE}" | |
| done | |
| # Auth | |
| - name: Configure AWS credentials | |
| id: auth | |
| uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1 | |
| with: | |
| role-to-assume: "arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/${{ env.GITHUB_ACTIONS_ROLE_NAME }}" | |
| aws-region: ${{ env.AWS_REGION }} | |
| role-session-name: GitHubActions-UAT-AICR | |
| # Capacity check (reads desired count and reservation ID from cluster config). | |
| # Uses the runner-preinstalled yq because Install tools runs later; if the | |
| # cluster-config schema ever drifts, the explicit empty-value guard below | |
| # surfaces the mismatch immediately instead of letting yq return empty | |
| # silently and the AWS API reject a malformed reservation ID. | |
| - name: Check EC2 capacity reservation | |
| id: capacity | |
| run: | | |
| set -euo pipefail | |
| GPU_WORKER='.compute.eks.nodeGroups.workers[] | select(.name == "gpu-worker")' | |
| DESIRED=$(yq eval "${GPU_WORKER} | .capacity.desired" "$CLUSTER_CONFIG") | |
| RESERVATION=$(yq eval "${GPU_WORKER} | .capacity.reservation.target" "$CLUSTER_CONFIG") | |
| if [[ -z "${DESIRED}" || "${DESIRED}" == "null" || \ | |
| -z "${RESERVATION}" || "${RESERVATION}" == "null" ]]; then | |
| echo "::error::Could not extract gpu-worker capacity from ${CLUSTER_CONFIG}." \ | |
| "Check the .compute.eks.nodeGroups.workers schema." \ | |
| "yq returned: desired='${DESIRED}', reservation='${RESERVATION}'" | |
| exit 1 | |
| fi | |
| echo "Reservation: ${RESERVATION}, desired: ${DESIRED}" | |
| AVAILABLE=$(aws ec2 describe-capacity-reservations \ | |
| --capacity-reservation-ids "${RESERVATION}" \ | |
| --query 'CapacityReservations[0].AvailableInstanceCount' \ | |
| --output text) | |
| echo "available=${AVAILABLE}" >> "$GITHUB_OUTPUT" | |
| echo "Available instances in reservation ${RESERVATION}: ${AVAILABLE}" | |
| if [[ "${AVAILABLE}" -lt "${DESIRED}" ]]; then | |
| echo "::error::Insufficient capacity: need ${DESIRED} instances, only ${AVAILABLE} available in reservation ${RESERVATION}" | |
| exit 1 | |
| fi | |
| # Deps | |
| - name: Install tools | |
| id: deps | |
| uses: ./.github/actions/setup-build-tools | |
| with: | |
| install_kubectl: 'true' | |
| kubectl_version: '${{ steps.versions.outputs.kubectl }}' | |
| install_yq: 'true' | |
| yq_version: '${{ steps.versions.outputs.yq }}' | |
| install_helm: 'true' | |
| helm_version: '${{ steps.versions.outputs.helm }}' | |
| install_helmfile: 'true' | |
| helmfile_version: '${{ steps.versions.outputs.helmfile }}' | |
| helmfile_sha256: '${{ steps.versions.outputs.helmfile_sha256_linux_amd64 }}' | |
| install_ko: 'true' | |
| ko_version: ${{ steps.versions.outputs.ko }} | |
| # The snapshot agent runs in-cluster as its own image, separate from the | |
| # locally-built aicr binary that drives the UAT phases. Left at the | |
| # default (ghcr.io/nvidia/aicr:latest), it would be the last *released* | |
| # build — so any artifact-schema change in the PR under test (e.g. an | |
| # apiVersion bump) produces a snapshot the freshly-built binary rejects. | |
| # Build the agent from the PR source and point AICR_IMAGE at it so the | |
| # snapshot is stamped by the code under test, not a stale release. Single | |
| # arch: the GPU worker (p5.48xlarge) is amd64. Lives in the aicr-validators | |
| # namespace (not the release ghcr.io/nvidia/aicr repo); cleaned up below. | |
| - name: Build and push snapshot-agent image | |
| if: inputs.skip_tests != true | |
| env: | |
| GOFLAGS: -mod=vendor | |
| KO_DOCKER_REPO: ghcr.io/nvidia/aicr-validators/agent | |
| run: | | |
| set -euo pipefail | |
| ko build ./cmd/aicr \ | |
| --bare \ | |
| --platform=linux/amd64 \ | |
| --image-label="org.opencontainers.image.revision=${{ github.sha }}" \ | |
| --tags="${VALIDATOR_TAG}" | |
| # Config — v0.4.23+ of the EKS actuator derives the cluster name from | |
| # .deployment.id (same as GKE); the previous yq write to .cluster.name | |
| # is now an invalid sibling of the typed cluster.eks ProviderConfig | |
| # and rejected at YAML unmarshal. | |
| - name: Update Config | |
| id: config | |
| run: | | |
| set -euox pipefail | |
| yq -i '.deployment.id = strenv(DEPLOYMENT_ID)' "$CLUSTER_CONFIG" | |
| cat "$CLUSTER_CONFIG" | |
| # Bringup | |
| - name: Bringup Infra | |
| id: infra | |
| run: | | |
| set -euox pipefail | |
| docker run \ | |
| -e CONFIG_CONTENT="$(base64 < "$CLUSTER_CONFIG")" \ | |
| -e AUTO_APPROVE=true \ | |
| -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ | |
| -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ | |
| -e AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \ | |
| "${EKS_IMAGE}" apply | |
| # Connect | |
| - name: Connect to Cluster | |
| id: client | |
| if: steps.infra.outcome == 'success' | |
| shell: bash | |
| run: | | |
| set -euox pipefail | |
| echo "Updating kubeconfig..." | |
| aws eks update-kubeconfig --region "${AWS_REGION}" --name "${DEPLOYMENT_ID}" | |
| echo "Verifying cluster connection..." | |
| kubectl get nodes | |
| # UAT phases — each step invokes tests/uat/aws/run so the demo | |
| # (demos/cuj1-training.md) and CI stay in lockstep, and per-phase | |
| # failures surface as discrete steps in the GitHub Actions UI. Phases | |
| # chain on success: prep → install → conformance → train → verify. | |
| # The signed evidence bundle is produced by the conformance step; the | |
| # subsequent train step exercises the deployed stack so the recorded | |
| # evidence reflects a fully successful run. | |
| - name: UAT - prep (snapshot + recipe + validate + bundle) | |
| id: prep | |
| if: steps.client.outcome == 'success' && inputs.skip_tests != true | |
| timeout-minutes: 15 | |
| shell: bash | |
| env: | |
| AICR_BIN: ${{ github.workspace }}/aicr | |
| RUN_ID: ${{ github.run_id }} | |
| # Snapshot agent = PR-built image (see "Build and push snapshot-agent | |
| # image"), not the stale released default. Honored by `aicr snapshot`. | |
| AICR_IMAGE: ghcr.io/nvidia/aicr-validators/agent:${{ env.VALIDATOR_TAG }} | |
| run: ./tests/uat/aws/run prep "${TEST_CONFIG}" | |
| - name: UAT - install (helmfile apply) | |
| id: install | |
| if: steps.prep.outcome == 'success' | |
| # helmfile apply (up to HELMFILE_TIMEOUT_SECONDS) + the post-install | |
| # readiness gate, which runs `aicr validate --phase deployment` until it | |
| # passes (up to READINESS_TIMEOUT_SECONDS, 20m, spanning a nodewright | |
| # reboot). Must exceed one helmfile attempt + the gate so the gate's own | |
| # fail-closed path runs before GitHub Actions kills the step. | |
| timeout-minutes: 45 | |
| shell: bash | |
| env: | |
| AICR_BIN: ${{ github.workspace }}/aicr | |
| RUN_ID: ${{ github.run_id }} | |
| run: ./tests/uat/aws/run install "${TEST_CONFIG}" | |
| # Runs ALL validate phases (deployment + conformance + performance), not | |
| # just conformance. The deployment phase polls component health checks | |
| # until the GPU stack converges — the readiness barrier the helmfile | |
| # bundle lacks (the helm deploy.sh `kubectl wait`s instead). Step id stays | |
| # `conformance` so the train/verify gating below is unaffected. | |
| - name: UAT - validate (all phases) + emit signed evidence | |
| id: conformance | |
| if: steps.install.outcome == 'success' | |
| # Larger budget than conformance-only: the deployment phase waits out | |
| # cold-start GPU bring-up (driver install/migration). (The performance | |
| # phase's NCCL benchmark skips on single-GPU-node clusters like this | |
| # one, so it is not the long pole here.) | |
| timeout-minutes: 40 | |
| shell: bash | |
| env: | |
| AICR_BIN: ${{ github.workspace }}/aicr | |
| RUN_ID: ${{ github.run_id }} | |
| run: ./tests/uat/aws/run conformance "${TEST_CONFIG}" | |
| - name: UAT - train (TrainJob) | |
| id: train | |
| if: steps.conformance.outcome == 'success' | |
| timeout-minutes: 25 | |
| shell: bash | |
| env: | |
| AICR_BIN: ${{ github.workspace }}/aicr | |
| RUN_ID: ${{ github.run_id }} | |
| run: ./tests/uat/aws/run train "${TEST_CONFIG}" | |
| - name: UAT - verify (evidence verify) | |
| id: verify | |
| if: steps.conformance.outcome == 'success' | |
| timeout-minutes: 5 | |
| shell: bash | |
| env: | |
| AICR_BIN: ${{ github.workspace }}/aicr | |
| RUN_ID: ${{ github.run_id }} | |
| # Pin the expected Sigstore signer to this workflow's OIDC | |
| # identity so verify only passes for bundles signed by this | |
| # workflow — without these, any Fulcio identity would verify | |
| # and the security check becomes a no-op. | |
| EXPECTED_ISSUER: https://token.actions.githubusercontent.com | |
| EXPECTED_IDENTITY_REGEXP: '^https://github\.com/NVIDIA/aicr/\.github/workflows/uat-aws\.yaml@refs/heads/.+$' | |
| run: ./tests/uat/aws/run verify "${TEST_CONFIG}" | |
| # Export the digest-pinned bundle ref for the ingest-evidence job. | |
| # Reads the pointer the conformance step wrote; emits empty (job | |
| # skips) when no bundle/digest is present. | |
| - name: Export evidence bundle ref | |
| id: evidence_ref | |
| if: steps.conformance.outcome == 'success' | |
| uses: ./.github/actions/export-evidence-ref | |
| # Artifacts + summary run BEFORE teardown so a slow or failing | |
| # `Destroy Cluster` (which can take 10+ min) does not delay or | |
| # consume the job-timeout budget needed to persist evidence. | |
| # | |
| # The signed evidence bundle itself is pushed to OCI by the | |
| # conformance step (ghcr.io/nvidia/aicr-evidence/h100-eks-ubuntu-training-kubeflow:run-${{ | |
| # github.run_id }}) and is the source of truth; here we attach the | |
| # lightweight pointer so a reviewer can locate that bundle directly | |
| # from the Actions UI. Org policy clamps retention at 90d. | |
| - name: Upload evidence pointer | |
| if: always() && steps.conformance.outcome == 'success' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: uat-aws-evidence-pointer-${{ github.run_id }} | |
| path: evidence/pointer.yaml | |
| retention-days: 90 | |
| if-no-files-found: error | |
| - name: Upload failure debug | |
| if: failure() && steps.prep.outcome != 'skipped' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: uat-aws-debug-${{ github.run_id }} | |
| path: | | |
| snapshot.yaml | |
| recipe.yaml | |
| dry-run.json | |
| report.json | |
| evidence-result.json | |
| evidence/pointer.yaml | |
| train-logs/** | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| # Summary | |
| - name: Test Summary | |
| if: always() | |
| run: | | |
| { | |
| echo "## UAT Results (AWS)" | |
| echo "" | |
| echo "**Build:** \`${{ github.sha }}\` (branch: \`${{ github.ref_name }}\`)" | |
| echo "" | |
| echo "| Phase | Status |" | |
| echo "|-------|--------|" | |
| echo "| Capacity | ${{ steps.capacity.outcome }} |" | |
| echo "| Dependencies | ${{ steps.deps.outcome }} |" | |
| echo "| Config | ${{ steps.config.outcome }} |" | |
| echo "| Cluster | ${{ steps.infra.outcome }} |" | |
| echo "| Connection | ${{ steps.client.outcome }} |" | |
| echo "| Prep | ${{ steps.prep.outcome }} |" | |
| echo "| Install | ${{ steps.install.outcome }} |" | |
| echo "| Validate (all phases) | ${{ steps.conformance.outcome }} |" | |
| echo "| Train | ${{ steps.train.outcome }} |" | |
| echo "| Verify | ${{ steps.verify.outcome }} |" | |
| if [[ -n "${{ steps.evidence_ref.outputs.ref }}" ]]; then | |
| echo "" | |
| echo "**Evidence (OCI):** \`${{ steps.evidence_ref.outputs.ref }}\`" | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| # Cleanup (after artifacts so a failed cleanup does not lose evidence). | |
| - name: Cleanup validator image | |
| if: always() && inputs.skip_tests != true | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Delete UAT-tagged validator + snapshot-agent images to avoid GHCR clutter | |
| for phase in deployment performance conformance agent; do | |
| VERSION_ID=$(gh api \ | |
| "repos/${{ github.repository }}/packages/container/aicr-validators%2F${phase}/versions" \ | |
| --jq ".[] | select(.metadata.container.tags[] == \"${VALIDATOR_TAG}\") | .id" 2>/dev/null) || true | |
| if [[ -n "${VERSION_ID}" ]]; then | |
| gh api --method DELETE \ | |
| "repos/${{ github.repository }}/packages/container/aicr-validators%2F${phase}/versions/${VERSION_ID}" || true | |
| fi | |
| done | |
| # Re-assume the role immediately before teardown. The `Configure AWS | |
| # credentials` step at the top of the job mints a 1-hour session (the | |
| # action's default with no role-duration-seconds), but the UAT phases | |
| # (prep + install + conformance + train + verify) can run the full job | |
| # timeout before teardown, so by the time `Destroy Cluster` runs the | |
| # original session token has expired. Passing that stale token into the | |
| # actuator made every retry fail with ExpiredToken and leaked the GPU | |
| # node. The `id-token: write` permission is granted for the whole job, | |
| # so configure-aws-credentials can mint a fresh session here. | |
| - name: Refresh AWS credentials for teardown | |
| if: always() && steps.infra.outcome != 'skipped' && inputs.skip_delete != true | |
| uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1 | |
| with: | |
| role-to-assume: "arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/${{ env.GITHUB_ACTIONS_ROLE_NAME }}" | |
| aws-region: ${{ env.AWS_REGION }} | |
| role-session-name: GitHubActions-UAT-AICR-Teardown | |
| # Teardown (retry up to 3 times). Runs last so a slow destroy | |
| # cannot block evidence upload or summary rendering. | |
| # | |
| # The pinned actuator (EKS_IMAGE digest above) has no `destroy` | |
| # subcommand — teardown is `apply` with .deployment.destroy=true (see | |
| # the actuator's own help: "apply — Deploy or destroy infrastructure | |
| # via Terraform"). The prior `destroy` invocation hit urfave/cli's | |
| # unknown-command path, printed the help banner, and exited 0, so the | |
| # `if` matched, "destroyed successfully" printed, and the GPU node | |
| # leaked — holding the H100 capacity reservation across runs. Mirrors | |
| # the GCP UAT teardown. | |
| # | |
| # The retry loop must fail the step if all attempts fail: `docker run` | |
| # in an `if` condition does not trip `set -e`, so without the explicit | |
| # post-loop guard a genuine destroy failure (e.g. orphaned ENIs/SGs | |
| # blocking VPC deletion) would exit 0 and re-leak silently. | |
| - name: Destroy Cluster | |
| if: always() && steps.infra.outcome != 'skipped' && inputs.skip_delete != true | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| yq -i '.deployment.destroy = true' "$CLUSTER_CONFIG" | |
| cat "$CLUSTER_CONFIG" | |
| destroyed=false | |
| for attempt in 1 2 3; do | |
| echo "Destroy attempt ${attempt}..." | |
| if docker run \ | |
| -e CONFIG_CONTENT="$(base64 < "$CLUSTER_CONFIG")" \ | |
| -e AUTO_APPROVE=true \ | |
| -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ | |
| -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ | |
| -e AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \ | |
| "${EKS_IMAGE}" apply; then | |
| echo "Cluster destroyed successfully" | |
| destroyed=true | |
| break | |
| fi | |
| echo "Destroy attempt ${attempt} failed, retrying..." | |
| sleep 30 | |
| done | |
| if [ "$destroyed" != true ]; then | |
| echo "::error::Cluster ${DEPLOYMENT_ID} destroy failed after 3 attempts; GPU node may be leaking" | |
| exit 1 | |
| fi | |
| # Ingest the signed UAT evidence bundle into the corroboration bucket. | |
| # Runs as its own job (separate runner) only when the run produced a | |
| # bundle; the reusable workflow verifies signer + issuer + identity + | |
| # registry in a credential-free step before publishing to GCS. | |
| ingest-evidence: | |
| name: Ingest UAT evidence | |
| needs: uat-aws | |
| if: needs.uat-aws.outputs.bundle_ref != '' | |
| permissions: | |
| contents: read | |
| id-token: write # WIF for the publish job | |
| packages: read # GHCR pull for the verify job | |
| uses: ./.github/workflows/evidence-ingest.yaml | |
| with: | |
| bundle_ref: ${{ needs.uat-aws.outputs.bundle_ref }} | |
| recipe: eks-h100-training |