Skip to content

Commit e83d061

Browse files
yuanchen8911yuanchen8911
committed
ci: refactor and harden H100 GPU workflow
1 parent 3be21ed commit e83d061

67 files changed

Lines changed: 3859 additions & 932 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/README.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,13 @@ This directory contains a modular, reusable GitHub Actions architecture optimize
44

55
## Composite Actions
66

7+
### Script Conventions
8+
9+
Composite action helper scripts in this directory are intentionally portable
10+
across checkout modes: keep them mode `0644` and invoke them as
11+
`bash path/to/script.sh` from workflows or `action.yml` files. Do not rely on
12+
executable bits or `./script.sh` invocation.
13+
714
### Core CI/CD Actions
815

916
#### `security-scan/`
@@ -50,7 +57,8 @@ This action runs `tools/setup-tools --skip-go --skip-docker` in auto mode, which
5057
**When to use**: When you need version values in workflow steps
5158
**Outputs**:
5259
- `go`, `goreleaser`, `ko`, `crane`, `golangci_lint`, `yamllint`, `addlicense`
53-
- `grype`, `kubectl`, `kind`, `ctlptl`, `tilt`, `helm`
60+
- `grype`, `kubectl`, `kind`, `nvkind`, `ctlptl`, `tilt`, `helm`
61+
- `kind_node_image`, `h100_kind_node_image`
5462

5563
**Example**:
5664
```yaml

‎.github/actions/aicr-build/action.yml‎

Lines changed: 25 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,17 @@
1313
# limitations under the License.
1414

1515
name: 'AICR Build'
16-
description: 'Builds the aicr validator image (via Dockerfile) and CLI binary, and loads the image into kind.'
16+
description: 'Builds the aicr CLI and optional snapshot/validator images, and loads requested images into kind.'
1717

1818
inputs:
19+
build_cli:
20+
description: 'Build and stage the standalone aicr CLI binary at the repository root'
21+
required: false
22+
default: 'true'
23+
build_snapshot_agent:
24+
description: 'Build the CUDA-based snapshot agent image and load it into kind'
25+
required: false
26+
default: 'true'
1927
build_validators:
2028
description: 'Deprecated: use validator_phases instead. Ignored when validator_phases is set.'
2129
required: false
@@ -28,86 +36,34 @@ inputs:
2836
runs:
2937
using: 'composite'
3038
steps:
31-
32-
- name: Install ko
39+
- name: Build standalone aicr CLI binary
40+
if: inputs.build_cli == 'true'
3341
shell: bash
34-
run: |
35-
KO_VERSION=$(yq eval '.build_tools.ko' .settings.yaml)
36-
GOFLAGS= go install "github.com/google/ko@${KO_VERSION}"
42+
env:
43+
GOFLAGS: -mod=vendor
44+
run: bash "${{ github.action_path }}/build-cli.sh"
3745

38-
- name: Build snapshot agent image and load into kind
46+
- name: Build snapshot agent CLI binary
47+
if: inputs.build_cli != 'true' && inputs.build_snapshot_agent == 'true'
3948
shell: bash
4049
env:
4150
GOFLAGS: -mod=vendor
42-
run: |
43-
# Build snapshot agent image with CUDA base (provides nvidia-smi for GPU detection).
44-
# Uses cuda:base (~250MB) instead of cuda:runtime (~1.8GB) — only nvidia-smi is needed.
45-
# GPU test workflows use --image=ko.local:smoke-test for aicr snapshot.
46-
CGO_ENABLED=0 go build -trimpath -o dist/aicr ./cmd/aicr
47-
docker build -t ko.local:smoke-test -f - . <<'DOCKERFILE'
48-
FROM nvcr.io/nvidia/cuda:13.1.0-base-ubuntu24.04
49-
COPY dist/aicr /usr/local/bin/aicr
50-
ENTRYPOINT ["/usr/local/bin/aicr"]
51-
DOCKERFILE
51+
run: bash "${{ github.action_path }}/build-cli.sh"
5252

53-
# Load onto all nodes. The snapshot agent requests nvidia.com/gpu but
54-
# does not set a node selector, so it can land on any GPU-capable node
55-
# including the control-plane (e.g., T4 smoke test).
56-
#
57-
# Timeout is intentionally generous (900s per attempt). H100 self-hosted
58-
# runners transfer images over a shared Docker-in-Docker bridge; large
59-
# CUDA base images (~250MB compressed) combined with I/O contention from
60-
# parallel GPU operator pods regularly exceed the previous 600s limit.
61-
timeout 900 kind load docker-image ko.local:smoke-test --name "${KIND_CLUSTER_NAME}" || {
62-
echo "::warning::kind load attempt 1 failed for ko.local:smoke-test, retrying..."
63-
timeout 900 kind load docker-image ko.local:smoke-test --name "${KIND_CLUSTER_NAME}"
64-
}
53+
- name: Build snapshot agent image and load into kind
54+
if: inputs.build_snapshot_agent == 'true'
55+
shell: bash
56+
run: bash "${{ github.action_path }}/build-snapshot-agent.sh"
6557

6658
- name: Build validator images and load into kind
6759
if: "!(inputs.validator_phases == 'none' || (inputs.validator_phases == '' && inputs.build_validators == 'false'))"
6860
shell: bash
6961
env:
7062
GOFLAGS: -mod=vendor
71-
run: |
72-
# Determine which validator phases to build.
73-
# validator_phases takes precedence; build_validators is a deprecated fallback.
74-
if [[ -n "${{ inputs.validator_phases }}" ]]; then
75-
if [[ "${{ inputs.validator_phases }}" == "none" ]]; then
76-
echo "Skipping validator builds (validator_phases=none)"
77-
exit 0
78-
fi
79-
PHASES="${{ inputs.validator_phases }}"
80-
else
81-
# Default: build all phases (backwards compatible)
82-
PHASES="deployment,performance,conformance"
83-
fi
84-
85-
# Compile only the requested validator binaries.
86-
mkdir -p dist/validator
87-
for phase in ${PHASES//,/ }; do
88-
echo "Building validator binary: ${phase}"
89-
CGO_ENABLED=0 go build -trimpath -o "dist/validator/${phase}" "./validators/${phase}"
90-
done
91-
92-
for phase in ${PHASES//,/ }; do
93-
mkdir -p "validators/${phase}/testdata"
94-
docker build -t "ko.local/aicr-validators/${phase}:latest" -f - . <<DOCKERFILE
95-
FROM gcr.io/distroless/static-debian12:nonroot
96-
COPY dist/validator/${phase} /${phase}
97-
COPY validators/${phase}/testdata /app/testdata
98-
WORKDIR /app
99-
USER nonroot
100-
ENTRYPOINT ["/${phase}"]
101-
DOCKERFILE
102-
# Validator images are small (~30MB distroless), but share the same
103-
# Docker-in-Docker bridge as the smoke-test load above. 600s per
104-
# attempt accommodates I/O queuing behind concurrent image pulls.
105-
timeout 600 kind load docker-image "ko.local/aicr-validators/${phase}:latest" --name "${KIND_CLUSTER_NAME}" || {
106-
echo "::warning::kind load attempt 1 failed for ko.local/aicr-validators/${phase}:latest, retrying..."
107-
timeout 600 kind load docker-image "ko.local/aicr-validators/${phase}:latest" --name "${KIND_CLUSTER_NAME}"
108-
}
109-
done
63+
VALIDATOR_PHASES: ${{ inputs.validator_phases }}
64+
run: bash "${{ github.action_path }}/build-validator-images.sh"
11065

11166
- name: Stage aicr binary at repo root
67+
if: inputs.build_cli == 'true'
11268
shell: bash
113-
run: cp dist/aicr ./aicr
69+
run: bash "${{ github.action_path }}/stage-cli.sh"

‎.github/actions/aicr-build/build-cli.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -euo pipefail
17+
18+
mkdir -p dist
19+
CGO_ENABLED=0 go build -trimpath -o dist/aicr ./cmd/aicr

‎.github/actions/aicr-build/build-snapshot-agent.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -euo pipefail
17+
18+
# Build snapshot agent image with CUDA base (provides nvidia-smi for GPU detection).
19+
# Uses cuda:base (~250MB) instead of cuda:runtime (~1.8GB) because only nvidia-smi is needed.
20+
timeout 900s docker build -t ko.local:smoke-test -f - . <<'DOCKERFILE'
21+
FROM nvcr.io/nvidia/cuda:13.1.0-base-ubuntu24.04
22+
COPY dist/aicr /usr/local/bin/aicr
23+
ENTRYPOINT ["/usr/local/bin/aicr"]
24+
DOCKERFILE
25+
26+
# Load onto all nodes. The snapshot agent requests nvidia.com/gpu but does not
27+
# set a node selector, so it can land on any GPU-capable node including the
28+
# control-plane in the L40G smoke test.
29+
timeout 900 kind load docker-image ko.local:smoke-test --name "${KIND_CLUSTER_NAME}" || {
30+
echo "::warning::kind load attempt 1 failed for ko.local:smoke-test, retrying..."
31+
timeout 900 kind load docker-image ko.local:smoke-test --name "${KIND_CLUSTER_NAME}"
32+
}

‎.github/actions/aicr-build/build-validator-images.sh‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env bash
2+
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -euo pipefail
17+
18+
VALIDATOR_PHASES="${VALIDATOR_PHASES:-}"
19+
if [[ -n "${VALIDATOR_PHASES}" ]]; then
20+
if [[ "${VALIDATOR_PHASES}" == "none" ]]; then
21+
echo "Skipping validator builds (validator_phases=none)"
22+
exit 0
23+
fi
24+
PHASES="${VALIDATOR_PHASES}"
25+
else
26+
# Default: build all phases (backwards compatible).
27+
PHASES="deployment,performance,conformance"
28+
fi
29+
30+
: "${KIND_CLUSTER_NAME:?KIND_CLUSTER_NAME must be set}"
31+
32+
mkdir -p dist/validator
33+
for phase in ${PHASES//,/ }; do
34+
if ! [[ "${phase}" =~ ^[a-z][a-z0-9_-]*$ ]]; then
35+
echo "::error::invalid validator phase '${phase}'; expected ^[a-z][a-z0-9_-]*$"
36+
exit 1
37+
fi
38+
echo "Building validator binary: ${phase}"
39+
CGO_ENABLED=0 go build -trimpath -o "dist/validator/${phase}" "./validators/${phase}"
40+
done
41+
42+
for phase in ${PHASES//,/ }; do
43+
if [[ ! -d "validators/${phase}/testdata" ]]; then
44+
echo "::error::validators/${phase}/testdata is missing"
45+
exit 1
46+
fi
47+
docker build -t "ko.local/aicr-validators/${phase}:latest" -f - . <<DOCKERFILE
48+
FROM gcr.io/distroless/static-debian12:nonroot
49+
COPY dist/validator/${phase} /${phase}
50+
COPY validators/${phase}/testdata /app/testdata
51+
WORKDIR /app
52+
USER nonroot
53+
ENTRYPOINT ["/${phase}"]
54+
DOCKERFILE
55+
timeout 600 kind load docker-image "ko.local/aicr-validators/${phase}:latest" --name "${KIND_CLUSTER_NAME}" || {
56+
echo "::warning::kind load attempt 1 failed for ko.local/aicr-validators/${phase}:latest, retrying..."
57+
timeout 600 kind load docker-image "ko.local/aicr-validators/${phase}:latest" --name "${KIND_CLUSTER_NAME}"
58+
}
59+
done

‎.github/actions/aicr-build/stage-cli.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -euo pipefail
17+
18+
cp dist/aicr ./aicr

‎.github/actions/check-control-plane-health/action.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: 'Check Control Plane Health'
16+
description: 'Fails if Kind control-plane static pods are missing, unready, or unstable.'
17+
18+
inputs:
19+
cluster_name:
20+
description: 'Kind cluster name'
21+
required: true
22+
namespace:
23+
description: 'Namespace that contains the control-plane pods'
24+
required: false
25+
default: kube-system
26+
components:
27+
description: 'Space-separated component label values to check'
28+
required: false
29+
default: kube-apiserver kube-controller-manager kube-scheduler etcd
30+
wait_timeout:
31+
description: 'Timeout for each component readiness wait'
32+
required: false
33+
default: 60s
34+
max_restarts:
35+
description: 'Compatibility input; with stability_window=0s, fail if historical restartCount exceeds this ceiling'
36+
required: false
37+
default: '1'
38+
stability_window:
39+
description: 'Optional duration to watch for new control-plane restarts after pods are Ready'
40+
required: false
41+
default: '0s'
42+
stability_probe_interval:
43+
description: 'Interval for active API server probes during the stability window'
44+
required: false
45+
default: '10s'
46+
stability_probe_failure_threshold:
47+
description: 'Consecutive active stability probe failures allowed before failing'
48+
required: false
49+
default: '2'
50+
lease_components:
51+
description: 'Space-separated leader election lease names to check for freshness'
52+
required: false
53+
default: kube-controller-manager kube-scheduler
54+
lease_stale_timeout:
55+
description: 'Maximum allowed leader election lease age at the end of a stability window'
56+
required: false
57+
default: '120s'
58+
runtime_diagnostics:
59+
description: 'Collect expensive kind node runtime diagnostics such as docker stats, crictl, and journalctl on failure'
60+
required: false
61+
default: 'false'
62+
63+
runs:
64+
using: 'composite'
65+
steps:
66+
- name: Check control-plane pods
67+
shell: bash
68+
env:
69+
KIND_CLUSTER_NAME: ${{ inputs.cluster_name }}
70+
NAMESPACE: ${{ inputs.namespace }}
71+
COMPONENTS: ${{ inputs.components }}
72+
WAIT_TIMEOUT: ${{ inputs.wait_timeout }}
73+
MAX_RESTARTS: ${{ inputs.max_restarts }}
74+
STABILITY_WINDOW: ${{ inputs.stability_window }}
75+
STABILITY_PROBE_INTERVAL: ${{ inputs.stability_probe_interval }}
76+
STABILITY_PROBE_FAILURE_THRESHOLD: ${{ inputs.stability_probe_failure_threshold }}
77+
LEASE_COMPONENTS: ${{ inputs.lease_components }}
78+
LEASE_STALE_TIMEOUT: ${{ inputs.lease_stale_timeout }}
79+
RUNTIME_DIAGNOSTICS: ${{ inputs.runtime_diagnostics }}
80+
run: bash "${{ github.action_path }}/check-control-plane-health.sh"

0 commit comments

Comments
 (0)