Fixed doc tests #1729
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Doc tests | |
| on: | |
| pull_request: | |
| paths: | |
| - 'assets/**' | |
| - 'content/docs/**' | |
| schedule: | |
| # Every day at 06:00 UTC | |
| - cron: '0 6 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| slack_notification: | |
| description: 'Send Slack notification' | |
| type: boolean | |
| default: false | |
| slack_channel: | |
| description: 'Slack channel to notify' | |
| type: string | |
| default: 'product-excellence-test-notifications' | |
| jobs: | |
| discover: | |
| name: Discover test cases | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.list.outputs.matrix }} | |
| has_tests: ${{ steps.list.outputs.has_tests }} | |
| dev_version: ${{ steps.dev-version.outputs.dev_version }} | |
| steps: | |
| - name: Checkout website repo | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.x' | |
| - name: Install Python dependencies | |
| run: pip install pyyaml | |
| - name: Get changed markdown files | |
| id: changed-files | |
| if: github.event_name == 'pull_request' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| ALL_CHANGED=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files \ | |
| --paginate --jq '.[].filename' || true) | |
| # Start with directly changed content/*.md files | |
| FILES=$(echo "$ALL_CHANGED" | grep '^content/.*\.md$' | tr '\n' ' ' || true) | |
| # For each changed assets/agw-docs/pages/*.md file, check for corresponding | |
| # content files in the kubernetes/standalone main/latest directories | |
| ASSET_FILES=$(echo "$ALL_CHANGED" | grep '^assets/agw-docs/pages/.*\.md$' || true) | |
| if [ -n "$ASSET_FILES" ]; then | |
| while IFS= read -r asset_file; do | |
| rel_path="${asset_file#assets/agw-docs/pages/}" | |
| rel_path="${rel_path#agentgateway/}" | |
| found=0 | |
| for content_dir in "content/docs/kubernetes/main" "content/docs/kubernetes/latest" "content/docs/standalone/main" "content/docs/standalone/latest"; do | |
| candidate="${content_dir}/${rel_path}" | |
| if [ -f "$candidate" ] && ! echo "$FILES" | grep -qF "$candidate"; then | |
| FILES="$FILES $candidate" | |
| found=1 | |
| fi | |
| done | |
| if [ "$found" -eq 0 ]; then | |
| echo "No content candidates found for asset file: $asset_file" | |
| fi | |
| done <<< "$ASSET_FILES" | |
| fi | |
| FILES=$(echo "$FILES" | tr ' ' '\n' | sort -u | tr '\n' ' ' | xargs) | |
| echo "files=${FILES}" >> $GITHUB_OUTPUT | |
| echo "$FILES" | tr ' ' '\n' | |
| - name: List test cases | |
| id: list | |
| run: | | |
| if [ -n "$CHANGED_FILES" ]; then | |
| TESTS=$(python3 scripts/doc_test_run.py --repo-root . --list-tests --file $CHANGED_FILES) | |
| else | |
| TESTS=$(python3 scripts/doc_test_run.py --repo-root . --list-tests) | |
| fi | |
| COUNT=$(echo "$TESTS" | jq 'length') | |
| if [ "$COUNT" -gt 0 ]; then | |
| echo "has_tests=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_tests=false" >> $GITHUB_OUTPUT | |
| fi | |
| MATRIX=$(echo "$TESTS" | jq -c '(length | [., 40] | min) as $shards | . as $tests | {include: [range($shards) | . as $i | ([$tests | to_entries[] | select(.key % $shards == $i) | .value] as $shard | {shard_index: $i, test_count: ($shard | length), tests: ($shard | tojson), shard_name: ($shard | [.[].test] | join(", ") | if length > 80 then .[:77] + "..." else . end)})]}') | |
| echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT | |
| echo "Discovered $COUNT test case(s)" | |
| env: | |
| CHANGED_FILES: ${{ steps.changed-files.outputs.files }} | |
| # Resolve an installable version for the development ("main"/experimental) install | |
| # snippets. Those pages install the nightly chart pinned in | |
| # assets/agw-docs/versions/patch-dev.md (normally the floating `0.0.0-latest-dev`). | |
| # | |
| # A failed upstream nightly (agentgateway/agentgateway) still publishes the | |
| # `0.0.0-latest-dev` chart, but with an `appVersion` pointing at a proxy image whose | |
| # build/push was skipped, so the image never lands in the registry. The deployer then | |
| # tries to pull a `v0.0.0-alpha.<sha>` image that does not exist -> ImagePullBackOff -> | |
| # every dev-channel doc test times out. | |
| # | |
| # We can do nothing to fix an upstream nightly, so instead of going red we resolve the | |
| # newest *fully published* build: keep `0.0.0-latest-dev` when its chart's image | |
| # actually exists, otherwise fall back to the most recent successful nightly's immutable | |
| # `0.0.0-alpha.<sha>` chart (which has both a chart and a matching image). Any failure in | |
| # this step falls back to the documented default, i.e. today's behavior. | |
| - name: Resolve installable dev build version | |
| id: dev-version | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -uo pipefail | |
| REG=cr.agentgateway.dev | |
| CHART="oci://${REG}/charts/agentgateway" | |
| IMG="${REG}/agentgateway" | |
| UPSTREAM=agentgateway/agentgateway | |
| DEFAULT=$(cat assets/agw-docs/versions/patch-dev.md) | |
| RESOLVED="$DEFAULT" | |
| # True if the container image tag exists in the registry (anonymous, public). | |
| img_exists() { docker buildx imagetools inspect "${IMG}:$1" >/dev/null 2>&1; } | |
| # Echo a chart version's appVersion, or nothing if the chart tag does not exist. | |
| chart_appversion() { | |
| helm show chart "$CHART" --version "$1" 2>/dev/null \ | |
| | awk -F': *' '/^appVersion:/ { gsub(/"/,"",$2); print $2; exit }' | |
| } | |
| # True if a chart version exists AND the proxy image its appVersion resolves to | |
| # exists. The deployer prefixes the appVersion with `v` when it is absent. | |
| installable() { | |
| local av; av=$(chart_appversion "$1") | |
| [ -n "$av" ] || return 1 | |
| case "$av" in | |
| v*) img_exists "$av" ;; | |
| *) img_exists "v$av" || img_exists "$av" ;; | |
| esac | |
| } | |
| if installable "$DEFAULT"; then | |
| echo "Default dev build '$DEFAULT' is installable; using it." | |
| else | |
| echo "Default dev build '$DEFAULT' is not installable (broken upstream nightly?); searching last successful nightly." | |
| RUN_IDS=$(gh run list --repo "$UPSTREAM" --workflow nightly.yml \ | |
| --status success --limit 15 --json databaseId --jq '.[].databaseId' 2>/dev/null || true) | |
| for rid in $RUN_IDS; do | |
| # The nightly records its version as `Nightly version: 0.0.0-alpha.<sha>` in the log. | |
| # Capture the log first, then extract, so an early-exiting pipe (SIGPIPE under | |
| # `set -o pipefail`) can't fail the step. | |
| LOG=$(gh run view "$rid" --repo "$UPSTREAM" --log 2>/dev/null) || LOG="" | |
| # here-string + `grep -m1` avoids a pipe, so grep can't be killed by SIGPIPE | |
| # (which would fail the step under `set -o pipefail`); `|| true` covers no-match. | |
| VER=$(grep -m1 -oiE '0\.0\.0-alpha\.[0-9a-f]+' <<<"$LOG" || true) | |
| [ -n "$VER" ] || continue | |
| if installable "$VER"; then | |
| RESOLVED="$VER" | |
| echo "Falling back to last good nightly build '$VER' (run $rid)." | |
| break | |
| fi | |
| done | |
| if [ "$RESOLVED" = "$DEFAULT" ]; then | |
| echo "No installable fallback found; leaving default '$DEFAULT' in place." | |
| fi | |
| fi | |
| echo "dev_version=$RESOLVED" >> "$GITHUB_OUTPUT" | |
| echo "Resolved dev build version: $RESOLVED" | |
| run-test: | |
| name: "${{ matrix.shard_name }}" | |
| needs: discover | |
| if: needs.discover.outputs.has_tests == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| strategy: | |
| matrix: ${{ fromJson(needs.discover.outputs.matrix) }} | |
| fail-fast: false | |
| steps: | |
| - name: Checkout website repo | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.x' | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: 'stable' | |
| cache: false | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: 'lts/*' | |
| - name: Install Python dependencies | |
| run: pip install pyyaml | |
| - name: Install cloud-provider-kind | |
| run: go install sigs.k8s.io/cloud-provider-kind@latest | |
| - name: Install yamltest | |
| run: npm install -g yamltest@latest | |
| # Redirect the development install snippets to the version resolved in `discover`. | |
| # Only rewrites the checkout used by this run; the committed docs still ship | |
| # `0.0.0-latest-dev`. No-op when the default is already installable. | |
| - name: Pin dev build version | |
| if: needs.discover.outputs.dev_version != '' && needs.discover.outputs.dev_version != '0.0.0-latest-dev' | |
| run: | | |
| printf '%s' '${{ needs.discover.outputs.dev_version }}' > assets/agw-docs/versions/patch-dev.md | |
| echo "Pinned assets/agw-docs/versions/patch-dev.md to ${{ needs.discover.outputs.dev_version }} for this run." | |
| - name: Run doc tests | |
| env: | |
| SHARD_TESTS: ${{ matrix.tests }} | |
| DEBUG_MODE: true | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| PYTHONUNBUFFERED: '1' | |
| # Used by the install-agentgateway-binary snippet's `gh run download` of the | |
| # nightly release-binary-linux artifact from agentgateway/agentgateway. | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| FAILED=0 | |
| TEST_INDEX=0 | |
| while IFS=$'\t' read -r file test; do | |
| python3 scripts/doc_test_run.py --repo-root . --file "$file" --test "$test" \ | |
| --report-file "out/tests/generated/shard/${TEST_INDEX}/test-results.yaml" || FAILED=1 | |
| TEST_INDEX=$((TEST_INDEX + 1)) | |
| done < <(echo "$SHARD_TESTS" | jq -r '.[] | "\(.file)\t\(.test)"') | |
| python3 scripts/merge_test_results.py out/tests/generated/shard/ out/tests/generated/test-results.yaml | |
| exit $FAILED | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-result-${{ strategy.job-index }} | |
| path: out/tests/generated/test-results.yaml | |
| retention-days: 1 | |
| - name: Upload test context (on failure) | |
| if: failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-context-${{ strategy.job-index }} | |
| path: out/tests/generated/context/ | |
| retention-days: 1 | |
| if-no-files-found: ignore | |
| report: | |
| name: Aggregate results and report | |
| needs: [discover, run-test] | |
| if: always() && needs.discover.outputs.has_tests == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout website repo | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.x' | |
| - name: Install Python dependencies | |
| run: pip install pyyaml | |
| - name: Download all test results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: test-result-* | |
| path: collected-results/ | |
| - name: Merge test results | |
| run: python3 scripts/merge_test_results.py collected-results/ out/tests/generated/test-results.yaml | |
| - name: Generate job summary | |
| id: summary | |
| run: | | |
| RESULTS_FILE=out/tests/generated/test-results.yaml | |
| # GitHub Step Summary (Markdown) | |
| python3 scripts/report_summary.py "$RESULTS_FILE" >> "$GITHUB_STEP_SUMMARY" | |
| # Slack Block Kit payload | |
| RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| SLACK_PAYLOAD=$(python3 scripts/report_summary.py --slack --run-url "$RUN_URL" "$RESULTS_FILE") | |
| echo "slack_blocks<<EOF" >> "$GITHUB_OUTPUT" | |
| echo "$SLACK_PAYLOAD" | jq '.main' >> "$GITHUB_OUTPUT" | |
| echo "EOF" >> "$GITHUB_OUTPUT" | |
| echo "slack_thread_blocks<<EOF" >> "$GITHUB_OUTPUT" | |
| echo "$SLACK_PAYLOAD" | jq '.thread // empty' >> "$GITHUB_OUTPUT" | |
| echo "EOF" >> "$GITHUB_OUTPUT" | |
| - name: Notify Slack | |
| if: always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.slack_notification)) | |
| uses: ./.github/actions/notify-slack | |
| with: | |
| message: "Doc Test Results" | |
| blocks: ${{ steps.summary.outputs.slack_blocks }} | |
| thread_blocks: ${{ steps.summary.outputs.slack_thread_blocks }} | |
| channel: ${{ github.event_name == 'workflow_dispatch' && inputs.slack_channel || 'product-excellence-test-notifications,doctopus-tests' }} | |
| token: ${{ secrets.SLACK_BOT_TOKEN }} | |
| - name: List untested docs | |
| if: always() | |
| run: | | |
| python3 scripts/list_untested_docs.py \ | |
| --docs-dir content/docs \ | |
| --output out/tests/generated/untested-docs.txt | |
| - name: Upload merged results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: doc-test-results | |
| path: out/tests/generated/ | |
| retention-days: 14 | |
| - name: Fail if any test failed | |
| run: | | |
| python3 -c " | |
| import yaml, sys | |
| with open('out/tests/generated/test-results.yaml') as f: | |
| report = yaml.safe_load(f) or {} | |
| tests = report.get('tests', {}) | |
| failed = [k for k, v in tests.items() if v.get('status') != 'passed'] | |
| if failed: | |
| print(f'{len(failed)} test(s) failed:', file=sys.stderr) | |
| for name in failed: | |
| print(f' - {name}', file=sys.stderr) | |
| sys.exit(1) | |
| print(f'All {len(tests)} test(s) passed') | |
| " |