ROCm Nightly Tests #287
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright Advanced Micro Devices, Inc. | |
| # SPDX-License-Identifier: MIT | |
| # | |
| # e2e-nightly workflow — top-level trigger for ROCm end-to-end nightly tests. | |
| # | |
| # Layer structure (2-layer): | |
| # [Setup test matrix] | |
| # └── [E2E Nightly — <run_id>] (matrix — one cell per platform) | |
| # ├─ e2e: linux-gfx94x (green — target_available=true) | |
| # └─ e2e: linux-gfx110X (grey/skipped — target_available=false) | |
| # └── [Nightly Results — <run_id> — All Platforms] | |
| # | |
| # Each matrix cell calls e2e-tests.yml which contains a single job named | |
| # "e2e: <platform>". When target_available=false the job-level if: guard | |
| # skips the job entirely — no runner is consumed and the cell shows grey. | |
| # | |
| # Inputs: | |
| # source_repo — ROCm/TheRock or ROCm/rockrel | |
| # artifact_source — behaviour depends on source_repo: | |
| # ROCm/TheRock : "latest-release" (full artifact tarball, default) | |
| # OR a release tag (e.g. "nightly-2026-05-25") | |
| # ROCm/rockrel : workflow filename (e.g. "build.yml"); setup_matrix | |
| # resolves the latest successful run_id via the GitHub API | |
| # OR an all-numeric run id, used as-is to pin a build | |
| # include_dev — rockrel path only: also download _dev archives | |
| # (headers, CMake configs, static libs). | |
| name: ROCm Nightly Tests | |
| on: | |
| schedule: | |
| - cron: '0 11 * * *' # 11:00 AM UTC daily | |
| workflow_call: | |
| inputs: | |
| source_repo: | |
| description: 'Source repository: ROCm/TheRock or ROCm/rockrel' | |
| type: string | |
| default: ROCm/rockrel | |
| artifact_source: | |
| description: > | |
| ROCm/TheRock: "latest-release" OR a release tag. | |
| ROCm/rockrel: workflow filename (e.g. "multi_arch_release.yml") whose latest | |
| run_id is resolved automatically, OR a run id (e.g. "31136844092") to pin it. | |
| type: string | |
| default: multi_arch_release.yml | |
| include_dev: | |
| description: > | |
| For ROCm/rockrel: Download _dev artifacts needed for test binary compilation. | |
| type: boolean | |
| default: true | |
| workflow_dispatch: | |
| inputs: | |
| source_repo: | |
| description: 'GitHub repo to pull artifacts from' | |
| type: choice | |
| options: | |
| - ROCm/rockrel | |
| - ROCm/TheRock | |
| default: ROCm/rockrel | |
| artifact_source: | |
| description: > | |
| ROCm/TheRock: "latest-release" OR a release tag. | |
| ROCm/rockrel: workflow filename (e.g. "multi_arch_release.yml") whose latest | |
| run_id is resolved automatically, OR a run id (e.g. "31136844092") to pin it. | |
| type: string | |
| default: 'multi_arch_release.yml' | |
| include_dev: | |
| description: > | |
| For ROCm/rockrel: Download _dev artifacts needed for test binary compilation. | |
| type: boolean | |
| default: true | |
| permissions: | |
| contents: read | |
| actions: read | |
| env: | |
| RESULTS_TABLE: rocmtests_nightly_results | |
| jobs: | |
| setup_matrix: | |
| name: Setup test matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| targets: ${{ steps.testplan.outputs.targets }} | |
| skipped_targets: ${{ steps.testplan.outputs.skipped_targets }} | |
| run_id: ${{ steps.get_runid.outputs.run_id }} | |
| steps: | |
| - name: Checkout rocm-tests | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Resolve rockrel run_id from workflow name | |
| id: get_runid | |
| if: ${{ (inputs.source_repo || 'ROCm/rockrel') == 'ROCm/rockrel' }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SOURCE_REPO: ${{ inputs.source_repo || 'ROCm/rockrel' }} | |
| ARTIFACT_SOURCE: ${{ inputs.artifact_source || 'multi_arch_release.yml' }} | |
| shell: python | |
| run: | | |
| import json, os, sys | |
| from urllib.error import HTTPError, URLError | |
| from urllib.request import urlopen, Request | |
| repo = os.environ["SOURCE_REPO"] | |
| source = os.environ["ARTIFACT_SOURCE"].strip() | |
| if source.isdigit(): | |
| run_id = source | |
| print(f'artifact_source is a run id - pinning {repo} run {run_id}') | |
| else: | |
| url = ( | |
| f'https://api.github.com/repos/{repo}/actions/workflows/' | |
| f'{source}/runs?status=success&event=schedule&branch=main&per_page=1' | |
| ) | |
| req = Request(url, headers={ | |
| 'Authorization': f'Bearer {os.environ["GITHUB_TOKEN"]}', | |
| 'Accept': 'application/vnd.github+json', | |
| 'X-GitHub-Api-Version': '2022-11-28', | |
| }) | |
| try: | |
| with urlopen(req) as resp: | |
| data = json.loads(resp.read()) | |
| except HTTPError as exc: | |
| sys.exit( | |
| f'GitHub API returned {exc.code} for {url}\n' | |
| f"artifact_source must be a workflow filename in {repo} (e.g. " | |
| f'"multi_arch_release.yml") or an all-numeric run id.' | |
| ) | |
| except URLError as exc: | |
| sys.exit(f'Could not reach the GitHub API for {url}: {exc.reason}') | |
| runs = data.get('workflow_runs') or [] | |
| if not runs: | |
| sys.exit(f'No successful scheduled run on main for {repo}/{source}') | |
| run_id = runs[0]['id'] | |
| print(f'Latest successful run of {repo}/{source}: {run_id}') | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as f: | |
| f.write(f'run_id={run_id}\n') | |
| - name: Build test matrix from Test Configuration | |
| id: testplan | |
| shell: python | |
| run: | | |
| import os, configparser, json | |
| conf = configparser.ConfigParser() | |
| conf.read('testplan.ini') | |
| all_targets = [] | |
| skipped = [] | |
| for section in conf.sections(): | |
| entry = {'name': section, **dict(conf.items(section))} | |
| all_targets.append(entry) | |
| if entry.get('target_available', 'true').strip().lower() == 'false': | |
| skipped.append(section) | |
| print(f'all targets: {repr(all_targets)}') | |
| print(f'skipped: {repr(skipped)}') | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as f: | |
| f.write(f'targets={json.dumps(all_targets)}\n') | |
| f.write(f'skipped_targets={json.dumps(skipped)}\n') | |
| e2e_nightly: | |
| name: "${{ matrix.target.name }}" | |
| needs: setup_matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: ${{ fromJSON(needs.setup_matrix.outputs.targets) }} | |
| uses: ./.github/workflows/e2e-tests.yml | |
| with: | |
| name: ${{ matrix.target.name }} | |
| runs_on: ${{ matrix.target.runs_on }} | |
| artifact_group: ${{ matrix.target.artifact_group }} | |
| gpu_arch: ${{ matrix.target.gpu_arch || '' }} | |
| tests_filters: ${{ matrix.target.tests_filters }} | |
| source_repo: ${{ inputs.source_repo || 'ROCm/rockrel' }} | |
| artifact_source: ${{ inputs.artifact_source || 'multi_arch_release.yml' }} | |
| include_dev: ${{ inputs.include_dev || true }} | |
| run_id: ${{ needs.setup_matrix.outputs.run_id || '' }} | |
| target_available: ${{ matrix.target.target_available || 'true' }} | |
| report: | |
| name: "Nightly Test Summary — ${{ needs.setup_matrix.outputs.run_id || inputs.artifact_source || 'latest-release' }}" | |
| needs: [setup_matrix, e2e_nightly] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download all platform counts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: test-counts-* | |
| path: all-counts | |
| merge-multiple: false | |
| - name: Build master summary table | |
| env: | |
| RUN_ID: ${{ needs.setup_matrix.outputs.run_id || inputs.artifact_source || 'latest-release' }} | |
| SKIPPED_TARGETS: ${{ needs.setup_matrix.outputs.skipped_targets }} | |
| shell: python | |
| run: | | |
| import json, os, pathlib | |
| rows = [] | |
| for counts_file in sorted(pathlib.Path("all-counts").rglob("counts.json")): | |
| try: | |
| rows.append(json.loads(counts_file.read_text())) | |
| except Exception as e: | |
| print(f"Skipping {counts_file}: {e}") | |
| raw = os.environ.get("SKIPPED_TARGETS", "").strip() | |
| skipped_names = json.loads(raw) if raw else [] | |
| summary_path = os.environ["GITHUB_STEP_SUMMARY"] | |
| has_skip = any(r.get("skipped", 0) > 0 for r in rows) | |
| lines = [] | |
| run_id = os.environ["RUN_ID"] | |
| lines.append(f"## Nightly Results — {run_id} — All Platforms") | |
| lines.append("") | |
| if has_skip: | |
| lines.append("| Platform | Tests Run | Pass | Skip | Fail | Error | Status |") | |
| lines.append("|---|--:|--:|--:|--:|--:|---|") | |
| else: | |
| lines.append("| Platform | Tests Run | Pass | Fail | Error | Status |") | |
| lines.append("|---|--:|--:|--:|--:|---|") | |
| total_run = total_pass = total_skip = total_fail = total_err = 0 | |
| for r in rows: | |
| ok = r["failed"] == 0 and r["error"] == 0 and r["total"] > 0 | |
| status = "✅ Passed" if ok else "❌ Failed" | |
| if has_skip: | |
| lines.append( | |
| f"| {r['platform']} | {r['total']} | {r['passed']} | " | |
| f"{r.get('skipped', 0)} | {r['failed']} | {r['error']} | {status} |" | |
| ) | |
| else: | |
| lines.append( | |
| f"| {r['platform']} | {r['total']} | {r['passed']} | " | |
| f"{r['failed']} | {r['error']} | {status} |" | |
| ) | |
| total_run += r["total"] | |
| total_pass += r["passed"] | |
| total_skip += r.get("skipped", 0) | |
| total_fail += r["failed"] | |
| total_err += r["error"] | |
| for name in skipped_names: | |
| if has_skip: | |
| lines.append(f"| {name} | — | — | — | — | — | ⏭ Skipped — platform not available |") | |
| else: | |
| lines.append(f"| {name} | — | — | — | — | ⏭ Skipped — platform not available |") | |
| if rows: | |
| if has_skip: | |
| lines.append( | |
| f"| **TOTAL** | **{total_run}** | **{total_pass}** | " | |
| f"**{total_skip}** | **{total_fail}** | **{total_err}** | |" | |
| ) | |
| else: | |
| lines.append( | |
| f"| **TOTAL** | **{total_run}** | **{total_pass}** | " | |
| f"**{total_fail}** | **{total_err}** | |" | |
| ) | |
| else: | |
| lines.append("_No platform results collected._") | |
| lines.append("") | |
| with open(summary_path, "a") as f: | |
| f.write("\n".join(lines) + "\n") | |
| upload_results: | |
| name: "Upload Nightly Results to ClickHouse" | |
| needs: [setup_matrix, e2e_nightly, report] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| environment: clickhouse-ingest | |
| steps: | |
| - name: Checkout rocm-tests | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Download all platform counts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: test-counts-* | |
| path: all-counts | |
| merge-multiple: false | |
| - name: Download all platform json reports | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: test-report-* | |
| path: all-reports | |
| merge-multiple: false | |
| - name: Set up Python for results upload | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: 3.12 | |
| - name: Install ClickHouse results backend dependencies | |
| run: python -m pip install --root-user-action=ignore --no-cache-dir -r requirements-results.txt | |
| - name: Upload nightly results to ClickHouse | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| CLICKHOUSE_HOST: ${{ secrets.CLICKHOUSE_HOST }} | |
| CLICKHOUSE_PORT: ${{ secrets.CLICKHOUSE_PORT }} | |
| CLICKHOUSE_USERNAME: ${{ secrets.CLICKHOUSE_USERNAME }} | |
| CLICKHOUSE_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }} | |
| CLICKHOUSE_DATABASE: ${{ secrets.CLICKHOUSE_DATABASE }} | |
| SOURCE_REPO: ${{ inputs.source_repo || 'ROCm/rockrel' }} | |
| ARTIFACT_SOURCE: ${{ inputs.artifact_source || 'multi_arch_release.yml' }} | |
| run: | | |
| python -m framework.results.backends.clickhouse \ | |
| --run-id "${{ github.run_id }}" \ | |
| --github-repo "${{ github.repository }}" \ | |
| --source-repo "${SOURCE_REPO}" \ | |
| --artifact-source "${ARTIFACT_SOURCE}" \ | |
| --table "${RESULTS_TABLE}" \ | |
| --testplan testplan.ini \ | |
| --counts-dir all-counts \ | |
| --reports-dir all-reports |