docker-builds #188
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: docker-builds | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths: | |
| - .ci/docker/** | |
| - .github/workflows/docker-builds.yml | |
| - .lintrunner.toml | |
| push: | |
| branches: | |
| - main | |
| - release/* | |
| - landchecks/* | |
| paths: | |
| - .ci/docker/** | |
| - .github/workflows/docker-builds.yml | |
| - .lintrunner.toml | |
| schedule: | |
| - cron: 1 3 * * 3 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} | |
| cancel-in-progress: true | |
| env: | |
| ALPINE_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine | |
| AWS_DEFAULT_REGION: us-east-1 | |
| permissions: read-all | |
| jobs: | |
| get-label-type: | |
| if: github.repository_owner == 'pytorch' | |
| name: get-label-type | |
| uses: pytorch/pytorch/.github/workflows/_runner-determinator.yml@main | |
| with: | |
| triggering_actor: ${{ github.triggering_actor }} | |
| issue_owner: ${{ github.event.pull_request.user.login || github.event.issue.user.login }} | |
| curr_branch: ${{ github.head_ref || github.ref_name }} | |
| curr_ref_type: ${{ github.ref_type }} | |
| docker-build: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [linux.12xlarge] | |
| docker-image-name: [ | |
| pytorch-linux-jammy-cuda13.0-cudnn9-py3-gcc11, | |
| pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11-vllm, | |
| pytorch-linux-jammy-cuda13.0-cudnn9-py3-gcc11-inductor-benchmarks, | |
| pytorch-linux-jammy-py3.10-clang18, | |
| pytorch-linux-jammy-py3.11-clang18, | |
| pytorch-linux-jammy-py3.12-clang18, | |
| pytorch-linux-jammy-py3.13-clang18, | |
| pytorch-linux-jammy-py3.14-clang18, | |
| pytorch-linux-jammy-py3.14t-clang18, | |
| pytorch-linux-jammy-rocm-n-py3, | |
| pytorch-linux-noble-rocm-n-py3, | |
| pytorch-linux-noble-rocm-nightly-py3, | |
| pytorch-linux-jammy-rocm-n-py3-benchmarks, | |
| pytorch-linux-jammy-cuda13.0-cudnn9-py3.10-clang18, | |
| pytorch-linux-jammy-py3-gcc11-inductor-benchmarks, | |
| pytorch-linux-jammy-py3.12-halide, | |
| pytorch-linux-jammy-py3.12-pallas, | |
| pytorch-linux-jammy-cuda13.0-py3.12-pallas, | |
| pytorch-linux-jammy-tpu-py3.12-pallas, | |
| pytorch-linux-jammy-xpu-n-1-py3, | |
| pytorch-linux-noble-xpu-n-py3, | |
| pytorch-linux-noble-xpu-n-py3-client, | |
| pytorch-linux-noble-xpu-n-py3-inductor-benchmarks, | |
| pytorch-linux-jammy-linter, | |
| pytorch-linux-jammy-cuda13.0-cudnn9-py3.10-linter, | |
| # TODO: Re-enable me when docker pin update happens | |
| # pytorch-linux-jammy-py3-clang18-executorch, | |
| pytorch-linux-jammy-py3.12-triton-cpu, | |
| pytorch-linux-noble-riscv64-py3.12-gcc14 | |
| ] | |
| include: | |
| - docker-image-name: pytorch-linux-jammy-aarch64-py3.10-gcc13 | |
| runner: linux.arm64.m7g.4xlarge | |
| - docker-image-name: pytorch-linux-jammy-aarch64-py3.10-gcc13-inductor-benchmarks | |
| runner: linux.arm64.m7g.4xlarge | |
| timeout-minutes: 600 | |
| # Docker uploads fail from LF runners, see https://github.com/pytorch/pytorch/pull/137358 | |
| # runs-on: "${{ needs.get-label-type.outputs.label-type }}${{ matrix.runner }}" | |
| runs-on: "${{ matrix.runner }}" | |
| steps: | |
| - name: Clean workspace | |
| shell: bash | |
| run: | | |
| echo "${GITHUB_WORKSPACE}" | |
| sudo rm -rf "${GITHUB_WORKSPACE}" | |
| mkdir "${GITHUB_WORKSPACE}" | |
| - name: Setup Linux | |
| uses: pytorch/pytorch/.github/actions/setup-linux@main | |
| with: | |
| submodules: 'false' | |
| - name: Login to ECR | |
| uses: ./.github/actions/ecr-login | |
| - name: Build docker image | |
| id: build-docker-image | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: ci-image:${{ matrix.docker-image-name }} | |
| always-rebuild: true | |
| push: true | |
| - name: Pull docker image | |
| uses: pytorch/test-infra/.github/actions/pull-docker-image@main | |
| with: | |
| docker-image: ${{ steps.build-docker-image.outputs.docker-image }} | |
| - name: Generate output | |
| if: contains(matrix.docker-image-name, 'rocm') | |
| id: generate_output | |
| run: | | |
| docker_image_name="${{ matrix.docker-image-name }}" | |
| docker_image_tag="${{ steps.build-docker-image.outputs.docker-image }}" | |
| echo "${docker_image_name}=${docker_image_tag}" >> docker-builds-output-${docker_image_name}.txt | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4.4.0 | |
| if: contains(matrix.docker-image-name, 'rocm') | |
| with: | |
| name: docker-builds-artifacts-${{ matrix.docker-image-name }} | |
| retention-days: 14 | |
| path: ./docker-builds-output-${{ matrix.docker-image-name }}.txt | |
| - uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 | |
| name: Push to https://ghcr.io/ | |
| id: push-to-ghcr-io | |
| if: ${{ github.event_name == 'push' }} | |
| env: | |
| ECR_DOCKER_IMAGE: ${{ steps.build-docker-image.outputs.docker-image }} | |
| GHCR_PAT: ${{ secrets.GHCR_PAT }} | |
| with: | |
| shell: bash | |
| timeout_minutes: 60 | |
| max_attempts: 5 | |
| retry_wait_seconds: 90 | |
| command: | | |
| set -euo pipefail | |
| ghcr_image="ghcr.io/pytorch/ci-image" | |
| tag=${ECR_DOCKER_IMAGE##*:} | |
| # Push docker image to the ghcr.io | |
| echo "$GHCR_PAT" | docker login ghcr.io -u pytorch --password-stdin | |
| docker tag "${ECR_DOCKER_IMAGE}" "${ghcr_image}:${tag}" | |
| docker push "${ghcr_image}:${tag}" | |
| # Also push a tag without the hash for easier reference | |
| docker tag "${ECR_DOCKER_IMAGE}" "${ghcr_image}:${{ matrix.docker-image-name }}" | |
| docker push "${ghcr_image}:${{ matrix.docker-image-name }}" | |
| - name: Chown workspace | |
| uses: ./.github/actions/chown-workspace | |
| with: | |
| ALPINE_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/${{ contains(matrix.runner, 'arm64') && 'arm64v8' || 'tool' }}/alpine | |
| if: always() | |
| - name: Teardown Linux | |
| uses: pytorch/test-infra/.github/actions/teardown-linux@main | |
| if: always() |