diff --git a/.github/workflows/build_container_image.yml b/.github/workflows/build_container_image.yml new file mode 100644 index 00000000..6a523834 --- /dev/null +++ b/.github/workflows/build_container_image.yml @@ -0,0 +1,259 @@ +name: Build Container Image + +on: + workflow_call: + inputs: + registry: + description: 'Container registry' + required: true + type: string + name: + description: 'Image name, without registry or tags (repo/name)' + required: true + type: string + tag: + description: 'Tag for manifest creation' + required: true + type: string + build-runners: + description: 'JSON array of runners to build on (e.g. ["ubuntu-24.04", "ubuntu-24.04-arm"])' + required: false + type: string + default: '["ubuntu-24.04", "ubuntu-24.04-arm"]' + merge-runner: + description: 'Runner for merge job' + required: false + type: string + default: 'ubuntu-24.04' + dockerfile: + description: 'Path to Dockerfile' + required: false + type: string + default: 'Dockerfile' + context: + description: 'Build context path' + required: false + type: string + default: '.' + build-args: + description: 'Build arguments as multi-line string (e.g. "ARG1=value1\nARG2=value2")' + required: false + type: string + default: '' + build-args-for-arch: + description: 'Architecture-specific build arguments as JSON object (e.g. {"x86_64": "ARG1=value1\nARG2=value2", "aarch64": "ARG3=value3"})' + required: false + type: string + default: '{}' + target: + description: 'Target stage in multi-stage Dockerfile' + required: false + type: string + default: '' + buildkit-config: + description: 'BuildKit daemon configuration' + required: false + type: string + default: '' + cache-prefix: + description: 'Prefix for cache image names' + required: false + type: string + default: '' + update-cache: + description: 'Enable cache-to for pushing updated cache layers' + required: false + type: boolean + default: false + artifact-name: + description: 'Name of the artifact to be downloaded before build' + required: false + type: string + default: '' + artifact-path: + description: 'Directory where the artifact should be unpacked' + required: false + type: string + default: '.' + checkout-submodules: + description: 'Whether to checkout git submodules' + required: false + type: boolean + default: false + checkout-ref: + description: 'Git ref to checkout' + required: false + type: string + default: '' + checkout-path: + description: 'Path to checkout' + required: false + type: string + default: '.' + + outputs: + digest: + description: 'Digest of the multi-arch image' + value: ${{ jobs.merge.outputs.digest }} + ref-with-digest: + description: 'Full image reference with digest' + value: ${{ jobs.merge.outputs.ref-with-digest }} + + secrets: + checkout-token: + description: 'GitHub token for checkout' + required: false + registry-user: + required: false + registry-password: + required: false + build-secrets: + description: 'Build secrets as a multi-line string (e.g. "SECRET1=value1\nSECRET2=value2")' + required: false + +permissions: {} + +defaults: + run: + shell: bash -euo pipefail {0} + +jobs: + build-image: + runs-on: ${{ matrix.runner }} + + strategy: + fail-fast: false + matrix: + runner: ${{ fromJSON(inputs.build-runners) }} + + permissions: + contents: read + packages: write + + outputs: + digest_x86_64: ${{ steps.digest.outputs.digest_x86_64 }} + digest_aarch64: ${{ steps.digest.outputs.digest_aarch64 }} + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: ${{ inputs.checkout-submodules }} + ref: ${{ inputs.checkout-ref }} + token: ${{ secrets.checkout-token != '' && secrets.checkout-token || github.token }} + path: ${{ inputs.checkout-path }} + + - name: Download artifact + if: ${{ inputs.artifact-name != '' }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: ${{ inputs.artifact-path }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + with: + cache-binary: false + buildkitd-config-inline: ${{ inputs.buildkit-config || '' }} + + - name: Login to Container Registry + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ${{ inputs.registry }} + username: ${{ secrets.registry-user != '' && secrets.registry-user || (inputs.registry == 'ghcr.io' && github.actor) }} + password: ${{ secrets.registry-password != '' && secrets.registry-password || (inputs.registry == 'ghcr.io' && github.token) }} + + - name: Detect architecture + id: arch + run: | + arch=$(uname -m) + echo "arch=${arch}" | tee -a "$GITHUB_OUTPUT" + + - name: Build and push architecture-specific image + id: build + env: + CACHE_KEY: ${{ inputs.cache-prefix && format('type=registry,ref={0}/{1}:{2}-{3}', inputs.registry, inputs.name, inputs.cache-prefix, steps.arch.outputs.arch) || '' }} + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ inputs.context }} + file: ${{ inputs.dockerfile }} + push: true + pull: true + target: ${{ inputs.target }} + build-args: ${{ format('{0}\n{1}', inputs.build-args, fromJson(inputs.build-args-for-arch)[steps.arch.outputs.arch] || '') }} + secrets: ${{ secrets.build-secrets }} + cache-from: ${{ env.CACHE_KEY }} + cache-to: ${{ inputs.update-cache == 'true' && format('{0},image-manifest=true,oci-mediatypes=true,mode=max', env.CACHE_KEY) || '' }} + attests: | + type=provenance,mode=max + type=sbom,generator=${{ contains(inputs.registry, 'databricks.com') && format('{0}/brickstore/neon/docker/buildkit-syft-scanner:1', inputs.registry) || 'docker.io/docker/buildkit-syft-scanner:1' }} + outputs: type=registry,name=${{ inputs.registry }}/${{ inputs.name }},push-by-digest=true,name-canonical=true + + - name: Export digest for architecture + id: digest + run: | + digest="${{ steps.build.outputs.digest }}" + echo "digest_$(uname -m)=${digest}" | tee -a "$GITHUB_OUTPUT" + + merge: + runs-on: ${{ inputs.merge-runner }} + needs: [build-image] + if: always() && !cancelled() && !failure() + + permissions: + contents: read + packages: write + + env: + IMAGE_REF: ${{ inputs.registry }}/${{ inputs.name }} + + outputs: + digest: ${{ steps.merge.outputs.digest }} + ref-with-digest: ${{ steps.merge.outputs.ref-with-digest }} + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Login to Container Registry + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ${{ inputs.registry }} + username: ${{ secrets.registry-user != '' && secrets.registry-user || (inputs.registry == 'ghcr.io' && github.actor) }} + password: ${{ secrets.registry-password != '' && secrets.registry-password || (inputs.registry == 'ghcr.io' && github.token) }} + + - name: Prepare references + id: prepare + env: + DIGEST_X86_64: ${{ needs.build-image.outputs.digest_x86_64 }} + DIGEST_AARCH64: ${{ needs.build-image.outputs.digest_aarch64 }} + run: | + # Parse environment variables and create references in one go + references_data=$(printenv | jq -Rsc '[split("\n")[] | capture("^DIGEST_(?[^=]+)=(?.+)$") | .arch |= ascii_downcase]') + + # Verify we have at least one digest + if [[ "$(echo "$references_data" | jq -r 'length')" -eq 0 ]]; then + echo "::error::No digest values found! Cannot create manifest list without at least one input reference." + exit 1 + fi + + # Log found architectures and their digests + echo "$references_data" | jq -r '.[] | "Found digest for \(.arch): \(.digest)"' + + # Create space-separated references string for the composite action + references_string=$(echo "$references_data" | jq -r --arg ref "$IMAGE_REF" 'map($ref + "@" + .digest) | join(" ")') + echo "references=${references_string}" >> "$GITHUB_OUTPUT" + + - name: Merge OCI manifest lists + id: merge + uses: ./merge-oci-manifest-lists + with: + target: ${{ inputs.registry }}/${{ inputs.name }}:${{ inputs.tag }} + references: ${{ steps.prepare.outputs.references }} + + - name: Fetch manifest digest references + id: digests + uses: ./fetch-oci-manifest-list-digest-references \ No newline at end of file diff --git a/.github/workflows/mutexbot-cleanup.yml b/.github/workflows/mutexbot-cleanup.yml index 2d061630..89305019 100644 --- a/.github/workflows/mutexbot-cleanup.yml +++ b/.github/workflows/mutexbot-cleanup.yml @@ -4,7 +4,7 @@ on: pull_request: types: ["closed"] branches: ["main"] - paths: ["mutexbot/**"] + paths: ["mutexbot/**", ".github/workflows/mutexbot*.yml"] permissions: {} diff --git a/.github/workflows/mutexbot.yml b/.github/workflows/mutexbot.yml index bcb640fb..4effa94b 100644 --- a/.github/workflows/mutexbot.yml +++ b/.github/workflows/mutexbot.yml @@ -4,123 +4,39 @@ on: pull_request: types: ["opened", "synchronize", "reopened"] branches: ["main"] - paths: ["mutexbot/**"] + paths: ["mutexbot/**", ".github/workflows/mutexbot*.yml"] push: branches: ["main"] - paths: ["mutexbot/**"] + paths: ["mutexbot/**", ".github/workflows/mutexbot*.yml"] tags: ["mutexbot-v*.*.*"] permissions: contents: read -env: - GHCR_REPO: ghcr.io/neondatabase/dev-actions - jobs: build-image: - runs-on: ${{ matrix.runner }} - outputs: - digest_x86_64: ${{ steps.export_digest.outputs.digest_x86_64 }} - digest_aarch64: ${{ steps.export_digest.outputs.digest_aarch64 }} - strategy: - fail-fast: false - matrix: - runner: - - ubuntu-24.04 - - ubuntu-24.04-arm + uses: ./.github/workflows/build_container_image.yml permissions: contents: read packages: write - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 - with: - egress-policy: audit - - - name: Fetch mutexbot folder - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - sparse-checkout: mutexbot - - - name: Docker meta - id: meta - uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 - with: - images: ${{ env.GHCR_REPO }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - - name: Login to GHCR - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0 - with: - context: mutexbot - labels: ${{ steps.meta.outputs.labels }} - attests: | - type=provenance,mode=max - type=sbom,generator=docker.io/docker/buildkit-syft-scanner:1 - outputs: type=image,name=${{ env.GHCR_REPO }},push-by-digest=true,name-canonical=true,push=true - - - name: Export digest - id: export_digest - run: | - digest="${{ steps.build.outputs.digest }}" - echo "digest_$(uname -m)=${digest#sha256:}" | tee -a "$GITHUB_OUTPUT" - - merge-image: - runs-on: ["self-hosted", "small"] - needs: [build-image] - permissions: - contents: read - packages: write - outputs: - version: ${{ steps.meta.outputs.version }} - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 - with: - egress-policy: audit - - - name: Login to GHCR - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - - name: Docker meta - id: meta - uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 - with: - images: ${{ env.GHCR_REPO }} - tags: | - # branch event - type=ref,enable=true,priority=600,prefix=mutexbot-,suffix=,event=branch - # pull request event - type=ref,enable=true,priority=600,prefix=mutexbot-pr-,suffix=,event=pr - # tags event - type=match,pattern=mutexbot-v(.*) - - - name: Create manifest list and push - run: | - docker buildx imagetools create \ - $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - ${{ env.GHCR_REPO }}@sha256:${{ needs.build-image.outputs.digest_aarch64 }} \ - ${{ env.GHCR_REPO }}@sha256:${{ needs.build-image.outputs.digest_x86_64 }} - - - name: Inspect image - run: docker buildx imagetools inspect ${{ env.GHCR_REPO }}:${{ steps.meta.outputs.version }} + attestations: write + id-token: write + with: + registry: ghcr.io + name: neondatabase/dev-actions + tag: >- + ${{ + (github.event_name == 'pull_request' && format('mutexbot-pr-{0}', github.event.pull_request.number)) + || (github.ref_type == 'tag' && github.ref_name) + || format('mutexbot-{0}', github.ref_name) + }} + context: mutexbot + dockerfile: mutexbot/Dockerfile + cache-prefix: mutexbot + update-cache: ${{ github.ref == 'refs/heads/main' }} + secrets: + registry-user: ${{ github.actor }} + registry-password: ${{ secrets.GITHUB_TOKEN }} build-binary: runs-on: ${{ matrix.runner }} @@ -173,7 +89,7 @@ jobs: - name: Fetch artifacts uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - pattern: "!*.dockerbuild" + pattern: "mutexbot-*" - name: Create archives run: | diff --git a/fetch-oci-manifest-list-digest-references/action.yml b/fetch-oci-manifest-list-digest-references/action.yml new file mode 100644 index 00000000..62992408 --- /dev/null +++ b/fetch-oci-manifest-list-digest-references/action.yml @@ -0,0 +1,39 @@ +name: 'Fetch Manifest Digest References' +description: 'Downloads container image digest artifacts and creates a map from tag to digest reference' + +outputs: + map: + description: 'JSON map from image tag to digest reference (e.g., {"ghcr.io/myorg/myapp:v1.2.3": "ghcr.io/myorg/myapp@sha256:..."})' + value: ${{ steps.create-map.outputs.map }} + +runs: + using: 'composite' + steps: + - name: Download container image digests + uses: actions/download-artifact@v4 + with: + pattern: container-image-digest-* + merge-multiple: false + + - name: Create tag to digest reference map + id: create-map + shell: bash + run: | + tag_digest_map=$( + find . -name "digest-info*.json" -type f -exec jq -c --arg filepath {} ' + { + "tag": ($filepath | capture("container-image-digest-(?[^/]+)/")) | (.base64_key | @base64d), + "digest_ref": .["ref-with-digest"], + } + ' {} \; | jq -sc ' + reduce .[] as $item ({}; + .[$item.tag] = $item.digest_ref + ) + ' + ) + + # Log the mappings + echo "$tag_digest_map" | jq + + # Set output + echo "map=$tag_digest_map" >> "$GITHUB_OUTPUT" diff --git a/merge-oci-manifest-lists/action.yml b/merge-oci-manifest-lists/action.yml new file mode 100644 index 00000000..2a94bc94 --- /dev/null +++ b/merge-oci-manifest-lists/action.yml @@ -0,0 +1,62 @@ +name: 'Merge OCI Manifest Lists' +description: 'Creates multi-arch manifest from architecture-specific image references and uploads digest artifact' + +inputs: + target: + description: 'Target image reference (e.g., "ghcr.io/myorg/myapp:v1.2.3")' + required: true + references: + description: 'Space separated image references to merge (e.g., "ghcr.io/myorg/myapp@sha256:... ghcr.io/myorg/myapp@sha256:..." or "ghcr.io/myorg/myapp:v1.2.3-aarch64 ghcr.io/myorg/myapp:v1.2.3-x86_64")' + required: true + +outputs: + digest: + description: 'Manifest digest' + value: ${{ steps.create.outputs.digest }} + ref-with-digest: + description: 'Manifest reference with digest' + value: ${{ steps.create.outputs.ref-with-digest }} + +runs: + using: 'composite' + steps: + - name: Create multi-arch manifest and upload artifact + id: create + shell: bash + env: + TARGET: ${{ inputs.target }} + REFERENCES: ${{ inputs.references }} + run: | + # Calculate base64 key once for file naming + image_key=$(echo -n "${TARGET}" | base64) + + # Create multi-arch manifest and capture output + buildx_create_output_file="create_output_${image_key}.txt" + docker buildx imagetools create -t "${TARGET}" ${REFERENCES} 2>&1 | tee "${buildx_create_output_file}" + + # Extract digest and target from captured output using jq with regex + manifest_digest=$(cat "${buildx_create_output_file}" | jq -Rsr --arg target "${TARGET}" ' + capture("pushing (?sha256:[a-f0-9]{64}) to (?.+)") // ("Error: Found no pushing line in output\n" | halt_error(1)) + | if .target == $target then .digest else "Error: Found no digest for \($target)\n" | halt_error(1) end') + + echo "digest=${manifest_digest}" | tee -a "$GITHUB_OUTPUT" + echo "ref-with-digest=${TARGET}@${manifest_digest}" | tee -a "$GITHUB_OUTPUT" + + # Create digest info JSON file for artifact + cat > "digest-info-${image_key}.json" << EOF + { + "digest": "${manifest_digest}", + "ref-with-digest": "${TARGET}@${manifest_digest}" + } + EOF + + # Generate artifact name and store outputs + echo "image_key=${image_key}" | tee -a "$GITHUB_OUTPUT" + echo "Multi-arch image created: ${TARGET}@${manifest_digest}" + + - name: Upload digest artifact + uses: actions/upload-artifact@v4 + with: + name: container-image-digest-${{ steps.create.outputs.image_key }} + path: digest-info-${{ steps.create.outputs.image_key }}.json + retention-days: 1