Skip to content

Commit 10473d0

Browse files
erubbolierubboli
committed
ci(docker): support arm64 builds and multi-arch manifest assembly
build.py gains --platform (amd64|arm64) and --tag_suffix: tags are suffixed inside build/push (with a local alias kept unsuffixed so app image builds can still reference the base images), and the version input stays unsuffixed to avoid double-suffixing. release_docker.yml becomes platform-aware via workflow_dispatch inputs (version, platform, tag_suffix, runner) so arm64 images can be built natively (ubuntu-24.04-arm) and pushed as '<tag>-arm64' suffixed tags. The new assemble_manifests input runs a registry-side job that turns per-arch suffixed tags into multi-arch manifest lists under the unsuffixed tags (docker buildx imagetools create; nothing rebuilt, idempotent snapshot of the historical amd64 layout under explicit '-amd64' tags first). Kept in the same workflow because GitHub validates workflow registrations against the default branch, so a separate dispatch-only workflow could not be used from a release branch. Note: single-arch release builds (workflow_call) behave exactly as before.
1 parent 3b3867e commit 10473d0

3 files changed

Lines changed: 174 additions & 24 deletions

File tree

‎.github/workflows/release_docker.yml‎

Lines changed: 116 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,33 @@ name: Build Docker
22

33
on:
44
workflow_call:
5+
workflow_dispatch:
6+
inputs:
7+
version:
8+
description: 'Version for image tags, e.g. 1.4.1 (defaults to the short commit SHA when empty)'
9+
required: false
10+
default: ''
11+
platform:
12+
description: 'Target CPU architecture (amd64 or arm64)'
13+
required: false
14+
default: 'amd64'
15+
tag_suffix:
16+
description: 'Suffix appended to every image tag, e.g. arm64 (leave empty for the default arch)'
17+
required: false
18+
default: ''
19+
runner:
20+
description: 'Runner label, e.g. ubuntu-24.04-arm for a native arm64 build'
21+
required: false
22+
default: 'ubuntu-latest'
23+
assemble_manifests:
24+
description: "Skip image builds and only assemble multi-arch manifests from already-pushed '<tag>-amd64'/'<tag>-arm64' tags"
25+
required: false
26+
default: 'false'
527

628
jobs:
729
build-docker-images:
8-
runs-on: ubuntu-latest
30+
if: github.event_name == 'workflow_call' || inputs.assemble_manifests != 'true'
31+
runs-on: ${{ inputs.runner || 'ubuntu-latest' }}
932
steps:
1033
- name: Checkout code
1134
uses: actions/checkout@v5
@@ -33,20 +56,28 @@ jobs:
3356

3457
- name: Determine version
3558
id: version
59+
env:
60+
INPUT_VERSION: ${{ inputs.version }}
3661
run: |
37-
if [[ $GITHUB_REF == refs/tags/* ]]; then
38-
VERSION=${GITHUB_REF#refs/tags/}
62+
validate() {
3963
# Validate the grammar up front (the version flows into docker build/push
40-
# command lines inside build.py), so a non-semver tag fails here with a
64+
# command lines inside build.py), so a bad version fails here with a
4165
# clear message. The leading 'v' is allowed here; build.py strips it.
42-
if ! echo "$VERSION" | grep -Eq '^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$'; then
43-
echo "derived version '$VERSION' is not [v]X.Y.Z[-suffix]; tag a semver release" >&2
66+
if ! echo "$1" | grep -Eq '^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$'; then
67+
echo "derived version '$1' is not [v]X.Y.Z[-suffix]; tag a semver release or pass a valid version input" >&2
4468
exit 1
4569
fi
46-
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
70+
}
71+
if [[ $GITHUB_REF == refs/tags/* ]]; then
72+
VERSION=${GITHUB_REF#refs/tags/}
73+
validate "$VERSION"
74+
elif [[ -n "$INPUT_VERSION" ]]; then
75+
VERSION="$INPUT_VERSION"
76+
validate "$VERSION"
4777
else
48-
echo "VERSION=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
78+
VERSION=$(git rev-parse --short HEAD)
4979
fi
80+
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
5081
5182
# Note: the version (and the Docker Hub username) are passed through environment
5283
# variables instead of direct ${{ }} interpolation into the run block, so that a
@@ -55,9 +86,84 @@ jobs:
5586
env:
5687
VERSION: ${{ steps.version.outputs.VERSION }}
5788
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
89+
PLATFORM: ${{ inputs.platform || 'amd64' }}
90+
TAG_SUFFIX: ${{ inputs.tag_suffix }}
5891
run: |
59-
python build-tools/docker/build.py --push --docker-hub-user "$DOCKERHUB_USERNAME" \
60-
--version "$VERSION" --latest
92+
args=(--push --docker-hub-user "$DOCKERHUB_USERNAME" \
93+
--version "$VERSION" --latest --platform "$PLATFORM")
94+
if [[ -n "$TAG_SUFFIX" ]]; then
95+
args+=(--tag_suffix "$TAG_SUFFIX")
96+
fi
97+
python build-tools/docker/build.py "${args[@]}"
6198
6299
env:
63100
DOCKER_BUILDKIT: 1
101+
102+
# Registry-side assembly of multi-arch manifest lists from images that were
103+
# pushed earlier by separate per-arch dispatches of the build job above
104+
# ("--platform arm64 --tag_suffix arm64"). Nothing is rebuilt: "docker
105+
# buildx imagetools create" only references already-pushed manifests. Runs
106+
# from the default branch via "gh workflow run ... --ref <branch>": GitHub
107+
# validates the workflow registration against the default branch but reads
108+
# the file content from the dispatched ref.
109+
assemble-manifests:
110+
if: github.event_name == 'workflow_dispatch' && inputs.assemble_manifests == 'true'
111+
runs-on: ubuntu-latest
112+
steps:
113+
- name: Set up Docker Buildx
114+
uses: docker/setup-buildx-action@v2
115+
116+
- name: Login to Docker Hub
117+
uses: docker/login-action@v2
118+
with:
119+
username: ${{ secrets.DOCKERHUB_USERNAME }}
120+
password: ${{ secrets.DOCKERHUB_TOKEN }}
121+
122+
- name: Assemble and push multi-arch manifests
123+
env:
124+
INPUT_VERSION: ${{ inputs.version }}
125+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
126+
run: |
127+
set -euo pipefail
128+
129+
VERSION="${INPUT_VERSION#v}"
130+
if ! echo "$VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$'; then
131+
echo "invalid version '$INPUT_VERSION'; expected [v]X.Y.Z[-suffix]" >&2
132+
exit 1
133+
fi
134+
135+
assemble() {
136+
local repo="$1" tag="$2"
137+
# Snapshot the existing (historically amd64-only) manifest under an
138+
# explicit arch tag, unless it's already there (idempotent re-runs).
139+
if ! docker buildx imagetools inspect "$repo:$tag-amd64" >/dev/null 2>&1; then
140+
docker buildx imagetools create -t "$repo:$tag-amd64" "$repo:$tag"
141+
fi
142+
docker buildx imagetools create -t "$repo:$tag" "$repo:$tag-amd64" "$repo:$tag-arm64"
143+
}
144+
145+
# Only the app images are maintained on Docker Hub (build.py's
146+
# push_instances pushes these six); the builder/runner-base images
147+
# stay local to the CI runner, so there is nothing to assemble for
148+
# them.
149+
for repo in "$DOCKERHUB_USERNAME/node-daemon" \
150+
"$DOCKERHUB_USERNAME/api-blockchain-scanner-daemon" \
151+
"$DOCKERHUB_USERNAME/api-web-server" \
152+
"$DOCKERHUB_USERNAME/wallet-cli" \
153+
"$DOCKERHUB_USERNAME/wallet-rpc-daemon" \
154+
"$DOCKERHUB_USERNAME/dns-server"; do
155+
for tag in "$VERSION" "v$VERSION" latest; do
156+
assemble "$repo" "$tag"
157+
done
158+
done
159+
160+
- name: Verify manifests
161+
env:
162+
INPUT_VERSION: ${{ inputs.version }}
163+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
164+
run: |
165+
VERSION="${INPUT_VERSION#v}"
166+
for repo in node-daemon api-blockchain-scanner-daemon api-web-server wallet-cli wallet-rpc-daemon dns-server; do
167+
echo "=== $DOCKERHUB_USERNAME/$repo:v$VERSION ==="
168+
docker buildx imagetools inspect "$DOCKERHUB_USERNAME/$repo:v$VERSION" | grep -E '^Name:|Platform:' || true
169+
done

‎build-tools/docker/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,3 +61,12 @@ docker run -it --network=mintlayer-net -v ~/.mintlayer:/root/.mintlayer mintlaye
6161
replace `<NETWORK>` with `mainnet` or `testnet` depending on what network you're running and `<IP_ADDRESS>` with the result of the command above.
6262

6363
This command mounts the same `~/.mintlayer` directory as a volume in the `wallet-cli` container and uses the `--rpc-cookie-file` option to specify the path to the cookie file.
64+
65+
## Multi-arch (amd64 / arm64) images
66+
67+
By default the build produces `linux/amd64` images, exactly as it always has. Two extra knobs exist for building other architectures:
68+
69+
- `build.py --platform arm64` builds `linux/arm64` images (must run on an arm64 machine, e.g. the `ubuntu-24.04-arm` GitHub runner, since the compilation happens inside the container).
70+
- `build.py --tag_suffix arm64` appends the suffix to every built and pushed tag (e.g. `node-daemon:1.4.1-arm64`), so a non-default architecture never overwrites the default-arch tags.
71+
72+
To assemble the suffixed images into a proper multi-arch manifest list under the unsuffixed tags (`docker pull mintlayer/node-daemon` then auto-selects the architecture), dispatch the **"Assemble multi-arch Docker manifests"** workflow (`release_docker_manifest.yml`) with the release version after both architectures have been pushed.

‎build-tools/docker/build.py‎

Lines changed: 49 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,18 @@ def get_cargo_version(cargo_toml_path):
2626
return version
2727

2828

29-
def build_docker_image(dockerfile_path, image_name, tags, num_jobs=None):
29+
def apply_tag_suffix(tags, tag_suffix):
30+
# Append the suffix (e.g. "arm64") to every tag, so that images built for a
31+
# non-default architecture never overwrite the tags of the default-arch images.
32+
if not tag_suffix:
33+
return list(tags)
34+
return [f"{tag}-{tag_suffix}" for tag in tags]
35+
36+
37+
def build_docker_image(dockerfile_path, image_name, tags, num_jobs=None, platform="amd64", tag_suffix=None):
38+
raw_tags = list(tags)
39+
tags = apply_tag_suffix(raw_tags, tag_suffix)
40+
3041
# Docker build command
3142
command = f"docker build"
3243

@@ -40,8 +51,7 @@ def build_docker_image(dockerfile_path, image_name, tags, num_jobs=None):
4051
if num_jobs:
4152
command += f" --build-arg NUM_JOBS={num_jobs}"
4253

43-
# Force the amd64 platform in case we're building on an arm-based one.
44-
command += " --platform linux/amd64"
54+
command += f" --platform linux/{platform}"
4555

4656
# Note: "plain" output is more verbose, but it makes it easier to understand what went wrong
4757
# when a problem occurs.
@@ -56,6 +66,24 @@ def build_docker_image(dockerfile_path, image_name, tags, num_jobs=None):
5666
print(f"Failed to build {image_name}: {error}")
5767
exit(1) # stop the build
5868

69+
if tag_suffix:
70+
# The app images reference "mintlayer-builder:latest" / "mintlayer-runner-base"
71+
# without a suffix in their FROM lines. When building for a non-default
72+
# architecture, the suffixed base image exists locally but the unsuffixed tag
73+
# would be resolved from the registry (where only the amd64 variant exists).
74+
# Add a local unsuffixed alias pointing to the image we've just built, so the
75+
# subsequent app image builds resolve it locally. These local aliases are never
76+
# pushed (push_instances only pushes the suffixed tags), so they cannot
77+
# overwrite the registry's default-arch tags.
78+
local_alias = f"{image_name}:{raw_tags[0]}"
79+
alias_command = f"docker tag {full_tags[0]} {local_alias}"
80+
try:
81+
subprocess.check_call(alias_command, shell=True)
82+
print(f"Tagged {full_tags[0]} locally as {local_alias}.")
83+
except subprocess.CalledProcessError as error:
84+
print(f"Failed to tag {image_name}: {error}")
85+
exit(1) # stop the build
86+
5987

6088
def push_docker_image(image_name, tags):
6189
for tag in tags:
@@ -86,24 +114,24 @@ def delete_docker_image(image_name, version):
86114
# No need to fail the build here
87115

88116

89-
def build_instances(tags, docker_hub_user, num_jobs):
117+
def build_instances(tags, docker_hub_user, num_jobs, platform="amd64", tag_suffix=None):
90118
build_docker_image("build-tools/docker/Dockerfile.builder",
91-
"mintlayer-builder", ["latest"], num_jobs)
119+
"mintlayer-builder", ["latest"], num_jobs, platform, tag_suffix)
92120
build_docker_image("build-tools/docker/Dockerfile.runner-base",
93-
"mintlayer-runner-base", ["latest"], num_jobs)
121+
"mintlayer-runner-base", ["latest"], num_jobs, platform, tag_suffix)
94122

95123
build_docker_image("build-tools/docker/Dockerfile.node-daemon",
96-
f"{docker_hub_user}/node-daemon", tags)
124+
f"{docker_hub_user}/node-daemon", tags, num_jobs, platform, tag_suffix)
97125
build_docker_image("build-tools/docker/Dockerfile.api-blockchain-scanner-daemon",
98-
f"{docker_hub_user}/api-blockchain-scanner-daemon", tags)
126+
f"{docker_hub_user}/api-blockchain-scanner-daemon", tags, num_jobs, platform, tag_suffix)
99127
build_docker_image("build-tools/docker/Dockerfile.api-web-server",
100-
f"{docker_hub_user}/api-web-server", tags)
128+
f"{docker_hub_user}/api-web-server", tags, num_jobs, platform, tag_suffix)
101129
build_docker_image("build-tools/docker/Dockerfile.wallet-cli",
102-
f"{docker_hub_user}/wallet-cli", tags)
130+
f"{docker_hub_user}/wallet-cli", tags, num_jobs, platform, tag_suffix)
103131
build_docker_image("build-tools/docker/Dockerfile.wallet-rpc-daemon",
104-
f"{docker_hub_user}/wallet-rpc-daemon", tags)
132+
f"{docker_hub_user}/wallet-rpc-daemon", tags, num_jobs, platform, tag_suffix)
105133
build_docker_image("build-tools/docker/Dockerfile.dns-server",
106-
f"{docker_hub_user}/dns-server", tags)
134+
f"{docker_hub_user}/dns-server", tags, num_jobs, platform, tag_suffix)
107135
# delete_docker_image("mintlayer-builder", "latest")
108136

109137

@@ -125,6 +153,10 @@ def main():
125153
parser.add_argument('--version', help='Override version number', default=None)
126154
parser.add_argument('--num_jobs', help='Number of parallel jobs', default=(os.cpu_count() or 1))
127155
parser.add_argument('--local_tags', nargs='*', help='Additional tags to apply (these won\'t be pushed)', default=[])
156+
parser.add_argument('--platform', choices=['amd64', 'arm64'], default='amd64',
157+
help='Target CPU architecture for the images (the default keeps the historical behavior)')
158+
parser.add_argument('--tag_suffix', help='Suffix appended to every built and pushed image tag (e.g. "arm64")',
159+
default=None)
128160
args = parser.parse_args()
129161

130162
version = args.version if args.version else get_cargo_version(ROOT_CARGO_TOML)
@@ -140,11 +172,14 @@ def main():
140172
all_tags = args.local_tags + tags_to_push
141173

142174
if args.build:
143-
build_instances(all_tags, args.docker_hub_user, args.num_jobs)
175+
# Note: the suffix is applied inside build_docker_image, so it must NOT be
176+
# pre-applied to these tags (that would produce doubled suffixes).
177+
build_instances(all_tags, args.docker_hub_user, args.num_jobs,
178+
args.platform, args.tag_suffix)
144179

145180
# Only push the image if the --push flag is provided
146181
if args.push:
147-
push_instances(args.docker_hub_user, tags_to_push)
182+
push_instances(args.docker_hub_user, apply_tag_suffix(tags_to_push, args.tag_suffix))
148183

149184

150185
if __name__ == "__main__":

0 commit comments

Comments
 (0)