Skip to content

Commit f7c3695

Browse files
authored
Merge pull request #265 from Calnode/fix/manifest-selftest-network-vs-missing
fix(ns8): tell a registry blip apart from a missing image in the manifest selftest
2 parents fcf6c3d + 80f7503 commit f7c3695

1 file changed

Lines changed: 59 additions & 6 deletions

File tree

‎packaging/ns8/push-manifest.sh‎

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,50 @@ if [ "${#arches[@]}" -eq 0 ]; then
4747
exit 1
4848
fi
4949

50+
# A network blip and a missing image both come back as a failed `manifest add`,
51+
# and telling them apart matters in both directions. Retrying a
52+
# genuine 404 hides a build that never ran. Reporting a TCP reset as a missing
53+
# tag sends the reader to rebuild and push an image that already exists, which
54+
# is real pointless work: that is exactly what a reset during this script's CI
55+
# selftest used to say.
56+
#
57+
# So: classify, retry only transport failures, and let the message name the case.
58+
retry_attempts="${RETRY_ATTEMPTS:-3}"
59+
retry_delay="${RETRY_DELAY:-5}"
60+
61+
# Transport failures. Deliberately a specific list rather than a broad "error"
62+
# match, because the fallback has to stay fail-fast: anything we do not recognise
63+
# is a real failure, not something to retry until the job times out.
64+
is_network_error() {
65+
grep -qiE 'connection reset|connection refused|i/o timeout|context deadline exceeded|timed out|temporary failure in name resolution|no such host|network is unreachable|tls handshake|unexpected eof|(^|[^0-9])(502|503|504)([^0-9]|$)|bad gateway|service unavailable|gateway timeout|internal server error'
66+
}
67+
68+
# add_arch RETRIES the add, then records why it gave up in ADD_OUTPUT/ADD_KIND
69+
# so the caller can print the right message. Returns 0 on success.
70+
add_arch() {
71+
local src="$1" attempt=1 delay="$retry_delay" output
72+
while :; do
73+
if output=$(buildah manifest add "${manifest}" "${src}" 2>&1); then
74+
return 0
75+
fi
76+
if printf '%s' "$output" | is_network_error; then
77+
if [ "$attempt" -lt "$retry_attempts" ]; then
78+
echo "Registry unreachable (attempt ${attempt}/${retry_attempts})," \
79+
"retrying in ${delay}s: ${src}" >&2
80+
sleep "$delay"
81+
attempt=$((attempt + 1))
82+
delay=$((delay * 2))
83+
continue
84+
fi
85+
ADD_KIND=network
86+
else
87+
ADD_KIND=missing
88+
fi
89+
ADD_OUTPUT="$output"
90+
return 1
91+
done
92+
}
93+
5094
for imagetag in "${imagetags[@]}"; do
5195
manifest="${repobase}/${reponame}:${imagetag}"
5296
# Drop any stale manifest so a retry cannot inherit old entries.
@@ -74,12 +118,21 @@ for imagetag in "${imagetags[@]}"; do
74118
#
75119
# Exactly two arguments: buildah 5.x takes only the list and the image,
76120
# with no instance-name argument.
77-
if ! buildah manifest add "${manifest}" \
78-
"${source_prefix}${repobase}/${reponame}:build-${arch}"; then
79-
echo "Failed to add ${source_prefix}${repobase}/${reponame}:build-${arch}" \
80-
"to ${manifest}." >&2
81-
echo "If that tag does not exist in the registry, run build-images.sh" \
82-
"with TARGETARCH=${arch} PUSH=1 first." >&2
121+
if ! add_arch "${source_prefix}${repobase}/${reponame}:build-${arch}"; then
122+
src="${source_prefix}${repobase}/${reponame}:build-${arch}"
123+
# Show what the tool actually said, then say which case it is.
124+
printf '%s\n' "${ADD_OUTPUT}" | tail -n 4 | sed 's/^/ /' >&2
125+
if [ "${ADD_KIND}" = "network" ]; then
126+
echo "The registry was unreachable while reading ${src}." >&2
127+
echo "This is a TRANSPORT failure, not a missing image: the tag was" >&2
128+
echo "never reported absent and nothing is wrong with the build. Do" >&2
129+
echo "not rebuild or push the image. Re-run the job; if it keeps" >&2
130+
echo "failing, check connectivity to the registry from this host." >&2
131+
else
132+
echo "Failed to add ${src}" "to ${manifest}." >&2
133+
echo "If that tag does not exist in the registry, run build-images.sh" \
134+
"with TARGETARCH=${arch} PUSH=1 first." >&2
135+
fi
83136
buildah manifest rm "${manifest}" 2>/dev/null || true
84137
exit 1
85138
fi

0 commit comments

Comments
 (0)