Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
273 changes: 211 additions & 62 deletions task/iib-image-builder-oci-ta/multi-arch-builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,11 @@
import shutil
import subprocess
import sys
import tempfile
from concurrent.futures import ThreadPoolExecutor, as_completed

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[low] code-organization

Import ordering: from typing import Any placed before from dataclasses breaks the established alphabetical convention for from-import lines.

Suggested fix: Reorder the from-import block: concurrent.futures, dataclasses, pathlib, typing.

from typing import Any
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

from tenacity import (
before_sleep_log,
Expand Down Expand Up @@ -133,9 +135,10 @@ def run_cmd(
):
raise IIBError("Error deleting packages from database")
elif cmd[0] == "buildah":
# Check for HTTP 403 or 50X errors on buildah
# Check for transient network/registry errors on any buildah operation
network_regexes = [
r".*([e,E]rror:? creating build container).*(:?(403|50[0-9]|125)\s?.*$)",
r".*((?:403 Forbidden|50[0-9] \w+).*$)",

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[low] edge-case

The new regex classifies HTTP 403 as transient ExternalServiceError and retries it, but 403 Forbidden typically indicates permanent authorization failure.

r".*(read\/write on closed pipe.*$)",
]
for regex in network_regexes:
Expand Down Expand Up @@ -327,6 +330,9 @@ def arches_from_metadata(metadata: dict[str, Any]) -> list[str] | None:
"array must contain at least one architecture"
)

if len(arches) != len(set(arches)):
raise IIBError("Invalid arches in IIB build metadata file: duplicate architectures")

return arches


Expand Down Expand Up @@ -510,7 +516,6 @@ def _build_image(self, arch: str, destination: str) -> None:
destination,
)

# Prepare buildah command with improved options
cmd = [
"buildah",
"bud",
Expand All @@ -530,20 +535,16 @@ def _build_image(self, arch: str, destination: str) -> None:
self.config.dockerfile_path,
]

# Add labels
for label in self.config.labels:
cmd.extend(["--label", label.strip()])

if self.config.binary_image:
cmd.extend(["--build-arg", f"BINARY_IMAGE={self.config.binary_image}"])

# Add context
cmd.append(self.config.context_path)

# Execute build with retry logic
run_cmd(cmd, {"timeout": 3600}, f"build for {arch} failed")

# Verify architecture was set correctly
logger.debug("Verifying that %s was built with expected arch %s", destination, arch)
self._verify_image_architecture(destination, arch)

Expand All @@ -554,9 +555,12 @@ def _verify_image_architecture(self, image_name: str, expected_arch: str) -> Non
:param str image_name: the image name to verify
:param str expected_arch: the expected architecture
"""

# Get image architecture using skopeo inspect
inspect_cmd = ["skopeo", "inspect", "--no-tags", f"containers-storage:{image_name}"]
inspect_cmd = [
"skopeo",
"inspect",
"--no-tags",
f"containers-storage:{image_name}",
]
result = run_cmd(inspect_cmd, {"timeout": 60}, f"inspect {image_name} failed")
image_data = json.loads(result)

Expand Down Expand Up @@ -585,6 +589,82 @@ def _verify_image_architecture(self, image_name: str, expected_arch: str) -> Non

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[medium] correctness

_push_image (3 retries on ExternalServiceError) is called inside _build_image (also 3 retries on ExternalServiceError). A persistent push failure exhausts _push_image retries, then _build_image retries the entire method including a full rebuild. This causes up to 3 unnecessary rebuilds and 9 total push attempts for a pure push failure.

Suggested fix: Move _push_image out of _build_image and call it separately in build_all parallel executor, or change _push_image to raise a non-retryable wrapper exception that _build_image retry decorator does not catch.

logger.info(f"✓ Architecture verification passed for {image_name}: {arch}")

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[medium] correctness

_push_image retry decorator uses retry_if_exception_type(ExternalServiceError), but run_cmd raises IIBError (not ExternalServiceError) for most buildah push failures because the network-error regex patterns in run_cmd only match 'buildah bud' error messages. The retry is effectively a no-op for transient push failures.

Suggested fix: Change to retry_if_exception_type(IIBError) to match _create_and_push_manifest_list's retry strategy, or extend run_cmd's network-error regexes to cover buildah push error patterns.

@retry(
before_sleep=before_sleep_log(logger, logging.WARNING),
reraise=True,
retry=retry_if_exception_type(ExternalServiceError),
stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=2),
)
def _push_image(self, image: str) -> str:
"""Push an architecture image and return its remote manifest digest."""

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[low] doc-style

_push_image uses a single-line docstring; the codebase convention is multi-line Sphinx format with :param, :return:, :rtype:, and :raises: entries.

Suggested fix: Expand the docstring to multi-line Sphinx style with parameter, return, and exception documentation.

logger.info("Pushing %s to registry", image)
with tempfile.TemporaryDirectory(prefix="iib-push-") as directory:
digest_file = Path(directory) / "digest"
cmd = [
"buildah",
"push",
"--tls-verify=true",
"--digestfile",
str(digest_file),
image,
f"docker://{image}",
]
run_cmd(cmd, {"timeout": 3600}, exc_msg=f"Failed to push {image}")
digest = digest_file.read_text(encoding="utf-8").strip() if digest_file.exists() else ""
if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
raise IIBError(f"Invalid push digest for {image}: {digest!r}")
logger.info("✓ Pushed %s at %s", image, digest)
return digest

def _verify_manifest_list(
self,
manifest_name: str,
platform_images: list[str],
pushed_digests: dict[str, str],
remote_digest: str | None = None,
) -> None:
"""Verify a local or published list points to this run's pushed images."""

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[low] doc-style

_verify_manifest_list uses a single-line docstring for a four-parameter method; the codebase convention is multi-line Sphinx format.

Suggested fix: Expand to multi-line Sphinx style with :param:, :raises: entries.

if len(platform_images) != len(self.config.platforms) or set(platform_images) != set(
pushed_digests
):
raise IIBError("Platform images and pushed digests do not match configured platforms")
if remote_digest:
image_repo = manifest_name.rsplit(":", 1)[0]
result = run_cmd(
["skopeo", "inspect", "--raw", f"docker://{image_repo}@{remote_digest}"],
{"timeout": 60},
f"Failed to inspect published manifest {image_repo}@{remote_digest}",
)
else:
result = run_cmd(
["buildah", "manifest", "inspect", manifest_name],
exc_msg=f"Failed to inspect manifest list {manifest_name}",
)
manifest_data = json.loads(result)
manifests = manifest_data.get("manifests", [])
found = sorted(
(
m.get("platform", {}).get("os", ""),
m.get("platform", {}).get("architecture", ""),
m.get("digest", ""),
)
for m in manifests
)
expected = sorted(
("linux", self.config.arch_map.get(platform, platform), pushed_digests[image])

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[low] edge-case

_verify_manifest_list uses self.config.platforms for arch_map lookup, but build_all() normalizes platforms only to a local variable. Whitespace-padded platforms would cause false verification failures, though mitigated in practice by arches_from_metadata already stripping.

Suggested fix: Normalize self.config.platforms in place at the start of build_all() so all downstream consumers use clean platform names.

for platform, image in zip(self.config.platforms, platform_images, strict=True)
)
if found != expected:
raise IIBError(
f"Manifest list {manifest_name} has entries {found}, expected {expected}"
)
logger.info(
"✓ Manifest list %s verified: %s",
manifest_name,
", ".join(arch for _, arch, _ in found),
)

@retry(
before_sleep=before_sleep_log(logger, logging.WARNING),
reraise=True,
Expand All @@ -595,52 +675,58 @@ def _verify_image_architecture(self, image_name: str, expected_arch: str) -> Non
def _create_and_push_manifest_list(
self,
platform_images: list[str],
) -> None:
pushed_digests: dict[str, str],
) -> str:
"""

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[medium] doc-style

The docstring for _create_and_push_manifest_list was not updated: the new pushed_digests parameter and the changed return type (str instead of None) are undocumented.

Suggested fix: Add :param dict pushed_digests:, :return:, and :rtype: str to the docstring.

Create and push the manifest list to the configured registry.

:param list platform_images: list of platform-specific image names
:raises IIBError: if creating or pushing the manifest list fails
"""
buildah_manifest_cmd = ["buildah", "manifest"]
image_name_repo, image_name_tag = self.config.image_name.split(":", 1)
# Initialize _tags with the output image tag
image_name_repo, image_name_tag = self.config.image_name.rsplit(":", 1)
_tags = [image_name_tag]
if self.config.commit_sha:
_tags.append(self.config.commit_sha)

output_pull_specs = []
for tag in _tags:
output_pull_spec = f"{image_name_repo}:{tag}"
output_pull_specs.append(output_pull_spec)
try:
run_cmd(
buildah_manifest_cmd + ["rm", output_pull_spec],
exc_msg=(
f"Failed to remove local manifest list. {output_pull_spec} does not exist"
),
)
except IIBError as e:
error_msg = str(e)
if "Manifest list not found locally." not in error_msg:
raise IIBError(f"Error removing local manifest list: {error_msg}")
logger.debug(
"Manifest list cannot be removed. No manifest list %s found", output_pull_spec
)
logger.info("Creating the manifest list %s locally", output_pull_spec)
primary_pull_spec = f"{image_name_repo}:{_tags[0]}"

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[medium] design-direction

The multi-tag manifest strategy was silently redesigned from an independent manifest-list-per-tag loop to push-primary plus skopeo copy --preserve-digests. Tags now share a single manifest digest rather than being independently built. This behavioral change has registry semantics implications not documented in the PR body.

Suggested fix: Document this strategy change explicitly in the PR body and explain why shared-digest aliasing is preferable to independent manifest creation.


# Build and push the manifest list for the primary tag
try:
run_cmd(
buildah_manifest_cmd + ["create", output_pull_spec],
exc_msg=f"Failed to create the manifest list locally: {output_pull_spec}",
buildah_manifest_cmd + ["rm", primary_pull_spec],
exc_msg=(
f"Failed to remove local manifest list. {primary_pull_spec} does not exist"
),
)
except IIBError as e:
error_msg = str(e)
if "Manifest list not found locally." not in error_msg:
raise IIBError(f"Error removing local manifest list: {error_msg}")
logger.debug(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't the order incorrect? and should not be here error instead of debug?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or this is just in case when the message is different and exception will not be raised?

"Manifest list cannot be removed. No manifest list %s found",
primary_pull_spec,
)
logger.info("Creating the manifest list %s locally", primary_pull_spec)
run_cmd(
buildah_manifest_cmd + ["create", primary_pull_spec],
exc_msg=f"Failed to create the manifest list locally: {primary_pull_spec}",
)
for arch_image in platform_images:
image_repo = arch_image.rsplit(":", 1)[0]
run_cmd(
buildah_manifest_cmd
+ ["add", primary_pull_spec, f"docker://{image_repo}@{pushed_digests[arch_image]}"],
exc_msg=(
f"Failed to add {arch_image} to the local manifest list: {primary_pull_spec}"
),
)
for arch_image in platform_images:
run_cmd(
buildah_manifest_cmd + ["add", output_pull_spec, arch_image],
exc_msg=(
f"Failed to add {arch_image} to the local manifest list: {output_pull_spec}"
),
)

logger.debug("Pushing manifest list %s", output_pull_spec)
self._verify_manifest_list(primary_pull_spec, platform_images, pushed_digests)

logger.debug("Pushing manifest list %s", primary_pull_spec)
with tempfile.TemporaryDirectory(prefix="iib-manifest-") as directory:
digest_file = Path(directory) / "digest"
run_cmd(
buildah_manifest_cmd
+ [
Expand All @@ -649,11 +735,58 @@ def _create_and_push_manifest_list(
"--format",
"v2s2",
"--tls-verify=true",
output_pull_spec,
f"docker://{output_pull_spec}",
"--digestfile",
str(digest_file),
primary_pull_spec,
f"docker://{primary_pull_spec}",
],
exc_msg=f"Failed to push the manifest list to {primary_pull_spec}",
)
manifest_digest = (
digest_file.read_text(encoding="utf-8").strip() if digest_file.exists() else ""
)
if not re.fullmatch(r"sha256:[0-9a-f]{64}", manifest_digest):
raise IIBError(
f"Invalid manifest push digest for {primary_pull_spec}: {manifest_digest!r}"
)

primary_inspect = run_cmd(
["skopeo", "inspect", "--no-tags", f"docker://{primary_pull_spec}"],
{"timeout": 60},
f"Failed to inspect pushed manifest {primary_pull_spec}",
)
if json.loads(primary_inspect).get("Digest") != manifest_digest:
raise IIBError(f"Primary manifest tag {primary_pull_spec} has a different digest")

self._verify_manifest_list(
primary_pull_spec, platform_images, pushed_digests, remote_digest=manifest_digest
)

# Copy the complete manifest list under each additional tag.
for tag in _tags[1:]:
additional_pull_spec = f"{image_name_repo}:{tag}"
logger.info("Copying manifest list to %s", additional_pull_spec)
run_cmd(
[

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[low] style/conventions

The skopeo copy command omits --tls-verify=true. While skopeo defaults to TLS verification, every other registry-facing command in the codebase explicitly sets this flag for consistency.

Suggested fix: Add '--tls-verify=true' to the skopeo copy command arguments.

"skopeo",
"copy",
"--all",

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[low] consistency

The skopeo copy --all command for additional tags omits --tls-verify=true. While skopeo defaults to TLS verification, every other registry-interacting command in this codebase explicitly sets this flag.

Suggested fix: Add "--tls-verify=true" to the skopeo copy command arguments for consistency with buildah push and buildah bud commands.

"--preserve-digests",
f"docker://{image_name_repo}@{manifest_digest}",
f"docker://{additional_pull_spec}",
],
exc_msg=f"Failed to push the manifest list to {output_pull_spec}",
{"timeout": 3600},
exc_msg=f"Failed to copy manifest list to {additional_pull_spec}",
)
additional_inspect = run_cmd(
["skopeo", "inspect", "--no-tags", f"docker://{additional_pull_spec}"],
{"timeout": 60},
f"Failed to inspect manifest tag {additional_pull_spec}",
)
if json.loads(additional_inspect).get("Digest") != manifest_digest:
raise IIBError(f"Manifest tag {additional_pull_spec} has a different digest")

return manifest_digest

def build_all(self, ca_bundle_path: str | None = None) -> dict[str, Any]:
"""
Expand All @@ -667,6 +800,15 @@ def build_all(self, ca_bundle_path: str | None = None) -> dict[str, Any]:
"""
logger.info("Starting multi-architecture build")

normalized_platforms = [platform.strip() for platform in self.config.platforms]
if len(normalized_platforms) != len(set(normalized_platforms)):
duplicates = sorted(
platform
for platform in set(normalized_platforms)
if normalized_platforms.count(platform) > 1
)
raise IIBError(f"Duplicate architectures: {', '.join(duplicates)}")

# Validate Dockerfile exists
if not self.validate_dockerfile():
raise RuntimeError("Dockerfile validation failed")
Expand Down Expand Up @@ -735,30 +877,37 @@ def build_all(self, ca_bundle_path: str | None = None) -> dict[str, Any]:
logger.error(f"Failed to copy cache to build context: {e}")
raise IIBError(f"Failed to copy cache to build context: {e}")

# Build images for each platform
# Build and verify images sequentially, then push in parallel
platform_images = []
for platform in self.config.platforms:
try:
platform_clean = platform.strip()
# output-image:tag-platform
platform_image = f"{self.config.image_name}-{platform_clean}"
self._build_image(platform_clean, platform_image)
platform_images.append(platform_image)
except (IIBError, ExternalServiceError) as e:
logger.error(f"Failed to build for {platform}: {e}")
raise
platform_clean = platform.strip()
platform_image = f"{self.config.image_name}-{platform_clean}"
self._build_image(platform_clean, platform_image)
platform_images.append(platform_image)

# Push all per-arch images in parallel so registry layers are
# present before manifest list creation
logger.info("Pushing per-arch images in parallel")
pushed_digests: dict[str, str] = {}
with ThreadPoolExecutor(max_workers=len(platform_images)) as executor:
futures = {executor.submit(self._push_image, img): img for img in platform_images}
errors = {}
for future in as_completed(futures):
img = futures[future]
try:
pushed_digests[img] = future.result()
except Exception as e:
logger.error("Failed to push %s: %s", img, e)
errors[img] = e
if errors:
failed = ", ".join(sorted(errors))
raise IIBError(f"Failed to push per-arch images: {failed}")

# Create and push manifest
logger.info("Creating and pushing multi-arch manifest")

# Create and push manifest list
self._create_and_push_manifest_list(platform_images)

# Get manifest digest
inspect_cmd = ["skopeo", "inspect", "--no-tags", f"docker://{self.config.image_name}"]
result = run_cmd(inspect_cmd, {"timeout": 60}, "inspect manifest failed")
manifest_data = json.loads(result)
digest = manifest_data.get("Digest", "")
digest = self._create_and_push_manifest_list(platform_images, pushed_digests)

results = {
"image_name": self.config.image_name,
Expand Down
Loading
Loading