Repository navigation
feat: push per-arch images in parallel before manifest list creation #27
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,9 +14,11 @@ | |
| import shutil | ||
| import subprocess | ||
| import sys | ||
| import tempfile | ||
| from concurrent.futures import ThreadPoolExecutor, as_completed | ||
| from typing import Any | ||
| from dataclasses import dataclass, field | ||
| from pathlib import Path | ||
| from typing import Any | ||
|
|
||
| from tenacity import ( | ||
| before_sleep_log, | ||
|
|
@@ -133,9 +135,10 @@ def run_cmd( | |
| ): | ||
| raise IIBError("Error deleting packages from database") | ||
| elif cmd[0] == "buildah": | ||
| # Check for HTTP 403 or 50X errors on buildah | ||
| # Check for transient network/registry errors on any buildah operation | ||
| network_regexes = [ | ||
| r".*([e,E]rror:? creating build container).*(:?(403|50[0-9]|125)\s?.*$)", | ||
| r".*((?:403 Forbidden|50[0-9] \w+).*$)", | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [low] edge-case The new regex classifies HTTP 403 as transient ExternalServiceError and retries it, but 403 Forbidden typically indicates permanent authorization failure. |
||
| r".*(read\/write on closed pipe.*$)", | ||
| ] | ||
| for regex in network_regexes: | ||
|
|
@@ -327,6 +330,9 @@ def arches_from_metadata(metadata: dict[str, Any]) -> list[str] | None: | |
| "array must contain at least one architecture" | ||
| ) | ||
|
|
||
| if len(arches) != len(set(arches)): | ||
| raise IIBError("Invalid arches in IIB build metadata file: duplicate architectures") | ||
|
|
||
| return arches | ||
|
|
||
|
|
||
|
|
@@ -510,7 +516,6 @@ def _build_image(self, arch: str, destination: str) -> None: | |
| destination, | ||
| ) | ||
|
|
||
| # Prepare buildah command with improved options | ||
| cmd = [ | ||
| "buildah", | ||
| "bud", | ||
|
|
@@ -530,20 +535,16 @@ def _build_image(self, arch: str, destination: str) -> None: | |
| self.config.dockerfile_path, | ||
| ] | ||
|
|
||
| # Add labels | ||
| for label in self.config.labels: | ||
| cmd.extend(["--label", label.strip()]) | ||
|
|
||
| if self.config.binary_image: | ||
| cmd.extend(["--build-arg", f"BINARY_IMAGE={self.config.binary_image}"]) | ||
|
|
||
| # Add context | ||
| cmd.append(self.config.context_path) | ||
|
|
||
| # Execute build with retry logic | ||
| run_cmd(cmd, {"timeout": 3600}, f"build for {arch} failed") | ||
|
|
||
| # Verify architecture was set correctly | ||
| logger.debug("Verifying that %s was built with expected arch %s", destination, arch) | ||
| self._verify_image_architecture(destination, arch) | ||
|
|
||
|
|
@@ -554,9 +555,12 @@ def _verify_image_architecture(self, image_name: str, expected_arch: str) -> Non | |
| :param str image_name: the image name to verify | ||
| :param str expected_arch: the expected architecture | ||
| """ | ||
|
|
||
| # Get image architecture using skopeo inspect | ||
| inspect_cmd = ["skopeo", "inspect", "--no-tags", f"containers-storage:{image_name}"] | ||
| inspect_cmd = [ | ||
| "skopeo", | ||
| "inspect", | ||
| "--no-tags", | ||
| f"containers-storage:{image_name}", | ||
| ] | ||
| result = run_cmd(inspect_cmd, {"timeout": 60}, f"inspect {image_name} failed") | ||
| image_data = json.loads(result) | ||
|
|
||
|
|
@@ -585,6 +589,82 @@ def _verify_image_architecture(self, image_name: str, expected_arch: str) -> Non | |
|
|
||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [medium] correctness _push_image (3 retries on ExternalServiceError) is called inside _build_image (also 3 retries on ExternalServiceError). A persistent push failure exhausts _push_image retries, then _build_image retries the entire method including a full rebuild. This causes up to 3 unnecessary rebuilds and 9 total push attempts for a pure push failure. Suggested fix: Move _push_image out of _build_image and call it separately in build_all parallel executor, or change _push_image to raise a non-retryable wrapper exception that _build_image retry decorator does not catch. |
||
| logger.info(f"✓ Architecture verification passed for {image_name}: {arch}") | ||
|
|
||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [medium] correctness _push_image retry decorator uses retry_if_exception_type(ExternalServiceError), but run_cmd raises IIBError (not ExternalServiceError) for most buildah push failures because the network-error regex patterns in run_cmd only match 'buildah bud' error messages. The retry is effectively a no-op for transient push failures. Suggested fix: Change to retry_if_exception_type(IIBError) to match _create_and_push_manifest_list's retry strategy, or extend run_cmd's network-error regexes to cover buildah push error patterns. |
||
| @retry( | ||
| before_sleep=before_sleep_log(logger, logging.WARNING), | ||
| reraise=True, | ||
| retry=retry_if_exception_type(ExternalServiceError), | ||
| stop=stop_after_attempt(3), | ||
| wait=wait_exponential(multiplier=2), | ||
| ) | ||
| def _push_image(self, image: str) -> str: | ||
| """Push an architecture image and return its remote manifest digest.""" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [low] doc-style _push_image uses a single-line docstring; the codebase convention is multi-line Sphinx format with :param, :return:, :rtype:, and :raises: entries. Suggested fix: Expand the docstring to multi-line Sphinx style with parameter, return, and exception documentation. |
||
| logger.info("Pushing %s to registry", image) | ||
| with tempfile.TemporaryDirectory(prefix="iib-push-") as directory: | ||
| digest_file = Path(directory) / "digest" | ||
| cmd = [ | ||
| "buildah", | ||
| "push", | ||
| "--tls-verify=true", | ||
| "--digestfile", | ||
| str(digest_file), | ||
| image, | ||
| f"docker://{image}", | ||
| ] | ||
| run_cmd(cmd, {"timeout": 3600}, exc_msg=f"Failed to push {image}") | ||
| digest = digest_file.read_text(encoding="utf-8").strip() if digest_file.exists() else "" | ||
| if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): | ||
| raise IIBError(f"Invalid push digest for {image}: {digest!r}") | ||
| logger.info("✓ Pushed %s at %s", image, digest) | ||
| return digest | ||
|
|
||
| def _verify_manifest_list( | ||
| self, | ||
| manifest_name: str, | ||
| platform_images: list[str], | ||
| pushed_digests: dict[str, str], | ||
| remote_digest: str | None = None, | ||
| ) -> None: | ||
| """Verify a local or published list points to this run's pushed images.""" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [low] doc-style _verify_manifest_list uses a single-line docstring for a four-parameter method; the codebase convention is multi-line Sphinx format. Suggested fix: Expand to multi-line Sphinx style with :param:, :raises: entries. |
||
| if len(platform_images) != len(self.config.platforms) or set(platform_images) != set( | ||
| pushed_digests | ||
| ): | ||
| raise IIBError("Platform images and pushed digests do not match configured platforms") | ||
| if remote_digest: | ||
| image_repo = manifest_name.rsplit(":", 1)[0] | ||
| result = run_cmd( | ||
| ["skopeo", "inspect", "--raw", f"docker://{image_repo}@{remote_digest}"], | ||
| {"timeout": 60}, | ||
| f"Failed to inspect published manifest {image_repo}@{remote_digest}", | ||
| ) | ||
| else: | ||
| result = run_cmd( | ||
| ["buildah", "manifest", "inspect", manifest_name], | ||
| exc_msg=f"Failed to inspect manifest list {manifest_name}", | ||
| ) | ||
| manifest_data = json.loads(result) | ||
| manifests = manifest_data.get("manifests", []) | ||
| found = sorted( | ||
| ( | ||
| m.get("platform", {}).get("os", ""), | ||
| m.get("platform", {}).get("architecture", ""), | ||
| m.get("digest", ""), | ||
| ) | ||
| for m in manifests | ||
| ) | ||
| expected = sorted( | ||
| ("linux", self.config.arch_map.get(platform, platform), pushed_digests[image]) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [low] edge-case _verify_manifest_list uses self.config.platforms for arch_map lookup, but build_all() normalizes platforms only to a local variable. Whitespace-padded platforms would cause false verification failures, though mitigated in practice by arches_from_metadata already stripping. Suggested fix: Normalize self.config.platforms in place at the start of build_all() so all downstream consumers use clean platform names. |
||
| for platform, image in zip(self.config.platforms, platform_images, strict=True) | ||
| ) | ||
| if found != expected: | ||
| raise IIBError( | ||
| f"Manifest list {manifest_name} has entries {found}, expected {expected}" | ||
| ) | ||
| logger.info( | ||
| "✓ Manifest list %s verified: %s", | ||
| manifest_name, | ||
| ", ".join(arch for _, arch, _ in found), | ||
| ) | ||
|
|
||
| @retry( | ||
| before_sleep=before_sleep_log(logger, logging.WARNING), | ||
| reraise=True, | ||
|
|
@@ -595,52 +675,58 @@ def _verify_image_architecture(self, image_name: str, expected_arch: str) -> Non | |
| def _create_and_push_manifest_list( | ||
| self, | ||
| platform_images: list[str], | ||
| ) -> None: | ||
| pushed_digests: dict[str, str], | ||
| ) -> str: | ||
| """ | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [medium] doc-style The docstring for _create_and_push_manifest_list was not updated: the new pushed_digests parameter and the changed return type (str instead of None) are undocumented. Suggested fix: Add :param dict pushed_digests:, :return:, and :rtype: str to the docstring. |
||
| Create and push the manifest list to the configured registry. | ||
|
|
||
| :param list platform_images: list of platform-specific image names | ||
| :raises IIBError: if creating or pushing the manifest list fails | ||
| """ | ||
| buildah_manifest_cmd = ["buildah", "manifest"] | ||
| image_name_repo, image_name_tag = self.config.image_name.split(":", 1) | ||
| # Initialize _tags with the output image tag | ||
| image_name_repo, image_name_tag = self.config.image_name.rsplit(":", 1) | ||
| _tags = [image_name_tag] | ||
| if self.config.commit_sha: | ||
| _tags.append(self.config.commit_sha) | ||
|
|
||
| output_pull_specs = [] | ||
| for tag in _tags: | ||
| output_pull_spec = f"{image_name_repo}:{tag}" | ||
| output_pull_specs.append(output_pull_spec) | ||
| try: | ||
| run_cmd( | ||
| buildah_manifest_cmd + ["rm", output_pull_spec], | ||
| exc_msg=( | ||
| f"Failed to remove local manifest list. {output_pull_spec} does not exist" | ||
| ), | ||
| ) | ||
| except IIBError as e: | ||
| error_msg = str(e) | ||
| if "Manifest list not found locally." not in error_msg: | ||
| raise IIBError(f"Error removing local manifest list: {error_msg}") | ||
| logger.debug( | ||
| "Manifest list cannot be removed. No manifest list %s found", output_pull_spec | ||
| ) | ||
| logger.info("Creating the manifest list %s locally", output_pull_spec) | ||
| primary_pull_spec = f"{image_name_repo}:{_tags[0]}" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [medium] design-direction The multi-tag manifest strategy was silently redesigned from an independent manifest-list-per-tag loop to push-primary plus skopeo copy --preserve-digests. Tags now share a single manifest digest rather than being independently built. This behavioral change has registry semantics implications not documented in the PR body. Suggested fix: Document this strategy change explicitly in the PR body and explain why shared-digest aliasing is preferable to independent manifest creation. |
||
|
|
||
| # Build and push the manifest list for the primary tag | ||
| try: | ||
| run_cmd( | ||
| buildah_manifest_cmd + ["create", output_pull_spec], | ||
| exc_msg=f"Failed to create the manifest list locally: {output_pull_spec}", | ||
| buildah_manifest_cmd + ["rm", primary_pull_spec], | ||
| exc_msg=( | ||
| f"Failed to remove local manifest list. {primary_pull_spec} does not exist" | ||
| ), | ||
| ) | ||
| except IIBError as e: | ||
| error_msg = str(e) | ||
| if "Manifest list not found locally." not in error_msg: | ||
| raise IIBError(f"Error removing local manifest list: {error_msg}") | ||
| logger.debug( | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Isn't the order incorrect? and should not be here error instead of debug?
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Or this is just in case when the message is different and exception will not be raised? |
||
| "Manifest list cannot be removed. No manifest list %s found", | ||
| primary_pull_spec, | ||
| ) | ||
| logger.info("Creating the manifest list %s locally", primary_pull_spec) | ||
| run_cmd( | ||
| buildah_manifest_cmd + ["create", primary_pull_spec], | ||
| exc_msg=f"Failed to create the manifest list locally: {primary_pull_spec}", | ||
| ) | ||
| for arch_image in platform_images: | ||
| image_repo = arch_image.rsplit(":", 1)[0] | ||
| run_cmd( | ||
| buildah_manifest_cmd | ||
| + ["add", primary_pull_spec, f"docker://{image_repo}@{pushed_digests[arch_image]}"], | ||
| exc_msg=( | ||
| f"Failed to add {arch_image} to the local manifest list: {primary_pull_spec}" | ||
| ), | ||
| ) | ||
| for arch_image in platform_images: | ||
| run_cmd( | ||
| buildah_manifest_cmd + ["add", output_pull_spec, arch_image], | ||
| exc_msg=( | ||
| f"Failed to add {arch_image} to the local manifest list: {output_pull_spec}" | ||
| ), | ||
| ) | ||
|
|
||
| logger.debug("Pushing manifest list %s", output_pull_spec) | ||
| self._verify_manifest_list(primary_pull_spec, platform_images, pushed_digests) | ||
|
|
||
| logger.debug("Pushing manifest list %s", primary_pull_spec) | ||
| with tempfile.TemporaryDirectory(prefix="iib-manifest-") as directory: | ||
| digest_file = Path(directory) / "digest" | ||
| run_cmd( | ||
| buildah_manifest_cmd | ||
| + [ | ||
|
|
@@ -649,11 +735,58 @@ def _create_and_push_manifest_list( | |
| "--format", | ||
| "v2s2", | ||
| "--tls-verify=true", | ||
| output_pull_spec, | ||
| f"docker://{output_pull_spec}", | ||
| "--digestfile", | ||
| str(digest_file), | ||
| primary_pull_spec, | ||
| f"docker://{primary_pull_spec}", | ||
| ], | ||
| exc_msg=f"Failed to push the manifest list to {primary_pull_spec}", | ||
| ) | ||
| manifest_digest = ( | ||
| digest_file.read_text(encoding="utf-8").strip() if digest_file.exists() else "" | ||
| ) | ||
| if not re.fullmatch(r"sha256:[0-9a-f]{64}", manifest_digest): | ||
| raise IIBError( | ||
| f"Invalid manifest push digest for {primary_pull_spec}: {manifest_digest!r}" | ||
| ) | ||
|
|
||
| primary_inspect = run_cmd( | ||
| ["skopeo", "inspect", "--no-tags", f"docker://{primary_pull_spec}"], | ||
| {"timeout": 60}, | ||
| f"Failed to inspect pushed manifest {primary_pull_spec}", | ||
| ) | ||
| if json.loads(primary_inspect).get("Digest") != manifest_digest: | ||
| raise IIBError(f"Primary manifest tag {primary_pull_spec} has a different digest") | ||
|
|
||
| self._verify_manifest_list( | ||
| primary_pull_spec, platform_images, pushed_digests, remote_digest=manifest_digest | ||
| ) | ||
|
|
||
| # Copy the complete manifest list under each additional tag. | ||
| for tag in _tags[1:]: | ||
| additional_pull_spec = f"{image_name_repo}:{tag}" | ||
| logger.info("Copying manifest list to %s", additional_pull_spec) | ||
| run_cmd( | ||
| [ | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [low] style/conventions The skopeo copy command omits --tls-verify=true. While skopeo defaults to TLS verification, every other registry-facing command in the codebase explicitly sets this flag for consistency. Suggested fix: Add '--tls-verify=true' to the skopeo copy command arguments. |
||
| "skopeo", | ||
| "copy", | ||
| "--all", | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [low] consistency The skopeo copy --all command for additional tags omits --tls-verify=true. While skopeo defaults to TLS verification, every other registry-interacting command in this codebase explicitly sets this flag. Suggested fix: Add "--tls-verify=true" to the skopeo copy command arguments for consistency with buildah push and buildah bud commands. |
||
| "--preserve-digests", | ||
| f"docker://{image_name_repo}@{manifest_digest}", | ||
| f"docker://{additional_pull_spec}", | ||
| ], | ||
| exc_msg=f"Failed to push the manifest list to {output_pull_spec}", | ||
| {"timeout": 3600}, | ||
| exc_msg=f"Failed to copy manifest list to {additional_pull_spec}", | ||
| ) | ||
| additional_inspect = run_cmd( | ||
| ["skopeo", "inspect", "--no-tags", f"docker://{additional_pull_spec}"], | ||
| {"timeout": 60}, | ||
| f"Failed to inspect manifest tag {additional_pull_spec}", | ||
| ) | ||
| if json.loads(additional_inspect).get("Digest") != manifest_digest: | ||
| raise IIBError(f"Manifest tag {additional_pull_spec} has a different digest") | ||
|
|
||
| return manifest_digest | ||
|
|
||
| def build_all(self, ca_bundle_path: str | None = None) -> dict[str, Any]: | ||
| """ | ||
|
|
@@ -667,6 +800,15 @@ def build_all(self, ca_bundle_path: str | None = None) -> dict[str, Any]: | |
| """ | ||
| logger.info("Starting multi-architecture build") | ||
|
|
||
| normalized_platforms = [platform.strip() for platform in self.config.platforms] | ||
| if len(normalized_platforms) != len(set(normalized_platforms)): | ||
| duplicates = sorted( | ||
| platform | ||
| for platform in set(normalized_platforms) | ||
| if normalized_platforms.count(platform) > 1 | ||
| ) | ||
| raise IIBError(f"Duplicate architectures: {', '.join(duplicates)}") | ||
|
|
||
| # Validate Dockerfile exists | ||
| if not self.validate_dockerfile(): | ||
| raise RuntimeError("Dockerfile validation failed") | ||
|
|
@@ -735,30 +877,37 @@ def build_all(self, ca_bundle_path: str | None = None) -> dict[str, Any]: | |
| logger.error(f"Failed to copy cache to build context: {e}") | ||
| raise IIBError(f"Failed to copy cache to build context: {e}") | ||
|
|
||
| # Build images for each platform | ||
| # Build and verify images sequentially, then push in parallel | ||
| platform_images = [] | ||
| for platform in self.config.platforms: | ||
| try: | ||
| platform_clean = platform.strip() | ||
| # output-image:tag-platform | ||
| platform_image = f"{self.config.image_name}-{platform_clean}" | ||
| self._build_image(platform_clean, platform_image) | ||
| platform_images.append(platform_image) | ||
| except (IIBError, ExternalServiceError) as e: | ||
| logger.error(f"Failed to build for {platform}: {e}") | ||
| raise | ||
| platform_clean = platform.strip() | ||
| platform_image = f"{self.config.image_name}-{platform_clean}" | ||
| self._build_image(platform_clean, platform_image) | ||
| platform_images.append(platform_image) | ||
|
|
||
| # Push all per-arch images in parallel so registry layers are | ||
| # present before manifest list creation | ||
| logger.info("Pushing per-arch images in parallel") | ||
| pushed_digests: dict[str, str] = {} | ||
| with ThreadPoolExecutor(max_workers=len(platform_images)) as executor: | ||
| futures = {executor.submit(self._push_image, img): img for img in platform_images} | ||
| errors = {} | ||
| for future in as_completed(futures): | ||
| img = futures[future] | ||
| try: | ||
| pushed_digests[img] = future.result() | ||
| except Exception as e: | ||
| logger.error("Failed to push %s: %s", img, e) | ||
| errors[img] = e | ||
| if errors: | ||
| failed = ", ".join(sorted(errors)) | ||
| raise IIBError(f"Failed to push per-arch images: {failed}") | ||
|
|
||
| # Create and push manifest | ||
| logger.info("Creating and pushing multi-arch manifest") | ||
|
|
||
| # Create and push manifest list | ||
| self._create_and_push_manifest_list(platform_images) | ||
|
|
||
| # Get manifest digest | ||
| inspect_cmd = ["skopeo", "inspect", "--no-tags", f"docker://{self.config.image_name}"] | ||
| result = run_cmd(inspect_cmd, {"timeout": 60}, "inspect manifest failed") | ||
| manifest_data = json.loads(result) | ||
| digest = manifest_data.get("Digest", "") | ||
| digest = self._create_and_push_manifest_list(platform_images, pushed_digests) | ||
|
|
||
| results = { | ||
| "image_name": self.config.image_name, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[low] code-organization
Import ordering: from typing import Any placed before from dataclasses breaks the established alphabetical convention for from-import lines.
Suggested fix: Reorder the from-import block: concurrent.futures, dataclasses, pathlib, typing.