From d46a91e5f478e437b3dd3d747f4198da86c3fe9a Mon Sep 17 00:00:00 2001 From: wayne Date: Wed, 23 Sep 2026 14:07:12 +0800 Subject: [PATCH 1/2] feat: publish native arm64 static distill-fs releases Build and test both amd64 and arm64 musl binaries on native CI runners. Verify their static ELF format and FUSE behavior before publishing the already-tested archives under a shared release checksum file. Advance to v0.1.3 so the new dual-architecture release does not reuse v0.1.2 artifacts. Signed-off-by: wayne --- .github/workflows/release.yml | 37 ++++++++++++++++++++++++++--------- AGENTS.md | 4 ++-- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 10 +++++----- ci/package-release.py | 14 ++++++++++--- ci/release.Dockerfile | 27 ++++++++++++++++++------- 7 files changed, 68 insertions(+), 28 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6a172e6..7379905 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,19 @@ concurrency: jobs: build: name: Build and test - runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + nydus_sha256: e4f7de672ced6031c13c3c38a599993f8a8103b0cf85f9b0783f5f5fa7c8b3dc + rust_target: x86_64-unknown-linux-musl + - arch: arm64 + runner: ubuntu-24.04-arm + nydus_sha256: dcc43724cc5c4632ed111be3d09f6ee3b063f1e291a750fd7cf71c910e6b78ce + rust_target: aarch64-unknown-linux-musl + runs-on: ${{ matrix.runner }} timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -39,33 +51,33 @@ jobs: PY git merge-base --is-ancestor HEAD origin/main - name: Check formatting, build, and test musl binary - run: docker build --platform linux/amd64 --target test -f ci/release.Dockerfile -t distill-fs-static-test . + run: docker build --platform linux/${{ matrix.arch }} --target test -f ci/release.Dockerfile -t distill-fs-static-test:${{ matrix.arch }} . - name: Test raw and Nydus FUSE mounts run: | sudo modprobe fuse test -c /dev/fuse mkdir -p target/release-test-tools curl -fSL --retry 5 \ - https://github.com/dragonflyoss/nydus/releases/download/v2.4.0/nydus-static-v2.4.0-linux-amd64.tgz \ + https://github.com/dragonflyoss/nydus/releases/download/v2.4.0/nydus-static-v2.4.0-linux-${{ matrix.arch }}.tgz \ -o target/release-test-tools/nydus.tgz - echo "e4f7de672ced6031c13c3c38a599993f8a8103b0cf85f9b0783f5f5fa7c8b3dc target/release-test-tools/nydus.tgz" | sha256sum -c - + echo "${{ matrix.nydus_sha256 }} target/release-test-tools/nydus.tgz" | sha256sum -c - tar -xzf target/release-test-tools/nydus.tgz -C target/release-test-tools docker run --rm --device /dev/fuse --cap-add SYS_ADMIN \ --security-opt apparmor=unconfined --network none \ -v "$PWD/target/release-test-tools/nydus-static:/nydus:ro" \ -e DISTILL_FS_NYDUS_IMAGE=/nydus/nydus-image \ - distill-fs-static-test cargo test --locked --release \ - --target x86_64-unknown-linux-musl --test mount -- --ignored + distill-fs-static-test:${{ matrix.arch }} cargo test --locked --release \ + --target ${{ matrix.rust_target }} --test mount -- --ignored - name: Build release archive and verify static executable run: | - docker build --platform linux/amd64 -f ci/release.Dockerfile \ + docker build --platform linux/${{ matrix.arch }} -f ci/release.Dockerfile \ --build-arg "SOURCE_REVISION=$(git rev-parse HEAD)" \ --output type=local,dest=dist . cd dist sha256sum -c SHA256SUMS - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: distill-fs-linux-amd64 + name: distill-fs-linux-${{ matrix.arch }} path: dist/* if-no-files-found: error retention-days: 14 @@ -80,7 +92,7 @@ jobs: steps: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: distill-fs-linux-amd64 + pattern: distill-fs-linux-* path: dist - name: Publish tested bytes env: @@ -89,6 +101,13 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} run: | cd dist + test -f distill-fs-linux-amd64/SHA256SUMS + test -f distill-fs-linux-arm64/SHA256SUMS + for arch in amd64 arm64; do + (cd "distill-fs-linux-${arch}" && sha256sum -c SHA256SUMS) + done + cat distill-fs-linux-{amd64,arm64}/SHA256SUMS > SHA256SUMS + mv distill-fs-linux-{amd64,arm64}/*.tar.gz . sha256sum -c SHA256SUMS prerelease=() if [[ "$RELEASE_TAG" == *-rc.* ]]; then prerelease=(--prerelease); fi diff --git a/AGENTS.md b/AGENTS.md index 9f564af..79684c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,9 +66,9 @@ ## Static Release Contract -- `.github/workflows/release.yml` uses one build job and the pinned musl container pipeline for formatting, static compilation, unit/integration tests, and privileged FUSE checks. Keep release publication gated on a version-tag **push** and a successful build job; manual workflow runs must never publish, even when selecting a tag. +- `.github/workflows/release.yml` uses native amd64 and arm64 build jobs with the pinned musl container pipeline for formatting, static compilation, unit/integration tests, and privileged FUSE checks. Keep release publication gated on a version-tag **push** and both successful build jobs; manual workflow runs must never publish, even when selecting a tag. - Strip release symbols at compilation using `[profile.release]`. Verify the stripped ELF before tests and package that same executable; do not strip or rebuild a different binary in the release job. -- `ci/release.Dockerfile` builds Linux/amd64 musl artifacts with the pinned Rust image and `Cargo.lock`. Keep static ELF checks and empty-root smoke tests mandatory for packaging. +- `ci/release.Dockerfile` builds native Linux/amd64 and Linux/arm64 musl artifacts with the pinned Rust image and `Cargo.lock`. Keep static ELF checks and empty-root smoke tests mandatory for both architectures. - `ci/package-release.py` defines archive names and provenance. Keep these synchronized with AKernel's `builder/scripts/install-distill-fs.sh` and `builder/distill-fs-versions.env`. - Version tags must match `Cargo.toml` and point to commits on `main`. The workflow publishes already-tested bytes and must not overwrite release assets. - Publish a release before updating consumer checksum pins. Do not use guessed digests or present locally rebuilt artifacts as published binaries. diff --git a/Cargo.lock b/Cargo.lock index a132b27..7dc82b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -528,7 +528,7 @@ dependencies = [ [[package]] name = "distill_fs" -version = "0.1.2" +version = "0.1.3" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index d291105..089d143 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "distill_fs" -version = "0.1.2" +version = "0.1.3" edition = "2021" rust-version = "1.85" license = "Apache-2.0" diff --git a/README.md b/README.md index 791acd0..79c7b3b 100644 --- a/README.md +++ b/README.md @@ -84,15 +84,15 @@ cargo run --bin distill_fs -- --help ## Static releases -GitHub releases provide `distill-fs-vX.Y.Z-linux-amd64.tar.gz` and `SHA256SUMS`. The archive contains the `distill_fs` executable, `manifest.json`, `LICENSE`, `NOTICE`, and the exact `Cargo.lock`. The executable uses musl and has no dynamic loader or shared-library dependencies. Mounting still requires Linux FUSE support and suitable privileges; remote HTTPS backends still need a CA certificate store on the host. +GitHub releases provide `distill-fs-vX.Y.Z-linux-amd64.tar.gz`, `distill-fs-vX.Y.Z-linux-arm64.tar.gz`, and one `SHA256SUMS` covering both archives. Each archive contains the `distill_fs` executable, `manifest.json`, `LICENSE`, `NOTICE`, and the exact `Cargo.lock`. The executable uses musl and has no dynamic loader or shared-library dependencies. Mounting still requires Linux FUSE support and suitable privileges; remote HTTPS backends still need a CA certificate store on the host. -The first release is pending publication. Once published, download from `https://github.com/inclusionAI/distill-fs/releases/download/vX.Y.Z/`, verify the archive against a reviewed SHA-256 pin, extract it, and install `distill_fs` on `PATH`. Keep the license and provenance files with the binary. Production consumers must pin the release URL and checksum in source control; fetching a checksum alongside an unpinned binary is not an independent integrity check. +Download a published release from `https://github.com/inclusionAI/distill-fs/releases/download/vX.Y.Z/`, verify the archive against a reviewed SHA-256 pin, extract it, and install `distill_fs` on `PATH`. Keep the license and provenance files with the binary. Production consumers must pin the release URL and checksum in source control; fetching a checksum alongside an unpinned binary is not an independent integrity check. -Build and test the same Linux/amd64 artifact locally with Docker: +Build and test an artifact locally with Docker, selecting `amd64` or `arm64` to match the target host: ```bash -docker build --platform linux/amd64 --target test -f ci/release.Dockerfile . -docker build --platform linux/amd64 -f ci/release.Dockerfile \ +docker build --platform linux/arm64 --target test -f ci/release.Dockerfile . +docker build --platform linux/arm64 -f ci/release.Dockerfile \ --build-arg "SOURCE_REVISION=$(git rev-parse HEAD)" \ --output type=local,dest=dist . (cd dist && sha256sum -c SHA256SUMS) diff --git a/ci/package-release.py b/ci/package-release.py index b6ff7e4..1f2e931 100644 --- a/ci/package-release.py +++ b/ci/package-release.py @@ -13,18 +13,26 @@ root = Path(__file__).resolve().parent.parent version = tomllib.loads((root / "Cargo.toml").read_text())["package"]["version"] revision = sys.argv[1] +arch = sys.argv[2] +targets = { + "amd64": "x86_64-unknown-linux-musl", + "arm64": "aarch64-unknown-linux-musl", +} +if arch not in targets: + raise SystemExit(f"unsupported release architecture: {arch}") +target = targets[arch] if not re.fullmatch(r"[0-9a-f]{40}", revision): raise SystemExit("SOURCE_REVISION must be the full source commit") if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", version): raise SystemExit("release version must be X.Y.Z or X.Y.Z-rc.N") -binary = root / "target/x86_64-unknown-linux-musl/release/distill_fs" +binary = root / "target" / target / "release/distill_fs" manifest = { "component": "distill-fs", "version": version, "release_tag": f"v{version}", "source_revision": revision, "repository": "inclusionAI/distill-fs", - "target": "x86_64-unknown-linux-musl", + "target": target, "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), } files = { @@ -36,7 +44,7 @@ } output = root / "dist" output.mkdir(exist_ok=True) -archive = output / f"distill-fs-v{version}-linux-amd64.tar.gz" +archive = output / f"distill-fs-v{version}-linux-{arch}.tar.gz" # Stable metadata lets a single build be compared and promoted byte for byte. with archive.open("wb") as raw, gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as zipped: with tarfile.open(fileobj=zipped, mode="w") as tar: diff --git a/ci/release.Dockerfile b/ci/release.Dockerfile index 116e07b..014bde4 100644 --- a/ci/release.Dockerfile +++ b/ci/release.Dockerfile @@ -1,21 +1,31 @@ # Build on musl so the C dependencies and Rust use the same libc. FROM rust:1.85.0-alpine3.21@sha256:bea885d2711087e67a9f7a7cd1a164976f4c35389478512af170730014d2452a AS build +ARG TARGETARCH RUN apk add --no-cache build-base cmake git perl pkgconf python3 binutils && \ rustup component add rustfmt WORKDIR /work ENV CARGO_NET_GIT_FETCH_WITH_CLI=true \ CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=+crt-static" \ + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=+crt-static" \ TMPDIR=/work/test-tmp COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ COPY .cargo/ .cargo/ COPY src/ src/ COPY tests/ tests/ -RUN mkdir -p "$TMPDIR" && \ - cargo fmt --all -- --check && \ - cargo build --locked --release --target x86_64-unknown-linux-musl --bin distill_fs +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) target=x86_64-unknown-linux-musl ;; \ + arm64) target=aarch64-unknown-linux-musl ;; \ + *) echo "unsupported target architecture: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + mkdir -p "$TMPDIR"; \ + cargo fmt --all -- --check; \ + rustup target add "$target"; \ + cargo build --locked --release --target "$target" --bin distill_fs # Verify the final stripped binary before tests and packaging. # Reject both dynamically linked executables and static PIEs with shared deps. -RUN binary=target/x86_64-unknown-linux-musl/release/distill_fs; \ +RUN case "$TARGETARCH" in amd64) target=x86_64-unknown-linux-musl ;; arm64) target=aarch64-unknown-linux-musl ;; esac; \ + binary="target/$target/release/distill_fs"; \ readelf -h "$binary" && \ ! readelf -l "$binary" | grep -q INTERP && \ ! readelf -d "$binary" | grep -q NEEDED && \ @@ -23,15 +33,18 @@ RUN binary=target/x86_64-unknown-linux-musl/release/distill_fs; \ ! grep -Eq '[[:space:]]\.(symtab|debug_[^[:space:]]*)[[:space:]]' /work/release-sections.txt FROM build AS test -RUN cargo test --locked --release --target x86_64-unknown-linux-musl +RUN case "$TARGETARCH" in amd64) target=x86_64-unknown-linux-musl ;; arm64) target=aarch64-unknown-linux-musl ;; esac; \ + cargo test --locked --release --target "$target" FROM test AS package ARG SOURCE_REVISION +ARG TARGETARCH COPY ci/package-release.py ci/package-release.py COPY LICENSE NOTICE ./ -RUN python3 ci/package-release.py "$SOURCE_REVISION" && \ +RUN case "$TARGETARCH" in amd64) target=x86_64-unknown-linux-musl ;; arm64) target=aarch64-unknown-linux-musl ;; esac; \ + python3 ci/package-release.py "$SOURCE_REVISION" "$TARGETARCH" && \ mkdir /empty-root && \ - cp target/x86_64-unknown-linux-musl/release/distill_fs /empty-root/distill_fs && \ + cp "target/$target/release/distill_fs" /empty-root/distill_fs && \ chroot /empty-root /distill_fs --version && \ chroot /empty-root /distill_fs mount --help From fe4b5639141bd79f24794b2e58c79679dec152dd Mon Sep 17 00:00:00 2001 From: wayne Date: Wed, 23 Sep 2026 14:38:18 +0800 Subject: [PATCH 2/2] fix: validate both static release artifacts before publishing Check archive provenance, ELF architecture, and tag matching before promoting the tested amd64 and arm64 binaries. Signed-off-by: wayne --- .github/workflows/release.yml | 44 ++++++++++++++++++++++++++++++++--- README.md | 4 ++-- ci/release.Dockerfile | 6 ++++- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7379905..6c13c16 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,8 +46,10 @@ jobs: with open("Cargo.toml", "rb") as source: version = tomllib.load(source)["package"]["version"] tag = os.environ["RELEASE_TAG"] - assert re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", tag), tag - assert tag == f"v{version}", "tag must match Cargo.toml" + if not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", tag): + raise SystemExit(f"invalid release tag: {tag}") + if tag != f"v{version}": + raise SystemExit("tag must match Cargo.toml") PY git merge-base --is-ancestor HEAD origin/main - name: Check formatting, build, and test musl binary @@ -105,9 +107,45 @@ jobs: test -f distill-fs-linux-arm64/SHA256SUMS for arch in amd64 arm64; do (cd "distill-fs-linux-${arch}" && sha256sum -c SHA256SUMS) + case "$arch" in + amd64) target=x86_64-unknown-linux-musl ;; + arm64) target=aarch64-unknown-linux-musl ;; + esac + python3 - "distill-fs-linux-${arch}" "$RELEASE_TAG" "$GITHUB_SHA" "$arch" "$target" <<'PY' + import hashlib + import json + from pathlib import Path + import sys + import tarfile + + directory, tag, revision, arch, target = sys.argv[1:] + directory = Path(directory) + archive = directory / f"distill-fs-{tag}-linux-{arch}.tar.gz" + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + if (directory / "SHA256SUMS").read_text() != f"{digest} {archive.name}\n": + raise SystemExit(f"unexpected checksum file for {arch}") + with tarfile.open(archive, "r:gz") as bundle: + if bundle.getnames() != ["distill_fs", "manifest.json", "LICENSE", "NOTICE", "Cargo.lock"]: + raise SystemExit(f"unexpected archive contents for {arch}") + manifest = json.load(bundle.extractfile("manifest.json")) + binary_digest = hashlib.sha256(bundle.extractfile("distill_fs").read()).hexdigest() + expected_manifest = { + "component": "distill-fs", + "version": tag.removeprefix("v"), + "release_tag": tag, + "source_revision": revision, + "repository": "inclusionAI/distill-fs", + "target": target, + "binary_sha256": binary_digest, + } + if manifest != expected_manifest: + raise SystemExit(f"unexpected release manifest for {arch}") + PY done cat distill-fs-linux-{amd64,arm64}/SHA256SUMS > SHA256SUMS - mv distill-fs-linux-{amd64,arm64}/*.tar.gz . + for arch in amd64 arm64; do + mv "distill-fs-linux-${arch}/distill-fs-${RELEASE_TAG}-linux-${arch}.tar.gz" . + done sha256sum -c SHA256SUMS prerelease=() if [[ "$RELEASE_TAG" == *-rc.* ]]; then prerelease=(--prerelease); fi diff --git a/README.md b/README.md index 79c7b3b..89340e1 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,9 @@ docker build --platform linux/arm64 -f ci/release.Dockerfile \ Use a clean source checkout for release builds. `manifest.json` records the package version, source commit, musl target, and binary SHA-256. The Docker build checks the ELF for a dynamic interpreter and shared dependencies, then runs the executable in an empty chroot. The test target runs the default test suite against the musl release build; ignored FUSE tests need an explicitly privileged test environment. The pinned build image and `Cargo.lock` fix the toolchain and Rust dependency graph; Alpine build packages are resolved when the build runs, so independently rebuilding a tag is not a byte-for-byte reproducibility guarantee. -The `Static binary` workflow builds and tests pull requests, `main`, manual runs, and version tags, including real raw and RAFS v5 FUSE mount tests using the checksum-pinned Nydus v2.4.0 tool. Non-tag runs upload a reviewable Actions artifact without publishing a release. Pushing a feature branch alone does not trigger the workflow; opening or updating a pull request does. Manual dispatch becomes available once the workflow file exists on the default branch: use Actions → Static binary → Run workflow, or `gh workflow run release.yml --repo inclusionAI/distill-fs --ref main`. To publish, update the package version and lockfile, merge to `main`, and push a matching `vX.Y.Z` tag (or `vX.Y.Z-rc.N` for a prerelease). The workflow rejects version mismatches and tags outside `main`, then publishes the tested archive without rebuilding in the release job. Existing release assets are never overwritten. Review changes before pushing a release tag. +The `CI` workflow builds and tests pull requests, `main`, manual runs, and version tags, including real raw and RAFS v5 FUSE mount tests using the checksum-pinned Nydus v2.4.0 tool. Non-tag runs upload reviewable Actions artifacts without publishing a release. Pushing a feature branch alone does not trigger the workflow; opening or updating a pull request does. Manual dispatch becomes available once the workflow file exists on the default branch: use Actions → CI → Run workflow, or `gh workflow run release.yml --repo inclusionAI/distill-fs --ref main`. To publish, update the package version and lockfile, merge to `main`, and push a matching `vX.Y.Z` tag (or `vX.Y.Z-rc.N` for a prerelease). The workflow rejects version mismatches and tags outside `main`, then publishes both tested archives without rebuilding in the release job. Existing release assets are never overwritten. Review changes before pushing a release tag. -AKernel consumes this archive through its own `builder/distill-fs-versions.env` and `builder/scripts/install-distill-fs.sh`. Publish the distill-fs release first, verify its checksum, and update the AKernel pin. No sandboxd pipeline or gitlink change is required. The distill-fs submodule in AKernel is optional source reference and is no longer a build input. +AKernel selects the architecture-specific archive through its own `builder/distill-fs-versions.env` and `builder/scripts/install-distill-fs.sh`. Publish the distill-fs release first, verify its checksums, and update the AKernel pins. No sandboxd pipeline or gitlink change is required. The distill-fs submodule in AKernel is optional source reference and is no longer a build input. ## CLI Overview diff --git a/ci/release.Dockerfile b/ci/release.Dockerfile index 014bde4..54c4a0d 100644 --- a/ci/release.Dockerfile +++ b/ci/release.Dockerfile @@ -24,9 +24,13 @@ RUN set -eu; \ cargo build --locked --release --target "$target" --bin distill_fs # Verify the final stripped binary before tests and packaging. # Reject both dynamically linked executables and static PIEs with shared deps. -RUN case "$TARGETARCH" in amd64) target=x86_64-unknown-linux-musl ;; arm64) target=aarch64-unknown-linux-musl ;; esac; \ +RUN case "$TARGETARCH" in \ + amd64) target=x86_64-unknown-linux-musl; machine='Advanced Micro Devices X86-64' ;; \ + arm64) target=aarch64-unknown-linux-musl; machine=AArch64 ;; \ + esac; \ binary="target/$target/release/distill_fs"; \ readelf -h "$binary" && \ + readelf -h "$binary" | grep -Eq "Machine:[[:space:]]+$machine$" && \ ! readelf -l "$binary" | grep -q INTERP && \ ! readelf -d "$binary" | grep -q NEEDED && \ readelf --wide --sections "$binary" > /work/release-sections.txt && \