Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 68 additions & 11 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,19 @@ concurrency:
jobs:
build:
name: Build and test
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
runner: ubuntu-24.04
nydus_sha256: e4f7de672ced6031c13c3c38a599993f8a8103b0cf85f9b0783f5f5fa7c8b3dc
rust_target: x86_64-unknown-linux-musl
- arch: arm64
runner: ubuntu-24.04-arm
nydus_sha256: dcc43724cc5c4632ed111be3d09f6ee3b063f1e291a750fd7cf71c910e6b78ce
rust_target: aarch64-unknown-linux-musl
runs-on: ${{ matrix.runner }}
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -34,38 +46,40 @@ jobs:
with open("Cargo.toml", "rb") as source:
version = tomllib.load(source)["package"]["version"]
tag = os.environ["RELEASE_TAG"]
assert re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", tag), tag
assert tag == f"v{version}", "tag must match Cargo.toml"
if not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", tag):
raise SystemExit(f"invalid release tag: {tag}")
if tag != f"v{version}":
raise SystemExit("tag must match Cargo.toml")
PY
git merge-base --is-ancestor HEAD origin/main
- name: Check formatting, build, and test musl binary
run: docker build --platform linux/amd64 --target test -f ci/release.Dockerfile -t distill-fs-static-test .
run: docker build --platform linux/${{ matrix.arch }} --target test -f ci/release.Dockerfile -t distill-fs-static-test:${{ matrix.arch }} .
- name: Test raw and Nydus FUSE mounts
run: |
sudo modprobe fuse
test -c /dev/fuse
mkdir -p target/release-test-tools
curl -fSL --retry 5 \
https://github.com/dragonflyoss/nydus/releases/download/v2.4.0/nydus-static-v2.4.0-linux-amd64.tgz \
https://github.com/dragonflyoss/nydus/releases/download/v2.4.0/nydus-static-v2.4.0-linux-${{ matrix.arch }}.tgz \
-o target/release-test-tools/nydus.tgz
echo "e4f7de672ced6031c13c3c38a599993f8a8103b0cf85f9b0783f5f5fa7c8b3dc target/release-test-tools/nydus.tgz" | sha256sum -c -
echo "${{ matrix.nydus_sha256 }} target/release-test-tools/nydus.tgz" | sha256sum -c -
tar -xzf target/release-test-tools/nydus.tgz -C target/release-test-tools
docker run --rm --device /dev/fuse --cap-add SYS_ADMIN \
--security-opt apparmor=unconfined --network none \
-v "$PWD/target/release-test-tools/nydus-static:/nydus:ro" \
-e DISTILL_FS_NYDUS_IMAGE=/nydus/nydus-image \
distill-fs-static-test cargo test --locked --release \
--target x86_64-unknown-linux-musl --test mount -- --ignored
distill-fs-static-test:${{ matrix.arch }} cargo test --locked --release \
--target ${{ matrix.rust_target }} --test mount -- --ignored
- name: Build release archive and verify static executable
run: |
docker build --platform linux/amd64 -f ci/release.Dockerfile \
docker build --platform linux/${{ matrix.arch }} -f ci/release.Dockerfile \
--build-arg "SOURCE_REVISION=$(git rev-parse HEAD)" \
--output type=local,dest=dist .
cd dist
sha256sum -c SHA256SUMS
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: distill-fs-linux-amd64
name: distill-fs-linux-${{ matrix.arch }}
path: dist/*
if-no-files-found: error
retention-days: 14
Expand All @@ -80,7 +94,7 @@ jobs:
steps:
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: distill-fs-linux-amd64
pattern: distill-fs-linux-*
path: dist
- name: Publish tested bytes
env:
Expand All @@ -89,6 +103,49 @@ jobs:
RELEASE_TAG: ${{ github.ref_name }}
run: |
cd dist
test -f distill-fs-linux-amd64/SHA256SUMS
test -f distill-fs-linux-arm64/SHA256SUMS
for arch in amd64 arm64; do
(cd "distill-fs-linux-${arch}" && sha256sum -c SHA256SUMS)
case "$arch" in
amd64) target=x86_64-unknown-linux-musl ;;
arm64) target=aarch64-unknown-linux-musl ;;
esac
python3 - "distill-fs-linux-${arch}" "$RELEASE_TAG" "$GITHUB_SHA" "$arch" "$target" <<'PY'
import hashlib
import json
from pathlib import Path
import sys
import tarfile

directory, tag, revision, arch, target = sys.argv[1:]
directory = Path(directory)
archive = directory / f"distill-fs-{tag}-linux-{arch}.tar.gz"
digest = hashlib.sha256(archive.read_bytes()).hexdigest()
if (directory / "SHA256SUMS").read_text() != f"{digest} {archive.name}\n":
raise SystemExit(f"unexpected checksum file for {arch}")
with tarfile.open(archive, "r:gz") as bundle:
if bundle.getnames() != ["distill_fs", "manifest.json", "LICENSE", "NOTICE", "Cargo.lock"]:
raise SystemExit(f"unexpected archive contents for {arch}")
manifest = json.load(bundle.extractfile("manifest.json"))
binary_digest = hashlib.sha256(bundle.extractfile("distill_fs").read()).hexdigest()
expected_manifest = {
"component": "distill-fs",
"version": tag.removeprefix("v"),
"release_tag": tag,
"source_revision": revision,
"repository": "inclusionAI/distill-fs",
"target": target,
"binary_sha256": binary_digest,
}
if manifest != expected_manifest:
raise SystemExit(f"unexpected release manifest for {arch}")
PY
done
cat distill-fs-linux-{amd64,arm64}/SHA256SUMS > SHA256SUMS
for arch in amd64 arm64; do
mv "distill-fs-linux-${arch}/distill-fs-${RELEASE_TAG}-linux-${arch}.tar.gz" .
done
sha256sum -c SHA256SUMS
prerelease=()
if [[ "$RELEASE_TAG" == *-rc.* ]]; then prerelease=(--prerelease); fi
Expand Down
4 changes: 2 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@

## Static Release Contract

- `.github/workflows/release.yml` uses one build job and the pinned musl container pipeline for formatting, static compilation, unit/integration tests, and privileged FUSE checks. Keep release publication gated on a version-tag **push** and a successful build job; manual workflow runs must never publish, even when selecting a tag.
- `.github/workflows/release.yml` uses native amd64 and arm64 build jobs with the pinned musl container pipeline for formatting, static compilation, unit/integration tests, and privileged FUSE checks. Keep release publication gated on a version-tag **push** and both successful build jobs; manual workflow runs must never publish, even when selecting a tag.
- Strip release symbols at compilation using `[profile.release]`. Verify the stripped ELF before tests and package that same executable; do not strip or rebuild a different binary in the release job.
- `ci/release.Dockerfile` builds Linux/amd64 musl artifacts with the pinned Rust image and `Cargo.lock`. Keep static ELF checks and empty-root smoke tests mandatory for packaging.
- `ci/release.Dockerfile` builds native Linux/amd64 and Linux/arm64 musl artifacts with the pinned Rust image and `Cargo.lock`. Keep static ELF checks and empty-root smoke tests mandatory for both architectures.
- `ci/package-release.py` defines archive names and provenance. Keep these synchronized with AKernel's `builder/scripts/install-distill-fs.sh` and `builder/distill-fs-versions.env`.
- Version tags must match `Cargo.toml` and point to commits on `main`. The workflow publishes already-tested bytes and must not overwrite release assets.
- Publish a release before updating consumer checksum pins. Do not use guessed digests or present locally rebuilt artifacts as published binaries.
Expand Down
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "distill_fs"
version = "0.1.2"
version = "0.1.3"
edition = "2021"
rust-version = "1.85"
license = "Apache-2.0"
Expand Down
14 changes: 7 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,25 +84,25 @@ cargo run --bin distill_fs -- --help

## Static releases

GitHub releases provide `distill-fs-vX.Y.Z-linux-amd64.tar.gz` and `SHA256SUMS`. The archive contains the `distill_fs` executable, `manifest.json`, `LICENSE`, `NOTICE`, and the exact `Cargo.lock`. The executable uses musl and has no dynamic loader or shared-library dependencies. Mounting still requires Linux FUSE support and suitable privileges; remote HTTPS backends still need a CA certificate store on the host.
GitHub releases provide `distill-fs-vX.Y.Z-linux-amd64.tar.gz`, `distill-fs-vX.Y.Z-linux-arm64.tar.gz`, and one `SHA256SUMS` covering both archives. Each archive contains the `distill_fs` executable, `manifest.json`, `LICENSE`, `NOTICE`, and the exact `Cargo.lock`. The executable uses musl and has no dynamic loader or shared-library dependencies. Mounting still requires Linux FUSE support and suitable privileges; remote HTTPS backends still need a CA certificate store on the host.

The first release is pending publication. Once published, download from `https://github.com/inclusionAI/distill-fs/releases/download/vX.Y.Z/`, verify the archive against a reviewed SHA-256 pin, extract it, and install `distill_fs` on `PATH`. Keep the license and provenance files with the binary. Production consumers must pin the release URL and checksum in source control; fetching a checksum alongside an unpinned binary is not an independent integrity check.
Download a published release from `https://github.com/inclusionAI/distill-fs/releases/download/vX.Y.Z/`, verify the archive against a reviewed SHA-256 pin, extract it, and install `distill_fs` on `PATH`. Keep the license and provenance files with the binary. Production consumers must pin the release URL and checksum in source control; fetching a checksum alongside an unpinned binary is not an independent integrity check.

Build and test the same Linux/amd64 artifact locally with Docker:
Build and test an artifact locally with Docker, selecting `amd64` or `arm64` to match the target host:

```bash
docker build --platform linux/amd64 --target test -f ci/release.Dockerfile .
docker build --platform linux/amd64 -f ci/release.Dockerfile \
docker build --platform linux/arm64 --target test -f ci/release.Dockerfile .
docker build --platform linux/arm64 -f ci/release.Dockerfile \
--build-arg "SOURCE_REVISION=$(git rev-parse HEAD)" \
--output type=local,dest=dist .
(cd dist && sha256sum -c SHA256SUMS)
```

Use a clean source checkout for release builds. `manifest.json` records the package version, source commit, musl target, and binary SHA-256. The Docker build checks the ELF for a dynamic interpreter and shared dependencies, then runs the executable in an empty chroot. The test target runs the default test suite against the musl release build; ignored FUSE tests need an explicitly privileged test environment. The pinned build image and `Cargo.lock` fix the toolchain and Rust dependency graph; Alpine build packages are resolved when the build runs, so independently rebuilding a tag is not a byte-for-byte reproducibility guarantee.

The `Static binary` workflow builds and tests pull requests, `main`, manual runs, and version tags, including real raw and RAFS v5 FUSE mount tests using the checksum-pinned Nydus v2.4.0 tool. Non-tag runs upload a reviewable Actions artifact without publishing a release. Pushing a feature branch alone does not trigger the workflow; opening or updating a pull request does. Manual dispatch becomes available once the workflow file exists on the default branch: use Actions → Static binary → Run workflow, or `gh workflow run release.yml --repo inclusionAI/distill-fs --ref main`. To publish, update the package version and lockfile, merge to `main`, and push a matching `vX.Y.Z` tag (or `vX.Y.Z-rc.N` for a prerelease). The workflow rejects version mismatches and tags outside `main`, then publishes the tested archive without rebuilding in the release job. Existing release assets are never overwritten. Review changes before pushing a release tag.
The `CI` workflow builds and tests pull requests, `main`, manual runs, and version tags, including real raw and RAFS v5 FUSE mount tests using the checksum-pinned Nydus v2.4.0 tool. Non-tag runs upload reviewable Actions artifacts without publishing a release. Pushing a feature branch alone does not trigger the workflow; opening or updating a pull request does. Manual dispatch becomes available once the workflow file exists on the default branch: use Actions → CI → Run workflow, or `gh workflow run release.yml --repo inclusionAI/distill-fs --ref main`. To publish, update the package version and lockfile, merge to `main`, and push a matching `vX.Y.Z` tag (or `vX.Y.Z-rc.N` for a prerelease). The workflow rejects version mismatches and tags outside `main`, then publishes both tested archives without rebuilding in the release job. Existing release assets are never overwritten. Review changes before pushing a release tag.

AKernel consumes this archive through its own `builder/distill-fs-versions.env` and `builder/scripts/install-distill-fs.sh`. Publish the distill-fs release first, verify its checksum, and update the AKernel pin. No sandboxd pipeline or gitlink change is required. The distill-fs submodule in AKernel is optional source reference and is no longer a build input.
AKernel selects the architecture-specific archive through its own `builder/distill-fs-versions.env` and `builder/scripts/install-distill-fs.sh`. Publish the distill-fs release first, verify its checksums, and update the AKernel pins. No sandboxd pipeline or gitlink change is required. The distill-fs submodule in AKernel is optional source reference and is no longer a build input.

## CLI Overview

Expand Down
14 changes: 11 additions & 3 deletions ci/package-release.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,26 @@
root = Path(__file__).resolve().parent.parent
version = tomllib.loads((root / "Cargo.toml").read_text())["package"]["version"]
revision = sys.argv[1]
arch = sys.argv[2]
targets = {
"amd64": "x86_64-unknown-linux-musl",
"arm64": "aarch64-unknown-linux-musl",
}
if arch not in targets:
raise SystemExit(f"unsupported release architecture: {arch}")
target = targets[arch]
if not re.fullmatch(r"[0-9a-f]{40}", revision):
raise SystemExit("SOURCE_REVISION must be the full source commit")
if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", version):
raise SystemExit("release version must be X.Y.Z or X.Y.Z-rc.N")
binary = root / "target/x86_64-unknown-linux-musl/release/distill_fs"
binary = root / "target" / target / "release/distill_fs"
manifest = {
"component": "distill-fs",
"version": version,
"release_tag": f"v{version}",
"source_revision": revision,
"repository": "inclusionAI/distill-fs",
"target": "x86_64-unknown-linux-musl",
"target": target,
"binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(),
}
files = {
Expand All @@ -36,7 +44,7 @@
}
output = root / "dist"
output.mkdir(exist_ok=True)
archive = output / f"distill-fs-v{version}-linux-amd64.tar.gz"
archive = output / f"distill-fs-v{version}-linux-{arch}.tar.gz"
# Stable metadata lets a single build be compared and promoted byte for byte.
with archive.open("wb") as raw, gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as zipped:
with tarfile.open(fileobj=zipped, mode="w") as tar:
Expand Down
31 changes: 24 additions & 7 deletions ci/release.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,37 +1,54 @@
# Build on musl so the C dependencies and Rust use the same libc.
FROM rust:1.85.0-alpine3.21@sha256:bea885d2711087e67a9f7a7cd1a164976f4c35389478512af170730014d2452a AS build
ARG TARGETARCH
RUN apk add --no-cache build-base cmake git perl pkgconf python3 binutils && \
rustup component add rustfmt
WORKDIR /work
ENV CARGO_NET_GIT_FETCH_WITH_CLI=true \
CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=+crt-static" \
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=+crt-static" \
TMPDIR=/work/test-tmp
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
COPY .cargo/ .cargo/
COPY src/ src/
COPY tests/ tests/
RUN mkdir -p "$TMPDIR" && \
cargo fmt --all -- --check && \
cargo build --locked --release --target x86_64-unknown-linux-musl --bin distill_fs
RUN set -eu; \
case "$TARGETARCH" in \
amd64) target=x86_64-unknown-linux-musl ;; \
arm64) target=aarch64-unknown-linux-musl ;; \
*) echo "unsupported target architecture: $TARGETARCH" >&2; exit 1 ;; \
esac; \
mkdir -p "$TMPDIR"; \
cargo fmt --all -- --check; \
rustup target add "$target"; \
cargo build --locked --release --target "$target" --bin distill_fs
# Verify the final stripped binary before tests and packaging.
# Reject both dynamically linked executables and static PIEs with shared deps.
RUN binary=target/x86_64-unknown-linux-musl/release/distill_fs; \
RUN case "$TARGETARCH" in \
amd64) target=x86_64-unknown-linux-musl; machine='Advanced Micro Devices X86-64' ;; \
arm64) target=aarch64-unknown-linux-musl; machine=AArch64 ;; \
esac; \
binary="target/$target/release/distill_fs"; \
readelf -h "$binary" && \
readelf -h "$binary" | grep -Eq "Machine:[[:space:]]+$machine$" && \
! readelf -l "$binary" | grep -q INTERP && \
! readelf -d "$binary" | grep -q NEEDED && \
readelf --wide --sections "$binary" > /work/release-sections.txt && \
! grep -Eq '[[:space:]]\.(symtab|debug_[^[:space:]]*)[[:space:]]' /work/release-sections.txt

FROM build AS test
RUN cargo test --locked --release --target x86_64-unknown-linux-musl
RUN case "$TARGETARCH" in amd64) target=x86_64-unknown-linux-musl ;; arm64) target=aarch64-unknown-linux-musl ;; esac; \
cargo test --locked --release --target "$target"

FROM test AS package
ARG SOURCE_REVISION
ARG TARGETARCH
COPY ci/package-release.py ci/package-release.py
COPY LICENSE NOTICE ./
RUN python3 ci/package-release.py "$SOURCE_REVISION" && \
RUN case "$TARGETARCH" in amd64) target=x86_64-unknown-linux-musl ;; arm64) target=aarch64-unknown-linux-musl ;; esac; \
python3 ci/package-release.py "$SOURCE_REVISION" "$TARGETARCH" && \
mkdir /empty-root && \
cp target/x86_64-unknown-linux-musl/release/distill_fs /empty-root/distill_fs && \
cp "target/$target/release/distill_fs" /empty-root/distill_fs && \
chroot /empty-root /distill_fs --version && \
chroot /empty-root /distill_fs mount --help

Expand Down