diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4578d1c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +target +dist diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9af99fc --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,96 @@ +name: Static binary + +on: + pull_request: + push: + branches: [main] + tags: ['v*'] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: static-${{ github.ref }} + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Validate release tag + if: startsWith(github.ref, 'refs/tags/') + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + python3 - <<'PY' + import os, re, tomllib + with open("Cargo.toml", "rb") as source: + version = tomllib.load(source)["package"]["version"] + tag = os.environ["RELEASE_TAG"] + assert re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", tag), tag + assert tag == f"v{version}", "tag must match Cargo.toml" + PY + git merge-base --is-ancestor HEAD origin/main + - name: Test musl build + run: docker build --platform linux/amd64 --target test -f ci/release.Dockerfile -t distill-fs-static-test . + - name: Test raw and Nydus FUSE mounts + run: | + sudo modprobe fuse + test -c /dev/fuse + mkdir -p target/release-test-tools + curl -fSL --retry 5 \ + https://github.com/dragonflyoss/nydus/releases/download/v2.4.0/nydus-static-v2.4.0-linux-amd64.tgz \ + -o target/release-test-tools/nydus.tgz + echo "e4f7de672ced6031c13c3c38a599993f8a8103b0cf85f9b0783f5f5fa7c8b3dc target/release-test-tools/nydus.tgz" | sha256sum -c - + tar -xzf target/release-test-tools/nydus.tgz -C target/release-test-tools + docker run --rm --device /dev/fuse --cap-add SYS_ADMIN \ + --security-opt apparmor=unconfined --network none \ + -v "$PWD/target/release-test-tools/nydus-static:/nydus:ro" \ + -e DISTILL_FS_NYDUS_IMAGE=/nydus/nydus-image \ + distill-fs-static-test cargo test --locked --release \ + --target x86_64-unknown-linux-musl --test mount -- --ignored + - name: Build release archive and verify static executable + run: | + docker build --platform linux/amd64 -f ci/release.Dockerfile \ + --build-arg "SOURCE_REVISION=$(git rev-parse HEAD)" \ + --output type=local,dest=dist . + cd dist + sha256sum -c SHA256SUMS + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: distill-fs-linux-amd64 + path: dist/* + if-no-files-found: error + retention-days: 14 + + release: + if: startsWith(github.ref, 'refs/tags/') + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: distill-fs-linux-amd64 + path: dist + - name: Publish tested bytes + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RELEASE_TAG: ${{ github.ref_name }} + run: | + cd dist + sha256sum -c SHA256SUMS + prerelease=() + if [[ "$RELEASE_TAG" == *-rc.* ]]; then prerelease=(--prerelease); fi + gh release create "$RELEASE_TAG" --verify-tag \ + --title "distill-fs $RELEASE_TAG" --generate-notes \ + "${prerelease[@]}" distill-fs-*.tar.gz SHA256SUMS diff --git a/.gitignore b/.gitignore index be0e453..f838e54 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .idea/ todo.txt /todo + +/dist/ diff --git a/AGENTS.md b/AGENTS.md index f0316af..61a4007 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,3 +63,10 @@ - When resolving a skill, check the current project first, then the global Codex configuration directory. - Project-local skills live under `.codex/skills/` in this repository and take precedence when a skill with the same name exists in multiple locations. - If the skill is not present in the project, fall back to `$CODEX_HOME/skills/` (for example, `~/.codex/skills/`). + +## Static Release Contract + +- `ci/release.Dockerfile` builds Linux/amd64 musl artifacts with the pinned Rust image and `Cargo.lock`. Keep static ELF checks and empty-root smoke tests mandatory for packaging. +- `ci/package-release.py` defines archive names and provenance. Keep these synchronized with AKernel's `builder/scripts/install-distill-fs.sh` and `builder/distill-fs-versions.env`. +- Version tags must match `Cargo.toml` and point to commits on `main`. The workflow publishes already-tested bytes and must not overwrite release assets. +- Publish a release before updating consumer checksum pins. Do not use guessed digests or present locally rebuilt artifacts as published binaries. diff --git a/README.md b/README.md index b4a7835..0a9d6da 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,28 @@ Run the CLI help: cargo run --bin distill_fs -- --help ``` +## Static releases + +GitHub releases provide `distill-fs-vX.Y.Z-linux-amd64.tar.gz` and `SHA256SUMS`. The archive contains the `distill_fs` executable, `manifest.json`, `LICENSE`, `NOTICE`, and the exact `Cargo.lock`. The executable uses musl and has no dynamic loader or shared-library dependencies. Mounting still requires Linux FUSE support and suitable privileges; remote HTTPS backends still need a CA certificate store on the host. + +The first release is pending publication. Once published, download from `https://github.com/inclusionAI/distill-fs/releases/download/vX.Y.Z/`, verify the archive against a reviewed SHA-256 pin, extract it, and install `distill_fs` on `PATH`. Keep the license and provenance files with the binary. Production consumers must pin the release URL and checksum in source control; fetching a checksum alongside an unpinned binary is not an independent integrity check. + +Build and test the same Linux/amd64 artifact locally with Docker: + +```bash +docker build --platform linux/amd64 --target test -f ci/release.Dockerfile . +docker build --platform linux/amd64 -f ci/release.Dockerfile \ + --build-arg "SOURCE_REVISION=$(git rev-parse HEAD)" \ + --output type=local,dest=dist . +(cd dist && sha256sum -c SHA256SUMS) +``` + +Use a clean source checkout for release builds. `manifest.json` records the package version, source commit, musl target, and binary SHA-256. The Docker build checks the ELF for a dynamic interpreter and shared dependencies, then runs the executable in an empty chroot. The test target runs the default test suite against the musl release build; ignored FUSE tests need an explicitly privileged test environment. The pinned build image and `Cargo.lock` fix the toolchain and Rust dependency graph; Alpine build packages are resolved when the build runs, so independently rebuilding a tag is not a byte-for-byte reproducibility guarantee. + +The `Static binary` workflow builds and tests pull requests, `main`, manual runs, and version tags, including real raw and RAFS v5 FUSE mount tests using the checksum-pinned Nydus v2.4.0 tool. Non-tag runs upload a reviewable Actions artifact without publishing a release. Pushing a feature branch alone does not trigger the workflow; opening or updating a pull request does. Manual dispatch becomes available once the workflow file exists on the default branch: use Actions → Static binary → Run workflow, or `gh workflow run release.yml --repo inclusionAI/distill-fs --ref main`. To publish, update the package version and lockfile, merge to `main`, and push a matching `vX.Y.Z` tag (or `vX.Y.Z-rc.N` for a prerelease). The workflow rejects version mismatches and tags outside `main`, then publishes the tested archive without rebuilding in the release job. Existing release assets are never overwritten. Review changes before pushing a release tag. + +AKernel consumes this archive through its own `builder/distill-fs-versions.env` and `builder/scripts/install-distill-fs.sh`. Publish the distill-fs release first, verify its checksum, and update the AKernel pin. No sandboxd pipeline or gitlink change is required. The distill-fs submodule in AKernel is optional source reference and is no longer a build input. + ## CLI Overview The main binary exposes four subcommands: diff --git a/ci/package-release.py b/ci/package-release.py new file mode 100644 index 0000000..b6ff7e4 --- /dev/null +++ b/ci/package-release.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Package the verified musl executable; no network access or source rebuild.""" +import gzip +import hashlib +import io +import json +from pathlib import Path +import re +import sys +import tarfile +import tomllib + +root = Path(__file__).resolve().parent.parent +version = tomllib.loads((root / "Cargo.toml").read_text())["package"]["version"] +revision = sys.argv[1] +if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise SystemExit("SOURCE_REVISION must be the full source commit") +if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", version): + raise SystemExit("release version must be X.Y.Z or X.Y.Z-rc.N") +binary = root / "target/x86_64-unknown-linux-musl/release/distill_fs" +manifest = { + "component": "distill-fs", + "version": version, + "release_tag": f"v{version}", + "source_revision": revision, + "repository": "inclusionAI/distill-fs", + "target": "x86_64-unknown-linux-musl", + "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), +} +files = { + "distill_fs": binary.read_bytes(), + "manifest.json": (json.dumps(manifest, indent=2) + "\n").encode(), + "LICENSE": (root / "LICENSE").read_bytes(), + "NOTICE": (root / "NOTICE").read_bytes(), + "Cargo.lock": (root / "Cargo.lock").read_bytes(), +} +output = root / "dist" +output.mkdir(exist_ok=True) +archive = output / f"distill-fs-v{version}-linux-amd64.tar.gz" +# Stable metadata lets a single build be compared and promoted byte for byte. +with archive.open("wb") as raw, gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as zipped: + with tarfile.open(fileobj=zipped, mode="w") as tar: + for name, data in files.items(): + entry = tarfile.TarInfo(name) + entry.size = len(data) + entry.mode = 0o755 if name == "distill_fs" else 0o644 + tar.addfile(entry, io.BytesIO(data)) +checksum = hashlib.sha256(archive.read_bytes()).hexdigest() +(output / "SHA256SUMS").write_text(f"{checksum} {archive.name}\n") +print(f"{checksum} {archive.name}") diff --git a/ci/release.Dockerfile b/ci/release.Dockerfile new file mode 100644 index 0000000..7779e11 --- /dev/null +++ b/ci/release.Dockerfile @@ -0,0 +1,34 @@ +# Build on musl so the C dependencies and Rust use the same libc. +FROM rust:1.85.0-alpine3.21@sha256:bea885d2711087e67a9f7a7cd1a164976f4c35389478512af170730014d2452a AS build +RUN apk add --no-cache build-base cmake git perl pkgconf python3 binutils +WORKDIR /work +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true \ + CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=+crt-static" \ + TMPDIR=/work/test-tmp +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY .cargo/ .cargo/ +COPY src/ src/ +COPY tests/ tests/ +RUN mkdir -p "$TMPDIR" && \ + cargo build --locked --release --target x86_64-unknown-linux-musl --bin distill_fs +# Reject both dynamically linked executables and static PIEs with shared deps. +RUN binary=target/x86_64-unknown-linux-musl/release/distill_fs; \ + readelf -h "$binary" && \ + ! readelf -l "$binary" | grep -q INTERP && \ + ! readelf -d "$binary" | grep -q NEEDED + +FROM build AS test +RUN cargo test --locked --release --target x86_64-unknown-linux-musl + +FROM test AS package +ARG SOURCE_REVISION +COPY ci/package-release.py ci/package-release.py +COPY LICENSE NOTICE ./ +RUN python3 ci/package-release.py "$SOURCE_REVISION" && \ + mkdir /empty-root && \ + cp target/x86_64-unknown-linux-musl/release/distill_fs /empty-root/distill_fs && \ + chroot /empty-root /distill_fs --version && \ + chroot /empty-root /distill_fs mount --help + +FROM scratch AS artifact +COPY --from=package /work/dist/ / diff --git a/src/backend/dedup/tests.rs b/src/backend/dedup/tests.rs index 78a943e..09b2641 100644 --- a/src/backend/dedup/tests.rs +++ b/src/backend/dedup/tests.rs @@ -36,7 +36,8 @@ struct MockBackend { struct MockBackendState { data: HashMap, size: u64, - fetch_calls: u64, + foreground_thread: std::thread::ThreadId, + foreground_fetch_calls: u64, invalidate_calls: Vec, } @@ -52,7 +53,8 @@ impl MockBackend { state: Arc::new(Mutex::new(MockBackendState { data, size, - fetch_calls: 0, + foreground_thread: std::thread::current().id(), + foreground_fetch_calls: 0, invalidate_calls: vec![], })), } @@ -66,7 +68,11 @@ impl Backend for MockBackend { fn fetch(&self, off: usize, data: &mut [u8]) -> io::Result { let mut state = self.state.lock().unwrap(); - state.fetch_calls += 1; + // Reads also enqueue background dedup/sync work. Count only calls + // from the test thread so assertions do not depend on scheduling. + if std::thread::current().id() == state.foreground_thread { + state.foreground_fetch_calls += 1; + } if off >= state.size as usize { return Ok(0); @@ -247,14 +253,14 @@ fn test_dedup_and_fetch_simple() { .dedup(off as u64, len as u32, None, CheckSumMethod::Blake3) .unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 1); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 1); let mut buf = vec![0; len as usize]; dedup.fetch(off as usize, &mut buf).unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 1); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 1); let mut original_data = vec![0; len as usize]; - backend.state.lock().unwrap().fetch_calls = 0; + backend.state.lock().unwrap().foreground_fetch_calls = 0; backend.fetch(off as usize, &mut original_data).unwrap(); assert_eq!(buf, original_data); } @@ -401,12 +407,12 @@ fn test_deduplication_prevents_refetch() { dedup .dedup(off1, len as u32, None, CheckSumMethod::Sha256) .unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 1); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 1); dedup .dedup(off2, len as u32, None, CheckSumMethod::Sha256) .unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 2); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 2); let mut buf1 = vec![0; len as usize]; let mut buf2 = vec![0; len as usize]; @@ -416,7 +422,7 @@ fn test_deduplication_prevents_refetch() { assert_eq!(buf1, data_to_write); assert_eq!(buf2, data_to_write); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 2); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 2); } #[test] @@ -426,22 +432,20 @@ fn test_fetch_mixed_data() { dedup .dedup(0, CHUNK_SIZE as u32, None, CheckSumMethod::Blake3) .unwrap(); - let initial_fetch_count = backend.state.lock().unwrap().fetch_calls; + let initial_fetch_count = backend.state.lock().unwrap().foreground_fetch_calls; assert_eq!(initial_fetch_count, 1); - std::thread::sleep(std::time::Duration::from_millis(100)); - backend.state.lock().unwrap().fetch_calls = 0; + backend.state.lock().unwrap().foreground_fetch_calls = 0; let fetch_off = CHUNK_SIZE / 2; let fetch_len = CHUNK_SIZE; let mut buf = vec![0; fetch_len]; dedup.fetch(fetch_off, &mut buf).unwrap(); - let fetch_count = backend.state.lock().unwrap().fetch_calls; - assert!( - fetch_count >= 1, - "Expected at least 1 fetch for chunk 1, got {}", - fetch_count + let fetch_count = backend.state.lock().unwrap().foreground_fetch_calls; + assert_eq!( + fetch_count, 1, + "Only chunk 1 should fall back to the backend" ); let mut original_data = vec![0; fetch_len]; @@ -545,14 +549,14 @@ fn test_fetch_partial_deduped_block() { dedup .dedup(off as u64, len as u32, None, CheckSumMethod::Blake3) .unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 1); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 1); let fetch_off = CHUNK_SIZE / 4; let fetch_len = CHUNK_SIZE / 2; let mut buf = vec![0; fetch_len]; dedup.fetch(fetch_off, &mut buf).unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 1); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 1); let mut original_data = vec![0; fetch_len]; backend.fetch(fetch_off, &mut original_data).unwrap(); @@ -573,17 +577,17 @@ fn test_dedup_with_precomputed_checksum() { dedup .dedup(off1 as u64, len, None, CheckSumMethod::Sha256) .unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 2); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 2); dedup .dedup(off2 as u64, len, Some(cs), CheckSumMethod::Sha256) .unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 2); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 2); let mut buf = vec![0; len as usize]; dedup.fetch(off2 as usize, &mut buf).unwrap(); assert_eq!(buf, data); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 2); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 2); } #[test] @@ -618,11 +622,11 @@ fn test_dedup_instances_isolate_by_data_id() { dedup_a .dedup(off as u64, len, None, CheckSumMethod::Blake3) .unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 1); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 1); let mut buf = vec![0_u8; len as usize]; dedup_b.fetch(off as usize, &mut buf).unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 2); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 2); } #[test] @@ -632,15 +636,17 @@ fn test_fetch_fallback_on_missing_global_data() { let len = CHUNK_SIZE; let data_cs = CheckSum::from_data(b"some fake data", CheckSumMethod::Blake3); - let range = DedupRange::new(off as u64, len as u32); + let range = DedupRange::new_with_id(dedup.core.id, off as u64, len as u32); dedup.core.add_local_dedup_info(&range, &data_cs).unwrap(); + assert!(dedup.core.dedup_check_range(&range).unwrap()); + assert!(!dedup.core.chunk_db.has_chunk(&data_cs).unwrap()); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 0); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 0); let mut buf = vec![0; len as usize]; dedup.fetch(off as usize, &mut buf).unwrap(); - assert_eq!(backend.state.lock().unwrap().fetch_calls, 1); + assert_eq!(backend.state.lock().unwrap().foreground_fetch_calls, 1); let mut original_data = vec![0; len as usize]; backend.fetch(off as usize, &mut original_data).unwrap();