From cf78ff336c40bc6860ff9bb65041bd3e6b2fcdf0 Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 06:46:46 +0100 Subject: [PATCH 01/14] feat: cache native dependency builds --- ...tive-build-cache-20260712-macos-arm64.json | 53 ++ ...native-build-cache-20260712-macos-arm64.md | 24 + bench/scripts/README.md | 14 + .../scripts/native-build-cache-benchmark.mjs | 271 +++++++ crates/lpm-cli/src/commands/cache_prune.rs | 189 ++++- .../src/commands/rebuild/build_cache.rs | 517 +++++++++++++ crates/lpm-cli/src/commands/rebuild/mod.rs | 271 ++++++- .../src/commands/rebuild/script_execution.rs | 12 +- .../lpm-cli/src/commands/rebuild/scripts.rs | 47 ++ .../lpm-cli/src/commands/rebuild/tests/mod.rs | 20 + crates/lpm-cli/src/sandbox_config.rs | 1 + crates/lpm-common/src/lib.rs | 6 +- crates/lpm-common/src/paths.rs | 11 + crates/lpm-sandbox/src/landlock_rules.rs | 71 +- crates/lpm-sandbox/src/lib.rs | 6 + crates/lpm-sandbox/src/linux.rs | 21 +- crates/lpm-sandbox/src/macos.rs | 6 +- crates/lpm-sandbox/src/seatbelt.rs | 73 +- crates/lpm-sandbox/src/windows.rs | 2 + .../tests/seatbelt_secret_deny_runtime.rs | 2 + .../lpm-sandbox/tests/seccomp_socket_deny.rs | 1 + .../tests/secret_overlay_bind_mount.rs | 2 + crates/lpm-store/src/v2/build_cache.rs | 720 ++++++++++++++++++ crates/lpm-store/src/v2/mod.rs | 6 + crates/lpm-store/src/v2/store.rs | 49 ++ tests/workflows/tests/native_build_cache.rs | 261 +++++++ ...cache__native_build_cache_hit_metrics.snap | 27 + 27 files changed, 2633 insertions(+), 50 deletions(-) create mode 100644 bench/perf-results/native-build-cache-20260712-macos-arm64.json create mode 100644 bench/perf-results/native-build-cache-20260712-macos-arm64.md create mode 100644 bench/scripts/native-build-cache-benchmark.mjs create mode 100644 crates/lpm-cli/src/commands/rebuild/build_cache.rs create mode 100644 crates/lpm-store/src/v2/build_cache.rs create mode 100644 tests/workflows/tests/native_build_cache.rs create mode 100644 tests/workflows/tests/snapshots/native_build_cache__native_build_cache_hit_metrics.snap diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64.json b/bench/perf-results/native-build-cache-20260712-macos-arm64.json new file mode 100644 index 00000000..092407ad --- /dev/null +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64.json @@ -0,0 +1,53 @@ +{ + "generated_at": "2026-07-12T05:36:38.673Z", + "platform": "darwin-arm64", + "node": "v22.22.1", + "lpm_bin": "target/release/lpm-rs", + "fixture": "bench/audit-fixtures/native/esbuild-prebuilt", + "samples": 10, + "artifact_bytes": 134444, + "scenarios": { + "cache_disabled": { + "wall_ms_median": 114.99, + "wall_ms_p95": 417.43, + "max_rss_bytes_median": 52248576, + "user_cpu_ms_median": 40, + "system_cpu_ms_median": 10, + "cache_hits": 0, + "cache_misses": 0 + }, + "cache_miss": { + "wall_ms_median": 134.58, + "wall_ms_p95": 148.42, + "max_rss_bytes_median": 52264960, + "user_cpu_ms_median": 50, + "system_cpu_ms_median": 20, + "cache_hits": 0, + "cache_misses": 10 + }, + "local_hit": { + "wall_ms_median": 29.45, + "wall_ms_p95": 51.91, + "max_rss_bytes_median": 41500672, + "user_cpu_ms_median": 10, + "system_cpu_ms_median": 0, + "cache_hits": 10, + "cache_misses": 0, + "lifecycle_ms_avoided": 1028, + "speedup_vs_disabled": 3.9, + "wall_reduction_percent": 74.39 + }, + "ci_warm_store_hit": { + "wall_ms_median": 29.19, + "wall_ms_p95": 31.29, + "max_rss_bytes_median": 41566208, + "user_cpu_ms_median": 10, + "system_cpu_ms_median": 0, + "cache_hits": 10, + "cache_misses": 0, + "lifecycle_ms_avoided": 1028, + "speedup_vs_disabled": 3.94, + "wall_reduction_percent": 74.62 + } + } +} diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64.md b/bench/perf-results/native-build-cache-20260712-macos-arm64.md new file mode 100644 index 00000000..1b2bac01 --- /dev/null +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64.md @@ -0,0 +1,24 @@ +# Native lifecycle-build cache benchmark + +- Platform: macOS arm64 +- Node: v22.22.1 +- Fixture: `bench/audit-fixtures/native/esbuild-prebuilt` +- Release-build samples per scenario: 10 +- Build artifact size: 134,444 bytes + +| Scenario | Median wall | p95 wall | Median RSS | User CPU | System CPU | Result | +|---|---:|---:|---:|---:|---:|---:| +| Cache disabled | 114.99 ms | 417.43 ms | 52,248,576 B | 40 ms | 10 ms | baseline | +| Cache miss | 134.58 ms | 148.42 ms | 52,264,960 B | 50 ms | 20 ms | 10 misses | +| Local artifact hit | 29.45 ms | 51.91 ms | 41,500,672 B | 10 ms | 0 ms | 3.90×; 74.39% lower wall time | +| Fresh-project warm-store hit | 29.19 ms | 31.29 ms | 41,566,208 B | 10 ms | 0 ms | 3.94×; 74.62% lower wall time | + +The disabled and miss scenarios rematerialized pristine dependencies before +every sample. Local-hit samples rematerialized the same project. CI-like hit +samples created a fresh project while retaining the warm LPM store. Across ten +hits, the artifact metadata recorded 1,028 ms of lifecycle execution avoided. + +The cache miss adds about 19.6 ms over the disabled median for keying, +pristine rematerialization, integrity validation, and atomic publication. This +fixture has a short lifecycle command; native compilation workloads should +have a substantially larger absolute hit benefit. diff --git a/bench/scripts/README.md b/bench/scripts/README.md index 1bbf5fe3..e59e666c 100644 --- a/bench/scripts/README.md +++ b/bench/scripts/README.md @@ -1,5 +1,19 @@ # Install benchmark scripts +## Native lifecycle-build cache + +Build the release CLI, then compare uncached rebuilds, cache misses, local +artifact hits, and CI-like fresh-project hits against the native esbuild +fixture: + +```bash +cargo build --release --locked -p lpm-cli --bin lpm-rs +node bench/scripts/native-build-cache-benchmark.mjs --samples 10 +``` + +The harness writes raw rows plus JSON and Markdown summaries under +`bench/perf-results/native-build-cache-/`. + ## Production-readiness harness `run-install-readiness.mjs` is the current install-readiness harness. It is diff --git a/bench/scripts/native-build-cache-benchmark.mjs b/bench/scripts/native-build-cache-benchmark.mjs new file mode 100644 index 00000000..b77663be --- /dev/null +++ b/bench/scripts/native-build-cache-benchmark.mjs @@ -0,0 +1,271 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = path.resolve(fileURLToPath(new URL('../..', import.meta.url))); +const options = parseArgs(process.argv.slice(2)); +const samples = Number(options.samples ?? 10); +const fixture = path.resolve( + repoRoot, + options.fixture ?? 'bench/audit-fixtures/native/esbuild-prebuilt', +); +const lpmBin = path.resolve(options['lpm-bin'] ?? path.join(repoRoot, 'target/release/lpm-rs')); +const timestamp = new Date().toISOString().replaceAll(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); +const outputDir = path.resolve( + options.output ?? path.join(repoRoot, 'bench/perf-results', `native-build-cache-${timestamp}`), +); + +if (!Number.isInteger(samples) || samples < 2) { + throw new Error('--samples must be an integer >= 2'); +} +for (const required of [lpmBin, path.join(fixture, 'package.json')]) { + if (!fs.existsSync(required)) throw new Error(`missing required path: ${required}`); +} + +const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'lpm-native-build-cache-bench-')); +const home = path.join(tempRoot, 'home'); +const lpmHome = path.join(tempRoot, 'lpm-home'); +const primary = path.join(tempRoot, 'primary'); +fs.mkdirSync(home, { recursive: true }); +materializeProject(primary); + +const env = { + ...process.env, + HOME: home, + USERPROFILE: home, + LPM_HOME: lpmHome, + LPM_STORE_VERSION: 'v2', + LPM_TYPOSQUAT_GUARD: '0', + CI: '1', +}; + +const rows = []; +try { + runChecked(['install'], primary); + for (let sample = 1; sample <= samples; sample += 1) { + rematerialize(primary); + rows.push(measureRebuild('cache_disabled', sample, primary, false)); + + fs.rmSync(path.join(lpmHome, 'store/v2/builds'), { recursive: true, force: true }); + rematerialize(primary); + rows.push(measureRebuild('cache_miss', sample, primary, true)); + + rematerialize(primary); + rows.push(measureRebuild('local_hit', sample, primary, true)); + + const ciProject = path.join(tempRoot, `ci-project-${sample}`); + materializeProject(ciProject); + runChecked(['install'], ciProject); + rows.push(measureRebuild('ci_warm_store_hit', sample, ciProject, true)); + } + + const artifactBytes = directorySize(path.join(lpmHome, 'store/v2/builds')); + const summary = summarize(rows, artifactBytes); + fs.mkdirSync(outputDir, { recursive: true }); + fs.writeFileSync(path.join(outputDir, 'rows.json'), `${JSON.stringify(rows, null, 2)}\n`); + fs.writeFileSync(path.join(outputDir, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`); + fs.writeFileSync(path.join(outputDir, 'summary.md'), `${renderMarkdown(summary)}\n`); + console.log(outputDir); + console.log(renderMarkdown(summary)); +} finally { + if (!options.keep) { + fs.rmSync(tempRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +} + +function materializeProject(projectDir) { + fs.mkdirSync(projectDir, { recursive: true }); + fs.copyFileSync(path.join(fixture, 'package.json'), path.join(projectDir, 'package.json')); +} + +function rematerialize(projectDir) { + fs.rmSync(path.join(projectDir, 'node_modules'), { recursive: true, force: true }); + runChecked(['install', '--force', '--no-frozen-lockfile'], projectDir); +} + +function measureRebuild(scenario, sample, cwd, strict) { + const args = ['--json', 'rebuild', '--all']; + if (strict) args.push('--strict-sandbox'); + const started = process.hrtime.bigint(); + const result = runTimed(args, cwd); + const wallMs = Number(process.hrtime.bigint() - started) / 1e6; + if (result.status !== 0) { + throw new Error( + `${scenario} failed (${result.status})\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } + const envelope = JSON.parse(result.stdout); + return { + scenario, + sample, + wall_ms: round(wallMs), + user_cpu_ms: result.userCpuMs, + system_cpu_ms: result.systemCpuMs, + max_rss_bytes: result.maxRssBytes, + build_cache: envelope.build_cache, + }; +} + +function runChecked(args, cwd) { + const result = spawnSync(lpmBin, args, { + cwd, + env, + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error( + `lpm ${args.join(' ')} failed (${result.status})\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function runTimed(args, cwd) { + if (process.platform === 'darwin' && fs.existsSync('/usr/bin/time')) { + const result = spawnSync('/usr/bin/time', ['-l', lpmBin, ...args], { + cwd, + env, + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }); + const cpu = result.stderr.match(/([\d.]+) real\s+([\d.]+) user\s+([\d.]+) sys/); + const rss = result.stderr.match(/\s+(\d+)\s+maximum resident set size/); + return { + ...result, + userCpuMs: cpu ? Number(cpu[2]) * 1000 : null, + systemCpuMs: cpu ? Number(cpu[3]) * 1000 : null, + maxRssBytes: rss ? Number(rss[1]) : null, + }; + } + if (process.platform === 'linux' && fs.existsSync('/usr/bin/time')) { + const result = spawnSync('/usr/bin/time', ['-v', lpmBin, ...args], { + cwd, + env, + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }); + const user = result.stderr.match(/User time \(seconds\): ([\d.]+)/); + const system = result.stderr.match(/System time \(seconds\): ([\d.]+)/); + const rss = result.stderr.match(/Maximum resident set size \(kbytes\): (\d+)/); + return { + ...result, + userCpuMs: user ? Number(user[1]) * 1000 : null, + systemCpuMs: system ? Number(system[1]) * 1000 : null, + maxRssBytes: rss ? Number(rss[1]) * 1024 : null, + }; + } + const result = spawnSync(lpmBin, args, { + cwd, + env, + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }); + return { ...result, userCpuMs: null, systemCpuMs: null, maxRssBytes: null }; +} + +function summarize(allRows, artifactBytes) { + const scenarios = {}; + for (const scenario of [...new Set(allRows.map((row) => row.scenario))]) { + const selected = allRows.filter((row) => row.scenario === scenario); + const walls = selected.map((row) => row.wall_ms).sort((a, b) => a - b); + const rss = selected + .map((row) => row.max_rss_bytes) + .filter((value) => value != null) + .sort((a, b) => a - b); + scenarios[scenario] = { + samples: selected.length, + wall_ms_median: percentile(walls, 0.5), + wall_ms_p95: percentile(walls, 0.95), + max_rss_bytes_median: rss.length ? percentile(rss, 0.5) : null, + user_cpu_ms_median: medianNullable(selected.map((row) => row.user_cpu_ms)), + system_cpu_ms_median: medianNullable(selected.map((row) => row.system_cpu_ms)), + cache_hits: selected.reduce((sum, row) => sum + row.build_cache.hits, 0), + cache_misses: selected.reduce((sum, row) => sum + row.build_cache.misses, 0), + lifecycle_ms_avoided: selected.reduce( + (sum, row) => sum + row.build_cache.lifecycle_ms_avoided, + 0, + ), + }; + } + const baseline = scenarios.cache_disabled.wall_ms_median; + for (const scenario of ['local_hit', 'ci_warm_store_hit']) { + scenarios[scenario].speedup_vs_disabled = round(baseline / scenarios[scenario].wall_ms_median); + scenarios[scenario].wall_reduction_percent = round( + 100 * (1 - scenarios[scenario].wall_ms_median / baseline), + ); + } + return { + generated_at: new Date().toISOString(), + platform: `${process.platform}-${process.arch}`, + node: process.version, + lpm_bin: lpmBin, + fixture: path.relative(repoRoot, fixture), + samples, + artifact_bytes: artifactBytes, + scenarios, + }; +} + +function renderMarkdown(summary) { + const lines = [ + '# Native lifecycle-build cache benchmark', + '', + `- Platform: ${summary.platform}`, + `- Fixture: \`${summary.fixture}\``, + `- Samples per scenario: ${summary.samples}`, + `- Build artifact bytes: ${summary.artifact_bytes}`, + '', + '| Scenario | Median wall | p95 wall | Median RSS | Hits | Misses | Speedup |', + '|---|---:|---:|---:|---:|---:|---:|', + ]; + for (const [name, value] of Object.entries(summary.scenarios)) { + lines.push( + `| ${name} | ${value.wall_ms_median} ms | ${value.wall_ms_p95} ms | ${value.max_rss_bytes_median ?? 'n/a'} B | ${value.cache_hits} | ${value.cache_misses} | ${value.speedup_vs_disabled ? `${value.speedup_vs_disabled}×` : '—'} |`, + ); + } + lines.push( + '', + 'The disabled and miss scenarios rematerialize pristine dependencies before each sample. Local-hit samples rematerialize the same project; CI-hit samples create a fresh project while retaining the warm LPM store.', + ); + return lines.join('\n'); +} + +function percentile(sorted, fraction) { + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)]; +} + +function medianNullable(values) { + const present = values.filter((value) => value != null).sort((a, b) => a - b); + return present.length ? percentile(present, 0.5) : null; +} + +function directorySize(root) { + if (!fs.existsSync(root)) return 0; + let total = 0; + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + const candidate = path.join(root, entry.name); + if (entry.isDirectory()) total += directorySize(candidate); + else if (entry.isFile()) total += fs.statSync(candidate).size; + } + return total; +} + +function round(value) { + return Math.round(value * 100) / 100; +} + +function parseArgs(argv) { + const parsed = {}; + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (!token.startsWith('--')) throw new Error(`unexpected argument: ${token}`); + const key = token.slice(2); + if (key === 'keep') parsed.keep = true; + else parsed[key] = argv[++index]; + } + return parsed; +} diff --git a/crates/lpm-cli/src/commands/cache_prune.rs b/crates/lpm-cli/src/commands/cache_prune.rs index 217cb282..5a7e2ce2 100644 --- a/crates/lpm-cli/src/commands/cache_prune.rs +++ b/crates/lpm-cli/src/commands/cache_prune.rs @@ -92,6 +92,10 @@ pub struct PruneSummary { /// cutoff. Without reachability we evict islands by LRU, never by the /// project walk, so a complete recently-used island is always kept. pub compat_islands_orphaned: Vec, + /// Content-addressed lifecycle-build artifacts on disk. + pub build_artifacts_total: usize, + /// Incomplete or age-expired lifecycle-build artifacts. + pub build_artifacts_orphaned: Vec, /// Total bytes that would be / were freed by deleting orphans. /// Sum of `link_entries_orphaned` + `object_entries_orphaned` + /// `compat_islands_orphaned` directory sizes. @@ -246,6 +250,15 @@ fn run_locked( )) })?; } + for artifact in &summary.build_artifacts_orphaned { + std::fs::remove_dir_all(artifact).map_err(|e| { + LpmError::Store(format!( + "cache prune: failed to remove build artifact {}: {e}", + artifact.display() + )) + })?; + } + remove_orphaned_build_locks(v2_store)?; // Sweep deferred global-uninstall tombstones. Errors are // surfaced via `summary.tombstone_sweep_error` (and a @@ -276,6 +289,33 @@ fn run_locked( Ok(summary) } +fn remove_orphaned_build_locks(store: &V2Store) -> Result<(), LpmError> { + let entries = match std::fs::read_dir(store.paths().build_locks_root()) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + for entry in entries { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let path = entry.path(); + let Some(key) = path.file_stem().and_then(|stem| stem.to_str()) else { + continue; + }; + if key.len() == 64 + && key + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + && !store.paths().builds_root().join(key).is_dir() + { + std::fs::remove_file(path)?; + } + } + Ok(()) +} + /// Compute (but don't apply) the prune plan. Pulled out so unit tests /// exercise the algorithm against synthetic stores without touching /// the filesystem outside of a tempdir. @@ -357,8 +397,11 @@ pub fn compute_prune_plan( // independent), so it is safe even with no usable registry. let (compat_islands_total, compat_islands_orphaned) = compute_compat_island_orphans(v2_store.paths().compat_root(), max_age); + let (build_artifacts_total, build_artifacts_orphaned) = + compute_build_artifact_orphans(v2_store.paths().builds_root(), max_age); let bytes_freed_or_eligible = compat_islands_orphaned .iter() + .chain(&build_artifacts_orphaned) .map(|dir| dir_size(dir).unwrap_or(0)) .fold(0u64, u64::saturating_add); return Ok(PruneSummary { @@ -373,6 +416,8 @@ pub fn compute_prune_plan( object_entries_orphaned: Vec::new(), compat_islands_total, compat_islands_orphaned, + build_artifacts_total, + build_artifacts_orphaned, bytes_freed_or_eligible, tombstones_pending, tombstones_swept: 0, @@ -514,6 +559,8 @@ pub fn compute_prune_plan( // are a macOS clonefile feature) → the dir is missing → no-op. let (compat_islands_total, compat_islands_orphaned) = compute_compat_island_orphans(v2_store.paths().compat_root(), max_age); + let (build_artifacts_total, build_artifacts_orphaned) = + compute_build_artifact_orphans(v2_store.paths().builds_root(), max_age); let mut bytes_freed_or_eligible = 0u64; for dir in &link_entries_orphaned { @@ -528,6 +575,10 @@ pub fn compute_prune_plan( bytes_freed_or_eligible = bytes_freed_or_eligible.saturating_add(dir_size(dir).unwrap_or(0)); } + for dir in &build_artifacts_orphaned { + bytes_freed_or_eligible = + bytes_freed_or_eligible.saturating_add(dir_size(dir).unwrap_or(0)); + } let (tombstones_pending, tombstone_count_error) = count_tombstones_with_error_capture(root); @@ -543,6 +594,8 @@ pub fn compute_prune_plan( object_entries_orphaned, compat_islands_total, compat_islands_orphaned, + build_artifacts_total, + build_artifacts_orphaned, bytes_freed_or_eligible, tombstones_pending, tombstones_swept: 0, @@ -683,7 +736,30 @@ fn compute_compat_island_orphans( compat_root: &Path, max_age: Option, ) -> (usize, Vec) { - let read = match std::fs::read_dir(compat_root) { + compute_lru_artifact_orphans( + compat_root, + lpm_store::v2::COMPAT_ISLAND_COMPLETE_FILENAME, + max_age, + ) +} + +fn compute_build_artifact_orphans( + builds_root: &Path, + max_age: Option, +) -> (usize, Vec) { + compute_lru_artifact_orphans( + builds_root, + lpm_store::v2::BUILD_ARTIFACT_COMPLETE_FILENAME, + max_age, + ) +} + +fn compute_lru_artifact_orphans( + root: &Path, + completion_sentinel: &str, + max_age: Option, +) -> (usize, Vec) { + let read = match std::fs::read_dir(root) { Ok(read) => read, Err(_) => return (0, Vec::new()), }; @@ -696,7 +772,7 @@ fn compute_compat_island_orphans( } let island = entry.path(); total += 1; - let sentinel = island.join(lpm_store::v2::COMPAT_ISLAND_COMPLETE_FILENAME); + let sentinel = island.join(completion_sentinel); let Ok(meta) = std::fs::metadata(&sentinel) else { orphaned.push(island); continue; @@ -833,6 +909,18 @@ fn emit_human(summary: &PruneSummary, applied: bool, elapsed: Duration) { }, )); } + if !summary.build_artifacts_orphaned.is_empty() { + install_ui::phase(&format!( + "{} stale build artifact(s) {} {}", + summary.build_artifacts_orphaned.len(), + if applied { "evicted" } else { "eligible —" }, + if applied { + String::new() + } else { + "`lpm cache prune --apply`".to_string() + }, + )); + } if !applied && summary.link_entries_orphaned.len() <= 20 { for dir in &summary.link_entries_orphaned { println!( @@ -852,6 +940,12 @@ fn emit_human(summary: &PruneSummary, applied: bool, elapsed: Duration) { sanitize_for_terminal(&dir.display().to_string()) ); } + for dir in &summary.build_artifacts_orphaned { + println!( + " build artifact orphan: {}", + sanitize_for_terminal(&dir.display().to_string()) + ); + } } if !applied { install_ui::done(&format!("Done · checked cache in {elapsed}")); @@ -884,6 +978,12 @@ fn emit_json(summary: &PruneSummary) { .iter() .map(|p| p.display().to_string()) .collect::>(), + "build_artifacts_total": summary.build_artifacts_total, + "build_artifacts_orphaned": summary + .build_artifacts_orphaned + .iter() + .map(|p| p.display().to_string()) + .collect::>(), "bytes_freed_or_eligible": summary.bytes_freed_or_eligible, "registry_missing": summary.registry_missing, "registry_corrupt": summary.registry_corrupt, @@ -1099,6 +1199,91 @@ mod tests { ); } + #[test] + fn build_artifact_prune_evicts_incomplete_and_stale_entries() { + let tmp = tempfile::tempdir().unwrap(); + let builds_root = tmp.path().join("builds"); + std::fs::create_dir_all(&builds_root).unwrap(); + let sentinel = lpm_store::v2::BUILD_ARTIFACT_COMPLETE_FILENAME; + + let fresh = builds_root.join("fresh"); + std::fs::create_dir_all(&fresh).unwrap(); + std::fs::write(fresh.join(sentinel), b"").unwrap(); + + let incomplete = builds_root.join("incomplete"); + std::fs::create_dir_all(&incomplete).unwrap(); + + let stale = builds_root.join("stale"); + std::fs::create_dir_all(&stale).unwrap(); + let stale_sentinel = stale.join(sentinel); + std::fs::write(&stale_sentinel, b"").unwrap(); + let ten_days_ago = + std::time::SystemTime::now() - std::time::Duration::from_secs(10 * 86_400); + std::fs::File::options() + .write(true) + .open(&stale_sentinel) + .unwrap() + .set_modified(ten_days_ago) + .unwrap(); + + let (total, orphans) = compute_build_artifact_orphans(&builds_root, None); + assert_eq!(total, 3); + assert_eq!(orphans, vec![incomplete.clone()]); + + let (total, mut orphans) = + compute_build_artifact_orphans(&builds_root, Some(ChronoDuration::days(7))); + assert_eq!(total, 3); + orphans.sort(); + let mut expected = vec![incomplete, stale]; + expected.sort(); + assert_eq!(orphans, expected); + } + + #[test] + fn build_artifact_crash_recovery_applies_without_project_registry() { + let tmp = tempfile::tempdir().unwrap(); + let root = LpmRoot::from_dir(tmp.path()); + let store = V2Store::from_lpm_root(&root); + let incomplete = store.paths().builds_root().join("incomplete"); + std::fs::create_dir_all(&incomplete).unwrap(); + std::fs::write(incomplete.join("partial"), b"partial").unwrap(); + + let summary = run_locked( + &root, + &store, + &PruneFlags { + apply: true, + ..PruneFlags::default() + }, + None, + ) + .unwrap(); + + assert!(summary.registry_missing); + assert_eq!(summary.build_artifacts_orphaned, vec![incomplete.clone()]); + assert!(!incomplete.exists()); + } + + #[test] + fn build_lock_cleanup_removes_only_locks_without_artifacts() { + let tmp = tempfile::tempdir().unwrap(); + let root = LpmRoot::from_dir(tmp.path()); + let store = V2Store::from_lpm_root(&root); + let orphan_key = "a".repeat(64); + let live_key = "b".repeat(64); + std::fs::create_dir_all(store.paths().build_locks_root()).unwrap(); + std::fs::create_dir_all(store.paths().builds_root().join(&live_key)).unwrap(); + let orphan_lock = store.paths().build_lock_path_for_key(&orphan_key); + let live_lock = store.paths().build_lock_path_for_key(&live_key); + std::fs::write(&orphan_lock, b"").unwrap(); + std::fs::write(&live_lock, b"").unwrap(); + + remove_orphaned_build_locks(&store).unwrap(); + + assert!(!orphan_lock.exists()); + assert!(live_lock.exists()); + } + #[test] #[cfg(unix)] fn compute_prune_plan_follows_dep_edges_via_sidecar() { diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache.rs b/crates/lpm-cli/src/commands/rebuild/build_cache.rs new file mode 100644 index 00000000..94aa03ce --- /dev/null +++ b/crates/lpm-cli/src/commands/rebuild/build_cache.rs @@ -0,0 +1,517 @@ +use super::scripts::ScriptablePackage; +use crate::capability::CapabilitySet; +use lpm_sandbox::{SandboxMode, SandboxOptions, SandboxPosture}; +use lpm_security::EXECUTED_INSTALL_PHASES; +use lpm_store::v2::{ + BuildCacheKey, BuildKeyInputs, BuildPlatformFingerprint, BuildRuntimeFingerprint, + BuildSandboxFingerprint, BuildScriptFingerprint, PlatformTuple, +}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; + +pub(super) struct BuildCacheScratch { + path: PathBuf, +} + +impl BuildCacheScratch { + pub(super) fn create(package_dir: &Path) -> std::io::Result { + let link_dir = package_dir + .ancestors() + .find(|ancestor| ancestor.join(lpm_store::v2::LINK_META_FILENAME).is_file()) + .ok_or_else(|| std::io::Error::other("package is not inside a v2 link entry"))?; + let path = link_dir.join(format!( + ".lpm-build-tmp.{}.{:016x}", + std::process::id(), + rand::random::() + )); + std::fs::create_dir(&path)?; + Ok(Self { path }) + } + + #[inline] + pub(super) fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for BuildCacheScratch { + fn drop(&mut self) { + if let Err(error) = std::fs::remove_dir_all(&self.path) { + tracing::debug!( + "failed to remove build-cache scratch directory at {}: {error}", + self.path.display() + ); + } + } +} + +pub(super) struct BuildCacheInvocation { + dependency_closure_hash: String, + platform: BuildPlatformFingerprint, + runtime: BuildRuntimeFingerprint, + sandbox: BuildSandboxFingerprint, + environment_hash: String, + environment: HashMap, + native_toolchain_hash: OnceLock>, +} + +impl BuildCacheInvocation { + #[allow(clippy::too_many_arguments)] + pub(super) fn prepare( + lockfile: &lpm_lockfile::Lockfile, + environment: &HashMap, + sandbox_mode: SandboxMode, + posture: &SandboxPosture, + sandbox_options: &SandboxOptions, + extra_write_dirs: &[PathBuf], + extra_read_paths: &[PathBuf], + capabilities: &CapabilitySet, + ) -> Option { + if sandbox_mode != SandboxMode::Enforce { + debug_bypass("sandbox is not enforcing"); + return None; + } + if !matches!(posture, SandboxPosture::Strict) || !sandbox_options.deny_outbound_network { + debug_bypass("outbound network denial is not effective"); + return None; + } + if !extra_write_dirs.is_empty() || !extra_read_paths.is_empty() { + debug_bypass("project widened sandbox paths"); + return None; + } + if !capabilities.is_at_baseline() { + debug_bypass("project widened lifecycle capabilities"); + return None; + } + let Some(runtime) = detect_node_runtime(environment) else { + debug_bypass("Node runtime fingerprint unavailable"); + return None; + }; + let platform = PlatformTuple::current(); + Some(Self { + dependency_closure_hash: hash_lockfile_graph(lockfile), + platform: BuildPlatformFingerprint { + os: platform.os, + architecture: platform.cpu, + libc: platform.libc.unwrap_or_default(), + cpu_features_hash: cpu_features_hash(), + }, + runtime, + sandbox: BuildSandboxFingerprint { + mode: "enforce".into(), + posture: "strict".into(), + network_denied: true, + environment_scrubbed: true, + allowed_inputs_hash: hash_allowed_inputs(capabilities), + }, + environment_hash: hash_build_environment(environment), + environment: environment.clone(), + native_toolchain_hash: OnceLock::new(), + }) + } +} + +fn debug_bypass(reason: &str) { + tracing::debug!(target: "lpm_cli::build_cache", "bypass: {reason}"); +} + +pub(super) fn is_cacheable_native_build(package: &ScriptablePackage) -> bool { + if !cfg!(any(target_os = "macos", target_os = "linux")) { + return false; + } + package.scripts.values().any(|command| { + command.contains("node-gyp") + || command.contains("node-gyp-build") + || command.contains("prebuild-install") + || (package.name == "esbuild" && command.contains("install.js")) + || (package.name == "sharp" && command.contains("install/")) + }) +} + +pub(super) fn marker_requires_key_validation(package: &ScriptablePackage) -> bool { + package.build_marker_key.is_some() && is_cacheable_native_build(package) +} + +pub(super) fn read_build_marker_key(marker_path: &Path) -> Option { + let bytes = std::fs::read(marker_path).ok()?; + if bytes.len() != 64 + || !bytes + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + { + return None; + } + String::from_utf8(bytes).ok() +} + +pub(super) fn build_key_for_package( + invocation: &BuildCacheInvocation, + package: &ScriptablePackage, +) -> Option { + let Some(graph_key_digest) = package.graph_key_digest.as_ref() else { + debug_bypass(&format!("{} has no v2 graph identity", package.name)); + return None; + }; + if !is_cacheable_native_build(package) { + debug_bypass(&format!( + "{} lifecycle command is not a recognized native build", + package.name + )); + return None; + } + let mut scripts = Vec::with_capacity(EXECUTED_INSTALL_PHASES.len()); + for phase in EXECUTED_INSTALL_PHASES { + if let Some(command) = package.scripts.get(*phase) { + scripts.push(BuildScriptFingerprint { + phase: (*phase).to_string(), + command: command.clone(), + }); + } + } + let toolchain_hash = if package + .scripts + .values() + .any(|command| command.contains("node-gyp")) + { + let Some(hash) = invocation + .native_toolchain_hash + .get_or_init(|| hash_toolchain(&invocation.environment).ok()) + .clone() + else { + debug_bypass("native toolchain fingerprint unavailable"); + return None; + }; + hash + } else { + "runtime-and-dependency-graph-v1".to_string() + }; + Some(BuildCacheKey::derive(&BuildKeyInputs { + source_integrity: package.source_integrity.clone(), + graph_key_digest: graph_key_digest.clone(), + dependency_closure_hash: invocation.dependency_closure_hash.clone(), + scripts, + platform: invocation.platform.clone(), + runtime: invocation.runtime.clone(), + sandbox: invocation.sandbox.clone(), + environment_hash: invocation.environment_hash.clone(), + toolchain_hash, + })) +} + +pub(super) fn read_v2_graph_key_digest(package_dir: &Path) -> Option { + let link_dir = package_dir.parent()?.parent()?; + lpm_store::v2::LinkMeta::read_from(link_dir) + .ok() + .map(|metadata| metadata.graph_key_digest_hex) +} + +fn detect_node_runtime(environment: &HashMap) -> Option { + let output = Command::new("node") + .arg("-p") + .arg("JSON.stringify({version:process.version,modules:process.versions.modules||'',napi:process.versions.napi||'',engine:process.versions.v8||'',execPath:process.execPath})") + .env_clear() + .envs(environment) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let value: serde_json::Value = serde_json::from_slice(&output.stdout).ok()?; + let executable = PathBuf::from(value.get("execPath")?.as_str()?); + Some(BuildRuntimeFingerprint { + runtime: "node".into(), + version: value + .get("version")? + .as_str()? + .trim_start_matches('v') + .into(), + modules_abi: value.get("modules")?.as_str()?.into(), + napi: value.get("napi")?.as_str()?.into(), + engine: value.get("engine")?.as_str()?.into(), + executable_hash: runtime_executable_identity(&executable).ok()?, + }) +} + +fn hash_lockfile_graph(lockfile: &lpm_lockfile::Lockfile) -> String { + let mut rows = lockfile + .packages + .iter() + .map(|package| { + format!( + "{}\0{}\0{}\0{}", + package.name, + package.version, + package.integrity.as_deref().unwrap_or(""), + package.source.as_deref().unwrap_or("") + ) + }) + .collect::>(); + rows.sort_unstable(); + hash_records(rows.iter().map(String::as_bytes)) +} + +fn hash_build_environment(environment: &HashMap) -> String { + const BUILD_ENVIRONMENT: &[&str] = &[ + "AR", + "CC", + "CFLAGS", + "CPPFLAGS", + "CXX", + "CXXFLAGS", + "LDFLAGS", + "MACOSX_DEPLOYMENT_TARGET", + "MAKEFLAGS", + "npm_config_arch", + "npm_config_build_from_source", + "npm_config_devdir", + "npm_config_libc", + "npm_config_nodedir", + "npm_config_node_gyp", + "npm_config_runtime", + "npm_config_target", + "npm_config_target_arch", + ]; + let mut records = Vec::with_capacity(BUILD_ENVIRONMENT.len()); + for name in BUILD_ENVIRONMENT { + let value = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map_or("", |(_, value)| value.as_str()); + records.push(format!("{name}\0{value}")); + } + hash_records(records.iter().map(String::as_bytes)) +} + +fn hash_toolchain(environment: &HashMap) -> std::io::Result { + const PROBES: &[(&str, &[&str])] = &[ + ("python3", &["--version"]), + ("make", &["--version"]), + ("cc", &["--version"]), + ("c++", &["--version"]), + ("clang", &["--version"]), + ("ld", &["-v"]), + ("xcrun", &["--show-sdk-path"]), + ("xcrun", &["--show-sdk-version"]), + ("rustc", &["-vV"]), + ("cargo", &["-V"]), + ]; + let mut hasher = Sha256::new(); + hasher.update(b"lpm-toolchain-v1\0"); + for (program, args) in PROBES { + hasher.update(program.as_bytes()); + hasher.update(b"\0"); + if let Ok(output) = Command::new(program) + .args(*args) + .env_clear() + .envs(environment) + .output() + { + hasher.update(&output.stdout[..output.stdout.len().min(16 * 1024)]); + hasher.update(&output.stderr[..output.stderr.len().min(16 * 1024)]); + } + hasher.update(b"\x1e"); + } + if let Some(home) = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("HOME")) + .map(|(_, value)| PathBuf::from(value)) + .or_else(dirs::home_dir) + { + hash_directory_tree(&home.join(".node-gyp"), &mut hasher)?; + } + Ok(format!("sha256-{}", hex::encode(hasher.finalize()))) +} + +fn hash_directory_tree(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hasher.update(b"directory-tree-v1\0"); + if !root.exists() { + hasher.update(b"absent"); + return Ok(()); + } + let mut pending = vec![root.to_path_buf()]; + let mut buffer = [0_u8; 64 * 1024]; + while let Some(directory) = pending.pop() { + let mut entries = std::fs::read_dir(&directory)?.collect::, _>>()?; + entries.sort_unstable_by_key(std::fs::DirEntry::file_name); + for entry in entries { + let path = entry.path(); + let relative = path.strip_prefix(root).unwrap_or(&path); + hash_os_path(hasher, relative); + let file_type = entry.file_type()?; + if file_type.is_dir() { + hasher.update(b"d"); + pending.push(path); + } else if file_type.is_file() { + hasher.update(b"f"); + let mut file = std::fs::File::open(path)?; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + } else if file_type.is_symlink() { + hasher.update(b"l"); + hash_os_path(hasher, &std::fs::read_link(path)?); + } else { + hasher.update(b"s"); + } + hasher.update(b"\x1e"); + } + } + Ok(()) +} + +fn hash_os_path(hasher: &mut Sha256, path: &Path) { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + let bytes = path.as_os_str().as_bytes(); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + } + #[cfg(not(unix))] + { + let value = path.to_string_lossy(); + hasher.update((value.len() as u64).to_le_bytes()); + hasher.update(value.as_bytes()); + } +} + +fn cpu_features_hash() -> String { + let mut features = Vec::new(); + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + for (name, enabled) in [ + ("sse2", std::is_x86_feature_detected!("sse2")), + ("sse4.2", std::is_x86_feature_detected!("sse4.2")), + ("avx", std::is_x86_feature_detected!("avx")), + ("avx2", std::is_x86_feature_detected!("avx2")), + ("fma", std::is_x86_feature_detected!("fma")), + ] { + if enabled { + features.push(name); + } + } + } + #[cfg(target_arch = "aarch64")] + { + features.push("aarch64"); + } + hash_records(features.iter().map(|feature| feature.as_bytes())) +} + +fn hash_allowed_inputs(capabilities: &CapabilitySet) -> String { + let records = [ + capabilities.canonical_hash(), + "package-local-writes".into(), + "graph-local-reads".into(), + "network-denied".into(), + ]; + hash_records(records.iter().map(String::as_bytes)) +} + +fn hash_records<'a>(records: impl IntoIterator) -> String { + let mut hasher = Sha256::new(); + for record in records { + hasher.update((record.len() as u64).to_le_bytes()); + hasher.update(record); + } + format!("sha256-{}", hex::encode(hasher.finalize())) +} + +fn runtime_executable_identity(path: &Path) -> std::io::Result { + let canonical = std::fs::canonicalize(path)?; + let metadata = std::fs::metadata(&canonical)?; + let mut hasher = Sha256::new(); + hasher.update(b"lpm-runtime-executable-v1\0"); + hash_os_path(&mut hasher, &canonical); + hasher.update(metadata.len().to_le_bytes()); + if let Ok(modified) = metadata.modified() + && let Ok(duration) = modified.duration_since(std::time::UNIX_EPOCH) + { + hasher.update(duration.as_secs().to_le_bytes()); + hasher.update(duration.subsec_nanos().to_le_bytes()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + hasher.update(metadata.dev().to_le_bytes()); + hasher.update(metadata.ino().to_le_bytes()); + hasher.update(metadata.ctime().to_le_bytes()); + hasher.update(metadata.ctime_nsec().to_le_bytes()); + } + Ok(format!("sha256-{}", hex::encode(hasher.finalize()))) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn build_environment_hash_ignores_unrelated_values() { + let mut first = HashMap::new(); + first.insert("UNRELATED".into(), "one".into()); + let mut second = HashMap::new(); + second.insert("UNRELATED".into(), "two".into()); + assert_eq!( + hash_build_environment(&first), + hash_build_environment(&second) + ); + } + + #[test] + fn build_environment_hash_changes_with_compiler_flags() { + let mut first = HashMap::new(); + first.insert("CFLAGS".into(), "-O2".into()); + let mut second = HashMap::new(); + second.insert("CFLAGS".into(), "-O3".into()); + assert_ne!( + hash_build_environment(&first), + hash_build_environment(&second) + ); + } + + #[test] + fn build_marker_reader_accepts_only_cache_key_shape() { + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join(".lpm-built"); + let key = "a".repeat(64); + std::fs::write(&marker, &key).unwrap(); + + assert_eq!(read_build_marker_key(&marker), Some(key)); + } + + #[test] + fn build_marker_reader_preserves_legacy_and_corrupt_markers_as_unkeyed() { + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join(".lpm-built"); + std::fs::write(&marker, b"").unwrap(); + assert!(read_build_marker_key(&marker).is_none()); + + std::fs::write(&marker, "G".repeat(64)).unwrap(); + assert!(read_build_marker_key(&marker).is_none()); + } + + #[test] + fn directory_tree_hash_changes_when_node_headers_change() { + let temp = tempfile::tempdir().unwrap(); + let headers = temp.path().join("include/node"); + std::fs::create_dir_all(&headers).unwrap(); + std::fs::write(headers.join("node.h"), b"first").unwrap(); + let mut first = Sha256::new(); + hash_directory_tree(temp.path(), &mut first).unwrap(); + + std::fs::write(headers.join("node.h"), b"second").unwrap(); + let mut second = Sha256::new(); + hash_directory_tree(temp.path(), &mut second).unwrap(); + + assert_ne!(first.finalize(), second.finalize()); + } +} diff --git a/crates/lpm-cli/src/commands/rebuild/mod.rs b/crates/lpm-cli/src/commands/rebuild/mod.rs index 03252d65..ad4d3be7 100644 --- a/crates/lpm-cli/src/commands/rebuild/mod.rs +++ b/crates/lpm-cli/src/commands/rebuild/mod.rs @@ -29,6 +29,7 @@ //! entire group (not just the direct child), preventing orphaned subprocesses //! - On Windows: `Child::kill()` terminates the process tree via `TerminateProcess` +mod build_cache; mod hints; mod package_dir; mod process_tree; @@ -40,6 +41,10 @@ mod trust; #[cfg(test)] mod tests; +use self::build_cache::{ + BuildCacheInvocation, BuildCacheScratch, build_key_for_package, is_cacheable_native_build, + marker_requires_key_validation, read_build_marker_key, read_v2_graph_key_digest, +}; #[cfg(test)] pub(crate) use self::hints::scriptable_package_rows; pub use self::hints::{all_scripted_packages_trusted, show_install_build_hint}; @@ -47,7 +52,7 @@ use self::package_dir::prepare_live_package_dir; use self::sandbox_env::build_sanitized_env; use self::script_execution::execute_script; use self::scripts::{ - BUILD_MARKER, ScriptablePackage, count_untrusted_unbuilt, package_baseline_dir_indexed, + BUILD_MARKER, BuildCacheMetrics, ScriptablePackage, count_untrusted_unbuilt, read_lifecycle_scripts, rebuild_dry_run_envelope, rebuild_package_failure_message, rebuild_package_label, rebuild_summary_envelope, scripts_word, toposort_packages, warn_stale_trusted_deps, widen_to_build_by_policy, @@ -61,13 +66,17 @@ use lpm_common::LpmError; use lpm_common::color::Painted; use lpm_sandbox::SandboxMode; use lpm_security::{EXECUTED_INSTALL_PHASES, SecurityPolicy}; -use lpm_store::V2BaselineIndex; +use lpm_store::{PackageBaselineLayout, V2BaselineIndex}; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::time::Duration; const DEFAULT_SCRIPT_TIMEOUT_SECS: u64 = 300; +fn elapsed_millis(duration: Duration) -> u64 { + duration.as_millis().min(u128::from(u64::MAX)) as u64 +} + #[allow(clippy::too_many_arguments)] pub async fn run( project_dir: &Path, @@ -287,11 +296,22 @@ async fn run_under_store_lock( // installs); silent skip preserves the previously // `pkg_json_path.exists()` semantic for non-store sources // while fixing the v2-installed-and-skipped data-loss bug. - let pkg_dir = - match package_baseline_dir_indexed(&baseline_index, &lpm_root, &lp.name, &lp.version) { - Some(p) => p, - None => continue, - }; + let baseline = match lpm_store::find_installed_package_baseline_indexed( + &baseline_index, + &lpm_root, + &lp.name, + &lp.version, + ) { + Some(baseline) => baseline, + None => continue, + }; + let graph_key_digest = if baseline.layout == PackageBaselineLayout::V2 { + read_v2_graph_key_digest(&baseline.package_dir) + } else { + None + }; + let pkg_dir = baseline.package_dir; + let pristine_path = baseline.pristine_dir; let pkg_json_path = pkg_dir.join("package.json"); if !pkg_json_path.exists() { @@ -303,7 +323,11 @@ async fn run_under_store_lock( _ => continue, }; - let is_built = pkg_dir.join(BUILD_MARKER).exists(); + let marker_path = pkg_dir.join(BUILD_MARKER); + let is_built = marker_path.exists(); + let build_marker_key = is_built + .then(|| read_build_marker_key(&marker_path)) + .flatten(); // trust decision // now flows through the shared [`evaluate_trust`] helper so @@ -373,8 +397,12 @@ async fn run_under_store_lock( integrity: lp.integrity.clone(), wrapper_id, store_path: pkg_dir, + pristine_path, + source_integrity: baseline.integrity, + graph_key_digest, scripts, is_built, + build_marker_key, is_trusted, trust_reason, }); @@ -407,7 +435,7 @@ async fn run_under_store_lock( let result = if dry_run { rebuild_dry_run_envelope(&[], force_security_floor) } else { - rebuild_summary_envelope(0, 0, force_security_floor) + rebuild_summary_envelope(0, 0, force_security_floor, &BuildCacheMetrics::default()) }; println!("{}", serde_json::to_string_pretty(&result).unwrap()); } else { @@ -470,7 +498,7 @@ async fn run_under_store_lock( } else { selected_for_policy .into_iter() - .filter(|p| !p.is_built) + .filter(|p| !p.is_built || (!dry_run && marker_requires_key_validation(p))) .collect() }; @@ -482,7 +510,7 @@ async fn run_under_store_lock( let result = if dry_run { rebuild_dry_run_envelope(&to_build, force_security_floor) } else { - rebuild_summary_envelope(0, 0, force_security_floor) + rebuild_summary_envelope(0, 0, force_security_floor, &BuildCacheMetrics::default()) }; println!("{}", serde_json::to_string_pretty(&result).unwrap()); } else { @@ -706,6 +734,7 @@ async fn run_under_store_lock( let mut successes = 0usize; let mut failures = 0usize; let mut completed_scripts = 0usize; + let mut build_cache_metrics = BuildCacheMetrics::default(); let mut built_packages = Vec::with_capacity(to_build.len()); let package_label_width = to_build .iter() @@ -902,6 +931,8 @@ async fn run_under_store_lock( lpm_sandbox::prepare_writable_dirs(&prepare_spec) .map_err(|e| LpmError::Registry(format!("{e}")))?; + let mut effective_sandbox_posture = lpm_sandbox::SandboxPosture::Disabled; + // `sandbox_options` already carries `allow-degraded` and // `deny_outbound_network` from the resolver call above. Reusing // it here preserves the resolved mode for both the pre-probe and @@ -941,6 +972,7 @@ async fn run_under_store_lock( sandbox_options.clone(), ) .map_err(|e| LpmError::Registry(format!("sandbox unavailable: {e}")))?; + effective_sandbox_posture = probe_sandbox.posture(); // per-install warning: emitted once when the // probe's effective posture is `Degraded`. The structured // line names kernel + active ABI + missing dimension so log @@ -973,6 +1005,19 @@ async fn run_under_store_lock( drop(probe_sandbox); } + let cache_preparation_start = std::time::Instant::now(); + let build_cache_invocation = BuildCacheInvocation::prepare( + &lockfile, + &sanitized_env, + sandbox_mode, + &effective_sandbox_posture, + &sandbox_options, + &extra_write_dirs, + &extra_secret_read_allow, + &requested_capabilities, + ); + build_cache_metrics.preparation_ms = elapsed_millis(cache_preparation_start.elapsed()); + // Banners fire AFTER the probe succeeds. On Linux + LogOnly the // probe above bailed with ModeNotSupportedOnPlatform, so this // banner's "logged but NOT enforced" promise never reaches a @@ -1079,6 +1124,175 @@ async fn run_under_store_lock( } }; + let key_start = std::time::Instant::now(); + let mut build_key = build_cache_invocation + .as_ref() + .and_then(|invocation| build_key_for_package(invocation, pkg)); + build_cache_metrics.key_ms += elapsed_millis(key_start.elapsed()); + if build_key.is_some() { + build_cache_metrics.eligible += 1; + } else if is_cacheable_native_build(pkg) { + build_cache_metrics.bypassed += 1; + } + let _build_key_lock = if let Some(key) = build_key.as_ref() { + let store = lpm_store::v2::Store::from_lpm_root(&lpm_root); + match lpm_common::acquire_exclusive_lock(store.paths().build_lock_path(key)) { + Ok(lock) => Some(lock), + Err(error) => { + tracing::warn!( + "build cache bypassed for {}@{} because the per-key lock failed: {error}", + pkg.name, + pkg.version + ); + build_cache_metrics.bypassed += 1; + build_key = None; + None + } + } + } else { + None + }; + if force { + let marker_path = pkg.store_path.join(BUILD_MARKER); + if let Err(error) = std::fs::remove_file(&marker_path) + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!( + "failed to clear build marker for forced rebuild of {}: {error}", + pkg.name + ); + } + } else if let Some(marker_key) = pkg.build_marker_key.as_deref() { + match build_key.as_ref() { + Some(key) if marker_key == key.as_str() => { + build_cache_metrics.local_state_hits += 1; + continue; + } + None => continue, + Some(_) => { + if let Err(error) = std::fs::remove_file(pkg.store_path.join(BUILD_MARKER)) + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!( + "failed to clear stale build marker for {}: {error}", + pkg.name + ); + } + } + } + } + if !force && let Some(key) = build_key.as_ref() { + let store = lpm_store::v2::Store::from_lpm_root(&lpm_root); + let lookup_start = std::time::Instant::now(); + match store.reusable_build_artifact(key) { + Ok(Some(artifact)) => { + build_cache_metrics.lookup_ms += elapsed_millis(lookup_start.elapsed()); + let restore_start = std::time::Instant::now(); + match store.restore_build_artifact(&artifact, &pkg.store_path) { + Ok(()) => { + build_cache_metrics.restore_ms += + elapsed_millis(restore_start.elapsed()); + build_cache_metrics.hits += 1; + build_cache_metrics.scripts_avoided += pkg.scripts.len(); + build_cache_metrics.restored_bytes += artifact.manifest.unpacked_bytes; + build_cache_metrics.lifecycle_ms_avoided += + artifact.manifest.lifecycle_duration_ms; + if let Err(error) = std::fs::write( + pkg.store_path.join(BUILD_MARKER), + key.as_str().as_bytes(), + ) { + tracing::warn!( + "failed to write restored build marker for {}: {error}", + pkg.name + ); + } + if !json_output { + let label = rebuild_package_label(pkg); + install_ui::detail(&format!( + " {} {label: { + build_cache_metrics.restore_ms += + elapsed_millis(restore_start.elapsed()); + build_cache_metrics.misses += 1; + tracing::warn!( + "failed to restore build cache entry for {}@{}: {error}", + pkg.name, + pkg.version + ); + } + } + } + Ok(None) => { + build_cache_metrics.lookup_ms += elapsed_millis(lookup_start.elapsed()); + build_cache_metrics.misses += 1; + } + Err(error) => { + build_cache_metrics.lookup_ms += elapsed_millis(lookup_start.elapsed()); + build_cache_metrics.misses += 1; + tracing::warn!( + "failed to inspect build cache entry for {}@{}: {error}", + pkg.name, + pkg.version + ); + } + } + } + + if build_key.is_some() { + let store = lpm_store::v2::Store::from_lpm_root(&lpm_root); + let rematerialize_start = std::time::Instant::now(); + if let Err(error) = store.restore_pristine_package(&pkg.pristine_path, &pkg.store_path) + { + build_cache_metrics.rematerialize_ms += + elapsed_millis(rematerialize_start.elapsed()); + if !json_output { + let label = rebuild_package_label(pkg); + install_ui::detail(&format!( + " {} {label: Some(scratch), + Err(error) => { + tracing::warn!( + "build cache bypassed for {}@{} because isolated scratch setup failed: {error}", + pkg.name, + pkg.version + ); + build_key = None; + None + } + } + } else { + None + }; + let script_tmpdir = build_cache_scratch + .as_ref() + .map_or(tmpdir.as_path(), BuildCacheScratch::path); + let mut package_sandbox_options = sandbox_options.clone(); + package_sandbox_options.build_cache_isolation = build_key.is_some(); + + let lifecycle_start = std::time::Instant::now(); for phase in EXECUTED_INSTALL_PHASES { let cmd = match pkg.scripts.get(*phase) { Some(c) => c, @@ -1094,12 +1308,12 @@ async fn run_under_store_lock( &sanitized_env, &timeout, sandbox_mode, - &sandbox_options, + &package_sandbox_options, &extra_write_dirs, &extra_secret_read_allow, &store_root, &home_dir, - &tmpdir, + script_tmpdir, ) { Ok(()) => { if !json_output { @@ -1128,9 +1342,29 @@ async fn run_under_store_lock( } if pkg_success { - // Write .lpm-built marker + if let Some(key) = build_key.as_ref() { + let store = lpm_store::v2::Store::from_lpm_root(&lpm_root); + let publish_start = std::time::Instant::now(); + if let Err(error) = store.publish_build_artifact( + key, + &pkg.source_integrity, + &pkg.store_path, + elapsed_millis(lifecycle_start.elapsed()), + force, + ) { + tracing::warn!( + "failed to publish build cache entry for {}@{}: {error}", + pkg.name, + pkg.version + ); + } + build_cache_metrics.publish_ms += elapsed_millis(publish_start.elapsed()); + } let marker_path = pkg.store_path.join(BUILD_MARKER); - if let Err(e) = std::fs::write(&marker_path, "") { + let marker = build_key + .as_ref() + .map_or(&[][..], |key| key.as_str().as_bytes()); + if let Err(e) = std::fs::write(&marker_path, marker) { tracing::warn!("failed to write build marker for {}: {e}", pkg.name); } successes += 1; @@ -1142,7 +1376,12 @@ async fn run_under_store_lock( // Summary if json_output { - let json = rebuild_summary_envelope(successes, failures, force_security_floor); + let json = rebuild_summary_envelope( + successes, + failures, + force_security_floor, + &build_cache_metrics, + ); println!("{}", serde_json::to_string_pretty(&json).unwrap()); } else if failures == 0 { eprintln!(); diff --git a/crates/lpm-cli/src/commands/rebuild/script_execution.rs b/crates/lpm-cli/src/commands/rebuild/script_execution.rs index f617c0be..8bf750b4 100644 --- a/crates/lpm-cli/src/commands/rebuild/script_execution.rs +++ b/crates/lpm-cli/src/commands/rebuild/script_execution.rs @@ -55,11 +55,21 @@ pub(super) fn execute_script( let path_value = build_lifecycle_path(project_dir, parent_path); let mut envs: Vec<(String, String)> = env .iter() - .filter(|(k, _)| !k.eq_ignore_ascii_case("PATH") && !k.eq_ignore_ascii_case("INIT_CWD")) + .filter(|(k, _)| { + !k.eq_ignore_ascii_case("PATH") + && !k.eq_ignore_ascii_case("INIT_CWD") + && !k.eq_ignore_ascii_case("TMPDIR") + && !k.eq_ignore_ascii_case("TMP") + && !k.eq_ignore_ascii_case("TEMP") + }) .map(|(k, v)| (k.clone(), v.clone())) .collect(); envs.push(("INIT_CWD".to_string(), project_dir.display().to_string())); envs.push(("PATH".to_string(), path_value)); + let tmp = tmpdir.display().to_string(); + envs.push(("TMPDIR".to_string(), tmp.clone())); + envs.push(("TMP".to_string(), tmp.clone())); + envs.push(("TEMP".to_string(), tmp)); let start = std::time::Instant::now(); diff --git a/crates/lpm-cli/src/commands/rebuild/scripts.rs b/crates/lpm-cli/src/commands/rebuild/scripts.rs index a4cd426b..0210bec9 100644 --- a/crates/lpm-cli/src/commands/rebuild/scripts.rs +++ b/crates/lpm-cli/src/commands/rebuild/scripts.rs @@ -65,8 +65,12 @@ pub(super) struct ScriptablePackage { /// Now the lookup is correct for every source kind. pub(super) wrapper_id: Option, pub(super) store_path: std::path::PathBuf, + pub(super) pristine_path: std::path::PathBuf, + pub(super) source_integrity: String, + pub(super) graph_key_digest: Option, pub(super) scripts: HashMap, pub(super) is_built: bool, + pub(super) build_marker_key: Option, pub(super) is_trusted: bool, /// the specific basis on which /// `is_trusted` was decided. Preserved so the dry-run output and @@ -79,6 +83,47 @@ pub(super) struct ScriptablePackage { pub(super) trust_reason: TrustReason, } +#[derive(Debug, Clone, Default)] +pub(super) struct BuildCacheMetrics { + pub(super) eligible: usize, + pub(super) hits: usize, + pub(super) misses: usize, + pub(super) bypassed: usize, + pub(super) local_state_hits: usize, + pub(super) scripts_avoided: usize, + pub(super) restored_bytes: u64, + pub(super) lifecycle_ms_avoided: u64, + pub(super) preparation_ms: u64, + pub(super) key_ms: u64, + pub(super) lookup_ms: u64, + pub(super) restore_ms: u64, + pub(super) rematerialize_ms: u64, + pub(super) publish_ms: u64, +} + +impl BuildCacheMetrics { + fn to_json(&self) -> serde_json::Value { + serde_json::json!({ + "eligible": self.eligible, + "hits": self.hits, + "misses": self.misses, + "bypassed": self.bypassed, + "local_state_hits": self.local_state_hits, + "scripts_avoided": self.scripts_avoided, + "restored_bytes": self.restored_bytes, + "lifecycle_ms_avoided": self.lifecycle_ms_avoided, + "timings_ms": { + "preparation": self.preparation_ms, + "key": self.key_ms, + "lookup": self.lookup_ms, + "restore": self.restore_ms, + "rematerialize": self.rematerialize_ms, + "publish": self.publish_ms, + } + }) + } +} + pub(crate) type RebuildPackageIdentity = (String, String, Option); #[derive(Debug, Clone, Default)] @@ -120,11 +165,13 @@ pub(super) fn rebuild_summary_envelope( successes: usize, failures: usize, force_security_floor: bool, + build_cache: &BuildCacheMetrics, ) -> serde_json::Value { let mut json = serde_json::json!({ "success": failures == 0, "built": successes, "failed": failures, + "build_cache": build_cache.to_json(), }); crate::security_floor::attach_security_posture(&mut json, force_security_floor); json diff --git a/crates/lpm-cli/src/commands/rebuild/tests/mod.rs b/crates/lpm-cli/src/commands/rebuild/tests/mod.rs index d103b302..8e1d551a 100644 --- a/crates/lpm-cli/src/commands/rebuild/tests/mod.rs +++ b/crates/lpm-cli/src/commands/rebuild/tests/mod.rs @@ -646,8 +646,12 @@ fn toposort_respects_dependency_order() { integrity: None, wrapper_id: None, store_path: PathBuf::new(), + pristine_path: PathBuf::new(), + source_integrity: "sha512-unused".into(), + graph_key_digest: None, scripts: HashMap::new(), is_built: false, + build_marker_key: None, is_trusted: true, trust_reason: TrustReason::StrictBinding, }, @@ -657,8 +661,12 @@ fn toposort_respects_dependency_order() { integrity: None, wrapper_id: None, store_path: PathBuf::new(), + pristine_path: PathBuf::new(), + source_integrity: "sha512-unused".into(), + graph_key_digest: None, scripts: HashMap::new(), is_built: false, + build_marker_key: None, is_trusted: true, trust_reason: TrustReason::StrictBinding, }, @@ -1102,8 +1110,12 @@ fn stale_detection_finds_packages_without_scripts() { integrity: None, wrapper_id: None, store_path: std::path::PathBuf::new(), + pristine_path: std::path::PathBuf::new(), + source_integrity: "sha512-unused".into(), + graph_key_digest: None, scripts: HashMap::from([("postinstall".into(), "node setup".into())]), is_built: false, + build_marker_key: None, is_trusted: true, trust_reason: TrustReason::StrictBinding, }, @@ -1113,8 +1125,12 @@ fn stale_detection_finds_packages_without_scripts() { integrity: None, wrapper_id: None, store_path: std::path::PathBuf::new(), + pristine_path: std::path::PathBuf::new(), + source_integrity: "sha512-unused".into(), + graph_key_digest: None, scripts: HashMap::from([("postinstall".into(), "node install.js".into())]), is_built: false, + build_marker_key: None, is_trusted: true, trust_reason: TrustReason::StrictBinding, }, @@ -1383,8 +1399,12 @@ fn synthetic_scriptable(name: &str, is_built: bool, is_trusted: bool) -> Scripta integrity: None, wrapper_id: None, store_path: std::path::PathBuf::from("/unused"), + pristine_path: std::path::PathBuf::from("/unused"), + source_integrity: "sha512-unused".into(), + graph_key_digest: None, scripts: HashMap::from([("postinstall".into(), "node x.js".into())]), is_built, + build_marker_key: None, is_trusted, trust_reason: if is_trusted { TrustReason::StrictBinding diff --git a/crates/lpm-cli/src/sandbox_config.rs b/crates/lpm-cli/src/sandbox_config.rs index 6c6974b9..a32dfc18 100644 --- a/crates/lpm-cli/src/sandbox_config.rs +++ b/crates/lpm-cli/src/sandbox_config.rs @@ -469,6 +469,7 @@ fn merge(project: RawSandboxKeys, global: RawSandboxKeys) -> (SandboxOptions, Re .or(global.allow_degraded) .unwrap_or(false), deny_outbound_network, + build_cache_isolation: false, }; (options, resolved) } diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index 05f56091..bd70bf5e 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -25,9 +25,9 @@ pub use integrity::Integrity; pub use package_name::PackageName; pub use paths::{ ExclusiveLockHandle, FsKind, GLOBAL_INSTALL_PATH_BUDGET, INSTALL_READY_MARKER, LpmRoot, - SharedLockHandle, as_extended_path, check_install_path_budget, is_local_fs, - project_install_lock, try_with_exclusive_lock, with_exclusive_lock, with_exclusive_lock_async, - with_shared_lock, with_shared_lock_async, + SharedLockHandle, acquire_exclusive_lock, as_extended_path, check_install_path_budget, + is_local_fs, project_install_lock, try_with_exclusive_lock, with_exclusive_lock, + with_exclusive_lock_async, with_shared_lock, with_shared_lock_async, }; pub use provenance::{ProvenanceSnapshot, ProvenanceStatus}; pub use symlink::{create_dir_symlink_or_junction, create_symlink}; diff --git a/crates/lpm-common/src/paths.rs b/crates/lpm-common/src/paths.rs index 9f04e8a0..1c73495b 100644 --- a/crates/lpm-common/src/paths.rs +++ b/crates/lpm-common/src/paths.rs @@ -712,6 +712,17 @@ where body() } +/// Acquire and return an exclusive advisory lock handle. +/// +/// The caller controls the critical-section lifetime by retaining the returned +/// handle. This is useful when existing control flow contains early returns or +/// loop `continue` paths that would make a closure-scoped lock cumbersome. +pub fn acquire_exclusive_lock( + lock_path: impl AsRef, +) -> Result { + acquire_exclusive_with_hint(lock_path.as_ref(), default_wait_hint) +} + /// Async variant of [`with_exclusive_lock`]. The lock acquisition runs on /// a `tokio::task::spawn_blocking` worker so a contended lock doesn't /// block the tokio reactor. The acquired handle is held across `body`'s diff --git a/crates/lpm-sandbox/src/landlock_rules.rs b/crates/lpm-sandbox/src/landlock_rules.rs index e1d6eba7..1bee0ff9 100644 --- a/crates/lpm-sandbox/src/landlock_rules.rs +++ b/crates/lpm-sandbox/src/landlock_rules.rs @@ -95,6 +95,13 @@ pub(crate) const SYSTEM_READ_PATHS: &[&str] = &[ /// 2. Free of unsafe `PathFd` opens at rule-description time (those /// happen in the child's pre_exec hook in [`crate::linux`]). pub(crate) fn describe_rules(spec: &SandboxSpec) -> Vec<(PathBuf, RuleAccess)> { + describe_rules_with_isolation(spec, false) +} + +pub(crate) fn describe_rules_with_isolation( + spec: &SandboxSpec, + build_cache_isolation: bool, +) -> Vec<(PathBuf, RuleAccess)> { let mut rules = Vec::with_capacity(32 + spec.extra_write_dirs.len()); // Read-only system baseline. @@ -115,7 +122,11 @@ pub(crate) fn describe_rules(spec: &SandboxSpec) -> Vec<(PathBuf, RuleAccess)> { // widening to `store_root` would expose every other package // the user has installed. Two remediations exist for that corner: // switch to hardlink, or widen reads with an explicit store rule. - rules.push((spec.project_dir.clone(), RuleAccess::Read)); + if build_cache_isolation { + rules.push((package_dependency_root(spec), RuleAccess::Read)); + } else { + rules.push((spec.project_dir.clone(), RuleAccess::Read)); + } // NVM-installed toolchain. Only added if the host has a matching // dir — [`crate::linux::spawn`] filters missing paths at FD-open // time; the description layer stays complete. @@ -143,13 +154,17 @@ pub(crate) fn describe_rules(spec: &SandboxSpec) -> Vec<(PathBuf, RuleAccess)> { // would require either a per-tool allowlist or migrating those // tools to LPM-controlled cache directories. rules.push((spec.package_dir.clone(), RuleAccess::ReadWrite)); - rules.push((spec.project_dir.join("node_modules"), RuleAccess::ReadWrite)); - rules.push((spec.project_dir.join(".husky"), RuleAccess::ReadWrite)); - rules.push((spec.project_dir.join(".lpm"), RuleAccess::ReadWrite)); - rules.push((spec.home_dir.join(".cache"), RuleAccess::ReadWrite)); - rules.push((spec.home_dir.join(".node-gyp"), RuleAccess::ReadWrite)); - rules.push((spec.home_dir.join(".npm"), RuleAccess::ReadWrite)); - rules.push((PathBuf::from("/tmp"), RuleAccess::ReadWrite)); + if build_cache_isolation { + rules.push((spec.home_dir.join(".node-gyp"), RuleAccess::Read)); + } else { + rules.push((spec.project_dir.join("node_modules"), RuleAccess::ReadWrite)); + rules.push((spec.project_dir.join(".husky"), RuleAccess::ReadWrite)); + rules.push((spec.project_dir.join(".lpm"), RuleAccess::ReadWrite)); + rules.push((spec.home_dir.join(".cache"), RuleAccess::ReadWrite)); + rules.push((spec.home_dir.join(".node-gyp"), RuleAccess::ReadWrite)); + rules.push((spec.home_dir.join(".npm"), RuleAccess::ReadWrite)); + rules.push((PathBuf::from("/tmp"), RuleAccess::ReadWrite)); + } rules.push((spec.tmpdir.clone(), RuleAccess::ReadWrite)); // `/dev/null` and `/dev/tty` as writable — shells redirect to // them constantly. The broader `/dev` Read rule already covers @@ -161,13 +176,26 @@ pub(crate) fn describe_rules(spec: &SandboxSpec) -> Vec<(PathBuf, RuleAccess)> { // Per-project extras from `package.json > lpm > scripts > // sandboxWriteDirs`. Loader guarantees absolute paths. - for p in &spec.extra_write_dirs { - rules.push((p.clone(), RuleAccess::ReadWrite)); + if !build_cache_isolation { + for p in &spec.extra_write_dirs { + rules.push((p.clone(), RuleAccess::ReadWrite)); + } } rules } +fn package_dependency_root(spec: &SandboxSpec) -> PathBuf { + let mut root = spec.package_dir.as_path(); + while let Some(parent) = root.parent() { + if root.file_name().is_some_and(|name| name == "node_modules") { + return root.to_path_buf(); + } + root = parent; + } + spec.package_dir.clone() +} + #[cfg(test)] mod tests { use super::*; @@ -347,6 +375,29 @@ mod tests { assert_eq!(last.0.as_os_str(), "/home/u/.cache/ms-playwright"); } + #[test] + fn build_cache_isolation_keeps_persistent_writes_package_local() { + let s = spec(); + let rules = describe_rules_with_isolation(&s, true); + + assert!(contains_rule( + &rules, + "/lpm-store/prisma@5.22.0", + RuleAccess::ReadWrite + )); + assert!(contains_rule(&rules, "/tmp", RuleAccess::ReadWrite)); + assert!(!contains_rule( + &rules, + "/home/u/proj/node_modules", + RuleAccess::ReadWrite + )); + assert!(!contains_rule( + &rules, + "/home/u/.node-gyp", + RuleAccess::ReadWrite + )); + } + #[test] fn tmpdir_distinct_from_slash_tmp_gets_its_own_rule() { let mut s = spec(); diff --git a/crates/lpm-sandbox/src/lib.rs b/crates/lpm-sandbox/src/lib.rs index da3737bb..39a96b32 100644 --- a/crates/lpm-sandbox/src/lib.rs +++ b/crates/lpm-sandbox/src/lib.rs @@ -286,6 +286,12 @@ pub struct SandboxOptions { /// /// Participates in the sandbox CLI / config / env precedence chain. pub deny_outbound_network: bool, + + /// Restrict persistent writes to the package directory and the supplied + /// temporary directory. This is used only while producing a reusable + /// lifecycle-build artifact; normal lifecycle execution keeps the broader + /// compatibility write set. + pub build_cache_isolation: bool, } impl SandboxOptions { diff --git a/crates/lpm-sandbox/src/linux.rs b/crates/lpm-sandbox/src/linux.rs index 10a9f70c..d25b30aa 100644 --- a/crates/lpm-sandbox/src/linux.rs +++ b/crates/lpm-sandbox/src/linux.rs @@ -237,6 +237,7 @@ pub(crate) struct LandlockSandbox { spec: SandboxSpec, mode: SandboxMode, posture: BackendPosture, + build_cache_isolation: bool, } impl LandlockSandbox { @@ -245,6 +246,7 @@ impl LandlockSandbox { mode: SandboxMode, options: SandboxOptions, ) -> Result { + let build_cache_isolation = options.build_cache_isolation; match mode { SandboxMode::Enforce => { let posture = if options.deny_outbound_network { @@ -315,6 +317,7 @@ impl LandlockSandbox { spec, mode, posture, + build_cache_isolation, }) } // Landlock has no native observe-only primitive @@ -425,11 +428,11 @@ impl Sandbox for LandlockSandbox { // `landlock_add_rule` via Ruleset::add_rule) happens here in // normal multi-threaded context. The child's pre_exec body // only touches direct syscalls. - let ruleset = build_parent_side_ruleset(&self.spec, &self.posture).map_err(|e| { - SandboxError::ProfileRenderFailed { - reason: format!("landlock ruleset build failed: {e}"), - } - })?; + let ruleset = + build_parent_side_ruleset(&self.spec, &self.posture, self.build_cache_isolation) + .map_err(|e| SandboxError::ProfileRenderFailed { + reason: format!("landlock ruleset build failed: {e}"), + })?; // compile the seccomp socket(2) deny filter // parent-side. Like the landlock ruleset, all the @@ -752,6 +755,7 @@ fn probe_landlock_at(abi: ABI, with_network: bool) -> Result<(), RulesetError> { fn build_parent_side_ruleset( spec: &SandboxSpec, posture: &BackendPosture, + build_cache_isolation: bool, ) -> Result { let abi = posture.abi(); let rw = AccessFs::from_all(abi); @@ -767,7 +771,9 @@ fn build_parent_side_ruleset( builder = builder.handle_access(AccessNet::from_all(abi))?; } let mut ruleset = builder.create()?; - for (path, access) in describe_rules(spec) { + for (path, access) in + crate::landlock_rules::describe_rules_with_isolation(spec, build_cache_isolation) + { let fd = match PathFd::new(&path) { Ok(fd) => fd, Err(e) => { @@ -976,6 +982,7 @@ mod tests { let options = SandboxOptions { deny_outbound_network: true, allow_degraded: true, + build_cache_isolation: false, }; match LandlockSandbox::new(realistic_spec(), SandboxMode::Enforce, options) { Ok(sb) => { @@ -1035,7 +1042,7 @@ mod tests { detected_kernel: "5.15.0-test".to_string(), }, ] { - match build_parent_side_ruleset(&spec, &posture) { + match build_parent_side_ruleset(&spec, &posture, false) { Ok(_) => {} // ruleset built, missing extra was skipped Err(e) => { // Only acceptable error: the kernel doesn't support diff --git a/crates/lpm-sandbox/src/macos.rs b/crates/lpm-sandbox/src/macos.rs index 98e09f16..2424d3a3 100644 --- a/crates/lpm-sandbox/src/macos.rs +++ b/crates/lpm-sandbox/src/macos.rs @@ -49,7 +49,11 @@ impl SeatbeltSandbox { ) -> Result { let deny_outbound_network = options.deny_outbound_network; let profile = match mode { - SandboxMode::Enforce => seatbelt::render_profile(&spec, deny_outbound_network)?, + SandboxMode::Enforce => seatbelt::render_profile_with_isolation( + &spec, + deny_outbound_network, + options.build_cache_isolation, + )?, SandboxMode::LogOnly => seatbelt::render_logonly_profile(&spec, deny_outbound_network)?, // Disabled never reaches this backend — the factory in // [`crate::new_for_platform`] short-circuits to diff --git a/crates/lpm-sandbox/src/seatbelt.rs b/crates/lpm-sandbox/src/seatbelt.rs index 7fcdd2eb..80635592 100644 --- a/crates/lpm-sandbox/src/seatbelt.rs +++ b/crates/lpm-sandbox/src/seatbelt.rs @@ -29,6 +29,14 @@ use std::path::{Path, PathBuf}; pub(crate) fn render_profile( spec: &SandboxSpec, deny_outbound_network: bool, +) -> Result { + render_profile_with_isolation(spec, deny_outbound_network, false) +} + +pub(crate) fn render_profile_with_isolation( + spec: &SandboxSpec, + deny_outbound_network: bool, + build_cache_isolation: bool, ) -> Result { // Canonicalize base paths so Seatbelt rules match against the // same form the kernel uses at enforcement time. macOS symlinks @@ -124,7 +132,16 @@ pub(crate) fn render_profile( // deny-default profile. out.push_str(" (literal \"/\")\n"); out.push_str(&format!(" (subpath {package_dir})\n")); - out.push_str(&format!(" (subpath {project_dir})\n")); + if build_cache_isolation { + let dependency_root = quoted_path( + &package_dependency_root(spec, &canon_package_dir), + "package dependency root", + )?; + out.push_str(&format!(" (subpath {dependency_root})\n")); + out.push_str(&format!(" (subpath {home_node_gyp})\n")); + } else { + out.push_str(&format!(" (subpath {project_dir})\n")); + } out.push_str(" (subpath \"/usr\")\n"); out.push_str(" (subpath \"/bin\")\n"); out.push_str(" (subpath \"/sbin\")\n"); @@ -172,18 +189,22 @@ pub(crate) fn render_profile( // form. `/dev/null` is writable so `>/dev/null` redirects work. out.push_str("(allow file-write*\n"); out.push_str(&format!(" (subpath {package_dir})\n")); - out.push_str(&format!(" (subpath {project_node_modules})\n")); - out.push_str(&format!(" (subpath {project_husky})\n")); - out.push_str(&format!(" (subpath {project_lpm})\n")); - out.push_str(&format!(" (subpath {home_cache})\n")); - out.push_str(&format!(" (subpath {home_node_gyp})\n")); - out.push_str(&format!(" (subpath {home_npm})\n")); - out.push_str(" (subpath \"/tmp\")\n"); + if !build_cache_isolation { + out.push_str(&format!(" (subpath {project_node_modules})\n")); + out.push_str(&format!(" (subpath {project_husky})\n")); + out.push_str(&format!(" (subpath {project_lpm})\n")); + out.push_str(&format!(" (subpath {home_cache})\n")); + out.push_str(&format!(" (subpath {home_node_gyp})\n")); + out.push_str(&format!(" (subpath {home_npm})\n")); + out.push_str(" (subpath \"/tmp\")\n"); + } out.push_str(&format!(" (subpath {tmpdir})\n")); out.push_str(" (literal \"/dev/null\")\n"); out.push_str(" (literal \"/dev/tty\")\n"); - for e in &extras { - out.push_str(&format!(" (subpath {e})\n")); + if !build_cache_isolation { + for e in &extras { + out.push_str(&format!(" (subpath {e})\n")); + } } out.push_str(")\n"); out.push('\n'); @@ -247,6 +268,19 @@ pub(crate) fn render_profile( Ok(out) } +fn package_dependency_root(spec: &SandboxSpec, canonical_package_dir: &Path) -> PathBuf { + let name_depth = spec + .package_name + .bytes() + .filter(|byte| *byte == b'/') + .count(); + canonical_package_dir + .ancestors() + .nth(name_depth + 1) + .unwrap_or(canonical_package_dir) + .to_path_buf() +} + /// Emit the `(deny file-read* ...)` block for well-known secret /// conventions. Driven by the shared catalog in /// [`crate::secret_paths`]; all entries are rendered project-rooted @@ -643,6 +677,25 @@ mod tests { ); } + #[test] + fn build_cache_profile_omits_project_and_home_cache_writes() { + let s = spec(); + let profile = render_profile_with_isolation(&s, true, true).unwrap(); + let write_section = profile + .split("(allow file-write*\n") + .nth(1) + .unwrap() + .split("\n)\n") + .next() + .unwrap(); + + assert!(write_section.contains("/lpm-store/prisma@5.22.0")); + assert!(write_section.contains("/var/folders/xx/T")); + assert!(!write_section.contains(".husky")); + assert!(!write_section.contains(".node-gyp")); + assert!(!write_section.contains("node_modules")); + } + #[test] fn profile_allows_process_and_signal_primitives() { let p = render_profile(&spec(), false).unwrap(); diff --git a/crates/lpm-sandbox/src/windows.rs b/crates/lpm-sandbox/src/windows.rs index 92b1f7b6..df519b70 100644 --- a/crates/lpm-sandbox/src/windows.rs +++ b/crates/lpm-sandbox/src/windows.rs @@ -1680,6 +1680,7 @@ mod tests { let opts = SandboxOptions { deny_outbound_network: true, allow_degraded: false, + build_cache_isolation: false, }; match WindowsSandbox::new(realistic_spec(), SandboxMode::Enforce, opts) { Err(SandboxError::UnsupportedPlatform { platform, .. }) => { @@ -1694,6 +1695,7 @@ mod tests { let opts = SandboxOptions { deny_outbound_network: true, allow_degraded: true, + build_cache_isolation: false, }; let sb = WindowsSandbox::new(realistic_spec(), SandboxMode::Enforce, opts) .expect("strict + allow_degraded must construct cleanly"); diff --git a/crates/lpm-sandbox/tests/seatbelt_secret_deny_runtime.rs b/crates/lpm-sandbox/tests/seatbelt_secret_deny_runtime.rs index 50712add..761e9367 100644 --- a/crates/lpm-sandbox/tests/seatbelt_secret_deny_runtime.rs +++ b/crates/lpm-sandbox/tests/seatbelt_secret_deny_runtime.rs @@ -53,6 +53,7 @@ fn cat_through_sandbox(project_dir: &Path, path_to_cat: &Path) -> (i32, Vec, let options = SandboxOptions { allow_degraded: false, deny_outbound_network: false, + build_cache_isolation: false, }; let sandbox = new_for_platform_with_options(spec, SandboxMode::Enforce, options) .expect("sandbox construction"); @@ -227,6 +228,7 @@ fn seatbelt_secret_read_allow_restores_dotenv_at_runtime() { let options = SandboxOptions { allow_degraded: false, deny_outbound_network: false, + build_cache_isolation: false, }; let sandbox = new_for_platform_with_options(spec, SandboxMode::Enforce, options) .expect("sandbox construction"); diff --git a/crates/lpm-sandbox/tests/seccomp_socket_deny.rs b/crates/lpm-sandbox/tests/seccomp_socket_deny.rs index 8d319287..899a7c6a 100644 --- a/crates/lpm-sandbox/tests/seccomp_socket_deny.rs +++ b/crates/lpm-sandbox/tests/seccomp_socket_deny.rs @@ -125,6 +125,7 @@ fn denies_udp_socket_under_strict() { let options = SandboxOptions { deny_outbound_network: true, allow_degraded: false, + build_cache_isolation: false, }; let sb = match new_for_platform_with_options(realistic_spec(), SandboxMode::Enforce, options) { Ok(sb) => sb, diff --git a/crates/lpm-sandbox/tests/secret_overlay_bind_mount.rs b/crates/lpm-sandbox/tests/secret_overlay_bind_mount.rs index e31cda6b..9f8fd30b 100644 --- a/crates/lpm-sandbox/tests/secret_overlay_bind_mount.rs +++ b/crates/lpm-sandbox/tests/secret_overlay_bind_mount.rs @@ -188,6 +188,7 @@ fn default_mode_bind_mounts_dotenv_to_empty() { let options = SandboxOptions { allow_degraded: false, deny_outbound_network: false, + build_cache_isolation: false, }; let sandbox = new_for_platform_with_options(spec, SandboxMode::Enforce, options) .expect("sandbox construction"); @@ -230,6 +231,7 @@ fn default_mode_does_not_overlay_source_files() { let options = SandboxOptions { allow_degraded: false, deny_outbound_network: false, + build_cache_isolation: false, }; let sandbox = new_for_platform_with_options(spec, SandboxMode::Enforce, options) .expect("sandbox construction"); diff --git a/crates/lpm-store/src/v2/build_cache.rs b/crates/lpm-store/src/v2/build_cache.rs new file mode 100644 index 00000000..d81cbfa4 --- /dev/null +++ b/crates/lpm-store/src/v2/build_cache.rs @@ -0,0 +1,720 @@ +//! Content-addressed artifacts produced by dependency lifecycle builds. + +use std::path::{Component, Path, PathBuf}; + +use chrono::{DateTime, Utc}; +use lpm_common::LpmError; +use serde::{Deserialize, Serialize}; + +use super::fs_util::{ + create_tmp_dir_locked, ensure_store_tier_dir_locked, materialize_into, materialize_into_inner, + tmp_sibling, +}; +use super::store::Store; +use super::tree_hash::{ObjectTreeStats, compute_object_tree_integrities}; + +const BUILD_KEY_SCHEMA: &[u8] = b"lpm-build-key-v1\0"; +const BUILD_ARTIFACT_SCHEMA: u32 = 1; +const BUILD_PACKAGE_DIR: &str = "package"; +const BUILD_MANIFEST_FILE: &str = "manifest.json"; +/// Sentinel written inside every atomically published build artifact. +pub const BUILD_ARTIFACT_COMPLETE_FILENAME: &str = ".lpm-build-complete"; +const BUILD_MANIFEST_MAX_BYTES: u64 = 256 * 1024; + +/// One lifecycle phase and its exact command body. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildScriptFingerprint { + /// Lifecycle phase name, normally `preinstall`, `install`, or `postinstall`. + pub phase: String, + /// Exact command passed to the lifecycle shell. + pub command: String, +} + +/// Host compatibility dimensions that can affect native output. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildPlatformFingerprint { + /// npm-compatible operating-system name. + pub os: String, + /// npm-compatible CPU architecture name. + pub architecture: String, + /// Linux libc family, or an empty string on other systems. + pub libc: String, + /// Canonical digest of the host's compatible CPU feature set. + pub cpu_features_hash: String, +} + +/// Runtime identity observed from the executable used by lifecycle scripts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildRuntimeFingerprint { + /// Runtime family, initially `node`. + pub runtime: String, + /// Exact runtime version. + pub version: String, + /// Native-module ABI reported by the runtime. + pub modules_abi: String, + /// N-API version reported by the runtime. + pub napi: String, + /// Embedded JavaScript-engine version. + pub engine: String, + /// Digest identifying the resolved runtime executable. + pub executable_hash: String, +} + +/// Effective containment and allowed-input policy for one build. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildSandboxFingerprint { + /// Requested runtime mode. + pub mode: String, + /// Effective backend posture after platform degradation decisions. + pub posture: String, + /// Whether outbound network access was denied. + pub network_denied: bool, + /// Whether credential and ambient environment scrubbing was active. + pub environment_scrubbed: bool, + /// Canonical hash of readable and writable input policy. + pub allowed_inputs_hash: String, +} + +/// Complete deterministic input set for a dependency build. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildKeyInputs { + /// Integrity of the pristine source object. + pub source_integrity: String, + /// Full graph-key digest for the materialized dependency context. + pub graph_key_digest: String, + /// Digest of dependency sources and dependency build identities. + pub dependency_closure_hash: String, + /// Executed lifecycle phases in execution order. + pub scripts: Vec, + /// Host compatibility fingerprint. + pub platform: BuildPlatformFingerprint, + /// Runtime/ABI fingerprint. + pub runtime: BuildRuntimeFingerprint, + /// Effective sandbox fingerprint. + pub sandbox: BuildSandboxFingerprint, + /// Digest of explicitly permitted build-environment values. + pub environment_hash: String, + /// Digest of discoverable compiler, SDK, and build-tool identities. + pub toolchain_hash: String, +} + +/// Filesystem-safe digest identifying interchangeable build output. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct BuildCacheKey(String); + +impl BuildCacheKey { + /// Derive a key from every input that can affect lifecycle output. + pub fn derive(inputs: &BuildKeyInputs) -> Self { + let mut hasher = blake3::Hasher::new(); + hasher.update(BUILD_KEY_SCHEMA); + hash_field(&mut hasher, b"source", inputs.source_integrity.as_bytes()); + hash_field(&mut hasher, b"graph", inputs.graph_key_digest.as_bytes()); + hash_field( + &mut hasher, + b"dependency_closure", + inputs.dependency_closure_hash.as_bytes(), + ); + for script in &inputs.scripts { + hash_field(&mut hasher, b"phase", script.phase.as_bytes()); + hash_field(&mut hasher, b"command", script.command.as_bytes()); + } + hash_field(&mut hasher, b"os", inputs.platform.os.as_bytes()); + hash_field( + &mut hasher, + b"architecture", + inputs.platform.architecture.as_bytes(), + ); + hash_field(&mut hasher, b"libc", inputs.platform.libc.as_bytes()); + hash_field( + &mut hasher, + b"cpu_features", + inputs.platform.cpu_features_hash.as_bytes(), + ); + hash_field(&mut hasher, b"runtime", inputs.runtime.runtime.as_bytes()); + hash_field( + &mut hasher, + b"runtime_version", + inputs.runtime.version.as_bytes(), + ); + hash_field( + &mut hasher, + b"modules_abi", + inputs.runtime.modules_abi.as_bytes(), + ); + hash_field(&mut hasher, b"napi", inputs.runtime.napi.as_bytes()); + hash_field(&mut hasher, b"engine", inputs.runtime.engine.as_bytes()); + hash_field( + &mut hasher, + b"runtime_executable", + inputs.runtime.executable_hash.as_bytes(), + ); + hash_field(&mut hasher, b"sandbox_mode", inputs.sandbox.mode.as_bytes()); + hash_field( + &mut hasher, + b"sandbox_posture", + inputs.sandbox.posture.as_bytes(), + ); + hash_bool( + &mut hasher, + b"network_denied", + inputs.sandbox.network_denied, + ); + hash_bool( + &mut hasher, + b"environment_scrubbed", + inputs.sandbox.environment_scrubbed, + ); + hash_field( + &mut hasher, + b"allowed_inputs", + inputs.sandbox.allowed_inputs_hash.as_bytes(), + ); + hash_field( + &mut hasher, + b"environment", + inputs.environment_hash.as_bytes(), + ); + hash_field(&mut hasher, b"toolchain", inputs.toolchain_hash.as_bytes()); + Self(hasher.finalize().to_hex().to_string()) + } + + /// Lowercase hexadecimal representation used as the directory name. + #[inline] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +fn hash_field(hasher: &mut blake3::Hasher, label: &[u8], value: &[u8]) { + hasher.update(label); + hasher.update(b"\0"); + hasher.update(&(value.len() as u64).to_le_bytes()); + hasher.update(value); +} + +fn hash_bool(hasher: &mut blake3::Hasher, label: &[u8], value: bool) { + hash_field(hasher, label, &[u8::from(value)]); +} + +/// Integrity and provenance metadata for a completed build artifact. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BuildArtifactManifest { + /// On-disk manifest schema. + pub schema: u32, + /// Cache key owning this artifact. + pub key: String, + /// Integrity of the pristine source object. + pub source_integrity: String, + /// SHA-256 tree digest of the complete built package. + pub output_integrity: String, + /// Number of files in the built package. + pub file_count: u64, + /// Total bytes in regular files. + pub unpacked_bytes: u64, + /// Time spent executing lifecycle commands when this artifact was created. + pub lifecycle_duration_ms: u64, + /// Artifact publication time. + pub created_at: DateTime, +} + +/// A validated build artifact ready to restore. +#[derive(Debug, Clone)] +pub struct BuildArtifact { + /// Content-addressed artifact directory. + pub directory: PathBuf, + /// Validated artifact manifest. + pub manifest: BuildArtifactManifest, +} + +impl BuildArtifact { + /// Directory containing the complete built package tree. + #[inline] + pub fn package_dir(&self) -> PathBuf { + self.directory.join(BUILD_PACKAGE_DIR) + } +} + +/// Result of publishing a successful lifecycle build. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BuildArtifactPublish { + /// This call published a new artifact. + Published, + /// An equivalent complete artifact already existed. + Existing, +} + +impl Store { + /// Return a validated local build artifact, or `None` on a miss or corruption. + pub fn reusable_build_artifact( + &self, + key: &BuildCacheKey, + ) -> Result, LpmError> { + let dir = self.paths().build_artifact_dir(key); + if !dir.is_dir() || !dir.join(BUILD_ARTIFACT_COMPLETE_FILENAME).is_file() { + return Ok(None); + } + let manifest = match read_manifest(&dir) { + Ok(manifest) if manifest.key == key.as_str() => manifest, + Ok(_) | Err(_) => return Ok(None), + }; + let package_dir = dir.join(BUILD_PACKAGE_DIR); + if !package_dir.is_dir() || !tree_has_safe_symlinks(&package_dir)? { + return Ok(None); + } + let actual = compute_object_tree_integrities(&package_dir)?; + if actual.content != manifest.output_integrity + || actual.stats.file_count != manifest.file_count + || actual.stats.unpacked_bytes != manifest.unpacked_bytes + { + return Ok(None); + } + if let Ok(file) = std::fs::File::options() + .write(true) + .open(dir.join(BUILD_ARTIFACT_COMPLETE_FILENAME)) + { + let _ = file.set_modified(std::time::SystemTime::now()); + } + Ok(Some(BuildArtifact { + directory: dir, + manifest, + })) + } + + /// Publish a successful package-local lifecycle build atomically. + pub fn publish_build_artifact( + &self, + key: &BuildCacheKey, + source_integrity: &str, + built_package_dir: &Path, + lifecycle_duration_ms: u64, + replace_existing: bool, + ) -> Result { + if !replace_existing && self.reusable_build_artifact(key)?.is_some() { + return Ok(BuildArtifactPublish::Existing); + } + if !tree_has_safe_symlinks(built_package_dir)? { + return Err(LpmError::Store(format!( + "refusing to cache lifecycle output at {} because it contains an escaping symlink", + built_package_dir.display() + ))); + } + ensure_store_tier_dir_locked(self.paths().builds_root()).map_err(|error| { + LpmError::Store(format!("failed to create v2 builds directory: {error}")) + })?; + + let final_dir = self.paths().build_artifact_dir(key); + if final_dir.exists() { + std::fs::remove_dir_all(&final_dir).map_err(|error| { + LpmError::Store(format!( + "failed to remove corrupt build artifact at {}: {error}", + final_dir.display() + )) + })?; + } + let tmp_dir = tmp_sibling(&final_dir); + create_tmp_dir_locked(&tmp_dir).map_err(|error| { + LpmError::Store(format!( + "failed to create build artifact staging directory at {}: {error}", + tmp_dir.display() + )) + })?; + + let staged_package = tmp_dir.join(BUILD_PACKAGE_DIR); + let publish_result = (|| { + materialize_into_inner(built_package_dir, built_package_dir, &staged_package, true)?; + let tree = compute_object_tree_integrities(&staged_package)?; + let manifest = manifest_from_tree( + key, + source_integrity, + &tree.stats, + tree.content, + lifecycle_duration_ms, + ); + let bytes = serde_json::to_vec_pretty(&manifest).map_err(|error| { + LpmError::Store(format!( + "failed to serialize build artifact manifest: {error}" + )) + })?; + std::fs::write(tmp_dir.join(BUILD_MANIFEST_FILE), bytes)?; + std::fs::write(tmp_dir.join(BUILD_ARTIFACT_COMPLETE_FILENAME), b"")?; + Ok::<(), LpmError>(()) + })(); + if let Err(error) = publish_result { + let _ = std::fs::remove_dir_all(&tmp_dir); + return Err(error); + } + + match std::fs::rename(&tmp_dir, &final_dir) { + Ok(()) => Ok(BuildArtifactPublish::Published), + Err(error) if final_dir.exists() => { + let _ = std::fs::remove_dir_all(&tmp_dir); + if self.reusable_build_artifact(key)?.is_some() { + Ok(BuildArtifactPublish::Existing) + } else { + Err(LpmError::Store(format!( + "concurrent build artifact publication left an invalid entry at {}: {error}", + final_dir.display() + ))) + } + } + Err(error) => { + let _ = std::fs::remove_dir_all(&tmp_dir); + Err(LpmError::Store(format!( + "failed to publish build artifact at {}: {error}", + final_dir.display() + ))) + } + } + } + + /// Restore a validated build artifact over a materialized package tree. + pub fn restore_build_artifact( + &self, + artifact: &BuildArtifact, + destination: &Path, + ) -> Result<(), LpmError> { + let staged = tmp_sibling(destination); + materialize_into_inner( + &artifact.package_dir(), + &artifact.package_dir(), + &staged, + true, + )?; + replace_with_staged_tree(destination, &staged) + } + + /// Replace a built link-package tree with its verified pristine object. + pub fn restore_pristine_package( + &self, + pristine_object_dir: &Path, + destination: &Path, + ) -> Result<(), LpmError> { + let staged = tmp_sibling(destination); + materialize_into(pristine_object_dir, &staged)?; + replace_with_staged_tree(destination, &staged) + } +} + +fn replace_with_staged_tree(destination: &Path, staged: &Path) -> Result<(), LpmError> { + let parent = destination.parent().ok_or_else(|| { + LpmError::Store(format!( + "package-tree destination has no parent: {}", + destination.display() + )) + })?; + let backup = + destination.with_extension(format!("lpm-build-backup.{:016x}", rand::random::())); + if destination.exists() { + std::fs::rename(destination, &backup).map_err(|error| { + let _ = std::fs::remove_dir_all(staged); + LpmError::Store(format!( + "failed to stage existing package tree at {}: {error}", + destination.display() + )) + })?; + } + if let Err(error) = std::fs::rename(staged, destination) { + if backup.exists() { + let _ = std::fs::rename(&backup, destination); + } + let _ = std::fs::remove_dir_all(staged); + return Err(LpmError::Store(format!( + "failed to restore build artifact into {} from {}: {error}", + destination.display(), + parent.display() + ))); + } + if backup.exists() { + std::fs::remove_dir_all(&backup).map_err(|error| { + LpmError::Store(format!( + "restored build artifact but failed to remove backup at {}: {error}", + backup.display() + )) + })?; + } + Ok(()) +} + +fn manifest_from_tree( + key: &BuildCacheKey, + source_integrity: &str, + stats: &ObjectTreeStats, + output_integrity: String, + lifecycle_duration_ms: u64, +) -> BuildArtifactManifest { + BuildArtifactManifest { + schema: BUILD_ARTIFACT_SCHEMA, + key: key.as_str().to_string(), + source_integrity: source_integrity.to_string(), + output_integrity, + file_count: stats.file_count, + unpacked_bytes: stats.unpacked_bytes, + lifecycle_duration_ms, + created_at: Utc::now(), + } +} + +fn read_manifest(dir: &Path) -> Result { + let path = dir.join(BUILD_MANIFEST_FILE); + let metadata = std::fs::metadata(&path)?; + if metadata.len() > BUILD_MANIFEST_MAX_BYTES { + return Err(LpmError::Store(format!( + "build artifact manifest exceeds {} bytes at {}", + BUILD_MANIFEST_MAX_BYTES, + path.display() + ))); + } + let bytes = std::fs::read(&path)?; + let manifest: BuildArtifactManifest = serde_json::from_slice(&bytes).map_err(|error| { + LpmError::Store(format!( + "failed to parse build artifact manifest at {}: {error}", + path.display() + )) + })?; + if manifest.schema != BUILD_ARTIFACT_SCHEMA { + return Err(LpmError::Store(format!( + "unsupported build artifact schema {} at {}", + manifest.schema, + path.display() + ))); + } + Ok(manifest) +} + +fn tree_has_safe_symlinks(root: &Path) -> Result { + let mut pending = vec![root.to_path_buf()]; + while let Some(dir) = pending.pop() { + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let path = entry.path(); + let file_type = entry.file_type()?; + if file_type.is_dir() { + pending.push(path); + } else if file_type.is_symlink() { + let target = std::fs::read_link(&path)?; + let relative_parent = path + .parent() + .and_then(|parent| parent.strip_prefix(root).ok()) + .unwrap_or_else(|| Path::new("")); + if !relative_target_stays_in_root(relative_parent, &target) { + return Ok(false); + } + } + } + } + Ok(true) +} + +fn relative_target_stays_in_root(parent: &Path, target: &Path) -> bool { + if target.is_absolute() { + return false; + } + let mut depth = parent.components().count(); + for component in target.components() { + match component { + Component::CurDir => {} + Component::Normal(_) => depth = depth.saturating_add(1), + Component::ParentDir if depth > 0 => depth -= 1, + Component::ParentDir | Component::RootDir | Component::Prefix(_) => return false, + } + } + true +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_inputs() -> BuildKeyInputs { + BuildKeyInputs { + source_integrity: "sha512-source".into(), + graph_key_digest: "graph".into(), + dependency_closure_hash: "closure".into(), + scripts: vec![BuildScriptFingerprint { + phase: "install".into(), + command: "node-gyp rebuild".into(), + }], + platform: BuildPlatformFingerprint { + os: "linux".into(), + architecture: "x64".into(), + libc: "glibc".into(), + cpu_features_hash: "cpu".into(), + }, + runtime: BuildRuntimeFingerprint { + runtime: "node".into(), + version: "22.12.0".into(), + modules_abi: "127".into(), + napi: "10".into(), + engine: "12.4".into(), + executable_hash: "node-binary".into(), + }, + sandbox: BuildSandboxFingerprint { + mode: "enforce".into(), + posture: "strict".into(), + network_denied: true, + environment_scrubbed: true, + allowed_inputs_hash: "policy".into(), + }, + environment_hash: "env".into(), + toolchain_hash: "toolchain".into(), + } + } + + #[test] + fn build_key_is_deterministic_for_identical_inputs() { + let inputs = sample_inputs(); + assert_eq!( + BuildCacheKey::derive(&inputs), + BuildCacheKey::derive(&inputs) + ); + } + + #[test] + fn build_key_changes_when_node_abi_changes() { + let first = sample_inputs(); + let mut second = first.clone(); + second.runtime.modules_abi = "131".into(); + assert_ne!( + BuildCacheKey::derive(&first), + BuildCacheKey::derive(&second) + ); + } + + #[test] + fn build_key_changes_when_sandbox_policy_changes() { + let first = sample_inputs(); + let mut second = first.clone(); + second.sandbox.network_denied = false; + assert_ne!( + BuildCacheKey::derive(&first), + BuildCacheKey::derive(&second) + ); + } + + #[test] + fn build_artifact_round_trip_restores_complete_tree() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let source = temp.path().join("source"); + let destination = temp.path().join("destination"); + std::fs::create_dir_all(source.join("build/Release")).unwrap(); + std::fs::write(source.join("package.json"), b"{}").unwrap(); + std::fs::write(source.join("build/Release/addon.node"), b"native").unwrap(); + std::fs::create_dir_all(&destination).unwrap(); + std::fs::write(destination.join("stale"), b"stale").unwrap(); + let key = BuildCacheKey::derive(&sample_inputs()); + + let published = store + .publish_build_artifact(&key, "sha512-source", &source, 125, false) + .unwrap(); + let artifact = store.reusable_build_artifact(&key).unwrap().unwrap(); + store + .restore_build_artifact(&artifact, &destination) + .unwrap(); + + assert_eq!(published, BuildArtifactPublish::Published); + assert_eq!(artifact.manifest.lifecycle_duration_ms, 125); + assert_eq!( + std::fs::read(destination.join("build/Release/addon.node")).unwrap(), + b"native" + ); + assert!(!destination.join("stale").exists()); + } + + #[test] + fn build_lock_path_is_stable_and_separate_from_artifacts() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let key = BuildCacheKey::derive(&sample_inputs()); + + let lock_path = store.paths().build_lock_path(&key); + + assert_eq!( + lock_path, + temp.path() + .join("store/build-locks") + .join(format!("{}.lock", key.as_str())) + ); + assert!(!lock_path.starts_with(store.paths().builds_root())); + } + + #[test] + fn build_artifact_corruption_is_a_cache_miss() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let source = temp.path().join("source"); + std::fs::create_dir_all(&source).unwrap(); + std::fs::write(source.join("package.json"), b"{}").unwrap(); + let key = BuildCacheKey::derive(&sample_inputs()); + store + .publish_build_artifact(&key, "sha512-source", &source, 125, false) + .unwrap(); + let artifact_dir = store.paths().build_artifact_dir(&key); + std::fs::write(artifact_dir.join("package/package.json"), b"corrupt").unwrap(); + + assert!(store.reusable_build_artifact(&key).unwrap().is_none()); + } + + #[test] + fn pristine_restore_replaces_previous_build_outputs() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let pristine = temp.path().join("pristine"); + let destination = temp.path().join("destination"); + std::fs::create_dir_all(&pristine).unwrap(); + std::fs::create_dir_all(&destination).unwrap(); + std::fs::write(pristine.join("package.json"), b"{}").unwrap(); + std::fs::write(destination.join("old-build.node"), b"old").unwrap(); + + store + .restore_pristine_package(&pristine, &destination) + .unwrap(); + + assert!(destination.join("package.json").is_file()); + assert!(!destination.join("old-build.node").exists()); + } + + #[test] + fn forced_publication_replaces_existing_artifact_for_same_key() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let source = temp.path().join("source"); + std::fs::create_dir_all(&source).unwrap(); + std::fs::write(source.join("output.node"), b"first").unwrap(); + let key = BuildCacheKey::derive(&sample_inputs()); + store + .publish_build_artifact(&key, "sha512-source", &source, 10, false) + .unwrap(); + + std::fs::write(source.join("output.node"), b"second").unwrap(); + store + .publish_build_artifact(&key, "sha512-source", &source, 20, true) + .unwrap(); + let artifact = store.reusable_build_artifact(&key).unwrap().unwrap(); + + assert_eq!( + std::fs::read(artifact.package_dir().join("output.node")).unwrap(), + b"second" + ); + assert_eq!(artifact.manifest.lifecycle_duration_ms, 20); + } + + #[cfg(unix)] + #[test] + fn build_artifact_rejects_symlink_that_escapes_package() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let source = temp.path().join("source"); + std::fs::create_dir_all(&source).unwrap(); + std::fs::write(source.join("package.json"), b"{}").unwrap(); + symlink("../secret", source.join("escape")).unwrap(); + let key = BuildCacheKey::derive(&sample_inputs()); + + let error = store + .publish_build_artifact(&key, "sha512-source", &source, 125, false) + .unwrap_err(); + assert!(error.to_string().contains("escaping symlink")); + } +} diff --git a/crates/lpm-store/src/v2/mod.rs b/crates/lpm-store/src/v2/mod.rs index 58762217..9463cb0a 100644 --- a/crates/lpm-store/src/v2/mod.rs +++ b/crates/lpm-store/src/v2/mod.rs @@ -37,6 +37,7 @@ //! //! See [`GraphKey::derive`] for the exact byte layout fed into BLAKE3. +pub mod build_cache; pub(crate) mod finalize_permits; pub(crate) mod fs_util; pub mod graph_key; @@ -47,6 +48,11 @@ pub mod platform; pub mod store; pub(crate) mod tree_hash; +pub use build_cache::{ + BUILD_ARTIFACT_COMPLETE_FILENAME, BuildArtifact, BuildArtifactManifest, BuildArtifactPublish, + BuildCacheKey, BuildKeyInputs, BuildPlatformFingerprint, BuildRuntimeFingerprint, + BuildSandboxFingerprint, BuildScriptFingerprint, +}; pub use graph_key::{DepEdge, GraphKey, GraphKeyInputs, LinkerModeTag, PeerEntry}; pub use link_meta::{ LINK_META_FILENAME, LINK_META_SCHEMA_VERSION, LinkMeta, LinkMetaDep, LinkMetaPlatform, diff --git a/crates/lpm-store/src/v2/store.rs b/crates/lpm-store/src/v2/store.rs index d24aabb7..a6081238 100644 --- a/crates/lpm-store/src/v2/store.rs +++ b/crates/lpm-store/src/v2/store.rs @@ -62,6 +62,13 @@ const LINKS_DIR: &str = "links"; /// re-copying every package. const COMPAT_DIR: &str = "compat"; +/// Subdirectory holding lifecycle-build artifacts keyed by their complete +/// execution inputs. +const BUILDS_DIR: &str = "builds"; + +/// Subdirectory holding per-build-key advisory lock files. +const BUILD_LOCKS_DIR: &str = "build-locks"; + /// Schema tag folded into [`compat_island_key`] so a change to the island's /// on-disk layout invalidates every cached island instead of silently /// reusing a stale shape. @@ -137,6 +144,10 @@ pub struct StoreV2Paths { links_root: PathBuf, /// `~/.lpm/store/v2/compat/` — precomputed for the same reason. compat_root: PathBuf, + /// `~/.lpm/store/v2/builds/` — precomputed for build-cache operations. + builds_root: PathBuf, + /// `~/.lpm/store/v2/build-locks/` — precomputed for per-key serialization. + build_locks_root: PathBuf, } impl StoreV2Paths { @@ -146,11 +157,15 @@ impl StoreV2Paths { let objects_root = root.join(OBJECTS_DIR); let links_root = root.join(LINKS_DIR); let compat_root = root.join(COMPAT_DIR); + let builds_root = root.join(BUILDS_DIR); + let build_locks_root = root.join(BUILD_LOCKS_DIR); Self { root, objects_root, links_root, compat_root, + builds_root, + build_locks_root, } } @@ -160,11 +175,15 @@ impl StoreV2Paths { let objects_root = root.join(OBJECTS_DIR); let links_root = root.join(LINKS_DIR); let compat_root = root.join(COMPAT_DIR); + let builds_root = root.join(BUILDS_DIR); + let build_locks_root = root.join(BUILD_LOCKS_DIR); Self { root, objects_root, links_root, compat_root, + builds_root, + build_locks_root, } } @@ -196,6 +215,36 @@ impl StoreV2Paths { &self.compat_root } + /// `~/.lpm/store/v2/builds/` — root of lifecycle-build artifacts. + #[inline] + pub fn builds_root(&self) -> &Path { + &self.builds_root + } + + /// Return the content-addressed directory for a lifecycle-build key. + #[inline] + pub fn build_artifact_dir(&self, key: &crate::v2::BuildCacheKey) -> PathBuf { + self.builds_root.join(key.as_str()) + } + + /// Advisory lock path serializing work for one lifecycle-build key. + #[inline] + pub fn build_lock_path(&self, key: &crate::v2::BuildCacheKey) -> PathBuf { + self.build_lock_path_for_key(key.as_str()) + } + + /// Advisory lock path for a validated hexadecimal build-key string. + #[inline] + pub fn build_lock_path_for_key(&self, key: &str) -> PathBuf { + self.build_locks_root.join(format!("{key}.lock")) + } + + /// `~/.lpm/store/v2/build-locks/` — per-key advisory locks. + #[inline] + pub fn build_locks_root(&self) -> &Path { + &self.build_locks_root + } + /// `~/.lpm/store/v2/compat//` — the cached island for a /// given entry-set content key. `key` is a hex digest the caller /// derives from the island's entry set, so the path is filesystem-safe. diff --git a/tests/workflows/tests/native_build_cache.rs b/tests/workflows/tests/native_build_cache.rs new file mode 100644 index 00000000..910e3e0a --- /dev/null +++ b/tests/workflows/tests/native_build_cache.rs @@ -0,0 +1,261 @@ +//! Workflow coverage for dependency lifecycle-build artifact reuse. + +#![cfg(target_os = "macos")] + +mod support; + +use support::mock_registry::MockRegistry; +use support::{TempProject, lpm_spawnable_with_registry, lpm_with_registry}; + +const PACKAGE_NAME: &str = "synthetic-native-cache"; +const PACKAGE_VERSION: &str = "1.0.0"; +const BUILD_SCRIPT: &str = r#" +const fs = require('fs'); +fs.writeFileSync('native-output.txt', process.hrtime.bigint().toString()); +"#; + +fn node_available() -> bool { + std::process::Command::new("node") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) +} + +fn project_manifest() -> String { + format!( + r#"{{ + "name": "native-build-cache-workflow", + "version": "1.0.0", + "dependencies": {{ + "{PACKAGE_NAME}": "{PACKAGE_VERSION}" + }} +}}"# + ) +} + +fn installed_package_dir(project: &TempProject) -> std::path::PathBuf { + std::fs::canonicalize(project.path().join("node_modules").join(PACKAGE_NAME)) + .expect("installed dependency must resolve into the v2 link entry") +} + +#[tokio::test] +async fn native_lifecycle_output_is_restored_after_pristine_rematerialization() { + if !node_available() { + eprintln!("skipping: node is required for the native build-cache workflow"); + return; + } + + let registry = MockRegistry::start().await; + registry + .with_manifest_package( + serde_json::json!({ + "name": PACKAGE_NAME, + "version": PACKAGE_VERSION, + "scripts": { + "postinstall": "node build.js # node-gyp rebuild" + } + }), + &[("build.js", BUILD_SCRIPT.as_bytes())], + ) + .await; + let project = TempProject::empty(&project_manifest()); + + let install = lpm_with_registry(&project, ®istry.url()) + .arg("install") + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("initial install"); + assert!( + install.status.success(), + "initial install failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&install.stdout), + String::from_utf8_lossy(&install.stderr) + ); + + let first_rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("first native rebuild"); + assert!( + first_rebuild.status.success(), + "first rebuild failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&first_rebuild.stdout), + String::from_utf8_lossy(&first_rebuild.stderr) + ); + let first_output = + std::fs::read_to_string(installed_package_dir(&project).join("native-output.txt")) + .expect("first build must produce native-output.txt"); + let builds_root = project.store_dir().join("v2/builds"); + assert!( + builds_root.is_dir(), + "first rebuild did not create the build cache:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&first_rebuild.stdout), + String::from_utf8_lossy(&first_rebuild.stderr) + ); + assert_eq!( + std::fs::read_dir(&builds_root) + .expect("build artifact root") + .count(), + 1, + "first rebuild must publish exactly one artifact" + ); + + std::fs::remove_dir_all(project.path().join("node_modules")) + .expect("remove project materialization"); + let reinstall = lpm_with_registry(&project, ®istry.url()) + .args(["install", "--force"]) + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("pristine reinstall"); + assert!( + reinstall.status.success(), + "reinstall failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&reinstall.stdout), + String::from_utf8_lossy(&reinstall.stderr) + ); + assert!( + !installed_package_dir(&project) + .join("native-output.txt") + .exists(), + "reinstall must rematerialize pristine source bytes before cache restore" + ); + + let second_rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("cached native rebuild"); + assert!( + second_rebuild.status.success(), + "cached rebuild failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&second_rebuild.stdout), + String::from_utf8_lossy(&second_rebuild.stderr) + ); + let mut envelope: serde_json::Value = serde_json::from_slice(&second_rebuild.stdout) + .expect("cached rebuild stdout must be a JSON envelope"); + assert_eq!(envelope["build_cache"]["hits"], 1); + assert_eq!(envelope["build_cache"]["misses"], 0); + assert_eq!(envelope["build_cache"]["scripts_avoided"], 1); + envelope["build_cache"]["restored_bytes"] = serde_json::json!(""); + envelope["build_cache"]["lifecycle_ms_avoided"] = serde_json::json!(""); + for field in [ + "preparation", + "key", + "lookup", + "restore", + "rematerialize", + "publish", + ] { + envelope["build_cache"]["timings_ms"][field] = serde_json::json!(""); + } + insta::assert_json_snapshot!("native_build_cache_hit_metrics", envelope); + assert_eq!( + std::fs::read_to_string(installed_package_dir(&project).join("native-output.txt")) + .expect("cached output must be restored"), + first_output, + "cache hit must restore the first build output instead of executing the script again" + ); + + let invalidated_rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .env("CFLAGS", "-DLPM_BUILD_CACHE_INVALIDATION=1") + .output() + .expect("rebuild with changed build environment"); + assert!( + invalidated_rebuild.status.success(), + "invalidated rebuild failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&invalidated_rebuild.stdout), + String::from_utf8_lossy(&invalidated_rebuild.stderr) + ); + let invalidated_envelope: serde_json::Value = + serde_json::from_slice(&invalidated_rebuild.stdout) + .expect("invalidated rebuild stdout must be JSON"); + assert_eq!(invalidated_envelope["build_cache"]["hits"], 0); + assert_eq!(invalidated_envelope["build_cache"]["misses"], 1); + assert_ne!( + std::fs::read_to_string(installed_package_dir(&project).join("native-output.txt")) + .expect("invalidated rebuild must produce native-output.txt"), + first_output, + "a changed build-environment key must invalidate keyed local build state and rerun the lifecycle command" + ); +} + +#[tokio::test] +async fn concurrent_native_rebuilds_execute_one_lifecycle_build_per_key() { + if !node_available() { + eprintln!("skipping: node is required for the native build-cache workflow"); + return; + } + + let registry = MockRegistry::start().await; + let script = br#" +const fs = require('fs'); +fs.appendFileSync('executions.txt', `${process.pid}\n`); +Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500); +"#; + registry + .with_manifest_package( + serde_json::json!({ + "name": PACKAGE_NAME, + "version": PACKAGE_VERSION, + "scripts": { + "postinstall": "node build.js # node-gyp rebuild" + } + }), + &[("build.js", script)], + ) + .await; + let project = TempProject::empty(&project_manifest()); + let install = lpm_with_registry(&project, ®istry.url()) + .arg("install") + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("initial install"); + assert!(install.status.success()); + + let mut first_command = lpm_spawnable_with_registry(&project, ®istry.url()); + first_command + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2"); + let mut second_command = lpm_spawnable_with_registry(&project, ®istry.url()); + second_command + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2"); + + let first = first_command.spawn().expect("spawn first rebuild"); + let second = second_command.spawn().expect("spawn second rebuild"); + let first_output = first.wait_with_output().expect("wait for first rebuild"); + let second_output = second.wait_with_output().expect("wait for second rebuild"); + assert!( + first_output.status.success() && second_output.status.success(), + "concurrent rebuild failed:\nfirst stderr:\n{}\nsecond stderr:\n{}", + String::from_utf8_lossy(&first_output.stderr), + String::from_utf8_lossy(&second_output.stderr) + ); + + let executions = + std::fs::read_to_string(installed_package_dir(&project).join("executions.txt")) + .expect("the winning lifecycle build must produce executions.txt"); + assert_eq!( + executions.lines().count(), + 1, + "the per-key lock must prevent duplicate lifecycle execution" + ); + let envelopes = [&first_output.stdout, &second_output.stdout] + .into_iter() + .map(|stdout| serde_json::from_slice::(stdout).unwrap()) + .collect::>(); + assert_eq!( + envelopes + .iter() + .map(|value| { + value["build_cache"]["hits"].as_u64().unwrap() + + value["build_cache"]["local_state_hits"].as_u64().unwrap() + }) + .sum::(), + 1, + "one contender must build and the other must reuse its keyed result" + ); +} diff --git a/tests/workflows/tests/snapshots/native_build_cache__native_build_cache_hit_metrics.snap b/tests/workflows/tests/snapshots/native_build_cache__native_build_cache_hit_metrics.snap new file mode 100644 index 00000000..b3f350c2 --- /dev/null +++ b/tests/workflows/tests/snapshots/native_build_cache__native_build_cache_hit_metrics.snap @@ -0,0 +1,27 @@ +--- +source: tests/workflows/tests/native_build_cache.rs +expression: envelope +--- +{ + "success": true, + "built": 1, + "failed": 0, + "build_cache": { + "eligible": 1, + "hits": 1, + "misses": 0, + "bypassed": 0, + "local_state_hits": 0, + "scripts_avoided": 1, + "restored_bytes": "", + "lifecycle_ms_avoided": "", + "timings_ms": { + "preparation": "", + "key": "", + "lookup": "", + "restore": "", + "rematerialize": "", + "publish": "" + } + } +} From b6339f3aa8cac4b2373be817a5e6034151a4169d Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 06:50:06 +0100 Subject: [PATCH 02/14] fix: keep Linux sandbox helpers cfg-clean --- crates/lpm-sandbox/src/landlock_rules.rs | 1 + crates/lpm-sandbox/src/linux.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/lpm-sandbox/src/landlock_rules.rs b/crates/lpm-sandbox/src/landlock_rules.rs index 1bee0ff9..8a2fe19c 100644 --- a/crates/lpm-sandbox/src/landlock_rules.rs +++ b/crates/lpm-sandbox/src/landlock_rules.rs @@ -94,6 +94,7 @@ pub(crate) const SYSTEM_READ_PATHS: &[&str] = &[ /// 1. Testable without a Linux kernel (runs on macOS host). /// 2. Free of unsafe `PathFd` opens at rule-description time (those /// happen in the child's pre_exec hook in [`crate::linux`]). +#[cfg(test)] pub(crate) fn describe_rules(spec: &SandboxSpec) -> Vec<(PathBuf, RuleAccess)> { describe_rules_with_isolation(spec, false) } diff --git a/crates/lpm-sandbox/src/linux.rs b/crates/lpm-sandbox/src/linux.rs index d25b30aa..e7e08363 100644 --- a/crates/lpm-sandbox/src/linux.rs +++ b/crates/lpm-sandbox/src/linux.rs @@ -117,7 +117,7 @@ #![cfg(target_os = "linux")] -use crate::landlock_rules::{RuleAccess, describe_rules}; +use crate::landlock_rules::RuleAccess; use crate::posture_decision::{PostureDecision, REQUIRED_KERNEL_FOR_STRICT, decide_posture}; use crate::{ Sandbox, SandboxError, SandboxMode, SandboxOptions, SandboxPosture, SandboxSpec, From 9377ef5758d31543a6aab1f4645aff835129198e Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 07:05:36 +0100 Subject: [PATCH 03/14] test: update missing-registry prune snapshot --- ...une__cache_prune_dry_run_json_envelope_missing_registry.snap | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/workflows/tests/snapshots/cache_prune__cache_prune_dry_run_json_envelope_missing_registry.snap b/tests/workflows/tests/snapshots/cache_prune__cache_prune_dry_run_json_envelope_missing_registry.snap index a70fffd5..1597e32b 100644 --- a/tests/workflows/tests/snapshots/cache_prune__cache_prune_dry_run_json_envelope_missing_registry.snap +++ b/tests/workflows/tests/snapshots/cache_prune__cache_prune_dry_run_json_envelope_missing_registry.snap @@ -15,6 +15,8 @@ expression: json "object_entries_orphaned": [], "compat_islands_total": 0, "compat_islands_orphaned": [], + "build_artifacts_total": 0, + "build_artifacts_orphaned": [], "bytes_freed_or_eligible": 0, "registry_missing": true, "registry_corrupt": false, From b68596e4e0a2f0a9703d05a2497d73cc24671f91 Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 13:25:02 +0100 Subject: [PATCH 04/14] fix: harden native build cache correctness --- ...tive-build-cache-20260712-macos-arm64.json | 34 +- ...native-build-cache-20260712-macos-arm64.md | 12 +- crates/lpm-cli/src/commands/cache_prune.rs | 106 ++++- .../src/commands/rebuild/build_cache.rs | 364 ++++++++++++++++-- crates/lpm-cli/src/commands/rebuild/mod.rs | 98 +++-- crates/lpm-store/src/v2/build_cache.rs | 14 + crates/lpm-store/src/v2/store.rs | 22 ++ tests/workflows/tests/native_build_cache.rs | 98 ++++- 8 files changed, 646 insertions(+), 102 deletions(-) diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64.json b/bench/perf-results/native-build-cache-20260712-macos-arm64.json index 092407ad..2124357f 100644 --- a/bench/perf-results/native-build-cache-20260712-macos-arm64.json +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-07-12T05:36:38.673Z", + "generated_at": "2026-07-12T11:27:12.697Z", "platform": "darwin-arm64", "node": "v22.22.1", "lpm_bin": "target/release/lpm-rs", @@ -8,17 +8,17 @@ "artifact_bytes": 134444, "scenarios": { "cache_disabled": { - "wall_ms_median": 114.99, - "wall_ms_p95": 417.43, - "max_rss_bytes_median": 52248576, + "wall_ms_median": 112.35, + "wall_ms_p95": 423.22, + "max_rss_bytes_median": 52264960, "user_cpu_ms_median": 40, "system_cpu_ms_median": 10, "cache_hits": 0, "cache_misses": 0 }, "cache_miss": { - "wall_ms_median": 134.58, - "wall_ms_p95": 148.42, + "wall_ms_median": 137.53, + "wall_ms_p95": 147.84, "max_rss_bytes_median": 52264960, "user_cpu_ms_median": 50, "system_cpu_ms_median": 20, @@ -26,28 +26,28 @@ "cache_misses": 10 }, "local_hit": { - "wall_ms_median": 29.45, - "wall_ms_p95": 51.91, - "max_rss_bytes_median": 41500672, + "wall_ms_median": 29.66, + "wall_ms_p95": 31.39, + "max_rss_bytes_median": 41517056, "user_cpu_ms_median": 10, "system_cpu_ms_median": 0, "cache_hits": 10, "cache_misses": 0, - "lifecycle_ms_avoided": 1028, - "speedup_vs_disabled": 3.9, - "wall_reduction_percent": 74.39 + "lifecycle_ms_avoided": 1046, + "speedup_vs_disabled": 3.79, + "wall_reduction_percent": 73.6 }, "ci_warm_store_hit": { "wall_ms_median": 29.19, - "wall_ms_p95": 31.29, - "max_rss_bytes_median": 41566208, + "wall_ms_p95": 30.51, + "max_rss_bytes_median": 41533440, "user_cpu_ms_median": 10, "system_cpu_ms_median": 0, "cache_hits": 10, "cache_misses": 0, - "lifecycle_ms_avoided": 1028, - "speedup_vs_disabled": 3.94, - "wall_reduction_percent": 74.62 + "lifecycle_ms_avoided": 1046, + "speedup_vs_disabled": 3.85, + "wall_reduction_percent": 74.02 } } } diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64.md b/bench/perf-results/native-build-cache-20260712-macos-arm64.md index 1b2bac01..4e80cdac 100644 --- a/bench/perf-results/native-build-cache-20260712-macos-arm64.md +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64.md @@ -8,17 +8,17 @@ | Scenario | Median wall | p95 wall | Median RSS | User CPU | System CPU | Result | |---|---:|---:|---:|---:|---:|---:| -| Cache disabled | 114.99 ms | 417.43 ms | 52,248,576 B | 40 ms | 10 ms | baseline | -| Cache miss | 134.58 ms | 148.42 ms | 52,264,960 B | 50 ms | 20 ms | 10 misses | -| Local artifact hit | 29.45 ms | 51.91 ms | 41,500,672 B | 10 ms | 0 ms | 3.90×; 74.39% lower wall time | -| Fresh-project warm-store hit | 29.19 ms | 31.29 ms | 41,566,208 B | 10 ms | 0 ms | 3.94×; 74.62% lower wall time | +| Cache disabled | 112.35 ms | 423.22 ms | 52,264,960 B | 40 ms | 10 ms | baseline | +| Cache miss | 137.53 ms | 147.84 ms | 52,264,960 B | 50 ms | 20 ms | 10 misses | +| Local artifact hit | 29.66 ms | 31.39 ms | 41,517,056 B | 10 ms | 0 ms | 3.79×; 73.60% lower wall time | +| Fresh-project warm-store hit | 29.19 ms | 30.51 ms | 41,533,440 B | 10 ms | 0 ms | 3.85×; 74.02% lower wall time | The disabled and miss scenarios rematerialized pristine dependencies before every sample. Local-hit samples rematerialized the same project. CI-like hit samples created a fresh project while retaining the warm LPM store. Across ten -hits, the artifact metadata recorded 1,028 ms of lifecycle execution avoided. +hits, the artifact metadata recorded 1,046 ms of lifecycle execution avoided. -The cache miss adds about 19.6 ms over the disabled median for keying, +The cache miss adds about 25.2 ms over the disabled median for keying, pristine rematerialization, integrity validation, and atomic publication. This fixture has a short lifecycle command; native compilation workloads should have a substantially larger absolute hit benefit. diff --git a/crates/lpm-cli/src/commands/cache_prune.rs b/crates/lpm-cli/src/commands/cache_prune.rs index 5a7e2ce2..035aaa54 100644 --- a/crates/lpm-cli/src/commands/cache_prune.rs +++ b/crates/lpm-cli/src/commands/cache_prune.rs @@ -259,6 +259,7 @@ fn run_locked( })?; } remove_orphaned_build_locks(v2_store)?; + remove_orphaned_build_entry_locks(v2_store)?; // Sweep deferred global-uninstall tombstones. Errors are // surfaced via `summary.tombstone_sweep_error` (and a @@ -301,21 +302,65 @@ fn remove_orphaned_build_locks(store: &V2Store) -> Result<(), LpmError> { continue; } let path = entry.path(); - let Some(key) = path.file_stem().and_then(|stem| stem.to_str()) else { + let Some(key) = advisory_lock_key(&path) else { continue; }; - if key.len() == 64 - && key - .bytes() - .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) - && !store.paths().builds_root().join(key).is_dir() - { + if !store.paths().builds_root().join(key).is_dir() { + std::fs::remove_file(path)?; + } + } + Ok(()) +} + +fn remove_orphaned_build_entry_locks(store: &V2Store) -> Result<(), LpmError> { + let mut live_graph_keys = HashSet::new(); + match std::fs::read_dir(store.paths().links_root()) { + Ok(entries) => { + for entry in entries.flatten() { + if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { + continue; + } + if let Ok(metadata) = lpm_store::v2::LinkMeta::read_from(&entry.path()) { + live_graph_keys.insert(metadata.graph_key_digest_hex); + } + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let entries = match std::fs::read_dir(store.paths().build_entry_locks_root()) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + for entry in entries { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let path = entry.path(); + let Some(key) = advisory_lock_key(&path) else { + continue; + }; + if !live_graph_keys.contains(key) { std::fs::remove_file(path)?; } } Ok(()) } +fn advisory_lock_key(path: &Path) -> Option<&str> { + let name = path.file_name()?.to_str()?; + let key = [".lock.writer-intent", ".lock.writer-queue", ".lock"] + .into_iter() + .find_map(|suffix| name.strip_suffix(suffix))?; + (key.len() == 64 + && key + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f'))) + .then_some(key) +} + /// Compute (but don't apply) the prune plan. Pulled out so unit tests /// exercise the algorithm against synthetic stores without touching /// the filesystem outside of a tempdir. @@ -1276,12 +1321,59 @@ mod tests { let orphan_lock = store.paths().build_lock_path_for_key(&orphan_key); let live_lock = store.paths().build_lock_path_for_key(&live_key); std::fs::write(&orphan_lock, b"").unwrap(); + std::fs::write(format!("{}.writer-intent", orphan_lock.display()), b"").unwrap(); + std::fs::write(format!("{}.writer-queue", orphan_lock.display()), b"").unwrap(); std::fs::write(&live_lock, b"").unwrap(); + std::fs::write(format!("{}.writer-intent", live_lock.display()), b"").unwrap(); + std::fs::write(format!("{}.writer-queue", live_lock.display()), b"").unwrap(); remove_orphaned_build_locks(&store).unwrap(); assert!(!orphan_lock.exists()); + assert!(!std::fs::exists(format!("{}.writer-intent", orphan_lock.display())).unwrap()); + assert!(!std::fs::exists(format!("{}.writer-queue", orphan_lock.display())).unwrap()); + assert!(live_lock.exists()); + assert!(std::fs::exists(format!("{}.writer-intent", live_lock.display())).unwrap()); + assert!(std::fs::exists(format!("{}.writer-queue", live_lock.display())).unwrap()); + } + + #[test] + fn build_entry_lock_cleanup_removes_only_locks_without_link_entries() { + let tmp = tempfile::tempdir().unwrap(); + let root = LpmRoot::from_dir(tmp.path()); + let store = V2Store::from_lpm_root(&root); + let orphan_key = "a".repeat(64); + let live_key = "b".repeat(64); + std::fs::create_dir_all(store.paths().build_entry_locks_root()).unwrap(); + let orphan_lock = store.paths().build_entry_lock_path(&orphan_key); + let live_lock = store.paths().build_entry_lock_path(&live_key); + std::fs::write(&orphan_lock, b"").unwrap(); + std::fs::write(format!("{}.writer-intent", orphan_lock.display()), b"").unwrap(); + std::fs::write(format!("{}.writer-queue", orphan_lock.display()), b"").unwrap(); + std::fs::write(&live_lock, b"").unwrap(); + std::fs::write(format!("{}.writer-intent", live_lock.display()), b"").unwrap(); + std::fs::write(format!("{}.writer-queue", live_lock.display()), b"").unwrap(); + let link_dir = store + .paths() + .links_root() + .join("example@1.0.0+bbbbbbbbbbbbbbbb"); + std::fs::create_dir_all(&link_dir).unwrap(); + std::fs::write( + link_dir.join(lpm_store::v2::LINK_META_FILENAME), + format!( + r#"{{"schema":1,"graph_key":"example@1.0.0+bbbbbbbbbbbbbbbb","graph_key_digest_hex":"{live_key}","name":"example","version":"1.0.0","source_sri":"sha512-example","object_path":"objects/sha512-example","deps":[],"platform":{{"os":"darwin","cpu":"arm64","libc":null}},"created_at":"2026-01-01T00:00:00Z","last_referenced_at":"2026-01-01T00:00:00Z"}}"# + ), + ) + .unwrap(); + + remove_orphaned_build_entry_locks(&store).unwrap(); + + assert!(!orphan_lock.exists()); + assert!(!std::fs::exists(format!("{}.writer-intent", orphan_lock.display())).unwrap()); + assert!(!std::fs::exists(format!("{}.writer-queue", orphan_lock.display())).unwrap()); assert!(live_lock.exists()); + assert!(std::fs::exists(format!("{}.writer-intent", live_lock.display())).unwrap()); + assert!(std::fs::exists(format!("{}.writer-queue", live_lock.display())).unwrap()); } #[test] diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache.rs b/crates/lpm-cli/src/commands/rebuild/build_cache.rs index 94aa03ce..8e13d38c 100644 --- a/crates/lpm-cli/src/commands/rebuild/build_cache.rs +++ b/crates/lpm-cli/src/commands/rebuild/build_cache.rs @@ -23,11 +23,15 @@ impl BuildCacheScratch { .ancestors() .find(|ancestor| ancestor.join(lpm_store::v2::LINK_META_FILENAME).is_file()) .ok_or_else(|| std::io::Error::other("package is not inside a v2 link entry"))?; - let path = link_dir.join(format!( - ".lpm-build-tmp.{}.{:016x}", - std::process::id(), - rand::random::() - )); + let path = link_dir.join(".lpm-build-tmp"); + match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => { + std::fs::remove_dir_all(&path)?; + } + Ok(_) => std::fs::remove_file(&path)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } std::fs::create_dir(&path)?; Ok(Self { path }) } @@ -120,16 +124,155 @@ fn debug_bypass(reason: &str) { } pub(super) fn is_cacheable_native_build(package: &ScriptablePackage) -> bool { + native_build_kind(package).is_some() +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum NativeBuildKind { + RuntimeOnly, + NativeToolchain, +} + +fn native_build_kind(package: &ScriptablePackage) -> Option { if !cfg!(any(target_os = "macos", target_os = "linux")) { + return None; + } + let mut kind = None; + for command in package.scripts.values() { + for segment in shell_command_segments(command)? { + let Some(segment_kind) = native_segment_kind(&package.name, &segment) else { + continue; + }; + if segment_kind == NativeBuildKind::NativeToolchain { + return Some(segment_kind); + } + kind = Some(segment_kind); + } + } + kind +} + +fn native_segment_kind(package_name: &str, words: &[String]) -> Option { + let command_index = words + .iter() + .position(|word| !is_environment_assignment(word))?; + let executable = Path::new(&words[command_index]) + .file_name() + .and_then(|name| name.to_str())?; + let arguments = &words[command_index + 1..]; + match executable { + "node-gyp" + | "node-gyp-build" + | "node-gyp-build-optional-packages" + | "prebuild-install" + | "electron-rebuild" + | "cmake-js" => Some(NativeBuildKind::NativeToolchain), + "node" if package_name == "esbuild" => arguments + .first() + .and_then(|argument| Path::new(argument).file_name()) + .and_then(|name| name.to_str()) + .filter(|name| *name == "install.js") + .map(|_| NativeBuildKind::RuntimeOnly), + "node" if package_name == "sharp" => arguments + .first() + .filter(|argument| { + let normalized = argument.replace('\\', "/"); + normalized == "install/check" || normalized.starts_with("install/") + }) + .map(|_| NativeBuildKind::NativeToolchain), + _ => None, + } +} + +fn is_environment_assignment(word: &str) -> bool { + let Some((name, _)) = word.split_once('=') else { return false; + }; + !name.is_empty() + && name.bytes().enumerate().all(|(index, byte)| { + byte == b'_' || byte.is_ascii_alphabetic() || (index > 0 && byte.is_ascii_digit()) + }) +} + +fn shell_command_segments(command: &str) -> Option>> { + let normalized = normalize_shell_structure(command); + let words = shlex::split(&normalized)?; + let mut segments = Vec::new(); + let mut current = Vec::new(); + for word in words { + if matches!(word.as_str(), "&&" | "||" | ";" | "|" | "&" | "(" | ")") { + if !current.is_empty() { + segments.push(std::mem::take(&mut current)); + } + } else { + current.push(word); + } } - package.scripts.values().any(|command| { - command.contains("node-gyp") - || command.contains("node-gyp-build") - || command.contains("prebuild-install") - || (package.name == "esbuild" && command.contains("install.js")) - || (package.name == "sharp" && command.contains("install/")) - }) + if !current.is_empty() { + segments.push(current); + } + Some(segments) +} + +fn normalize_shell_structure(command: &str) -> String { + let mut normalized = String::with_capacity(command.len() * 2); + let mut chars = command.chars().peekable(); + let mut single_quoted = false; + let mut double_quoted = false; + let mut word_boundary = true; + while let Some(character) = chars.next() { + if character == '\\' && !single_quoted { + normalized.push(character); + if let Some(escaped) = chars.next() { + normalized.push(escaped); + } + word_boundary = false; + continue; + } + if character == '\'' && !double_quoted { + single_quoted = !single_quoted; + normalized.push(character); + word_boundary = false; + continue; + } + if character == '"' && !single_quoted { + double_quoted = !double_quoted; + normalized.push(character); + word_boundary = false; + continue; + } + if !single_quoted && !double_quoted && character == '#' && word_boundary { + while chars.next().is_some_and(|next| next != '\n') {} + normalized.push_str(" ; "); + word_boundary = true; + continue; + } + if !single_quoted && !double_quoted { + match character { + '\n' | ';' | '(' | ')' => { + normalized.push(' '); + normalized.push(if character == '\n' { ';' } else { character }); + normalized.push(' '); + word_boundary = true; + continue; + } + '|' | '&' => { + normalized.push(' '); + normalized.push(character); + if chars.peek().copied() == Some(character) { + normalized.push(chars.next().expect("peeked shell operator")); + } + normalized.push(' '); + word_boundary = true; + continue; + } + _ => {} + } + } + normalized.push(character); + word_boundary = character.is_whitespace(); + } + normalized } pub(super) fn marker_requires_key_validation(package: &ScriptablePackage) -> bool { @@ -156,13 +299,13 @@ pub(super) fn build_key_for_package( debug_bypass(&format!("{} has no v2 graph identity", package.name)); return None; }; - if !is_cacheable_native_build(package) { + let Some(native_kind) = native_build_kind(package) else { debug_bypass(&format!( "{} lifecycle command is not a recognized native build", package.name )); return None; - } + }; let mut scripts = Vec::with_capacity(EXECUTED_INSTALL_PHASES.len()); for phase in EXECUTED_INSTALL_PHASES { if let Some(command) = package.scripts.get(*phase) { @@ -172,11 +315,7 @@ pub(super) fn build_key_for_package( }); } } - let toolchain_hash = if package - .scripts - .values() - .any(|command| command.contains("node-gyp")) - { + let toolchain_hash = if native_kind == NativeBuildKind::NativeToolchain { let Some(hash) = invocation .native_toolchain_hash .get_or_init(|| hash_toolchain(&invocation.environment).ok()) @@ -237,19 +376,39 @@ fn detect_node_runtime(environment: &HashMap) -> Option String { - let mut rows = lockfile - .packages - .iter() - .map(|package| { - format!( - "{}\0{}\0{}\0{}", - package.name, - package.version, - package.integrity.as_deref().unwrap_or(""), - package.source.as_deref().unwrap_or("") - ) - }) - .collect::>(); + let mut rows = Vec::with_capacity(lockfile.packages.len() + lockfile.patches.len() + 1); + rows.push("graph-schema\0v2".to_string()); + for package in &lockfile.packages { + let mut dependencies = package.dependencies.clone(); + dependencies.sort_unstable(); + let mut aliases = package.alias_dependencies.clone(); + aliases.sort_unstable(); + let mut peers = package.peers.clone(); + peers.sort_unstable(); + rows.push(format!( + "package\0{}\0{}\0{}\0{}\0deps\0{}\0aliases\0{}\0peers\0{}", + package.name, + package.version, + package.integrity.as_deref().unwrap_or(""), + package.source.as_deref().unwrap_or(""), + dependencies.join("\0"), + aliases + .iter() + .map(|[local, target]| format!("{local}\0{target}")) + .collect::>() + .join("\0"), + peers.join("\0") + )); + } + for (package, patch) in &lockfile.patches { + rows.push(format!( + "patch\0{package}\0{}\0{}\0{}", + patch.path, patch.sha256, patch.original_integrity + )); + } + for (local, target) in &lockfile.root_aliases { + rows.push(format!("root-alias\0{local}\0{target}")); + } rows.sort_unstable(); hash_records(rows.iter().map(String::as_bytes)) } @@ -454,6 +613,49 @@ mod tests { use super::*; use std::collections::HashMap; + fn scriptable_package(name: &str, command: &str) -> ScriptablePackage { + ScriptablePackage { + name: name.into(), + version: "1.0.0".into(), + integrity: None, + wrapper_id: None, + store_path: PathBuf::new(), + pristine_path: PathBuf::new(), + source_integrity: "sha512-source".into(), + graph_key_digest: Some("a".repeat(64)), + scripts: [("postinstall".into(), command.into())] + .into_iter() + .collect(), + is_built: false, + build_marker_key: None, + is_trusted: true, + trust_reason: super::super::trust::TrustReason::StrictBinding, + } + } + + fn graph_lockfile( + left_dependencies: &[&str], + right_dependencies: &[&str], + ) -> lpm_lockfile::Lockfile { + let mut lockfile = lpm_lockfile::Lockfile::new(); + for (name, dependencies) in [ + ("root", vec!["left@1.0.0", "right@1.0.0"]), + ("left", left_dependencies.to_vec()), + ("right", right_dependencies.to_vec()), + ("leaf", Vec::new()), + ] { + lockfile.add_package(lpm_lockfile::LockedPackage { + name: name.into(), + version: "1.0.0".into(), + source: Some("registry+https://registry.npmjs.org".into()), + integrity: Some(format!("sha512-{name}")), + dependencies: dependencies.into_iter().map(str::to_owned).collect(), + ..Default::default() + }); + } + lockfile + } + #[test] fn build_environment_hash_ignores_unrelated_values() { let mut first = HashMap::new(); @@ -514,4 +716,102 @@ mod tests { assert_ne!(first.finalize(), second.finalize()); } + + #[test] + fn lockfile_graph_hash_changes_when_dependency_edge_is_rewired() { + let first = graph_lockfile(&["leaf@1.0.0"], &[]); + let second = graph_lockfile(&[], &["leaf@1.0.0"]); + + assert_ne!(hash_lockfile_graph(&first), hash_lockfile_graph(&second)); + } + + #[test] + fn lockfile_graph_hash_changes_when_patch_identity_changes() { + let mut first = graph_lockfile(&["leaf@1.0.0"], &[]); + first.patches.insert( + "leaf@1.0.0".into(), + lpm_lockfile::LockfilePatch { + path: "patches/leaf.patch".into(), + sha256: "sha256-first".into(), + original_integrity: "sha512-leaf".into(), + }, + ); + let mut second = first.clone(); + second.patches.get_mut("leaf@1.0.0").unwrap().sha256 = "sha256-second".into(); + + assert_ne!(hash_lockfile_graph(&first), hash_lockfile_graph(&second)); + } + + #[test] + fn build_cache_scratch_path_is_stable_for_one_link_entry() { + let temp = tempfile::tempdir().unwrap(); + let link_dir = temp.path().join("links/example"); + let package_dir = link_dir.join("node_modules/example"); + std::fs::create_dir_all(&package_dir).unwrap(); + std::fs::write(link_dir.join(lpm_store::v2::LINK_META_FILENAME), b"{}").unwrap(); + + let first_path = { + let scratch = BuildCacheScratch::create(&package_dir).unwrap(); + scratch.path().to_path_buf() + }; + let second_path = { + let scratch = BuildCacheScratch::create(&package_dir).unwrap(); + scratch.path().to_path_buf() + }; + + assert_eq!(first_path, second_path); + } + + #[test] + #[cfg(unix)] + fn build_cache_scratch_replaces_symlink_without_touching_target() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let link_dir = temp.path().join("links/example"); + let package_dir = link_dir.join("node_modules/example"); + let outside = temp.path().join("outside"); + std::fs::create_dir_all(&package_dir).unwrap(); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::write(outside.join("sentinel"), b"keep").unwrap(); + std::fs::write(link_dir.join(lpm_store::v2::LINK_META_FILENAME), b"{}").unwrap(); + symlink(&outside, link_dir.join(".lpm-build-tmp")).unwrap(); + + let scratch = BuildCacheScratch::create(&package_dir).unwrap(); + + assert!(outside.join("sentinel").is_file()); + assert_ne!(std::fs::canonicalize(scratch.path()).unwrap(), outside); + } + + #[test] + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn native_build_detection_ignores_shell_comments() { + let package = scriptable_package("fixture", "node build.js # node-gyp rebuild"); + + assert!(!is_cacheable_native_build(&package)); + } + + #[test] + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn native_build_detection_ignores_argument_text() { + let package = scriptable_package("fixture", "node build.js node-gyp"); + + assert!(!is_cacheable_native_build(&package)); + } + + #[test] + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn native_build_detection_recognizes_package_specific_esbuild_installer() { + let package = scriptable_package("esbuild", "node install.js"); + + assert!(is_cacheable_native_build(&package)); + } + + #[test] + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn native_build_detection_recognizes_compound_fallback_command_position() { + let package = scriptable_package("fixture", "prebuild-install || node-gyp rebuild"); + + assert!(is_cacheable_native_build(&package)); + } } diff --git a/crates/lpm-cli/src/commands/rebuild/mod.rs b/crates/lpm-cli/src/commands/rebuild/mod.rs index ad4d3be7..fc355692 100644 --- a/crates/lpm-cli/src/commands/rebuild/mod.rs +++ b/crates/lpm-cli/src/commands/rebuild/mod.rs @@ -1083,6 +1083,58 @@ async fn run_under_store_lock( for pkg in &to_build { let mut pkg_success = true; + let key_start = std::time::Instant::now(); + let mut build_key = build_cache_invocation + .as_ref() + .and_then(|invocation| build_key_for_package(invocation, pkg)); + build_cache_metrics.key_ms += elapsed_millis(key_start.elapsed()); + if build_key.is_some() { + build_cache_metrics.eligible += 1; + } else if is_cacheable_native_build(pkg) { + build_cache_metrics.bypassed += 1; + } + let v2_store = lpm_store::v2::Store::from_lpm_root(&lpm_root); + let _build_entry_lock = if let Some(graph_key_digest) = pkg.graph_key_digest.as_deref() { + match lpm_common::acquire_exclusive_lock( + v2_store.paths().build_entry_lock_path(graph_key_digest), + ) { + Ok(lock) => Some(lock), + Err(error) => { + if !json_output { + let label = rebuild_package_label(pkg); + install_ui::detail(&format!( + " {} {label: Some(lock), + Err(error) => { + tracing::warn!( + "build cache bypassed for {}@{} because the per-key lock failed: {error}", + pkg.name, + pkg.version + ); + build_cache_metrics.bypassed += 1; + build_key = None; + None + } + } + } else { + None + }; + // fix: lifecycle scripts must run from the LIVE // per-package directory (where the symlinked sibling // node_modules/ exists), not the global content-addressable @@ -1124,36 +1176,12 @@ async fn run_under_store_lock( } }; - let key_start = std::time::Instant::now(); - let mut build_key = build_cache_invocation - .as_ref() - .and_then(|invocation| build_key_for_package(invocation, pkg)); - build_cache_metrics.key_ms += elapsed_millis(key_start.elapsed()); - if build_key.is_some() { - build_cache_metrics.eligible += 1; - } else if is_cacheable_native_build(pkg) { - build_cache_metrics.bypassed += 1; - } - let _build_key_lock = if let Some(key) = build_key.as_ref() { - let store = lpm_store::v2::Store::from_lpm_root(&lpm_root); - match lpm_common::acquire_exclusive_lock(store.paths().build_lock_path(key)) { - Ok(lock) => Some(lock), - Err(error) => { - tracing::warn!( - "build cache bypassed for {}@{} because the per-key lock failed: {error}", - pkg.name, - pkg.version - ); - build_cache_metrics.bypassed += 1; - build_key = None; - None - } - } - } else { - None - }; + let marker_path = pkg.store_path.join(BUILD_MARKER); + let current_marker_exists = marker_path.is_file(); + let current_marker_key = current_marker_exists + .then(|| read_build_marker_key(&marker_path)) + .flatten(); if force { - let marker_path = pkg.store_path.join(BUILD_MARKER); if let Err(error) = std::fs::remove_file(&marker_path) && error.kind() != std::io::ErrorKind::NotFound { @@ -1162,15 +1190,15 @@ async fn run_under_store_lock( pkg.name ); } - } else if let Some(marker_key) = pkg.build_marker_key.as_deref() { - match build_key.as_ref() { - Some(key) if marker_key == key.as_str() => { + } else if current_marker_exists { + match (current_marker_key.as_deref(), build_key.as_ref()) { + (Some(marker_key), Some(key)) if marker_key == key.as_str() => { build_cache_metrics.local_state_hits += 1; continue; } - None => continue, - Some(_) => { - if let Err(error) = std::fs::remove_file(pkg.store_path.join(BUILD_MARKER)) + (_, None) => continue, + _ => { + if let Err(error) = std::fs::remove_file(&marker_path) && error.kind() != std::io::ErrorKind::NotFound { tracing::warn!( diff --git a/crates/lpm-store/src/v2/build_cache.rs b/crates/lpm-store/src/v2/build_cache.rs index d81cbfa4..dfa1b19d 100644 --- a/crates/lpm-store/src/v2/build_cache.rs +++ b/crates/lpm-store/src/v2/build_cache.rs @@ -638,6 +638,20 @@ mod tests { assert!(!lock_path.starts_with(store.paths().builds_root())); } + #[test] + fn build_entry_lock_path_is_keyed_by_graph_identity() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let graph_key = "a".repeat(64); + + assert_eq!( + store.paths().build_entry_lock_path(&graph_key), + temp.path() + .join("store/build-entry-locks") + .join(format!("{graph_key}.lock")) + ); + } + #[test] fn build_artifact_corruption_is_a_cache_miss() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/lpm-store/src/v2/store.rs b/crates/lpm-store/src/v2/store.rs index a6081238..9469cbc9 100644 --- a/crates/lpm-store/src/v2/store.rs +++ b/crates/lpm-store/src/v2/store.rs @@ -69,6 +69,9 @@ const BUILDS_DIR: &str = "builds"; /// Subdirectory holding per-build-key advisory lock files. const BUILD_LOCKS_DIR: &str = "build-locks"; +/// Subdirectory holding per-graph-entry advisory lock files. +const BUILD_ENTRY_LOCKS_DIR: &str = "build-entry-locks"; + /// Schema tag folded into [`compat_island_key`] so a change to the island's /// on-disk layout invalidates every cached island instead of silently /// reusing a stale shape. @@ -148,6 +151,8 @@ pub struct StoreV2Paths { builds_root: PathBuf, /// `~/.lpm/store/v2/build-locks/` — precomputed for per-key serialization. build_locks_root: PathBuf, + /// `~/.lpm/store/v2/build-entry-locks/` — precomputed for link-tree mutation. + build_entry_locks_root: PathBuf, } impl StoreV2Paths { @@ -159,6 +164,7 @@ impl StoreV2Paths { let compat_root = root.join(COMPAT_DIR); let builds_root = root.join(BUILDS_DIR); let build_locks_root = root.join(BUILD_LOCKS_DIR); + let build_entry_locks_root = root.join(BUILD_ENTRY_LOCKS_DIR); Self { root, objects_root, @@ -166,6 +172,7 @@ impl StoreV2Paths { compat_root, builds_root, build_locks_root, + build_entry_locks_root, } } @@ -177,6 +184,7 @@ impl StoreV2Paths { let compat_root = root.join(COMPAT_DIR); let builds_root = root.join(BUILDS_DIR); let build_locks_root = root.join(BUILD_LOCKS_DIR); + let build_entry_locks_root = root.join(BUILD_ENTRY_LOCKS_DIR); Self { root, objects_root, @@ -184,6 +192,7 @@ impl StoreV2Paths { compat_root, builds_root, build_locks_root, + build_entry_locks_root, } } @@ -245,6 +254,19 @@ impl StoreV2Paths { &self.build_locks_root } + /// Advisory lock path serializing mutation of one v2 graph entry. + #[inline] + pub fn build_entry_lock_path(&self, graph_key_digest: &str) -> PathBuf { + self.build_entry_locks_root + .join(format!("{graph_key_digest}.lock")) + } + + /// `~/.lpm/store/v2/build-entry-locks/` — per-graph-entry advisory locks. + #[inline] + pub fn build_entry_locks_root(&self) -> &Path { + &self.build_entry_locks_root + } + /// `~/.lpm/store/v2/compat//` — the cached island for a /// given entry-set content key. `key` is a hex digest the caller /// derives from the island's entry set, so the path is filesystem-safe. diff --git a/tests/workflows/tests/native_build_cache.rs b/tests/workflows/tests/native_build_cache.rs index 910e3e0a..4a6d7b6a 100644 --- a/tests/workflows/tests/native_build_cache.rs +++ b/tests/workflows/tests/native_build_cache.rs @@ -7,7 +7,7 @@ mod support; use support::mock_registry::MockRegistry; use support::{TempProject, lpm_spawnable_with_registry, lpm_with_registry}; -const PACKAGE_NAME: &str = "synthetic-native-cache"; +const PACKAGE_NAME: &str = "esbuild"; const PACKAGE_VERSION: &str = "1.0.0"; const BUILD_SCRIPT: &str = r#" const fs = require('fs'); @@ -52,10 +52,10 @@ async fn native_lifecycle_output_is_restored_after_pristine_rematerialization() "name": PACKAGE_NAME, "version": PACKAGE_VERSION, "scripts": { - "postinstall": "node build.js # node-gyp rebuild" + "postinstall": "node install.js" } }), - &[("build.js", BUILD_SCRIPT.as_bytes())], + &[("install.js", BUILD_SCRIPT.as_bytes())], ) .await; let project = TempProject::empty(&project_manifest()); @@ -201,10 +201,10 @@ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500); "name": PACKAGE_NAME, "version": PACKAGE_VERSION, "scripts": { - "postinstall": "node build.js # node-gyp rebuild" + "postinstall": "node install.js" } }), - &[("build.js", script)], + &[("install.js", script)], ) .await; let project = TempProject::empty(&project_manifest()); @@ -259,3 +259,91 @@ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500); "one contender must build and the other must reuse its keyed result" ); } + +#[tokio::test] +async fn concurrent_native_rebuilds_with_different_keys_serialize_shared_package_mutation() { + if !node_available() { + eprintln!("skipping: node is required for the native build-cache workflow"); + return; + } + + let registry = MockRegistry::start().await; + let script = br#" +const fs = require('fs'); +const label = process.env.CFLAGS; +fs.writeFileSync('active-build.txt', label); +Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 6000); +const active = fs.existsSync('active-build.txt') + ? fs.readFileSync('active-build.txt', 'utf8') + : ''; +if (active !== label) { + throw new Error(`shared package tree was mutated by ${active} while ${label} was building`); +} +fs.rmSync('active-build.txt'); +fs.writeFileSync('native-output.txt', label); +"#; + registry + .with_manifest_package( + serde_json::json!({ + "name": PACKAGE_NAME, + "version": PACKAGE_VERSION, + "scripts": { + "postinstall": "node install.js" + } + }), + &[("install.js", script)], + ) + .await; + let project = TempProject::empty(&project_manifest()); + let install = lpm_with_registry(&project, ®istry.url()) + .arg("install") + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("initial install"); + assert!(install.status.success()); + + let mut first_command = lpm_spawnable_with_registry(&project, ®istry.url()); + first_command + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .env("CFLAGS", "build-a"); + let first = first_command.spawn().expect("spawn first rebuild"); + let active_path = installed_package_dir(&project).join("active-build.txt"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !active_path.is_file() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!( + active_path.is_file(), + "first build did not enter its lifecycle script" + ); + let mut second_command = lpm_spawnable_with_registry(&project, ®istry.url()); + second_command + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .env("CFLAGS", "build-b"); + let second = second_command.spawn().expect("spawn second rebuild"); + let first_output = first.wait_with_output().expect("wait for first rebuild"); + let second_output = second.wait_with_output().expect("wait for second rebuild"); + + assert!( + first_output.status.success() && second_output.status.success(), + "different-key rebuilds raced on one package tree:\nfirst stderr:\n{}\nsecond stderr:\n{}", + String::from_utf8_lossy(&first_output.stderr), + String::from_utf8_lossy(&second_output.stderr) + ); + let mut artifact_outputs = std::fs::read_dir(project.store_dir().join("v2/builds")) + .expect("build artifacts") + .map(|entry| { + std::fs::read_to_string( + entry + .expect("build artifact entry") + .path() + .join("package/native-output.txt"), + ) + .expect("each key must preserve its own native output") + }) + .collect::>(); + artifact_outputs.sort_unstable(); + assert_eq!(artifact_outputs, ["build-a", "build-b"]); +} From 0c8853643172b0d94f4ab6fd6c5ebc286f27700a Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 14:37:36 +0100 Subject: [PATCH 05/14] fix: close native build cache input gaps --- ...tive-build-cache-20260712-macos-arm64.json | 52 ++- ...native-build-cache-20260712-macos-arm64.md | 26 +- bench/scripts/README.md | 6 +- .../scripts/native-build-cache-benchmark.mjs | 60 +++- crates/lpm-cli/src/commands/cache_prune.rs | 4 +- .../src/commands/rebuild/build_cache.rs | 332 +++++++++++++++--- crates/lpm-cli/src/commands/rebuild/mod.rs | 18 +- .../src/commands/rebuild/script_execution.rs | 47 ++- crates/lpm-store/src/v2/build_cache.rs | 17 +- crates/lpm-store/src/v2/link_meta.rs | 29 ++ crates/lpm-store/src/v2/store.rs | 12 +- tests/workflows/tests/native_build_cache.rs | 70 +++- 12 files changed, 527 insertions(+), 146 deletions(-) diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64.json b/bench/perf-results/native-build-cache-20260712-macos-arm64.json index 2124357f..4fc474bd 100644 --- a/bench/perf-results/native-build-cache-20260712-macos-arm64.json +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64.json @@ -1,53 +1,49 @@ { - "generated_at": "2026-07-12T11:27:12.697Z", + "generated_at": "2026-07-12T13:19:13.671Z", "platform": "darwin-arm64", "node": "v22.22.1", "lpm_bin": "target/release/lpm-rs", "fixture": "bench/audit-fixtures/native/esbuild-prebuilt", "samples": 10, + "artifact_count": 2, "artifact_bytes": 134444, "scenarios": { - "cache_disabled": { - "wall_ms_median": 112.35, - "wall_ms_p95": 423.22, - "max_rss_bytes_median": 52264960, - "user_cpu_ms_median": 40, - "system_cpu_ms_median": 10, - "cache_hits": 0, - "cache_misses": 0 - }, "cache_miss": { - "wall_ms_median": 137.53, - "wall_ms_p95": 147.84, - "max_rss_bytes_median": 52264960, + "samples": 10, + "wall_ms_median": 138.03, + "wall_ms_p95": 449.94, + "max_rss_bytes_median": 52248576, "user_cpu_ms_median": 50, "system_cpu_ms_median": 20, "cache_hits": 0, - "cache_misses": 10 + "cache_misses": 10, + "lifecycle_ms_avoided": 0 }, "local_hit": { - "wall_ms_median": 29.66, - "wall_ms_p95": 31.39, - "max_rss_bytes_median": 41517056, + "samples": 10, + "wall_ms_median": 32.33, + "wall_ms_p95": 37.27, + "max_rss_bytes_median": 41467904, "user_cpu_ms_median": 10, - "system_cpu_ms_median": 0, + "system_cpu_ms_median": 10, "cache_hits": 10, "cache_misses": 0, - "lifecycle_ms_avoided": 1046, - "speedup_vs_disabled": 3.79, - "wall_reduction_percent": 73.6 + "lifecycle_ms_avoided": 1356, + "speedup_vs_miss": 4.27, + "wall_reduction_percent": 76.58 }, "ci_warm_store_hit": { - "wall_ms_median": 29.19, - "wall_ms_p95": 30.51, - "max_rss_bytes_median": 41533440, + "samples": 10, + "wall_ms_median": 29.5, + "wall_ms_p95": 30.12, + "max_rss_bytes_median": 41418752, "user_cpu_ms_median": 10, - "system_cpu_ms_median": 0, + "system_cpu_ms_median": 10, "cache_hits": 10, "cache_misses": 0, - "lifecycle_ms_avoided": 1046, - "speedup_vs_disabled": 3.85, - "wall_reduction_percent": 74.02 + "lifecycle_ms_avoided": 1053, + "speedup_vs_miss": 4.68, + "wall_reduction_percent": 78.63 } } } diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64.md b/bench/perf-results/native-build-cache-20260712-macos-arm64.md index 4e80cdac..a331a0af 100644 --- a/bench/perf-results/native-build-cache-20260712-macos-arm64.md +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64.md @@ -4,21 +4,21 @@ - Node: v22.22.1 - Fixture: `bench/audit-fixtures/native/esbuild-prebuilt` - Release-build samples per scenario: 10 -- Build artifact size: 134,444 bytes +- Median build artifact size: 134,444 bytes | Scenario | Median wall | p95 wall | Median RSS | User CPU | System CPU | Result | |---|---:|---:|---:|---:|---:|---:| -| Cache disabled | 112.35 ms | 423.22 ms | 52,264,960 B | 40 ms | 10 ms | baseline | -| Cache miss | 137.53 ms | 147.84 ms | 52,264,960 B | 50 ms | 20 ms | 10 misses | -| Local artifact hit | 29.66 ms | 31.39 ms | 41,517,056 B | 10 ms | 0 ms | 3.79×; 73.60% lower wall time | -| Fresh-project warm-store hit | 29.19 ms | 30.51 ms | 41,533,440 B | 10 ms | 0 ms | 3.85×; 74.02% lower wall time | +| Strict cache miss | 138.03 ms | 449.94 ms | 52,248,576 B | 50 ms | 20 ms | 10 misses | +| Local artifact hit | 32.33 ms | 37.27 ms | 41,467,904 B | 10 ms | 10 ms | 4.27×; 76.58% lower wall time | +| Stable-path CI warm-store hit | 29.50 ms | 30.12 ms | 41,418,752 B | 10 ms | 10 ms | 4.68×; 78.63% lower wall time | -The disabled and miss scenarios rematerialized pristine dependencies before -every sample. Local-hit samples rematerialized the same project. CI-like hit -samples created a fresh project while retaining the warm LPM store. Across ten -hits, the artifact metadata recorded 1,046 ms of lifecycle execution avoided. +Every measured scenario used the same strict sandbox. Miss and local-hit +samples rematerialized the same project. CI-like samples deleted and recreated +a checkout at one stable path while retaining the warm LPM store; observable +lifecycle paths are part of the cache key, so moving a checkout intentionally +invalidates the artifact. -The cache miss adds about 25.2 ms over the disabled median for keying, -pristine rematerialization, integrity validation, and atomic publication. This -fixture has a short lifecycle command; native compilation workloads should -have a substantially larger absolute hit benefit. +Across ten local hits, artifact metadata recorded 1,356 ms of lifecycle +execution avoided. Across ten stable-path CI hits, it recorded 1,053 ms +avoided. This fixture has a short lifecycle command; native compilation +workloads should have a substantially larger absolute hit benefit. diff --git a/bench/scripts/README.md b/bench/scripts/README.md index e59e666c..852728ae 100644 --- a/bench/scripts/README.md +++ b/bench/scripts/README.md @@ -2,9 +2,9 @@ ## Native lifecycle-build cache -Build the release CLI, then compare uncached rebuilds, cache misses, local -artifact hits, and CI-like fresh-project hits against the native esbuild -fixture: +Build the release CLI, then compare strict cache misses, local artifact hits, +and CI-like stable-path checkout recreation against the native esbuild fixture. +Every measured scenario uses the same strict sandbox: ```bash cargo build --release --locked -p lpm-cli --bin lpm-rs diff --git a/bench/scripts/native-build-cache-benchmark.mjs b/bench/scripts/native-build-cache-benchmark.mjs index b77663be..8748c0ae 100644 --- a/bench/scripts/native-build-cache-benchmark.mjs +++ b/bench/scripts/native-build-cache-benchmark.mjs @@ -47,24 +47,29 @@ const rows = []; try { runChecked(['install'], primary); for (let sample = 1; sample <= samples; sample += 1) { - rematerialize(primary); - rows.push(measureRebuild('cache_disabled', sample, primary, false)); - fs.rmSync(path.join(lpmHome, 'store/v2/builds'), { recursive: true, force: true }); rematerialize(primary); - rows.push(measureRebuild('cache_miss', sample, primary, true)); + rows.push(measureRebuild('cache_miss', sample, primary)); rematerialize(primary); - rows.push(measureRebuild('local_hit', sample, primary, true)); + rows.push(measureRebuild('local_hit', sample, primary)); - const ciProject = path.join(tempRoot, `ci-project-${sample}`); + const ciProject = path.join(tempRoot, 'ci-project'); + fs.rmSync(ciProject, { recursive: true, force: true }); materializeProject(ciProject); runChecked(['install'], ciProject); - rows.push(measureRebuild('ci_warm_store_hit', sample, ciProject, true)); + runChecked(['rebuild', '--all', '--strict-sandbox'], ciProject); + fs.rmSync(ciProject, { recursive: true, force: true }); + materializeProject(ciProject); + runChecked(['install'], ciProject); + rows.push(measureRebuild('ci_warm_store_hit', sample, ciProject)); } - const artifactBytes = directorySize(path.join(lpmHome, 'store/v2/builds')); - const summary = summarize(rows, artifactBytes); + const artifactSizes = childDirectorySizes(path.join(lpmHome, 'store/v2/builds')); + if (artifactSizes.length === 0) { + throw new Error('benchmark completed without publishing a build artifact'); + } + const summary = summarize(rows, artifactSizes); fs.mkdirSync(outputDir, { recursive: true }); fs.writeFileSync(path.join(outputDir, 'rows.json'), `${JSON.stringify(rows, null, 2)}\n`); fs.writeFileSync(path.join(outputDir, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`); @@ -87,9 +92,8 @@ function rematerialize(projectDir) { runChecked(['install', '--force', '--no-frozen-lockfile'], projectDir); } -function measureRebuild(scenario, sample, cwd, strict) { - const args = ['--json', 'rebuild', '--all']; - if (strict) args.push('--strict-sandbox'); +function measureRebuild(scenario, sample, cwd) { + const args = ['--json', 'rebuild', '--all', '--strict-sandbox']; const started = process.hrtime.bigint(); const result = runTimed(args, cwd); const wallMs = Number(process.hrtime.bigint() - started) / 1e6; @@ -99,6 +103,15 @@ function measureRebuild(scenario, sample, cwd, strict) { ); } const envelope = JSON.parse(result.stdout); + const expectedHit = scenario.endsWith('_hit'); + if ( + (expectedHit && (envelope.build_cache.hits !== 1 || envelope.build_cache.misses !== 0)) || + (!expectedHit && (envelope.build_cache.hits !== 0 || envelope.build_cache.misses !== 1)) + ) { + throw new Error( + `${scenario} produced unexpected cache metrics: ${JSON.stringify(envelope.build_cache)}`, + ); + } return { scenario, sample, @@ -167,7 +180,7 @@ function runTimed(args, cwd) { return { ...result, userCpuMs: null, systemCpuMs: null, maxRssBytes: null }; } -function summarize(allRows, artifactBytes) { +function summarize(allRows, artifactSizes) { const scenarios = {}; for (const scenario of [...new Set(allRows.map((row) => row.scenario))]) { const selected = allRows.filter((row) => row.scenario === scenario); @@ -191,9 +204,9 @@ function summarize(allRows, artifactBytes) { ), }; } - const baseline = scenarios.cache_disabled.wall_ms_median; + const baseline = scenarios.cache_miss.wall_ms_median; for (const scenario of ['local_hit', 'ci_warm_store_hit']) { - scenarios[scenario].speedup_vs_disabled = round(baseline / scenarios[scenario].wall_ms_median); + scenarios[scenario].speedup_vs_miss = round(baseline / scenarios[scenario].wall_ms_median); scenarios[scenario].wall_reduction_percent = round( 100 * (1 - scenarios[scenario].wall_ms_median / baseline), ); @@ -205,7 +218,8 @@ function summarize(allRows, artifactBytes) { lpm_bin: lpmBin, fixture: path.relative(repoRoot, fixture), samples, - artifact_bytes: artifactBytes, + artifact_count: artifactSizes.length, + artifact_bytes: percentile(artifactSizes.sort((a, b) => a - b), 0.5), scenarios, }; } @@ -217,19 +231,19 @@ function renderMarkdown(summary) { `- Platform: ${summary.platform}`, `- Fixture: \`${summary.fixture}\``, `- Samples per scenario: ${summary.samples}`, - `- Build artifact bytes: ${summary.artifact_bytes}`, + `- Median build artifact bytes: ${summary.artifact_bytes} (${summary.artifact_count} keys retained)`, '', '| Scenario | Median wall | p95 wall | Median RSS | Hits | Misses | Speedup |', '|---|---:|---:|---:|---:|---:|---:|', ]; for (const [name, value] of Object.entries(summary.scenarios)) { lines.push( - `| ${name} | ${value.wall_ms_median} ms | ${value.wall_ms_p95} ms | ${value.max_rss_bytes_median ?? 'n/a'} B | ${value.cache_hits} | ${value.cache_misses} | ${value.speedup_vs_disabled ? `${value.speedup_vs_disabled}×` : '—'} |`, + `| ${name} | ${value.wall_ms_median} ms | ${value.wall_ms_p95} ms | ${value.max_rss_bytes_median ?? 'n/a'} B | ${value.cache_hits} | ${value.cache_misses} | ${value.speedup_vs_miss ? `${value.speedup_vs_miss}×` : '—'} |`, ); } lines.push( '', - 'The disabled and miss scenarios rematerialize pristine dependencies before each sample. Local-hit samples rematerialize the same project; CI-hit samples create a fresh project while retaining the warm LPM store.', + 'Every measured scenario uses the same strict sandbox. Miss and local-hit samples rematerialize the same project; CI-hit samples recreate a checkout at the same stable path while retaining the warm LPM store.', ); return lines.join('\n'); } @@ -254,6 +268,14 @@ function directorySize(root) { return total; } +function childDirectorySizes(root) { + if (!fs.existsSync(root)) return []; + return fs + .readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => directorySize(path.join(root, entry.name))); +} + function round(value) { return Math.round(value * 100) / 100; } diff --git a/crates/lpm-cli/src/commands/cache_prune.rs b/crates/lpm-cli/src/commands/cache_prune.rs index 035aaa54..fd9d0ce0 100644 --- a/crates/lpm-cli/src/commands/cache_prune.rs +++ b/crates/lpm-cli/src/commands/cache_prune.rs @@ -1345,8 +1345,8 @@ mod tests { let orphan_key = "a".repeat(64); let live_key = "b".repeat(64); std::fs::create_dir_all(store.paths().build_entry_locks_root()).unwrap(); - let orphan_lock = store.paths().build_entry_lock_path(&orphan_key); - let live_lock = store.paths().build_entry_lock_path(&live_key); + let orphan_lock = store.paths().build_entry_lock_path(&orphan_key).unwrap(); + let live_lock = store.paths().build_entry_lock_path(&live_key).unwrap(); std::fs::write(&orphan_lock, b"").unwrap(); std::fs::write(format!("{}.writer-intent", orphan_lock.display()), b"").unwrap(); std::fs::write(format!("{}.writer-queue", orphan_lock.display()), b"").unwrap(); diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache.rs b/crates/lpm-cli/src/commands/rebuild/build_cache.rs index 8e13d38c..ca7b13b3 100644 --- a/crates/lpm-cli/src/commands/rebuild/build_cache.rs +++ b/crates/lpm-cli/src/commands/rebuild/build_cache.rs @@ -1,3 +1,4 @@ +use super::script_execution::{build_lifecycle_environment, build_lifecycle_path}; use super::scripts::ScriptablePackage; use crate::capability::CapabilitySet; use lpm_sandbox::{SandboxMode, SandboxOptions, SandboxPosture}; @@ -18,12 +19,35 @@ pub(super) struct BuildCacheScratch { } impl BuildCacheScratch { - pub(super) fn create(package_dir: &Path) -> std::io::Result { - let link_dir = package_dir + fn path_for(package_dir: &Path, graph_key_digest: &str) -> std::io::Result { + let node_modules_dir = package_dir .ancestors() - .find(|ancestor| ancestor.join(lpm_store::v2::LINK_META_FILENAME).is_file()) - .ok_or_else(|| std::io::Error::other("package is not inside a v2 link entry"))?; - let path = link_dir.join(".lpm-build-tmp"); + .find(|ancestor| { + ancestor + .file_name() + .is_some_and(|name| name == "node_modules") + }) + .ok_or_else(|| std::io::Error::other("package is not inside a node_modules tree"))?; + let link_dir = node_modules_dir + .parent() + .ok_or_else(|| std::io::Error::other("package has no v2 link-entry parent"))?; + let metadata = lpm_store::v2::LinkMeta::read_from(link_dir) + .map_err(|error| std::io::Error::other(error.to_string()))?; + if metadata.graph_key_digest_hex != graph_key_digest { + return Err(std::io::Error::other( + "package graph identity does not match its v2 link sidecar", + )); + } + if node_modules_dir.join(&metadata.name) != package_dir { + return Err(std::io::Error::other( + "package path does not match its v2 link sidecar", + )); + } + Ok(link_dir.join(".lpm-build-tmp")) + } + + pub(super) fn create(package_dir: &Path, graph_key_digest: &str) -> std::io::Result { + let path = Self::path_for(package_dir, graph_key_digest)?; match std::fs::symlink_metadata(&path) { Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => { std::fs::remove_dir_all(&path)?; @@ -58,8 +82,8 @@ pub(super) struct BuildCacheInvocation { platform: BuildPlatformFingerprint, runtime: BuildRuntimeFingerprint, sandbox: BuildSandboxFingerprint, - environment_hash: String, environment: HashMap, + toolchain_environment: HashMap, native_toolchain_hash: OnceLock>, } @@ -68,6 +92,7 @@ impl BuildCacheInvocation { pub(super) fn prepare( lockfile: &lpm_lockfile::Lockfile, environment: &HashMap, + project_dir: &Path, sandbox_mode: SandboxMode, posture: &SandboxPosture, sandbox_options: &SandboxOptions, @@ -96,6 +121,16 @@ impl BuildCacheInvocation { return None; }; let platform = PlatformTuple::current(); + let mut toolchain_environment = environment.clone(); + let parent_path = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("PATH")) + .map(|(_, value)| value.as_str()); + toolchain_environment.retain(|key, _| !key.eq_ignore_ascii_case("PATH")); + toolchain_environment.insert( + "PATH".into(), + build_lifecycle_path(project_dir, parent_path), + ); Some(Self { dependency_closure_hash: hash_lockfile_graph(lockfile), platform: BuildPlatformFingerprint { @@ -112,8 +147,8 @@ impl BuildCacheInvocation { environment_scrubbed: true, allowed_inputs_hash: hash_allowed_inputs(capabilities), }, - environment_hash: hash_build_environment(environment), environment: environment.clone(), + toolchain_environment, native_toolchain_hash: OnceLock::new(), }) } @@ -294,6 +329,7 @@ pub(super) fn read_build_marker_key(marker_path: &Path) -> Option { pub(super) fn build_key_for_package( invocation: &BuildCacheInvocation, package: &ScriptablePackage, + project_dir: &Path, ) -> Option { let Some(graph_key_digest) = package.graph_key_digest.as_ref() else { debug_bypass(&format!("{} has no v2 graph identity", package.name)); @@ -306,6 +342,18 @@ pub(super) fn build_key_for_package( )); return None; }; + let scratch_path = match BuildCacheScratch::path_for(&package.store_path, graph_key_digest) { + Ok(path) => path, + Err(error) => { + debug_bypass(&format!( + "{} has invalid v2 build-cache context: {error}", + package.name + )); + return None; + } + }; + let lifecycle_environment = + build_lifecycle_environment(&invocation.environment, project_dir, &scratch_path); let mut scripts = Vec::with_capacity(EXECUTED_INSTALL_PHASES.len()); for phase in EXECUTED_INSTALL_PHASES { if let Some(command) = package.scripts.get(*phase) { @@ -318,17 +366,20 @@ pub(super) fn build_key_for_package( let toolchain_hash = if native_kind == NativeBuildKind::NativeToolchain { let Some(hash) = invocation .native_toolchain_hash - .get_or_init(|| hash_toolchain(&invocation.environment).ok()) + .get_or_init(|| hash_toolchain(&invocation.toolchain_environment).ok()) .clone() else { debug_bypass("native toolchain fingerprint unavailable"); return None; }; - hash + let invoked_executables = + hash_invoked_build_executables(package, &invocation.toolchain_environment); + hash_records([hash.as_bytes(), invoked_executables.as_bytes()]) } else { "runtime-and-dependency-graph-v1".to_string() }; - Some(BuildCacheKey::derive(&BuildKeyInputs { + let environment_hash = hash_environment_pairs(&lifecycle_environment); + let key = BuildCacheKey::derive(&BuildKeyInputs { source_integrity: package.source_integrity.clone(), graph_key_digest: graph_key_digest.clone(), dependency_closure_hash: invocation.dependency_closure_hash.clone(), @@ -336,9 +387,17 @@ pub(super) fn build_key_for_package( platform: invocation.platform.clone(), runtime: invocation.runtime.clone(), sandbox: invocation.sandbox.clone(), - environment_hash: invocation.environment_hash.clone(), + environment_hash: environment_hash.clone(), toolchain_hash, - })) + }); + tracing::debug!( + target: "lpm_cli::build_cache", + package = %package.name, + %environment_hash, + build_key = %key.as_str(), + "derived native build-cache key" + ); + Some(key) } pub(super) fn read_v2_graph_key_digest(package_dir: &Path) -> Option { @@ -413,36 +472,29 @@ fn hash_lockfile_graph(lockfile: &lpm_lockfile::Lockfile) -> String { hash_records(rows.iter().map(String::as_bytes)) } +#[cfg(test)] fn hash_build_environment(environment: &HashMap) -> String { - const BUILD_ENVIRONMENT: &[&str] = &[ - "AR", - "CC", - "CFLAGS", - "CPPFLAGS", - "CXX", - "CXXFLAGS", - "LDFLAGS", - "MACOSX_DEPLOYMENT_TARGET", - "MAKEFLAGS", - "npm_config_arch", - "npm_config_build_from_source", - "npm_config_devdir", - "npm_config_libc", - "npm_config_nodedir", - "npm_config_node_gyp", - "npm_config_runtime", - "npm_config_target", - "npm_config_target_arch", - ]; - let mut records = Vec::with_capacity(BUILD_ENVIRONMENT.len()); - for name in BUILD_ENVIRONMENT { - let value = environment - .iter() - .find(|(key, _)| key.eq_ignore_ascii_case(name)) - .map_or("", |(_, value)| value.as_str()); - records.push(format!("{name}\0{value}")); - } - hash_records(records.iter().map(String::as_bytes)) + let mut records = environment.iter().collect::>(); + records.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + hash_records(records.into_iter().map(|(name, value)| { + let mut record = String::with_capacity(name.len() + value.len() + 1); + record.push_str(name); + record.push('\0'); + record.push_str(value); + record + })) +} + +fn hash_environment_pairs(environment: &[(String, String)]) -> String { + let mut records = environment.iter().collect::>(); + records.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + hash_records(records.into_iter().map(|(name, value)| { + let mut record = String::with_capacity(name.len() + value.len() + 1); + record.push_str(name); + record.push('\0'); + record.push_str(value); + record + })) } fn hash_toolchain(environment: &HashMap) -> std::io::Result { @@ -457,9 +509,14 @@ fn hash_toolchain(environment: &HashMap) -> std::io::Result) -> std::io::Result) -> std::io::Result std::io::Result<()> { + hash_os_path(hasher, path); + match std::fs::File::open(path) { + Ok(mut file) => { + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => hasher.update(b"absent"), + Err(error) => return Err(error), + } + hasher.update(b"\x1e"); + Ok(()) +} + +fn hash_invoked_build_executables( + package: &ScriptablePackage, + environment: &HashMap, +) -> String { + let mut identities = Vec::new(); + for command in package.scripts.values() { + let Some(segments) = shell_command_segments(command) else { + continue; + }; + for words in segments { + let Some(command_index) = words + .iter() + .position(|word| !is_environment_assignment(word)) + else { + continue; + }; + let executable = &words[command_index]; + let Some(path) = resolve_executable(executable, environment) else { + continue; + }; + if let Ok(identity) = runtime_executable_identity(&path) { + identities.push(format!("{executable}\0{identity}")); + } + } + } + identities.sort_unstable(); + hash_records(identities.iter().map(String::as_bytes)) +} + +fn resolve_executable(executable: &str, environment: &HashMap) -> Option { + let executable_path = Path::new(executable); + if executable_path.components().count() > 1 { + return executable_path + .is_file() + .then(|| executable_path.to_path_buf()); + } + let path = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("PATH")) + .map(|(_, value)| value)?; + std::env::split_paths(path) + .map(|directory| directory.join(executable)) + .find(|candidate| candidate.is_file()) +} + fn hash_directory_tree(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { hasher.update(b"directory-tree-v1\0"); if !root.exists() { @@ -569,15 +703,20 @@ fn hash_allowed_inputs(capabilities: &CapabilitySet) -> String { let records = [ capabilities.canonical_hash(), "package-local-writes".into(), - "graph-local-reads".into(), + "graph-and-fingerprinted-system-reads".into(), "network-denied".into(), ]; hash_records(records.iter().map(String::as_bytes)) } -fn hash_records<'a>(records: impl IntoIterator) -> String { +fn hash_records(records: T) -> String +where + T: IntoIterator, + B: AsRef<[u8]>, +{ let mut hasher = Sha256::new(); for record in records { + let record = record.as_ref(); hasher.update((record.len() as u64).to_le_bytes()); hasher.update(record); } @@ -656,13 +795,36 @@ mod tests { lockfile } + fn write_link_meta(link_dir: &Path, package_name: &str) -> String { + let inputs = lpm_store::v2::GraphKeyInputs::new( + package_name, + "1.0.0", + lpm_store::v2::PlatformTuple::current(), + lpm_store::v2::LinkerModeTag::Isolated, + ); + let key = lpm_store::v2::GraphKey::derive(&inputs); + let metadata = lpm_store::v2::LinkMeta::new( + &key, + "sha512-source", + "objects/sha512-source", + Vec::new(), + std::sync::Arc::new(lpm_store::v2::LinkMetaPlatform { + os: std::env::consts::OS.into(), + cpu: std::env::consts::ARCH.into(), + libc: None, + }), + ); + metadata.write_to(link_dir).unwrap(); + metadata.graph_key_digest_hex + } + #[test] - fn build_environment_hash_ignores_unrelated_values() { + fn build_environment_hash_changes_with_custom_values() { let mut first = HashMap::new(); first.insert("UNRELATED".into(), "one".into()); let mut second = HashMap::new(); second.insert("UNRELATED".into(), "two".into()); - assert_eq!( + assert_ne!( hash_build_environment(&first), hash_build_environment(&second) ); @@ -717,6 +879,50 @@ mod tests { assert_ne!(first.finalize(), second.finalize()); } + #[test] + #[cfg(unix)] + fn toolchain_hash_changes_when_cmake_executable_changes() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let bin = temp.path().join("bin"); + std::fs::create_dir(&bin).unwrap(); + let cmake = bin.join("cmake"); + std::fs::write(&cmake, "#!/bin/sh\necho cmake-one\n").unwrap(); + std::fs::set_permissions(&cmake, std::fs::Permissions::from_mode(0o755)).unwrap(); + let environment = HashMap::from([ + ("PATH".to_string(), bin.display().to_string()), + ("HOME".to_string(), temp.path().display().to_string()), + ]); + let first = hash_toolchain(&environment).unwrap(); + + std::fs::write(&cmake, "#!/bin/sh\necho cmake-two\n").unwrap(); + let second = hash_toolchain(&environment).unwrap(); + + assert_ne!(first, second); + } + + #[test] + #[cfg(unix)] + fn invoked_tool_hash_changes_when_cmake_js_executable_changes() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let bin = temp.path().join("bin"); + std::fs::create_dir(&bin).unwrap(); + let cmake_js = bin.join("cmake-js"); + std::fs::write(&cmake_js, "#!/bin/sh\necho cmake-js-one\n").unwrap(); + std::fs::set_permissions(&cmake_js, std::fs::Permissions::from_mode(0o755)).unwrap(); + let environment = HashMap::from([("PATH".to_string(), bin.display().to_string())]); + let package = scriptable_package("fixture", "cmake-js compile"); + let first = hash_invoked_build_executables(&package, &environment); + + std::fs::write(&cmake_js, "#!/bin/sh\necho cmake-js-two\n").unwrap(); + let second = hash_invoked_build_executables(&package, &environment); + + assert_ne!(first, second); + } + #[test] fn lockfile_graph_hash_changes_when_dependency_edge_is_rewired() { let first = graph_lockfile(&["leaf@1.0.0"], &[]); @@ -748,20 +954,44 @@ mod tests { let link_dir = temp.path().join("links/example"); let package_dir = link_dir.join("node_modules/example"); std::fs::create_dir_all(&package_dir).unwrap(); - std::fs::write(link_dir.join(lpm_store::v2::LINK_META_FILENAME), b"{}").unwrap(); + let digest = write_link_meta(&link_dir, "example"); let first_path = { - let scratch = BuildCacheScratch::create(&package_dir).unwrap(); + let scratch = BuildCacheScratch::create(&package_dir, &digest).unwrap(); scratch.path().to_path_buf() }; let second_path = { - let scratch = BuildCacheScratch::create(&package_dir).unwrap(); + let scratch = BuildCacheScratch::create(&package_dir, &digest).unwrap(); scratch.path().to_path_buf() }; assert_eq!(first_path, second_path); } + #[test] + fn build_cache_scratch_supports_scoped_package_path() { + let temp = tempfile::tempdir().unwrap(); + let link_dir = temp.path().join("links/scoped"); + let package_dir = link_dir.join("node_modules/@scope/example"); + std::fs::create_dir_all(&package_dir).unwrap(); + let digest = write_link_meta(&link_dir, "@scope/example"); + + let scratch = BuildCacheScratch::create(&package_dir, &digest).unwrap(); + + assert_eq!(scratch.path(), link_dir.join(".lpm-build-tmp")); + } + + #[test] + fn build_cache_scratch_rejects_package_controlled_sidecar() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("links/example/node_modules/example"); + std::fs::create_dir_all(&package_dir).unwrap(); + std::fs::write(package_dir.join(lpm_store::v2::LINK_META_FILENAME), b"{}").unwrap(); + + assert!(BuildCacheScratch::create(&package_dir, &"a".repeat(64)).is_err()); + assert!(!package_dir.join(".lpm-build-tmp").exists()); + } + #[test] #[cfg(unix)] fn build_cache_scratch_replaces_symlink_without_touching_target() { @@ -774,10 +1004,10 @@ mod tests { std::fs::create_dir_all(&package_dir).unwrap(); std::fs::create_dir_all(&outside).unwrap(); std::fs::write(outside.join("sentinel"), b"keep").unwrap(); - std::fs::write(link_dir.join(lpm_store::v2::LINK_META_FILENAME), b"{}").unwrap(); + let digest = write_link_meta(&link_dir, "example"); symlink(&outside, link_dir.join(".lpm-build-tmp")).unwrap(); - let scratch = BuildCacheScratch::create(&package_dir).unwrap(); + let scratch = BuildCacheScratch::create(&package_dir, &digest).unwrap(); assert!(outside.join("sentinel").is_file()); assert_ne!(std::fs::canonicalize(scratch.path()).unwrap(), outside); diff --git a/crates/lpm-cli/src/commands/rebuild/mod.rs b/crates/lpm-cli/src/commands/rebuild/mod.rs index fc355692..f8dbc3c4 100644 --- a/crates/lpm-cli/src/commands/rebuild/mod.rs +++ b/crates/lpm-cli/src/commands/rebuild/mod.rs @@ -1009,6 +1009,7 @@ async fn run_under_store_lock( let build_cache_invocation = BuildCacheInvocation::prepare( &lockfile, &sanitized_env, + project_dir, sandbox_mode, &effective_sandbox_posture, &sandbox_options, @@ -1086,7 +1087,7 @@ async fn run_under_store_lock( let key_start = std::time::Instant::now(); let mut build_key = build_cache_invocation .as_ref() - .and_then(|invocation| build_key_for_package(invocation, pkg)); + .and_then(|invocation| build_key_for_package(invocation, pkg, project_dir)); build_cache_metrics.key_ms += elapsed_millis(key_start.elapsed()); if build_key.is_some() { build_cache_metrics.eligible += 1; @@ -1095,9 +1096,11 @@ async fn run_under_store_lock( } let v2_store = lpm_store::v2::Store::from_lpm_root(&lpm_root); let _build_entry_lock = if let Some(graph_key_digest) = pkg.graph_key_digest.as_deref() { - match lpm_common::acquire_exclusive_lock( - v2_store.paths().build_entry_lock_path(graph_key_digest), - ) { + match v2_store + .paths() + .build_entry_lock_path(graph_key_digest) + .and_then(lpm_common::acquire_exclusive_lock) + { Ok(lock) => Some(lock), Err(error) => { if !json_output { @@ -1299,7 +1302,12 @@ async fn run_under_store_lock( } let build_cache_scratch = if build_key.is_some() { - match BuildCacheScratch::create(&pkg.store_path) { + let scratch = pkg + .graph_key_digest + .as_deref() + .ok_or_else(|| std::io::Error::other("package has no v2 graph identity")) + .and_then(|digest| BuildCacheScratch::create(&pkg.store_path, digest)); + match scratch { Ok(scratch) => Some(scratch), Err(error) => { tracing::warn!( diff --git a/crates/lpm-cli/src/commands/rebuild/script_execution.rs b/crates/lpm-cli/src/commands/rebuild/script_execution.rs index 8bf750b4..e2902dfa 100644 --- a/crates/lpm-cli/src/commands/rebuild/script_execution.rs +++ b/crates/lpm-cli/src/commands/rebuild/script_execution.rs @@ -51,25 +51,7 @@ pub(super) fn execute_script( // worked accidentally because `std::process::Command::env` on // Windows does case-insensitive deduplication and our explicit // `"PATH"` overrides won, but the LOOKUP path was still broken. - let parent_path = find_env_case_insensitive(env, "PATH"); - let path_value = build_lifecycle_path(project_dir, parent_path); - let mut envs: Vec<(String, String)> = env - .iter() - .filter(|(k, _)| { - !k.eq_ignore_ascii_case("PATH") - && !k.eq_ignore_ascii_case("INIT_CWD") - && !k.eq_ignore_ascii_case("TMPDIR") - && !k.eq_ignore_ascii_case("TMP") - && !k.eq_ignore_ascii_case("TEMP") - }) - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - envs.push(("INIT_CWD".to_string(), project_dir.display().to_string())); - envs.push(("PATH".to_string(), path_value)); - let tmp = tmpdir.display().to_string(); - envs.push(("TMPDIR".to_string(), tmp.clone())); - envs.push(("TMP".to_string(), tmp.clone())); - envs.push(("TEMP".to_string(), tmp)); + let envs = build_lifecycle_environment(env, project_dir, tmpdir); let start = std::time::Instant::now(); @@ -107,6 +89,33 @@ pub(super) fn execute_script( } } +pub(super) fn build_lifecycle_environment( + env: &HashMap, + project_dir: &Path, + tmpdir: &Path, +) -> Vec<(String, String)> { + let parent_path = find_env_case_insensitive(env, "PATH"); + let path_value = build_lifecycle_path(project_dir, parent_path); + let mut envs: Vec<(String, String)> = env + .iter() + .filter(|(k, _)| { + !k.eq_ignore_ascii_case("PATH") + && !k.eq_ignore_ascii_case("INIT_CWD") + && !k.eq_ignore_ascii_case("TMPDIR") + && !k.eq_ignore_ascii_case("TMP") + && !k.eq_ignore_ascii_case("TEMP") + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + envs.push(("INIT_CWD".to_string(), project_dir.display().to_string())); + envs.push(("PATH".to_string(), path_value)); + let tmp = tmpdir.display().to_string(); + envs.push(("TMPDIR".to_string(), tmp.clone())); + envs.push(("TMP".to_string(), tmp.clone())); + envs.push(("TEMP".to_string(), tmp)); + envs +} + /// — resolve the live per-package directory where lifecycle /// scripts should `current_dir` to. /// diff --git a/crates/lpm-store/src/v2/build_cache.rs b/crates/lpm-store/src/v2/build_cache.rs index dfa1b19d..b1b16dc3 100644 --- a/crates/lpm-store/src/v2/build_cache.rs +++ b/crates/lpm-store/src/v2/build_cache.rs @@ -92,7 +92,7 @@ pub struct BuildKeyInputs { pub runtime: BuildRuntimeFingerprint, /// Effective sandbox fingerprint. pub sandbox: BuildSandboxFingerprint, - /// Digest of explicitly permitted build-environment values. + /// Digest of the complete environment exposed to the lifecycle script. pub environment_hash: String, /// Digest of discoverable compiler, SDK, and build-tool identities. pub toolchain_hash: String, @@ -645,13 +645,26 @@ mod tests { let graph_key = "a".repeat(64); assert_eq!( - store.paths().build_entry_lock_path(&graph_key), + store.paths().build_entry_lock_path(&graph_key).unwrap(), temp.path() .join("store/build-entry-locks") .join(format!("{graph_key}.lock")) ); } + #[test] + fn build_entry_lock_path_rejects_traversal_digest() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + + assert!( + store + .paths() + .build_entry_lock_path("../../outside") + .is_err() + ); + } + #[test] fn build_artifact_corruption_is_a_cache_miss() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/lpm-store/src/v2/link_meta.rs b/crates/lpm-store/src/v2/link_meta.rs index f4062f9c..2df1d7ad 100644 --- a/crates/lpm-store/src/v2/link_meta.rs +++ b/crates/lpm-store/src/v2/link_meta.rs @@ -265,6 +265,12 @@ impl LinkMeta { parsed.name ))); } + if !is_lower_hex_digest(&parsed.graph_key_digest_hex) { + return Err(LpmError::Store(format!( + "v2 link sidecar at {} has invalid graph-key digest", + path.display() + ))); + } Ok(parsed) } @@ -399,6 +405,13 @@ fn validate_path_segment(s: &str) -> Result<(), &'static str> { Ok(()) } +pub(crate) fn is_lower_hex_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + #[cfg(test)] mod tests { use super::*; @@ -684,4 +697,20 @@ mod tests { let reconstructed = GraphKey::from_recorded(&meta.name, &meta.version, digest_bytes); assert_eq!(reconstructed, key); } + + #[test] + fn read_from_rejects_graph_key_digest_outside_exact_lower_hex_shape() { + for invalid in ["../../outside".into(), "A".repeat(64), "a".repeat(63)] { + let dir = tempfile::tempdir().unwrap(); + let mut meta = sample_meta(); + meta.graph_key_digest_hex = invalid; + std::fs::write( + dir.path().join(LINK_META_FILENAME), + serde_json::to_vec(&meta).unwrap(), + ) + .unwrap(); + + assert!(LinkMeta::read_from(dir.path()).is_err()); + } + } } diff --git a/crates/lpm-store/src/v2/store.rs b/crates/lpm-store/src/v2/store.rs index 9469cbc9..46f134a7 100644 --- a/crates/lpm-store/src/v2/store.rs +++ b/crates/lpm-store/src/v2/store.rs @@ -256,9 +256,15 @@ impl StoreV2Paths { /// Advisory lock path serializing mutation of one v2 graph entry. #[inline] - pub fn build_entry_lock_path(&self, graph_key_digest: &str) -> PathBuf { - self.build_entry_locks_root - .join(format!("{graph_key_digest}.lock")) + pub fn build_entry_lock_path(&self, graph_key_digest: &str) -> Result { + if !crate::v2::link_meta::is_lower_hex_digest(graph_key_digest) { + return Err(LpmError::Store( + "invalid graph-key digest for build-entry lock".into(), + )); + } + Ok(self + .build_entry_locks_root + .join(format!("{graph_key_digest}.lock"))) } /// `~/.lpm/store/v2/build-entry-locks/` — per-graph-entry advisory locks. diff --git a/tests/workflows/tests/native_build_cache.rs b/tests/workflows/tests/native_build_cache.rs index 4a6d7b6a..1d3caea0 100644 --- a/tests/workflows/tests/native_build_cache.rs +++ b/tests/workflows/tests/native_build_cache.rs @@ -1,6 +1,6 @@ //! Workflow coverage for dependency lifecycle-build artifact reuse. -#![cfg(target_os = "macos")] +#![cfg(any(target_os = "macos", target_os = "linux"))] mod support; @@ -182,6 +182,74 @@ async fn native_lifecycle_output_is_restored_after_pristine_rematerialization() ); } +#[tokio::test] +async fn custom_lifecycle_environment_change_invalidates_cached_output() { + if !node_available() { + eprintln!("skipping: node is required for the native build-cache workflow"); + return; + } + + let registry = MockRegistry::start().await; + let script = br#" +const fs = require('fs'); +fs.writeFileSync('native-output.txt', process.env.LPM_NATIVE_TEST_INPUT); +"#; + registry + .with_manifest_package( + serde_json::json!({ + "name": PACKAGE_NAME, + "version": PACKAGE_VERSION, + "scripts": { + "postinstall": "node install.js" + } + }), + &[("install.js", script)], + ) + .await; + let project = TempProject::empty(&project_manifest()); + let install = lpm_with_registry(&project, ®istry.url()) + .arg("install") + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("initial install"); + assert!(install.status.success()); + + for value in ["first", "second"] { + let rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .env("LPM_NATIVE_TEST_INPUT", value) + .output() + .expect("native rebuild"); + assert!( + rebuild.status.success(), + "rebuild failed for {value}: {}", + String::from_utf8_lossy(&rebuild.stderr) + ); + let envelope: serde_json::Value = + serde_json::from_slice(&rebuild.stdout).expect("native rebuild stdout must be JSON"); + assert_eq!( + envelope["build_cache"]["misses"], + 1, + "{value} must execute under its own cache key; envelope: {envelope}; stderr: {}", + String::from_utf8_lossy(&rebuild.stderr) + ); + } + + assert_eq!( + std::fs::read_to_string(installed_package_dir(&project).join("native-output.txt")) + .expect("second build output"), + "second" + ); + assert_eq!( + std::fs::read_dir(project.store_dir().join("v2/builds")) + .expect("build artifacts") + .count(), + 2, + "each visible custom environment must have a distinct cache key" + ); +} + #[tokio::test] async fn concurrent_native_rebuilds_execute_one_lifecycle_build_per_key() { if !node_available() { From 98c76660996bfb51b1b62ba33da0e8a3f97c8e73 Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 15:40:30 +0100 Subject: [PATCH 06/14] fix: fingerprint native host build state --- .github/workflows/ci.yml | 3 + .../src/commands/rebuild/build_cache.rs | 257 +++++++++++++++++- 2 files changed, 246 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2f71ee0..1df2bc11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,6 +154,9 @@ jobs: - name: CLI binary-surface tests run: cargo nextest run --locked -p lpm-cli -E 'not binary_id(/bin\/lpm-rs/)' --no-fail-fast --status-level slow --final-status-level fail + - name: Native build cache workflow (Linux sandbox) + run: cargo nextest run --locked -p lpm-workflows --test native_build_cache --no-fail-fast --status-level slow --final-status-level fail + macos-workspace: name: macOS workspace checks runs-on: macos-latest diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache.rs b/crates/lpm-cli/src/commands/rebuild/build_cache.rs index ca7b13b3..88ff4b21 100644 --- a/crates/lpm-cli/src/commands/rebuild/build_cache.rs +++ b/crates/lpm-cli/src/commands/rebuild/build_cache.rs @@ -401,10 +401,14 @@ pub(super) fn build_key_for_package( } pub(super) fn read_v2_graph_key_digest(package_dir: &Path) -> Option { - let link_dir = package_dir.parent()?.parent()?; - lpm_store::v2::LinkMeta::read_from(link_dir) - .ok() - .map(|metadata| metadata.graph_key_digest_hex) + let node_modules_dir = package_dir.ancestors().find(|ancestor| { + ancestor + .file_name() + .is_some_and(|name| name == "node_modules") + })?; + let link_dir = node_modules_dir.parent()?; + let metadata = lpm_store::v2::LinkMeta::read_from(link_dir).ok()?; + (node_modules_dir.join(&metadata.name) == package_dir).then_some(metadata.graph_key_digest_hex) } fn detect_node_runtime(environment: &HashMap) -> Option { @@ -502,7 +506,11 @@ fn hash_toolchain(environment: &HashMap) -> std::io::Result) -> std::io::Result>(); + handles + .into_iter() + .map(|handle| handle.join().ok().flatten()) + .collect::>() + }); + for ((program, _), output) in PROBES.iter().zip(probe_outputs) { hasher.update(program.as_bytes()); hasher.update(b"\0"); - if let Ok(output) = Command::new(program) - .args(*args) - .env_clear() - .envs(environment) - .output() - { - hasher.update(&output.stdout[..output.stdout.len().min(16 * 1024)]); - hasher.update(&output.stderr[..output.stderr.len().min(16 * 1024)]); + if let Some(output) = output { + hasher.update(&output.stdout[..output.stdout.len().min(4 * 1024 * 1024)]); + hasher.update(&output.stderr[..output.stderr.len().min(4 * 1024 * 1024)]); } if let Some(executable) = resolve_executable(program, environment) && let Ok(identity) = runtime_executable_identity(&executable) @@ -543,6 +584,26 @@ fn hash_toolchain(environment: &HashMap) -> std::io::Result>(); + hash_host_package_state(&host_state_paths, &mut hasher)?; + hash_pkg_config_directories(environment, &mut hasher)?; + for cellar in [ + Path::new("/opt/homebrew/Cellar"), + Path::new("/usr/local/Cellar"), + Path::new("/home/linuxbrew/.linuxbrew/Cellar"), + ] { + hash_homebrew_state(cellar, &mut hasher)?; + } + #[cfg(target_os = "macos")] + for receipts in [ + Path::new("/var/db/receipts"), + Path::new("/Library/Apple/System/Library/Receipts"), + ] { + hash_directory_files_with_extension(receipts, "plist", &mut hasher)?; + } if let Some(home) = environment .iter() .find(|(key, _)| key.eq_ignore_ascii_case("HOME")) @@ -554,6 +615,124 @@ fn hash_toolchain(environment: &HashMap) -> std::io::Result &'static [&'static str] { + &[ + "/var/lib/dpkg/status", + "/var/lib/apt/extended_states", + "/var/lib/rpm/Packages", + "/var/lib/rpm/rpmdb.sqlite", + "/usr/lib/sysimage/rpm/Packages", + "/usr/lib/sysimage/rpm/rpmdb.sqlite", + "/lib/apk/db/installed", + "/var/lib/pacman/local", + ] +} + +#[cfg(not(target_os = "linux"))] +fn host_package_state_paths() -> &'static [&'static str] { + &[] +} + +fn hash_host_package_state(paths: &[&Path], hasher: &mut Sha256) -> std::io::Result<()> { + hasher.update(b"host-package-state-v1\0"); + for path in paths { + if path.is_dir() { + hash_directory_tree(path, hasher)?; + } else { + hash_optional_file(path, hasher)?; + } + } + Ok(()) +} + +fn hash_pkg_config_directories( + environment: &HashMap, + hasher: &mut Sha256, +) -> std::io::Result<()> { + let mut search_path = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("PKG_CONFIG_PATH")) + .map(|(_, value)| value.clone()) + .unwrap_or_default(); + if let Ok(output) = Command::new("pkg-config") + .args(["--variable=pc_path", "pkg-config"]) + .env_clear() + .envs(environment) + .output() + && output.status.success() + { + if !search_path.is_empty() { + search_path.push(':'); + } + search_path.push_str(String::from_utf8_lossy(&output.stdout).trim()); + } + let mut paths = std::env::split_paths(&search_path).collect::>(); + paths.sort_unstable(); + paths.dedup(); + for path in paths { + hash_directory_tree(&path, hasher)?; + } + Ok(()) +} + +fn hash_homebrew_state(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hash_os_path(hasher, root); + let mut formulae = match std::fs::read_dir(root) { + Ok(entries) => entries.collect::, _>>()?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hasher.update(b"absent\x1e"); + return Ok(()); + } + Err(error) => return Err(error), + }; + formulae.sort_unstable_by_key(std::fs::DirEntry::file_name); + for formula in formulae { + if !formula.file_type()?.is_dir() { + continue; + } + let formula_name = formula.file_name(); + hash_os_path(hasher, Path::new(&formula_name)); + let mut versions = std::fs::read_dir(formula.path())?.collect::, _>>()?; + versions.sort_unstable_by_key(std::fs::DirEntry::file_name); + for version in versions { + if !version.file_type()?.is_dir() { + continue; + } + let version_name = version.file_name(); + hash_os_path(hasher, Path::new(&version_name)); + hash_optional_file(&version.path().join("INSTALL_RECEIPT.json"), hasher)?; + } + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn hash_directory_files_with_extension( + root: &Path, + extension: &str, + hasher: &mut Sha256, +) -> std::io::Result<()> { + hash_os_path(hasher, root); + let mut paths = match std::fs::read_dir(root) { + Ok(entries) => entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|value| value == extension)) + .collect::>(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hasher.update(b"absent\x1e"); + return Ok(()); + } + Err(error) => return Err(error), + }; + paths.sort_unstable(); + for path in paths { + hash_optional_file(&path, hasher)?; + } + Ok(()) +} + fn hash_optional_file(path: &Path, hasher: &mut Sha256) -> std::io::Result<()> { hash_os_path(hasher, path); match std::fs::File::open(path) { @@ -879,6 +1058,45 @@ mod tests { assert_ne!(first.finalize(), second.finalize()); } + #[test] + fn host_package_state_hash_changes_when_installed_state_changes() { + let temp = tempfile::tempdir().unwrap(); + let state = temp.path().join("status"); + std::fs::write(&state, b"Version: 1\n").unwrap(); + let mut first = Sha256::new(); + hash_host_package_state(&[state.as_path()], &mut first).unwrap(); + + std::fs::write(&state, b"Version: 2\n").unwrap(); + let mut second = Sha256::new(); + hash_host_package_state(&[state.as_path()], &mut second).unwrap(); + + assert_ne!(first.finalize(), second.finalize()); + } + + #[test] + fn homebrew_state_hash_changes_when_install_receipt_changes() { + let temp = tempfile::tempdir().unwrap(); + let version = temp.path().join("example/1.0.0"); + std::fs::create_dir_all(&version).unwrap(); + std::fs::write( + version.join("INSTALL_RECEIPT.json"), + b"{\"source\":\"first\"}", + ) + .unwrap(); + let mut first = Sha256::new(); + hash_homebrew_state(temp.path(), &mut first).unwrap(); + + std::fs::write( + version.join("INSTALL_RECEIPT.json"), + b"{\"source\":\"second\"}", + ) + .unwrap(); + let mut second = Sha256::new(); + hash_homebrew_state(temp.path(), &mut second).unwrap(); + + assert_ne!(first.finalize(), second.finalize()); + } + #[test] #[cfg(unix)] fn toolchain_hash_changes_when_cmake_executable_changes() { @@ -981,6 +1199,17 @@ mod tests { assert_eq!(scratch.path(), link_dir.join(".lpm-build-tmp")); } + #[test] + fn graph_identity_discovery_supports_scoped_package_path() { + let temp = tempfile::tempdir().unwrap(); + let link_dir = temp.path().join("links/scoped"); + let package_dir = link_dir.join("node_modules/@scope/example"); + std::fs::create_dir_all(&package_dir).unwrap(); + let digest = write_link_meta(&link_dir, "@scope/example"); + + assert_eq!(read_v2_graph_key_digest(&package_dir), Some(digest)); + } + #[test] fn build_cache_scratch_rejects_package_controlled_sidecar() { let temp = tempfile::tempdir().unwrap(); From 84e7d99a55d8ed9c682f74a88e51b57729df588a Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 15:53:22 +0100 Subject: [PATCH 07/14] ci: run native cache tests with Linux sandbox support --- .github/workflows/ci.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1df2bc11..97043da6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,7 +155,25 @@ jobs: run: cargo nextest run --locked -p lpm-cli -E 'not binary_id(/bin\/lpm-rs/)' --no-fail-fast --status-level slow --final-status-level fail - name: Native build cache workflow (Linux sandbox) - run: cargo nextest run --locked -p lpm-workflows --test native_build_cache --no-fail-fast --status-level slow --final-status-level fail + run: | + test_bin="$( + cargo test --locked -p lpm-workflows --test native_build_cache --no-run --message-format=json | + jq -r 'select(.reason == "compiler-artifact" and .target.name == "native_build_cache" and .profile.test) | .executable' | + tail -n 1 + )" + test -n "$test_bin" + test_bin="$(realpath "$test_bin")" + docker run --rm --privileged \ + --security-opt apparmor=unconfined \ + --security-opt seccomp=unconfined \ + --volume "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \ + --workdir "$GITHUB_WORKSPACE" \ + ubuntu:24.04 \ + bash -euxo pipefail -c ' + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install --yes nodejs libdbus-1-3 + "$1" --test-threads=1 + ' bash "$test_bin" macos-workspace: name: macOS workspace checks From 288320c9b22648a65bb223f557c6a767c5f69aa2 Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 16:03:21 +0100 Subject: [PATCH 08/14] ci: trace Linux sandbox startup failure --- .github/workflows/ci.yml | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97043da6..57fe9797 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,15 +145,6 @@ jobs: - name: Build workspace run: cargo build --workspace --locked - - name: Unit and integration tests (non-workflow, non-CLI) - run: cargo nextest run --locked --workspace --exclude lpm-workflows --exclude lpm-cli --no-fail-fast --status-level slow --final-status-level fail - - - name: CLI unit tests (serial libtest) - run: cargo test --locked -p lpm-cli --bin lpm-rs -- --test-threads=1 - - - name: CLI binary-surface tests - run: cargo nextest run --locked -p lpm-cli -E 'not binary_id(/bin\/lpm-rs/)' --no-fail-fast --status-level slow --final-status-level fail - - name: Native build cache workflow (Linux sandbox) run: | test_bin="$( @@ -171,10 +162,28 @@ jobs: ubuntu:24.04 \ bash -euxo pipefail -c ' apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install --yes nodejs libdbus-1-3 - "$1" --test-threads=1 + DEBIAN_FRONTEND=noninteractive apt-get install --yes nodejs libdbus-1-3 strace + set +e + strace -ff -o /tmp/native-build-cache.strace \ + -e trace=prctl,seccomp,landlock_restrict_self,unshare,setpgid,execve \ + "$1" --test-threads=1 + status=$? + set -e + if [ "$status" -ne 0 ]; then + grep -H " = -1 " /tmp/native-build-cache.strace* || true + fi + exit "$status" ' bash "$test_bin" + - name: Unit and integration tests (non-workflow, non-CLI) + run: cargo nextest run --locked --workspace --exclude lpm-workflows --exclude lpm-cli --no-fail-fast --status-level slow --final-status-level fail + + - name: CLI unit tests (serial libtest) + run: cargo test --locked -p lpm-cli --bin lpm-rs -- --test-threads=1 + + - name: CLI binary-surface tests + run: cargo nextest run --locked -p lpm-cli -E 'not binary_id(/bin\/lpm-rs/)' --no-fail-fast --status-level slow --final-status-level fail + macos-workspace: name: macOS workspace checks runs-on: macos-latest From 713d6cb4087364697d7e7ff1f4f1b1ead4df31e0 Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 16:11:03 +0100 Subject: [PATCH 09/14] fix: tailor Landlock access to inode type --- .github/workflows/ci.yml | 29 +------------------- crates/lpm-sandbox/src/linux.rs | 48 ++++++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57fe9797..59326f38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,34 +146,7 @@ jobs: run: cargo build --workspace --locked - name: Native build cache workflow (Linux sandbox) - run: | - test_bin="$( - cargo test --locked -p lpm-workflows --test native_build_cache --no-run --message-format=json | - jq -r 'select(.reason == "compiler-artifact" and .target.name == "native_build_cache" and .profile.test) | .executable' | - tail -n 1 - )" - test -n "$test_bin" - test_bin="$(realpath "$test_bin")" - docker run --rm --privileged \ - --security-opt apparmor=unconfined \ - --security-opt seccomp=unconfined \ - --volume "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \ - --workdir "$GITHUB_WORKSPACE" \ - ubuntu:24.04 \ - bash -euxo pipefail -c ' - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install --yes nodejs libdbus-1-3 strace - set +e - strace -ff -o /tmp/native-build-cache.strace \ - -e trace=prctl,seccomp,landlock_restrict_self,unshare,setpgid,execve \ - "$1" --test-threads=1 - status=$? - set -e - if [ "$status" -ne 0 ]; then - grep -H " = -1 " /tmp/native-build-cache.strace* || true - fi - exit "$status" - ' bash "$test_bin" + run: cargo nextest run --locked -p lpm-workflows --test native_build_cache --no-fail-fast --status-level slow --final-status-level fail - name: Unit and integration tests (non-workflow, non-CLI) run: cargo nextest run --locked --workspace --exclude lpm-workflows --exclude lpm-cli --no-fail-fast --status-level slow --final-status-level fail diff --git a/crates/lpm-sandbox/src/linux.rs b/crates/lpm-sandbox/src/linux.rs index e7e08363..56532d4e 100644 --- a/crates/lpm-sandbox/src/linux.rs +++ b/crates/lpm-sandbox/src/linux.rs @@ -124,8 +124,8 @@ use crate::{ SandboxedCommand, }; use landlock::{ - ABI, Access, AccessFs, AccessNet, CompatLevel, Compatible, PathBeneath, PathFd, Ruleset, - RulesetAttr, RulesetCreated, RulesetCreatedAttr, RulesetError, RulesetStatus, + ABI, Access, AccessFs, AccessNet, BitFlags, CompatLevel, Compatible, PathBeneath, PathFd, + Ruleset, RulesetAttr, RulesetCreated, RulesetCreatedAttr, RulesetError, RulesetStatus, }; use std::os::unix::process::CommandExt; use std::process::{Child, Command, Stdio}; @@ -759,7 +759,6 @@ fn build_parent_side_ruleset( ) -> Result { let abi = posture.abi(); let rw = AccessFs::from_all(abi); - let read = AccessFs::from_read(abi); let mut builder = Ruleset::default().handle_access(rw)?; if posture.enforces_network() { // Strict V4 only: declare the network access classes so @@ -781,15 +780,25 @@ fn build_parent_side_ruleset( continue; } }; - let access_bits = match access { - RuleAccess::Read => read, - RuleAccess::ReadWrite => rw, - }; + let is_file = std::fs::metadata(&path).is_ok_and(|metadata| !metadata.is_dir()); + let access_bits = rule_access_bits(access, abi, is_file); ruleset = ruleset.add_rule(PathBeneath::new(fd, access_bits))?; } Ok(ruleset) } +fn rule_access_bits(access: RuleAccess, abi: ABI, is_file: bool) -> BitFlags { + let valid = if is_file { + AccessFs::from_file(abi) + } else { + AccessFs::from_all(abi) + }; + match access { + RuleAccess::Read => valid & AccessFs::from_read(abi), + RuleAccess::ReadWrite => valid, + } +} + /// Async-signal-safe stderr write. Bypasses [`std::io::Stderr::lock`] /// (which holds a userspace mutex and deadlocks post-fork in /// multi-threaded processes) by issuing a direct `write(2)` to fd 2. @@ -1074,6 +1083,31 @@ mod tests { assert!(status.success(), "/usr/bin/true under landlock must exit 0"); } + #[test] + fn regular_file_rules_exclude_directory_only_access_rights() { + let read = rule_access_bits(RuleAccess::Read, ABI::V4, true); + let read_write = rule_access_bits(RuleAccess::ReadWrite, ABI::V4, true); + + assert_eq!(read, AccessFs::Execute | AccessFs::ReadFile); + assert_eq!(read_write, AccessFs::from_file(ABI::V4)); + assert!(!read_write.contains(AccessFs::ReadDir)); + assert!(!read_write.contains(AccessFs::RemoveDir)); + assert!(!read_write.contains(AccessFs::MakeDir)); + assert!(!read_write.contains(AccessFs::Refer)); + } + + #[test] + fn directory_rules_keep_full_requested_access_rights() { + assert_eq!( + rule_access_bits(RuleAccess::Read, ABI::V4, false), + AccessFs::from_read(ABI::V4) + ); + assert_eq!( + rule_access_bits(RuleAccess::ReadWrite, ABI::V4, false), + AccessFs::from_all(ABI::V4) + ); + } + #[test] fn enforces_deny_on_read_outside_allow_list() { // Forbidden target MUST live at a path no sandbox rule From 990d78c13e5160cc83b46f299267209cebaefbb2 Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 16:52:30 +0100 Subject: [PATCH 10/14] fix: contain native toolchain probes --- ...20260712-macos-arm64-native-toolchain.json | 49 +++ ...e-20260712-macos-arm64-native-toolchain.md | 25 ++ bench/scripts/README.md | 7 +- .../scripts/native-build-cache-benchmark.mjs | 31 +- .../src/commands/rebuild/build_cache.rs | 373 +++++++++++++++--- 5 files changed, 434 insertions(+), 51 deletions(-) create mode 100644 bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json create mode 100644 bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json new file mode 100644 index 00000000..655d234e --- /dev/null +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json @@ -0,0 +1,49 @@ +{ + "generated_at": "2026-07-12T15:47:06.739Z", + "platform": "darwin-arm64", + "node": "v22.22.1", + "lpm_bin": "target/release/lpm-rs", + "fixture": "bench/audit-fixtures/native/sharp-image", + "samples": 10, + "artifact_count": 2, + "artifact_bytes": 503469, + "scenarios": { + "cache_miss": { + "samples": 10, + "wall_ms_median": 245.04, + "wall_ms_p95": 319.67, + "max_rss_bytes_median": 47136768, + "user_cpu_ms_median": 70, + "system_cpu_ms_median": 60, + "cache_hits": 0, + "cache_misses": 10, + "lifecycle_ms_avoided": 0 + }, + "local_hit": { + "samples": 10, + "wall_ms_median": 132.8, + "wall_ms_p95": 156.1, + "max_rss_bytes_median": 41566208, + "user_cpu_ms_median": 50, + "system_cpu_ms_median": 50, + "cache_hits": 10, + "cache_misses": 0, + "lifecycle_ms_avoided": 1048, + "speedup_vs_miss": 1.85, + "wall_reduction_percent": 45.8 + }, + "ci_warm_store_hit": { + "samples": 10, + "wall_ms_median": 134.28, + "wall_ms_p95": 140.65, + "max_rss_bytes_median": 41549824, + "user_cpu_ms_median": 50, + "system_cpu_ms_median": 50, + "cache_hits": 10, + "cache_misses": 0, + "lifecycle_ms_avoided": 1053, + "speedup_vs_miss": 1.82, + "wall_reduction_percent": 45.2 + } + } +} diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md new file mode 100644 index 00000000..5e8e26e5 --- /dev/null +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md @@ -0,0 +1,25 @@ +# Native lifecycle-build cache benchmark — NativeToolchain + +- Platform: macOS arm64 +- Node: v22.22.1 +- Fixture: `bench/audit-fixtures/native/sharp-image` +- Release-build samples per scenario: 10 +- Median build artifact size: 503,469 bytes + +| Scenario | Median wall | p95 wall | Median RSS | User CPU | System CPU | Result | +|---|---:|---:|---:|---:|---:|---:| +| Strict cache miss | 245.04 ms | 319.67 ms | 47,136,768 B | 70 ms | 60 ms | 10 misses | +| Local artifact hit | 132.80 ms | 156.10 ms | 41,566,208 B | 50 ms | 50 ms | 1.85×; 45.8% lower wall time | +| Stable-path CI warm-store hit | 134.28 ms | 140.65 ms | 41,549,824 B | 50 ms | 50 ms | 1.82×; 45.2% lower wall time | + +Every measured scenario used the same strict sandbox. Sharp follows the +`NativeToolchain` cache path, so every hit includes trusted compiler, SDK, +package-database, pkg-config, Homebrew receipt, and Node-header fingerprinting. +Miss and local-hit samples rematerialized the same project. CI-like samples +deleted and recreated a checkout at one stable path while retaining the warm +LPM store. + +Across ten local hits, artifact metadata recorded 1,048 ms of lifecycle +execution avoided. Across ten stable-path CI hits, it recorded 1,053 ms +avoided. The measured hit remains beneficial after paying the complete native +toolchain key cost. diff --git a/bench/scripts/README.md b/bench/scripts/README.md index 852728ae..f2dfb983 100644 --- a/bench/scripts/README.md +++ b/bench/scripts/README.md @@ -3,11 +3,14 @@ ## Native lifecycle-build cache Build the release CLI, then compare strict cache misses, local artifact hits, -and CI-like stable-path checkout recreation against the native esbuild fixture. -Every measured scenario uses the same strict sandbox: +and CI-like stable-path checkout recreation against sharp's NativeToolchain +cache path. Every measured scenario uses the same strict sandbox, so cache-hit +timings include host compiler, SDK, package database, and pkg-config +fingerprinting: ```bash cargo build --release --locked -p lpm-cli --bin lpm-rs +node bench/scripts/native-build-cache-benchmark.mjs --self-test node bench/scripts/native-build-cache-benchmark.mjs --samples 10 ``` diff --git a/bench/scripts/native-build-cache-benchmark.mjs b/bench/scripts/native-build-cache-benchmark.mjs index 8748c0ae..7beb4282 100644 --- a/bench/scripts/native-build-cache-benchmark.mjs +++ b/bench/scripts/native-build-cache-benchmark.mjs @@ -8,10 +8,14 @@ import { fileURLToPath } from 'node:url'; const repoRoot = path.resolve(fileURLToPath(new URL('../..', import.meta.url))); const options = parseArgs(process.argv.slice(2)); +if (options['self-test']) { + runSelfTest(); + process.exit(0); +} const samples = Number(options.samples ?? 10); const fixture = path.resolve( repoRoot, - options.fixture ?? 'bench/audit-fixtures/native/esbuild-prebuilt', + options.fixture ?? 'bench/audit-fixtures/native/sharp-image', ); const lpmBin = path.resolve(options['lpm-bin'] ?? path.join(repoRoot, 'target/release/lpm-rs')); const timestamp = new Date().toISOString().replaceAll(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); @@ -102,7 +106,7 @@ function measureRebuild(scenario, sample, cwd) { `${scenario} failed (${result.status})\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, ); } - const envelope = JSON.parse(result.stdout); + const envelope = parseJsonEnvelope(result.stdout); const expectedHit = scenario.endsWith('_hit'); if ( (expectedHit && (envelope.build_cache.hits !== 1 || envelope.build_cache.misses !== 0)) || @@ -123,6 +127,27 @@ function measureRebuild(scenario, sample, cwd) { }; } +function parseJsonEnvelope(stdout) { + const lines = stdout.trimEnd().split(/\r?\n/); + for (let index = lines.length - 1; index >= 0; index -= 1) { + if (!lines[index].trimStart().startsWith('{')) continue; + try { + const value = JSON.parse(lines.slice(index).join('\n')); + if (value && typeof value === 'object' && value.build_cache) return value; + } catch {} + } + throw new Error(`rebuild output did not contain a JSON cache envelope:\n${stdout}`); +} + +function runSelfTest() { + const envelope = parseJsonEnvelope( + 'native lifecycle output\n{\n "success": true,\n "build_cache": {"hits": 1}\n}\n', + ); + if (envelope.build_cache.hits !== 1) { + throw new Error('JSON envelope parser did not preserve cache metrics'); + } +} + function runChecked(args, cwd) { const result = spawnSync(lpmBin, args, { cwd, @@ -286,7 +311,7 @@ function parseArgs(argv) { const token = argv[index]; if (!token.startsWith('--')) throw new Error(`unexpected argument: ${token}`); const key = token.slice(2); - if (key === 'keep') parsed.keep = true; + if (key === 'keep' || key === 'self-test') parsed[key] = true; else parsed[key] = argv[++index]; } return parsed; diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache.rs b/crates/lpm-cli/src/commands/rebuild/build_cache.rs index 88ff4b21..efae0d78 100644 --- a/crates/lpm-cli/src/commands/rebuild/build_cache.rs +++ b/crates/lpm-cli/src/commands/rebuild/build_cache.rs @@ -11,8 +11,12 @@ use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::io::Read; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +const TOOLCHAIN_PROBE_TIMEOUT: Duration = Duration::from_secs(2); +const TOOLCHAIN_PROBE_OUTPUT_LIMIT: usize = 4 * 1024 * 1024; pub(super) struct BuildCacheScratch { path: PathBuf, @@ -83,7 +87,9 @@ pub(super) struct BuildCacheInvocation { runtime: BuildRuntimeFingerprint, sandbox: BuildSandboxFingerprint, environment: HashMap, - toolchain_environment: HashMap, + trusted_toolchain_environment: HashMap, + build_executable_environment: HashMap, + project_dir: PathBuf, native_toolchain_hash: OnceLock>, } @@ -116,18 +122,19 @@ impl BuildCacheInvocation { debug_bypass("project widened lifecycle capabilities"); return None; } - let Some(runtime) = detect_node_runtime(environment) else { + let trusted_toolchain_environment = trusted_toolchain_environment(environment, project_dir); + let Some(runtime) = detect_node_runtime(&trusted_toolchain_environment, project_dir) else { debug_bypass("Node runtime fingerprint unavailable"); return None; }; let platform = PlatformTuple::current(); - let mut toolchain_environment = environment.clone(); + let mut build_executable_environment = environment.clone(); let parent_path = environment .iter() .find(|(key, _)| key.eq_ignore_ascii_case("PATH")) .map(|(_, value)| value.as_str()); - toolchain_environment.retain(|key, _| !key.eq_ignore_ascii_case("PATH")); - toolchain_environment.insert( + build_executable_environment.retain(|key, _| !key.eq_ignore_ascii_case("PATH")); + build_executable_environment.insert( "PATH".into(), build_lifecycle_path(project_dir, parent_path), ); @@ -148,7 +155,9 @@ impl BuildCacheInvocation { allowed_inputs_hash: hash_allowed_inputs(capabilities), }, environment: environment.clone(), - toolchain_environment, + trusted_toolchain_environment, + build_executable_environment, + project_dir: project_dir.to_path_buf(), native_toolchain_hash: OnceLock::new(), }) } @@ -366,14 +375,21 @@ pub(super) fn build_key_for_package( let toolchain_hash = if native_kind == NativeBuildKind::NativeToolchain { let Some(hash) = invocation .native_toolchain_hash - .get_or_init(|| hash_toolchain(&invocation.toolchain_environment).ok()) + .get_or_init(|| { + hash_toolchain( + &invocation.trusted_toolchain_environment, + &invocation.build_executable_environment, + &invocation.project_dir, + ) + .ok() + }) .clone() else { debug_bypass("native toolchain fingerprint unavailable"); return None; }; let invoked_executables = - hash_invoked_build_executables(package, &invocation.toolchain_environment); + hash_invoked_build_executables(package, &invocation.build_executable_environment); hash_records([hash.as_bytes(), invoked_executables.as_bytes()]) } else { "runtime-and-dependency-graph-v1".to_string() @@ -411,14 +427,20 @@ pub(super) fn read_v2_graph_key_digest(package_dir: &Path) -> Option { (node_modules_dir.join(&metadata.name) == package_dir).then_some(metadata.graph_key_digest_hex) } -fn detect_node_runtime(environment: &HashMap) -> Option { - let output = Command::new("node") - .arg("-p") - .arg("JSON.stringify({version:process.version,modules:process.versions.modules||'',napi:process.versions.napi||'',engine:process.versions.v8||'',execPath:process.execPath})") - .env_clear() - .envs(environment) - .output() - .ok()?; +fn detect_node_runtime( + environment: &HashMap, + project_dir: &Path, +) -> Option { + let executable = resolve_trusted_executable("node", environment, project_dir)?; + let probe_environment = toolchain_probe_environment(environment); + let output = run_bounded_toolchain_probe( + &executable, + &[ + "-p", + "JSON.stringify({version:process.version,modules:process.versions.modules||'',napi:process.versions.napi||'',engine:process.versions.v8||'',execPath:process.execPath})", + ], + &probe_environment, + )?; if !output.status.success() { return None; } @@ -501,7 +523,180 @@ fn hash_environment_pairs(environment: &[(String, String)]) -> String { })) } -fn hash_toolchain(environment: &HashMap) -> std::io::Result { +fn trusted_toolchain_environment( + environment: &HashMap, + project_dir: &Path, +) -> HashMap { + let mut trusted = environment.clone(); + let inherited_path = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("PATH")) + .map_or(default_toolchain_path(), |(_, value)| value.as_str()); + let search_dirs = std::env::split_paths(inherited_path) + .filter(|directory| !is_project_controlled_path(directory, project_dir)) + .collect::>(); + let path = std::env::join_paths(search_dirs) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| default_toolchain_path().into()); + trusted.retain(|key, _| !key.eq_ignore_ascii_case("PATH")); + trusted.insert("PATH".into(), path.to_string_lossy().into_owned()); + trusted +} + +fn default_toolchain_path() -> &'static str { + #[cfg(unix)] + { + "/usr/local/bin:/usr/bin:/bin" + } + #[cfg(windows)] + { + r"C:\Windows\System32;C:\Windows;C:\Windows\System32\Wbem" + } + #[cfg(not(any(unix, windows)))] + { + "/usr/bin:/bin" + } +} + +fn is_project_controlled_path(path: &Path, project_dir: &Path) -> bool { + if !path.is_absolute() + || path + .components() + .any(|component| component.as_os_str() == "node_modules") + || path.starts_with(project_dir) + { + return true; + } + let Ok(canonical) = std::fs::canonicalize(path) else { + return false; + }; + let canonical_project = + std::fs::canonicalize(project_dir).unwrap_or_else(|_| project_dir.to_path_buf()); + canonical.starts_with(canonical_project) + || canonical + .components() + .any(|component| component.as_os_str() == "node_modules") +} + +fn resolve_trusted_executable( + executable: &str, + environment: &HashMap, + project_dir: &Path, +) -> Option { + let path = resolve_executable(executable, environment)?; + let canonical = std::fs::canonicalize(path).ok()?; + (!is_project_controlled_path(&canonical, project_dir)).then_some(canonical) +} + +fn toolchain_probe_environment( + trusted_environment: &HashMap, +) -> HashMap { + // The full lifecycle environment can contain loader/runtime hooks that execute code + // before the lifecycle sandbox exists. Probes receive only data-bearing inputs. + const PASSTHROUGH: &[&str] = &[ + "PATH", + "SDKROOT", + "DEVELOPER_DIR", + "PKG_CONFIG_PATH", + "PKG_CONFIG_LIBDIR", + "PKG_CONFIG_SYSROOT_DIR", + ]; + let mut probe_environment = HashMap::with_capacity(PASSTHROUGH.len() + 2); + for name in PASSTHROUGH { + if let Some((key, value)) = trusted_environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + { + probe_environment.insert(key.clone(), value.clone()); + } + } + probe_environment.insert("LANG".into(), "C".into()); + probe_environment.insert("LC_ALL".into(), "C".into()); + probe_environment +} + +fn read_bounded_probe_stream(mut reader: R) -> Vec { + let mut retained = Vec::with_capacity(8 * 1024); + let mut chunk = [0_u8; 64 * 1024]; + while let Ok(read) = reader.read(&mut chunk) { + if read == 0 { + break; + } + let remaining = TOOLCHAIN_PROBE_OUTPUT_LIMIT.saturating_sub(retained.len()); + retained.extend_from_slice(&chunk[..read.min(remaining)]); + } + retained +} + +fn terminate_toolchain_probe(child: &mut std::process::Child) { + #[cfg(unix)] + { + let pid = child.id() as i32; + // SAFETY: the child is placed in a fresh process group immediately before spawn. + unsafe { + libc::kill(-pid, libc::SIGKILL); + } + } + let _ = child.kill(); +} + +fn run_bounded_toolchain_probe( + executable: &Path, + args: &[&str], + environment: &HashMap, +) -> Option { + let mut command = Command::new(executable); + command + .args(args) + .env_clear() + .envs(environment) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let mut child = command.spawn().ok()?; + let (Some(stdout), Some(stderr)) = (child.stdout.take(), child.stderr.take()) else { + terminate_toolchain_probe(&mut child); + let _ = child.wait(); + return None; + }; + let stdout_reader = std::thread::spawn(move || read_bounded_probe_stream(stdout)); + let stderr_reader = std::thread::spawn(move || read_bounded_probe_stream(stderr)); + let started = Instant::now(); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) if started.elapsed() < TOOLCHAIN_PROBE_TIMEOUT => { + std::thread::sleep(Duration::from_millis(10)); + } + Ok(None) | Err(_) => { + terminate_toolchain_probe(&mut child); + let _ = child.wait(); + let _ = stdout_reader.join(); + let _ = stderr_reader.join(); + return None; + } + } + }; + let stdout = stdout_reader.join().ok()?; + let stderr = stderr_reader.join().ok()?; + Some(std::process::Output { + status, + stdout, + stderr, + }) +} + +fn hash_toolchain( + trusted_environment: &HashMap, + build_environment: &HashMap, + project_dir: &Path, +) -> std::io::Result { const PROBES: &[(&str, &[&str])] = &[ ("python3", &["--version"]), ("make", &["--version"]), @@ -539,7 +734,8 @@ fn hash_toolchain(environment: &HashMap) -> std::io::Result) -> std::io::Result>(); @@ -563,16 +757,28 @@ fn hash_toolchain(environment: &HashMap) -> std::io::Result>() }); - for ((program, _), output) in PROBES.iter().zip(probe_outputs) { + for ((program, args), output) in PROBES.iter().zip(&probe_outputs) { hasher.update(program.as_bytes()); hasher.update(b"\0"); + for argument in *args { + hasher.update(argument.as_bytes()); + hasher.update(b"\0"); + } if let Some(output) = output { - hasher.update(&output.stdout[..output.stdout.len().min(4 * 1024 * 1024)]); - hasher.update(&output.stderr[..output.stderr.len().min(4 * 1024 * 1024)]); + hasher.update(&output.stdout); + hasher.update(&output.stderr); } - if let Some(executable) = resolve_executable(program, environment) + if let Some(executable) = + resolve_trusted_executable(program, trusted_environment, project_dir) && let Ok(identity) = runtime_executable_identity(&executable) { + hasher.update(b"trusted\0"); + hasher.update(identity.as_bytes()); + } + if let Some(executable) = resolve_executable(program, build_environment) + && let Ok(identity) = runtime_executable_identity(&executable) + { + hasher.update(b"build-visible\0"); hasher.update(identity.as_bytes()); } hasher.update(b"\x1e"); @@ -589,7 +795,18 @@ fn hash_toolchain(environment: &HashMap) -> std::io::Result>(); hash_host_package_state(&host_state_paths, &mut hasher)?; - hash_pkg_config_directories(environment, &mut hasher)?; + let pkg_config_pc_path = + PROBES + .iter() + .zip(&probe_outputs) + .find_map(|((program, args), output)| { + (*program == "pkg-config" + && *args == ["--variable=pc_path", "pkg-config"] + && output.as_ref().is_some_and(|value| value.status.success())) + .then(|| output.as_ref().map(|value| value.stdout.as_slice())) + .flatten() + }); + hash_pkg_config_directories(trusted_environment, pkg_config_pc_path, &mut hasher)?; for cellar in [ Path::new("/opt/homebrew/Cellar"), Path::new("/usr/local/Cellar"), @@ -604,7 +821,7 @@ fn hash_toolchain(environment: &HashMap) -> std::io::Result std::io::Res fn hash_pkg_config_directories( environment: &HashMap, + default_search_path: Option<&[u8]>, hasher: &mut Sha256, ) -> std::io::Result<()> { let mut search_path = environment @@ -655,17 +873,11 @@ fn hash_pkg_config_directories( .find(|(key, _)| key.eq_ignore_ascii_case("PKG_CONFIG_PATH")) .map(|(_, value)| value.clone()) .unwrap_or_default(); - if let Ok(output) = Command::new("pkg-config") - .args(["--variable=pc_path", "pkg-config"]) - .env_clear() - .envs(environment) - .output() - && output.status.success() - { + if let Some(default_search_path) = default_search_path { if !search_path.is_empty() { search_path.push(':'); } - search_path.push_str(String::from_utf8_lossy(&output.stdout).trim()); + search_path.push_str(String::from_utf8_lossy(default_search_path).trim()); } let mut paths = std::env::split_paths(&search_path).collect::>(); paths.sort_unstable(); @@ -1099,25 +1311,94 @@ mod tests { #[test] #[cfg(unix)] - fn toolchain_hash_changes_when_cmake_executable_changes() { + fn toolchain_hash_tracks_project_cmake_without_executing_it() { use std::os::unix::fs::PermissionsExt; let temp = tempfile::tempdir().unwrap(); - let bin = temp.path().join("bin"); + let project = temp.path().join("project"); + let bin = project.join("node_modules/.bin"); + std::fs::create_dir_all(&bin).unwrap(); + let cmake = bin.join("cmake"); + let marker = temp.path().join("executed"); + std::fs::write( + &cmake, + format!("#!/bin/sh\n: > '{}'\necho cmake-one\n", marker.display()), + ) + .unwrap(); + std::fs::set_permissions(&cmake, std::fs::Permissions::from_mode(0o755)).unwrap(); + let build_environment = HashMap::from([ + ("PATH".to_string(), bin.display().to_string()), + ("HOME".to_string(), temp.path().display().to_string()), + ]); + let trusted_environment = trusted_toolchain_environment(&build_environment, &project); + let first = hash_toolchain(&trusted_environment, &build_environment, &project).unwrap(); + assert!(!marker.exists()); + + std::fs::write( + &cmake, + format!("#!/bin/sh\n: > '{}'\necho cmake-two\n", marker.display()), + ) + .unwrap(); + let second = hash_toolchain(&trusted_environment, &build_environment, &project).unwrap(); + + assert_ne!(first, second); + assert!(!marker.exists()); + } + + #[test] + #[cfg(unix)] + fn toolchain_probe_timeout_bounds_stalled_executable() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + std::fs::create_dir(&project).unwrap(); + let bin = temp.path().join("host-bin"); std::fs::create_dir(&bin).unwrap(); let cmake = bin.join("cmake"); - std::fs::write(&cmake, "#!/bin/sh\necho cmake-one\n").unwrap(); + std::fs::write(&cmake, "#!/bin/sh\n/bin/sleep 5\n").unwrap(); std::fs::set_permissions(&cmake, std::fs::Permissions::from_mode(0o755)).unwrap(); let environment = HashMap::from([ ("PATH".to_string(), bin.display().to_string()), ("HOME".to_string(), temp.path().display().to_string()), ]); - let first = hash_toolchain(&environment).unwrap(); - std::fs::write(&cmake, "#!/bin/sh\necho cmake-two\n").unwrap(); - let second = hash_toolchain(&environment).unwrap(); + let started = std::time::Instant::now(); + hash_toolchain(&environment, &environment, &project).unwrap(); - assert_ne!(first, second); + assert!(started.elapsed() < std::time::Duration::from_secs(3)); + } + + #[test] + fn toolchain_probe_reader_discards_output_beyond_limit() { + let input = vec![b'x'; TOOLCHAIN_PROBE_OUTPUT_LIMIT + 1024]; + + let output = read_bounded_probe_stream(std::io::Cursor::new(input)); + + assert_eq!(output.len(), TOOLCHAIN_PROBE_OUTPUT_LIMIT); + } + + #[test] + fn toolchain_probe_environment_excludes_code_injection_variables() { + let trusted = HashMap::from([ + ("PATH".into(), "/usr/bin:/bin".into()), + ( + "NODE_OPTIONS".into(), + "--require=/project/payload.js".into(), + ), + ("LD_PRELOAD".into(), "/project/payload.so".into()), + ( + "DYLD_INSERT_LIBRARIES".into(), + "/project/payload.dylib".into(), + ), + ]); + + let probe = toolchain_probe_environment(&trusted); + + assert_eq!(probe.get("PATH").map(String::as_str), Some("/usr/bin:/bin")); + assert!(!probe.contains_key("NODE_OPTIONS")); + assert!(!probe.contains_key("LD_PRELOAD")); + assert!(!probe.contains_key("DYLD_INSERT_LIBRARIES")); } #[test] From 5e1ee1e4cb5f0cdd8220732f87878ed91236bb25 Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 20:09:16 +0100 Subject: [PATCH 11/14] fix: harden native runtime probes --- ...20260712-macos-arm64-native-toolchain.json | 44 ++-- ...e-20260712-macos-arm64-native-toolchain.md | 10 +- .../src/commands/rebuild/build_cache.rs | 207 ++++++++++++++++-- 3 files changed, 212 insertions(+), 49 deletions(-) diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json index 655d234e..bd51275f 100644 --- a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-07-12T15:47:06.739Z", + "generated_at": "2026-07-12T19:08:31.460Z", "platform": "darwin-arm64", "node": "v22.22.1", "lpm_bin": "target/release/lpm-rs", @@ -10,40 +10,40 @@ "scenarios": { "cache_miss": { "samples": 10, - "wall_ms_median": 245.04, - "wall_ms_p95": 319.67, - "max_rss_bytes_median": 47136768, - "user_cpu_ms_median": 70, - "system_cpu_ms_median": 60, + "wall_ms_median": 229.99, + "wall_ms_p95": 444.35, + "max_rss_bytes_median": 47104000, + "user_cpu_ms_median": 120, + "system_cpu_ms_median": 170, "cache_hits": 0, "cache_misses": 10, "lifecycle_ms_avoided": 0 }, "local_hit": { "samples": 10, - "wall_ms_median": 132.8, - "wall_ms_p95": 156.1, - "max_rss_bytes_median": 41566208, - "user_cpu_ms_median": 50, - "system_cpu_ms_median": 50, + "wall_ms_median": 122.88, + "wall_ms_p95": 131.98, + "max_rss_bytes_median": 41631744, + "user_cpu_ms_median": 100, + "system_cpu_ms_median": 160, "cache_hits": 10, "cache_misses": 0, - "lifecycle_ms_avoided": 1048, - "speedup_vs_miss": 1.85, - "wall_reduction_percent": 45.8 + "lifecycle_ms_avoided": 1072, + "speedup_vs_miss": 1.87, + "wall_reduction_percent": 46.57 }, "ci_warm_store_hit": { "samples": 10, - "wall_ms_median": 134.28, - "wall_ms_p95": 140.65, - "max_rss_bytes_median": 41549824, - "user_cpu_ms_median": 50, - "system_cpu_ms_median": 50, + "wall_ms_median": 125.39, + "wall_ms_p95": 135.02, + "max_rss_bytes_median": 41615360, + "user_cpu_ms_median": 100, + "system_cpu_ms_median": 160, "cache_hits": 10, "cache_misses": 0, - "lifecycle_ms_avoided": 1053, - "speedup_vs_miss": 1.82, - "wall_reduction_percent": 45.2 + "lifecycle_ms_avoided": 1073, + "speedup_vs_miss": 1.83, + "wall_reduction_percent": 45.48 } } } diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md index 5e8e26e5..4c9ad7a6 100644 --- a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md @@ -8,9 +8,9 @@ | Scenario | Median wall | p95 wall | Median RSS | User CPU | System CPU | Result | |---|---:|---:|---:|---:|---:|---:| -| Strict cache miss | 245.04 ms | 319.67 ms | 47,136,768 B | 70 ms | 60 ms | 10 misses | -| Local artifact hit | 132.80 ms | 156.10 ms | 41,566,208 B | 50 ms | 50 ms | 1.85×; 45.8% lower wall time | -| Stable-path CI warm-store hit | 134.28 ms | 140.65 ms | 41,549,824 B | 50 ms | 50 ms | 1.82×; 45.2% lower wall time | +| Strict cache miss | 229.99 ms | 444.35 ms | 47,104,000 B | 120 ms | 170 ms | 10 misses | +| Local artifact hit | 122.88 ms | 131.98 ms | 41,631,744 B | 100 ms | 160 ms | 1.87×; 46.57% lower wall time | +| Stable-path CI warm-store hit | 125.39 ms | 135.02 ms | 41,615,360 B | 100 ms | 160 ms | 1.83×; 45.48% lower wall time | Every measured scenario used the same strict sandbox. Sharp follows the `NativeToolchain` cache path, so every hit includes trusted compiler, SDK, @@ -19,7 +19,7 @@ Miss and local-hit samples rematerialized the same project. CI-like samples deleted and recreated a checkout at one stable path while retaining the warm LPM store. -Across ten local hits, artifact metadata recorded 1,048 ms of lifecycle -execution avoided. Across ten stable-path CI hits, it recorded 1,053 ms +Across ten local hits, artifact metadata recorded 1,072 ms of lifecycle +execution avoided. Across ten stable-path CI hits, it recorded 1,073 ms avoided. The measured hit remains beneficial after paying the complete native toolchain key cost. diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache.rs b/crates/lpm-cli/src/commands/rebuild/build_cache.rs index efae0d78..914942a6 100644 --- a/crates/lpm-cli/src/commands/rebuild/build_cache.rs +++ b/crates/lpm-cli/src/commands/rebuild/build_cache.rs @@ -372,6 +372,10 @@ pub(super) fn build_key_for_package( }); } } + if !build_visible_node_matches_trusted(invocation) { + debug_bypass("build-visible Node does not match the trusted runtime"); + return None; + } let toolchain_hash = if native_kind == NativeBuildKind::NativeToolchain { let Some(hash) = invocation .native_toolchain_hash @@ -416,6 +420,12 @@ pub(super) fn build_key_for_package( Some(key) } +fn build_visible_node_matches_trusted(invocation: &BuildCacheInvocation) -> bool { + let trusted = resolve_executable("node", &invocation.trusted_toolchain_environment); + let build_visible = resolve_executable("node", &invocation.build_executable_environment); + trusted.is_some() && trusted == build_visible +} + pub(super) fn read_v2_graph_key_digest(package_dir: &Path) -> Option { let node_modules_dir = package_dir.ancestors().find(|ancestor| { ancestor @@ -434,7 +444,7 @@ fn detect_node_runtime( let executable = resolve_trusted_executable("node", environment, project_dir)?; let probe_environment = toolchain_probe_environment(environment); let output = run_bounded_toolchain_probe( - &executable, + &executable.invocation_path, &[ "-p", "JSON.stringify({version:process.version,modules:process.versions.modules||'',napi:process.versions.napi||'',engine:process.versions.v8||'',execPath:process.execPath})", @@ -579,14 +589,24 @@ fn is_project_controlled_path(path: &Path, project_dir: &Path) -> bool { .any(|component| component.as_os_str() == "node_modules") } +struct TrustedExecutable { + invocation_path: PathBuf, + canonical_target: PathBuf, +} + fn resolve_trusted_executable( executable: &str, environment: &HashMap, project_dir: &Path, -) -> Option { - let path = resolve_executable(executable, environment)?; - let canonical = std::fs::canonicalize(path).ok()?; - (!is_project_controlled_path(&canonical, project_dir)).then_some(canonical) +) -> Option { + let invocation_path = resolve_executable(executable, environment)?; + let canonical_target = std::fs::canonicalize(&invocation_path).ok()?; + (!is_project_controlled_path(&invocation_path, project_dir) + && !is_project_controlled_path(&canonical_target, project_dir)) + .then_some(TrustedExecutable { + invocation_path, + canonical_target, + }) } fn toolchain_probe_environment( @@ -657,6 +677,7 @@ fn run_bounded_toolchain_probe( #[cfg(unix)] { use std::os::unix::process::CommandExt; + command.current_dir("/"); command.process_group(0); } let mut child = command.spawn().ok()?; @@ -668,25 +689,37 @@ fn run_bounded_toolchain_probe( let stdout_reader = std::thread::spawn(move || read_bounded_probe_stream(stdout)); let stderr_reader = std::thread::spawn(move || read_bounded_probe_stream(stderr)); let started = Instant::now(); - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break status, - Ok(None) if started.elapsed() < TOOLCHAIN_PROBE_TIMEOUT => { - std::thread::sleep(Duration::from_millis(10)); - } - Ok(None) | Err(_) => { - terminate_toolchain_probe(&mut child); - let _ = child.wait(); - let _ = stdout_reader.join(); - let _ = stderr_reader.join(); - return None; + let mut status = None; + loop { + if status.is_none() { + match child.try_wait() { + Ok(Some(exit_status)) => status = Some(exit_status), + Ok(None) => {} + Err(_) => { + terminate_toolchain_probe(&mut child); + let _ = child.wait(); + let _ = stdout_reader.join(); + let _ = stderr_reader.join(); + return None; + } } } - }; + if status.is_some() && stdout_reader.is_finished() && stderr_reader.is_finished() { + break; + } + if started.elapsed() >= TOOLCHAIN_PROBE_TIMEOUT { + terminate_toolchain_probe(&mut child); + let _ = child.wait(); + let _ = stdout_reader.join(); + let _ = stderr_reader.join(); + return None; + } + std::thread::sleep(Duration::from_millis(10)); + } let stdout = stdout_reader.join().ok()?; let stderr = stderr_reader.join().ok()?; Some(std::process::Output { - status, + status: status?, stdout, stderr, }) @@ -734,7 +767,7 @@ fn hash_toolchain( ("ldd", &["--version"]), ]; let mut hasher = Sha256::new(); - hasher.update(b"lpm-toolchain-v4\0"); + hasher.update(b"lpm-toolchain-v5\0"); let probe_environment = toolchain_probe_environment(trusted_environment); let probe_outputs = std::thread::scope(|scope| { #[expect( @@ -748,7 +781,11 @@ fn hash_toolchain( scope.spawn(move || { let executable = resolve_trusted_executable(program, trusted_environment, project_dir)?; - run_bounded_toolchain_probe(&executable, args, probe_environment) + run_bounded_toolchain_probe( + &executable.invocation_path, + args, + probe_environment, + ) }) }) .collect::>(); @@ -770,9 +807,10 @@ fn hash_toolchain( } if let Some(executable) = resolve_trusted_executable(program, trusted_environment, project_dir) - && let Ok(identity) = runtime_executable_identity(&executable) + && let Ok(identity) = runtime_executable_identity(&executable.canonical_target) { hasher.update(b"trusted\0"); + hash_os_path(&mut hasher, &executable.invocation_path); hasher.update(identity.as_bytes()); } if let Some(executable) = resolve_executable(program, build_environment) @@ -1369,6 +1407,131 @@ mod tests { assert!(started.elapsed() < std::time::Duration::from_secs(3)); } + #[test] + #[cfg(unix)] + fn toolchain_probe_timeout_kills_descendant_holding_output_pipe() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let probe = temp.path().join("probe"); + std::fs::write(&probe, "#!/bin/sh\n(/bin/sleep 5) &\necho ready\n").unwrap(); + std::fs::set_permissions(&probe, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let started = std::time::Instant::now(); + let output = run_bounded_toolchain_probe(&probe, &[], &HashMap::new()); + + assert!(output.is_none()); + assert!(started.elapsed() < std::time::Duration::from_secs(3)); + } + + #[test] + #[cfg(unix)] + fn trusted_proxy_executes_with_symlink_invocation_name() { + use std::os::unix::fs::{PermissionsExt, symlink}; + + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let bin = temp.path().join("host-bin"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&bin).unwrap(); + let manager = bin.join("multicall"); + std::fs::write(&manager, "#!/bin/sh\necho \"${0##*/}\"\n").unwrap(); + std::fs::set_permissions(&manager, std::fs::Permissions::from_mode(0o755)).unwrap(); + symlink(&manager, bin.join("rustc")).unwrap(); + let environment = HashMap::from([("PATH".to_string(), bin.display().to_string())]); + let executable = resolve_trusted_executable("rustc", &environment, &project).unwrap(); + + let output = + run_bounded_toolchain_probe(&executable.invocation_path, &[], &environment).unwrap(); + + assert_eq!(String::from_utf8(output.stdout).unwrap().trim(), "rustc"); + } + + #[test] + #[cfg(unix)] + fn toolchain_probe_runs_from_neutral_directory() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let probe = temp.path().join("probe"); + std::fs::write(&probe, "#!/bin/sh\npwd\n").unwrap(); + std::fs::set_permissions(&probe, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let output = run_bounded_toolchain_probe(&probe, &[], &HashMap::new()).unwrap(); + + assert_eq!(String::from_utf8(output.stdout).unwrap().trim(), "/"); + } + + #[test] + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn runtime_only_key_is_rejected_when_project_node_shadows_trusted_runtime() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let project_bin = project.join("node_modules/.bin"); + let trusted_bin = temp.path().join("trusted-bin"); + let link_dir = temp.path().join("links/esbuild"); + let package_dir = link_dir.join("node_modules/esbuild"); + std::fs::create_dir_all(&project_bin).unwrap(); + std::fs::create_dir(&trusted_bin).unwrap(); + std::fs::create_dir_all(&package_dir).unwrap(); + std::fs::write(project_bin.join("node"), b"project node").unwrap(); + std::fs::write(trusted_bin.join("node"), b"trusted node").unwrap(); + std::fs::set_permissions( + project_bin.join("node"), + std::fs::Permissions::from_mode(0o755), + ) + .unwrap(); + std::fs::set_permissions( + trusted_bin.join("node"), + std::fs::Permissions::from_mode(0o755), + ) + .unwrap(); + let graph_key_digest = write_link_meta(&link_dir, "esbuild"); + let platform = PlatformTuple::current(); + let trusted_path = trusted_bin.display().to_string(); + let trusted_environment = HashMap::from([("PATH".to_string(), trusted_path.clone())]); + let build_executable_environment = HashMap::from([( + "PATH".to_string(), + build_lifecycle_path(&project, Some(&trusted_path)), + )]); + let invocation = BuildCacheInvocation { + dependency_closure_hash: "graph".into(), + platform: BuildPlatformFingerprint { + os: platform.os, + architecture: platform.cpu, + libc: platform.libc.unwrap_or_default(), + cpu_features_hash: "cpu".into(), + }, + runtime: BuildRuntimeFingerprint { + runtime: "node".into(), + version: "22.0.0".into(), + modules_abi: "127".into(), + napi: "10".into(), + engine: "v8".into(), + executable_hash: "trusted-node".into(), + }, + sandbox: BuildSandboxFingerprint { + mode: "enforce".into(), + posture: "strict".into(), + network_denied: true, + environment_scrubbed: true, + allowed_inputs_hash: "inputs".into(), + }, + environment: trusted_environment.clone(), + trusted_toolchain_environment: trusted_environment, + build_executable_environment, + project_dir: project.clone(), + native_toolchain_hash: OnceLock::new(), + }; + let mut package = scriptable_package("esbuild", "node install.js"); + package.store_path = package_dir; + package.graph_key_digest = Some(graph_key_digest); + + assert!(build_key_for_package(&invocation, &package, &project).is_none()); + } + #[test] fn toolchain_probe_reader_discards_output_beyond_limit() { let input = vec![b'x'; TOOLCHAIN_PROBE_OUTPUT_LIMIT + 1024]; From 8895d7d2d563189fe70340fbaa7e379ef6c8b9c3 Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 21:02:07 +0100 Subject: [PATCH 12/14] fix: invalidate pre-hardening runtime artifacts --- .../src/commands/rebuild/build_cache.rs | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache.rs b/crates/lpm-cli/src/commands/rebuild/build_cache.rs index 914942a6..a6d6bd35 100644 --- a/crates/lpm-cli/src/commands/rebuild/build_cache.rs +++ b/crates/lpm-cli/src/commands/rebuild/build_cache.rs @@ -17,6 +17,7 @@ use std::time::{Duration, Instant}; const TOOLCHAIN_PROBE_TIMEOUT: Duration = Duration::from_secs(2); const TOOLCHAIN_PROBE_OUTPUT_LIMIT: usize = 4 * 1024 * 1024; +const RUNTIME_ONLY_TOOLCHAIN_FINGERPRINT: &str = "runtime-and-dependency-graph-v2"; pub(super) struct BuildCacheScratch { path: PathBuf, @@ -396,7 +397,7 @@ pub(super) fn build_key_for_package( hash_invoked_build_executables(package, &invocation.build_executable_environment); hash_records([hash.as_bytes(), invoked_executables.as_bytes()]) } else { - "runtime-and-dependency-graph-v1".to_string() + RUNTIME_ONLY_TOOLCHAIN_FINGERPRINT.to_string() }; let environment_hash = hash_environment_pairs(&lifecycle_environment); let key = BuildCacheKey::derive(&BuildKeyInputs { @@ -1271,6 +1272,49 @@ mod tests { ); } + #[test] + fn runtime_only_key_namespace_invalidates_pre_hardening_artifacts() { + fn derive(toolchain_hash: &str) -> BuildCacheKey { + BuildCacheKey::derive(&BuildKeyInputs { + source_integrity: "sha512-source".into(), + graph_key_digest: "graph".into(), + dependency_closure_hash: "closure".into(), + scripts: vec![BuildScriptFingerprint { + phase: "postinstall".into(), + command: "node install.js".into(), + }], + platform: BuildPlatformFingerprint { + os: "linux".into(), + architecture: "x86_64".into(), + libc: "glibc".into(), + cpu_features_hash: "cpu".into(), + }, + runtime: BuildRuntimeFingerprint { + runtime: "node".into(), + version: "22.0.0".into(), + modules_abi: "127".into(), + napi: "10".into(), + engine: "v8".into(), + executable_hash: "node".into(), + }, + sandbox: BuildSandboxFingerprint { + mode: "enforce".into(), + posture: "strict".into(), + network_denied: true, + environment_scrubbed: true, + allowed_inputs_hash: "inputs".into(), + }, + environment_hash: "environment".into(), + toolchain_hash: toolchain_hash.into(), + }) + } + + let pre_hardening = derive("runtime-and-dependency-graph-v1"); + let current = derive(RUNTIME_ONLY_TOOLCHAIN_FINGERPRINT); + + assert_ne!(current, pre_hardening); + } + #[test] fn build_marker_reader_accepts_only_cache_key_shape() { let temp = tempfile::tempdir().unwrap(); From b8bf6611a8324f5d955b42b9ebb61dee88be0762 Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 22:43:08 +0100 Subject: [PATCH 13/14] perf: cache validated native toolchain fingerprints --- ...20260712-macos-arm64-native-toolchain.json | 40 +- ...e-20260712-macos-arm64-native-toolchain.md | 20 +- .../src/commands/rebuild/build_cache.rs | 320 +++++++--- .../rebuild/build_cache/toolchain_snapshot.rs | 559 ++++++++++++++++++ tests/workflows/tests/native_build_cache.rs | 87 +++ 5 files changed, 930 insertions(+), 96 deletions(-) create mode 100644 crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json index bd51275f..941e45d3 100644 --- a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-07-12T19:08:31.460Z", + "generated_at": "2026-07-12T21:38:02.229Z", "platform": "darwin-arm64", "node": "v22.22.1", "lpm_bin": "target/release/lpm-rs", @@ -10,40 +10,40 @@ "scenarios": { "cache_miss": { "samples": 10, - "wall_ms_median": 229.99, - "wall_ms_p95": 444.35, - "max_rss_bytes_median": 47104000, - "user_cpu_ms_median": 120, - "system_cpu_ms_median": 170, + "wall_ms_median": 173.1, + "wall_ms_p95": 280, + "max_rss_bytes_median": 47169536, + "user_cpu_ms_median": 40, + "system_cpu_ms_median": 20, "cache_hits": 0, "cache_misses": 10, "lifecycle_ms_avoided": 0 }, "local_hit": { "samples": 10, - "wall_ms_median": 122.88, - "wall_ms_p95": 131.98, + "wall_ms_median": 59.07, + "wall_ms_p95": 72.25, "max_rss_bytes_median": 41631744, - "user_cpu_ms_median": 100, - "system_cpu_ms_median": 160, + "user_cpu_ms_median": 20, + "system_cpu_ms_median": 10, "cache_hits": 10, "cache_misses": 0, - "lifecycle_ms_avoided": 1072, - "speedup_vs_miss": 1.87, - "wall_reduction_percent": 46.57 + "lifecycle_ms_avoided": 1049, + "speedup_vs_miss": 2.93, + "wall_reduction_percent": 65.88 }, "ci_warm_store_hit": { "samples": 10, - "wall_ms_median": 125.39, - "wall_ms_p95": 135.02, + "wall_ms_median": 58.49, + "wall_ms_p95": 61.69, "max_rss_bytes_median": 41615360, - "user_cpu_ms_median": 100, - "system_cpu_ms_median": 160, + "user_cpu_ms_median": 10, + "system_cpu_ms_median": 10, "cache_hits": 10, "cache_misses": 0, - "lifecycle_ms_avoided": 1073, - "speedup_vs_miss": 1.83, - "wall_reduction_percent": 45.48 + "lifecycle_ms_avoided": 1051, + "speedup_vs_miss": 2.96, + "wall_reduction_percent": 66.21 } } } diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md index 4c9ad7a6..6f11aa15 100644 --- a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md @@ -8,18 +8,20 @@ | Scenario | Median wall | p95 wall | Median RSS | User CPU | System CPU | Result | |---|---:|---:|---:|---:|---:|---:| -| Strict cache miss | 229.99 ms | 444.35 ms | 47,104,000 B | 120 ms | 170 ms | 10 misses | -| Local artifact hit | 122.88 ms | 131.98 ms | 41,631,744 B | 100 ms | 160 ms | 1.87×; 46.57% lower wall time | -| Stable-path CI warm-store hit | 125.39 ms | 135.02 ms | 41,615,360 B | 100 ms | 160 ms | 1.83×; 45.48% lower wall time | +| Strict cache miss | 173.10 ms | 280.00 ms | 47,169,536 B | 40 ms | 20 ms | 10 misses | +| Local artifact hit | 59.07 ms | 72.25 ms | 41,631,744 B | 20 ms | 10 ms | 2.93×; 65.88% lower wall time | +| Stable-path CI warm-store hit | 58.49 ms | 61.69 ms | 41,615,360 B | 10 ms | 10 ms | 2.96×; 66.21% lower wall time | Every measured scenario used the same strict sandbox. Sharp follows the -`NativeToolchain` cache path, so every hit includes trusted compiler, SDK, -package-database, pkg-config, Homebrew receipt, and Node-header fingerprinting. +`NativeToolchain` cache path, so every hit validates the persisted compiler, +SDK, package-database, pkg-config, Homebrew receipt, and Node-header snapshot. +The first measured miss computed the complete host fingerprint in 126 ms; +subsequent unchanged-host key derivations took 15–24 ms. Miss and local-hit samples rematerialized the same project. CI-like samples deleted and recreated a checkout at one stable path while retaining the warm LPM store. -Across ten local hits, artifact metadata recorded 1,072 ms of lifecycle -execution avoided. Across ten stable-path CI hits, it recorded 1,073 ms -avoided. The measured hit remains beneficial after paying the complete native -toolchain key cost. +Across ten local hits, artifact metadata recorded 1,049 ms of lifecycle +execution avoided. Across ten stable-path CI hits, it recorded 1,051 ms +avoided. The measured hit remains beneficial after paying the conservative +host-snapshot validation cost. diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache.rs b/crates/lpm-cli/src/commands/rebuild/build_cache.rs index a6d6bd35..6bcc5244 100644 --- a/crates/lpm-cli/src/commands/rebuild/build_cache.rs +++ b/crates/lpm-cli/src/commands/rebuild/build_cache.rs @@ -1,3 +1,6 @@ +use self::toolchain_snapshot::{ + ComputedToolchainFingerprint, ToolchainFingerprintCache, cached_toolchain_fingerprint, +}; use super::script_execution::{build_lifecycle_environment, build_lifecycle_path}; use super::scripts::ScriptablePackage; use crate::capability::CapabilitySet; @@ -15,10 +18,49 @@ use std::process::{Command, Stdio}; use std::sync::OnceLock; use std::time::{Duration, Instant}; +mod toolchain_snapshot; + const TOOLCHAIN_PROBE_TIMEOUT: Duration = Duration::from_secs(2); const TOOLCHAIN_PROBE_OUTPUT_LIMIT: usize = 4 * 1024 * 1024; const RUNTIME_ONLY_TOOLCHAIN_FINGERPRINT: &str = "runtime-and-dependency-graph-v2"; +const TOOLCHAIN_PROBES: &[(&str, &[&str])] = &[ + ("python3", &["--version"]), + ("make", &["--version"]), + ("cc", &["--version"]), + ("cc", &["-print-sysroot"]), + ("cc", &["-print-search-dirs"]), + ("c++", &["--version"]), + ("c++", &["-print-sysroot"]), + ("c++", &["-print-search-dirs"]), + ("clang", &["--version"]), + ("ld", &["-v"]), + ("xcrun", &["--show-sdk-path"]), + ("xcrun", &["--show-sdk-version"]), + ("rustc", &["-vV"]), + ("cargo", &["-V"]), + ("cmake", &["--version"]), + ("ninja", &["--version"]), + ("pkg-config", &["--version"]), + ("pkg-config", &["--variable=pc_path", "pkg-config"]), + ("pkg-config", &["--list-all"]), + ( + "dpkg-query", + &["-W", "-f=${Package}\t${Version}\t${Architecture}\\n"], + ), + ( + "rpm", + &["-qa", "--qf", "%{NAME}\t%{VERSION}-%{RELEASE}\t%{ARCH}\\n"], + ), + ("pacman", &["-Q"]), + ("apk", &["info", "-vv"]), + ("pkgutil", &["--pkgs"]), + ("xcodebuild", &["-version"]), + ("uname", &["-a"]), + ("sw_vers", &[]), + ("ldd", &["--version"]), +]; + pub(super) struct BuildCacheScratch { path: PathBuf, } @@ -91,6 +133,7 @@ pub(super) struct BuildCacheInvocation { trusted_toolchain_environment: HashMap, build_executable_environment: HashMap, project_dir: PathBuf, + toolchain_cache: Option, native_toolchain_hash: OnceLock>, } @@ -159,6 +202,9 @@ impl BuildCacheInvocation { trusted_toolchain_environment, build_executable_environment, project_dir: project_dir.to_path_buf(), + toolchain_cache: lpm_common::LpmRoot::from_env() + .ok() + .map(ToolchainFingerprintCache::from_lpm_root), native_toolchain_hash: OnceLock::new(), }) } @@ -381,10 +427,11 @@ pub(super) fn build_key_for_package( let Some(hash) = invocation .native_toolchain_hash .get_or_init(|| { - hash_toolchain( + hash_toolchain_cached( &invocation.trusted_toolchain_environment, &invocation.build_executable_environment, &invocation.project_dir, + invocation.toolchain_cache.as_ref(), ) .ok() }) @@ -590,9 +637,10 @@ fn is_project_controlled_path(path: &Path, project_dir: &Path) -> bool { .any(|component| component.as_os_str() == "node_modules") } +#[derive(Clone)] struct TrustedExecutable { invocation_path: PathBuf, - canonical_target: PathBuf, + identity: String, } fn resolve_trusted_executable( @@ -602,11 +650,14 @@ fn resolve_trusted_executable( ) -> Option { let invocation_path = resolve_executable(executable, environment)?; let canonical_target = std::fs::canonicalize(&invocation_path).ok()?; - (!is_project_controlled_path(&invocation_path, project_dir) - && !is_project_controlled_path(&canonical_target, project_dir)) - .then_some(TrustedExecutable { + if is_project_controlled_path(&invocation_path, project_dir) + || is_project_controlled_path(&canonical_target, project_dir) + { + return None; + } + Some(TrustedExecutable { invocation_path, - canonical_target, + identity: runtime_executable_identity(&canonical_target).ok()?, }) } @@ -726,62 +777,141 @@ fn run_bounded_toolchain_probe( }) } +#[cfg(test)] fn hash_toolchain( trusted_environment: &HashMap, build_environment: &HashMap, project_dir: &Path, ) -> std::io::Result { - const PROBES: &[(&str, &[&str])] = &[ - ("python3", &["--version"]), - ("make", &["--version"]), - ("cc", &["--version"]), - ("cc", &["-print-sysroot"]), - ("cc", &["-print-search-dirs"]), - ("c++", &["--version"]), - ("c++", &["-print-sysroot"]), - ("c++", &["-print-search-dirs"]), - ("clang", &["--version"]), - ("ld", &["-v"]), - ("xcrun", &["--show-sdk-path"]), - ("xcrun", &["--show-sdk-version"]), - ("rustc", &["-vV"]), - ("cargo", &["-V"]), - ("cmake", &["--version"]), - ("ninja", &["--version"]), - ("pkg-config", &["--version"]), - ("pkg-config", &["--variable=pc_path", "pkg-config"]), - ("pkg-config", &["--list-all"]), - ( - "dpkg-query", - &["-W", "-f=${Package}\t${Version}\t${Architecture}\\n"], - ), - ( - "rpm", - &["-qa", "--qf", "%{NAME}\t%{VERSION}-%{RELEASE}\t%{ARCH}\\n"], - ), - ("pacman", &["-Q"]), - ("apk", &["info", "-vv"]), - ("pkgutil", &["--pkgs"]), - ("xcodebuild", &["-version"]), - ("uname", &["-a"]), - ("sw_vers", &[]), - ("ldd", &["--version"]), - ]; + hash_toolchain_cached(trusted_environment, build_environment, project_dir, None) +} + +fn hash_toolchain_cached( + trusted_environment: &HashMap, + build_environment: &HashMap, + project_dir: &Path, + cache: Option<&ToolchainFingerprintCache>, +) -> std::io::Result { + let mut resolved = HashMap::with_capacity(TOOLCHAIN_PROBES.len()); + let trusted_executables = TOOLCHAIN_PROBES + .iter() + .map(|(program, _)| { + resolved + .entry(*program) + .or_insert_with(|| { + resolve_trusted_executable(program, trusted_environment, project_dir) + }) + .clone() + }) + .collect::>(); + let base_key = + toolchain_snapshot_base_key(trusted_environment, &trusted_executables, project_dir); + let mut compute = || compute_toolchain_fingerprint(trusted_environment, &trusted_executables); + let host_fingerprint = match cache { + Some(cache) => cached_toolchain_fingerprint( + cache, + &base_key, + trusted_environment, + project_dir, + &mut compute, + )?, + None => compute()?.fingerprint, + }; + let build_visible_fingerprint = hash_build_visible_toolchain_executables(build_environment); + Ok(hash_records([ + host_fingerprint.as_bytes(), + build_visible_fingerprint.as_bytes(), + ])) +} + +fn toolchain_snapshot_base_key( + trusted_environment: &HashMap, + trusted_executables: &[Option], + project_dir: &Path, +) -> String { let mut hasher = Sha256::new(); - hasher.update(b"lpm-toolchain-v5\0"); + hasher.update(b"lpm-toolchain-snapshot-base-v1\0"); + hasher.update(std::env::consts::OS.as_bytes()); + hasher.update(b"\0"); + hasher.update(std::env::consts::ARCH.as_bytes()); + hasher.update(b"\0"); + hasher.update(env!("CARGO_PKG_VERSION").as_bytes()); + hasher.update(b"\0"); + let probe_environment = toolchain_probe_environment(trusted_environment); + hasher.update(hash_string_map(&probe_environment).as_bytes()); + if let Some(home) = trusted_environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("HOME")) + .map(|(_, value)| value) + { + hash_os_path(&mut hasher, Path::new(home)); + } + for (index, ((program, args), executable)) in + TOOLCHAIN_PROBES.iter().zip(trusted_executables).enumerate() + { + hasher.update(index.to_le_bytes()); + hasher.update(program.as_bytes()); + for argument in *args { + hasher.update(b"\0"); + hasher.update(argument.as_bytes()); + } + if let Some(executable) = executable + && !is_project_controlled_path(&executable.invocation_path, project_dir) + { + hash_os_path(&mut hasher, &executable.invocation_path); + hasher.update(executable.identity.as_bytes()); + } + hasher.update(b"\x1e"); + } + format!("sha256-{}", hex::encode(hasher.finalize())) +} + +fn hash_build_visible_toolchain_executables(build_environment: &HashMap) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"lpm-build-visible-toolchain-v1\0"); + let mut identities = HashMap::with_capacity(TOOLCHAIN_PROBES.len()); + for (program, args) in TOOLCHAIN_PROBES { + hasher.update(program.as_bytes()); + for argument in *args { + hasher.update(b"\0"); + hasher.update(argument.as_bytes()); + } + if let Some((executable, identity)) = identities + .entry(*program) + .or_insert_with(|| { + let executable = resolve_executable(program, build_environment)?; + let identity = runtime_executable_identity(&executable).ok()?; + Some((executable, identity)) + }) + .as_ref() + { + hash_os_path(&mut hasher, executable); + hasher.update(identity.as_bytes()); + } + hasher.update(b"\x1e"); + } + format!("sha256-{}", hex::encode(hasher.finalize())) +} + +fn compute_toolchain_fingerprint( + trusted_environment: &HashMap, + trusted_executables: &[Option], +) -> std::io::Result { + let mut hasher = Sha256::new(); + hasher.update(b"lpm-toolchain-host-v6\0"); let probe_environment = toolchain_probe_environment(trusted_environment); let probe_outputs = std::thread::scope(|scope| { #[expect( clippy::needless_collect, reason = "all probes must start before any join blocks" )] - let handles = PROBES + let handles = TOOLCHAIN_PROBES .iter() - .map(|(program, args)| { + .zip(trusted_executables) + .map(|((_, args), executable)| { let probe_environment = &probe_environment; scope.spawn(move || { - let executable = - resolve_trusted_executable(program, trusted_environment, project_dir)?; + let executable = executable.as_ref()?; run_bounded_toolchain_probe( &executable.invocation_path, args, @@ -795,7 +925,11 @@ fn hash_toolchain( .map(|handle| handle.join().ok().flatten()) .collect::>() }); - for ((program, args), output) in PROBES.iter().zip(&probe_outputs) { + for (((program, args), executable), output) in TOOLCHAIN_PROBES + .iter() + .zip(trusted_executables) + .zip(&probe_outputs) + { hasher.update(program.as_bytes()); hasher.update(b"\0"); for argument in *args { @@ -806,19 +940,10 @@ fn hash_toolchain( hasher.update(&output.stdout); hasher.update(&output.stderr); } - if let Some(executable) = - resolve_trusted_executable(program, trusted_environment, project_dir) - && let Ok(identity) = runtime_executable_identity(&executable.canonical_target) - { + if let Some(executable) = executable { hasher.update(b"trusted\0"); hash_os_path(&mut hasher, &executable.invocation_path); - hasher.update(identity.as_bytes()); - } - if let Some(executable) = resolve_executable(program, build_environment) - && let Ok(identity) = runtime_executable_identity(&executable) - { - hasher.update(b"build-visible\0"); - hasher.update(identity.as_bytes()); + hasher.update(executable.identity.as_bytes()); } hasher.update(b"\x1e"); } @@ -835,7 +960,7 @@ fn hash_toolchain( .collect::>(); hash_host_package_state(&host_state_paths, &mut hasher)?; let pkg_config_pc_path = - PROBES + TOOLCHAIN_PROBES .iter() .zip(&probe_outputs) .find_map(|((program, args), output)| { @@ -845,7 +970,8 @@ fn hash_toolchain( .then(|| output.as_ref().map(|value| value.stdout.as_slice())) .flatten() }); - hash_pkg_config_directories(trusted_environment, pkg_config_pc_path, &mut hasher)?; + let pkg_config_paths = pkg_config_search_paths(trusted_environment, pkg_config_pc_path); + hash_pkg_config_directories(&pkg_config_paths, &mut hasher)?; for cellar in [ Path::new("/opt/homebrew/Cellar"), Path::new("/usr/local/Cellar"), @@ -868,7 +994,22 @@ fn hash_toolchain( { hash_directory_tree(&home.join(".node-gyp"), &mut hasher)?; } - Ok(format!("sha256-{}", hex::encode(hasher.finalize()))) + Ok(ComputedToolchainFingerprint { + fingerprint: format!("sha256-{}", hex::encode(hasher.finalize())), + pkg_config_paths, + }) +} + +fn hash_string_map(values: &HashMap) -> String { + let mut values = values.iter().collect::>(); + values.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + hash_records(values.into_iter().map(|(key, value)| { + let mut record = String::with_capacity(key.len() + value.len() + 1); + record.push_str(key); + record.push('\0'); + record.push_str(value); + record + })) } #[cfg(target_os = "linux")] @@ -902,11 +1043,10 @@ fn hash_host_package_state(paths: &[&Path], hasher: &mut Sha256) -> std::io::Res Ok(()) } -fn hash_pkg_config_directories( +fn pkg_config_search_paths( environment: &HashMap, default_search_path: Option<&[u8]>, - hasher: &mut Sha256, -) -> std::io::Result<()> { +) -> Vec { let mut search_path = environment .iter() .find(|(key, _)| key.eq_ignore_ascii_case("PKG_CONFIG_PATH")) @@ -918,11 +1058,17 @@ fn hash_pkg_config_directories( } search_path.push_str(String::from_utf8_lossy(default_search_path).trim()); } - let mut paths = std::env::split_paths(&search_path).collect::>(); + let mut paths = std::env::split_paths(&search_path) + .filter(|path| !path.as_os_str().is_empty()) + .collect::>(); paths.sort_unstable(); paths.dedup(); + paths +} + +fn hash_pkg_config_directories(paths: &[PathBuf], hasher: &mut Sha256) -> std::io::Result<()> { for path in paths { - hash_directory_tree(&path, hasher)?; + hash_directory_tree(path, hasher)?; } Ok(()) } @@ -1367,6 +1513,45 @@ mod tests { assert_ne!(first.finalize(), second.finalize()); } + #[test] + #[cfg(unix)] + fn persisted_toolchain_snapshot_skips_unchanged_trusted_probe() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let host_bin = temp.path().join("host-bin"); + let home = temp.path().join("home"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&host_bin).unwrap(); + std::fs::create_dir(&home).unwrap(); + let marker = temp.path().join("cmake-probes"); + let cmake = host_bin.join("cmake"); + std::fs::write( + &cmake, + format!( + "#!/bin/sh\nprintf 'probe\\n' >> '{}'\necho cmake-one\n", + marker.display() + ), + ) + .unwrap(); + std::fs::set_permissions(&cmake, std::fs::Permissions::from_mode(0o755)).unwrap(); + let environment = HashMap::from([ + ("PATH".to_string(), host_bin.display().to_string()), + ("HOME".to_string(), home.display().to_string()), + ]); + let cache = ToolchainFingerprintCache::for_test(temp.path().join("cache")); + + let first = + hash_toolchain_cached(&environment, &environment, &project, Some(&cache)).unwrap(); + assert_eq!(std::fs::read_to_string(&marker).unwrap(), "probe\n"); + let second = + hash_toolchain_cached(&environment, &environment, &project, Some(&cache)).unwrap(); + + assert_eq!(first, second); + assert_eq!(std::fs::read_to_string(marker).unwrap(), "probe\n"); + } + #[test] fn homebrew_state_hash_changes_when_install_receipt_changes() { let temp = tempfile::tempdir().unwrap(); @@ -1567,6 +1752,7 @@ mod tests { trusted_toolchain_environment: trusted_environment, build_executable_environment, project_dir: project.clone(), + toolchain_cache: None, native_toolchain_hash: OnceLock::new(), }; let mut package = scriptable_package("esbuild", "node install.js"); diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs b/crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs new file mode 100644 index 00000000..c5a5ccfc --- /dev/null +++ b/crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs @@ -0,0 +1,559 @@ +use super::{ + hash_optional_file, hash_os_path, hash_records, host_package_state_paths, + is_project_controlled_path, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +const MAX_AGE: Duration = Duration::from_secs(60 * 60); +const MAX_BYTES: u64 = 256 * 1024; +const SCHEMA: u32 = 1; + +#[derive(Clone)] +pub(super) struct ToolchainFingerprintCache { + directory: PathBuf, +} + +impl ToolchainFingerprintCache { + pub(super) fn from_lpm_root(root: lpm_common::LpmRoot) -> Self { + Self { + directory: root.cache_metadata().join("native-toolchains/v1"), + } + } + + #[cfg(test)] + pub(super) fn for_test(directory: PathBuf) -> Self { + Self { directory } + } + + fn entry_path(&self, base_key: &str) -> Option { + let digest = base_key.strip_prefix("sha256-")?; + is_lower_hex_digest(digest).then(|| self.directory.join(format!("{digest}.json"))) + } +} + +#[derive(Serialize, Deserialize)] +struct ToolchainFingerprintSnapshot { + schema: u32, + base_key: String, + fingerprint: String, + validation_hash: String, + pkg_config_paths: Vec, + created_unix_seconds: u64, + entry_hash: String, +} + +pub(super) struct ComputedToolchainFingerprint { + pub(super) fingerprint: String, + pub(super) pkg_config_paths: Vec, +} + +pub(super) fn cached_toolchain_fingerprint( + cache: &ToolchainFingerprintCache, + base_key: &str, + environment: &HashMap, + project_dir: &Path, + compute: &mut F, +) -> std::io::Result +where + F: FnMut() -> std::io::Result, +{ + let Some(entry_path) = cache.entry_path(base_key) else { + return Ok(compute()?.fingerprint); + }; + if std::fs::create_dir_all(&cache.directory).is_err() { + return Ok(compute()?.fingerprint); + } + let lock_path = entry_path.with_extension("lock"); + let Ok(_lock) = lpm_common::acquire_exclusive_lock(&lock_path) else { + return Ok(compute()?.fingerprint); + }; + if let Some(snapshot) = read_snapshot(&entry_path, base_key) + && snapshot_paths_are_allowed(&snapshot, environment, project_dir) + && validation_hash(environment, &snapshot.pkg_config_paths) + .is_ok_and(|current| current == snapshot.validation_hash) + { + tracing::debug!(target: "lpm_cli::build_cache", "native toolchain snapshot hit"); + return Ok(snapshot.fingerprint); + } + tracing::debug!(target: "lpm_cli::build_cache", "native toolchain snapshot miss"); + let computed = compute()?; + let validation_hash = validation_hash(environment, &computed.pkg_config_paths)?; + let created_unix_seconds = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let mut snapshot = ToolchainFingerprintSnapshot { + schema: SCHEMA, + base_key: base_key.to_owned(), + fingerprint: computed.fingerprint.clone(), + validation_hash, + pkg_config_paths: computed.pkg_config_paths, + created_unix_seconds, + entry_hash: String::new(), + }; + snapshot.entry_hash = entry_hash(&snapshot); + if let Err(error) = write_snapshot(&entry_path, &snapshot) { + tracing::debug!( + target: "lpm_cli::build_cache", + path = %entry_path.display(), + "failed to persist native toolchain snapshot: {error}" + ); + } + Ok(computed.fingerprint) +} + +fn read_snapshot(path: &Path, expected_base_key: &str) -> Option { + let file = std::fs::File::open(path).ok()?; + let metadata = file.metadata().ok()?; + if metadata.len() > MAX_BYTES { + return None; + } + let snapshot: ToolchainFingerprintSnapshot = serde_json::from_reader(file).ok()?; + if snapshot.schema != SCHEMA + || snapshot.base_key != expected_base_key + || !is_sha256_fingerprint(&snapshot.fingerprint) + || !is_sha256_fingerprint(&snapshot.validation_hash) + || snapshot.pkg_config_paths.len() > 128 + || snapshot.entry_hash != entry_hash(&snapshot) + { + return None; + } + let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?.as_secs(); + let age = now.checked_sub(snapshot.created_unix_seconds)?; + (age <= MAX_AGE.as_secs()).then_some(snapshot) +} + +fn snapshot_paths_are_allowed( + snapshot: &ToolchainFingerprintSnapshot, + environment: &HashMap, + project_dir: &Path, +) -> bool { + let configured_paths = environment + .iter() + .filter(|(key, _)| { + key.eq_ignore_ascii_case("PKG_CONFIG_PATH") + || key.eq_ignore_ascii_case("PKG_CONFIG_LIBDIR") + }) + .flat_map(|(_, value)| std::env::split_paths(value)) + .collect::>(); + snapshot.pkg_config_paths.iter().all(|path| { + let is_configured = configured_paths.iter().any(|configured| configured == path); + path.is_absolute() + && path.as_os_str().len() <= 4096 + && (is_configured + || (path.file_name().is_some_and(|name| name == "pkgconfig") + && !is_project_controlled_path(path, project_dir))) + }) +} + +fn entry_hash(snapshot: &ToolchainFingerprintSnapshot) -> String { + let mut records = Vec::with_capacity(snapshot.pkg_config_paths.len() + 6); + records.push(format!("schema\0{}", snapshot.schema)); + records.push(format!("base\0{}", snapshot.base_key)); + records.push(format!("fingerprint\0{}", snapshot.fingerprint)); + records.push(format!("validation\0{}", snapshot.validation_hash)); + records.push(format!("created\0{}", snapshot.created_unix_seconds)); + records.extend( + snapshot + .pkg_config_paths + .iter() + .map(|path| format!("pkg-config\0{}", path.display())), + ); + hash_records(records.iter().map(String::as_bytes)) +} + +fn write_snapshot(path: &Path, snapshot: &ToolchainFingerprintSnapshot) -> std::io::Result<()> { + let parent = path + .parent() + .ok_or_else(|| std::io::Error::other("toolchain snapshot has no parent directory"))?; + let mut temporary = tempfile::Builder::new() + .prefix(".native-toolchain-") + .tempfile_in(parent)?; + serde_json::to_writer(temporary.as_file_mut(), snapshot)?; + temporary.as_file_mut().write_all(b"\n")?; + temporary.as_file_mut().sync_all()?; + temporary + .persist(path) + .map_err(|error| error.error) + .map(|_| ()) +} + +fn validation_hash( + environment: &HashMap, + pkg_config_paths: &[PathBuf], +) -> std::io::Result { + let mut hasher = Sha256::new(); + hasher.update(b"lpm-toolchain-validation-v1\0"); + for path in [ + Path::new("/etc/os-release"), + Path::new("/etc/ld.so.cache"), + Path::new("/etc/hostname"), + Path::new("/proc/sys/kernel/osrelease"), + Path::new("/proc/sys/kernel/version"), + Path::new("/System/Library/CoreServices/SystemVersion.plist"), + Path::new("/var/db/xcode_select_link"), + ] { + hash_path_metadata(path, &mut hasher)?; + if path != Path::new("/var/db/xcode_select_link") { + hash_optional_file(path, &mut hasher)?; + } + } + for path in host_package_state_paths().iter().map(Path::new) { + hash_metadata_tree(path, &mut hasher)?; + } + for path in pkg_config_paths { + hash_metadata_tree(path, &mut hasher)?; + } + for cellar in [ + Path::new("/opt/homebrew/Cellar"), + Path::new("/usr/local/Cellar"), + Path::new("/home/linuxbrew/.linuxbrew/Cellar"), + ] { + hash_homebrew_metadata(cellar, &mut hasher)?; + } + #[cfg(target_os = "macos")] + for receipts in [ + Path::new("/var/db/receipts"), + Path::new("/Library/Apple/System/Library/Receipts"), + ] { + hash_extension_metadata(receipts, "plist", &mut hasher)?; + } + if let Some(home) = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("HOME")) + .map(|(_, value)| PathBuf::from(value)) + .or_else(dirs::home_dir) + { + hash_metadata_tree(&home.join(".node-gyp"), &mut hasher)?; + hash_rustup_metadata(&home.join(".rustup"), &mut hasher)?; + } + Ok(format!("sha256-{}", hex::encode(hasher.finalize()))) +} + +fn hash_metadata_tree(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hasher.update(b"metadata-tree-v1\0"); + hash_os_path(hasher, root); + let root_metadata = match std::fs::symlink_metadata(root) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hasher.update(b"absent\x1e"); + return Ok(()); + } + Err(error) => return Err(error), + }; + hash_metadata_record(hasher, &root_metadata); + if !root_metadata.is_dir() || root_metadata.file_type().is_symlink() { + return Ok(()); + } + let mut pending = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + let mut entries = std::fs::read_dir(&directory)?.collect::, _>>()?; + entries.sort_unstable_by_key(std::fs::DirEntry::file_name); + for entry in entries { + let path = entry.path(); + hash_os_path(hasher, path.strip_prefix(root).unwrap_or(&path)); + let metadata = std::fs::symlink_metadata(&path)?; + hash_metadata_record(hasher, &metadata); + if metadata.file_type().is_symlink() { + hash_os_path(hasher, &std::fs::read_link(path)?); + } else if metadata.is_dir() { + pending.push(path); + } + hasher.update(b"\x1e"); + } + } + Ok(()) +} + +fn hash_path_metadata(path: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hash_os_path(hasher, path); + match std::fs::symlink_metadata(path) { + Ok(metadata) => { + hash_metadata_record(hasher, &metadata); + if metadata.file_type().is_symlink() { + hash_os_path(hasher, &std::fs::read_link(path)?); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => hasher.update(b"absent"), + Err(error) => return Err(error), + } + hasher.update(b"\x1e"); + Ok(()) +} + +fn hash_metadata_record(hasher: &mut Sha256, metadata: &std::fs::Metadata) { + let file_type = metadata.file_type(); + hasher.update(if file_type.is_dir() { + b"d".as_slice() + } else if file_type.is_file() { + b"f".as_slice() + } else if file_type.is_symlink() { + b"l".as_slice() + } else { + b"s".as_slice() + }); + hasher.update(metadata.len().to_le_bytes()); + if let Ok(modified) = metadata.modified() + && let Ok(duration) = modified.duration_since(UNIX_EPOCH) + { + hasher.update(duration.as_secs().to_le_bytes()); + hasher.update(duration.subsec_nanos().to_le_bytes()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + hasher.update(metadata.dev().to_le_bytes()); + hasher.update(metadata.ino().to_le_bytes()); + hasher.update(metadata.mode().to_le_bytes()); + hasher.update(metadata.ctime().to_le_bytes()); + hasher.update(metadata.ctime_nsec().to_le_bytes()); + } +} + +fn hash_homebrew_metadata(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hash_os_path(hasher, root); + let mut formulae = match std::fs::read_dir(root) { + Ok(entries) => entries.collect::, _>>()?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hasher.update(b"absent\x1e"); + return Ok(()); + } + Err(error) => return Err(error), + }; + formulae.sort_unstable_by_key(std::fs::DirEntry::file_name); + for formula in formulae { + if !formula.file_type()?.is_dir() { + continue; + } + hash_os_path(hasher, Path::new(&formula.file_name())); + let mut versions = std::fs::read_dir(formula.path())?.collect::, _>>()?; + versions.sort_unstable_by_key(std::fs::DirEntry::file_name); + for version in versions { + if version.file_type()?.is_dir() { + hash_os_path(hasher, Path::new(&version.file_name())); + hash_path_metadata(&version.path().join("INSTALL_RECEIPT.json"), hasher)?; + } + } + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn hash_extension_metadata( + root: &Path, + extension: &str, + hasher: &mut Sha256, +) -> std::io::Result<()> { + hash_os_path(hasher, root); + let mut paths = match std::fs::read_dir(root) { + Ok(entries) => entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|value| value == extension)) + .collect::>(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hasher.update(b"absent\x1e"); + return Ok(()); + } + Err(error) => return Err(error), + }; + paths.sort_unstable(); + for path in paths { + hash_path_metadata(&path, hasher)?; + } + Ok(()) +} + +fn hash_rustup_metadata(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hash_path_metadata(&root.join("settings.toml"), hasher)?; + let toolchains = root.join("toolchains"); + let mut entries = match std::fs::read_dir(&toolchains) { + Ok(entries) => entries.collect::, _>>()?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + entries.sort_unstable_by_key(std::fs::DirEntry::file_name); + for entry in entries { + if !entry.file_type()?.is_dir() { + continue; + } + hash_os_path(hasher, Path::new(&entry.file_name())); + for executable in ["rustc", "cargo"] { + hash_path_metadata(&entry.path().join("bin").join(executable), hasher)?; + } + } + Ok(()) +} + +fn is_lower_hex_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + +fn is_sha256_fingerprint(value: &str) -> bool { + value + .strip_prefix("sha256-") + .is_some_and(is_lower_hex_digest) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + fn environment(temp: &tempfile::TempDir) -> HashMap { + HashMap::from([( + "HOME".to_string(), + temp.path().join("home").display().to_string(), + )]) + } + + #[test] + fn unchanged_tracked_inputs_reuse_persisted_fingerprint() { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let tracked = temp.path().join("pkgconfig"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&tracked).unwrap(); + std::fs::write(tracked.join("native.pc"), b"Version: 1\n").unwrap(); + let environment = environment(&temp); + let cache = ToolchainFingerprintCache::for_test(temp.path().join("cache")); + let base_key = format!("sha256-{}", "b".repeat(64)); + let computations = Cell::new(0_u32); + let mut compute = || { + computations.set(computations.get() + 1); + Ok(ComputedToolchainFingerprint { + fingerprint: format!("sha256-{}", "1".repeat(64)), + pkg_config_paths: vec![tracked.clone()], + }) + }; + + let first = + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + let second = + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + + assert_eq!(first, second); + assert_eq!(computations.get(), 1); + } + + #[test] + fn changed_tracked_input_recomputes_persisted_fingerprint() { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let tracked = temp.path().join("pkgconfig"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&tracked).unwrap(); + let tracked_file = tracked.join("native.pc"); + std::fs::write(&tracked_file, b"Version: 1\n").unwrap(); + let environment = environment(&temp); + let cache = ToolchainFingerprintCache::for_test(temp.path().join("cache")); + let base_key = format!("sha256-{}", "b".repeat(64)); + let computations = Cell::new(0_u32); + let mut compute = || { + let count = computations.get() + 1; + computations.set(count); + Ok(ComputedToolchainFingerprint { + fingerprint: format!("sha256-{}", count.to_string().repeat(64)), + pkg_config_paths: vec![tracked.clone()], + }) + }; + + let first = + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + std::fs::write(&tracked_file, b"Version: 2 with a different size\n").unwrap(); + let second = + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + + assert_ne!(first, second); + assert_eq!(computations.get(), 2); + } + + #[test] + fn corrupt_snapshot_is_ignored_and_replaced() { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + std::fs::create_dir(&project).unwrap(); + let environment = environment(&temp); + let cache = ToolchainFingerprintCache::for_test(temp.path().join("cache")); + let base_key = format!("sha256-{}", "b".repeat(64)); + let entry = cache.entry_path(&base_key).unwrap(); + std::fs::create_dir_all(entry.parent().unwrap()).unwrap(); + std::fs::write(&entry, b"{not valid json").unwrap(); + let computations = Cell::new(0_u32); + let mut compute = || { + computations.set(computations.get() + 1); + Ok(ComputedToolchainFingerprint { + fingerprint: format!("sha256-{}", "1".repeat(64)), + pkg_config_paths: Vec::new(), + }) + }; + + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + + assert_eq!(computations.get(), 1); + assert!(read_snapshot(&entry, &base_key).is_some()); + } + + #[test] + fn expired_snapshot_recomputes_fingerprint() { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + std::fs::create_dir(&project).unwrap(); + let environment = environment(&temp); + let cache = ToolchainFingerprintCache::for_test(temp.path().join("cache")); + let base_key = format!("sha256-{}", "b".repeat(64)); + let computations = Cell::new(0_u32); + let mut compute = || { + let count = computations.get() + 1; + computations.set(count); + Ok(ComputedToolchainFingerprint { + fingerprint: format!("sha256-{}", count.to_string().repeat(64)), + pkg_config_paths: Vec::new(), + }) + }; + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + let entry = cache.entry_path(&base_key).unwrap(); + let mut snapshot = read_snapshot(&entry, &base_key).unwrap(); + snapshot.created_unix_seconds = 0; + snapshot.entry_hash = entry_hash(&snapshot); + write_snapshot(&entry, &snapshot).unwrap(); + + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + + assert_eq!(computations.get(), 2); + } + + #[test] + fn snapshot_rejects_unconfigured_broad_scan_root() { + let snapshot = ToolchainFingerprintSnapshot { + schema: SCHEMA, + base_key: format!("sha256-{}", "b".repeat(64)), + fingerprint: format!("sha256-{}", "1".repeat(64)), + validation_hash: format!("sha256-{}", "2".repeat(64)), + pkg_config_paths: vec![PathBuf::from("/")], + created_unix_seconds: 1, + entry_hash: String::new(), + }; + + assert!(!snapshot_paths_are_allowed( + &snapshot, + &HashMap::new(), + Path::new("/project") + )); + } +} diff --git a/tests/workflows/tests/native_build_cache.rs b/tests/workflows/tests/native_build_cache.rs index 1d3caea0..c7794506 100644 --- a/tests/workflows/tests/native_build_cache.rs +++ b/tests/workflows/tests/native_build_cache.rs @@ -250,6 +250,93 @@ fs.writeFileSync('native-output.txt', process.env.LPM_NATIVE_TEST_INPUT); ); } +#[tokio::test] +async fn native_toolchain_snapshot_is_reused_across_rebuild_processes() { + use std::os::unix::fs::MetadataExt; + + if !node_available() { + eprintln!("skipping: node is required for the native build-cache workflow"); + return; + } + + let registry = MockRegistry::start().await; + registry + .with_manifest_package( + serde_json::json!({ + "name": "sharp", + "version": PACKAGE_VERSION, + "scripts": { + "postinstall": "node install/check" + } + }), + &[( + "install/check", + b"require('fs').writeFileSync('native-output.txt', 'built');", + )], + ) + .await; + let project = TempProject::empty(&format!( + r#"{{ + "name": "native-toolchain-snapshot-workflow", + "version": "1.0.0", + "dependencies": {{ + "sharp": "{PACKAGE_VERSION}" + }} +}}"# + )); + let install = lpm_with_registry(&project, ®istry.url()) + .arg("install") + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("initial install"); + assert!( + install.status.success(), + "initial install failed: {}", + String::from_utf8_lossy(&install.stderr) + ); + + let first_rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("first native rebuild"); + assert!( + first_rebuild.status.success(), + "first rebuild failed: {}", + String::from_utf8_lossy(&first_rebuild.stderr) + ); + + let snapshot_dir = project.cache_dir().join("metadata/native-toolchains/v1"); + let snapshots = std::fs::read_dir(&snapshot_dir) + .expect("native toolchain snapshot directory") + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "json") + }) + .collect::>(); + assert_eq!(snapshots.len(), 1); + let inode_after_first_rebuild = std::fs::metadata(&snapshots[0]).unwrap().ino(); + + let second_rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("second native rebuild"); + assert!( + second_rebuild.status.success(), + "second rebuild failed: {}", + String::from_utf8_lossy(&second_rebuild.stderr) + ); + + assert_eq!( + std::fs::metadata(&snapshots[0]).unwrap().ino(), + inode_after_first_rebuild, + "an unchanged host snapshot must be read without being replaced" + ); +} + #[tokio::test] async fn concurrent_native_rebuilds_execute_one_lifecycle_build_per_key() { if !node_available() { From 2f6d7e78a21f9c55e9d1012509150e5719ceb20c Mon Sep 17 00:00:00 2001 From: Codex Merge Simulation Date: Sun, 12 Jul 2026 23:06:39 +0100 Subject: [PATCH 14/14] test: allow conservative toolchain snapshot refresh --- .../rebuild/build_cache/toolchain_snapshot.rs | 26 ++++++++++++++----- tests/workflows/tests/native_build_cache.rs | 21 ++++++++------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs b/crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs index c5a5ccfc..ace1d5c5 100644 --- a/crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs +++ b/crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs @@ -424,7 +424,8 @@ mod tests { std::fs::create_dir(&tracked).unwrap(); std::fs::write(tracked.join("native.pc"), b"Version: 1\n").unwrap(); let environment = environment(&temp); - let cache = ToolchainFingerprintCache::for_test(temp.path().join("cache")); + let cache_dir = temp.path().join("cache"); + let first_cache = ToolchainFingerprintCache::for_test(cache_dir.clone()); let base_key = format!("sha256-{}", "b".repeat(64)); let computations = Cell::new(0_u32); let mut compute = || { @@ -435,12 +436,23 @@ mod tests { }) }; - let first = - cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) - .unwrap(); - let second = - cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) - .unwrap(); + let first = cached_toolchain_fingerprint( + &first_cache, + &base_key, + &environment, + &project, + &mut compute, + ) + .unwrap(); + let second_cache = ToolchainFingerprintCache::for_test(cache_dir); + let second = cached_toolchain_fingerprint( + &second_cache, + &base_key, + &environment, + &project, + &mut compute, + ) + .unwrap(); assert_eq!(first, second); assert_eq!(computations.get(), 1); diff --git a/tests/workflows/tests/native_build_cache.rs b/tests/workflows/tests/native_build_cache.rs index c7794506..b1a0f57e 100644 --- a/tests/workflows/tests/native_build_cache.rs +++ b/tests/workflows/tests/native_build_cache.rs @@ -251,9 +251,7 @@ fs.writeFileSync('native-output.txt', process.env.LPM_NATIVE_TEST_INPUT); } #[tokio::test] -async fn native_toolchain_snapshot_is_reused_across_rebuild_processes() { - use std::os::unix::fs::MetadataExt; - +async fn native_toolchain_snapshot_persists_across_rebuild_processes() { if !node_available() { eprintln!("skipping: node is required for the native build-cache workflow"); return; @@ -317,7 +315,11 @@ async fn native_toolchain_snapshot_is_reused_across_rebuild_processes() { }) .collect::>(); assert_eq!(snapshots.len(), 1); - let inode_after_first_rebuild = std::fs::metadata(&snapshots[0]).unwrap().ino(); + let first_snapshot: serde_json::Value = serde_json::from_slice( + &std::fs::read(&snapshots[0]).expect("persisted native toolchain snapshot"), + ) + .expect("native toolchain snapshot must be valid JSON"); + assert_eq!(first_snapshot["schema"], 1); let second_rebuild = lpm_with_registry(&project, ®istry.url()) .args(["rebuild", "--all", "--strict-sandbox"]) @@ -330,11 +332,12 @@ async fn native_toolchain_snapshot_is_reused_across_rebuild_processes() { String::from_utf8_lossy(&second_rebuild.stderr) ); - assert_eq!( - std::fs::metadata(&snapshots[0]).unwrap().ino(), - inode_after_first_rebuild, - "an unchanged host snapshot must be read without being replaced" - ); + let second_snapshot: serde_json::Value = serde_json::from_slice( + &std::fs::read(&snapshots[0]).expect("native toolchain snapshot after second rebuild"), + ) + .expect("native toolchain snapshot must remain valid JSON"); + assert_eq!(second_snapshot["schema"], 1); + assert_eq!(second_snapshot["base_key"], first_snapshot["base_key"]); } #[tokio::test]