diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2f71ee0..59326f38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,9 @@ jobs: - name: Build workspace run: cargo build --workspace --locked + - name: Native build cache workflow (Linux sandbox) + run: cargo nextest run --locked -p lpm-workflows --test native_build_cache --no-fail-fast --status-level slow --final-status-level fail + - name: Unit and integration tests (non-workflow, non-CLI) run: cargo nextest run --locked --workspace --exclude lpm-workflows --exclude lpm-cli --no-fail-fast --status-level slow --final-status-level fail diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json new file mode 100644 index 00000000..941e45d3 --- /dev/null +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.json @@ -0,0 +1,49 @@ +{ + "generated_at": "2026-07-12T21:38:02.229Z", + "platform": "darwin-arm64", + "node": "v22.22.1", + "lpm_bin": "target/release/lpm-rs", + "fixture": "bench/audit-fixtures/native/sharp-image", + "samples": 10, + "artifact_count": 2, + "artifact_bytes": 503469, + "scenarios": { + "cache_miss": { + "samples": 10, + "wall_ms_median": 173.1, + "wall_ms_p95": 280, + "max_rss_bytes_median": 47169536, + "user_cpu_ms_median": 40, + "system_cpu_ms_median": 20, + "cache_hits": 0, + "cache_misses": 10, + "lifecycle_ms_avoided": 0 + }, + "local_hit": { + "samples": 10, + "wall_ms_median": 59.07, + "wall_ms_p95": 72.25, + "max_rss_bytes_median": 41631744, + "user_cpu_ms_median": 20, + "system_cpu_ms_median": 10, + "cache_hits": 10, + "cache_misses": 0, + "lifecycle_ms_avoided": 1049, + "speedup_vs_miss": 2.93, + "wall_reduction_percent": 65.88 + }, + "ci_warm_store_hit": { + "samples": 10, + "wall_ms_median": 58.49, + "wall_ms_p95": 61.69, + "max_rss_bytes_median": 41615360, + "user_cpu_ms_median": 10, + "system_cpu_ms_median": 10, + "cache_hits": 10, + "cache_misses": 0, + "lifecycle_ms_avoided": 1051, + "speedup_vs_miss": 2.96, + "wall_reduction_percent": 66.21 + } + } +} diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md new file mode 100644 index 00000000..6f11aa15 --- /dev/null +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64-native-toolchain.md @@ -0,0 +1,27 @@ +# Native lifecycle-build cache benchmark — NativeToolchain + +- Platform: macOS arm64 +- Node: v22.22.1 +- Fixture: `bench/audit-fixtures/native/sharp-image` +- Release-build samples per scenario: 10 +- Median build artifact size: 503,469 bytes + +| Scenario | Median wall | p95 wall | Median RSS | User CPU | System CPU | Result | +|---|---:|---:|---:|---:|---:|---:| +| Strict cache miss | 173.10 ms | 280.00 ms | 47,169,536 B | 40 ms | 20 ms | 10 misses | +| Local artifact hit | 59.07 ms | 72.25 ms | 41,631,744 B | 20 ms | 10 ms | 2.93×; 65.88% lower wall time | +| Stable-path CI warm-store hit | 58.49 ms | 61.69 ms | 41,615,360 B | 10 ms | 10 ms | 2.96×; 66.21% lower wall time | + +Every measured scenario used the same strict sandbox. Sharp follows the +`NativeToolchain` cache path, so every hit validates the persisted compiler, +SDK, package-database, pkg-config, Homebrew receipt, and Node-header snapshot. +The first measured miss computed the complete host fingerprint in 126 ms; +subsequent unchanged-host key derivations took 15–24 ms. +Miss and local-hit samples rematerialized the same project. CI-like samples +deleted and recreated a checkout at one stable path while retaining the warm +LPM store. + +Across ten local hits, artifact metadata recorded 1,049 ms of lifecycle +execution avoided. Across ten stable-path CI hits, it recorded 1,051 ms +avoided. The measured hit remains beneficial after paying the conservative +host-snapshot validation cost. diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64.json b/bench/perf-results/native-build-cache-20260712-macos-arm64.json new file mode 100644 index 00000000..4fc474bd --- /dev/null +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64.json @@ -0,0 +1,49 @@ +{ + "generated_at": "2026-07-12T13:19:13.671Z", + "platform": "darwin-arm64", + "node": "v22.22.1", + "lpm_bin": "target/release/lpm-rs", + "fixture": "bench/audit-fixtures/native/esbuild-prebuilt", + "samples": 10, + "artifact_count": 2, + "artifact_bytes": 134444, + "scenarios": { + "cache_miss": { + "samples": 10, + "wall_ms_median": 138.03, + "wall_ms_p95": 449.94, + "max_rss_bytes_median": 52248576, + "user_cpu_ms_median": 50, + "system_cpu_ms_median": 20, + "cache_hits": 0, + "cache_misses": 10, + "lifecycle_ms_avoided": 0 + }, + "local_hit": { + "samples": 10, + "wall_ms_median": 32.33, + "wall_ms_p95": 37.27, + "max_rss_bytes_median": 41467904, + "user_cpu_ms_median": 10, + "system_cpu_ms_median": 10, + "cache_hits": 10, + "cache_misses": 0, + "lifecycle_ms_avoided": 1356, + "speedup_vs_miss": 4.27, + "wall_reduction_percent": 76.58 + }, + "ci_warm_store_hit": { + "samples": 10, + "wall_ms_median": 29.5, + "wall_ms_p95": 30.12, + "max_rss_bytes_median": 41418752, + "user_cpu_ms_median": 10, + "system_cpu_ms_median": 10, + "cache_hits": 10, + "cache_misses": 0, + "lifecycle_ms_avoided": 1053, + "speedup_vs_miss": 4.68, + "wall_reduction_percent": 78.63 + } + } +} diff --git a/bench/perf-results/native-build-cache-20260712-macos-arm64.md b/bench/perf-results/native-build-cache-20260712-macos-arm64.md new file mode 100644 index 00000000..a331a0af --- /dev/null +++ b/bench/perf-results/native-build-cache-20260712-macos-arm64.md @@ -0,0 +1,24 @@ +# Native lifecycle-build cache benchmark + +- Platform: macOS arm64 +- Node: v22.22.1 +- Fixture: `bench/audit-fixtures/native/esbuild-prebuilt` +- Release-build samples per scenario: 10 +- Median build artifact size: 134,444 bytes + +| Scenario | Median wall | p95 wall | Median RSS | User CPU | System CPU | Result | +|---|---:|---:|---:|---:|---:|---:| +| Strict cache miss | 138.03 ms | 449.94 ms | 52,248,576 B | 50 ms | 20 ms | 10 misses | +| Local artifact hit | 32.33 ms | 37.27 ms | 41,467,904 B | 10 ms | 10 ms | 4.27×; 76.58% lower wall time | +| Stable-path CI warm-store hit | 29.50 ms | 30.12 ms | 41,418,752 B | 10 ms | 10 ms | 4.68×; 78.63% lower wall time | + +Every measured scenario used the same strict sandbox. Miss and local-hit +samples rematerialized the same project. CI-like samples deleted and recreated +a checkout at one stable path while retaining the warm LPM store; observable +lifecycle paths are part of the cache key, so moving a checkout intentionally +invalidates the artifact. + +Across ten local hits, artifact metadata recorded 1,356 ms of lifecycle +execution avoided. Across ten stable-path CI hits, it recorded 1,053 ms +avoided. This fixture has a short lifecycle command; native compilation +workloads should have a substantially larger absolute hit benefit. diff --git a/bench/scripts/README.md b/bench/scripts/README.md index 1bbf5fe3..f2dfb983 100644 --- a/bench/scripts/README.md +++ b/bench/scripts/README.md @@ -1,5 +1,22 @@ # Install benchmark scripts +## Native lifecycle-build cache + +Build the release CLI, then compare strict cache misses, local artifact hits, +and CI-like stable-path checkout recreation against sharp's NativeToolchain +cache path. Every measured scenario uses the same strict sandbox, so cache-hit +timings include host compiler, SDK, package database, and pkg-config +fingerprinting: + +```bash +cargo build --release --locked -p lpm-cli --bin lpm-rs +node bench/scripts/native-build-cache-benchmark.mjs --self-test +node bench/scripts/native-build-cache-benchmark.mjs --samples 10 +``` + +The harness writes raw rows plus JSON and Markdown summaries under +`bench/perf-results/native-build-cache-/`. + ## Production-readiness harness `run-install-readiness.mjs` is the current install-readiness harness. It is diff --git a/bench/scripts/native-build-cache-benchmark.mjs b/bench/scripts/native-build-cache-benchmark.mjs new file mode 100644 index 00000000..7beb4282 --- /dev/null +++ b/bench/scripts/native-build-cache-benchmark.mjs @@ -0,0 +1,318 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = path.resolve(fileURLToPath(new URL('../..', import.meta.url))); +const options = parseArgs(process.argv.slice(2)); +if (options['self-test']) { + runSelfTest(); + process.exit(0); +} +const samples = Number(options.samples ?? 10); +const fixture = path.resolve( + repoRoot, + options.fixture ?? 'bench/audit-fixtures/native/sharp-image', +); +const lpmBin = path.resolve(options['lpm-bin'] ?? path.join(repoRoot, 'target/release/lpm-rs')); +const timestamp = new Date().toISOString().replaceAll(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); +const outputDir = path.resolve( + options.output ?? path.join(repoRoot, 'bench/perf-results', `native-build-cache-${timestamp}`), +); + +if (!Number.isInteger(samples) || samples < 2) { + throw new Error('--samples must be an integer >= 2'); +} +for (const required of [lpmBin, path.join(fixture, 'package.json')]) { + if (!fs.existsSync(required)) throw new Error(`missing required path: ${required}`); +} + +const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'lpm-native-build-cache-bench-')); +const home = path.join(tempRoot, 'home'); +const lpmHome = path.join(tempRoot, 'lpm-home'); +const primary = path.join(tempRoot, 'primary'); +fs.mkdirSync(home, { recursive: true }); +materializeProject(primary); + +const env = { + ...process.env, + HOME: home, + USERPROFILE: home, + LPM_HOME: lpmHome, + LPM_STORE_VERSION: 'v2', + LPM_TYPOSQUAT_GUARD: '0', + CI: '1', +}; + +const rows = []; +try { + runChecked(['install'], primary); + for (let sample = 1; sample <= samples; sample += 1) { + fs.rmSync(path.join(lpmHome, 'store/v2/builds'), { recursive: true, force: true }); + rematerialize(primary); + rows.push(measureRebuild('cache_miss', sample, primary)); + + rematerialize(primary); + rows.push(measureRebuild('local_hit', sample, primary)); + + const ciProject = path.join(tempRoot, 'ci-project'); + fs.rmSync(ciProject, { recursive: true, force: true }); + materializeProject(ciProject); + runChecked(['install'], ciProject); + runChecked(['rebuild', '--all', '--strict-sandbox'], ciProject); + fs.rmSync(ciProject, { recursive: true, force: true }); + materializeProject(ciProject); + runChecked(['install'], ciProject); + rows.push(measureRebuild('ci_warm_store_hit', sample, ciProject)); + } + + const artifactSizes = childDirectorySizes(path.join(lpmHome, 'store/v2/builds')); + if (artifactSizes.length === 0) { + throw new Error('benchmark completed without publishing a build artifact'); + } + const summary = summarize(rows, artifactSizes); + fs.mkdirSync(outputDir, { recursive: true }); + fs.writeFileSync(path.join(outputDir, 'rows.json'), `${JSON.stringify(rows, null, 2)}\n`); + fs.writeFileSync(path.join(outputDir, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`); + fs.writeFileSync(path.join(outputDir, 'summary.md'), `${renderMarkdown(summary)}\n`); + console.log(outputDir); + console.log(renderMarkdown(summary)); +} finally { + if (!options.keep) { + fs.rmSync(tempRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +} + +function materializeProject(projectDir) { + fs.mkdirSync(projectDir, { recursive: true }); + fs.copyFileSync(path.join(fixture, 'package.json'), path.join(projectDir, 'package.json')); +} + +function rematerialize(projectDir) { + fs.rmSync(path.join(projectDir, 'node_modules'), { recursive: true, force: true }); + runChecked(['install', '--force', '--no-frozen-lockfile'], projectDir); +} + +function measureRebuild(scenario, sample, cwd) { + const args = ['--json', 'rebuild', '--all', '--strict-sandbox']; + const started = process.hrtime.bigint(); + const result = runTimed(args, cwd); + const wallMs = Number(process.hrtime.bigint() - started) / 1e6; + if (result.status !== 0) { + throw new Error( + `${scenario} failed (${result.status})\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } + const envelope = parseJsonEnvelope(result.stdout); + const expectedHit = scenario.endsWith('_hit'); + if ( + (expectedHit && (envelope.build_cache.hits !== 1 || envelope.build_cache.misses !== 0)) || + (!expectedHit && (envelope.build_cache.hits !== 0 || envelope.build_cache.misses !== 1)) + ) { + throw new Error( + `${scenario} produced unexpected cache metrics: ${JSON.stringify(envelope.build_cache)}`, + ); + } + return { + scenario, + sample, + wall_ms: round(wallMs), + user_cpu_ms: result.userCpuMs, + system_cpu_ms: result.systemCpuMs, + max_rss_bytes: result.maxRssBytes, + build_cache: envelope.build_cache, + }; +} + +function parseJsonEnvelope(stdout) { + const lines = stdout.trimEnd().split(/\r?\n/); + for (let index = lines.length - 1; index >= 0; index -= 1) { + if (!lines[index].trimStart().startsWith('{')) continue; + try { + const value = JSON.parse(lines.slice(index).join('\n')); + if (value && typeof value === 'object' && value.build_cache) return value; + } catch {} + } + throw new Error(`rebuild output did not contain a JSON cache envelope:\n${stdout}`); +} + +function runSelfTest() { + const envelope = parseJsonEnvelope( + 'native lifecycle output\n{\n "success": true,\n "build_cache": {"hits": 1}\n}\n', + ); + if (envelope.build_cache.hits !== 1) { + throw new Error('JSON envelope parser did not preserve cache metrics'); + } +} + +function runChecked(args, cwd) { + const result = spawnSync(lpmBin, args, { + cwd, + env, + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error( + `lpm ${args.join(' ')} failed (${result.status})\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function runTimed(args, cwd) { + if (process.platform === 'darwin' && fs.existsSync('/usr/bin/time')) { + const result = spawnSync('/usr/bin/time', ['-l', lpmBin, ...args], { + cwd, + env, + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }); + const cpu = result.stderr.match(/([\d.]+) real\s+([\d.]+) user\s+([\d.]+) sys/); + const rss = result.stderr.match(/\s+(\d+)\s+maximum resident set size/); + return { + ...result, + userCpuMs: cpu ? Number(cpu[2]) * 1000 : null, + systemCpuMs: cpu ? Number(cpu[3]) * 1000 : null, + maxRssBytes: rss ? Number(rss[1]) : null, + }; + } + if (process.platform === 'linux' && fs.existsSync('/usr/bin/time')) { + const result = spawnSync('/usr/bin/time', ['-v', lpmBin, ...args], { + cwd, + env, + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }); + const user = result.stderr.match(/User time \(seconds\): ([\d.]+)/); + const system = result.stderr.match(/System time \(seconds\): ([\d.]+)/); + const rss = result.stderr.match(/Maximum resident set size \(kbytes\): (\d+)/); + return { + ...result, + userCpuMs: user ? Number(user[1]) * 1000 : null, + systemCpuMs: system ? Number(system[1]) * 1000 : null, + maxRssBytes: rss ? Number(rss[1]) * 1024 : null, + }; + } + const result = spawnSync(lpmBin, args, { + cwd, + env, + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }); + return { ...result, userCpuMs: null, systemCpuMs: null, maxRssBytes: null }; +} + +function summarize(allRows, artifactSizes) { + const scenarios = {}; + for (const scenario of [...new Set(allRows.map((row) => row.scenario))]) { + const selected = allRows.filter((row) => row.scenario === scenario); + const walls = selected.map((row) => row.wall_ms).sort((a, b) => a - b); + const rss = selected + .map((row) => row.max_rss_bytes) + .filter((value) => value != null) + .sort((a, b) => a - b); + scenarios[scenario] = { + samples: selected.length, + wall_ms_median: percentile(walls, 0.5), + wall_ms_p95: percentile(walls, 0.95), + max_rss_bytes_median: rss.length ? percentile(rss, 0.5) : null, + user_cpu_ms_median: medianNullable(selected.map((row) => row.user_cpu_ms)), + system_cpu_ms_median: medianNullable(selected.map((row) => row.system_cpu_ms)), + cache_hits: selected.reduce((sum, row) => sum + row.build_cache.hits, 0), + cache_misses: selected.reduce((sum, row) => sum + row.build_cache.misses, 0), + lifecycle_ms_avoided: selected.reduce( + (sum, row) => sum + row.build_cache.lifecycle_ms_avoided, + 0, + ), + }; + } + const baseline = scenarios.cache_miss.wall_ms_median; + for (const scenario of ['local_hit', 'ci_warm_store_hit']) { + scenarios[scenario].speedup_vs_miss = round(baseline / scenarios[scenario].wall_ms_median); + scenarios[scenario].wall_reduction_percent = round( + 100 * (1 - scenarios[scenario].wall_ms_median / baseline), + ); + } + return { + generated_at: new Date().toISOString(), + platform: `${process.platform}-${process.arch}`, + node: process.version, + lpm_bin: lpmBin, + fixture: path.relative(repoRoot, fixture), + samples, + artifact_count: artifactSizes.length, + artifact_bytes: percentile(artifactSizes.sort((a, b) => a - b), 0.5), + scenarios, + }; +} + +function renderMarkdown(summary) { + const lines = [ + '# Native lifecycle-build cache benchmark', + '', + `- Platform: ${summary.platform}`, + `- Fixture: \`${summary.fixture}\``, + `- Samples per scenario: ${summary.samples}`, + `- Median build artifact bytes: ${summary.artifact_bytes} (${summary.artifact_count} keys retained)`, + '', + '| Scenario | Median wall | p95 wall | Median RSS | Hits | Misses | Speedup |', + '|---|---:|---:|---:|---:|---:|---:|', + ]; + for (const [name, value] of Object.entries(summary.scenarios)) { + lines.push( + `| ${name} | ${value.wall_ms_median} ms | ${value.wall_ms_p95} ms | ${value.max_rss_bytes_median ?? 'n/a'} B | ${value.cache_hits} | ${value.cache_misses} | ${value.speedup_vs_miss ? `${value.speedup_vs_miss}×` : '—'} |`, + ); + } + lines.push( + '', + 'Every measured scenario uses the same strict sandbox. Miss and local-hit samples rematerialize the same project; CI-hit samples recreate a checkout at the same stable path while retaining the warm LPM store.', + ); + return lines.join('\n'); +} + +function percentile(sorted, fraction) { + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)]; +} + +function medianNullable(values) { + const present = values.filter((value) => value != null).sort((a, b) => a - b); + return present.length ? percentile(present, 0.5) : null; +} + +function directorySize(root) { + if (!fs.existsSync(root)) return 0; + let total = 0; + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + const candidate = path.join(root, entry.name); + if (entry.isDirectory()) total += directorySize(candidate); + else if (entry.isFile()) total += fs.statSync(candidate).size; + } + return total; +} + +function childDirectorySizes(root) { + if (!fs.existsSync(root)) return []; + return fs + .readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => directorySize(path.join(root, entry.name))); +} + +function round(value) { + return Math.round(value * 100) / 100; +} + +function parseArgs(argv) { + const parsed = {}; + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (!token.startsWith('--')) throw new Error(`unexpected argument: ${token}`); + const key = token.slice(2); + if (key === 'keep' || key === 'self-test') parsed[key] = true; + else parsed[key] = argv[++index]; + } + return parsed; +} diff --git a/crates/lpm-cli/src/commands/cache_prune.rs b/crates/lpm-cli/src/commands/cache_prune.rs index 217cb282..fd9d0ce0 100644 --- a/crates/lpm-cli/src/commands/cache_prune.rs +++ b/crates/lpm-cli/src/commands/cache_prune.rs @@ -92,6 +92,10 @@ pub struct PruneSummary { /// cutoff. Without reachability we evict islands by LRU, never by the /// project walk, so a complete recently-used island is always kept. pub compat_islands_orphaned: Vec, + /// Content-addressed lifecycle-build artifacts on disk. + pub build_artifacts_total: usize, + /// Incomplete or age-expired lifecycle-build artifacts. + pub build_artifacts_orphaned: Vec, /// Total bytes that would be / were freed by deleting orphans. /// Sum of `link_entries_orphaned` + `object_entries_orphaned` + /// `compat_islands_orphaned` directory sizes. @@ -246,6 +250,16 @@ fn run_locked( )) })?; } + for artifact in &summary.build_artifacts_orphaned { + std::fs::remove_dir_all(artifact).map_err(|e| { + LpmError::Store(format!( + "cache prune: failed to remove build artifact {}: {e}", + artifact.display() + )) + })?; + } + remove_orphaned_build_locks(v2_store)?; + remove_orphaned_build_entry_locks(v2_store)?; // Sweep deferred global-uninstall tombstones. Errors are // surfaced via `summary.tombstone_sweep_error` (and a @@ -276,6 +290,77 @@ fn run_locked( Ok(summary) } +fn remove_orphaned_build_locks(store: &V2Store) -> Result<(), LpmError> { + let entries = match std::fs::read_dir(store.paths().build_locks_root()) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + for entry in entries { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let path = entry.path(); + let Some(key) = advisory_lock_key(&path) else { + continue; + }; + if !store.paths().builds_root().join(key).is_dir() { + std::fs::remove_file(path)?; + } + } + Ok(()) +} + +fn remove_orphaned_build_entry_locks(store: &V2Store) -> Result<(), LpmError> { + let mut live_graph_keys = HashSet::new(); + match std::fs::read_dir(store.paths().links_root()) { + Ok(entries) => { + for entry in entries.flatten() { + if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { + continue; + } + if let Ok(metadata) = lpm_store::v2::LinkMeta::read_from(&entry.path()) { + live_graph_keys.insert(metadata.graph_key_digest_hex); + } + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let entries = match std::fs::read_dir(store.paths().build_entry_locks_root()) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + for entry in entries { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let path = entry.path(); + let Some(key) = advisory_lock_key(&path) else { + continue; + }; + if !live_graph_keys.contains(key) { + std::fs::remove_file(path)?; + } + } + Ok(()) +} + +fn advisory_lock_key(path: &Path) -> Option<&str> { + let name = path.file_name()?.to_str()?; + let key = [".lock.writer-intent", ".lock.writer-queue", ".lock"] + .into_iter() + .find_map(|suffix| name.strip_suffix(suffix))?; + (key.len() == 64 + && key + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f'))) + .then_some(key) +} + /// Compute (but don't apply) the prune plan. Pulled out so unit tests /// exercise the algorithm against synthetic stores without touching /// the filesystem outside of a tempdir. @@ -357,8 +442,11 @@ pub fn compute_prune_plan( // independent), so it is safe even with no usable registry. let (compat_islands_total, compat_islands_orphaned) = compute_compat_island_orphans(v2_store.paths().compat_root(), max_age); + let (build_artifacts_total, build_artifacts_orphaned) = + compute_build_artifact_orphans(v2_store.paths().builds_root(), max_age); let bytes_freed_or_eligible = compat_islands_orphaned .iter() + .chain(&build_artifacts_orphaned) .map(|dir| dir_size(dir).unwrap_or(0)) .fold(0u64, u64::saturating_add); return Ok(PruneSummary { @@ -373,6 +461,8 @@ pub fn compute_prune_plan( object_entries_orphaned: Vec::new(), compat_islands_total, compat_islands_orphaned, + build_artifacts_total, + build_artifacts_orphaned, bytes_freed_or_eligible, tombstones_pending, tombstones_swept: 0, @@ -514,6 +604,8 @@ pub fn compute_prune_plan( // are a macOS clonefile feature) → the dir is missing → no-op. let (compat_islands_total, compat_islands_orphaned) = compute_compat_island_orphans(v2_store.paths().compat_root(), max_age); + let (build_artifacts_total, build_artifacts_orphaned) = + compute_build_artifact_orphans(v2_store.paths().builds_root(), max_age); let mut bytes_freed_or_eligible = 0u64; for dir in &link_entries_orphaned { @@ -528,6 +620,10 @@ pub fn compute_prune_plan( bytes_freed_or_eligible = bytes_freed_or_eligible.saturating_add(dir_size(dir).unwrap_or(0)); } + for dir in &build_artifacts_orphaned { + bytes_freed_or_eligible = + bytes_freed_or_eligible.saturating_add(dir_size(dir).unwrap_or(0)); + } let (tombstones_pending, tombstone_count_error) = count_tombstones_with_error_capture(root); @@ -543,6 +639,8 @@ pub fn compute_prune_plan( object_entries_orphaned, compat_islands_total, compat_islands_orphaned, + build_artifacts_total, + build_artifacts_orphaned, bytes_freed_or_eligible, tombstones_pending, tombstones_swept: 0, @@ -683,7 +781,30 @@ fn compute_compat_island_orphans( compat_root: &Path, max_age: Option, ) -> (usize, Vec) { - let read = match std::fs::read_dir(compat_root) { + compute_lru_artifact_orphans( + compat_root, + lpm_store::v2::COMPAT_ISLAND_COMPLETE_FILENAME, + max_age, + ) +} + +fn compute_build_artifact_orphans( + builds_root: &Path, + max_age: Option, +) -> (usize, Vec) { + compute_lru_artifact_orphans( + builds_root, + lpm_store::v2::BUILD_ARTIFACT_COMPLETE_FILENAME, + max_age, + ) +} + +fn compute_lru_artifact_orphans( + root: &Path, + completion_sentinel: &str, + max_age: Option, +) -> (usize, Vec) { + let read = match std::fs::read_dir(root) { Ok(read) => read, Err(_) => return (0, Vec::new()), }; @@ -696,7 +817,7 @@ fn compute_compat_island_orphans( } let island = entry.path(); total += 1; - let sentinel = island.join(lpm_store::v2::COMPAT_ISLAND_COMPLETE_FILENAME); + let sentinel = island.join(completion_sentinel); let Ok(meta) = std::fs::metadata(&sentinel) else { orphaned.push(island); continue; @@ -833,6 +954,18 @@ fn emit_human(summary: &PruneSummary, applied: bool, elapsed: Duration) { }, )); } + if !summary.build_artifacts_orphaned.is_empty() { + install_ui::phase(&format!( + "{} stale build artifact(s) {} {}", + summary.build_artifacts_orphaned.len(), + if applied { "evicted" } else { "eligible —" }, + if applied { + String::new() + } else { + "`lpm cache prune --apply`".to_string() + }, + )); + } if !applied && summary.link_entries_orphaned.len() <= 20 { for dir in &summary.link_entries_orphaned { println!( @@ -852,6 +985,12 @@ fn emit_human(summary: &PruneSummary, applied: bool, elapsed: Duration) { sanitize_for_terminal(&dir.display().to_string()) ); } + for dir in &summary.build_artifacts_orphaned { + println!( + " build artifact orphan: {}", + sanitize_for_terminal(&dir.display().to_string()) + ); + } } if !applied { install_ui::done(&format!("Done · checked cache in {elapsed}")); @@ -884,6 +1023,12 @@ fn emit_json(summary: &PruneSummary) { .iter() .map(|p| p.display().to_string()) .collect::>(), + "build_artifacts_total": summary.build_artifacts_total, + "build_artifacts_orphaned": summary + .build_artifacts_orphaned + .iter() + .map(|p| p.display().to_string()) + .collect::>(), "bytes_freed_or_eligible": summary.bytes_freed_or_eligible, "registry_missing": summary.registry_missing, "registry_corrupt": summary.registry_corrupt, @@ -1099,6 +1244,138 @@ mod tests { ); } + #[test] + fn build_artifact_prune_evicts_incomplete_and_stale_entries() { + let tmp = tempfile::tempdir().unwrap(); + let builds_root = tmp.path().join("builds"); + std::fs::create_dir_all(&builds_root).unwrap(); + let sentinel = lpm_store::v2::BUILD_ARTIFACT_COMPLETE_FILENAME; + + let fresh = builds_root.join("fresh"); + std::fs::create_dir_all(&fresh).unwrap(); + std::fs::write(fresh.join(sentinel), b"").unwrap(); + + let incomplete = builds_root.join("incomplete"); + std::fs::create_dir_all(&incomplete).unwrap(); + + let stale = builds_root.join("stale"); + std::fs::create_dir_all(&stale).unwrap(); + let stale_sentinel = stale.join(sentinel); + std::fs::write(&stale_sentinel, b"").unwrap(); + let ten_days_ago = + std::time::SystemTime::now() - std::time::Duration::from_secs(10 * 86_400); + std::fs::File::options() + .write(true) + .open(&stale_sentinel) + .unwrap() + .set_modified(ten_days_ago) + .unwrap(); + + let (total, orphans) = compute_build_artifact_orphans(&builds_root, None); + assert_eq!(total, 3); + assert_eq!(orphans, vec![incomplete.clone()]); + + let (total, mut orphans) = + compute_build_artifact_orphans(&builds_root, Some(ChronoDuration::days(7))); + assert_eq!(total, 3); + orphans.sort(); + let mut expected = vec![incomplete, stale]; + expected.sort(); + assert_eq!(orphans, expected); + } + + #[test] + fn build_artifact_crash_recovery_applies_without_project_registry() { + let tmp = tempfile::tempdir().unwrap(); + let root = LpmRoot::from_dir(tmp.path()); + let store = V2Store::from_lpm_root(&root); + let incomplete = store.paths().builds_root().join("incomplete"); + std::fs::create_dir_all(&incomplete).unwrap(); + std::fs::write(incomplete.join("partial"), b"partial").unwrap(); + + let summary = run_locked( + &root, + &store, + &PruneFlags { + apply: true, + ..PruneFlags::default() + }, + None, + ) + .unwrap(); + + assert!(summary.registry_missing); + assert_eq!(summary.build_artifacts_orphaned, vec![incomplete.clone()]); + assert!(!incomplete.exists()); + } + + #[test] + fn build_lock_cleanup_removes_only_locks_without_artifacts() { + let tmp = tempfile::tempdir().unwrap(); + let root = LpmRoot::from_dir(tmp.path()); + let store = V2Store::from_lpm_root(&root); + let orphan_key = "a".repeat(64); + let live_key = "b".repeat(64); + std::fs::create_dir_all(store.paths().build_locks_root()).unwrap(); + std::fs::create_dir_all(store.paths().builds_root().join(&live_key)).unwrap(); + let orphan_lock = store.paths().build_lock_path_for_key(&orphan_key); + let live_lock = store.paths().build_lock_path_for_key(&live_key); + std::fs::write(&orphan_lock, b"").unwrap(); + std::fs::write(format!("{}.writer-intent", orphan_lock.display()), b"").unwrap(); + std::fs::write(format!("{}.writer-queue", orphan_lock.display()), b"").unwrap(); + std::fs::write(&live_lock, b"").unwrap(); + std::fs::write(format!("{}.writer-intent", live_lock.display()), b"").unwrap(); + std::fs::write(format!("{}.writer-queue", live_lock.display()), b"").unwrap(); + + remove_orphaned_build_locks(&store).unwrap(); + + assert!(!orphan_lock.exists()); + assert!(!std::fs::exists(format!("{}.writer-intent", orphan_lock.display())).unwrap()); + assert!(!std::fs::exists(format!("{}.writer-queue", orphan_lock.display())).unwrap()); + assert!(live_lock.exists()); + assert!(std::fs::exists(format!("{}.writer-intent", live_lock.display())).unwrap()); + assert!(std::fs::exists(format!("{}.writer-queue", live_lock.display())).unwrap()); + } + + #[test] + fn build_entry_lock_cleanup_removes_only_locks_without_link_entries() { + let tmp = tempfile::tempdir().unwrap(); + let root = LpmRoot::from_dir(tmp.path()); + let store = V2Store::from_lpm_root(&root); + let orphan_key = "a".repeat(64); + let live_key = "b".repeat(64); + std::fs::create_dir_all(store.paths().build_entry_locks_root()).unwrap(); + let orphan_lock = store.paths().build_entry_lock_path(&orphan_key).unwrap(); + let live_lock = store.paths().build_entry_lock_path(&live_key).unwrap(); + std::fs::write(&orphan_lock, b"").unwrap(); + std::fs::write(format!("{}.writer-intent", orphan_lock.display()), b"").unwrap(); + std::fs::write(format!("{}.writer-queue", orphan_lock.display()), b"").unwrap(); + std::fs::write(&live_lock, b"").unwrap(); + std::fs::write(format!("{}.writer-intent", live_lock.display()), b"").unwrap(); + std::fs::write(format!("{}.writer-queue", live_lock.display()), b"").unwrap(); + let link_dir = store + .paths() + .links_root() + .join("example@1.0.0+bbbbbbbbbbbbbbbb"); + std::fs::create_dir_all(&link_dir).unwrap(); + std::fs::write( + link_dir.join(lpm_store::v2::LINK_META_FILENAME), + format!( + r#"{{"schema":1,"graph_key":"example@1.0.0+bbbbbbbbbbbbbbbb","graph_key_digest_hex":"{live_key}","name":"example","version":"1.0.0","source_sri":"sha512-example","object_path":"objects/sha512-example","deps":[],"platform":{{"os":"darwin","cpu":"arm64","libc":null}},"created_at":"2026-01-01T00:00:00Z","last_referenced_at":"2026-01-01T00:00:00Z"}}"# + ), + ) + .unwrap(); + + remove_orphaned_build_entry_locks(&store).unwrap(); + + assert!(!orphan_lock.exists()); + assert!(!std::fs::exists(format!("{}.writer-intent", orphan_lock.display())).unwrap()); + assert!(!std::fs::exists(format!("{}.writer-queue", orphan_lock.display())).unwrap()); + assert!(live_lock.exists()); + assert!(std::fs::exists(format!("{}.writer-intent", live_lock.display())).unwrap()); + assert!(std::fs::exists(format!("{}.writer-queue", live_lock.display())).unwrap()); + } + #[test] #[cfg(unix)] fn compute_prune_plan_follows_dep_edges_via_sidecar() { diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache.rs b/crates/lpm-cli/src/commands/rebuild/build_cache.rs new file mode 100644 index 00000000..6bcc5244 --- /dev/null +++ b/crates/lpm-cli/src/commands/rebuild/build_cache.rs @@ -0,0 +1,1950 @@ +use self::toolchain_snapshot::{ + ComputedToolchainFingerprint, ToolchainFingerprintCache, cached_toolchain_fingerprint, +}; +use super::script_execution::{build_lifecycle_environment, build_lifecycle_path}; +use super::scripts::ScriptablePackage; +use crate::capability::CapabilitySet; +use lpm_sandbox::{SandboxMode, SandboxOptions, SandboxPosture}; +use lpm_security::EXECUTED_INSTALL_PHASES; +use lpm_store::v2::{ + BuildCacheKey, BuildKeyInputs, BuildPlatformFingerprint, BuildRuntimeFingerprint, + BuildSandboxFingerprint, BuildScriptFingerprint, PlatformTuple, +}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +mod toolchain_snapshot; + +const TOOLCHAIN_PROBE_TIMEOUT: Duration = Duration::from_secs(2); +const TOOLCHAIN_PROBE_OUTPUT_LIMIT: usize = 4 * 1024 * 1024; +const RUNTIME_ONLY_TOOLCHAIN_FINGERPRINT: &str = "runtime-and-dependency-graph-v2"; + +const TOOLCHAIN_PROBES: &[(&str, &[&str])] = &[ + ("python3", &["--version"]), + ("make", &["--version"]), + ("cc", &["--version"]), + ("cc", &["-print-sysroot"]), + ("cc", &["-print-search-dirs"]), + ("c++", &["--version"]), + ("c++", &["-print-sysroot"]), + ("c++", &["-print-search-dirs"]), + ("clang", &["--version"]), + ("ld", &["-v"]), + ("xcrun", &["--show-sdk-path"]), + ("xcrun", &["--show-sdk-version"]), + ("rustc", &["-vV"]), + ("cargo", &["-V"]), + ("cmake", &["--version"]), + ("ninja", &["--version"]), + ("pkg-config", &["--version"]), + ("pkg-config", &["--variable=pc_path", "pkg-config"]), + ("pkg-config", &["--list-all"]), + ( + "dpkg-query", + &["-W", "-f=${Package}\t${Version}\t${Architecture}\\n"], + ), + ( + "rpm", + &["-qa", "--qf", "%{NAME}\t%{VERSION}-%{RELEASE}\t%{ARCH}\\n"], + ), + ("pacman", &["-Q"]), + ("apk", &["info", "-vv"]), + ("pkgutil", &["--pkgs"]), + ("xcodebuild", &["-version"]), + ("uname", &["-a"]), + ("sw_vers", &[]), + ("ldd", &["--version"]), +]; + +pub(super) struct BuildCacheScratch { + path: PathBuf, +} + +impl BuildCacheScratch { + fn path_for(package_dir: &Path, graph_key_digest: &str) -> std::io::Result { + let node_modules_dir = package_dir + .ancestors() + .find(|ancestor| { + ancestor + .file_name() + .is_some_and(|name| name == "node_modules") + }) + .ok_or_else(|| std::io::Error::other("package is not inside a node_modules tree"))?; + let link_dir = node_modules_dir + .parent() + .ok_or_else(|| std::io::Error::other("package has no v2 link-entry parent"))?; + let metadata = lpm_store::v2::LinkMeta::read_from(link_dir) + .map_err(|error| std::io::Error::other(error.to_string()))?; + if metadata.graph_key_digest_hex != graph_key_digest { + return Err(std::io::Error::other( + "package graph identity does not match its v2 link sidecar", + )); + } + if node_modules_dir.join(&metadata.name) != package_dir { + return Err(std::io::Error::other( + "package path does not match its v2 link sidecar", + )); + } + Ok(link_dir.join(".lpm-build-tmp")) + } + + pub(super) fn create(package_dir: &Path, graph_key_digest: &str) -> std::io::Result { + let path = Self::path_for(package_dir, graph_key_digest)?; + match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => { + std::fs::remove_dir_all(&path)?; + } + Ok(_) => std::fs::remove_file(&path)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + std::fs::create_dir(&path)?; + Ok(Self { path }) + } + + #[inline] + pub(super) fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for BuildCacheScratch { + fn drop(&mut self) { + if let Err(error) = std::fs::remove_dir_all(&self.path) { + tracing::debug!( + "failed to remove build-cache scratch directory at {}: {error}", + self.path.display() + ); + } + } +} + +pub(super) struct BuildCacheInvocation { + dependency_closure_hash: String, + platform: BuildPlatformFingerprint, + runtime: BuildRuntimeFingerprint, + sandbox: BuildSandboxFingerprint, + environment: HashMap, + trusted_toolchain_environment: HashMap, + build_executable_environment: HashMap, + project_dir: PathBuf, + toolchain_cache: Option, + native_toolchain_hash: OnceLock>, +} + +impl BuildCacheInvocation { + #[allow(clippy::too_many_arguments)] + pub(super) fn prepare( + lockfile: &lpm_lockfile::Lockfile, + environment: &HashMap, + project_dir: &Path, + sandbox_mode: SandboxMode, + posture: &SandboxPosture, + sandbox_options: &SandboxOptions, + extra_write_dirs: &[PathBuf], + extra_read_paths: &[PathBuf], + capabilities: &CapabilitySet, + ) -> Option { + if sandbox_mode != SandboxMode::Enforce { + debug_bypass("sandbox is not enforcing"); + return None; + } + if !matches!(posture, SandboxPosture::Strict) || !sandbox_options.deny_outbound_network { + debug_bypass("outbound network denial is not effective"); + return None; + } + if !extra_write_dirs.is_empty() || !extra_read_paths.is_empty() { + debug_bypass("project widened sandbox paths"); + return None; + } + if !capabilities.is_at_baseline() { + debug_bypass("project widened lifecycle capabilities"); + return None; + } + let trusted_toolchain_environment = trusted_toolchain_environment(environment, project_dir); + let Some(runtime) = detect_node_runtime(&trusted_toolchain_environment, project_dir) else { + debug_bypass("Node runtime fingerprint unavailable"); + return None; + }; + let platform = PlatformTuple::current(); + let mut build_executable_environment = environment.clone(); + let parent_path = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("PATH")) + .map(|(_, value)| value.as_str()); + build_executable_environment.retain(|key, _| !key.eq_ignore_ascii_case("PATH")); + build_executable_environment.insert( + "PATH".into(), + build_lifecycle_path(project_dir, parent_path), + ); + Some(Self { + dependency_closure_hash: hash_lockfile_graph(lockfile), + platform: BuildPlatformFingerprint { + os: platform.os, + architecture: platform.cpu, + libc: platform.libc.unwrap_or_default(), + cpu_features_hash: cpu_features_hash(), + }, + runtime, + sandbox: BuildSandboxFingerprint { + mode: "enforce".into(), + posture: "strict".into(), + network_denied: true, + environment_scrubbed: true, + allowed_inputs_hash: hash_allowed_inputs(capabilities), + }, + environment: environment.clone(), + trusted_toolchain_environment, + build_executable_environment, + project_dir: project_dir.to_path_buf(), + toolchain_cache: lpm_common::LpmRoot::from_env() + .ok() + .map(ToolchainFingerprintCache::from_lpm_root), + native_toolchain_hash: OnceLock::new(), + }) + } +} + +fn debug_bypass(reason: &str) { + tracing::debug!(target: "lpm_cli::build_cache", "bypass: {reason}"); +} + +pub(super) fn is_cacheable_native_build(package: &ScriptablePackage) -> bool { + native_build_kind(package).is_some() +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum NativeBuildKind { + RuntimeOnly, + NativeToolchain, +} + +fn native_build_kind(package: &ScriptablePackage) -> Option { + if !cfg!(any(target_os = "macos", target_os = "linux")) { + return None; + } + let mut kind = None; + for command in package.scripts.values() { + for segment in shell_command_segments(command)? { + let Some(segment_kind) = native_segment_kind(&package.name, &segment) else { + continue; + }; + if segment_kind == NativeBuildKind::NativeToolchain { + return Some(segment_kind); + } + kind = Some(segment_kind); + } + } + kind +} + +fn native_segment_kind(package_name: &str, words: &[String]) -> Option { + let command_index = words + .iter() + .position(|word| !is_environment_assignment(word))?; + let executable = Path::new(&words[command_index]) + .file_name() + .and_then(|name| name.to_str())?; + let arguments = &words[command_index + 1..]; + match executable { + "node-gyp" + | "node-gyp-build" + | "node-gyp-build-optional-packages" + | "prebuild-install" + | "electron-rebuild" + | "cmake-js" => Some(NativeBuildKind::NativeToolchain), + "node" if package_name == "esbuild" => arguments + .first() + .and_then(|argument| Path::new(argument).file_name()) + .and_then(|name| name.to_str()) + .filter(|name| *name == "install.js") + .map(|_| NativeBuildKind::RuntimeOnly), + "node" if package_name == "sharp" => arguments + .first() + .filter(|argument| { + let normalized = argument.replace('\\', "/"); + normalized == "install/check" || normalized.starts_with("install/") + }) + .map(|_| NativeBuildKind::NativeToolchain), + _ => None, + } +} + +fn is_environment_assignment(word: &str) -> bool { + let Some((name, _)) = word.split_once('=') else { + return false; + }; + !name.is_empty() + && name.bytes().enumerate().all(|(index, byte)| { + byte == b'_' || byte.is_ascii_alphabetic() || (index > 0 && byte.is_ascii_digit()) + }) +} + +fn shell_command_segments(command: &str) -> Option>> { + let normalized = normalize_shell_structure(command); + let words = shlex::split(&normalized)?; + let mut segments = Vec::new(); + let mut current = Vec::new(); + for word in words { + if matches!(word.as_str(), "&&" | "||" | ";" | "|" | "&" | "(" | ")") { + if !current.is_empty() { + segments.push(std::mem::take(&mut current)); + } + } else { + current.push(word); + } + } + if !current.is_empty() { + segments.push(current); + } + Some(segments) +} + +fn normalize_shell_structure(command: &str) -> String { + let mut normalized = String::with_capacity(command.len() * 2); + let mut chars = command.chars().peekable(); + let mut single_quoted = false; + let mut double_quoted = false; + let mut word_boundary = true; + while let Some(character) = chars.next() { + if character == '\\' && !single_quoted { + normalized.push(character); + if let Some(escaped) = chars.next() { + normalized.push(escaped); + } + word_boundary = false; + continue; + } + if character == '\'' && !double_quoted { + single_quoted = !single_quoted; + normalized.push(character); + word_boundary = false; + continue; + } + if character == '"' && !single_quoted { + double_quoted = !double_quoted; + normalized.push(character); + word_boundary = false; + continue; + } + if !single_quoted && !double_quoted && character == '#' && word_boundary { + while chars.next().is_some_and(|next| next != '\n') {} + normalized.push_str(" ; "); + word_boundary = true; + continue; + } + if !single_quoted && !double_quoted { + match character { + '\n' | ';' | '(' | ')' => { + normalized.push(' '); + normalized.push(if character == '\n' { ';' } else { character }); + normalized.push(' '); + word_boundary = true; + continue; + } + '|' | '&' => { + normalized.push(' '); + normalized.push(character); + if chars.peek().copied() == Some(character) { + normalized.push(chars.next().expect("peeked shell operator")); + } + normalized.push(' '); + word_boundary = true; + continue; + } + _ => {} + } + } + normalized.push(character); + word_boundary = character.is_whitespace(); + } + normalized +} + +pub(super) fn marker_requires_key_validation(package: &ScriptablePackage) -> bool { + package.build_marker_key.is_some() && is_cacheable_native_build(package) +} + +pub(super) fn read_build_marker_key(marker_path: &Path) -> Option { + let bytes = std::fs::read(marker_path).ok()?; + if bytes.len() != 64 + || !bytes + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + { + return None; + } + String::from_utf8(bytes).ok() +} + +pub(super) fn build_key_for_package( + invocation: &BuildCacheInvocation, + package: &ScriptablePackage, + project_dir: &Path, +) -> Option { + let Some(graph_key_digest) = package.graph_key_digest.as_ref() else { + debug_bypass(&format!("{} has no v2 graph identity", package.name)); + return None; + }; + let Some(native_kind) = native_build_kind(package) else { + debug_bypass(&format!( + "{} lifecycle command is not a recognized native build", + package.name + )); + return None; + }; + let scratch_path = match BuildCacheScratch::path_for(&package.store_path, graph_key_digest) { + Ok(path) => path, + Err(error) => { + debug_bypass(&format!( + "{} has invalid v2 build-cache context: {error}", + package.name + )); + return None; + } + }; + let lifecycle_environment = + build_lifecycle_environment(&invocation.environment, project_dir, &scratch_path); + let mut scripts = Vec::with_capacity(EXECUTED_INSTALL_PHASES.len()); + for phase in EXECUTED_INSTALL_PHASES { + if let Some(command) = package.scripts.get(*phase) { + scripts.push(BuildScriptFingerprint { + phase: (*phase).to_string(), + command: command.clone(), + }); + } + } + if !build_visible_node_matches_trusted(invocation) { + debug_bypass("build-visible Node does not match the trusted runtime"); + return None; + } + let toolchain_hash = if native_kind == NativeBuildKind::NativeToolchain { + let Some(hash) = invocation + .native_toolchain_hash + .get_or_init(|| { + hash_toolchain_cached( + &invocation.trusted_toolchain_environment, + &invocation.build_executable_environment, + &invocation.project_dir, + invocation.toolchain_cache.as_ref(), + ) + .ok() + }) + .clone() + else { + debug_bypass("native toolchain fingerprint unavailable"); + return None; + }; + let invoked_executables = + hash_invoked_build_executables(package, &invocation.build_executable_environment); + hash_records([hash.as_bytes(), invoked_executables.as_bytes()]) + } else { + RUNTIME_ONLY_TOOLCHAIN_FINGERPRINT.to_string() + }; + let environment_hash = hash_environment_pairs(&lifecycle_environment); + let key = BuildCacheKey::derive(&BuildKeyInputs { + source_integrity: package.source_integrity.clone(), + graph_key_digest: graph_key_digest.clone(), + dependency_closure_hash: invocation.dependency_closure_hash.clone(), + scripts, + platform: invocation.platform.clone(), + runtime: invocation.runtime.clone(), + sandbox: invocation.sandbox.clone(), + environment_hash: environment_hash.clone(), + toolchain_hash, + }); + tracing::debug!( + target: "lpm_cli::build_cache", + package = %package.name, + %environment_hash, + build_key = %key.as_str(), + "derived native build-cache key" + ); + Some(key) +} + +fn build_visible_node_matches_trusted(invocation: &BuildCacheInvocation) -> bool { + let trusted = resolve_executable("node", &invocation.trusted_toolchain_environment); + let build_visible = resolve_executable("node", &invocation.build_executable_environment); + trusted.is_some() && trusted == build_visible +} + +pub(super) fn read_v2_graph_key_digest(package_dir: &Path) -> Option { + let node_modules_dir = package_dir.ancestors().find(|ancestor| { + ancestor + .file_name() + .is_some_and(|name| name == "node_modules") + })?; + let link_dir = node_modules_dir.parent()?; + let metadata = lpm_store::v2::LinkMeta::read_from(link_dir).ok()?; + (node_modules_dir.join(&metadata.name) == package_dir).then_some(metadata.graph_key_digest_hex) +} + +fn detect_node_runtime( + environment: &HashMap, + project_dir: &Path, +) -> Option { + let executable = resolve_trusted_executable("node", environment, project_dir)?; + let probe_environment = toolchain_probe_environment(environment); + let output = run_bounded_toolchain_probe( + &executable.invocation_path, + &[ + "-p", + "JSON.stringify({version:process.version,modules:process.versions.modules||'',napi:process.versions.napi||'',engine:process.versions.v8||'',execPath:process.execPath})", + ], + &probe_environment, + )?; + if !output.status.success() { + return None; + } + let value: serde_json::Value = serde_json::from_slice(&output.stdout).ok()?; + let executable = PathBuf::from(value.get("execPath")?.as_str()?); + Some(BuildRuntimeFingerprint { + runtime: "node".into(), + version: value + .get("version")? + .as_str()? + .trim_start_matches('v') + .into(), + modules_abi: value.get("modules")?.as_str()?.into(), + napi: value.get("napi")?.as_str()?.into(), + engine: value.get("engine")?.as_str()?.into(), + executable_hash: runtime_executable_identity(&executable).ok()?, + }) +} + +fn hash_lockfile_graph(lockfile: &lpm_lockfile::Lockfile) -> String { + let mut rows = Vec::with_capacity(lockfile.packages.len() + lockfile.patches.len() + 1); + rows.push("graph-schema\0v2".to_string()); + for package in &lockfile.packages { + let mut dependencies = package.dependencies.clone(); + dependencies.sort_unstable(); + let mut aliases = package.alias_dependencies.clone(); + aliases.sort_unstable(); + let mut peers = package.peers.clone(); + peers.sort_unstable(); + rows.push(format!( + "package\0{}\0{}\0{}\0{}\0deps\0{}\0aliases\0{}\0peers\0{}", + package.name, + package.version, + package.integrity.as_deref().unwrap_or(""), + package.source.as_deref().unwrap_or(""), + dependencies.join("\0"), + aliases + .iter() + .map(|[local, target]| format!("{local}\0{target}")) + .collect::>() + .join("\0"), + peers.join("\0") + )); + } + for (package, patch) in &lockfile.patches { + rows.push(format!( + "patch\0{package}\0{}\0{}\0{}", + patch.path, patch.sha256, patch.original_integrity + )); + } + for (local, target) in &lockfile.root_aliases { + rows.push(format!("root-alias\0{local}\0{target}")); + } + rows.sort_unstable(); + hash_records(rows.iter().map(String::as_bytes)) +} + +#[cfg(test)] +fn hash_build_environment(environment: &HashMap) -> String { + let mut records = environment.iter().collect::>(); + records.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + hash_records(records.into_iter().map(|(name, value)| { + let mut record = String::with_capacity(name.len() + value.len() + 1); + record.push_str(name); + record.push('\0'); + record.push_str(value); + record + })) +} + +fn hash_environment_pairs(environment: &[(String, String)]) -> String { + let mut records = environment.iter().collect::>(); + records.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + hash_records(records.into_iter().map(|(name, value)| { + let mut record = String::with_capacity(name.len() + value.len() + 1); + record.push_str(name); + record.push('\0'); + record.push_str(value); + record + })) +} + +fn trusted_toolchain_environment( + environment: &HashMap, + project_dir: &Path, +) -> HashMap { + let mut trusted = environment.clone(); + let inherited_path = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("PATH")) + .map_or(default_toolchain_path(), |(_, value)| value.as_str()); + let search_dirs = std::env::split_paths(inherited_path) + .filter(|directory| !is_project_controlled_path(directory, project_dir)) + .collect::>(); + let path = std::env::join_paths(search_dirs) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| default_toolchain_path().into()); + trusted.retain(|key, _| !key.eq_ignore_ascii_case("PATH")); + trusted.insert("PATH".into(), path.to_string_lossy().into_owned()); + trusted +} + +fn default_toolchain_path() -> &'static str { + #[cfg(unix)] + { + "/usr/local/bin:/usr/bin:/bin" + } + #[cfg(windows)] + { + r"C:\Windows\System32;C:\Windows;C:\Windows\System32\Wbem" + } + #[cfg(not(any(unix, windows)))] + { + "/usr/bin:/bin" + } +} + +fn is_project_controlled_path(path: &Path, project_dir: &Path) -> bool { + if !path.is_absolute() + || path + .components() + .any(|component| component.as_os_str() == "node_modules") + || path.starts_with(project_dir) + { + return true; + } + let Ok(canonical) = std::fs::canonicalize(path) else { + return false; + }; + let canonical_project = + std::fs::canonicalize(project_dir).unwrap_or_else(|_| project_dir.to_path_buf()); + canonical.starts_with(canonical_project) + || canonical + .components() + .any(|component| component.as_os_str() == "node_modules") +} + +#[derive(Clone)] +struct TrustedExecutable { + invocation_path: PathBuf, + identity: String, +} + +fn resolve_trusted_executable( + executable: &str, + environment: &HashMap, + project_dir: &Path, +) -> Option { + let invocation_path = resolve_executable(executable, environment)?; + let canonical_target = std::fs::canonicalize(&invocation_path).ok()?; + if is_project_controlled_path(&invocation_path, project_dir) + || is_project_controlled_path(&canonical_target, project_dir) + { + return None; + } + Some(TrustedExecutable { + invocation_path, + identity: runtime_executable_identity(&canonical_target).ok()?, + }) +} + +fn toolchain_probe_environment( + trusted_environment: &HashMap, +) -> HashMap { + // The full lifecycle environment can contain loader/runtime hooks that execute code + // before the lifecycle sandbox exists. Probes receive only data-bearing inputs. + const PASSTHROUGH: &[&str] = &[ + "PATH", + "SDKROOT", + "DEVELOPER_DIR", + "PKG_CONFIG_PATH", + "PKG_CONFIG_LIBDIR", + "PKG_CONFIG_SYSROOT_DIR", + ]; + let mut probe_environment = HashMap::with_capacity(PASSTHROUGH.len() + 2); + for name in PASSTHROUGH { + if let Some((key, value)) = trusted_environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + { + probe_environment.insert(key.clone(), value.clone()); + } + } + probe_environment.insert("LANG".into(), "C".into()); + probe_environment.insert("LC_ALL".into(), "C".into()); + probe_environment +} + +fn read_bounded_probe_stream(mut reader: R) -> Vec { + let mut retained = Vec::with_capacity(8 * 1024); + let mut chunk = [0_u8; 64 * 1024]; + while let Ok(read) = reader.read(&mut chunk) { + if read == 0 { + break; + } + let remaining = TOOLCHAIN_PROBE_OUTPUT_LIMIT.saturating_sub(retained.len()); + retained.extend_from_slice(&chunk[..read.min(remaining)]); + } + retained +} + +fn terminate_toolchain_probe(child: &mut std::process::Child) { + #[cfg(unix)] + { + let pid = child.id() as i32; + // SAFETY: the child is placed in a fresh process group immediately before spawn. + unsafe { + libc::kill(-pid, libc::SIGKILL); + } + } + let _ = child.kill(); +} + +fn run_bounded_toolchain_probe( + executable: &Path, + args: &[&str], + environment: &HashMap, +) -> Option { + let mut command = Command::new(executable); + command + .args(args) + .env_clear() + .envs(environment) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.current_dir("/"); + command.process_group(0); + } + let mut child = command.spawn().ok()?; + let (Some(stdout), Some(stderr)) = (child.stdout.take(), child.stderr.take()) else { + terminate_toolchain_probe(&mut child); + let _ = child.wait(); + return None; + }; + let stdout_reader = std::thread::spawn(move || read_bounded_probe_stream(stdout)); + let stderr_reader = std::thread::spawn(move || read_bounded_probe_stream(stderr)); + let started = Instant::now(); + let mut status = None; + loop { + if status.is_none() { + match child.try_wait() { + Ok(Some(exit_status)) => status = Some(exit_status), + Ok(None) => {} + Err(_) => { + terminate_toolchain_probe(&mut child); + let _ = child.wait(); + let _ = stdout_reader.join(); + let _ = stderr_reader.join(); + return None; + } + } + } + if status.is_some() && stdout_reader.is_finished() && stderr_reader.is_finished() { + break; + } + if started.elapsed() >= TOOLCHAIN_PROBE_TIMEOUT { + terminate_toolchain_probe(&mut child); + let _ = child.wait(); + let _ = stdout_reader.join(); + let _ = stderr_reader.join(); + return None; + } + std::thread::sleep(Duration::from_millis(10)); + } + let stdout = stdout_reader.join().ok()?; + let stderr = stderr_reader.join().ok()?; + Some(std::process::Output { + status: status?, + stdout, + stderr, + }) +} + +#[cfg(test)] +fn hash_toolchain( + trusted_environment: &HashMap, + build_environment: &HashMap, + project_dir: &Path, +) -> std::io::Result { + hash_toolchain_cached(trusted_environment, build_environment, project_dir, None) +} + +fn hash_toolchain_cached( + trusted_environment: &HashMap, + build_environment: &HashMap, + project_dir: &Path, + cache: Option<&ToolchainFingerprintCache>, +) -> std::io::Result { + let mut resolved = HashMap::with_capacity(TOOLCHAIN_PROBES.len()); + let trusted_executables = TOOLCHAIN_PROBES + .iter() + .map(|(program, _)| { + resolved + .entry(*program) + .or_insert_with(|| { + resolve_trusted_executable(program, trusted_environment, project_dir) + }) + .clone() + }) + .collect::>(); + let base_key = + toolchain_snapshot_base_key(trusted_environment, &trusted_executables, project_dir); + let mut compute = || compute_toolchain_fingerprint(trusted_environment, &trusted_executables); + let host_fingerprint = match cache { + Some(cache) => cached_toolchain_fingerprint( + cache, + &base_key, + trusted_environment, + project_dir, + &mut compute, + )?, + None => compute()?.fingerprint, + }; + let build_visible_fingerprint = hash_build_visible_toolchain_executables(build_environment); + Ok(hash_records([ + host_fingerprint.as_bytes(), + build_visible_fingerprint.as_bytes(), + ])) +} + +fn toolchain_snapshot_base_key( + trusted_environment: &HashMap, + trusted_executables: &[Option], + project_dir: &Path, +) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"lpm-toolchain-snapshot-base-v1\0"); + hasher.update(std::env::consts::OS.as_bytes()); + hasher.update(b"\0"); + hasher.update(std::env::consts::ARCH.as_bytes()); + hasher.update(b"\0"); + hasher.update(env!("CARGO_PKG_VERSION").as_bytes()); + hasher.update(b"\0"); + let probe_environment = toolchain_probe_environment(trusted_environment); + hasher.update(hash_string_map(&probe_environment).as_bytes()); + if let Some(home) = trusted_environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("HOME")) + .map(|(_, value)| value) + { + hash_os_path(&mut hasher, Path::new(home)); + } + for (index, ((program, args), executable)) in + TOOLCHAIN_PROBES.iter().zip(trusted_executables).enumerate() + { + hasher.update(index.to_le_bytes()); + hasher.update(program.as_bytes()); + for argument in *args { + hasher.update(b"\0"); + hasher.update(argument.as_bytes()); + } + if let Some(executable) = executable + && !is_project_controlled_path(&executable.invocation_path, project_dir) + { + hash_os_path(&mut hasher, &executable.invocation_path); + hasher.update(executable.identity.as_bytes()); + } + hasher.update(b"\x1e"); + } + format!("sha256-{}", hex::encode(hasher.finalize())) +} + +fn hash_build_visible_toolchain_executables(build_environment: &HashMap) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"lpm-build-visible-toolchain-v1\0"); + let mut identities = HashMap::with_capacity(TOOLCHAIN_PROBES.len()); + for (program, args) in TOOLCHAIN_PROBES { + hasher.update(program.as_bytes()); + for argument in *args { + hasher.update(b"\0"); + hasher.update(argument.as_bytes()); + } + if let Some((executable, identity)) = identities + .entry(*program) + .or_insert_with(|| { + let executable = resolve_executable(program, build_environment)?; + let identity = runtime_executable_identity(&executable).ok()?; + Some((executable, identity)) + }) + .as_ref() + { + hash_os_path(&mut hasher, executable); + hasher.update(identity.as_bytes()); + } + hasher.update(b"\x1e"); + } + format!("sha256-{}", hex::encode(hasher.finalize())) +} + +fn compute_toolchain_fingerprint( + trusted_environment: &HashMap, + trusted_executables: &[Option], +) -> std::io::Result { + let mut hasher = Sha256::new(); + hasher.update(b"lpm-toolchain-host-v6\0"); + let probe_environment = toolchain_probe_environment(trusted_environment); + let probe_outputs = std::thread::scope(|scope| { + #[expect( + clippy::needless_collect, + reason = "all probes must start before any join blocks" + )] + let handles = TOOLCHAIN_PROBES + .iter() + .zip(trusted_executables) + .map(|((_, args), executable)| { + let probe_environment = &probe_environment; + scope.spawn(move || { + let executable = executable.as_ref()?; + run_bounded_toolchain_probe( + &executable.invocation_path, + args, + probe_environment, + ) + }) + }) + .collect::>(); + handles + .into_iter() + .map(|handle| handle.join().ok().flatten()) + .collect::>() + }); + for (((program, args), executable), output) in TOOLCHAIN_PROBES + .iter() + .zip(trusted_executables) + .zip(&probe_outputs) + { + hasher.update(program.as_bytes()); + hasher.update(b"\0"); + for argument in *args { + hasher.update(argument.as_bytes()); + hasher.update(b"\0"); + } + if let Some(output) = output { + hasher.update(&output.stdout); + hasher.update(&output.stderr); + } + if let Some(executable) = executable { + hasher.update(b"trusted\0"); + hash_os_path(&mut hasher, &executable.invocation_path); + hasher.update(executable.identity.as_bytes()); + } + hasher.update(b"\x1e"); + } + for path in [ + Path::new("/etc/os-release"), + Path::new("/etc/ld.so.cache"), + Path::new("/System/Library/CoreServices/SystemVersion.plist"), + ] { + hash_optional_file(path, &mut hasher)?; + } + let host_state_paths = host_package_state_paths() + .iter() + .map(Path::new) + .collect::>(); + hash_host_package_state(&host_state_paths, &mut hasher)?; + let pkg_config_pc_path = + TOOLCHAIN_PROBES + .iter() + .zip(&probe_outputs) + .find_map(|((program, args), output)| { + (*program == "pkg-config" + && *args == ["--variable=pc_path", "pkg-config"] + && output.as_ref().is_some_and(|value| value.status.success())) + .then(|| output.as_ref().map(|value| value.stdout.as_slice())) + .flatten() + }); + let pkg_config_paths = pkg_config_search_paths(trusted_environment, pkg_config_pc_path); + hash_pkg_config_directories(&pkg_config_paths, &mut hasher)?; + for cellar in [ + Path::new("/opt/homebrew/Cellar"), + Path::new("/usr/local/Cellar"), + Path::new("/home/linuxbrew/.linuxbrew/Cellar"), + ] { + hash_homebrew_state(cellar, &mut hasher)?; + } + #[cfg(target_os = "macos")] + for receipts in [ + Path::new("/var/db/receipts"), + Path::new("/Library/Apple/System/Library/Receipts"), + ] { + hash_directory_files_with_extension(receipts, "plist", &mut hasher)?; + } + if let Some(home) = trusted_environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("HOME")) + .map(|(_, value)| PathBuf::from(value)) + .or_else(dirs::home_dir) + { + hash_directory_tree(&home.join(".node-gyp"), &mut hasher)?; + } + Ok(ComputedToolchainFingerprint { + fingerprint: format!("sha256-{}", hex::encode(hasher.finalize())), + pkg_config_paths, + }) +} + +fn hash_string_map(values: &HashMap) -> String { + let mut values = values.iter().collect::>(); + values.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + hash_records(values.into_iter().map(|(key, value)| { + let mut record = String::with_capacity(key.len() + value.len() + 1); + record.push_str(key); + record.push('\0'); + record.push_str(value); + record + })) +} + +#[cfg(target_os = "linux")] +fn host_package_state_paths() -> &'static [&'static str] { + &[ + "/var/lib/dpkg/status", + "/var/lib/apt/extended_states", + "/var/lib/rpm/Packages", + "/var/lib/rpm/rpmdb.sqlite", + "/usr/lib/sysimage/rpm/Packages", + "/usr/lib/sysimage/rpm/rpmdb.sqlite", + "/lib/apk/db/installed", + "/var/lib/pacman/local", + ] +} + +#[cfg(not(target_os = "linux"))] +fn host_package_state_paths() -> &'static [&'static str] { + &[] +} + +fn hash_host_package_state(paths: &[&Path], hasher: &mut Sha256) -> std::io::Result<()> { + hasher.update(b"host-package-state-v1\0"); + for path in paths { + if path.is_dir() { + hash_directory_tree(path, hasher)?; + } else { + hash_optional_file(path, hasher)?; + } + } + Ok(()) +} + +fn pkg_config_search_paths( + environment: &HashMap, + default_search_path: Option<&[u8]>, +) -> Vec { + let mut search_path = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("PKG_CONFIG_PATH")) + .map(|(_, value)| value.clone()) + .unwrap_or_default(); + if let Some(default_search_path) = default_search_path { + if !search_path.is_empty() { + search_path.push(':'); + } + search_path.push_str(String::from_utf8_lossy(default_search_path).trim()); + } + let mut paths = std::env::split_paths(&search_path) + .filter(|path| !path.as_os_str().is_empty()) + .collect::>(); + paths.sort_unstable(); + paths.dedup(); + paths +} + +fn hash_pkg_config_directories(paths: &[PathBuf], hasher: &mut Sha256) -> std::io::Result<()> { + for path in paths { + hash_directory_tree(path, hasher)?; + } + Ok(()) +} + +fn hash_homebrew_state(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hash_os_path(hasher, root); + let mut formulae = match std::fs::read_dir(root) { + Ok(entries) => entries.collect::, _>>()?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hasher.update(b"absent\x1e"); + return Ok(()); + } + Err(error) => return Err(error), + }; + formulae.sort_unstable_by_key(std::fs::DirEntry::file_name); + for formula in formulae { + if !formula.file_type()?.is_dir() { + continue; + } + let formula_name = formula.file_name(); + hash_os_path(hasher, Path::new(&formula_name)); + let mut versions = std::fs::read_dir(formula.path())?.collect::, _>>()?; + versions.sort_unstable_by_key(std::fs::DirEntry::file_name); + for version in versions { + if !version.file_type()?.is_dir() { + continue; + } + let version_name = version.file_name(); + hash_os_path(hasher, Path::new(&version_name)); + hash_optional_file(&version.path().join("INSTALL_RECEIPT.json"), hasher)?; + } + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn hash_directory_files_with_extension( + root: &Path, + extension: &str, + hasher: &mut Sha256, +) -> std::io::Result<()> { + hash_os_path(hasher, root); + let mut paths = match std::fs::read_dir(root) { + Ok(entries) => entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|value| value == extension)) + .collect::>(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hasher.update(b"absent\x1e"); + return Ok(()); + } + Err(error) => return Err(error), + }; + paths.sort_unstable(); + for path in paths { + hash_optional_file(&path, hasher)?; + } + Ok(()) +} + +fn hash_optional_file(path: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hash_os_path(hasher, path); + match std::fs::File::open(path) { + Ok(mut file) => { + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => hasher.update(b"absent"), + Err(error) => return Err(error), + } + hasher.update(b"\x1e"); + Ok(()) +} + +fn hash_invoked_build_executables( + package: &ScriptablePackage, + environment: &HashMap, +) -> String { + let mut identities = Vec::new(); + for command in package.scripts.values() { + let Some(segments) = shell_command_segments(command) else { + continue; + }; + for words in segments { + let Some(command_index) = words + .iter() + .position(|word| !is_environment_assignment(word)) + else { + continue; + }; + let executable = &words[command_index]; + let Some(path) = resolve_executable(executable, environment) else { + continue; + }; + if let Ok(identity) = runtime_executable_identity(&path) { + identities.push(format!("{executable}\0{identity}")); + } + } + } + identities.sort_unstable(); + hash_records(identities.iter().map(String::as_bytes)) +} + +fn resolve_executable(executable: &str, environment: &HashMap) -> Option { + let executable_path = Path::new(executable); + if executable_path.components().count() > 1 { + return executable_path + .is_file() + .then(|| executable_path.to_path_buf()); + } + let path = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("PATH")) + .map(|(_, value)| value)?; + std::env::split_paths(path) + .map(|directory| directory.join(executable)) + .find(|candidate| candidate.is_file()) +} + +fn hash_directory_tree(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hasher.update(b"directory-tree-v1\0"); + if !root.exists() { + hasher.update(b"absent"); + return Ok(()); + } + let mut pending = vec![root.to_path_buf()]; + let mut buffer = [0_u8; 64 * 1024]; + while let Some(directory) = pending.pop() { + let mut entries = std::fs::read_dir(&directory)?.collect::, _>>()?; + entries.sort_unstable_by_key(std::fs::DirEntry::file_name); + for entry in entries { + let path = entry.path(); + let relative = path.strip_prefix(root).unwrap_or(&path); + hash_os_path(hasher, relative); + let file_type = entry.file_type()?; + if file_type.is_dir() { + hasher.update(b"d"); + pending.push(path); + } else if file_type.is_file() { + hasher.update(b"f"); + let mut file = std::fs::File::open(path)?; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + } else if file_type.is_symlink() { + hasher.update(b"l"); + hash_os_path(hasher, &std::fs::read_link(path)?); + } else { + hasher.update(b"s"); + } + hasher.update(b"\x1e"); + } + } + Ok(()) +} + +fn hash_os_path(hasher: &mut Sha256, path: &Path) { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + let bytes = path.as_os_str().as_bytes(); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + } + #[cfg(not(unix))] + { + let value = path.to_string_lossy(); + hasher.update((value.len() as u64).to_le_bytes()); + hasher.update(value.as_bytes()); + } +} + +fn cpu_features_hash() -> String { + let mut features = Vec::new(); + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + for (name, enabled) in [ + ("sse2", std::is_x86_feature_detected!("sse2")), + ("sse4.2", std::is_x86_feature_detected!("sse4.2")), + ("avx", std::is_x86_feature_detected!("avx")), + ("avx2", std::is_x86_feature_detected!("avx2")), + ("fma", std::is_x86_feature_detected!("fma")), + ] { + if enabled { + features.push(name); + } + } + } + #[cfg(target_arch = "aarch64")] + { + features.push("aarch64"); + } + hash_records(features.iter().map(|feature| feature.as_bytes())) +} + +fn hash_allowed_inputs(capabilities: &CapabilitySet) -> String { + let records = [ + capabilities.canonical_hash(), + "package-local-writes".into(), + "graph-and-fingerprinted-system-reads".into(), + "network-denied".into(), + ]; + hash_records(records.iter().map(String::as_bytes)) +} + +fn hash_records(records: T) -> String +where + T: IntoIterator, + B: AsRef<[u8]>, +{ + let mut hasher = Sha256::new(); + for record in records { + let record = record.as_ref(); + hasher.update((record.len() as u64).to_le_bytes()); + hasher.update(record); + } + format!("sha256-{}", hex::encode(hasher.finalize())) +} + +fn runtime_executable_identity(path: &Path) -> std::io::Result { + let canonical = std::fs::canonicalize(path)?; + let metadata = std::fs::metadata(&canonical)?; + let mut hasher = Sha256::new(); + hasher.update(b"lpm-runtime-executable-v1\0"); + hash_os_path(&mut hasher, &canonical); + hasher.update(metadata.len().to_le_bytes()); + if let Ok(modified) = metadata.modified() + && let Ok(duration) = modified.duration_since(std::time::UNIX_EPOCH) + { + hasher.update(duration.as_secs().to_le_bytes()); + hasher.update(duration.subsec_nanos().to_le_bytes()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + hasher.update(metadata.dev().to_le_bytes()); + hasher.update(metadata.ino().to_le_bytes()); + hasher.update(metadata.ctime().to_le_bytes()); + hasher.update(metadata.ctime_nsec().to_le_bytes()); + } + Ok(format!("sha256-{}", hex::encode(hasher.finalize()))) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + fn scriptable_package(name: &str, command: &str) -> ScriptablePackage { + ScriptablePackage { + name: name.into(), + version: "1.0.0".into(), + integrity: None, + wrapper_id: None, + store_path: PathBuf::new(), + pristine_path: PathBuf::new(), + source_integrity: "sha512-source".into(), + graph_key_digest: Some("a".repeat(64)), + scripts: [("postinstall".into(), command.into())] + .into_iter() + .collect(), + is_built: false, + build_marker_key: None, + is_trusted: true, + trust_reason: super::super::trust::TrustReason::StrictBinding, + } + } + + fn graph_lockfile( + left_dependencies: &[&str], + right_dependencies: &[&str], + ) -> lpm_lockfile::Lockfile { + let mut lockfile = lpm_lockfile::Lockfile::new(); + for (name, dependencies) in [ + ("root", vec!["left@1.0.0", "right@1.0.0"]), + ("left", left_dependencies.to_vec()), + ("right", right_dependencies.to_vec()), + ("leaf", Vec::new()), + ] { + lockfile.add_package(lpm_lockfile::LockedPackage { + name: name.into(), + version: "1.0.0".into(), + source: Some("registry+https://registry.npmjs.org".into()), + integrity: Some(format!("sha512-{name}")), + dependencies: dependencies.into_iter().map(str::to_owned).collect(), + ..Default::default() + }); + } + lockfile + } + + fn write_link_meta(link_dir: &Path, package_name: &str) -> String { + let inputs = lpm_store::v2::GraphKeyInputs::new( + package_name, + "1.0.0", + lpm_store::v2::PlatformTuple::current(), + lpm_store::v2::LinkerModeTag::Isolated, + ); + let key = lpm_store::v2::GraphKey::derive(&inputs); + let metadata = lpm_store::v2::LinkMeta::new( + &key, + "sha512-source", + "objects/sha512-source", + Vec::new(), + std::sync::Arc::new(lpm_store::v2::LinkMetaPlatform { + os: std::env::consts::OS.into(), + cpu: std::env::consts::ARCH.into(), + libc: None, + }), + ); + metadata.write_to(link_dir).unwrap(); + metadata.graph_key_digest_hex + } + + #[test] + fn build_environment_hash_changes_with_custom_values() { + let mut first = HashMap::new(); + first.insert("UNRELATED".into(), "one".into()); + let mut second = HashMap::new(); + second.insert("UNRELATED".into(), "two".into()); + assert_ne!( + hash_build_environment(&first), + hash_build_environment(&second) + ); + } + + #[test] + fn build_environment_hash_changes_with_compiler_flags() { + let mut first = HashMap::new(); + first.insert("CFLAGS".into(), "-O2".into()); + let mut second = HashMap::new(); + second.insert("CFLAGS".into(), "-O3".into()); + assert_ne!( + hash_build_environment(&first), + hash_build_environment(&second) + ); + } + + #[test] + fn runtime_only_key_namespace_invalidates_pre_hardening_artifacts() { + fn derive(toolchain_hash: &str) -> BuildCacheKey { + BuildCacheKey::derive(&BuildKeyInputs { + source_integrity: "sha512-source".into(), + graph_key_digest: "graph".into(), + dependency_closure_hash: "closure".into(), + scripts: vec![BuildScriptFingerprint { + phase: "postinstall".into(), + command: "node install.js".into(), + }], + platform: BuildPlatformFingerprint { + os: "linux".into(), + architecture: "x86_64".into(), + libc: "glibc".into(), + cpu_features_hash: "cpu".into(), + }, + runtime: BuildRuntimeFingerprint { + runtime: "node".into(), + version: "22.0.0".into(), + modules_abi: "127".into(), + napi: "10".into(), + engine: "v8".into(), + executable_hash: "node".into(), + }, + sandbox: BuildSandboxFingerprint { + mode: "enforce".into(), + posture: "strict".into(), + network_denied: true, + environment_scrubbed: true, + allowed_inputs_hash: "inputs".into(), + }, + environment_hash: "environment".into(), + toolchain_hash: toolchain_hash.into(), + }) + } + + let pre_hardening = derive("runtime-and-dependency-graph-v1"); + let current = derive(RUNTIME_ONLY_TOOLCHAIN_FINGERPRINT); + + assert_ne!(current, pre_hardening); + } + + #[test] + fn build_marker_reader_accepts_only_cache_key_shape() { + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join(".lpm-built"); + let key = "a".repeat(64); + std::fs::write(&marker, &key).unwrap(); + + assert_eq!(read_build_marker_key(&marker), Some(key)); + } + + #[test] + fn build_marker_reader_preserves_legacy_and_corrupt_markers_as_unkeyed() { + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join(".lpm-built"); + std::fs::write(&marker, b"").unwrap(); + assert!(read_build_marker_key(&marker).is_none()); + + std::fs::write(&marker, "G".repeat(64)).unwrap(); + assert!(read_build_marker_key(&marker).is_none()); + } + + #[test] + fn directory_tree_hash_changes_when_node_headers_change() { + let temp = tempfile::tempdir().unwrap(); + let headers = temp.path().join("include/node"); + std::fs::create_dir_all(&headers).unwrap(); + std::fs::write(headers.join("node.h"), b"first").unwrap(); + let mut first = Sha256::new(); + hash_directory_tree(temp.path(), &mut first).unwrap(); + + std::fs::write(headers.join("node.h"), b"second").unwrap(); + let mut second = Sha256::new(); + hash_directory_tree(temp.path(), &mut second).unwrap(); + + assert_ne!(first.finalize(), second.finalize()); + } + + #[test] + fn host_package_state_hash_changes_when_installed_state_changes() { + let temp = tempfile::tempdir().unwrap(); + let state = temp.path().join("status"); + std::fs::write(&state, b"Version: 1\n").unwrap(); + let mut first = Sha256::new(); + hash_host_package_state(&[state.as_path()], &mut first).unwrap(); + + std::fs::write(&state, b"Version: 2\n").unwrap(); + let mut second = Sha256::new(); + hash_host_package_state(&[state.as_path()], &mut second).unwrap(); + + assert_ne!(first.finalize(), second.finalize()); + } + + #[test] + #[cfg(unix)] + fn persisted_toolchain_snapshot_skips_unchanged_trusted_probe() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let host_bin = temp.path().join("host-bin"); + let home = temp.path().join("home"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&host_bin).unwrap(); + std::fs::create_dir(&home).unwrap(); + let marker = temp.path().join("cmake-probes"); + let cmake = host_bin.join("cmake"); + std::fs::write( + &cmake, + format!( + "#!/bin/sh\nprintf 'probe\\n' >> '{}'\necho cmake-one\n", + marker.display() + ), + ) + .unwrap(); + std::fs::set_permissions(&cmake, std::fs::Permissions::from_mode(0o755)).unwrap(); + let environment = HashMap::from([ + ("PATH".to_string(), host_bin.display().to_string()), + ("HOME".to_string(), home.display().to_string()), + ]); + let cache = ToolchainFingerprintCache::for_test(temp.path().join("cache")); + + let first = + hash_toolchain_cached(&environment, &environment, &project, Some(&cache)).unwrap(); + assert_eq!(std::fs::read_to_string(&marker).unwrap(), "probe\n"); + let second = + hash_toolchain_cached(&environment, &environment, &project, Some(&cache)).unwrap(); + + assert_eq!(first, second); + assert_eq!(std::fs::read_to_string(marker).unwrap(), "probe\n"); + } + + #[test] + fn homebrew_state_hash_changes_when_install_receipt_changes() { + let temp = tempfile::tempdir().unwrap(); + let version = temp.path().join("example/1.0.0"); + std::fs::create_dir_all(&version).unwrap(); + std::fs::write( + version.join("INSTALL_RECEIPT.json"), + b"{\"source\":\"first\"}", + ) + .unwrap(); + let mut first = Sha256::new(); + hash_homebrew_state(temp.path(), &mut first).unwrap(); + + std::fs::write( + version.join("INSTALL_RECEIPT.json"), + b"{\"source\":\"second\"}", + ) + .unwrap(); + let mut second = Sha256::new(); + hash_homebrew_state(temp.path(), &mut second).unwrap(); + + assert_ne!(first.finalize(), second.finalize()); + } + + #[test] + #[cfg(unix)] + fn toolchain_hash_tracks_project_cmake_without_executing_it() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let bin = project.join("node_modules/.bin"); + std::fs::create_dir_all(&bin).unwrap(); + let cmake = bin.join("cmake"); + let marker = temp.path().join("executed"); + std::fs::write( + &cmake, + format!("#!/bin/sh\n: > '{}'\necho cmake-one\n", marker.display()), + ) + .unwrap(); + std::fs::set_permissions(&cmake, std::fs::Permissions::from_mode(0o755)).unwrap(); + let build_environment = HashMap::from([ + ("PATH".to_string(), bin.display().to_string()), + ("HOME".to_string(), temp.path().display().to_string()), + ]); + let trusted_environment = trusted_toolchain_environment(&build_environment, &project); + let first = hash_toolchain(&trusted_environment, &build_environment, &project).unwrap(); + assert!(!marker.exists()); + + std::fs::write( + &cmake, + format!("#!/bin/sh\n: > '{}'\necho cmake-two\n", marker.display()), + ) + .unwrap(); + let second = hash_toolchain(&trusted_environment, &build_environment, &project).unwrap(); + + assert_ne!(first, second); + assert!(!marker.exists()); + } + + #[test] + #[cfg(unix)] + fn toolchain_probe_timeout_bounds_stalled_executable() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + std::fs::create_dir(&project).unwrap(); + let bin = temp.path().join("host-bin"); + std::fs::create_dir(&bin).unwrap(); + let cmake = bin.join("cmake"); + std::fs::write(&cmake, "#!/bin/sh\n/bin/sleep 5\n").unwrap(); + std::fs::set_permissions(&cmake, std::fs::Permissions::from_mode(0o755)).unwrap(); + let environment = HashMap::from([ + ("PATH".to_string(), bin.display().to_string()), + ("HOME".to_string(), temp.path().display().to_string()), + ]); + + let started = std::time::Instant::now(); + hash_toolchain(&environment, &environment, &project).unwrap(); + + assert!(started.elapsed() < std::time::Duration::from_secs(3)); + } + + #[test] + #[cfg(unix)] + fn toolchain_probe_timeout_kills_descendant_holding_output_pipe() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let probe = temp.path().join("probe"); + std::fs::write(&probe, "#!/bin/sh\n(/bin/sleep 5) &\necho ready\n").unwrap(); + std::fs::set_permissions(&probe, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let started = std::time::Instant::now(); + let output = run_bounded_toolchain_probe(&probe, &[], &HashMap::new()); + + assert!(output.is_none()); + assert!(started.elapsed() < std::time::Duration::from_secs(3)); + } + + #[test] + #[cfg(unix)] + fn trusted_proxy_executes_with_symlink_invocation_name() { + use std::os::unix::fs::{PermissionsExt, symlink}; + + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let bin = temp.path().join("host-bin"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&bin).unwrap(); + let manager = bin.join("multicall"); + std::fs::write(&manager, "#!/bin/sh\necho \"${0##*/}\"\n").unwrap(); + std::fs::set_permissions(&manager, std::fs::Permissions::from_mode(0o755)).unwrap(); + symlink(&manager, bin.join("rustc")).unwrap(); + let environment = HashMap::from([("PATH".to_string(), bin.display().to_string())]); + let executable = resolve_trusted_executable("rustc", &environment, &project).unwrap(); + + let output = + run_bounded_toolchain_probe(&executable.invocation_path, &[], &environment).unwrap(); + + assert_eq!(String::from_utf8(output.stdout).unwrap().trim(), "rustc"); + } + + #[test] + #[cfg(unix)] + fn toolchain_probe_runs_from_neutral_directory() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let probe = temp.path().join("probe"); + std::fs::write(&probe, "#!/bin/sh\npwd\n").unwrap(); + std::fs::set_permissions(&probe, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let output = run_bounded_toolchain_probe(&probe, &[], &HashMap::new()).unwrap(); + + assert_eq!(String::from_utf8(output.stdout).unwrap().trim(), "/"); + } + + #[test] + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn runtime_only_key_is_rejected_when_project_node_shadows_trusted_runtime() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let project_bin = project.join("node_modules/.bin"); + let trusted_bin = temp.path().join("trusted-bin"); + let link_dir = temp.path().join("links/esbuild"); + let package_dir = link_dir.join("node_modules/esbuild"); + std::fs::create_dir_all(&project_bin).unwrap(); + std::fs::create_dir(&trusted_bin).unwrap(); + std::fs::create_dir_all(&package_dir).unwrap(); + std::fs::write(project_bin.join("node"), b"project node").unwrap(); + std::fs::write(trusted_bin.join("node"), b"trusted node").unwrap(); + std::fs::set_permissions( + project_bin.join("node"), + std::fs::Permissions::from_mode(0o755), + ) + .unwrap(); + std::fs::set_permissions( + trusted_bin.join("node"), + std::fs::Permissions::from_mode(0o755), + ) + .unwrap(); + let graph_key_digest = write_link_meta(&link_dir, "esbuild"); + let platform = PlatformTuple::current(); + let trusted_path = trusted_bin.display().to_string(); + let trusted_environment = HashMap::from([("PATH".to_string(), trusted_path.clone())]); + let build_executable_environment = HashMap::from([( + "PATH".to_string(), + build_lifecycle_path(&project, Some(&trusted_path)), + )]); + let invocation = BuildCacheInvocation { + dependency_closure_hash: "graph".into(), + platform: BuildPlatformFingerprint { + os: platform.os, + architecture: platform.cpu, + libc: platform.libc.unwrap_or_default(), + cpu_features_hash: "cpu".into(), + }, + runtime: BuildRuntimeFingerprint { + runtime: "node".into(), + version: "22.0.0".into(), + modules_abi: "127".into(), + napi: "10".into(), + engine: "v8".into(), + executable_hash: "trusted-node".into(), + }, + sandbox: BuildSandboxFingerprint { + mode: "enforce".into(), + posture: "strict".into(), + network_denied: true, + environment_scrubbed: true, + allowed_inputs_hash: "inputs".into(), + }, + environment: trusted_environment.clone(), + trusted_toolchain_environment: trusted_environment, + build_executable_environment, + project_dir: project.clone(), + toolchain_cache: None, + native_toolchain_hash: OnceLock::new(), + }; + let mut package = scriptable_package("esbuild", "node install.js"); + package.store_path = package_dir; + package.graph_key_digest = Some(graph_key_digest); + + assert!(build_key_for_package(&invocation, &package, &project).is_none()); + } + + #[test] + fn toolchain_probe_reader_discards_output_beyond_limit() { + let input = vec![b'x'; TOOLCHAIN_PROBE_OUTPUT_LIMIT + 1024]; + + let output = read_bounded_probe_stream(std::io::Cursor::new(input)); + + assert_eq!(output.len(), TOOLCHAIN_PROBE_OUTPUT_LIMIT); + } + + #[test] + fn toolchain_probe_environment_excludes_code_injection_variables() { + let trusted = HashMap::from([ + ("PATH".into(), "/usr/bin:/bin".into()), + ( + "NODE_OPTIONS".into(), + "--require=/project/payload.js".into(), + ), + ("LD_PRELOAD".into(), "/project/payload.so".into()), + ( + "DYLD_INSERT_LIBRARIES".into(), + "/project/payload.dylib".into(), + ), + ]); + + let probe = toolchain_probe_environment(&trusted); + + assert_eq!(probe.get("PATH").map(String::as_str), Some("/usr/bin:/bin")); + assert!(!probe.contains_key("NODE_OPTIONS")); + assert!(!probe.contains_key("LD_PRELOAD")); + assert!(!probe.contains_key("DYLD_INSERT_LIBRARIES")); + } + + #[test] + #[cfg(unix)] + fn invoked_tool_hash_changes_when_cmake_js_executable_changes() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let bin = temp.path().join("bin"); + std::fs::create_dir(&bin).unwrap(); + let cmake_js = bin.join("cmake-js"); + std::fs::write(&cmake_js, "#!/bin/sh\necho cmake-js-one\n").unwrap(); + std::fs::set_permissions(&cmake_js, std::fs::Permissions::from_mode(0o755)).unwrap(); + let environment = HashMap::from([("PATH".to_string(), bin.display().to_string())]); + let package = scriptable_package("fixture", "cmake-js compile"); + let first = hash_invoked_build_executables(&package, &environment); + + std::fs::write(&cmake_js, "#!/bin/sh\necho cmake-js-two\n").unwrap(); + let second = hash_invoked_build_executables(&package, &environment); + + assert_ne!(first, second); + } + + #[test] + fn lockfile_graph_hash_changes_when_dependency_edge_is_rewired() { + let first = graph_lockfile(&["leaf@1.0.0"], &[]); + let second = graph_lockfile(&[], &["leaf@1.0.0"]); + + assert_ne!(hash_lockfile_graph(&first), hash_lockfile_graph(&second)); + } + + #[test] + fn lockfile_graph_hash_changes_when_patch_identity_changes() { + let mut first = graph_lockfile(&["leaf@1.0.0"], &[]); + first.patches.insert( + "leaf@1.0.0".into(), + lpm_lockfile::LockfilePatch { + path: "patches/leaf.patch".into(), + sha256: "sha256-first".into(), + original_integrity: "sha512-leaf".into(), + }, + ); + let mut second = first.clone(); + second.patches.get_mut("leaf@1.0.0").unwrap().sha256 = "sha256-second".into(); + + assert_ne!(hash_lockfile_graph(&first), hash_lockfile_graph(&second)); + } + + #[test] + fn build_cache_scratch_path_is_stable_for_one_link_entry() { + let temp = tempfile::tempdir().unwrap(); + let link_dir = temp.path().join("links/example"); + let package_dir = link_dir.join("node_modules/example"); + std::fs::create_dir_all(&package_dir).unwrap(); + let digest = write_link_meta(&link_dir, "example"); + + let first_path = { + let scratch = BuildCacheScratch::create(&package_dir, &digest).unwrap(); + scratch.path().to_path_buf() + }; + let second_path = { + let scratch = BuildCacheScratch::create(&package_dir, &digest).unwrap(); + scratch.path().to_path_buf() + }; + + assert_eq!(first_path, second_path); + } + + #[test] + fn build_cache_scratch_supports_scoped_package_path() { + let temp = tempfile::tempdir().unwrap(); + let link_dir = temp.path().join("links/scoped"); + let package_dir = link_dir.join("node_modules/@scope/example"); + std::fs::create_dir_all(&package_dir).unwrap(); + let digest = write_link_meta(&link_dir, "@scope/example"); + + let scratch = BuildCacheScratch::create(&package_dir, &digest).unwrap(); + + assert_eq!(scratch.path(), link_dir.join(".lpm-build-tmp")); + } + + #[test] + fn graph_identity_discovery_supports_scoped_package_path() { + let temp = tempfile::tempdir().unwrap(); + let link_dir = temp.path().join("links/scoped"); + let package_dir = link_dir.join("node_modules/@scope/example"); + std::fs::create_dir_all(&package_dir).unwrap(); + let digest = write_link_meta(&link_dir, "@scope/example"); + + assert_eq!(read_v2_graph_key_digest(&package_dir), Some(digest)); + } + + #[test] + fn build_cache_scratch_rejects_package_controlled_sidecar() { + let temp = tempfile::tempdir().unwrap(); + let package_dir = temp.path().join("links/example/node_modules/example"); + std::fs::create_dir_all(&package_dir).unwrap(); + std::fs::write(package_dir.join(lpm_store::v2::LINK_META_FILENAME), b"{}").unwrap(); + + assert!(BuildCacheScratch::create(&package_dir, &"a".repeat(64)).is_err()); + assert!(!package_dir.join(".lpm-build-tmp").exists()); + } + + #[test] + #[cfg(unix)] + fn build_cache_scratch_replaces_symlink_without_touching_target() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let link_dir = temp.path().join("links/example"); + let package_dir = link_dir.join("node_modules/example"); + let outside = temp.path().join("outside"); + std::fs::create_dir_all(&package_dir).unwrap(); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::write(outside.join("sentinel"), b"keep").unwrap(); + let digest = write_link_meta(&link_dir, "example"); + symlink(&outside, link_dir.join(".lpm-build-tmp")).unwrap(); + + let scratch = BuildCacheScratch::create(&package_dir, &digest).unwrap(); + + assert!(outside.join("sentinel").is_file()); + assert_ne!(std::fs::canonicalize(scratch.path()).unwrap(), outside); + } + + #[test] + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn native_build_detection_ignores_shell_comments() { + let package = scriptable_package("fixture", "node build.js # node-gyp rebuild"); + + assert!(!is_cacheable_native_build(&package)); + } + + #[test] + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn native_build_detection_ignores_argument_text() { + let package = scriptable_package("fixture", "node build.js node-gyp"); + + assert!(!is_cacheable_native_build(&package)); + } + + #[test] + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn native_build_detection_recognizes_package_specific_esbuild_installer() { + let package = scriptable_package("esbuild", "node install.js"); + + assert!(is_cacheable_native_build(&package)); + } + + #[test] + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn native_build_detection_recognizes_compound_fallback_command_position() { + let package = scriptable_package("fixture", "prebuild-install || node-gyp rebuild"); + + assert!(is_cacheable_native_build(&package)); + } +} diff --git a/crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs b/crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs new file mode 100644 index 00000000..ace1d5c5 --- /dev/null +++ b/crates/lpm-cli/src/commands/rebuild/build_cache/toolchain_snapshot.rs @@ -0,0 +1,571 @@ +use super::{ + hash_optional_file, hash_os_path, hash_records, host_package_state_paths, + is_project_controlled_path, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +const MAX_AGE: Duration = Duration::from_secs(60 * 60); +const MAX_BYTES: u64 = 256 * 1024; +const SCHEMA: u32 = 1; + +#[derive(Clone)] +pub(super) struct ToolchainFingerprintCache { + directory: PathBuf, +} + +impl ToolchainFingerprintCache { + pub(super) fn from_lpm_root(root: lpm_common::LpmRoot) -> Self { + Self { + directory: root.cache_metadata().join("native-toolchains/v1"), + } + } + + #[cfg(test)] + pub(super) fn for_test(directory: PathBuf) -> Self { + Self { directory } + } + + fn entry_path(&self, base_key: &str) -> Option { + let digest = base_key.strip_prefix("sha256-")?; + is_lower_hex_digest(digest).then(|| self.directory.join(format!("{digest}.json"))) + } +} + +#[derive(Serialize, Deserialize)] +struct ToolchainFingerprintSnapshot { + schema: u32, + base_key: String, + fingerprint: String, + validation_hash: String, + pkg_config_paths: Vec, + created_unix_seconds: u64, + entry_hash: String, +} + +pub(super) struct ComputedToolchainFingerprint { + pub(super) fingerprint: String, + pub(super) pkg_config_paths: Vec, +} + +pub(super) fn cached_toolchain_fingerprint( + cache: &ToolchainFingerprintCache, + base_key: &str, + environment: &HashMap, + project_dir: &Path, + compute: &mut F, +) -> std::io::Result +where + F: FnMut() -> std::io::Result, +{ + let Some(entry_path) = cache.entry_path(base_key) else { + return Ok(compute()?.fingerprint); + }; + if std::fs::create_dir_all(&cache.directory).is_err() { + return Ok(compute()?.fingerprint); + } + let lock_path = entry_path.with_extension("lock"); + let Ok(_lock) = lpm_common::acquire_exclusive_lock(&lock_path) else { + return Ok(compute()?.fingerprint); + }; + if let Some(snapshot) = read_snapshot(&entry_path, base_key) + && snapshot_paths_are_allowed(&snapshot, environment, project_dir) + && validation_hash(environment, &snapshot.pkg_config_paths) + .is_ok_and(|current| current == snapshot.validation_hash) + { + tracing::debug!(target: "lpm_cli::build_cache", "native toolchain snapshot hit"); + return Ok(snapshot.fingerprint); + } + tracing::debug!(target: "lpm_cli::build_cache", "native toolchain snapshot miss"); + let computed = compute()?; + let validation_hash = validation_hash(environment, &computed.pkg_config_paths)?; + let created_unix_seconds = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let mut snapshot = ToolchainFingerprintSnapshot { + schema: SCHEMA, + base_key: base_key.to_owned(), + fingerprint: computed.fingerprint.clone(), + validation_hash, + pkg_config_paths: computed.pkg_config_paths, + created_unix_seconds, + entry_hash: String::new(), + }; + snapshot.entry_hash = entry_hash(&snapshot); + if let Err(error) = write_snapshot(&entry_path, &snapshot) { + tracing::debug!( + target: "lpm_cli::build_cache", + path = %entry_path.display(), + "failed to persist native toolchain snapshot: {error}" + ); + } + Ok(computed.fingerprint) +} + +fn read_snapshot(path: &Path, expected_base_key: &str) -> Option { + let file = std::fs::File::open(path).ok()?; + let metadata = file.metadata().ok()?; + if metadata.len() > MAX_BYTES { + return None; + } + let snapshot: ToolchainFingerprintSnapshot = serde_json::from_reader(file).ok()?; + if snapshot.schema != SCHEMA + || snapshot.base_key != expected_base_key + || !is_sha256_fingerprint(&snapshot.fingerprint) + || !is_sha256_fingerprint(&snapshot.validation_hash) + || snapshot.pkg_config_paths.len() > 128 + || snapshot.entry_hash != entry_hash(&snapshot) + { + return None; + } + let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?.as_secs(); + let age = now.checked_sub(snapshot.created_unix_seconds)?; + (age <= MAX_AGE.as_secs()).then_some(snapshot) +} + +fn snapshot_paths_are_allowed( + snapshot: &ToolchainFingerprintSnapshot, + environment: &HashMap, + project_dir: &Path, +) -> bool { + let configured_paths = environment + .iter() + .filter(|(key, _)| { + key.eq_ignore_ascii_case("PKG_CONFIG_PATH") + || key.eq_ignore_ascii_case("PKG_CONFIG_LIBDIR") + }) + .flat_map(|(_, value)| std::env::split_paths(value)) + .collect::>(); + snapshot.pkg_config_paths.iter().all(|path| { + let is_configured = configured_paths.iter().any(|configured| configured == path); + path.is_absolute() + && path.as_os_str().len() <= 4096 + && (is_configured + || (path.file_name().is_some_and(|name| name == "pkgconfig") + && !is_project_controlled_path(path, project_dir))) + }) +} + +fn entry_hash(snapshot: &ToolchainFingerprintSnapshot) -> String { + let mut records = Vec::with_capacity(snapshot.pkg_config_paths.len() + 6); + records.push(format!("schema\0{}", snapshot.schema)); + records.push(format!("base\0{}", snapshot.base_key)); + records.push(format!("fingerprint\0{}", snapshot.fingerprint)); + records.push(format!("validation\0{}", snapshot.validation_hash)); + records.push(format!("created\0{}", snapshot.created_unix_seconds)); + records.extend( + snapshot + .pkg_config_paths + .iter() + .map(|path| format!("pkg-config\0{}", path.display())), + ); + hash_records(records.iter().map(String::as_bytes)) +} + +fn write_snapshot(path: &Path, snapshot: &ToolchainFingerprintSnapshot) -> std::io::Result<()> { + let parent = path + .parent() + .ok_or_else(|| std::io::Error::other("toolchain snapshot has no parent directory"))?; + let mut temporary = tempfile::Builder::new() + .prefix(".native-toolchain-") + .tempfile_in(parent)?; + serde_json::to_writer(temporary.as_file_mut(), snapshot)?; + temporary.as_file_mut().write_all(b"\n")?; + temporary.as_file_mut().sync_all()?; + temporary + .persist(path) + .map_err(|error| error.error) + .map(|_| ()) +} + +fn validation_hash( + environment: &HashMap, + pkg_config_paths: &[PathBuf], +) -> std::io::Result { + let mut hasher = Sha256::new(); + hasher.update(b"lpm-toolchain-validation-v1\0"); + for path in [ + Path::new("/etc/os-release"), + Path::new("/etc/ld.so.cache"), + Path::new("/etc/hostname"), + Path::new("/proc/sys/kernel/osrelease"), + Path::new("/proc/sys/kernel/version"), + Path::new("/System/Library/CoreServices/SystemVersion.plist"), + Path::new("/var/db/xcode_select_link"), + ] { + hash_path_metadata(path, &mut hasher)?; + if path != Path::new("/var/db/xcode_select_link") { + hash_optional_file(path, &mut hasher)?; + } + } + for path in host_package_state_paths().iter().map(Path::new) { + hash_metadata_tree(path, &mut hasher)?; + } + for path in pkg_config_paths { + hash_metadata_tree(path, &mut hasher)?; + } + for cellar in [ + Path::new("/opt/homebrew/Cellar"), + Path::new("/usr/local/Cellar"), + Path::new("/home/linuxbrew/.linuxbrew/Cellar"), + ] { + hash_homebrew_metadata(cellar, &mut hasher)?; + } + #[cfg(target_os = "macos")] + for receipts in [ + Path::new("/var/db/receipts"), + Path::new("/Library/Apple/System/Library/Receipts"), + ] { + hash_extension_metadata(receipts, "plist", &mut hasher)?; + } + if let Some(home) = environment + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case("HOME")) + .map(|(_, value)| PathBuf::from(value)) + .or_else(dirs::home_dir) + { + hash_metadata_tree(&home.join(".node-gyp"), &mut hasher)?; + hash_rustup_metadata(&home.join(".rustup"), &mut hasher)?; + } + Ok(format!("sha256-{}", hex::encode(hasher.finalize()))) +} + +fn hash_metadata_tree(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hasher.update(b"metadata-tree-v1\0"); + hash_os_path(hasher, root); + let root_metadata = match std::fs::symlink_metadata(root) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hasher.update(b"absent\x1e"); + return Ok(()); + } + Err(error) => return Err(error), + }; + hash_metadata_record(hasher, &root_metadata); + if !root_metadata.is_dir() || root_metadata.file_type().is_symlink() { + return Ok(()); + } + let mut pending = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + let mut entries = std::fs::read_dir(&directory)?.collect::, _>>()?; + entries.sort_unstable_by_key(std::fs::DirEntry::file_name); + for entry in entries { + let path = entry.path(); + hash_os_path(hasher, path.strip_prefix(root).unwrap_or(&path)); + let metadata = std::fs::symlink_metadata(&path)?; + hash_metadata_record(hasher, &metadata); + if metadata.file_type().is_symlink() { + hash_os_path(hasher, &std::fs::read_link(path)?); + } else if metadata.is_dir() { + pending.push(path); + } + hasher.update(b"\x1e"); + } + } + Ok(()) +} + +fn hash_path_metadata(path: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hash_os_path(hasher, path); + match std::fs::symlink_metadata(path) { + Ok(metadata) => { + hash_metadata_record(hasher, &metadata); + if metadata.file_type().is_symlink() { + hash_os_path(hasher, &std::fs::read_link(path)?); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => hasher.update(b"absent"), + Err(error) => return Err(error), + } + hasher.update(b"\x1e"); + Ok(()) +} + +fn hash_metadata_record(hasher: &mut Sha256, metadata: &std::fs::Metadata) { + let file_type = metadata.file_type(); + hasher.update(if file_type.is_dir() { + b"d".as_slice() + } else if file_type.is_file() { + b"f".as_slice() + } else if file_type.is_symlink() { + b"l".as_slice() + } else { + b"s".as_slice() + }); + hasher.update(metadata.len().to_le_bytes()); + if let Ok(modified) = metadata.modified() + && let Ok(duration) = modified.duration_since(UNIX_EPOCH) + { + hasher.update(duration.as_secs().to_le_bytes()); + hasher.update(duration.subsec_nanos().to_le_bytes()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + hasher.update(metadata.dev().to_le_bytes()); + hasher.update(metadata.ino().to_le_bytes()); + hasher.update(metadata.mode().to_le_bytes()); + hasher.update(metadata.ctime().to_le_bytes()); + hasher.update(metadata.ctime_nsec().to_le_bytes()); + } +} + +fn hash_homebrew_metadata(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hash_os_path(hasher, root); + let mut formulae = match std::fs::read_dir(root) { + Ok(entries) => entries.collect::, _>>()?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hasher.update(b"absent\x1e"); + return Ok(()); + } + Err(error) => return Err(error), + }; + formulae.sort_unstable_by_key(std::fs::DirEntry::file_name); + for formula in formulae { + if !formula.file_type()?.is_dir() { + continue; + } + hash_os_path(hasher, Path::new(&formula.file_name())); + let mut versions = std::fs::read_dir(formula.path())?.collect::, _>>()?; + versions.sort_unstable_by_key(std::fs::DirEntry::file_name); + for version in versions { + if version.file_type()?.is_dir() { + hash_os_path(hasher, Path::new(&version.file_name())); + hash_path_metadata(&version.path().join("INSTALL_RECEIPT.json"), hasher)?; + } + } + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn hash_extension_metadata( + root: &Path, + extension: &str, + hasher: &mut Sha256, +) -> std::io::Result<()> { + hash_os_path(hasher, root); + let mut paths = match std::fs::read_dir(root) { + Ok(entries) => entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|value| value == extension)) + .collect::>(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hasher.update(b"absent\x1e"); + return Ok(()); + } + Err(error) => return Err(error), + }; + paths.sort_unstable(); + for path in paths { + hash_path_metadata(&path, hasher)?; + } + Ok(()) +} + +fn hash_rustup_metadata(root: &Path, hasher: &mut Sha256) -> std::io::Result<()> { + hash_path_metadata(&root.join("settings.toml"), hasher)?; + let toolchains = root.join("toolchains"); + let mut entries = match std::fs::read_dir(&toolchains) { + Ok(entries) => entries.collect::, _>>()?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + entries.sort_unstable_by_key(std::fs::DirEntry::file_name); + for entry in entries { + if !entry.file_type()?.is_dir() { + continue; + } + hash_os_path(hasher, Path::new(&entry.file_name())); + for executable in ["rustc", "cargo"] { + hash_path_metadata(&entry.path().join("bin").join(executable), hasher)?; + } + } + Ok(()) +} + +fn is_lower_hex_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + +fn is_sha256_fingerprint(value: &str) -> bool { + value + .strip_prefix("sha256-") + .is_some_and(is_lower_hex_digest) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + fn environment(temp: &tempfile::TempDir) -> HashMap { + HashMap::from([( + "HOME".to_string(), + temp.path().join("home").display().to_string(), + )]) + } + + #[test] + fn unchanged_tracked_inputs_reuse_persisted_fingerprint() { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let tracked = temp.path().join("pkgconfig"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&tracked).unwrap(); + std::fs::write(tracked.join("native.pc"), b"Version: 1\n").unwrap(); + let environment = environment(&temp); + let cache_dir = temp.path().join("cache"); + let first_cache = ToolchainFingerprintCache::for_test(cache_dir.clone()); + let base_key = format!("sha256-{}", "b".repeat(64)); + let computations = Cell::new(0_u32); + let mut compute = || { + computations.set(computations.get() + 1); + Ok(ComputedToolchainFingerprint { + fingerprint: format!("sha256-{}", "1".repeat(64)), + pkg_config_paths: vec![tracked.clone()], + }) + }; + + let first = cached_toolchain_fingerprint( + &first_cache, + &base_key, + &environment, + &project, + &mut compute, + ) + .unwrap(); + let second_cache = ToolchainFingerprintCache::for_test(cache_dir); + let second = cached_toolchain_fingerprint( + &second_cache, + &base_key, + &environment, + &project, + &mut compute, + ) + .unwrap(); + + assert_eq!(first, second); + assert_eq!(computations.get(), 1); + } + + #[test] + fn changed_tracked_input_recomputes_persisted_fingerprint() { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let tracked = temp.path().join("pkgconfig"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&tracked).unwrap(); + let tracked_file = tracked.join("native.pc"); + std::fs::write(&tracked_file, b"Version: 1\n").unwrap(); + let environment = environment(&temp); + let cache = ToolchainFingerprintCache::for_test(temp.path().join("cache")); + let base_key = format!("sha256-{}", "b".repeat(64)); + let computations = Cell::new(0_u32); + let mut compute = || { + let count = computations.get() + 1; + computations.set(count); + Ok(ComputedToolchainFingerprint { + fingerprint: format!("sha256-{}", count.to_string().repeat(64)), + pkg_config_paths: vec![tracked.clone()], + }) + }; + + let first = + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + std::fs::write(&tracked_file, b"Version: 2 with a different size\n").unwrap(); + let second = + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + + assert_ne!(first, second); + assert_eq!(computations.get(), 2); + } + + #[test] + fn corrupt_snapshot_is_ignored_and_replaced() { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + std::fs::create_dir(&project).unwrap(); + let environment = environment(&temp); + let cache = ToolchainFingerprintCache::for_test(temp.path().join("cache")); + let base_key = format!("sha256-{}", "b".repeat(64)); + let entry = cache.entry_path(&base_key).unwrap(); + std::fs::create_dir_all(entry.parent().unwrap()).unwrap(); + std::fs::write(&entry, b"{not valid json").unwrap(); + let computations = Cell::new(0_u32); + let mut compute = || { + computations.set(computations.get() + 1); + Ok(ComputedToolchainFingerprint { + fingerprint: format!("sha256-{}", "1".repeat(64)), + pkg_config_paths: Vec::new(), + }) + }; + + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + + assert_eq!(computations.get(), 1); + assert!(read_snapshot(&entry, &base_key).is_some()); + } + + #[test] + fn expired_snapshot_recomputes_fingerprint() { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + std::fs::create_dir(&project).unwrap(); + let environment = environment(&temp); + let cache = ToolchainFingerprintCache::for_test(temp.path().join("cache")); + let base_key = format!("sha256-{}", "b".repeat(64)); + let computations = Cell::new(0_u32); + let mut compute = || { + let count = computations.get() + 1; + computations.set(count); + Ok(ComputedToolchainFingerprint { + fingerprint: format!("sha256-{}", count.to_string().repeat(64)), + pkg_config_paths: Vec::new(), + }) + }; + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + let entry = cache.entry_path(&base_key).unwrap(); + let mut snapshot = read_snapshot(&entry, &base_key).unwrap(); + snapshot.created_unix_seconds = 0; + snapshot.entry_hash = entry_hash(&snapshot); + write_snapshot(&entry, &snapshot).unwrap(); + + cached_toolchain_fingerprint(&cache, &base_key, &environment, &project, &mut compute) + .unwrap(); + + assert_eq!(computations.get(), 2); + } + + #[test] + fn snapshot_rejects_unconfigured_broad_scan_root() { + let snapshot = ToolchainFingerprintSnapshot { + schema: SCHEMA, + base_key: format!("sha256-{}", "b".repeat(64)), + fingerprint: format!("sha256-{}", "1".repeat(64)), + validation_hash: format!("sha256-{}", "2".repeat(64)), + pkg_config_paths: vec![PathBuf::from("/")], + created_unix_seconds: 1, + entry_hash: String::new(), + }; + + assert!(!snapshot_paths_are_allowed( + &snapshot, + &HashMap::new(), + Path::new("/project") + )); + } +} diff --git a/crates/lpm-cli/src/commands/rebuild/mod.rs b/crates/lpm-cli/src/commands/rebuild/mod.rs index 03252d65..f8dbc3c4 100644 --- a/crates/lpm-cli/src/commands/rebuild/mod.rs +++ b/crates/lpm-cli/src/commands/rebuild/mod.rs @@ -29,6 +29,7 @@ //! entire group (not just the direct child), preventing orphaned subprocesses //! - On Windows: `Child::kill()` terminates the process tree via `TerminateProcess` +mod build_cache; mod hints; mod package_dir; mod process_tree; @@ -40,6 +41,10 @@ mod trust; #[cfg(test)] mod tests; +use self::build_cache::{ + BuildCacheInvocation, BuildCacheScratch, build_key_for_package, is_cacheable_native_build, + marker_requires_key_validation, read_build_marker_key, read_v2_graph_key_digest, +}; #[cfg(test)] pub(crate) use self::hints::scriptable_package_rows; pub use self::hints::{all_scripted_packages_trusted, show_install_build_hint}; @@ -47,7 +52,7 @@ use self::package_dir::prepare_live_package_dir; use self::sandbox_env::build_sanitized_env; use self::script_execution::execute_script; use self::scripts::{ - BUILD_MARKER, ScriptablePackage, count_untrusted_unbuilt, package_baseline_dir_indexed, + BUILD_MARKER, BuildCacheMetrics, ScriptablePackage, count_untrusted_unbuilt, read_lifecycle_scripts, rebuild_dry_run_envelope, rebuild_package_failure_message, rebuild_package_label, rebuild_summary_envelope, scripts_word, toposort_packages, warn_stale_trusted_deps, widen_to_build_by_policy, @@ -61,13 +66,17 @@ use lpm_common::LpmError; use lpm_common::color::Painted; use lpm_sandbox::SandboxMode; use lpm_security::{EXECUTED_INSTALL_PHASES, SecurityPolicy}; -use lpm_store::V2BaselineIndex; +use lpm_store::{PackageBaselineLayout, V2BaselineIndex}; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::time::Duration; const DEFAULT_SCRIPT_TIMEOUT_SECS: u64 = 300; +fn elapsed_millis(duration: Duration) -> u64 { + duration.as_millis().min(u128::from(u64::MAX)) as u64 +} + #[allow(clippy::too_many_arguments)] pub async fn run( project_dir: &Path, @@ -287,11 +296,22 @@ async fn run_under_store_lock( // installs); silent skip preserves the previously // `pkg_json_path.exists()` semantic for non-store sources // while fixing the v2-installed-and-skipped data-loss bug. - let pkg_dir = - match package_baseline_dir_indexed(&baseline_index, &lpm_root, &lp.name, &lp.version) { - Some(p) => p, - None => continue, - }; + let baseline = match lpm_store::find_installed_package_baseline_indexed( + &baseline_index, + &lpm_root, + &lp.name, + &lp.version, + ) { + Some(baseline) => baseline, + None => continue, + }; + let graph_key_digest = if baseline.layout == PackageBaselineLayout::V2 { + read_v2_graph_key_digest(&baseline.package_dir) + } else { + None + }; + let pkg_dir = baseline.package_dir; + let pristine_path = baseline.pristine_dir; let pkg_json_path = pkg_dir.join("package.json"); if !pkg_json_path.exists() { @@ -303,7 +323,11 @@ async fn run_under_store_lock( _ => continue, }; - let is_built = pkg_dir.join(BUILD_MARKER).exists(); + let marker_path = pkg_dir.join(BUILD_MARKER); + let is_built = marker_path.exists(); + let build_marker_key = is_built + .then(|| read_build_marker_key(&marker_path)) + .flatten(); // trust decision // now flows through the shared [`evaluate_trust`] helper so @@ -373,8 +397,12 @@ async fn run_under_store_lock( integrity: lp.integrity.clone(), wrapper_id, store_path: pkg_dir, + pristine_path, + source_integrity: baseline.integrity, + graph_key_digest, scripts, is_built, + build_marker_key, is_trusted, trust_reason, }); @@ -407,7 +435,7 @@ async fn run_under_store_lock( let result = if dry_run { rebuild_dry_run_envelope(&[], force_security_floor) } else { - rebuild_summary_envelope(0, 0, force_security_floor) + rebuild_summary_envelope(0, 0, force_security_floor, &BuildCacheMetrics::default()) }; println!("{}", serde_json::to_string_pretty(&result).unwrap()); } else { @@ -470,7 +498,7 @@ async fn run_under_store_lock( } else { selected_for_policy .into_iter() - .filter(|p| !p.is_built) + .filter(|p| !p.is_built || (!dry_run && marker_requires_key_validation(p))) .collect() }; @@ -482,7 +510,7 @@ async fn run_under_store_lock( let result = if dry_run { rebuild_dry_run_envelope(&to_build, force_security_floor) } else { - rebuild_summary_envelope(0, 0, force_security_floor) + rebuild_summary_envelope(0, 0, force_security_floor, &BuildCacheMetrics::default()) }; println!("{}", serde_json::to_string_pretty(&result).unwrap()); } else { @@ -706,6 +734,7 @@ async fn run_under_store_lock( let mut successes = 0usize; let mut failures = 0usize; let mut completed_scripts = 0usize; + let mut build_cache_metrics = BuildCacheMetrics::default(); let mut built_packages = Vec::with_capacity(to_build.len()); let package_label_width = to_build .iter() @@ -902,6 +931,8 @@ async fn run_under_store_lock( lpm_sandbox::prepare_writable_dirs(&prepare_spec) .map_err(|e| LpmError::Registry(format!("{e}")))?; + let mut effective_sandbox_posture = lpm_sandbox::SandboxPosture::Disabled; + // `sandbox_options` already carries `allow-degraded` and // `deny_outbound_network` from the resolver call above. Reusing // it here preserves the resolved mode for both the pre-probe and @@ -941,6 +972,7 @@ async fn run_under_store_lock( sandbox_options.clone(), ) .map_err(|e| LpmError::Registry(format!("sandbox unavailable: {e}")))?; + effective_sandbox_posture = probe_sandbox.posture(); // per-install warning: emitted once when the // probe's effective posture is `Degraded`. The structured // line names kernel + active ABI + missing dimension so log @@ -973,6 +1005,20 @@ async fn run_under_store_lock( drop(probe_sandbox); } + let cache_preparation_start = std::time::Instant::now(); + let build_cache_invocation = BuildCacheInvocation::prepare( + &lockfile, + &sanitized_env, + project_dir, + sandbox_mode, + &effective_sandbox_posture, + &sandbox_options, + &extra_write_dirs, + &extra_secret_read_allow, + &requested_capabilities, + ); + build_cache_metrics.preparation_ms = elapsed_millis(cache_preparation_start.elapsed()); + // Banners fire AFTER the probe succeeds. On Linux + LogOnly the // probe above bailed with ModeNotSupportedOnPlatform, so this // banner's "logged but NOT enforced" promise never reaches a @@ -1038,6 +1084,60 @@ async fn run_under_store_lock( for pkg in &to_build { let mut pkg_success = true; + let key_start = std::time::Instant::now(); + let mut build_key = build_cache_invocation + .as_ref() + .and_then(|invocation| build_key_for_package(invocation, pkg, project_dir)); + build_cache_metrics.key_ms += elapsed_millis(key_start.elapsed()); + if build_key.is_some() { + build_cache_metrics.eligible += 1; + } else if is_cacheable_native_build(pkg) { + build_cache_metrics.bypassed += 1; + } + let v2_store = lpm_store::v2::Store::from_lpm_root(&lpm_root); + let _build_entry_lock = if let Some(graph_key_digest) = pkg.graph_key_digest.as_deref() { + match v2_store + .paths() + .build_entry_lock_path(graph_key_digest) + .and_then(lpm_common::acquire_exclusive_lock) + { + Ok(lock) => Some(lock), + Err(error) => { + if !json_output { + let label = rebuild_package_label(pkg); + install_ui::detail(&format!( + " {} {label: Some(lock), + Err(error) => { + tracing::warn!( + "build cache bypassed for {}@{} because the per-key lock failed: {error}", + pkg.name, + pkg.version + ); + build_cache_metrics.bypassed += 1; + build_key = None; + None + } + } + } else { + None + }; + // fix: lifecycle scripts must run from the LIVE // per-package directory (where the symlinked sibling // node_modules/ exists), not the global content-addressable @@ -1079,6 +1179,156 @@ async fn run_under_store_lock( } }; + let marker_path = pkg.store_path.join(BUILD_MARKER); + let current_marker_exists = marker_path.is_file(); + let current_marker_key = current_marker_exists + .then(|| read_build_marker_key(&marker_path)) + .flatten(); + if force { + if let Err(error) = std::fs::remove_file(&marker_path) + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!( + "failed to clear build marker for forced rebuild of {}: {error}", + pkg.name + ); + } + } else if current_marker_exists { + match (current_marker_key.as_deref(), build_key.as_ref()) { + (Some(marker_key), Some(key)) if marker_key == key.as_str() => { + build_cache_metrics.local_state_hits += 1; + continue; + } + (_, None) => continue, + _ => { + if let Err(error) = std::fs::remove_file(&marker_path) + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!( + "failed to clear stale build marker for {}: {error}", + pkg.name + ); + } + } + } + } + if !force && let Some(key) = build_key.as_ref() { + let store = lpm_store::v2::Store::from_lpm_root(&lpm_root); + let lookup_start = std::time::Instant::now(); + match store.reusable_build_artifact(key) { + Ok(Some(artifact)) => { + build_cache_metrics.lookup_ms += elapsed_millis(lookup_start.elapsed()); + let restore_start = std::time::Instant::now(); + match store.restore_build_artifact(&artifact, &pkg.store_path) { + Ok(()) => { + build_cache_metrics.restore_ms += + elapsed_millis(restore_start.elapsed()); + build_cache_metrics.hits += 1; + build_cache_metrics.scripts_avoided += pkg.scripts.len(); + build_cache_metrics.restored_bytes += artifact.manifest.unpacked_bytes; + build_cache_metrics.lifecycle_ms_avoided += + artifact.manifest.lifecycle_duration_ms; + if let Err(error) = std::fs::write( + pkg.store_path.join(BUILD_MARKER), + key.as_str().as_bytes(), + ) { + tracing::warn!( + "failed to write restored build marker for {}: {error}", + pkg.name + ); + } + if !json_output { + let label = rebuild_package_label(pkg); + install_ui::detail(&format!( + " {} {label: { + build_cache_metrics.restore_ms += + elapsed_millis(restore_start.elapsed()); + build_cache_metrics.misses += 1; + tracing::warn!( + "failed to restore build cache entry for {}@{}: {error}", + pkg.name, + pkg.version + ); + } + } + } + Ok(None) => { + build_cache_metrics.lookup_ms += elapsed_millis(lookup_start.elapsed()); + build_cache_metrics.misses += 1; + } + Err(error) => { + build_cache_metrics.lookup_ms += elapsed_millis(lookup_start.elapsed()); + build_cache_metrics.misses += 1; + tracing::warn!( + "failed to inspect build cache entry for {}@{}: {error}", + pkg.name, + pkg.version + ); + } + } + } + + if build_key.is_some() { + let store = lpm_store::v2::Store::from_lpm_root(&lpm_root); + let rematerialize_start = std::time::Instant::now(); + if let Err(error) = store.restore_pristine_package(&pkg.pristine_path, &pkg.store_path) + { + build_cache_metrics.rematerialize_ms += + elapsed_millis(rematerialize_start.elapsed()); + if !json_output { + let label = rebuild_package_label(pkg); + install_ui::detail(&format!( + " {} {label: Some(scratch), + Err(error) => { + tracing::warn!( + "build cache bypassed for {}@{} because isolated scratch setup failed: {error}", + pkg.name, + pkg.version + ); + build_key = None; + None + } + } + } else { + None + }; + let script_tmpdir = build_cache_scratch + .as_ref() + .map_or(tmpdir.as_path(), BuildCacheScratch::path); + let mut package_sandbox_options = sandbox_options.clone(); + package_sandbox_options.build_cache_isolation = build_key.is_some(); + + let lifecycle_start = std::time::Instant::now(); for phase in EXECUTED_INSTALL_PHASES { let cmd = match pkg.scripts.get(*phase) { Some(c) => c, @@ -1094,12 +1344,12 @@ async fn run_under_store_lock( &sanitized_env, &timeout, sandbox_mode, - &sandbox_options, + &package_sandbox_options, &extra_write_dirs, &extra_secret_read_allow, &store_root, &home_dir, - &tmpdir, + script_tmpdir, ) { Ok(()) => { if !json_output { @@ -1128,9 +1378,29 @@ async fn run_under_store_lock( } if pkg_success { - // Write .lpm-built marker + if let Some(key) = build_key.as_ref() { + let store = lpm_store::v2::Store::from_lpm_root(&lpm_root); + let publish_start = std::time::Instant::now(); + if let Err(error) = store.publish_build_artifact( + key, + &pkg.source_integrity, + &pkg.store_path, + elapsed_millis(lifecycle_start.elapsed()), + force, + ) { + tracing::warn!( + "failed to publish build cache entry for {}@{}: {error}", + pkg.name, + pkg.version + ); + } + build_cache_metrics.publish_ms += elapsed_millis(publish_start.elapsed()); + } let marker_path = pkg.store_path.join(BUILD_MARKER); - if let Err(e) = std::fs::write(&marker_path, "") { + let marker = build_key + .as_ref() + .map_or(&[][..], |key| key.as_str().as_bytes()); + if let Err(e) = std::fs::write(&marker_path, marker) { tracing::warn!("failed to write build marker for {}: {e}", pkg.name); } successes += 1; @@ -1142,7 +1412,12 @@ async fn run_under_store_lock( // Summary if json_output { - let json = rebuild_summary_envelope(successes, failures, force_security_floor); + let json = rebuild_summary_envelope( + successes, + failures, + force_security_floor, + &build_cache_metrics, + ); println!("{}", serde_json::to_string_pretty(&json).unwrap()); } else if failures == 0 { eprintln!(); diff --git a/crates/lpm-cli/src/commands/rebuild/script_execution.rs b/crates/lpm-cli/src/commands/rebuild/script_execution.rs index f617c0be..e2902dfa 100644 --- a/crates/lpm-cli/src/commands/rebuild/script_execution.rs +++ b/crates/lpm-cli/src/commands/rebuild/script_execution.rs @@ -51,15 +51,7 @@ pub(super) fn execute_script( // worked accidentally because `std::process::Command::env` on // Windows does case-insensitive deduplication and our explicit // `"PATH"` overrides won, but the LOOKUP path was still broken. - let parent_path = find_env_case_insensitive(env, "PATH"); - let path_value = build_lifecycle_path(project_dir, parent_path); - let mut envs: Vec<(String, String)> = env - .iter() - .filter(|(k, _)| !k.eq_ignore_ascii_case("PATH") && !k.eq_ignore_ascii_case("INIT_CWD")) - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - envs.push(("INIT_CWD".to_string(), project_dir.display().to_string())); - envs.push(("PATH".to_string(), path_value)); + let envs = build_lifecycle_environment(env, project_dir, tmpdir); let start = std::time::Instant::now(); @@ -97,6 +89,33 @@ pub(super) fn execute_script( } } +pub(super) fn build_lifecycle_environment( + env: &HashMap, + project_dir: &Path, + tmpdir: &Path, +) -> Vec<(String, String)> { + let parent_path = find_env_case_insensitive(env, "PATH"); + let path_value = build_lifecycle_path(project_dir, parent_path); + let mut envs: Vec<(String, String)> = env + .iter() + .filter(|(k, _)| { + !k.eq_ignore_ascii_case("PATH") + && !k.eq_ignore_ascii_case("INIT_CWD") + && !k.eq_ignore_ascii_case("TMPDIR") + && !k.eq_ignore_ascii_case("TMP") + && !k.eq_ignore_ascii_case("TEMP") + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + envs.push(("INIT_CWD".to_string(), project_dir.display().to_string())); + envs.push(("PATH".to_string(), path_value)); + let tmp = tmpdir.display().to_string(); + envs.push(("TMPDIR".to_string(), tmp.clone())); + envs.push(("TMP".to_string(), tmp.clone())); + envs.push(("TEMP".to_string(), tmp)); + envs +} + /// — resolve the live per-package directory where lifecycle /// scripts should `current_dir` to. /// diff --git a/crates/lpm-cli/src/commands/rebuild/scripts.rs b/crates/lpm-cli/src/commands/rebuild/scripts.rs index a4cd426b..0210bec9 100644 --- a/crates/lpm-cli/src/commands/rebuild/scripts.rs +++ b/crates/lpm-cli/src/commands/rebuild/scripts.rs @@ -65,8 +65,12 @@ pub(super) struct ScriptablePackage { /// Now the lookup is correct for every source kind. pub(super) wrapper_id: Option, pub(super) store_path: std::path::PathBuf, + pub(super) pristine_path: std::path::PathBuf, + pub(super) source_integrity: String, + pub(super) graph_key_digest: Option, pub(super) scripts: HashMap, pub(super) is_built: bool, + pub(super) build_marker_key: Option, pub(super) is_trusted: bool, /// the specific basis on which /// `is_trusted` was decided. Preserved so the dry-run output and @@ -79,6 +83,47 @@ pub(super) struct ScriptablePackage { pub(super) trust_reason: TrustReason, } +#[derive(Debug, Clone, Default)] +pub(super) struct BuildCacheMetrics { + pub(super) eligible: usize, + pub(super) hits: usize, + pub(super) misses: usize, + pub(super) bypassed: usize, + pub(super) local_state_hits: usize, + pub(super) scripts_avoided: usize, + pub(super) restored_bytes: u64, + pub(super) lifecycle_ms_avoided: u64, + pub(super) preparation_ms: u64, + pub(super) key_ms: u64, + pub(super) lookup_ms: u64, + pub(super) restore_ms: u64, + pub(super) rematerialize_ms: u64, + pub(super) publish_ms: u64, +} + +impl BuildCacheMetrics { + fn to_json(&self) -> serde_json::Value { + serde_json::json!({ + "eligible": self.eligible, + "hits": self.hits, + "misses": self.misses, + "bypassed": self.bypassed, + "local_state_hits": self.local_state_hits, + "scripts_avoided": self.scripts_avoided, + "restored_bytes": self.restored_bytes, + "lifecycle_ms_avoided": self.lifecycle_ms_avoided, + "timings_ms": { + "preparation": self.preparation_ms, + "key": self.key_ms, + "lookup": self.lookup_ms, + "restore": self.restore_ms, + "rematerialize": self.rematerialize_ms, + "publish": self.publish_ms, + } + }) + } +} + pub(crate) type RebuildPackageIdentity = (String, String, Option); #[derive(Debug, Clone, Default)] @@ -120,11 +165,13 @@ pub(super) fn rebuild_summary_envelope( successes: usize, failures: usize, force_security_floor: bool, + build_cache: &BuildCacheMetrics, ) -> serde_json::Value { let mut json = serde_json::json!({ "success": failures == 0, "built": successes, "failed": failures, + "build_cache": build_cache.to_json(), }); crate::security_floor::attach_security_posture(&mut json, force_security_floor); json diff --git a/crates/lpm-cli/src/commands/rebuild/tests/mod.rs b/crates/lpm-cli/src/commands/rebuild/tests/mod.rs index d103b302..8e1d551a 100644 --- a/crates/lpm-cli/src/commands/rebuild/tests/mod.rs +++ b/crates/lpm-cli/src/commands/rebuild/tests/mod.rs @@ -646,8 +646,12 @@ fn toposort_respects_dependency_order() { integrity: None, wrapper_id: None, store_path: PathBuf::new(), + pristine_path: PathBuf::new(), + source_integrity: "sha512-unused".into(), + graph_key_digest: None, scripts: HashMap::new(), is_built: false, + build_marker_key: None, is_trusted: true, trust_reason: TrustReason::StrictBinding, }, @@ -657,8 +661,12 @@ fn toposort_respects_dependency_order() { integrity: None, wrapper_id: None, store_path: PathBuf::new(), + pristine_path: PathBuf::new(), + source_integrity: "sha512-unused".into(), + graph_key_digest: None, scripts: HashMap::new(), is_built: false, + build_marker_key: None, is_trusted: true, trust_reason: TrustReason::StrictBinding, }, @@ -1102,8 +1110,12 @@ fn stale_detection_finds_packages_without_scripts() { integrity: None, wrapper_id: None, store_path: std::path::PathBuf::new(), + pristine_path: std::path::PathBuf::new(), + source_integrity: "sha512-unused".into(), + graph_key_digest: None, scripts: HashMap::from([("postinstall".into(), "node setup".into())]), is_built: false, + build_marker_key: None, is_trusted: true, trust_reason: TrustReason::StrictBinding, }, @@ -1113,8 +1125,12 @@ fn stale_detection_finds_packages_without_scripts() { integrity: None, wrapper_id: None, store_path: std::path::PathBuf::new(), + pristine_path: std::path::PathBuf::new(), + source_integrity: "sha512-unused".into(), + graph_key_digest: None, scripts: HashMap::from([("postinstall".into(), "node install.js".into())]), is_built: false, + build_marker_key: None, is_trusted: true, trust_reason: TrustReason::StrictBinding, }, @@ -1383,8 +1399,12 @@ fn synthetic_scriptable(name: &str, is_built: bool, is_trusted: bool) -> Scripta integrity: None, wrapper_id: None, store_path: std::path::PathBuf::from("/unused"), + pristine_path: std::path::PathBuf::from("/unused"), + source_integrity: "sha512-unused".into(), + graph_key_digest: None, scripts: HashMap::from([("postinstall".into(), "node x.js".into())]), is_built, + build_marker_key: None, is_trusted, trust_reason: if is_trusted { TrustReason::StrictBinding diff --git a/crates/lpm-cli/src/sandbox_config.rs b/crates/lpm-cli/src/sandbox_config.rs index 6c6974b9..a32dfc18 100644 --- a/crates/lpm-cli/src/sandbox_config.rs +++ b/crates/lpm-cli/src/sandbox_config.rs @@ -469,6 +469,7 @@ fn merge(project: RawSandboxKeys, global: RawSandboxKeys) -> (SandboxOptions, Re .or(global.allow_degraded) .unwrap_or(false), deny_outbound_network, + build_cache_isolation: false, }; (options, resolved) } diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index 05f56091..bd70bf5e 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -25,9 +25,9 @@ pub use integrity::Integrity; pub use package_name::PackageName; pub use paths::{ ExclusiveLockHandle, FsKind, GLOBAL_INSTALL_PATH_BUDGET, INSTALL_READY_MARKER, LpmRoot, - SharedLockHandle, as_extended_path, check_install_path_budget, is_local_fs, - project_install_lock, try_with_exclusive_lock, with_exclusive_lock, with_exclusive_lock_async, - with_shared_lock, with_shared_lock_async, + SharedLockHandle, acquire_exclusive_lock, as_extended_path, check_install_path_budget, + is_local_fs, project_install_lock, try_with_exclusive_lock, with_exclusive_lock, + with_exclusive_lock_async, with_shared_lock, with_shared_lock_async, }; pub use provenance::{ProvenanceSnapshot, ProvenanceStatus}; pub use symlink::{create_dir_symlink_or_junction, create_symlink}; diff --git a/crates/lpm-common/src/paths.rs b/crates/lpm-common/src/paths.rs index 9f04e8a0..1c73495b 100644 --- a/crates/lpm-common/src/paths.rs +++ b/crates/lpm-common/src/paths.rs @@ -712,6 +712,17 @@ where body() } +/// Acquire and return an exclusive advisory lock handle. +/// +/// The caller controls the critical-section lifetime by retaining the returned +/// handle. This is useful when existing control flow contains early returns or +/// loop `continue` paths that would make a closure-scoped lock cumbersome. +pub fn acquire_exclusive_lock( + lock_path: impl AsRef, +) -> Result { + acquire_exclusive_with_hint(lock_path.as_ref(), default_wait_hint) +} + /// Async variant of [`with_exclusive_lock`]. The lock acquisition runs on /// a `tokio::task::spawn_blocking` worker so a contended lock doesn't /// block the tokio reactor. The acquired handle is held across `body`'s diff --git a/crates/lpm-sandbox/src/landlock_rules.rs b/crates/lpm-sandbox/src/landlock_rules.rs index e1d6eba7..8a2fe19c 100644 --- a/crates/lpm-sandbox/src/landlock_rules.rs +++ b/crates/lpm-sandbox/src/landlock_rules.rs @@ -94,7 +94,15 @@ pub(crate) const SYSTEM_READ_PATHS: &[&str] = &[ /// 1. Testable without a Linux kernel (runs on macOS host). /// 2. Free of unsafe `PathFd` opens at rule-description time (those /// happen in the child's pre_exec hook in [`crate::linux`]). +#[cfg(test)] pub(crate) fn describe_rules(spec: &SandboxSpec) -> Vec<(PathBuf, RuleAccess)> { + describe_rules_with_isolation(spec, false) +} + +pub(crate) fn describe_rules_with_isolation( + spec: &SandboxSpec, + build_cache_isolation: bool, +) -> Vec<(PathBuf, RuleAccess)> { let mut rules = Vec::with_capacity(32 + spec.extra_write_dirs.len()); // Read-only system baseline. @@ -115,7 +123,11 @@ pub(crate) fn describe_rules(spec: &SandboxSpec) -> Vec<(PathBuf, RuleAccess)> { // widening to `store_root` would expose every other package // the user has installed. Two remediations exist for that corner: // switch to hardlink, or widen reads with an explicit store rule. - rules.push((spec.project_dir.clone(), RuleAccess::Read)); + if build_cache_isolation { + rules.push((package_dependency_root(spec), RuleAccess::Read)); + } else { + rules.push((spec.project_dir.clone(), RuleAccess::Read)); + } // NVM-installed toolchain. Only added if the host has a matching // dir — [`crate::linux::spawn`] filters missing paths at FD-open // time; the description layer stays complete. @@ -143,13 +155,17 @@ pub(crate) fn describe_rules(spec: &SandboxSpec) -> Vec<(PathBuf, RuleAccess)> { // would require either a per-tool allowlist or migrating those // tools to LPM-controlled cache directories. rules.push((spec.package_dir.clone(), RuleAccess::ReadWrite)); - rules.push((spec.project_dir.join("node_modules"), RuleAccess::ReadWrite)); - rules.push((spec.project_dir.join(".husky"), RuleAccess::ReadWrite)); - rules.push((spec.project_dir.join(".lpm"), RuleAccess::ReadWrite)); - rules.push((spec.home_dir.join(".cache"), RuleAccess::ReadWrite)); - rules.push((spec.home_dir.join(".node-gyp"), RuleAccess::ReadWrite)); - rules.push((spec.home_dir.join(".npm"), RuleAccess::ReadWrite)); - rules.push((PathBuf::from("/tmp"), RuleAccess::ReadWrite)); + if build_cache_isolation { + rules.push((spec.home_dir.join(".node-gyp"), RuleAccess::Read)); + } else { + rules.push((spec.project_dir.join("node_modules"), RuleAccess::ReadWrite)); + rules.push((spec.project_dir.join(".husky"), RuleAccess::ReadWrite)); + rules.push((spec.project_dir.join(".lpm"), RuleAccess::ReadWrite)); + rules.push((spec.home_dir.join(".cache"), RuleAccess::ReadWrite)); + rules.push((spec.home_dir.join(".node-gyp"), RuleAccess::ReadWrite)); + rules.push((spec.home_dir.join(".npm"), RuleAccess::ReadWrite)); + rules.push((PathBuf::from("/tmp"), RuleAccess::ReadWrite)); + } rules.push((spec.tmpdir.clone(), RuleAccess::ReadWrite)); // `/dev/null` and `/dev/tty` as writable — shells redirect to // them constantly. The broader `/dev` Read rule already covers @@ -161,13 +177,26 @@ pub(crate) fn describe_rules(spec: &SandboxSpec) -> Vec<(PathBuf, RuleAccess)> { // Per-project extras from `package.json > lpm > scripts > // sandboxWriteDirs`. Loader guarantees absolute paths. - for p in &spec.extra_write_dirs { - rules.push((p.clone(), RuleAccess::ReadWrite)); + if !build_cache_isolation { + for p in &spec.extra_write_dirs { + rules.push((p.clone(), RuleAccess::ReadWrite)); + } } rules } +fn package_dependency_root(spec: &SandboxSpec) -> PathBuf { + let mut root = spec.package_dir.as_path(); + while let Some(parent) = root.parent() { + if root.file_name().is_some_and(|name| name == "node_modules") { + return root.to_path_buf(); + } + root = parent; + } + spec.package_dir.clone() +} + #[cfg(test)] mod tests { use super::*; @@ -347,6 +376,29 @@ mod tests { assert_eq!(last.0.as_os_str(), "/home/u/.cache/ms-playwright"); } + #[test] + fn build_cache_isolation_keeps_persistent_writes_package_local() { + let s = spec(); + let rules = describe_rules_with_isolation(&s, true); + + assert!(contains_rule( + &rules, + "/lpm-store/prisma@5.22.0", + RuleAccess::ReadWrite + )); + assert!(contains_rule(&rules, "/tmp", RuleAccess::ReadWrite)); + assert!(!contains_rule( + &rules, + "/home/u/proj/node_modules", + RuleAccess::ReadWrite + )); + assert!(!contains_rule( + &rules, + "/home/u/.node-gyp", + RuleAccess::ReadWrite + )); + } + #[test] fn tmpdir_distinct_from_slash_tmp_gets_its_own_rule() { let mut s = spec(); diff --git a/crates/lpm-sandbox/src/lib.rs b/crates/lpm-sandbox/src/lib.rs index da3737bb..39a96b32 100644 --- a/crates/lpm-sandbox/src/lib.rs +++ b/crates/lpm-sandbox/src/lib.rs @@ -286,6 +286,12 @@ pub struct SandboxOptions { /// /// Participates in the sandbox CLI / config / env precedence chain. pub deny_outbound_network: bool, + + /// Restrict persistent writes to the package directory and the supplied + /// temporary directory. This is used only while producing a reusable + /// lifecycle-build artifact; normal lifecycle execution keeps the broader + /// compatibility write set. + pub build_cache_isolation: bool, } impl SandboxOptions { diff --git a/crates/lpm-sandbox/src/linux.rs b/crates/lpm-sandbox/src/linux.rs index 10a9f70c..56532d4e 100644 --- a/crates/lpm-sandbox/src/linux.rs +++ b/crates/lpm-sandbox/src/linux.rs @@ -117,15 +117,15 @@ #![cfg(target_os = "linux")] -use crate::landlock_rules::{RuleAccess, describe_rules}; +use crate::landlock_rules::RuleAccess; use crate::posture_decision::{PostureDecision, REQUIRED_KERNEL_FOR_STRICT, decide_posture}; use crate::{ Sandbox, SandboxError, SandboxMode, SandboxOptions, SandboxPosture, SandboxSpec, SandboxedCommand, }; use landlock::{ - ABI, Access, AccessFs, AccessNet, CompatLevel, Compatible, PathBeneath, PathFd, Ruleset, - RulesetAttr, RulesetCreated, RulesetCreatedAttr, RulesetError, RulesetStatus, + ABI, Access, AccessFs, AccessNet, BitFlags, CompatLevel, Compatible, PathBeneath, PathFd, + Ruleset, RulesetAttr, RulesetCreated, RulesetCreatedAttr, RulesetError, RulesetStatus, }; use std::os::unix::process::CommandExt; use std::process::{Child, Command, Stdio}; @@ -237,6 +237,7 @@ pub(crate) struct LandlockSandbox { spec: SandboxSpec, mode: SandboxMode, posture: BackendPosture, + build_cache_isolation: bool, } impl LandlockSandbox { @@ -245,6 +246,7 @@ impl LandlockSandbox { mode: SandboxMode, options: SandboxOptions, ) -> Result { + let build_cache_isolation = options.build_cache_isolation; match mode { SandboxMode::Enforce => { let posture = if options.deny_outbound_network { @@ -315,6 +317,7 @@ impl LandlockSandbox { spec, mode, posture, + build_cache_isolation, }) } // Landlock has no native observe-only primitive @@ -425,11 +428,11 @@ impl Sandbox for LandlockSandbox { // `landlock_add_rule` via Ruleset::add_rule) happens here in // normal multi-threaded context. The child's pre_exec body // only touches direct syscalls. - let ruleset = build_parent_side_ruleset(&self.spec, &self.posture).map_err(|e| { - SandboxError::ProfileRenderFailed { - reason: format!("landlock ruleset build failed: {e}"), - } - })?; + let ruleset = + build_parent_side_ruleset(&self.spec, &self.posture, self.build_cache_isolation) + .map_err(|e| SandboxError::ProfileRenderFailed { + reason: format!("landlock ruleset build failed: {e}"), + })?; // compile the seccomp socket(2) deny filter // parent-side. Like the landlock ruleset, all the @@ -752,10 +755,10 @@ fn probe_landlock_at(abi: ABI, with_network: bool) -> Result<(), RulesetError> { fn build_parent_side_ruleset( spec: &SandboxSpec, posture: &BackendPosture, + build_cache_isolation: bool, ) -> Result { let abi = posture.abi(); let rw = AccessFs::from_all(abi); - let read = AccessFs::from_read(abi); let mut builder = Ruleset::default().handle_access(rw)?; if posture.enforces_network() { // Strict V4 only: declare the network access classes so @@ -767,7 +770,9 @@ fn build_parent_side_ruleset( builder = builder.handle_access(AccessNet::from_all(abi))?; } let mut ruleset = builder.create()?; - for (path, access) in describe_rules(spec) { + for (path, access) in + crate::landlock_rules::describe_rules_with_isolation(spec, build_cache_isolation) + { let fd = match PathFd::new(&path) { Ok(fd) => fd, Err(e) => { @@ -775,15 +780,25 @@ fn build_parent_side_ruleset( continue; } }; - let access_bits = match access { - RuleAccess::Read => read, - RuleAccess::ReadWrite => rw, - }; + let is_file = std::fs::metadata(&path).is_ok_and(|metadata| !metadata.is_dir()); + let access_bits = rule_access_bits(access, abi, is_file); ruleset = ruleset.add_rule(PathBeneath::new(fd, access_bits))?; } Ok(ruleset) } +fn rule_access_bits(access: RuleAccess, abi: ABI, is_file: bool) -> BitFlags { + let valid = if is_file { + AccessFs::from_file(abi) + } else { + AccessFs::from_all(abi) + }; + match access { + RuleAccess::Read => valid & AccessFs::from_read(abi), + RuleAccess::ReadWrite => valid, + } +} + /// Async-signal-safe stderr write. Bypasses [`std::io::Stderr::lock`] /// (which holds a userspace mutex and deadlocks post-fork in /// multi-threaded processes) by issuing a direct `write(2)` to fd 2. @@ -976,6 +991,7 @@ mod tests { let options = SandboxOptions { deny_outbound_network: true, allow_degraded: true, + build_cache_isolation: false, }; match LandlockSandbox::new(realistic_spec(), SandboxMode::Enforce, options) { Ok(sb) => { @@ -1035,7 +1051,7 @@ mod tests { detected_kernel: "5.15.0-test".to_string(), }, ] { - match build_parent_side_ruleset(&spec, &posture) { + match build_parent_side_ruleset(&spec, &posture, false) { Ok(_) => {} // ruleset built, missing extra was skipped Err(e) => { // Only acceptable error: the kernel doesn't support @@ -1067,6 +1083,31 @@ mod tests { assert!(status.success(), "/usr/bin/true under landlock must exit 0"); } + #[test] + fn regular_file_rules_exclude_directory_only_access_rights() { + let read = rule_access_bits(RuleAccess::Read, ABI::V4, true); + let read_write = rule_access_bits(RuleAccess::ReadWrite, ABI::V4, true); + + assert_eq!(read, AccessFs::Execute | AccessFs::ReadFile); + assert_eq!(read_write, AccessFs::from_file(ABI::V4)); + assert!(!read_write.contains(AccessFs::ReadDir)); + assert!(!read_write.contains(AccessFs::RemoveDir)); + assert!(!read_write.contains(AccessFs::MakeDir)); + assert!(!read_write.contains(AccessFs::Refer)); + } + + #[test] + fn directory_rules_keep_full_requested_access_rights() { + assert_eq!( + rule_access_bits(RuleAccess::Read, ABI::V4, false), + AccessFs::from_read(ABI::V4) + ); + assert_eq!( + rule_access_bits(RuleAccess::ReadWrite, ABI::V4, false), + AccessFs::from_all(ABI::V4) + ); + } + #[test] fn enforces_deny_on_read_outside_allow_list() { // Forbidden target MUST live at a path no sandbox rule diff --git a/crates/lpm-sandbox/src/macos.rs b/crates/lpm-sandbox/src/macos.rs index 98e09f16..2424d3a3 100644 --- a/crates/lpm-sandbox/src/macos.rs +++ b/crates/lpm-sandbox/src/macos.rs @@ -49,7 +49,11 @@ impl SeatbeltSandbox { ) -> Result { let deny_outbound_network = options.deny_outbound_network; let profile = match mode { - SandboxMode::Enforce => seatbelt::render_profile(&spec, deny_outbound_network)?, + SandboxMode::Enforce => seatbelt::render_profile_with_isolation( + &spec, + deny_outbound_network, + options.build_cache_isolation, + )?, SandboxMode::LogOnly => seatbelt::render_logonly_profile(&spec, deny_outbound_network)?, // Disabled never reaches this backend — the factory in // [`crate::new_for_platform`] short-circuits to diff --git a/crates/lpm-sandbox/src/seatbelt.rs b/crates/lpm-sandbox/src/seatbelt.rs index 7fcdd2eb..80635592 100644 --- a/crates/lpm-sandbox/src/seatbelt.rs +++ b/crates/lpm-sandbox/src/seatbelt.rs @@ -29,6 +29,14 @@ use std::path::{Path, PathBuf}; pub(crate) fn render_profile( spec: &SandboxSpec, deny_outbound_network: bool, +) -> Result { + render_profile_with_isolation(spec, deny_outbound_network, false) +} + +pub(crate) fn render_profile_with_isolation( + spec: &SandboxSpec, + deny_outbound_network: bool, + build_cache_isolation: bool, ) -> Result { // Canonicalize base paths so Seatbelt rules match against the // same form the kernel uses at enforcement time. macOS symlinks @@ -124,7 +132,16 @@ pub(crate) fn render_profile( // deny-default profile. out.push_str(" (literal \"/\")\n"); out.push_str(&format!(" (subpath {package_dir})\n")); - out.push_str(&format!(" (subpath {project_dir})\n")); + if build_cache_isolation { + let dependency_root = quoted_path( + &package_dependency_root(spec, &canon_package_dir), + "package dependency root", + )?; + out.push_str(&format!(" (subpath {dependency_root})\n")); + out.push_str(&format!(" (subpath {home_node_gyp})\n")); + } else { + out.push_str(&format!(" (subpath {project_dir})\n")); + } out.push_str(" (subpath \"/usr\")\n"); out.push_str(" (subpath \"/bin\")\n"); out.push_str(" (subpath \"/sbin\")\n"); @@ -172,18 +189,22 @@ pub(crate) fn render_profile( // form. `/dev/null` is writable so `>/dev/null` redirects work. out.push_str("(allow file-write*\n"); out.push_str(&format!(" (subpath {package_dir})\n")); - out.push_str(&format!(" (subpath {project_node_modules})\n")); - out.push_str(&format!(" (subpath {project_husky})\n")); - out.push_str(&format!(" (subpath {project_lpm})\n")); - out.push_str(&format!(" (subpath {home_cache})\n")); - out.push_str(&format!(" (subpath {home_node_gyp})\n")); - out.push_str(&format!(" (subpath {home_npm})\n")); - out.push_str(" (subpath \"/tmp\")\n"); + if !build_cache_isolation { + out.push_str(&format!(" (subpath {project_node_modules})\n")); + out.push_str(&format!(" (subpath {project_husky})\n")); + out.push_str(&format!(" (subpath {project_lpm})\n")); + out.push_str(&format!(" (subpath {home_cache})\n")); + out.push_str(&format!(" (subpath {home_node_gyp})\n")); + out.push_str(&format!(" (subpath {home_npm})\n")); + out.push_str(" (subpath \"/tmp\")\n"); + } out.push_str(&format!(" (subpath {tmpdir})\n")); out.push_str(" (literal \"/dev/null\")\n"); out.push_str(" (literal \"/dev/tty\")\n"); - for e in &extras { - out.push_str(&format!(" (subpath {e})\n")); + if !build_cache_isolation { + for e in &extras { + out.push_str(&format!(" (subpath {e})\n")); + } } out.push_str(")\n"); out.push('\n'); @@ -247,6 +268,19 @@ pub(crate) fn render_profile( Ok(out) } +fn package_dependency_root(spec: &SandboxSpec, canonical_package_dir: &Path) -> PathBuf { + let name_depth = spec + .package_name + .bytes() + .filter(|byte| *byte == b'/') + .count(); + canonical_package_dir + .ancestors() + .nth(name_depth + 1) + .unwrap_or(canonical_package_dir) + .to_path_buf() +} + /// Emit the `(deny file-read* ...)` block for well-known secret /// conventions. Driven by the shared catalog in /// [`crate::secret_paths`]; all entries are rendered project-rooted @@ -643,6 +677,25 @@ mod tests { ); } + #[test] + fn build_cache_profile_omits_project_and_home_cache_writes() { + let s = spec(); + let profile = render_profile_with_isolation(&s, true, true).unwrap(); + let write_section = profile + .split("(allow file-write*\n") + .nth(1) + .unwrap() + .split("\n)\n") + .next() + .unwrap(); + + assert!(write_section.contains("/lpm-store/prisma@5.22.0")); + assert!(write_section.contains("/var/folders/xx/T")); + assert!(!write_section.contains(".husky")); + assert!(!write_section.contains(".node-gyp")); + assert!(!write_section.contains("node_modules")); + } + #[test] fn profile_allows_process_and_signal_primitives() { let p = render_profile(&spec(), false).unwrap(); diff --git a/crates/lpm-sandbox/src/windows.rs b/crates/lpm-sandbox/src/windows.rs index 92b1f7b6..df519b70 100644 --- a/crates/lpm-sandbox/src/windows.rs +++ b/crates/lpm-sandbox/src/windows.rs @@ -1680,6 +1680,7 @@ mod tests { let opts = SandboxOptions { deny_outbound_network: true, allow_degraded: false, + build_cache_isolation: false, }; match WindowsSandbox::new(realistic_spec(), SandboxMode::Enforce, opts) { Err(SandboxError::UnsupportedPlatform { platform, .. }) => { @@ -1694,6 +1695,7 @@ mod tests { let opts = SandboxOptions { deny_outbound_network: true, allow_degraded: true, + build_cache_isolation: false, }; let sb = WindowsSandbox::new(realistic_spec(), SandboxMode::Enforce, opts) .expect("strict + allow_degraded must construct cleanly"); diff --git a/crates/lpm-sandbox/tests/seatbelt_secret_deny_runtime.rs b/crates/lpm-sandbox/tests/seatbelt_secret_deny_runtime.rs index 50712add..761e9367 100644 --- a/crates/lpm-sandbox/tests/seatbelt_secret_deny_runtime.rs +++ b/crates/lpm-sandbox/tests/seatbelt_secret_deny_runtime.rs @@ -53,6 +53,7 @@ fn cat_through_sandbox(project_dir: &Path, path_to_cat: &Path) -> (i32, Vec, let options = SandboxOptions { allow_degraded: false, deny_outbound_network: false, + build_cache_isolation: false, }; let sandbox = new_for_platform_with_options(spec, SandboxMode::Enforce, options) .expect("sandbox construction"); @@ -227,6 +228,7 @@ fn seatbelt_secret_read_allow_restores_dotenv_at_runtime() { let options = SandboxOptions { allow_degraded: false, deny_outbound_network: false, + build_cache_isolation: false, }; let sandbox = new_for_platform_with_options(spec, SandboxMode::Enforce, options) .expect("sandbox construction"); diff --git a/crates/lpm-sandbox/tests/seccomp_socket_deny.rs b/crates/lpm-sandbox/tests/seccomp_socket_deny.rs index 8d319287..899a7c6a 100644 --- a/crates/lpm-sandbox/tests/seccomp_socket_deny.rs +++ b/crates/lpm-sandbox/tests/seccomp_socket_deny.rs @@ -125,6 +125,7 @@ fn denies_udp_socket_under_strict() { let options = SandboxOptions { deny_outbound_network: true, allow_degraded: false, + build_cache_isolation: false, }; let sb = match new_for_platform_with_options(realistic_spec(), SandboxMode::Enforce, options) { Ok(sb) => sb, diff --git a/crates/lpm-sandbox/tests/secret_overlay_bind_mount.rs b/crates/lpm-sandbox/tests/secret_overlay_bind_mount.rs index e31cda6b..9f8fd30b 100644 --- a/crates/lpm-sandbox/tests/secret_overlay_bind_mount.rs +++ b/crates/lpm-sandbox/tests/secret_overlay_bind_mount.rs @@ -188,6 +188,7 @@ fn default_mode_bind_mounts_dotenv_to_empty() { let options = SandboxOptions { allow_degraded: false, deny_outbound_network: false, + build_cache_isolation: false, }; let sandbox = new_for_platform_with_options(spec, SandboxMode::Enforce, options) .expect("sandbox construction"); @@ -230,6 +231,7 @@ fn default_mode_does_not_overlay_source_files() { let options = SandboxOptions { allow_degraded: false, deny_outbound_network: false, + build_cache_isolation: false, }; let sandbox = new_for_platform_with_options(spec, SandboxMode::Enforce, options) .expect("sandbox construction"); diff --git a/crates/lpm-store/src/v2/build_cache.rs b/crates/lpm-store/src/v2/build_cache.rs new file mode 100644 index 00000000..b1b16dc3 --- /dev/null +++ b/crates/lpm-store/src/v2/build_cache.rs @@ -0,0 +1,747 @@ +//! Content-addressed artifacts produced by dependency lifecycle builds. + +use std::path::{Component, Path, PathBuf}; + +use chrono::{DateTime, Utc}; +use lpm_common::LpmError; +use serde::{Deserialize, Serialize}; + +use super::fs_util::{ + create_tmp_dir_locked, ensure_store_tier_dir_locked, materialize_into, materialize_into_inner, + tmp_sibling, +}; +use super::store::Store; +use super::tree_hash::{ObjectTreeStats, compute_object_tree_integrities}; + +const BUILD_KEY_SCHEMA: &[u8] = b"lpm-build-key-v1\0"; +const BUILD_ARTIFACT_SCHEMA: u32 = 1; +const BUILD_PACKAGE_DIR: &str = "package"; +const BUILD_MANIFEST_FILE: &str = "manifest.json"; +/// Sentinel written inside every atomically published build artifact. +pub const BUILD_ARTIFACT_COMPLETE_FILENAME: &str = ".lpm-build-complete"; +const BUILD_MANIFEST_MAX_BYTES: u64 = 256 * 1024; + +/// One lifecycle phase and its exact command body. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildScriptFingerprint { + /// Lifecycle phase name, normally `preinstall`, `install`, or `postinstall`. + pub phase: String, + /// Exact command passed to the lifecycle shell. + pub command: String, +} + +/// Host compatibility dimensions that can affect native output. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildPlatformFingerprint { + /// npm-compatible operating-system name. + pub os: String, + /// npm-compatible CPU architecture name. + pub architecture: String, + /// Linux libc family, or an empty string on other systems. + pub libc: String, + /// Canonical digest of the host's compatible CPU feature set. + pub cpu_features_hash: String, +} + +/// Runtime identity observed from the executable used by lifecycle scripts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildRuntimeFingerprint { + /// Runtime family, initially `node`. + pub runtime: String, + /// Exact runtime version. + pub version: String, + /// Native-module ABI reported by the runtime. + pub modules_abi: String, + /// N-API version reported by the runtime. + pub napi: String, + /// Embedded JavaScript-engine version. + pub engine: String, + /// Digest identifying the resolved runtime executable. + pub executable_hash: String, +} + +/// Effective containment and allowed-input policy for one build. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildSandboxFingerprint { + /// Requested runtime mode. + pub mode: String, + /// Effective backend posture after platform degradation decisions. + pub posture: String, + /// Whether outbound network access was denied. + pub network_denied: bool, + /// Whether credential and ambient environment scrubbing was active. + pub environment_scrubbed: bool, + /// Canonical hash of readable and writable input policy. + pub allowed_inputs_hash: String, +} + +/// Complete deterministic input set for a dependency build. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildKeyInputs { + /// Integrity of the pristine source object. + pub source_integrity: String, + /// Full graph-key digest for the materialized dependency context. + pub graph_key_digest: String, + /// Digest of dependency sources and dependency build identities. + pub dependency_closure_hash: String, + /// Executed lifecycle phases in execution order. + pub scripts: Vec, + /// Host compatibility fingerprint. + pub platform: BuildPlatformFingerprint, + /// Runtime/ABI fingerprint. + pub runtime: BuildRuntimeFingerprint, + /// Effective sandbox fingerprint. + pub sandbox: BuildSandboxFingerprint, + /// Digest of the complete environment exposed to the lifecycle script. + pub environment_hash: String, + /// Digest of discoverable compiler, SDK, and build-tool identities. + pub toolchain_hash: String, +} + +/// Filesystem-safe digest identifying interchangeable build output. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct BuildCacheKey(String); + +impl BuildCacheKey { + /// Derive a key from every input that can affect lifecycle output. + pub fn derive(inputs: &BuildKeyInputs) -> Self { + let mut hasher = blake3::Hasher::new(); + hasher.update(BUILD_KEY_SCHEMA); + hash_field(&mut hasher, b"source", inputs.source_integrity.as_bytes()); + hash_field(&mut hasher, b"graph", inputs.graph_key_digest.as_bytes()); + hash_field( + &mut hasher, + b"dependency_closure", + inputs.dependency_closure_hash.as_bytes(), + ); + for script in &inputs.scripts { + hash_field(&mut hasher, b"phase", script.phase.as_bytes()); + hash_field(&mut hasher, b"command", script.command.as_bytes()); + } + hash_field(&mut hasher, b"os", inputs.platform.os.as_bytes()); + hash_field( + &mut hasher, + b"architecture", + inputs.platform.architecture.as_bytes(), + ); + hash_field(&mut hasher, b"libc", inputs.platform.libc.as_bytes()); + hash_field( + &mut hasher, + b"cpu_features", + inputs.platform.cpu_features_hash.as_bytes(), + ); + hash_field(&mut hasher, b"runtime", inputs.runtime.runtime.as_bytes()); + hash_field( + &mut hasher, + b"runtime_version", + inputs.runtime.version.as_bytes(), + ); + hash_field( + &mut hasher, + b"modules_abi", + inputs.runtime.modules_abi.as_bytes(), + ); + hash_field(&mut hasher, b"napi", inputs.runtime.napi.as_bytes()); + hash_field(&mut hasher, b"engine", inputs.runtime.engine.as_bytes()); + hash_field( + &mut hasher, + b"runtime_executable", + inputs.runtime.executable_hash.as_bytes(), + ); + hash_field(&mut hasher, b"sandbox_mode", inputs.sandbox.mode.as_bytes()); + hash_field( + &mut hasher, + b"sandbox_posture", + inputs.sandbox.posture.as_bytes(), + ); + hash_bool( + &mut hasher, + b"network_denied", + inputs.sandbox.network_denied, + ); + hash_bool( + &mut hasher, + b"environment_scrubbed", + inputs.sandbox.environment_scrubbed, + ); + hash_field( + &mut hasher, + b"allowed_inputs", + inputs.sandbox.allowed_inputs_hash.as_bytes(), + ); + hash_field( + &mut hasher, + b"environment", + inputs.environment_hash.as_bytes(), + ); + hash_field(&mut hasher, b"toolchain", inputs.toolchain_hash.as_bytes()); + Self(hasher.finalize().to_hex().to_string()) + } + + /// Lowercase hexadecimal representation used as the directory name. + #[inline] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +fn hash_field(hasher: &mut blake3::Hasher, label: &[u8], value: &[u8]) { + hasher.update(label); + hasher.update(b"\0"); + hasher.update(&(value.len() as u64).to_le_bytes()); + hasher.update(value); +} + +fn hash_bool(hasher: &mut blake3::Hasher, label: &[u8], value: bool) { + hash_field(hasher, label, &[u8::from(value)]); +} + +/// Integrity and provenance metadata for a completed build artifact. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BuildArtifactManifest { + /// On-disk manifest schema. + pub schema: u32, + /// Cache key owning this artifact. + pub key: String, + /// Integrity of the pristine source object. + pub source_integrity: String, + /// SHA-256 tree digest of the complete built package. + pub output_integrity: String, + /// Number of files in the built package. + pub file_count: u64, + /// Total bytes in regular files. + pub unpacked_bytes: u64, + /// Time spent executing lifecycle commands when this artifact was created. + pub lifecycle_duration_ms: u64, + /// Artifact publication time. + pub created_at: DateTime, +} + +/// A validated build artifact ready to restore. +#[derive(Debug, Clone)] +pub struct BuildArtifact { + /// Content-addressed artifact directory. + pub directory: PathBuf, + /// Validated artifact manifest. + pub manifest: BuildArtifactManifest, +} + +impl BuildArtifact { + /// Directory containing the complete built package tree. + #[inline] + pub fn package_dir(&self) -> PathBuf { + self.directory.join(BUILD_PACKAGE_DIR) + } +} + +/// Result of publishing a successful lifecycle build. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BuildArtifactPublish { + /// This call published a new artifact. + Published, + /// An equivalent complete artifact already existed. + Existing, +} + +impl Store { + /// Return a validated local build artifact, or `None` on a miss or corruption. + pub fn reusable_build_artifact( + &self, + key: &BuildCacheKey, + ) -> Result, LpmError> { + let dir = self.paths().build_artifact_dir(key); + if !dir.is_dir() || !dir.join(BUILD_ARTIFACT_COMPLETE_FILENAME).is_file() { + return Ok(None); + } + let manifest = match read_manifest(&dir) { + Ok(manifest) if manifest.key == key.as_str() => manifest, + Ok(_) | Err(_) => return Ok(None), + }; + let package_dir = dir.join(BUILD_PACKAGE_DIR); + if !package_dir.is_dir() || !tree_has_safe_symlinks(&package_dir)? { + return Ok(None); + } + let actual = compute_object_tree_integrities(&package_dir)?; + if actual.content != manifest.output_integrity + || actual.stats.file_count != manifest.file_count + || actual.stats.unpacked_bytes != manifest.unpacked_bytes + { + return Ok(None); + } + if let Ok(file) = std::fs::File::options() + .write(true) + .open(dir.join(BUILD_ARTIFACT_COMPLETE_FILENAME)) + { + let _ = file.set_modified(std::time::SystemTime::now()); + } + Ok(Some(BuildArtifact { + directory: dir, + manifest, + })) + } + + /// Publish a successful package-local lifecycle build atomically. + pub fn publish_build_artifact( + &self, + key: &BuildCacheKey, + source_integrity: &str, + built_package_dir: &Path, + lifecycle_duration_ms: u64, + replace_existing: bool, + ) -> Result { + if !replace_existing && self.reusable_build_artifact(key)?.is_some() { + return Ok(BuildArtifactPublish::Existing); + } + if !tree_has_safe_symlinks(built_package_dir)? { + return Err(LpmError::Store(format!( + "refusing to cache lifecycle output at {} because it contains an escaping symlink", + built_package_dir.display() + ))); + } + ensure_store_tier_dir_locked(self.paths().builds_root()).map_err(|error| { + LpmError::Store(format!("failed to create v2 builds directory: {error}")) + })?; + + let final_dir = self.paths().build_artifact_dir(key); + if final_dir.exists() { + std::fs::remove_dir_all(&final_dir).map_err(|error| { + LpmError::Store(format!( + "failed to remove corrupt build artifact at {}: {error}", + final_dir.display() + )) + })?; + } + let tmp_dir = tmp_sibling(&final_dir); + create_tmp_dir_locked(&tmp_dir).map_err(|error| { + LpmError::Store(format!( + "failed to create build artifact staging directory at {}: {error}", + tmp_dir.display() + )) + })?; + + let staged_package = tmp_dir.join(BUILD_PACKAGE_DIR); + let publish_result = (|| { + materialize_into_inner(built_package_dir, built_package_dir, &staged_package, true)?; + let tree = compute_object_tree_integrities(&staged_package)?; + let manifest = manifest_from_tree( + key, + source_integrity, + &tree.stats, + tree.content, + lifecycle_duration_ms, + ); + let bytes = serde_json::to_vec_pretty(&manifest).map_err(|error| { + LpmError::Store(format!( + "failed to serialize build artifact manifest: {error}" + )) + })?; + std::fs::write(tmp_dir.join(BUILD_MANIFEST_FILE), bytes)?; + std::fs::write(tmp_dir.join(BUILD_ARTIFACT_COMPLETE_FILENAME), b"")?; + Ok::<(), LpmError>(()) + })(); + if let Err(error) = publish_result { + let _ = std::fs::remove_dir_all(&tmp_dir); + return Err(error); + } + + match std::fs::rename(&tmp_dir, &final_dir) { + Ok(()) => Ok(BuildArtifactPublish::Published), + Err(error) if final_dir.exists() => { + let _ = std::fs::remove_dir_all(&tmp_dir); + if self.reusable_build_artifact(key)?.is_some() { + Ok(BuildArtifactPublish::Existing) + } else { + Err(LpmError::Store(format!( + "concurrent build artifact publication left an invalid entry at {}: {error}", + final_dir.display() + ))) + } + } + Err(error) => { + let _ = std::fs::remove_dir_all(&tmp_dir); + Err(LpmError::Store(format!( + "failed to publish build artifact at {}: {error}", + final_dir.display() + ))) + } + } + } + + /// Restore a validated build artifact over a materialized package tree. + pub fn restore_build_artifact( + &self, + artifact: &BuildArtifact, + destination: &Path, + ) -> Result<(), LpmError> { + let staged = tmp_sibling(destination); + materialize_into_inner( + &artifact.package_dir(), + &artifact.package_dir(), + &staged, + true, + )?; + replace_with_staged_tree(destination, &staged) + } + + /// Replace a built link-package tree with its verified pristine object. + pub fn restore_pristine_package( + &self, + pristine_object_dir: &Path, + destination: &Path, + ) -> Result<(), LpmError> { + let staged = tmp_sibling(destination); + materialize_into(pristine_object_dir, &staged)?; + replace_with_staged_tree(destination, &staged) + } +} + +fn replace_with_staged_tree(destination: &Path, staged: &Path) -> Result<(), LpmError> { + let parent = destination.parent().ok_or_else(|| { + LpmError::Store(format!( + "package-tree destination has no parent: {}", + destination.display() + )) + })?; + let backup = + destination.with_extension(format!("lpm-build-backup.{:016x}", rand::random::())); + if destination.exists() { + std::fs::rename(destination, &backup).map_err(|error| { + let _ = std::fs::remove_dir_all(staged); + LpmError::Store(format!( + "failed to stage existing package tree at {}: {error}", + destination.display() + )) + })?; + } + if let Err(error) = std::fs::rename(staged, destination) { + if backup.exists() { + let _ = std::fs::rename(&backup, destination); + } + let _ = std::fs::remove_dir_all(staged); + return Err(LpmError::Store(format!( + "failed to restore build artifact into {} from {}: {error}", + destination.display(), + parent.display() + ))); + } + if backup.exists() { + std::fs::remove_dir_all(&backup).map_err(|error| { + LpmError::Store(format!( + "restored build artifact but failed to remove backup at {}: {error}", + backup.display() + )) + })?; + } + Ok(()) +} + +fn manifest_from_tree( + key: &BuildCacheKey, + source_integrity: &str, + stats: &ObjectTreeStats, + output_integrity: String, + lifecycle_duration_ms: u64, +) -> BuildArtifactManifest { + BuildArtifactManifest { + schema: BUILD_ARTIFACT_SCHEMA, + key: key.as_str().to_string(), + source_integrity: source_integrity.to_string(), + output_integrity, + file_count: stats.file_count, + unpacked_bytes: stats.unpacked_bytes, + lifecycle_duration_ms, + created_at: Utc::now(), + } +} + +fn read_manifest(dir: &Path) -> Result { + let path = dir.join(BUILD_MANIFEST_FILE); + let metadata = std::fs::metadata(&path)?; + if metadata.len() > BUILD_MANIFEST_MAX_BYTES { + return Err(LpmError::Store(format!( + "build artifact manifest exceeds {} bytes at {}", + BUILD_MANIFEST_MAX_BYTES, + path.display() + ))); + } + let bytes = std::fs::read(&path)?; + let manifest: BuildArtifactManifest = serde_json::from_slice(&bytes).map_err(|error| { + LpmError::Store(format!( + "failed to parse build artifact manifest at {}: {error}", + path.display() + )) + })?; + if manifest.schema != BUILD_ARTIFACT_SCHEMA { + return Err(LpmError::Store(format!( + "unsupported build artifact schema {} at {}", + manifest.schema, + path.display() + ))); + } + Ok(manifest) +} + +fn tree_has_safe_symlinks(root: &Path) -> Result { + let mut pending = vec![root.to_path_buf()]; + while let Some(dir) = pending.pop() { + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let path = entry.path(); + let file_type = entry.file_type()?; + if file_type.is_dir() { + pending.push(path); + } else if file_type.is_symlink() { + let target = std::fs::read_link(&path)?; + let relative_parent = path + .parent() + .and_then(|parent| parent.strip_prefix(root).ok()) + .unwrap_or_else(|| Path::new("")); + if !relative_target_stays_in_root(relative_parent, &target) { + return Ok(false); + } + } + } + } + Ok(true) +} + +fn relative_target_stays_in_root(parent: &Path, target: &Path) -> bool { + if target.is_absolute() { + return false; + } + let mut depth = parent.components().count(); + for component in target.components() { + match component { + Component::CurDir => {} + Component::Normal(_) => depth = depth.saturating_add(1), + Component::ParentDir if depth > 0 => depth -= 1, + Component::ParentDir | Component::RootDir | Component::Prefix(_) => return false, + } + } + true +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_inputs() -> BuildKeyInputs { + BuildKeyInputs { + source_integrity: "sha512-source".into(), + graph_key_digest: "graph".into(), + dependency_closure_hash: "closure".into(), + scripts: vec![BuildScriptFingerprint { + phase: "install".into(), + command: "node-gyp rebuild".into(), + }], + platform: BuildPlatformFingerprint { + os: "linux".into(), + architecture: "x64".into(), + libc: "glibc".into(), + cpu_features_hash: "cpu".into(), + }, + runtime: BuildRuntimeFingerprint { + runtime: "node".into(), + version: "22.12.0".into(), + modules_abi: "127".into(), + napi: "10".into(), + engine: "12.4".into(), + executable_hash: "node-binary".into(), + }, + sandbox: BuildSandboxFingerprint { + mode: "enforce".into(), + posture: "strict".into(), + network_denied: true, + environment_scrubbed: true, + allowed_inputs_hash: "policy".into(), + }, + environment_hash: "env".into(), + toolchain_hash: "toolchain".into(), + } + } + + #[test] + fn build_key_is_deterministic_for_identical_inputs() { + let inputs = sample_inputs(); + assert_eq!( + BuildCacheKey::derive(&inputs), + BuildCacheKey::derive(&inputs) + ); + } + + #[test] + fn build_key_changes_when_node_abi_changes() { + let first = sample_inputs(); + let mut second = first.clone(); + second.runtime.modules_abi = "131".into(); + assert_ne!( + BuildCacheKey::derive(&first), + BuildCacheKey::derive(&second) + ); + } + + #[test] + fn build_key_changes_when_sandbox_policy_changes() { + let first = sample_inputs(); + let mut second = first.clone(); + second.sandbox.network_denied = false; + assert_ne!( + BuildCacheKey::derive(&first), + BuildCacheKey::derive(&second) + ); + } + + #[test] + fn build_artifact_round_trip_restores_complete_tree() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let source = temp.path().join("source"); + let destination = temp.path().join("destination"); + std::fs::create_dir_all(source.join("build/Release")).unwrap(); + std::fs::write(source.join("package.json"), b"{}").unwrap(); + std::fs::write(source.join("build/Release/addon.node"), b"native").unwrap(); + std::fs::create_dir_all(&destination).unwrap(); + std::fs::write(destination.join("stale"), b"stale").unwrap(); + let key = BuildCacheKey::derive(&sample_inputs()); + + let published = store + .publish_build_artifact(&key, "sha512-source", &source, 125, false) + .unwrap(); + let artifact = store.reusable_build_artifact(&key).unwrap().unwrap(); + store + .restore_build_artifact(&artifact, &destination) + .unwrap(); + + assert_eq!(published, BuildArtifactPublish::Published); + assert_eq!(artifact.manifest.lifecycle_duration_ms, 125); + assert_eq!( + std::fs::read(destination.join("build/Release/addon.node")).unwrap(), + b"native" + ); + assert!(!destination.join("stale").exists()); + } + + #[test] + fn build_lock_path_is_stable_and_separate_from_artifacts() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let key = BuildCacheKey::derive(&sample_inputs()); + + let lock_path = store.paths().build_lock_path(&key); + + assert_eq!( + lock_path, + temp.path() + .join("store/build-locks") + .join(format!("{}.lock", key.as_str())) + ); + assert!(!lock_path.starts_with(store.paths().builds_root())); + } + + #[test] + fn build_entry_lock_path_is_keyed_by_graph_identity() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let graph_key = "a".repeat(64); + + assert_eq!( + store.paths().build_entry_lock_path(&graph_key).unwrap(), + temp.path() + .join("store/build-entry-locks") + .join(format!("{graph_key}.lock")) + ); + } + + #[test] + fn build_entry_lock_path_rejects_traversal_digest() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + + assert!( + store + .paths() + .build_entry_lock_path("../../outside") + .is_err() + ); + } + + #[test] + fn build_artifact_corruption_is_a_cache_miss() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let source = temp.path().join("source"); + std::fs::create_dir_all(&source).unwrap(); + std::fs::write(source.join("package.json"), b"{}").unwrap(); + let key = BuildCacheKey::derive(&sample_inputs()); + store + .publish_build_artifact(&key, "sha512-source", &source, 125, false) + .unwrap(); + let artifact_dir = store.paths().build_artifact_dir(&key); + std::fs::write(artifact_dir.join("package/package.json"), b"corrupt").unwrap(); + + assert!(store.reusable_build_artifact(&key).unwrap().is_none()); + } + + #[test] + fn pristine_restore_replaces_previous_build_outputs() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let pristine = temp.path().join("pristine"); + let destination = temp.path().join("destination"); + std::fs::create_dir_all(&pristine).unwrap(); + std::fs::create_dir_all(&destination).unwrap(); + std::fs::write(pristine.join("package.json"), b"{}").unwrap(); + std::fs::write(destination.join("old-build.node"), b"old").unwrap(); + + store + .restore_pristine_package(&pristine, &destination) + .unwrap(); + + assert!(destination.join("package.json").is_file()); + assert!(!destination.join("old-build.node").exists()); + } + + #[test] + fn forced_publication_replaces_existing_artifact_for_same_key() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let source = temp.path().join("source"); + std::fs::create_dir_all(&source).unwrap(); + std::fs::write(source.join("output.node"), b"first").unwrap(); + let key = BuildCacheKey::derive(&sample_inputs()); + store + .publish_build_artifact(&key, "sha512-source", &source, 10, false) + .unwrap(); + + std::fs::write(source.join("output.node"), b"second").unwrap(); + store + .publish_build_artifact(&key, "sha512-source", &source, 20, true) + .unwrap(); + let artifact = store.reusable_build_artifact(&key).unwrap().unwrap(); + + assert_eq!( + std::fs::read(artifact.package_dir().join("output.node")).unwrap(), + b"second" + ); + assert_eq!(artifact.manifest.lifecycle_duration_ms, 20); + } + + #[cfg(unix)] + #[test] + fn build_artifact_rejects_symlink_that_escapes_package() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let store = Store::at(temp.path().join("store")); + let source = temp.path().join("source"); + std::fs::create_dir_all(&source).unwrap(); + std::fs::write(source.join("package.json"), b"{}").unwrap(); + symlink("../secret", source.join("escape")).unwrap(); + let key = BuildCacheKey::derive(&sample_inputs()); + + let error = store + .publish_build_artifact(&key, "sha512-source", &source, 125, false) + .unwrap_err(); + assert!(error.to_string().contains("escaping symlink")); + } +} diff --git a/crates/lpm-store/src/v2/link_meta.rs b/crates/lpm-store/src/v2/link_meta.rs index f4062f9c..2df1d7ad 100644 --- a/crates/lpm-store/src/v2/link_meta.rs +++ b/crates/lpm-store/src/v2/link_meta.rs @@ -265,6 +265,12 @@ impl LinkMeta { parsed.name ))); } + if !is_lower_hex_digest(&parsed.graph_key_digest_hex) { + return Err(LpmError::Store(format!( + "v2 link sidecar at {} has invalid graph-key digest", + path.display() + ))); + } Ok(parsed) } @@ -399,6 +405,13 @@ fn validate_path_segment(s: &str) -> Result<(), &'static str> { Ok(()) } +pub(crate) fn is_lower_hex_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + #[cfg(test)] mod tests { use super::*; @@ -684,4 +697,20 @@ mod tests { let reconstructed = GraphKey::from_recorded(&meta.name, &meta.version, digest_bytes); assert_eq!(reconstructed, key); } + + #[test] + fn read_from_rejects_graph_key_digest_outside_exact_lower_hex_shape() { + for invalid in ["../../outside".into(), "A".repeat(64), "a".repeat(63)] { + let dir = tempfile::tempdir().unwrap(); + let mut meta = sample_meta(); + meta.graph_key_digest_hex = invalid; + std::fs::write( + dir.path().join(LINK_META_FILENAME), + serde_json::to_vec(&meta).unwrap(), + ) + .unwrap(); + + assert!(LinkMeta::read_from(dir.path()).is_err()); + } + } } diff --git a/crates/lpm-store/src/v2/mod.rs b/crates/lpm-store/src/v2/mod.rs index 58762217..9463cb0a 100644 --- a/crates/lpm-store/src/v2/mod.rs +++ b/crates/lpm-store/src/v2/mod.rs @@ -37,6 +37,7 @@ //! //! See [`GraphKey::derive`] for the exact byte layout fed into BLAKE3. +pub mod build_cache; pub(crate) mod finalize_permits; pub(crate) mod fs_util; pub mod graph_key; @@ -47,6 +48,11 @@ pub mod platform; pub mod store; pub(crate) mod tree_hash; +pub use build_cache::{ + BUILD_ARTIFACT_COMPLETE_FILENAME, BuildArtifact, BuildArtifactManifest, BuildArtifactPublish, + BuildCacheKey, BuildKeyInputs, BuildPlatformFingerprint, BuildRuntimeFingerprint, + BuildSandboxFingerprint, BuildScriptFingerprint, +}; pub use graph_key::{DepEdge, GraphKey, GraphKeyInputs, LinkerModeTag, PeerEntry}; pub use link_meta::{ LINK_META_FILENAME, LINK_META_SCHEMA_VERSION, LinkMeta, LinkMetaDep, LinkMetaPlatform, diff --git a/crates/lpm-store/src/v2/store.rs b/crates/lpm-store/src/v2/store.rs index d24aabb7..46f134a7 100644 --- a/crates/lpm-store/src/v2/store.rs +++ b/crates/lpm-store/src/v2/store.rs @@ -62,6 +62,16 @@ const LINKS_DIR: &str = "links"; /// re-copying every package. const COMPAT_DIR: &str = "compat"; +/// Subdirectory holding lifecycle-build artifacts keyed by their complete +/// execution inputs. +const BUILDS_DIR: &str = "builds"; + +/// Subdirectory holding per-build-key advisory lock files. +const BUILD_LOCKS_DIR: &str = "build-locks"; + +/// Subdirectory holding per-graph-entry advisory lock files. +const BUILD_ENTRY_LOCKS_DIR: &str = "build-entry-locks"; + /// Schema tag folded into [`compat_island_key`] so a change to the island's /// on-disk layout invalidates every cached island instead of silently /// reusing a stale shape. @@ -137,6 +147,12 @@ pub struct StoreV2Paths { links_root: PathBuf, /// `~/.lpm/store/v2/compat/` — precomputed for the same reason. compat_root: PathBuf, + /// `~/.lpm/store/v2/builds/` — precomputed for build-cache operations. + builds_root: PathBuf, + /// `~/.lpm/store/v2/build-locks/` — precomputed for per-key serialization. + build_locks_root: PathBuf, + /// `~/.lpm/store/v2/build-entry-locks/` — precomputed for link-tree mutation. + build_entry_locks_root: PathBuf, } impl StoreV2Paths { @@ -146,11 +162,17 @@ impl StoreV2Paths { let objects_root = root.join(OBJECTS_DIR); let links_root = root.join(LINKS_DIR); let compat_root = root.join(COMPAT_DIR); + let builds_root = root.join(BUILDS_DIR); + let build_locks_root = root.join(BUILD_LOCKS_DIR); + let build_entry_locks_root = root.join(BUILD_ENTRY_LOCKS_DIR); Self { root, objects_root, links_root, compat_root, + builds_root, + build_locks_root, + build_entry_locks_root, } } @@ -160,11 +182,17 @@ impl StoreV2Paths { let objects_root = root.join(OBJECTS_DIR); let links_root = root.join(LINKS_DIR); let compat_root = root.join(COMPAT_DIR); + let builds_root = root.join(BUILDS_DIR); + let build_locks_root = root.join(BUILD_LOCKS_DIR); + let build_entry_locks_root = root.join(BUILD_ENTRY_LOCKS_DIR); Self { root, objects_root, links_root, compat_root, + builds_root, + build_locks_root, + build_entry_locks_root, } } @@ -196,6 +224,55 @@ impl StoreV2Paths { &self.compat_root } + /// `~/.lpm/store/v2/builds/` — root of lifecycle-build artifacts. + #[inline] + pub fn builds_root(&self) -> &Path { + &self.builds_root + } + + /// Return the content-addressed directory for a lifecycle-build key. + #[inline] + pub fn build_artifact_dir(&self, key: &crate::v2::BuildCacheKey) -> PathBuf { + self.builds_root.join(key.as_str()) + } + + /// Advisory lock path serializing work for one lifecycle-build key. + #[inline] + pub fn build_lock_path(&self, key: &crate::v2::BuildCacheKey) -> PathBuf { + self.build_lock_path_for_key(key.as_str()) + } + + /// Advisory lock path for a validated hexadecimal build-key string. + #[inline] + pub fn build_lock_path_for_key(&self, key: &str) -> PathBuf { + self.build_locks_root.join(format!("{key}.lock")) + } + + /// `~/.lpm/store/v2/build-locks/` — per-key advisory locks. + #[inline] + pub fn build_locks_root(&self) -> &Path { + &self.build_locks_root + } + + /// Advisory lock path serializing mutation of one v2 graph entry. + #[inline] + pub fn build_entry_lock_path(&self, graph_key_digest: &str) -> Result { + if !crate::v2::link_meta::is_lower_hex_digest(graph_key_digest) { + return Err(LpmError::Store( + "invalid graph-key digest for build-entry lock".into(), + )); + } + Ok(self + .build_entry_locks_root + .join(format!("{graph_key_digest}.lock"))) + } + + /// `~/.lpm/store/v2/build-entry-locks/` — per-graph-entry advisory locks. + #[inline] + pub fn build_entry_locks_root(&self) -> &Path { + &self.build_entry_locks_root + } + /// `~/.lpm/store/v2/compat//` — the cached island for a /// given entry-set content key. `key` is a hex digest the caller /// derives from the island's entry set, so the path is filesystem-safe. diff --git a/tests/workflows/tests/native_build_cache.rs b/tests/workflows/tests/native_build_cache.rs new file mode 100644 index 00000000..b1a0f57e --- /dev/null +++ b/tests/workflows/tests/native_build_cache.rs @@ -0,0 +1,507 @@ +//! Workflow coverage for dependency lifecycle-build artifact reuse. + +#![cfg(any(target_os = "macos", target_os = "linux"))] + +mod support; + +use support::mock_registry::MockRegistry; +use support::{TempProject, lpm_spawnable_with_registry, lpm_with_registry}; + +const PACKAGE_NAME: &str = "esbuild"; +const PACKAGE_VERSION: &str = "1.0.0"; +const BUILD_SCRIPT: &str = r#" +const fs = require('fs'); +fs.writeFileSync('native-output.txt', process.hrtime.bigint().toString()); +"#; + +fn node_available() -> bool { + std::process::Command::new("node") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) +} + +fn project_manifest() -> String { + format!( + r#"{{ + "name": "native-build-cache-workflow", + "version": "1.0.0", + "dependencies": {{ + "{PACKAGE_NAME}": "{PACKAGE_VERSION}" + }} +}}"# + ) +} + +fn installed_package_dir(project: &TempProject) -> std::path::PathBuf { + std::fs::canonicalize(project.path().join("node_modules").join(PACKAGE_NAME)) + .expect("installed dependency must resolve into the v2 link entry") +} + +#[tokio::test] +async fn native_lifecycle_output_is_restored_after_pristine_rematerialization() { + if !node_available() { + eprintln!("skipping: node is required for the native build-cache workflow"); + return; + } + + let registry = MockRegistry::start().await; + registry + .with_manifest_package( + serde_json::json!({ + "name": PACKAGE_NAME, + "version": PACKAGE_VERSION, + "scripts": { + "postinstall": "node install.js" + } + }), + &[("install.js", BUILD_SCRIPT.as_bytes())], + ) + .await; + let project = TempProject::empty(&project_manifest()); + + let install = lpm_with_registry(&project, ®istry.url()) + .arg("install") + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("initial install"); + assert!( + install.status.success(), + "initial install failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&install.stdout), + String::from_utf8_lossy(&install.stderr) + ); + + let first_rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("first native rebuild"); + assert!( + first_rebuild.status.success(), + "first rebuild failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&first_rebuild.stdout), + String::from_utf8_lossy(&first_rebuild.stderr) + ); + let first_output = + std::fs::read_to_string(installed_package_dir(&project).join("native-output.txt")) + .expect("first build must produce native-output.txt"); + let builds_root = project.store_dir().join("v2/builds"); + assert!( + builds_root.is_dir(), + "first rebuild did not create the build cache:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&first_rebuild.stdout), + String::from_utf8_lossy(&first_rebuild.stderr) + ); + assert_eq!( + std::fs::read_dir(&builds_root) + .expect("build artifact root") + .count(), + 1, + "first rebuild must publish exactly one artifact" + ); + + std::fs::remove_dir_all(project.path().join("node_modules")) + .expect("remove project materialization"); + let reinstall = lpm_with_registry(&project, ®istry.url()) + .args(["install", "--force"]) + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("pristine reinstall"); + assert!( + reinstall.status.success(), + "reinstall failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&reinstall.stdout), + String::from_utf8_lossy(&reinstall.stderr) + ); + assert!( + !installed_package_dir(&project) + .join("native-output.txt") + .exists(), + "reinstall must rematerialize pristine source bytes before cache restore" + ); + + let second_rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("cached native rebuild"); + assert!( + second_rebuild.status.success(), + "cached rebuild failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&second_rebuild.stdout), + String::from_utf8_lossy(&second_rebuild.stderr) + ); + let mut envelope: serde_json::Value = serde_json::from_slice(&second_rebuild.stdout) + .expect("cached rebuild stdout must be a JSON envelope"); + assert_eq!(envelope["build_cache"]["hits"], 1); + assert_eq!(envelope["build_cache"]["misses"], 0); + assert_eq!(envelope["build_cache"]["scripts_avoided"], 1); + envelope["build_cache"]["restored_bytes"] = serde_json::json!(""); + envelope["build_cache"]["lifecycle_ms_avoided"] = serde_json::json!(""); + for field in [ + "preparation", + "key", + "lookup", + "restore", + "rematerialize", + "publish", + ] { + envelope["build_cache"]["timings_ms"][field] = serde_json::json!(""); + } + insta::assert_json_snapshot!("native_build_cache_hit_metrics", envelope); + assert_eq!( + std::fs::read_to_string(installed_package_dir(&project).join("native-output.txt")) + .expect("cached output must be restored"), + first_output, + "cache hit must restore the first build output instead of executing the script again" + ); + + let invalidated_rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .env("CFLAGS", "-DLPM_BUILD_CACHE_INVALIDATION=1") + .output() + .expect("rebuild with changed build environment"); + assert!( + invalidated_rebuild.status.success(), + "invalidated rebuild failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&invalidated_rebuild.stdout), + String::from_utf8_lossy(&invalidated_rebuild.stderr) + ); + let invalidated_envelope: serde_json::Value = + serde_json::from_slice(&invalidated_rebuild.stdout) + .expect("invalidated rebuild stdout must be JSON"); + assert_eq!(invalidated_envelope["build_cache"]["hits"], 0); + assert_eq!(invalidated_envelope["build_cache"]["misses"], 1); + assert_ne!( + std::fs::read_to_string(installed_package_dir(&project).join("native-output.txt")) + .expect("invalidated rebuild must produce native-output.txt"), + first_output, + "a changed build-environment key must invalidate keyed local build state and rerun the lifecycle command" + ); +} + +#[tokio::test] +async fn custom_lifecycle_environment_change_invalidates_cached_output() { + if !node_available() { + eprintln!("skipping: node is required for the native build-cache workflow"); + return; + } + + let registry = MockRegistry::start().await; + let script = br#" +const fs = require('fs'); +fs.writeFileSync('native-output.txt', process.env.LPM_NATIVE_TEST_INPUT); +"#; + registry + .with_manifest_package( + serde_json::json!({ + "name": PACKAGE_NAME, + "version": PACKAGE_VERSION, + "scripts": { + "postinstall": "node install.js" + } + }), + &[("install.js", script)], + ) + .await; + let project = TempProject::empty(&project_manifest()); + let install = lpm_with_registry(&project, ®istry.url()) + .arg("install") + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("initial install"); + assert!(install.status.success()); + + for value in ["first", "second"] { + let rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .env("LPM_NATIVE_TEST_INPUT", value) + .output() + .expect("native rebuild"); + assert!( + rebuild.status.success(), + "rebuild failed for {value}: {}", + String::from_utf8_lossy(&rebuild.stderr) + ); + let envelope: serde_json::Value = + serde_json::from_slice(&rebuild.stdout).expect("native rebuild stdout must be JSON"); + assert_eq!( + envelope["build_cache"]["misses"], + 1, + "{value} must execute under its own cache key; envelope: {envelope}; stderr: {}", + String::from_utf8_lossy(&rebuild.stderr) + ); + } + + assert_eq!( + std::fs::read_to_string(installed_package_dir(&project).join("native-output.txt")) + .expect("second build output"), + "second" + ); + assert_eq!( + std::fs::read_dir(project.store_dir().join("v2/builds")) + .expect("build artifacts") + .count(), + 2, + "each visible custom environment must have a distinct cache key" + ); +} + +#[tokio::test] +async fn native_toolchain_snapshot_persists_across_rebuild_processes() { + if !node_available() { + eprintln!("skipping: node is required for the native build-cache workflow"); + return; + } + + let registry = MockRegistry::start().await; + registry + .with_manifest_package( + serde_json::json!({ + "name": "sharp", + "version": PACKAGE_VERSION, + "scripts": { + "postinstall": "node install/check" + } + }), + &[( + "install/check", + b"require('fs').writeFileSync('native-output.txt', 'built');", + )], + ) + .await; + let project = TempProject::empty(&format!( + r#"{{ + "name": "native-toolchain-snapshot-workflow", + "version": "1.0.0", + "dependencies": {{ + "sharp": "{PACKAGE_VERSION}" + }} +}}"# + )); + let install = lpm_with_registry(&project, ®istry.url()) + .arg("install") + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("initial install"); + assert!( + install.status.success(), + "initial install failed: {}", + String::from_utf8_lossy(&install.stderr) + ); + + let first_rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("first native rebuild"); + assert!( + first_rebuild.status.success(), + "first rebuild failed: {}", + String::from_utf8_lossy(&first_rebuild.stderr) + ); + + let snapshot_dir = project.cache_dir().join("metadata/native-toolchains/v1"); + let snapshots = std::fs::read_dir(&snapshot_dir) + .expect("native toolchain snapshot directory") + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "json") + }) + .collect::>(); + assert_eq!(snapshots.len(), 1); + let first_snapshot: serde_json::Value = serde_json::from_slice( + &std::fs::read(&snapshots[0]).expect("persisted native toolchain snapshot"), + ) + .expect("native toolchain snapshot must be valid JSON"); + assert_eq!(first_snapshot["schema"], 1); + + let second_rebuild = lpm_with_registry(&project, ®istry.url()) + .args(["rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("second native rebuild"); + assert!( + second_rebuild.status.success(), + "second rebuild failed: {}", + String::from_utf8_lossy(&second_rebuild.stderr) + ); + + let second_snapshot: serde_json::Value = serde_json::from_slice( + &std::fs::read(&snapshots[0]).expect("native toolchain snapshot after second rebuild"), + ) + .expect("native toolchain snapshot must remain valid JSON"); + assert_eq!(second_snapshot["schema"], 1); + assert_eq!(second_snapshot["base_key"], first_snapshot["base_key"]); +} + +#[tokio::test] +async fn concurrent_native_rebuilds_execute_one_lifecycle_build_per_key() { + if !node_available() { + eprintln!("skipping: node is required for the native build-cache workflow"); + return; + } + + let registry = MockRegistry::start().await; + let script = br#" +const fs = require('fs'); +fs.appendFileSync('executions.txt', `${process.pid}\n`); +Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500); +"#; + registry + .with_manifest_package( + serde_json::json!({ + "name": PACKAGE_NAME, + "version": PACKAGE_VERSION, + "scripts": { + "postinstall": "node install.js" + } + }), + &[("install.js", script)], + ) + .await; + let project = TempProject::empty(&project_manifest()); + let install = lpm_with_registry(&project, ®istry.url()) + .arg("install") + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("initial install"); + assert!(install.status.success()); + + let mut first_command = lpm_spawnable_with_registry(&project, ®istry.url()); + first_command + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2"); + let mut second_command = lpm_spawnable_with_registry(&project, ®istry.url()); + second_command + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2"); + + let first = first_command.spawn().expect("spawn first rebuild"); + let second = second_command.spawn().expect("spawn second rebuild"); + let first_output = first.wait_with_output().expect("wait for first rebuild"); + let second_output = second.wait_with_output().expect("wait for second rebuild"); + assert!( + first_output.status.success() && second_output.status.success(), + "concurrent rebuild failed:\nfirst stderr:\n{}\nsecond stderr:\n{}", + String::from_utf8_lossy(&first_output.stderr), + String::from_utf8_lossy(&second_output.stderr) + ); + + let executions = + std::fs::read_to_string(installed_package_dir(&project).join("executions.txt")) + .expect("the winning lifecycle build must produce executions.txt"); + assert_eq!( + executions.lines().count(), + 1, + "the per-key lock must prevent duplicate lifecycle execution" + ); + let envelopes = [&first_output.stdout, &second_output.stdout] + .into_iter() + .map(|stdout| serde_json::from_slice::(stdout).unwrap()) + .collect::>(); + assert_eq!( + envelopes + .iter() + .map(|value| { + value["build_cache"]["hits"].as_u64().unwrap() + + value["build_cache"]["local_state_hits"].as_u64().unwrap() + }) + .sum::(), + 1, + "one contender must build and the other must reuse its keyed result" + ); +} + +#[tokio::test] +async fn concurrent_native_rebuilds_with_different_keys_serialize_shared_package_mutation() { + if !node_available() { + eprintln!("skipping: node is required for the native build-cache workflow"); + return; + } + + let registry = MockRegistry::start().await; + let script = br#" +const fs = require('fs'); +const label = process.env.CFLAGS; +fs.writeFileSync('active-build.txt', label); +Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 6000); +const active = fs.existsSync('active-build.txt') + ? fs.readFileSync('active-build.txt', 'utf8') + : ''; +if (active !== label) { + throw new Error(`shared package tree was mutated by ${active} while ${label} was building`); +} +fs.rmSync('active-build.txt'); +fs.writeFileSync('native-output.txt', label); +"#; + registry + .with_manifest_package( + serde_json::json!({ + "name": PACKAGE_NAME, + "version": PACKAGE_VERSION, + "scripts": { + "postinstall": "node install.js" + } + }), + &[("install.js", script)], + ) + .await; + let project = TempProject::empty(&project_manifest()); + let install = lpm_with_registry(&project, ®istry.url()) + .arg("install") + .env("LPM_STORE_VERSION", "v2") + .output() + .expect("initial install"); + assert!(install.status.success()); + + let mut first_command = lpm_spawnable_with_registry(&project, ®istry.url()); + first_command + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .env("CFLAGS", "build-a"); + let first = first_command.spawn().expect("spawn first rebuild"); + let active_path = installed_package_dir(&project).join("active-build.txt"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !active_path.is_file() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!( + active_path.is_file(), + "first build did not enter its lifecycle script" + ); + let mut second_command = lpm_spawnable_with_registry(&project, ®istry.url()); + second_command + .args(["--json", "rebuild", "--all", "--strict-sandbox"]) + .env("LPM_STORE_VERSION", "v2") + .env("CFLAGS", "build-b"); + let second = second_command.spawn().expect("spawn second rebuild"); + let first_output = first.wait_with_output().expect("wait for first rebuild"); + let second_output = second.wait_with_output().expect("wait for second rebuild"); + + assert!( + first_output.status.success() && second_output.status.success(), + "different-key rebuilds raced on one package tree:\nfirst stderr:\n{}\nsecond stderr:\n{}", + String::from_utf8_lossy(&first_output.stderr), + String::from_utf8_lossy(&second_output.stderr) + ); + let mut artifact_outputs = std::fs::read_dir(project.store_dir().join("v2/builds")) + .expect("build artifacts") + .map(|entry| { + std::fs::read_to_string( + entry + .expect("build artifact entry") + .path() + .join("package/native-output.txt"), + ) + .expect("each key must preserve its own native output") + }) + .collect::>(); + artifact_outputs.sort_unstable(); + assert_eq!(artifact_outputs, ["build-a", "build-b"]); +} diff --git a/tests/workflows/tests/snapshots/cache_prune__cache_prune_dry_run_json_envelope_missing_registry.snap b/tests/workflows/tests/snapshots/cache_prune__cache_prune_dry_run_json_envelope_missing_registry.snap index a70fffd5..1597e32b 100644 --- a/tests/workflows/tests/snapshots/cache_prune__cache_prune_dry_run_json_envelope_missing_registry.snap +++ b/tests/workflows/tests/snapshots/cache_prune__cache_prune_dry_run_json_envelope_missing_registry.snap @@ -15,6 +15,8 @@ expression: json "object_entries_orphaned": [], "compat_islands_total": 0, "compat_islands_orphaned": [], + "build_artifacts_total": 0, + "build_artifacts_orphaned": [], "bytes_freed_or_eligible": 0, "registry_missing": true, "registry_corrupt": false, diff --git a/tests/workflows/tests/snapshots/native_build_cache__native_build_cache_hit_metrics.snap b/tests/workflows/tests/snapshots/native_build_cache__native_build_cache_hit_metrics.snap new file mode 100644 index 00000000..b3f350c2 --- /dev/null +++ b/tests/workflows/tests/snapshots/native_build_cache__native_build_cache_hit_metrics.snap @@ -0,0 +1,27 @@ +--- +source: tests/workflows/tests/native_build_cache.rs +expression: envelope +--- +{ + "success": true, + "built": 1, + "failed": 0, + "build_cache": { + "eligible": 1, + "hits": 1, + "misses": 0, + "bypassed": 0, + "local_state_hits": 0, + "scripts_avoided": 1, + "restored_bytes": "", + "lifecycle_ms_avoided": "", + "timings_ms": { + "preparation": "", + "key": "", + "lookup": "", + "restore": "", + "rematerialize": "", + "publish": "" + } + } +}