From fa9e69f6a9c95c73904e6a7316bfbec7293d0b1a Mon Sep 17 00:00:00 2001 From: Douglas Jarquin <8209+douglasjarquin@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:56:22 -0400 Subject: [PATCH 1/3] fix(benchmark): execute the frozen Go helper reference honestly --- docs/features/coordination.md | 2 +- docs/go-helper-prototype.md | 17 +++++++--- scripts/benchmark_fixture.py | 12 +++---- scripts/benchmark_go.py | 27 +++++++++++++--- tests/test_benchmark_go.py | 61 +++++++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 17 deletions(-) create mode 100644 tests/test_benchmark_go.py diff --git a/docs/features/coordination.md b/docs/features/coordination.md index 759bd32..ea378f0 100644 --- a/docs/features/coordination.md +++ b/docs/features/coordination.md @@ -14,6 +14,6 @@ Journeys the coordinator, workers, and developers take through `sumctl` and the | `live.herdr-smoke` | A named lab Herdr session runs a real dispatch, question, and refresh | manual: `mise run test-live` on a host with Herdr 0.9.0 | operator report | | `live.harness-canary` | An authenticated harness worker reads its brief and calls `sumctl ask` and `sumctl report` | manual: `docs/ACCEPTANCE.md` section 2 | operator report | | `performance.hot-path-baseline` | Opt-in local metrics and `mise run benchmark` measure production helper entrypoints in fake and isolated real-Herdr labs before any rewrite decision | automated: `tests/test_benchmark.py` | `lib/sum_measure.py`, `scripts/benchmark.py`, `scripts/benchmark_fixture.py`, `scripts/benchmark_real.py`, `scripts/benchmark_report.py`, `mise-tasks/benchmark`, and retained `benchmarks/issue-37/raw.json`, `benchmarks/issue-37/report.md`, `benchmarks/issue-37/real-herdr.log` | -| `performance.go-cobra-prototype` | A cgo-free compiled Cobra entrypoint keeps the frozen Python helper as an explicit compatibility boundary while measuring native version/help and representative fixture dispatch | automated: `go/internal/cli/root_test.go`, `go/cmd/sumctl-go/main_test.go` | `docs/go-helper-prototype.md`, `go/go.mod`, `go/go.sum`, `go/internal/cli/root.go`, `go/cmd/sumctl-go/main.go`, `scripts/benchmark_go.py`, and benchmark JSON | +| `performance.go-cobra-prototype` | The compiled Cobra benchmark executes a frozen Python snapshot for compatibility workloads and distinguishes exit-code smoke checks from unproven behavior parity | automated: `go/internal/cli/root_test.go`, `go/cmd/sumctl-go/main_test.go`, `tests/test_benchmark_go.py` | `docs/go-helper-prototype.md`, `go/go.mod`, `go/go.sum`, `go/internal/cli/root.go`, `go/cmd/sumctl-go/main.go`, `scripts/benchmark_go.py`, `scripts/benchmark_fixture.py`, and benchmark JSON | | `packaging.native-companion` | Setup and immutable release staging build a cgo-free native companion beside the existing Python/Node runtime, record dependency provenance, and reuse installed artifacts without replacing them | automated: `tests/test_packaging.py`, `tests/test_native_packaging.py`, `tests/test_core.py` | `docs/dependency-inventory.json`, `docs/DEPENDENCIES.md`, `scripts/setup.py`, `lib/sumctl.py`, `mise.toml`, `bin/herdr-mesh-go`, `go/cmd/herdr-mesh/main.go`, `go/internal/mesh/auth.go`, `go/internal/mesh/cli.go`, `go/internal/mesh/config.go`, `go/internal/mesh/helpers.go`, `go/internal/mesh/inputs.go`, `go/internal/mesh/mesh_test.go`, `go/internal/mesh/protocol.go`, `go/internal/mesh/protocol_test.go`, `go/internal/mesh/runner.go`, `go/internal/mesh/service.go`, `go/internal/mesh/service_test.go`, and the staged release manifest | | `skills.namespace` | Sum-owned skills use one canonical `sum-*` identity across source directories, frontmatter, projections, generated references, and immutable release compatibility paths; collisions are refused without replacing a working projection | automated: `tests/test_skill_namespace.py`, `tests/test_core.py` | offline suite | diff --git a/docs/go-helper-prototype.md b/docs/go-helper-prototype.md index 45211c5..c520310 100644 --- a/docs/go-helper-prototype.md +++ b/docs/go-helper-prototype.md @@ -1,6 +1,6 @@ # Go helper prototype -This prototype freezes the Python reference at `b03b8020621e0d417906402a5c7ecc5d63192541`. +The benchmark freezes the Python reference at `b03b8020621e0d417906402a5c7ecc5d63192541`. The production entrypoint remains `bin/sumctl` and continues to run `lib/sumctl.py`. @@ -9,7 +9,7 @@ The compiled prototype is `go/cmd/sumctl-go` and uses Cobra `v1.9.1` for command The prototype's retained native behavior is: - `--version` prints the reference-compatible `sum 0.1.0` value without opening state or invoking another process. -- Root `--help` is rendered by the fresh Cobra tree when no reference helper is configured, and forwards to the frozen reference when one is available so the command inventory and help text stay aligned. +- Root `--help` is rendered by the fresh Cobra tree when no reference helper is configured, and forwards to the configured reference when one is available. - Command construction, root help, unknown commands, context cancellation, repeated options, leading-dash values, `--`, and compatibility exit codes are covered by `go/internal/cli/root_test.go`. The following existing command paths are present in the Cobra tree and remain explicit Python compatibility commands in this prototype: @@ -18,7 +18,8 @@ The following existing command paths are present in the Cobra tree and remain ex No stateful domain operation is claimed as ported. -The compatibility boundary forwards the original argument tokens to `bin/sumctl` with `exec.CommandContext`, preserving stdout, stderr, exit status, cancellation, environment, and subprocess ordering. +The compatibility boundary forwards argument tokens to the helper selected by `SUM_PYTHON_HELPER`, or the working directory's `bin/sumctl`, with `exec.CommandContext`. +The Go tests exercise forwarding with shell fixtures; they do not establish differential parity against the Python implementation. The benchmark command is: @@ -28,10 +29,16 @@ python3 scripts/benchmark_go.py --binary /tmp/sumctl-go --output /tmp/sum-go-ben ``` The benchmark includes compiled version and help paths plus representative fixture reads and an expected failure through the compatibility boundary. +Before timing, it extracts the frozen commit from local Git into a temporary directory and uses that source for both fixture setup and compatibility calls. +The reference commit must exist locally; the benchmark does not download it or use an installed runtime in its place. +The temporary source and fixture are removed when the run exits. -It records binary size, latency, peak memory, exit codes, subprocess counts, allocation results, and the exact reference revision. +It records binary size, latency, peak memory, exit codes, declared compatibility-subprocess counts, allocation results, and the reference revision and helper path. +The subprocess counts describe the wrapper boundary, not an observed total of nested Python, Git, or Herdr processes. -The JSON also evaluates the #37 latency, frequency, behavior, and memory gates and records the explicit `defer` decision with its comparison inputs. +The JSON evaluates the #37 latency gate and records the explicit `defer` decision with its comparison inputs. +Frequency-weighted benefit remains unestablished, behavior parity is not evaluated, and compatibility-process memory is not comparable. +Expected exit codes are smoke checks, not evidence of zero behavior regressions. The #37 gate remains 50 ms and 35% on a measured interactive hot path, 500 ms of serial frequency-weighted opportunity, zero behavior regressions, and at most 10% peak-memory regression. diff --git a/scripts/benchmark_fixture.py b/scripts/benchmark_fixture.py index e6c7209..b9ac4e3 100644 --- a/scripts/benchmark_fixture.py +++ b/scripts/benchmark_fixture.py @@ -25,7 +25,7 @@ def __str__(self) -> str: return self.detail -def clean_environment(base: Path) -> dict[str, str]: +def clean_environment(base: Path, source_root: Path = ROOT) -> dict[str, str]: env = {key: value for key, value in os.environ.items() if not key.startswith(("SUM_", "HERDR_"))} env.update({ "HOME": str(base / "home"), @@ -34,9 +34,9 @@ def clean_environment(base: Path) -> dict[str, str]: "HERDR_PANE_ID": "w-parent:p1", "HERDR_SESSION": "sum-benchmark", "SUM_SESSION": "sum-benchmark", - "SUM_HERDR_BIN": str(FAKE_HERDR), + "SUM_HERDR_BIN": str(source_root / "tests" / "fixtures" / "herdr.py"), "FAKE_HERDR_ROOT": str(base / "fake"), - "FAKE_PARENT_CWD": str(ROOT), + "FAKE_PARENT_CWD": str(source_root), "FAKE_SESSION": "sum-benchmark", }) Path(env["HOME"]).mkdir(parents=True, exist_ok=True) @@ -106,8 +106,8 @@ def git_repo(path: Path) -> None: subprocess.run(["git", "-C", str(path), "commit", "-q", "-m", "fixture"], check=True) -def fixture(base: Path): - env = clean_environment(base) +def fixture(base: Path, source_root: Path = ROOT): + env = clean_environment(base, source_root) repo = base / "repo" git_repo(repo) brief = base / "brief.md" @@ -115,7 +115,7 @@ def fixture(base: Path): home = base / "state" home.mkdir(mode=0o700) (home / "state.json").write_text('{"schema":1,"sum_version":"0.1.0","created_at":"2026-09-07T00:00:00+00:00"}\n', encoding="utf-8") - prefix = [SUMCTL, "--home", home] + prefix = [source_root / "bin" / "sumctl", "--home", home] run_plain([*prefix, "init"], env) run_plain([*prefix, "settings", "set", "--global", "64", "--per-repository", "64"], env) task = run_plain([*prefix, "dispatch", "--repo", repo, "--brief", brief, "--harness", "codex", "--approved"], env) diff --git a/scripts/benchmark_go.py b/scripts/benchmark_go.py index 4f6d338..5c2a68a 100644 --- a/scripts/benchmark_go.py +++ b/scripts/benchmark_go.py @@ -2,10 +2,12 @@ import argparse import json +import os from pathlib import Path import statistics import subprocess import tempfile +import tarfile import time import re @@ -13,6 +15,7 @@ ROOT = Path(__file__).resolve().parents[1] +REFERENCE_REVISION = "b03b8020621e0d417906402a5c7ecc5d63192541" BASELINE = { "startup.version.cold": 136.568, "startup.help.warm-fs": 138.560, @@ -21,6 +24,18 @@ } +def reference_snapshot(destination: Path) -> Path: + destination.mkdir() + env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + with tempfile.TemporaryFile() as archive: + subprocess.run(["git", "--no-replace-objects", "-C", str(ROOT), "archive", "--format=tar", REFERENCE_REVISION], + env=env, stdout=archive, check=True) + archive.seek(0) + with tarfile.open(fileobj=archive) as source: + source.extractall(destination, filter="data") + return destination + + def allocations() -> dict[str, object]: command = ["go", "test", "-run", "^$", "-bench", "BenchmarkNewRoot", "-benchmem", "./internal/cli"] result = subprocess.run(command, cwd=ROOT / "go", text=True, capture_output=True) @@ -58,7 +73,7 @@ def gate(scenarios: list[dict[str, object]], allocation: dict[str, object]) -> d "pass": any(row["absolute_improvement_ms"] >= 50 and row["relative_improvement_percent"] >= 35 for row in comparisons.values()), } frequency = {"required_ms": 500, "observed_ms": 0, "pass": False, "reason": "No stateful command is native; status remains a compatibility subprocess."} - behavior = {"required_regressions": 0, "observed_regressions": 0, "pass": True, "basis": "all compiled scenarios returned their declared exit codes"} + behavior = {"required_regressions": 0, "observed_regressions": None, "pass": False, "status": "not-evaluated", "basis": "Expected exit codes are smoke checks, not differential output or effect parity."} memory = {"required_regression_percent": 10, "pass": False, "status": "not-comparable", "reason": "The compatibility child is outside the compiled parent's /usr/bin/time memory sample."} return { "outcome": "defer", @@ -70,7 +85,7 @@ def gate(scenarios: list[dict[str, object]], allocation: dict[str, object]) -> d "memory": memory, "allocations": allocation, "binary_and_entrypoint": {"version": version["id"], "help": help_row["id"]}, - "reasons": ["The native startup/help path crosses the latency gate.", "No stateful command is native, so the 500 ms frequency-weighted gate is not established.", "Compatibility memory is not comparable to the Python child process."], + "reasons": ["The native startup/help path crosses the latency gate." if interactive["pass"] else "The native startup/help path does not cross the latency gate.", "No stateful command is native, so the 500 ms frequency-weighted gate is not established.", "Behavior parity has not been evaluated.", "Compatibility memory is not comparable to the Python child process."], } @@ -117,9 +132,10 @@ def main() -> int: raise SystemExit(f"binary does not exist: {binary}") with tempfile.TemporaryDirectory(prefix="sum-go-benchmark-") as temporary: base = Path(temporary) - case = fixture(base) + reference = reference_snapshot(base / "reference") + case = fixture(base, source_root=reference) env = dict(case["env"]) - env["SUM_PYTHON_HELPER"] = str(ROOT / "bin" / "sumctl") + env["SUM_PYTHON_HELPER"] = str(reference / "bin" / "sumctl") commands = [ ("startup.version.cold", [str(binary), "--version"], dict(env)), ("startup.help.cobra", [str(binary), "--help"], {**env, "SUM_PYTHON_HELPER": str(base / "missing-reference")}), @@ -139,7 +155,8 @@ def main() -> int: "schema": 1, "binary": str(binary), "binary_bytes": binary.stat().st_size, - "reference_revision": "b03b8020621e0d417906402a5c7ecc5d63192541", + "reference_revision": REFERENCE_REVISION, + "reference_helper": str(reference / "bin" / "sumctl"), "samples": args.samples, "scenarios": results, "scope": "compiled Cobra entrypoint; status and missing-show use the explicit Python compatibility boundary", diff --git a/tests/test_benchmark_go.py b/tests/test_benchmark_go.py new file mode 100644 index 0000000..e03d3d2 --- /dev/null +++ b/tests/test_benchmark_go.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +SPEC = importlib.util.spec_from_file_location("benchmark_go", ROOT / "scripts" / "benchmark_go.py") +benchmark = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(benchmark) + + +class GoBenchmarkEvidenceTest(unittest.TestCase): + def scenarios(self, latency=6): + return [{"id": name, "p50_ms": latency} for name in ( + "startup.version.cold", "startup.help.cobra", "read.status.fixture", "failure.show-missing", + )] + + def test_exit_codes_alone_do_not_establish_behavior_parity(self): + result = benchmark.gate(self.scenarios(), {}) + self.assertFalse(result["behavior"]["pass"]) + self.assertIsNone(result["behavior"]["observed_regressions"]) + self.assertEqual(result["behavior"]["status"], "not-evaluated") + + def test_failed_latency_gate_is_not_described_as_crossed(self): + result = benchmark.gate(self.scenarios(latency=1000), {}) + self.assertFalse(result["interactive_hot_path"]["pass"]) + self.assertNotIn("The native startup/help path crosses the latency gate.", result["reasons"]) + + def test_reference_snapshot_uses_frozen_bytes_not_working_tree(self): + with tempfile.TemporaryDirectory(prefix="sum-go-reference-test-") as temporary: + base = Path(temporary) + repo = base / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + subprocess.run(["git", "-C", str(repo), "config", "user.name", "fixture"], check=True) + subprocess.run(["git", "-C", str(repo), "config", "user.email", "fixture@example.invalid"], check=True) + helper = repo / "bin" / "sumctl" + helper.parent.mkdir() + helper.write_text("#!/bin/sh\nprintf frozen\n") + helper.chmod(0o755) + subprocess.run(["git", "-C", str(repo), "add", "."], check=True) + subprocess.run(["git", "-C", str(repo), "commit", "-qm", "frozen"], check=True) + revision = subprocess.check_output(["git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip() + helper.write_text("#!/bin/sh\nprintf changed\n") + with patch.object(benchmark, "ROOT", repo), patch.object(benchmark, "REFERENCE_REVISION", revision, create=True): + snapshot = benchmark.reference_snapshot(base / "reference") + result = subprocess.run([str(snapshot / "bin" / "sumctl")], capture_output=True, text=True, check=True) + self.assertEqual(result.stdout, "frozen") + self.assertEqual(helper.read_text(), "#!/bin/sh\nprintf changed\n") + self.assertFalse((snapshot / ".git").exists()) + + +if __name__ == "__main__": + unittest.main() From 9578b6ad9430710f1c421bc1d9c010f5cfd5bbf3 Mon Sep 17 00:00:00 2001 From: Douglas Jarquin <8209+douglasjarquin@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:17:39 -0400 Subject: [PATCH 2/3] refactor(helper): retire the unadopted Go prototype --- benchmarks/issue-38/raw.json | 227 +++++++++++++++++++++++++++++++++ benchmarks/issue-38/report.md | 74 +++++++++++ docs/DEPENDENCIES.md | 6 +- docs/dependency-inventory.json | 12 -- docs/features/coordination.md | 2 +- docs/go-helper-prototype.md | 58 ++------- go/cmd/sumctl-go/main.go | 46 ------- go/cmd/sumctl-go/main_test.go | 59 --------- go/internal/cli/root.go | 120 ----------------- go/internal/cli/root_test.go | 147 --------------------- lib/sumctl.py | 72 ++++++----- scripts/benchmark_fixture.py | 12 +- scripts/benchmark_go.py | 173 ------------------------- tests/test_benchmark_go.py | 61 --------- tests/test_core.py | 61 +++++++-- tests/test_native_packaging.py | 10 +- tests/test_packaging.py | 4 +- 17 files changed, 425 insertions(+), 719 deletions(-) create mode 100644 benchmarks/issue-38/raw.json create mode 100644 benchmarks/issue-38/report.md delete mode 100644 go/cmd/sumctl-go/main.go delete mode 100644 go/cmd/sumctl-go/main_test.go delete mode 100644 go/internal/cli/root.go delete mode 100644 go/internal/cli/root_test.go delete mode 100644 scripts/benchmark_go.py delete mode 100644 tests/test_benchmark_go.py diff --git a/benchmarks/issue-38/raw.json b/benchmarks/issue-38/raw.json new file mode 100644 index 0000000..f71c705 --- /dev/null +++ b/benchmarks/issue-38/raw.json @@ -0,0 +1,227 @@ +{ + "schema": 1, + "binary": "/Users/douglasjarquin/github/sum/.sum/dev/issue38-audit/.artifacts/issue38/sumctl-go", + "binary_bytes": 6109458, + "reference_revision": "b03b8020621e0d417906402a5c7ecc5d63192541", + "reference_helper": "/var/folders/5j/n805d0vn43g2bjt7ykqmyxmm0000gn/T/sum-go-benchmark-1a1wn4kb/reference/bin/sumctl", + "samples": 15, + "scenarios": [ + { + "command": [ + "/Users/douglasjarquin/github/sum/.sum/dev/issue38-audit/.artifacts/issue38/sumctl-go", + "--version" + ], + "samples": 15, + "p50_ms": 5.919, + "p95_ms": 16.432, + "min_ms": 5.293, + "max_ms": 16.432, + "exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "peak_memory_bytes": 3490224, + "subprocesses": 0, + "id": "startup.version.cold" + }, + { + "command": [ + "/Users/douglasjarquin/github/sum/.sum/dev/issue38-audit/.artifacts/issue38/sumctl-go", + "--help" + ], + "samples": 15, + "p50_ms": 5.801, + "p95_ms": 6.585, + "min_ms": 5.251, + "max_ms": 6.585, + "exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "peak_memory_bytes": 3523016, + "subprocesses": 0, + "id": "startup.help.cobra" + }, + { + "command": [ + "/Users/douglasjarquin/github/sum/.sum/dev/issue38-audit/.artifacts/issue38/sumctl-go", + "--home", + "/var/folders/5j/n805d0vn43g2bjt7ykqmyxmm0000gn/T/sum-go-benchmark-1a1wn4kb/state", + "status" + ], + "samples": 15, + "p50_ms": 133.056, + "p95_ms": 156.934, + "min_ms": 126.481, + "max_ms": 156.934, + "exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "peak_memory_bytes": 3555784, + "subprocesses": 1, + "id": "read.status.fixture" + }, + { + "command": [ + "/Users/douglasjarquin/github/sum/.sum/dev/issue38-audit/.artifacts/issue38/sumctl-go", + "--home", + "/var/folders/5j/n805d0vn43g2bjt7ykqmyxmm0000gn/T/sum-go-benchmark-1a1wn4kb/state", + "show", + "t-000000000000" + ], + "samples": 15, + "p50_ms": 132.928, + "p95_ms": 164.723, + "min_ms": 127.212, + "max_ms": 164.723, + "exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "peak_memory_bytes": 3572168, + "subprocesses": 1, + "id": "failure.show-missing", + "expected_exit": 1 + } + ], + "scope": "compiled Cobra entrypoint; status and missing-show use the explicit Python compatibility boundary", + "allocations": { + "command": [ + "go", + "test", + "-run", + "^$", + "-bench", + "BenchmarkNewRoot", + "-benchmem", + "./internal/cli" + ], + "exit": 0, + "ns_per_op": 6299.0, + "bytes_per_op": 33080.0, + "allocs_per_op": 133.0 + }, + "gate": { + "outcome": "defer", + "pass": false, + "comparisons": { + "startup.version.cold": { + "baseline_p50_ms": 136.568, + "candidate_p50_ms": 5.919, + "absolute_improvement_ms": 130.649, + "relative_improvement_percent": 95.67 + }, + "startup.help.cobra": { + "baseline_p50_ms": 138.56, + "candidate_p50_ms": 5.801, + "absolute_improvement_ms": 132.759, + "relative_improvement_percent": 95.81 + } + }, + "interactive_hot_path": { + "required_absolute_ms": 50, + "required_relative_percent": 35, + "observed_absolute_improvement_ms": 132.759, + "observed_relative_improvement_percent": 95.81, + "pass": true + }, + "frequency_weighted": { + "required_ms": 500, + "observed_ms": 0, + "pass": false, + "reason": "No stateful command is native; status remains a compatibility subprocess." + }, + "behavior": { + "required_regressions": 0, + "observed_regressions": null, + "pass": false, + "status": "not-evaluated", + "basis": "Expected exit codes are smoke checks, not differential output or effect parity." + }, + "memory": { + "required_regression_percent": 10, + "pass": false, + "status": "not-comparable", + "reason": "The compatibility child is outside the compiled parent's /usr/bin/time memory sample." + }, + "allocations": { + "command": [ + "go", + "test", + "-run", + "^$", + "-bench", + "BenchmarkNewRoot", + "-benchmem", + "./internal/cli" + ], + "exit": 0, + "ns_per_op": 6299.0, + "bytes_per_op": 33080.0, + "allocs_per_op": 133.0 + }, + "binary_and_entrypoint": { + "version": "startup.version.cold", + "help": "startup.help.cobra" + }, + "reasons": [ + "The native startup/help path crosses the latency gate.", + "No stateful command is native, so the 500 ms frequency-weighted gate is not established.", + "Behavior parity has not been evaluated.", + "Compatibility memory is not comparable to the Python child process." + ] + } +} diff --git a/benchmarks/issue-38/report.md b/benchmarks/issue-38/report.md new file mode 100644 index 0000000..7bf3f82 --- /dev/null +++ b/benchmarks/issue-38/report.md @@ -0,0 +1,74 @@ +# Go helper decision: discard the unused prototype + +Do not adopt the Go helper or plan a production cutover under issue #39. +The experiment demonstrates faster native version/help handling, but no stateful operation moved to Go and the required adoption gate is not met. +Production remains on the existing Python helper, and the separately delivered Go Herdr Mesh remains available. + +## Measured candidate + +The corrected driver and recoverable experiment are at commit `fa9e69f6a9c95c73904e6a7316bfbec7293d0b1a`. +Its Go source is unchanged from the compiled source at `d9d0bd210aa06d4dd87215460a642e1a4a2979aa`. +The Python reference is `b03b8020621e0d417906402a5c7ecc5d63192541`, extracted from local Git and used for fixture setup and compatibility execution. +The build used Go 1.25.0 with `CGO_ENABLED=0`, `GOENV=off`, `GOTOOLCHAIN=local`, `-trimpath`, and `-buildvcs=false`. +The runtime used Python 3.13.5 on macOS arm64. +No model, native Herdr pane, live installation state, or installed runtime was used. + +The executable was 6,109,458 bytes with SHA-256 `1faf6062b68d2c9d3d06557f8f4e519e90e6169fe13d4bbd0195a2ffd6de66c4`. +The retained [raw result](raw.json) is the exact 15-sample output of the corrected benchmark, with SHA-256 `6aeb8054523f69d7e58a30236e41c7759c1c43a1e60663546ec75831aebef9a5`. +Recorded temporary paths identify the measured run; those fixtures were removed after it completed. + +| Compiled entrypoint | Median | Implementation | +| --- | ---: | --- | +| `--version` | 5.919 ms | Native Go | +| Native root `--help` | 5.801 ms | Native Cobra rendering, not Python help parity | +| Fixture `status` | 133.056 ms | Python compatibility subprocess | +| Missing-task `show` | 132.928 ms | Python compatibility subprocess, expected exit 1 | + +Fresh Cobra tree construction measured 6,299 ns/op, 33,080 B/op, and 133 allocations/op. +The reported subprocess counts are declared compatibility-boundary counts, not measured totals of nested Python, Git, or Herdr processes. +The memory samples omit the compatibility child's peak usage and cannot establish a whole-operation memory regression bound. + +## Unchanged adoption gate + +The [issue #37 baseline](../issue-37/report.md) and its thresholds were not rerun or altered. +The required gate remains at least 50 ms and 35% improvement on a measured interactive path, at least 500 ms of serial frequency-weighted opportunity, zero behavior regressions, and at most 10% peak-memory regression. + +- Native version/help cross the local latency threshold against the recorded baseline. +- No stateful command is native, so the required frequency-weighted opportunity is not established. +- Expected exit codes are smoke checks, not differential output, state, role, durability, or cancellation parity against Python. +- Whole-operation memory is not comparable. + +The gate therefore fails; this is a discard decision, not a claimed successful port. +No command is moved to Go, so no new implementation is accepted under a weaker parity standard. +All domain operations remain in their existing implementation. + +## Evidence correction + +The earlier driver labeled the Python reference with the frozen commit but invoked the working checkout's helper. +A trace of the actual compiled benchmark observed configured source hash `0391d1dc8cbe8f7b2c06d3ae03e5ba289479c1ac547f761cca378fb3120797fa`, differing from the claimed reference hash `bba3eb2d171da1dfd5193e1ade4f0fb4482c0383856b5f969d21065462ac90da`. +After the repair at `fa9e69f`, the same trace observed the latter hash for the configured helper, and the behavior verdict changed from an unsupported pass to `not-evaluated`. +Earlier labeled results are not used as frozen-reference evidence here. + +## Reproducing the historical experiment + +Use an isolated checkout of `fa9e69f6a9c95c73904e6a7316bfbec7293d0b1a` with the pinned Go and Python tools already available. +The repository must contain the frozen Python commit locally. +Run these commands in that historical checkout, with an owned temporary output directory replacing `/tmp/sum38-reproduction`: + +```sh +mkdir /tmp/sum38-reproduction +(cd go && env -u GOROOT -u GOBIN -u GOTOOLDIR -u GOOS -u GOARCH CGO_ENABLED=0 GOENV=off GOTOOLCHAIN=local GOPROXY=off go build -trimpath -buildvcs=false -o /tmp/sum38-reproduction/sumctl-go ./cmd/sumctl-go) +python3 scripts/benchmark_go.py --binary /tmp/sum38-reproduction/sumctl-go --output /tmp/sum38-reproduction/raw.json --samples 15 +``` + +The driver extracts and cleans its own temporary reference and fake-Herdr fixture. +Retain the output if needed for comparison and remove only the owned reproduction directory afterward. +Timing varies by host and run; this is reproducibility of the experiment and decision inputs, not bit-identical performance or binary output. + +## Delivery boundary + +The retirement removes the unused helper implementation and its current-tree benchmark driver from new source releases. +New staging retains the independently used Go Mesh artifact and its integrity checks. +Historical releases are checked against their own artifact inventory; this does not delete old installed binaries or change a running process. +Issue #39 is deferred/not planned because its prerequisite demonstrated helper win is absent. +This decision does not block Remainder, Pinchos, or other roadmap work, and it does not authorize installation updates or fleet restarts. diff --git a/docs/DEPENDENCIES.md b/docs/DEPENDENCIES.md index bf58ea7..51974d7 100644 --- a/docs/DEPENDENCIES.md +++ b/docs/DEPENDENCIES.md @@ -17,15 +17,17 @@ The SDK's reviewed transitive graph remains visible in `go.sum`; no Viper, gener 1. Creates local runtime symlinks under `.local/bin`, once. An existing link is never retargeted, because a running process may depend on it; setup reports a differing pin instead. 2. Clones Herdr Mesh at **54adef519aa6af4dcd0bbd72586d414abab90046** into a private staging directory, runs `npm ci --omit=dev --ignore-scripts` against the upstream committed lockfile there, applies the documented runtime overlay below, and renames the finished tree to `.deps/herdr-mesh`. An existing `.deps/herdr-mesh` is never rewritten, reinstalled, or re-patched; drift between its overlay and the current `patches/` is only reported. 3. Copies the release-matched Herdr skill from `herdr --skill`. -4. Builds the cgo-free `go/cmd/sumctl-go` companion as `.local/bin/sumctl-go`; it is not selected by `bin/sumctl` and does not replace the Python helper. +4. Builds the cgo-free `go/cmd/herdr-mesh` companion as `.local/bin/herdr-mesh-go`; its opt-in launcher is `bin/herdr-mesh-go`, and the existing Node Mesh entrypoint remains available. 5. Generates repository-local MCP settings and tests MCP initialization/discovery. Re-running setup is therefore safe while a coordinator, workers, or an MCP server are using the checkout; it changes nothing they hold open. Newer code or dependencies go into a staged release instead (below). The native companion is also built in a release staging directory with `CGO_ENABLED=0`. -The staged binary's source, build requirements, runtime requirements, and SHA-256 are recorded in `release.json` under `dependencies.native.sumctl-go`. +The staged binary's source, build requirements, runtime requirements, and SHA-256 are recorded in `release.json` under `dependencies.native.herdr-mesh-go`. Running it requires no Go toolchain, module download, Node, Python, or Cobra generator. +The unused Go helper experiment is [not adopted](go-helper-prototype.md); `bin/sumctl` remains the Python entrypoint. +Native artifact requirements come from each release's own dependency inventory, so a missing or corrupt declared artifact is refused without making a retired experiment mandatory for new bundles. The source revision and upstream lockfile are pinned. This does not claim bit-for-bit reproducibility of every OS/runtime installation. A mise lockfile has not been invented; generate/review it on a networked machine when updating dependency pins. diff --git a/docs/dependency-inventory.json b/docs/dependency-inventory.json index 722eda2..d238d4a 100644 --- a/docs/dependency-inventory.json +++ b/docs/dependency-inventory.json @@ -134,18 +134,6 @@ "owner": "Model Context Protocol", "contracts": {"cli": [], "mcp": ["MCP 2025-11-25 compatibility"]} }, - { - "id": "sumctl-go", - "source": "go/cmd/sumctl-go", - "version": "sum 0.1.0", - "checksum": "release.json#dependencies.native.sumctl-go.sha256", - "license": "MIT", - "platforms": ["darwin-arm64", "darwin-amd64", "linux-arm64", "linux-amd64"], - "requirements": ["setup/release staging", "CGO_ENABLED=0", "no runtime toolchain"], - "role": "runtime", - "owner": "sum", - "contracts": {"cli": ["sumctl compatibility argv/stdout/stderr"], "mcp": []} - }, { "id": "herdr-mesh-go", "source": "go/cmd/herdr-mesh", diff --git a/docs/features/coordination.md b/docs/features/coordination.md index ea378f0..c118c1a 100644 --- a/docs/features/coordination.md +++ b/docs/features/coordination.md @@ -14,6 +14,6 @@ Journeys the coordinator, workers, and developers take through `sumctl` and the | `live.herdr-smoke` | A named lab Herdr session runs a real dispatch, question, and refresh | manual: `mise run test-live` on a host with Herdr 0.9.0 | operator report | | `live.harness-canary` | An authenticated harness worker reads its brief and calls `sumctl ask` and `sumctl report` | manual: `docs/ACCEPTANCE.md` section 2 | operator report | | `performance.hot-path-baseline` | Opt-in local metrics and `mise run benchmark` measure production helper entrypoints in fake and isolated real-Herdr labs before any rewrite decision | automated: `tests/test_benchmark.py` | `lib/sum_measure.py`, `scripts/benchmark.py`, `scripts/benchmark_fixture.py`, `scripts/benchmark_real.py`, `scripts/benchmark_report.py`, `mise-tasks/benchmark`, and retained `benchmarks/issue-37/raw.json`, `benchmarks/issue-37/report.md`, `benchmarks/issue-37/real-herdr.log` | -| `performance.go-cobra-prototype` | The compiled Cobra benchmark executes a frozen Python snapshot for compatibility workloads and distinguishes exit-code smoke checks from unproven behavior parity | automated: `go/internal/cli/root_test.go`, `go/cmd/sumctl-go/main_test.go`, `tests/test_benchmark_go.py` | `docs/go-helper-prototype.md`, `go/go.mod`, `go/go.sum`, `go/internal/cli/root.go`, `go/cmd/sumctl-go/main.go`, `scripts/benchmark_go.py`, `scripts/benchmark_fixture.py`, and benchmark JSON | +| `performance.go-cobra-prototype` | The helper experiment is retired after a measured no-go; new builds retain Go Mesh without building the unused helper, while the decision and corrected measurements remain available | automated: `tests/test_native_packaging.py`, `tests/test_packaging.py` | `docs/go-helper-prototype.md`, `benchmarks/issue-38/report.md`, `benchmarks/issue-38/raw.json`, `go/go.mod`, `go/go.sum`, `lib/sumctl.py`; retired source locations under `go/cmd`, `go/internal`, and `scripts/` are covered by this removal | | `packaging.native-companion` | Setup and immutable release staging build a cgo-free native companion beside the existing Python/Node runtime, record dependency provenance, and reuse installed artifacts without replacing them | automated: `tests/test_packaging.py`, `tests/test_native_packaging.py`, `tests/test_core.py` | `docs/dependency-inventory.json`, `docs/DEPENDENCIES.md`, `scripts/setup.py`, `lib/sumctl.py`, `mise.toml`, `bin/herdr-mesh-go`, `go/cmd/herdr-mesh/main.go`, `go/internal/mesh/auth.go`, `go/internal/mesh/cli.go`, `go/internal/mesh/config.go`, `go/internal/mesh/helpers.go`, `go/internal/mesh/inputs.go`, `go/internal/mesh/mesh_test.go`, `go/internal/mesh/protocol.go`, `go/internal/mesh/protocol_test.go`, `go/internal/mesh/runner.go`, `go/internal/mesh/service.go`, `go/internal/mesh/service_test.go`, and the staged release manifest | | `skills.namespace` | Sum-owned skills use one canonical `sum-*` identity across source directories, frontmatter, projections, generated references, and immutable release compatibility paths; collisions are refused without replacing a working projection | automated: `tests/test_skill_namespace.py`, `tests/test_core.py` | offline suite | diff --git a/docs/go-helper-prototype.md b/docs/go-helper-prototype.md index c520310..d321e31 100644 --- a/docs/go-helper-prototype.md +++ b/docs/go-helper-prototype.md @@ -1,49 +1,19 @@ -# Go helper prototype +# Go helper prototype: not adopted -The benchmark freezes the Python reference at `b03b8020621e0d417906402a5c7ecc5d63192541`. +The unused Go helper experiment is retired; production `bin/sumctl` continues to run `lib/sumctl.py`. +No command, state operation, callback, or running process moves to a new implementation. -The production entrypoint remains `bin/sumctl` and continues to run `lib/sumctl.py`. +The [decision report and reproduction instructions](../benchmarks/issue-38/report.md) preserve the corrected measurements and the exact historical experiment revision. +The [raw benchmark](../benchmarks/issue-38/raw.json) records the frozen Python reference, actual compiled invocations, timings, allocation results, and unmet gates. -The compiled prototype is `go/cmd/sumctl-go` and uses Cobra `v1.9.1` for command construction, help, version handling, and the outer exit boundary. +Native version/help handling was faster, but every stateful operation still invoked Python. +The required frequency-weighted opportunity was not established, behavior parity was not evaluated, and whole-operation memory was not comparable. +Expected exit codes did not prove zero behavior regressions. +The unchanged issue #37 thresholds therefore do not justify adoption. -The prototype's retained native behavior is: +Issue #39 is deferred/not planned because its prerequisite demonstrated helper win is absent. +This is a measured no-go, not an incomplete runtime replacement presented as complete. -- `--version` prints the reference-compatible `sum 0.1.0` value without opening state or invoking another process. -- Root `--help` is rendered by the fresh Cobra tree when no reference helper is configured, and forwards to the configured reference when one is available. -- Command construction, root help, unknown commands, context cancellation, repeated options, leading-dash values, `--`, and compatibility exit codes are covered by `go/internal/cli/root_test.go`. - -The following existing command paths are present in the Cobra tree and remain explicit Python compatibility commands in this prototype: - -`doctor`, `init`, `status`, `inbox`, `prepare`, `dispatch`, `start`, `help`, `context`, `notes`, `env`, `show`, `notice`, `archive`, `ask`, `answer`, `report`, `resolve`, `review`, `verify`, `pr`, `cleanup`, `pump`, `hook`, `metadata`, `attention`, `bind`, `backup`, `settings`, `preset`, `project`, `herdr`, `graph`, `dev`, `brief`, `refresh`, `release`, and `update`. - -No stateful domain operation is claimed as ported. - -The compatibility boundary forwards argument tokens to the helper selected by `SUM_PYTHON_HELPER`, or the working directory's `bin/sumctl`, with `exec.CommandContext`. -The Go tests exercise forwarding with shell fixtures; they do not establish differential parity against the Python implementation. - -The benchmark command is: - -```sh -(cd go && go build -trimpath -buildvcs=false -o /tmp/sumctl-go ./cmd/sumctl-go) -python3 scripts/benchmark_go.py --binary /tmp/sumctl-go --output /tmp/sum-go-benchmark.json -``` - -The benchmark includes compiled version and help paths plus representative fixture reads and an expected failure through the compatibility boundary. -Before timing, it extracts the frozen commit from local Git into a temporary directory and uses that source for both fixture setup and compatibility calls. -The reference commit must exist locally; the benchmark does not download it or use an installed runtime in its place. -The temporary source and fixture are removed when the run exits. - -It records binary size, latency, peak memory, exit codes, declared compatibility-subprocess counts, allocation results, and the reference revision and helper path. -The subprocess counts describe the wrapper boundary, not an observed total of nested Python, Git, or Herdr processes. - -The JSON evaluates the #37 latency gate and records the explicit `defer` decision with its comparison inputs. -Frequency-weighted benefit remains unestablished, behavior parity is not evaluated, and compatibility-process memory is not comparable. -Expected exit codes are smoke checks, not evidence of zero behavior regressions. - -The #37 gate remains 50 ms and 35% on a measured interactive hot path, 500 ms of serial frequency-weighted opportunity, zero behavior regressions, and at most 10% peak-memory regression. - -The prototype decision is defer production adoption. - -The native `--version` path demonstrates a process-local reduction, but the measured stateful workloads still use the compatibility subprocess and therefore do not establish the required end-to-end gate. - -Issue #39 owns any production cutover after an independently verified native state boundary demonstrates the gate. +The independently delivered [Go Herdr Mesh](go-mesh.md) remains available and retains Cobra and the official MCP Go SDK. +New releases build that artifact, not the unused helper. +Existing immutable releases and running processes are not removed or retargeted by this source change. diff --git a/go/cmd/sumctl-go/main.go b/go/cmd/sumctl-go/main.go deleted file mode 100644 index 3b83729..0000000 --- a/go/cmd/sumctl-go/main.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "os/signal" - "path/filepath" - "syscall" - - "github.com/douglasjarquin/sum/go/internal/cli" -) - -func main() { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - - root := cli.NewRoot(referenceHelper(), os.Stdout, os.Stderr) - if err := root.ExecuteContext(ctx); err != nil { - var exitErr *cli.ExitError - if errors.As(err, &exitErr) { - os.Exit(exitErr.Code) - } - payload, marshalErr := json.Marshal(map[string]string{"error": err.Error()}) - if marshalErr != nil { - fmt.Fprintln(os.Stderr, `{"error":"sumctl-go failed"}`) - } else { - fmt.Fprintln(os.Stderr, string(payload)) - } - os.Exit(1) - } -} - -func referenceHelper() string { - if value := os.Getenv("SUM_PYTHON_HELPER"); value != "" { - return value - } - if value, err := filepath.Abs("bin/sumctl"); err == nil { - if _, statErr := os.Stat(value); statErr == nil { - return value - } - } - return "" -} diff --git a/go/cmd/sumctl-go/main_test.go b/go/cmd/sumctl-go/main_test.go deleted file mode 100644 index 88a36d6..0000000 --- a/go/cmd/sumctl-go/main_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package main - -import ( - "bytes" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "testing" - "time" -) - -func TestCompiledEntrypointCancellationExitsOnce(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("signal semantics differ on Windows") - } - dir := t.TempDir() - binary := filepath.Join(dir, "sumctl-go") - build := exec.Command("go", "build", "-trimpath", "-buildvcs=false", "-o", binary, ".") - build.Dir = "." - if output, err := build.CombinedOutput(); err != nil { - t.Fatalf("go build failed: %v\n%s", err, output) - } - reference := filepath.Join(dir, "reference.sh") - marker := filepath.Join(dir, "started") - if err := os.WriteFile(reference, []byte("#!/bin/sh\nprintf '%s' $$ > \"$SUM_GO_MARKER\"\nsleep 30\n"), 0o700); err != nil { - t.Fatal(err) - } - command := exec.Command(binary, "status") - command.Env = append(os.Environ(), "SUM_PYTHON_HELPER="+reference, "SUM_GO_MARKER="+marker) - var stdout, stderr bytes.Buffer - command.Stdout = &stdout - command.Stderr = &stderr - if err := command.Start(); err != nil { - t.Fatal(err) - } - deadline := time.Now().Add(2 * time.Second) - for { - if _, err := os.Stat(marker); err == nil { - break - } - if time.Now().After(deadline) { - _ = command.Process.Kill() - t.Fatal("reference helper did not start") - } - time.Sleep(10 * time.Millisecond) - } - if err := command.Process.Signal(os.Interrupt); err != nil { - t.Fatal(err) - } - err := command.Wait() - if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { - t.Fatalf("compiled exit = %v, want status 1", err) - } - if stdout.Len() != 0 || !strings.Contains(stderr.String(), `"error":"context canceled"`) { - t.Fatalf("compiled cancellation output stdout=%q stderr=%q", stdout.String(), stderr.String()) - } -} diff --git a/go/internal/cli/root.go b/go/internal/cli/root.go deleted file mode 100644 index b77241f..0000000 --- a/go/internal/cli/root.go +++ /dev/null @@ -1,120 +0,0 @@ -package cli - -import ( - "context" - "fmt" - "io" - "os" - "os/exec" - - "github.com/spf13/cobra" -) - -const Version = "sum 0.1.0" - -type ExitError struct { - Code int -} - -func (e *ExitError) Error() string { - return fmt.Sprintf("reference helper exited with status %d", e.Code) -} - -type rootOptions struct { - reference string - home string - homeSet bool - out io.Writer - err io.Writer -} - -func NewRoot(reference string, out, errOut io.Writer) *cobra.Command { - if out == nil { - out = io.Discard - } - if errOut == nil { - errOut = io.Discard - } - opts := &rootOptions{reference: reference, out: out, err: errOut} - root := &cobra.Command{ - Use: "sumctl", - Short: "Small, synchronous helpers for sum", - Version: Version, - SilenceErrors: true, - SilenceUsage: true, - DisableSuggestions: true, - TraverseChildren: true, - Args: cobra.NoArgs, - RunE: func(*cobra.Command, []string) error { - return fmt.Errorf("command is required") - }, - } - root.SetOut(out) - root.SetErr(errOut) - root.SetVersionTemplate("{{.Version}}\n") - root.CompletionOptions.DisableDefaultCmd = true - root.Flags().StringVar(&opts.home, "home", "", "state home") - root.Flags().Lookup("home").NoOptDefVal = "" - root.PersistentPreRun = func(cmd *cobra.Command, _ []string) { - opts.homeSet = root.Flags().Changed("home") - } - root.SetHelpCommand(nil) - root.SetHelpFunc(func(cmd *cobra.Command, _ []string) { - if opts.reference != "" { - if _, statErr := os.Stat(opts.reference); statErr == nil { - if err := opts.compat(cmd.Context(), []string{"--help"}); err != nil { - fmt.Fprintln(cmd.ErrOrStderr(), err) - } - return - } - } - _, _ = io.WriteString(cmd.OutOrStdout(), cmd.UsageString()) - }) - - for _, name := range compatibilityCommands { - command := &cobra.Command{ - Use: name, - DisableFlagParsing: true, - Args: cobra.ArbitraryArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return opts.compat(cmd.Context(), append([]string{cmd.Name()}, args...)) - }, - } - root.AddCommand(command) - } - return root -} - -var compatibilityCommands = []string{ - "doctor", "init", "status", "inbox", "prepare", "dispatch", "start", "help", "context", "notes", "env", "show", "notice", "archive", "ask", "answer", "report", "resolve", "review", "verify", "pr", "cleanup", "pump", "hook", "metadata", "attention", "bind", "backup", "settings", "preset", "project", "herdr", "graph", "dev", "brief", "refresh", "release", "update", -} - -func (o *rootOptions) compat(ctx context.Context, args []string) error { - if o.reference == "" { - return fmt.Errorf("sumctl reference helper is not configured") - } - argv := append([]string(nil), args...) - argv = normalizeHome(argv, o.home, o.homeSet) - command := exec.CommandContext(ctx, o.reference, argv...) - command.Stdin = os.Stdin - command.Stdout = o.out - command.Stderr = o.err - if err := command.Run(); err != nil { - if ctx.Err() != nil { - return ctx.Err() - } - if exit, ok := err.(*exec.ExitError); ok { - return &ExitError{Code: exit.ExitCode()} - } - return err - } - return nil -} - -func normalizeHome(args []string, home string, homeSet bool) []string { - var prefix []string - if homeSet { - prefix = append([]string{"--home", home}, prefix...) - } - return append(prefix, args...) -} diff --git a/go/internal/cli/root_test.go b/go/internal/cli/root_test.go deleted file mode 100644 index 73b066c..0000000 --- a/go/internal/cli/root_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package cli - -import ( - "bytes" - "context" - "io" - "os" - "path/filepath" - "testing" - "time" -) - -func BenchmarkNewRoot(b *testing.B) { - b.ReportAllocs() - for range b.N { - NewRoot("", io.Discard, io.Discard) - } -} - -func TestHelpBuildsWithoutTouchingReferenceOrHome(t *testing.T) { - var stdout, stderr bytes.Buffer - root := NewRoot("/path/that/must/not/be opened", &stdout, &stderr) - root.SetArgs([]string{"--help"}) - - if err := root.ExecuteContext(context.Background()); err != nil { - t.Fatalf("help failed: %v", err) - } - if stdout.Len() == 0 { - t.Fatal("help produced no output") - } -} - -func TestUnknownCommandDoesNotInvokeReference(t *testing.T) { - dir := t.TempDir() - marker := filepath.Join(dir, "invoked") - reference := filepath.Join(dir, "reference.sh") - if err := os.WriteFile(reference, []byte("#!/bin/sh\ntouch \"$SUM_GO_MARKER\"\n"), 0o700); err != nil { - t.Fatal(err) - } - t.Setenv("SUM_GO_MARKER", marker) - root := NewRoot(reference, &bytes.Buffer{}, &bytes.Buffer{}) - root.SetArgs([]string{"not-a-command"}) - if err := root.ExecuteContext(context.Background()); err == nil { - t.Fatal("unknown command unexpectedly succeeded") - } - if _, err := os.Stat(marker); !os.IsNotExist(err) { - t.Fatalf("reference marker exists after invalid invocation: %v", err) - } -} - -func TestCommandTreesDoNotLeakFlags(t *testing.T) { - first := NewRoot("", &bytes.Buffer{}, &bytes.Buffer{}) - second := NewRoot("", &bytes.Buffer{}, &bytes.Buffer{}) - - if first == second { - t.Fatal("NewRoot returned a shared command tree") - } - if first.PersistentFlags() == second.PersistentFlags() { - t.Fatal("NewRoot returned shared flag state") - } -} - -func TestCompatibilityCommandPreservesLeadingDashAndRepeatedArguments(t *testing.T) { - dir := t.TempDir() - argsFile := filepath.Join(dir, "args") - reference := filepath.Join(dir, "reference.sh") - if err := os.WriteFile(reference, []byte("#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$SUM_GO_ARGS_FILE\"\n"), 0o700); err != nil { - t.Fatal(err) - } - t.Setenv("SUM_GO_ARGS_FILE", argsFile) - var stdout, stderr bytes.Buffer - root := NewRoot(reference, &stdout, &stderr) - root.SetArgs([]string{"status", "--arg=-m", "--arg=-m", "--"}) - - if err := root.ExecuteContext(context.Background()); err != nil { - t.Fatalf("compatibility command failed: %v", err) - } - want := "status\n--arg=-m\n--arg=-m\n--\n" - got, err := os.ReadFile(argsFile) - if err != nil { - t.Fatal(err) - } - if string(got) != want { - t.Fatalf("reference argv = %q, want %q", got, want) - } -} - -func TestCompatibilityPreservesGlobalHomePosition(t *testing.T) { - dir := t.TempDir() - argsFile := filepath.Join(dir, "args") - reference := filepath.Join(dir, "reference.sh") - if err := os.WriteFile(reference, []byte("#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$SUM_GO_ARGS_FILE\"\n"), 0o700); err != nil { - t.Fatal(err) - } - t.Setenv("SUM_GO_ARGS_FILE", argsFile) - for _, input := range [][]string{ - {"--home", filepath.Join(dir, "before"), "status", "--arg=-m"}, - {"status", "--home=" + filepath.Join(dir, "after"), "--arg=-m"}, - } { - root := NewRoot(reference, &bytes.Buffer{}, &bytes.Buffer{}) - root.SetArgs(input) - if err := root.ExecuteContext(context.Background()); err != nil { - t.Fatalf("compatibility command failed for %q: %v", input, err) - } - got, err := os.ReadFile(argsFile) - if err != nil { - t.Fatal(err) - } - want := "--home\n" + input[1] + "\nstatus\n--arg=-m\n" - if input[0] == "status" { - want = "status\n" + input[1] + "\n--arg=-m\n" - } - if string(got) != want { - t.Fatalf("reference argv = %q, want %q", got, want) - } - } -} - -func TestCompatibilityHonorsCancellation(t *testing.T) { - dir := t.TempDir() - reference := filepath.Join(dir, "reference.sh") - if err := os.WriteFile(reference, []byte("#!/bin/sh\nsleep 10\n"), 0o700); err != nil { - t.Fatal(err) - } - root := NewRoot(reference, &bytes.Buffer{}, &bytes.Buffer{}) - root.SetArgs([]string{"status"}) - ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) - defer cancel() - if err := root.ExecuteContext(ctx); err == nil || err != context.DeadlineExceeded { - t.Fatalf("cancellation error = %v, want context deadline", err) - } -} - -func TestCompatibilityPreservesExitCode(t *testing.T) { - dir := t.TempDir() - reference := filepath.Join(dir, "reference.sh") - if err := os.WriteFile(reference, []byte("#!/bin/sh\nexit 7\n"), 0o700); err != nil { - t.Fatal(err) - } - root := NewRoot(reference, &bytes.Buffer{}, &bytes.Buffer{}) - root.SetArgs([]string{"status"}) - err := root.ExecuteContext(context.Background()) - exit, ok := err.(*ExitError) - if !ok || exit.Code != 7 { - t.Fatalf("error = %#v, want ExitError{Code: 7}", err) - } -} diff --git a/lib/sumctl.py b/lib/sumctl.py index e57fa0c..42d7931 100644 --- a/lib/sumctl.py +++ b/lib/sumctl.py @@ -5164,10 +5164,10 @@ def build_log(validated, observed, claimed, label, role, stamp): def upsert_log(record, row): - exists = any(l["id"] == row["id"] for l in record["logs"]) + exists = any(log["id"] == row["id"] for log in record["logs"]) if not exists and len(record["logs"]) >= ENVIRONMENT_LIMITS["logs"]: raise SumError(f"At most {ENVIRONMENT_LIMITS['logs']} log references per task.") - record["logs"] = [row if l["id"] == row["id"] else l for l in record["logs"]] if exists else record["logs"] + [row] + record["logs"] = [row if log["id"] == row["id"] else log for log in record["logs"]] if exists else record["logs"] + [row] return row @@ -5276,8 +5276,8 @@ def env_inspect(store, args): history_item = {"at": row["observed_at"], "state": row["state"], "ownership": row["ownership"]} if row.get("local"): observation = port_observations[row["id"]] - previous_pids = {l["pid"] for l in (row.get("observation") or {}).get("listeners", [])} - current_pids = {l["pid"] for l in observation["listeners"]} + previous_pids = {listener["pid"] for listener in (row.get("observation") or {}).get("listeners", [])} + current_pids = {listener["pid"] for listener in observation["listeners"]} if observation["state"] == "not-listening" and before[0] in ("observed", "stale"): row["state"], row["stale_reason"] = "stale", f"nothing listens on port {row['port']} any more" elif observation["state"] == "observed" and previous_pids and previous_pids != current_pids: @@ -5311,7 +5311,7 @@ def env_inspect(store, args): return {"task": task["id"], "path": str(environment_path(store, task["id"])), "inspected_at": stamp, "changes": changes, "discovery": {k: discovery.get(k) for k in ("config_revision", "current_revision", "stale", "stale_reason")} if discovery else None, "endpoints": [{k: e.get(k) for k in ("id", "url", "state", "ownership", "stale_reason", "config_stale")} for e in record["endpoints"]], - "logs": [{k: l.get(k) for k in ("id", "path", "state", "bytes")} for l in record["logs"]], + "logs": [{k: log.get(k) for k in ("id", "path", "state", "bytes")} for log in record["logs"]], "by": role, "touched": "nothing was started, stopped, or reconfigured", "note": ENVIRONMENT_NOTE} @@ -5335,15 +5335,15 @@ def command_view(row): return {**{k: row.get(k) for k in ("name", "kind", "source", "description", "image", "declared_ports")}, "command": bounded_view(text, limit) if limit else text, "redactions": row.get("redactions", 0) + redactions} stale = bool(discovery and discovery.get("stale")) or any(e["state"] in ("stale", "unverified") or e.get("config_stale") for e in record["endpoints"]) \ - or any(l["state"] != "present" for l in record["logs"]) or any(s["state"] in ("unknown", "stopping", "conflict", "failed") for s in record.get("services", [])) + or any(log["state"] != "present" for log in record["logs"]) or any(s["state"] in ("unknown", "stopping", "conflict", "failed") for s in record.get("services", [])) return {"present": True, "ok": True, "path": str(environment_path(store, task["id"])), "updated_at": record.get("updated_at"), "stale": stale, "discovery": {**{k: discovery.get(k) for k in ("observed_at", "head", "config_revision", "current_revision", "stale", "stale_reason", "checked_at", "summary", "problems", "task_origins", "verification_contract")}, "sources": [{k: s.get(k) for k in ("path", "bytes", "sha256", "skipped")} for s in discovery.get("sources", [])], "commands": [command_view(c) for c in discovery.get("commands", [])]} if discovery else None, "endpoints": [{**{k: e.get(k) for k in ("id", "url", "port", "local", "label", "ownership", "claimed_ownership", "state", "stale_reason", "config_stale", "observed_at", "recorded_by")}, - "listeners": [{k: l.get(k) for k in ("pid", "owner")} for l in (e.get("observation") or {}).get("listeners", [])], + "listeners": [{k: listener.get(k) for k in ("pid", "owner")} for listener in (e.get("observation") or {}).get("listeners", [])], "conflicts": [c["task"] for c in e.get("conflicts", [])]} for e in record["endpoints"]], - "logs": [{k: l.get(k) for k in ("id", "path", "scope", "label", "ownership", "state", "bytes", "modified_at", "observed_at")} for l in record["logs"]], + "logs": [{k: log.get(k) for k in ("id", "path", "scope", "label", "ownership", "state", "bytes", "modified_at", "observed_at")} for log in record["logs"]], "resources": [{k: r.get(k) for k in ("kind", "id", "session", "label", "ownership", "state", "cwd", "note", "service", "observed_at")} for r in record["resources"]], "services": services_view(record), "history": record.get("history", [])[-5:], "commands": commands, "authority": CLAIM_NOTE, "note": ENVIRONMENT_NOTE} @@ -5359,7 +5359,7 @@ def environment_outline(store, task): discovery = record.get("discovery") or {} return {"present": True, "updated_at": record.get("updated_at"), "config_stale": bool(discovery.get("stale")), "endpoints": {state: sum(1 for e in record["endpoints"] if e["state"] == state) for state in ENDPOINT_STATES if any(e["state"] == state for e in record["endpoints"])}, - "logs_missing": sum(1 for l in record["logs"] if l["state"] != "present"), "resources": len(record["resources"]), + "logs_missing": sum(1 for log in record["logs"] if log["state"] != "present"), "resources": len(record["resources"]), "services": {state: sum(1 for s in record.get("services", []) if s["state"] == state) for state in SERVICE_STATES if any(s["state"] == state for s in record.get("services", []))}} @@ -5568,16 +5568,17 @@ def wait_for_listener(store, task, parsed, session, pane_id, process, timeout): if misses: if time.monotonic() >= deadline: # Every path reaches the deadline. return {"ready": False, "checked": "listener", "waited_s": round(waited, 2), "changed": True, "processes": info["processes"], "reason": f"the recorded instance was not the pane foreground at the deadline ({timeout}s); the launch is unknown"} - time.sleep(SERVICE_POLL); waited += SERVICE_POLL + time.sleep(SERVICE_POLL) + waited += SERVICE_POLL continue pane_pids = {p["pid"] for p in info["processes"] if p.get("pid") is not None} observation = observe_port(store, task, parsed["port"]) if observation["state"] == "observed": - mine = [l for l in observation["listeners"] if l["pid"] in pane_pids] + mine = [listener for listener in observation["listeners"] if listener["pid"] in pane_pids] if mine: return {"ready": True, "checked": "listener", "waited_s": round(waited, 2), "observation": observation, "listener": mine[0]} return {"ready": False, "checked": "listener", "waited_s": round(waited, 2), "observation": observation, - "reason": f"port {parsed['port']} is taken by a process that is not in the service pane ({[(l['pid'], l.get('owner')) for l in observation['listeners']]}); a checkout cwd alone is not ownership, reported and not terminated"} + "reason": f"port {parsed['port']} is taken by a process that is not in the service pane ({[(listener['pid'], listener.get('owner')) for listener in observation['listeners']]}); a checkout cwd alone is not ownership, reported and not terminated"} if observation["state"] == "unverified": return {"ready": False, "checked": "listener", "waited_s": round(waited, 2), "observation": observation, "reason": f"listeners cannot be observed: {observation.get('error')}"} if time.monotonic() >= deadline: @@ -5684,7 +5685,7 @@ def env_start(store, args): current = ensure_environment(store, task) current["services"] = (current.get("services") or [])[-(SERVICE_LIMIT - 1):] + [service] write_environment(store, current, {"event": "start-conflict", "service": service["id"], "port": parsed["port"]}) - raise SumError(f"Port {parsed['port']} is already taken by {[(l['pid'], l.get('owner'), l.get('cwd')) for l in busy['listeners']]}; recorded as a conflict ({service['id']}). " + raise SumError(f"Port {parsed['port']} is already taken by {[(listener['pid'], listener.get('owner'), listener.get('cwd')) for listener in busy['listeners']]}; recorded as a conflict ({service['id']}). " f"sum never terminates the occupant; pick the port the repository's configuration reports or stop that service yourself.") # Intent is durable before any pane exists. service = {"id": "s-" + uuid.uuid4().hex[:10], "name": row["name"], "source": row["source"], "kind": row["kind"], "command": command, "launch": {**launch, "cwd": worktree, "pane_created": False}, @@ -5812,7 +5813,7 @@ def stop_service(store, task, service, timeout=STOP_TIMEOUT): port_check = observe_port(store, task, service["port"]) if port_check["state"] == "observed": row = update_service(store, task["id"], service["id"], "stop-port-still-taken", state="unknown", - stop={"action": "interrupt" if sent else "none", "result": f"process exited but port {service['port']} is still taken by {[(l['pid'], l.get('owner')) for l in port_check['listeners']]}"}) + stop={"action": "interrupt" if sent else "none", "result": f"process exited but port {service['port']} is still taken by {[(listener['pid'], listener.get('owner')) for listener in port_check['listeners']]}"}) return {**outcome, "action": "interrupt" if sent else "none", "state": "unknown", "closed_pane": False, "reason": f"port {service['port']} is still taken after the exit; a detached child or another process holds it, nothing is terminated"} closed = False @@ -8008,7 +8009,7 @@ def build_native_artifact(target): source = target / "go" if not (source / "go.mod").is_file(): raise SumError(f"Native Go source is missing from {source}") - outputs = {"sumctl-go": "./cmd/sumctl-go", "herdr-mesh-go": "./cmd/herdr-mesh"} + outputs = {"herdr-mesh-go": "./cmd/herdr-mesh"} output_dir = target / ".local" / "bin" output_dir.mkdir(parents=True, exist_ok=True) pending = [] @@ -8020,7 +8021,7 @@ def build_native_artifact(target): else: pending.append((name, package)) if not pending: - return output_dir / "sumctl-go" + return output_dir / "herdr-mesh-go" go = os.environ.get("SUM_GO_BIN") if not go: mise = shutil.which("mise") @@ -8034,7 +8035,7 @@ def build_native_artifact(target): platform_name = native_platform() goos, goarch = platform_name.split("-", 1) build_env = {**os.environ, "CGO_ENABLED": "0", "GOENV": "off", "GOOS": goos, "GOARCH": goarch} - for variable in ("GOROOT", "GOTOOLDIR", "GOTOOLCHAIN"): + for variable in ("GOROOT", "GOBIN", "GOTOOLDIR", "GOTOOLCHAIN"): build_env.pop(variable, None) for name, package in pending: output = output_dir / name @@ -8051,7 +8052,7 @@ def build_native_artifact(target): finally: if temporary.exists(): temporary.unlink() - return output_dir / "sumctl-go" + return output_dir / "herdr-mesh-go" def native_platform(): @@ -8173,6 +8174,11 @@ def validate_dependency_inventory(value): seen.add(entry["id"]) +def native_inventory_entries(inventory): + return {entry["id"]: entry for entry in inventory["dependencies"] + if entry["checksum"] == f"release.json#dependencies.native.{entry['id']}.sha256"} + + def build_manifest(store, root, sha, target): target = Path(target) contract = candidate_contract(target) @@ -8186,20 +8192,14 @@ def build_manifest(store, root, sha, target): mesh = target / ".deps" / "herdr-mesh" patched = read_json(mesh / ".sum-patched") inventory = dependency_inventory(target) - native = next((entry for entry in inventory["dependencies"] if entry["id"] == "sumctl-go"), None) - native_path = target / ".local" / "bin" / "sumctl-go" - if native is None or not native_path.is_file() or not os.access(native_path, os.X_OK): - raise SumError("Release is missing the staged native sumctl-go artifact") - native = {**native, "path": ".local/bin/sumctl-go", "sha256": sha256_file(native_path), - "platform": native_platform(), "build": {"cgo": False, "requires": ["go >= 1.25"]}, - "runtime": {"requires": []}} - native_artifacts = {"sumctl-go": native} - mesh_path = target / ".local" / "bin" / "herdr-mesh-go" - if mesh_path.is_file() and os.access(mesh_path, os.X_OK): - native_artifacts["herdr-mesh-go"] = {"source": "go/cmd/herdr-mesh", "version": "0.1.0", - "path": ".local/bin/herdr-mesh-go", "sha256": sha256_file(mesh_path), - "platform": native_platform(), "build": {"cgo": False, "requires": ["go >= 1.25"]}, - "runtime": {"requires": []}} + native_artifacts = {} + for name, entry in native_inventory_entries(inventory).items(): + native_path = target / ".local" / "bin" / name + if native_path.is_symlink() or not native_path.is_file() or not os.access(native_path, os.X_OK): + raise SumError(f"Release is missing the staged native artifact {name}") + native_artifacts[name] = {**entry, "path": f".local/bin/{name}", "sha256": sha256_file(native_path), + "platform": native_platform(), "build": {"cgo": False, "requires": ["go >= 1.25"]}, + "runtime": {"requires": []}} state = read_json(store.home / "state.json") if (store.home / "state.json").is_file() else {} return {"schema": RELEASE_SCHEMA, "kind": "sum-release", "sum_version": contract["sum_version"], "source": {"sha": sha, "tree": run(["git", "-C", root, "rev-parse", f"{sha}^{{tree}}"]).stdout.strip(), "repository": str(root)}, @@ -8267,9 +8267,13 @@ def verify_release(path, expected_sha=None): inventory = manifest.get("dependencies", {}).get("inventory") if inventory is not None: validate_dependency_inventory(inventory) + if not isinstance(native, dict): + raise SumError(f"{path}: native dependency metadata is incomplete") + required_native = native_inventory_entries(inventory) if inventory is not None else {} + missing_native = sorted(set(required_native) - set(native)) + if missing_native: + raise SumError(f"{path}: native artifact {missing_native[0]} is missing from the manifest") if native: - if not isinstance(native, dict) or "sumctl-go" not in native: - raise SumError(f"{path}: native dependency metadata is incomplete") for name, artifact in native.items(): relative = artifact.get("path") if isinstance(artifact, dict) else None if not isinstance(relative, str) or not relative or Path(relative).is_absolute() or ".." in PurePosixPath(relative).parts: diff --git a/scripts/benchmark_fixture.py b/scripts/benchmark_fixture.py index b9ac4e3..e6c7209 100644 --- a/scripts/benchmark_fixture.py +++ b/scripts/benchmark_fixture.py @@ -25,7 +25,7 @@ def __str__(self) -> str: return self.detail -def clean_environment(base: Path, source_root: Path = ROOT) -> dict[str, str]: +def clean_environment(base: Path) -> dict[str, str]: env = {key: value for key, value in os.environ.items() if not key.startswith(("SUM_", "HERDR_"))} env.update({ "HOME": str(base / "home"), @@ -34,9 +34,9 @@ def clean_environment(base: Path, source_root: Path = ROOT) -> dict[str, str]: "HERDR_PANE_ID": "w-parent:p1", "HERDR_SESSION": "sum-benchmark", "SUM_SESSION": "sum-benchmark", - "SUM_HERDR_BIN": str(source_root / "tests" / "fixtures" / "herdr.py"), + "SUM_HERDR_BIN": str(FAKE_HERDR), "FAKE_HERDR_ROOT": str(base / "fake"), - "FAKE_PARENT_CWD": str(source_root), + "FAKE_PARENT_CWD": str(ROOT), "FAKE_SESSION": "sum-benchmark", }) Path(env["HOME"]).mkdir(parents=True, exist_ok=True) @@ -106,8 +106,8 @@ def git_repo(path: Path) -> None: subprocess.run(["git", "-C", str(path), "commit", "-q", "-m", "fixture"], check=True) -def fixture(base: Path, source_root: Path = ROOT): - env = clean_environment(base, source_root) +def fixture(base: Path): + env = clean_environment(base) repo = base / "repo" git_repo(repo) brief = base / "brief.md" @@ -115,7 +115,7 @@ def fixture(base: Path, source_root: Path = ROOT): home = base / "state" home.mkdir(mode=0o700) (home / "state.json").write_text('{"schema":1,"sum_version":"0.1.0","created_at":"2026-09-07T00:00:00+00:00"}\n', encoding="utf-8") - prefix = [source_root / "bin" / "sumctl", "--home", home] + prefix = [SUMCTL, "--home", home] run_plain([*prefix, "init"], env) run_plain([*prefix, "settings", "set", "--global", "64", "--per-repository", "64"], env) task = run_plain([*prefix, "dispatch", "--repo", repo, "--brief", brief, "--harness", "codex", "--approved"], env) diff --git a/scripts/benchmark_go.py b/scripts/benchmark_go.py deleted file mode 100644 index 5c2a68a..0000000 --- a/scripts/benchmark_go.py +++ /dev/null @@ -1,173 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import os -from pathlib import Path -import statistics -import subprocess -import tempfile -import tarfile -import time -import re - -from benchmark_fixture import fixture - - -ROOT = Path(__file__).resolve().parents[1] -REFERENCE_REVISION = "b03b8020621e0d417906402a5c7ecc5d63192541" -BASELINE = { - "startup.version.cold": 136.568, - "startup.help.warm-fs": 138.560, - "read.status.empty": 145.848, - "failure.show-missing": 146.063, -} - - -def reference_snapshot(destination: Path) -> Path: - destination.mkdir() - env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - with tempfile.TemporaryFile() as archive: - subprocess.run(["git", "--no-replace-objects", "-C", str(ROOT), "archive", "--format=tar", REFERENCE_REVISION], - env=env, stdout=archive, check=True) - archive.seek(0) - with tarfile.open(fileobj=archive) as source: - source.extractall(destination, filter="data") - return destination - - -def allocations() -> dict[str, object]: - command = ["go", "test", "-run", "^$", "-bench", "BenchmarkNewRoot", "-benchmem", "./internal/cli"] - result = subprocess.run(command, cwd=ROOT / "go", text=True, capture_output=True) - match = re.search(r"BenchmarkNewRoot-\S+\s+\d+\s+([\d.]+) ns/op\s+([\d.]+) B/op\s+([\d.]+) allocs/op", result.stdout) - if result.returncode or not match: - raise RuntimeError(f"allocation benchmark failed: {(result.stderr or result.stdout)[-2000:]}") - return { - "command": command, - "exit": result.returncode, - "ns_per_op": float(match.group(1)), - "bytes_per_op": float(match.group(2)), - "allocs_per_op": float(match.group(3)), - } - - -def gate(scenarios: list[dict[str, object]], allocation: dict[str, object]) -> dict[str, object]: - by_id = {row["id"]: row for row in scenarios} - version = by_id["startup.version.cold"] - help_row = by_id["startup.help.cobra"] - comparisons = {} - for candidate_id, baseline_id in (("startup.version.cold", "startup.version.cold"), ("startup.help.cobra", "startup.help.warm-fs")): - baseline = BASELINE[baseline_id] - candidate = float(by_id[candidate_id]["p50_ms"]) - comparisons[candidate_id] = { - "baseline_p50_ms": baseline, - "candidate_p50_ms": candidate, - "absolute_improvement_ms": round(baseline - candidate, 3), - "relative_improvement_percent": round((baseline - candidate) / baseline * 100, 2), - } - interactive = { - "required_absolute_ms": 50, - "required_relative_percent": 35, - "observed_absolute_improvement_ms": round(max(row["absolute_improvement_ms"] for row in comparisons.values()), 3), - "observed_relative_improvement_percent": round(max(row["relative_improvement_percent"] for row in comparisons.values()), 2), - "pass": any(row["absolute_improvement_ms"] >= 50 and row["relative_improvement_percent"] >= 35 for row in comparisons.values()), - } - frequency = {"required_ms": 500, "observed_ms": 0, "pass": False, "reason": "No stateful command is native; status remains a compatibility subprocess."} - behavior = {"required_regressions": 0, "observed_regressions": None, "pass": False, "status": "not-evaluated", "basis": "Expected exit codes are smoke checks, not differential output or effect parity."} - memory = {"required_regression_percent": 10, "pass": False, "status": "not-comparable", "reason": "The compatibility child is outside the compiled parent's /usr/bin/time memory sample."} - return { - "outcome": "defer", - "pass": False, - "comparisons": comparisons, - "interactive_hot_path": interactive, - "frequency_weighted": frequency, - "behavior": behavior, - "memory": memory, - "allocations": allocation, - "binary_and_entrypoint": {"version": version["id"], "help": help_row["id"]}, - "reasons": ["The native startup/help path crosses the latency gate." if interactive["pass"] else "The native startup/help path does not cross the latency gate.", "No stateful command is native, so the 500 ms frequency-weighted gate is not established.", "Behavior parity has not been evaluated.", "Compatibility memory is not comparable to the Python child process."], - } - - -def measure(command: list[str], env: dict[str, str], samples: int, expected: int = 0, subprocesses: int = 0) -> dict[str, object]: - values = [] - exit_codes = [] - peak_memory = 0 - for _ in range(samples): - started = time.perf_counter_ns() - measured = ["/usr/bin/time", "-l", *command] - result = subprocess.run(measured, cwd=ROOT, env=env, text=True, capture_output=True) - values.append((time.perf_counter_ns() - started) / 1_000_000) - exit_codes.append(result.returncode) - for line in result.stderr.splitlines(): - if line.strip().endswith("peak memory footprint"): - peak_memory = max(peak_memory, int(line.split()[0])) - if result.returncode != expected: - raise RuntimeError(f"{command} exited {result.returncode}, expected {expected}: {(result.stderr or result.stdout)[-2000:]}") - ordered = sorted(values) - return { - "command": command, - "samples": samples, - "p50_ms": round(statistics.median(values), 3), - "p95_ms": round(ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))], 3), - "min_ms": round(min(values), 3), - "max_ms": round(max(values), 3), - "exit_codes": exit_codes, - "peak_memory_bytes": peak_memory, - "subprocesses": subprocesses, - } - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--binary", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--samples", type=int, default=15) - args = parser.parse_args() - if not 1 <= args.samples <= 200: - parser.error("--samples must be between 1 and 200") - - binary = args.binary.resolve() - if not binary.is_file(): - raise SystemExit(f"binary does not exist: {binary}") - with tempfile.TemporaryDirectory(prefix="sum-go-benchmark-") as temporary: - base = Path(temporary) - reference = reference_snapshot(base / "reference") - case = fixture(base, source_root=reference) - env = dict(case["env"]) - env["SUM_PYTHON_HELPER"] = str(reference / "bin" / "sumctl") - commands = [ - ("startup.version.cold", [str(binary), "--version"], dict(env)), - ("startup.help.cobra", [str(binary), "--help"], {**env, "SUM_PYTHON_HELPER": str(base / "missing-reference")}), - ("read.status.fixture", [str(binary), "--home", str(case["home"]), "status"], dict(env)), - ("failure.show-missing", [str(binary), "--home", str(case["home"]), "show", "t-000000000000"], dict(env)), - ] - results = [] - for case_id, command, command_env in commands: - expected = 1 if case_id == "failure.show-missing" else 0 - subprocesses = 0 if case_id.startswith("startup.") else 1 - row = measure(command, command_env, args.samples, expected, subprocesses) - row["id"] = case_id - if case_id == "failure.show-missing": - row["expected_exit"] = expected - results.append(row) - record = { - "schema": 1, - "binary": str(binary), - "binary_bytes": binary.stat().st_size, - "reference_revision": REFERENCE_REVISION, - "reference_helper": str(reference / "bin" / "sumctl"), - "samples": args.samples, - "scenarios": results, - "scope": "compiled Cobra entrypoint; status and missing-show use the explicit Python compatibility boundary", - "allocations": allocations(), - } - record["gate"] = gate(results, record["allocations"]) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") - print(json.dumps({"output": str(args.output), "binary_bytes": record["binary_bytes"], "scenarios": len(results)})) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/test_benchmark_go.py b/tests/test_benchmark_go.py deleted file mode 100644 index e03d3d2..0000000 --- a/tests/test_benchmark_go.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -import importlib.util -from pathlib import Path -import subprocess -import sys -import tempfile -import unittest -from unittest.mock import patch - - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "scripts")) -SPEC = importlib.util.spec_from_file_location("benchmark_go", ROOT / "scripts" / "benchmark_go.py") -benchmark = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(benchmark) - - -class GoBenchmarkEvidenceTest(unittest.TestCase): - def scenarios(self, latency=6): - return [{"id": name, "p50_ms": latency} for name in ( - "startup.version.cold", "startup.help.cobra", "read.status.fixture", "failure.show-missing", - )] - - def test_exit_codes_alone_do_not_establish_behavior_parity(self): - result = benchmark.gate(self.scenarios(), {}) - self.assertFalse(result["behavior"]["pass"]) - self.assertIsNone(result["behavior"]["observed_regressions"]) - self.assertEqual(result["behavior"]["status"], "not-evaluated") - - def test_failed_latency_gate_is_not_described_as_crossed(self): - result = benchmark.gate(self.scenarios(latency=1000), {}) - self.assertFalse(result["interactive_hot_path"]["pass"]) - self.assertNotIn("The native startup/help path crosses the latency gate.", result["reasons"]) - - def test_reference_snapshot_uses_frozen_bytes_not_working_tree(self): - with tempfile.TemporaryDirectory(prefix="sum-go-reference-test-") as temporary: - base = Path(temporary) - repo = base / "repo" - repo.mkdir() - subprocess.run(["git", "init", "-q", str(repo)], check=True) - subprocess.run(["git", "-C", str(repo), "config", "user.name", "fixture"], check=True) - subprocess.run(["git", "-C", str(repo), "config", "user.email", "fixture@example.invalid"], check=True) - helper = repo / "bin" / "sumctl" - helper.parent.mkdir() - helper.write_text("#!/bin/sh\nprintf frozen\n") - helper.chmod(0o755) - subprocess.run(["git", "-C", str(repo), "add", "."], check=True) - subprocess.run(["git", "-C", str(repo), "commit", "-qm", "frozen"], check=True) - revision = subprocess.check_output(["git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip() - helper.write_text("#!/bin/sh\nprintf changed\n") - with patch.object(benchmark, "ROOT", repo), patch.object(benchmark, "REFERENCE_REVISION", revision, create=True): - snapshot = benchmark.reference_snapshot(base / "reference") - result = subprocess.run([str(snapshot / "bin" / "sumctl")], capture_output=True, text=True, check=True) - self.assertEqual(result.stdout, "frozen") - self.assertEqual(helper.read_text(), "#!/bin/sh\nprintf changed\n") - self.assertFalse((snapshot / ".git").exists()) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_core.py b/tests/test_core.py index 8ae77e5..f6fb03b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1222,8 +1222,8 @@ def fake_installer(target, local_mesh=None): for name in sumctl.TOOLS: real = {"python3": sys.executable, "node": shutil.which("node") or sys.executable}.get(name, str(ROOT / "tests/fixtures/herdr.py")) sumctl.link_tool(target / ".local" / "bin" / name, real) - native = target / ".local" / "bin" / "sumctl-go" - native.write_text("#!/bin/sh\nprintf '%s\\n' 'sum 0.1.0'\n") + native = target / ".local" / "bin" / "herdr-mesh-go" + native.write_text("#!/bin/sh\nprintf '%s\\n' 'herdr-mesh 0.1.0'\n") native.chmod(0o755) (target / ".local" / "skills" / "herdr").mkdir(parents=True) (target / ".local" / "skills" / "herdr" / "SKILL.md").write_text("fake herdr skill\n") @@ -1261,6 +1261,8 @@ def installation(self, name="sum install dir", via_symlink=False): listing = subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"], capture_output=True, check=True).stdout for relative in filter(None, listing.decode().split("\0")): source, target = ROOT / relative, real / relative + if not source.exists() and not source.is_symlink(): + continue target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(source, target, follow_symlinks=False) self.git("init", "-b", "main", cwd=real) @@ -1314,6 +1316,9 @@ def test_verify_release_accepts_an_immutable_historical_worker_path(self): historical = self.root / "historical-release" sumctl.archive_source(ROOT, "de92361b87181837c58308acf2521fdae2677cec", historical) fake_installer(historical) + historical_native = historical / ".local" / "bin" / "sumctl-go" + historical_native.write_text("#!/bin/sh\nprintf '%s\\n' 'sum 0.1.0'\n") + historical_native.chmod(0o755) manifest = sumctl.build_manifest(store, ROOT, "de92361b87181837c58308acf2521fdae2677cec", historical) sumctl.atomic_json(historical / sumctl.RELEASE_MANIFEST, manifest) @@ -1353,25 +1358,65 @@ def test_stage_builds_a_validated_immutable_bundle_outside_state(self): self.assertEqual([(r["sha"], r["ok"]) for r in listing["releases"]], [(head, True)]) self.assertEqual(sumctl.release_show(store, head[:8])["sha"], head) - def test_stage_packages_native_bridge_with_runtime_provenance(self): + def test_stage_packages_required_mesh_with_runtime_provenance(self): root, store = self.installation() release = Path(self.stage(store)["release"]) - native = json.loads((release / "release.json").read_text())["dependencies"]["native"]["sumctl-go"] + native_dependencies = json.loads((release / "release.json").read_text())["dependencies"]["native"] + self.assertEqual(set(native_dependencies), {"herdr-mesh-go"}) + native = native_dependencies["herdr-mesh-go"] binary = release / native["path"] - self.assertEqual(native["source"], "go/cmd/sumctl-go") - self.assertEqual(native["version"], "sum 0.1.0") + self.assertEqual(native["source"], "go/cmd/herdr-mesh") + self.assertEqual(native["version"], "0.1.0") self.assertEqual(native["platform"], sumctl.native_platform()) self.assertEqual(native["build"], {"cgo": False, "requires": ["go >= 1.25"]}) self.assertEqual(native["runtime"], {"requires": []}) self.assertTrue(binary.is_file() and os.access(binary, os.X_OK)) self.assertEqual(native["sha256"], hashlib_sha(binary)) - self.assertEqual(self.cli([binary, "--version"]).stdout, "sum 0.1.0\n") + self.assertEqual(self.cli([binary, "--version"]).stdout, "herdr-mesh 0.1.0\n") sumctl.set_read_only(release, read_only=False) binary.write_text("corrupt\n") - with self.assertRaisesRegex(sumctl.SumError, "native artifact sumctl-go"): + with self.assertRaisesRegex(sumctl.SumError, "native artifact herdr-mesh-go"): sumctl.verify_release(release, release.name) + def test_verify_release_requires_every_native_artifact_declared_by_its_inventory(self): + root, store = self.installation() + release = Path(self.stage(store)["release"]) + sumctl.set_read_only(release, read_only=False) + (release / ".local" / "bin" / "herdr-mesh-go").unlink() + + with self.assertRaisesRegex(sumctl.SumError, "native artifact herdr-mesh-go"): + sumctl.verify_release(release, release.name) + + def test_verify_release_keeps_historical_native_inventory_contract(self): + root, store = self.installation() + release = Path(self.stage(store)["release"]) + sumctl.set_read_only(release, read_only=False) + manifest_path = release / "release.json" + manifest = json.loads(manifest_path.read_text()) + mesh = manifest["dependencies"]["native"].pop("herdr-mesh-go") + historical_binary = release / ".local" / "bin" / "sumctl-go" + historical_binary.write_text("#!/bin/sh\nprintf '%s\\n' 'sum 0.1.0'\n") + historical_binary.chmod(0o755) + historical_entry = { + **mesh, + "id": "sumctl-go", + "source": "go/cmd/sumctl-go", + "version": "sum 0.1.0", + "checksum": "release.json#dependencies.native.sumctl-go.sha256", + "contracts": {"cli": ["sumctl compatibility argv/stdout/stderr"], "mcp": []}, + "path": ".local/bin/sumctl-go", + "sha256": hashlib_sha(historical_binary), + } + manifest["dependencies"]["native"]["sumctl-go"] = historical_entry + inventory = manifest["dependencies"]["inventory"]["dependencies"] + inventory[:] = [entry for entry in inventory if entry["id"] != "herdr-mesh-go"] + inventory.append({key: value for key, value in historical_entry.items() if key not in {"path", "sha256", "platform", "build", "runtime"}}) + sumctl.atomic_json(manifest_path, manifest) + + verified = sumctl.verify_release(release, release.name) + self.assertEqual(set(verified["dependencies"]["native"]), {"sumctl-go"}) + def test_release_tree_never_owns_state_and_runs_only_for_its_installation(self): root, store = self.installation() release = Path(self.stage(store)["release"]) diff --git a/tests/test_native_packaging.py b/tests/test_native_packaging.py index 228cadb..536f18d 100644 --- a/tests/test_native_packaging.py +++ b/tests/test_native_packaging.py @@ -16,20 +16,22 @@ class NativePackagingTest(unittest.TestCase): - def test_native_build_produces_cgo_free_binary_without_runtime_tools(self): + def test_native_build_produces_only_required_mesh_binary_without_runtime_tools(self): with tempfile.TemporaryDirectory(prefix="sum-native-build-") as name: target = Path(name) (target / "go").symlink_to(ROOT / "go", target_is_directory=True) empty = target / "empty" empty.mkdir() expected = {"darwin": "darwin", "linux": "linux"}[sys.platform] + "-" + {"x86_64": "amd64", "aarch64": "arm64"}.get(platform.machine().lower(), platform.machine().lower()) - with mock.patch.dict(sumctl.os.environ, {"SUM_GO_BIN": shutil.which("go"), "GOROOT": "/stale/go", "GOTOOLDIR": "/stale/go/pkg/tool", "GOTOOLCHAIN": "local", "GOOS": "linux", "GOARCH": "amd64"}): + with mock.patch.dict(sumctl.os.environ, {"SUM_GO_BIN": shutil.which("go"), "GOROOT": "/stale/go", "GOBIN": "/stale/go/bin", "GOTOOLDIR": "/stale/go/pkg/tool", "GOTOOLCHAIN": "local", "GOOS": "linux", "GOARCH": "amd64"}): output = sumctl.build_native_artifact(target) result = subprocess.run([str(output), "--version"], env={"PATH": str(empty)}, capture_output=True, text=True, check=True) - self.assertEqual((result.stdout, result.stderr), ("sum 0.1.0\n", "")) + self.assertEqual((result.stdout, result.stderr), ("herdr-mesh 0.1.0\n", "")) + self.assertEqual(output, target / ".local" / "bin" / "herdr-mesh-go") + self.assertFalse((target / ".local" / "bin" / "sumctl-go").exists()) self.assertEqual(sumctl.native_platform(), expected) - def test_native_build_does_not_replace_existing_executable(self): + def test_native_build_does_not_replace_existing_mesh_executable(self): with tempfile.TemporaryDirectory(prefix="sum-native-") as name: target = Path(name) (target / "go").symlink_to(ROOT / "go", target_is_directory=True) diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 42595cc..05e4257 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,6 +1,5 @@ import json from pathlib import Path -import re import unittest @@ -18,7 +17,8 @@ def test_inventory_covers_pins_and_native_contracts_without_duplicate_ids(self): self.assertTrue(entry["checksum"], entry["id"]) self.assertIn(entry["role"], ("build", "runtime", "build-and-runtime"), entry["id"]) ids = {entry["id"] for entry in entries} - self.assertTrue({"go", "cobra", "mcp-go-sdk", "sumctl-go", "herdr-mesh", "quota-axi"} <= ids) + self.assertTrue({"go", "cobra", "mcp-go-sdk", "herdr-mesh-go", "herdr-mesh", "quota-axi"} <= ids) + self.assertNotIn("sumctl-go", ids) def test_go_module_uses_the_reviewed_official_mcp_sdk(self): go_mod = (ROOT / "go/go.mod").read_text() From 7cdb7a41abf8f7403162c392fabddf067adced39 Mon Sep 17 00:00:00 2001 From: Douglas Jarquin <8209+douglasjarquin@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:38:29 -0400 Subject: [PATCH 3/3] fix(release): bind native artifacts to bundled contracts --- docs/DEPENDENCIES.md | 1 + lib/sumctl.py | 12 +++++++- tests/test_core.py | 67 +++++++++++++++++++++++++++----------------- 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/docs/DEPENDENCIES.md b/docs/DEPENDENCIES.md index 51974d7..f8797c2 100644 --- a/docs/DEPENDENCIES.md +++ b/docs/DEPENDENCIES.md @@ -28,6 +28,7 @@ The staged binary's source, build requirements, runtime requirements, and SHA-25 Running it requires no Go toolchain, module download, Node, Python, or Cobra generator. The unused Go helper experiment is [not adopted](go-helper-prototype.md); `bin/sumctl` remains the Python entrypoint. Native artifact requirements come from each release's own dependency inventory, so a missing or corrupt declared artifact is refused without making a retired experiment mandatory for new bundles. +Verification compares the manifest's inventory with the hash-checked bundled inventory and requires each native artifact's canonical path, source identity, version, and current-host platform to agree. The source revision and upstream lockfile are pinned. This does not claim bit-for-bit reproducibility of every OS/runtime installation. A mise lockfile has not been invented; generate/review it on a networked machine when updating dependency pins. diff --git a/lib/sumctl.py b/lib/sumctl.py index 42d7931..2989e89 100644 --- a/lib/sumctl.py +++ b/lib/sumctl.py @@ -8265,6 +8265,10 @@ def verify_release(path, expected_sha=None): raise SumError(f"{path}: pinned tool {name} is missing or does not resolve") native = manifest.get("dependencies", {}).get("native", {}) inventory = manifest.get("dependencies", {}).get("inventory") + inventory_file = "docs/dependency-inventory.json" + if inventory is not None or (path / inventory_file).exists(): + if inventory_file not in files or inventory != dependency_inventory(path): + raise SumError(f"{path}: dependency inventory does not match the hash-verified bundled inventory") if inventory is not None: validate_dependency_inventory(inventory) if not isinstance(native, dict): @@ -8278,6 +8282,8 @@ def verify_release(path, expected_sha=None): relative = artifact.get("path") if isinstance(artifact, dict) else None if not isinstance(relative, str) or not relative or Path(relative).is_absolute() or ".." in PurePosixPath(relative).parts: raise SumError(f"{path}: native artifact {name} has an invalid path") + if relative != f".local/bin/{name}": + raise SumError(f"{path}: native artifact {name} does not use its canonical path") member = path / relative if member.is_symlink() or not member.is_file() or not os.access(member, os.X_OK): raise SumError(f"{path}: native artifact {name} is missing or not executable") @@ -8286,9 +8292,13 @@ def verify_release(path, expected_sha=None): target = artifact.get("platform") if not isinstance(target, str) or not re.fullmatch(r"[a-z0-9]+-[a-z0-9]+", target): raise SumError(f"{path}: native artifact {name} lacks a valid GOOS-GOARCH target") - catalog = next((entry for entry in (inventory or {}).get("dependencies", []) if entry.get("id") == name), None) + catalog = required_native.get(name) if catalog is None or target not in catalog.get("platforms", []): raise SumError(f"{path}: native artifact {name} target {target} is not in the dependency inventory") + if any(artifact.get(key) != catalog.get(key) for key in ("source", "version")): + raise SumError(f"{path}: native artifact {name} identity does not match its inventory") + if target != native_platform(): + raise SumError(f"{path}: native artifact {name} targets {target}, not this host's {native_platform()}") return manifest diff --git a/tests/test_core.py b/tests/test_core.py index f6fb03b..e518cd2 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1261,8 +1261,6 @@ def installation(self, name="sum install dir", via_symlink=False): listing = subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"], capture_output=True, check=True).stdout for relative in filter(None, listing.decode().split("\0")): source, target = ROOT / relative, real / relative - if not source.exists() and not source.is_symlink(): - continue target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(source, target, follow_symlinks=False) self.git("init", "-b", "main", cwd=real) @@ -1388,34 +1386,53 @@ def test_verify_release_requires_every_native_artifact_declared_by_its_inventory with self.assertRaisesRegex(sumctl.SumError, "native artifact herdr-mesh-go"): sumctl.verify_release(release, release.name) + def test_release_show_refuses_native_contract_tampering(self): + for variant in ("inventory-checksum", "inventory-omitted", "native-path", "native-platform"): + with self.subTest(variant=variant): + root, store = self.installation(name=variant) + release = Path(self.stage(store)["release"]) + command = [root / "bin" / "sumctl", "--home", store.home, "release", "show", release.name] + control = self.cli(command) + self.assertEqual(control.returncode, 0, control.stderr) + sumctl.set_read_only(release, read_only=False) + manifest_path = release / "release.json" + manifest = json.loads(manifest_path.read_text()) + native = manifest["dependencies"]["native"]["herdr-mesh-go"] + if variant == "inventory-checksum": + entry = next(row for row in manifest["dependencies"]["inventory"]["dependencies"] if row["id"] == "herdr-mesh-go") + entry["checksum"] = "sha256:changed" + manifest["dependencies"]["native"] = {} + elif variant == "inventory-omitted": + manifest["dependencies"].pop("inventory") + manifest["dependencies"]["native"] = {} + elif variant == "native-path": + native["path"] = "bin/sumctl" + native["sha256"] = hashlib_sha(release / "bin" / "sumctl") + else: + native["platform"] = "linux-amd64" if sumctl.native_platform() != "linux-amd64" else "darwin-arm64" + sumctl.atomic_json(manifest_path, manifest) + before = self.snapshot(store.home) + result = self.cli(command) + self.assertEqual(result.returncode, 1, variant) + self.assertEqual(result.stdout, "") + self.assertTrue(json.loads(result.stderr)["error"]) + self.assertEqual(self.snapshot(store.home), before) + def test_verify_release_keeps_historical_native_inventory_contract(self): - root, store = self.installation() - release = Path(self.stage(store)["release"]) - sumctl.set_read_only(release, read_only=False) - manifest_path = release / "release.json" - manifest = json.loads(manifest_path.read_text()) - mesh = manifest["dependencies"]["native"].pop("herdr-mesh-go") + _, store = self.installation() + revision = "fe57809a3b80e8327df337ca8acc5d87c4bb71f0" + release = self.root / "historical-native-release" + sumctl.archive_source(ROOT, revision, release) + fake_installer(release) historical_binary = release / ".local" / "bin" / "sumctl-go" historical_binary.write_text("#!/bin/sh\nprintf '%s\\n' 'sum 0.1.0'\n") historical_binary.chmod(0o755) - historical_entry = { - **mesh, - "id": "sumctl-go", - "source": "go/cmd/sumctl-go", - "version": "sum 0.1.0", - "checksum": "release.json#dependencies.native.sumctl-go.sha256", - "contracts": {"cli": ["sumctl compatibility argv/stdout/stderr"], "mcp": []}, - "path": ".local/bin/sumctl-go", - "sha256": hashlib_sha(historical_binary), - } - manifest["dependencies"]["native"]["sumctl-go"] = historical_entry - inventory = manifest["dependencies"]["inventory"]["dependencies"] - inventory[:] = [entry for entry in inventory if entry["id"] != "herdr-mesh-go"] - inventory.append({key: value for key, value in historical_entry.items() if key not in {"path", "sha256", "platform", "build", "runtime"}}) - sumctl.atomic_json(manifest_path, manifest) - - verified = sumctl.verify_release(release, release.name) + manifest = sumctl.build_manifest(store, ROOT, revision, release) + sumctl.atomic_json(release / "release.json", manifest) + + verified = sumctl.verify_release(release, revision) self.assertEqual(set(verified["dependencies"]["native"]), {"sumctl-go"}) + self.assertEqual(verified["dependencies"]["inventory"], sumctl.dependency_inventory(release)) def test_release_tree_never_owns_state_and_runs_only_for_its_installation(self): root, store = self.installation()