test: add end-to-end rewrite-retention regression check (#200)

scripts/rewrite-retention-check/README.md

@@ -0,0 +1,54 @@
+# Rewrite-retention regression check (Claude Code)
+
+End-to-end guard for the #204 fix (issue #200): Claude Code rewrites session
+JSONL files in place on resume/compact, and splitrail's totals must not drift
+when that happens, because the local history store restores retained records.
+
+Everything runs against a synthetic corpus under an isolated `$HOME` — no real
+`~/.claude` data is read, no config is loaded, no upload path exists.
+
+## Run
+
+```bash
+cargo build --release
+scripts/rewrite-retention-check/run_check.sh
+```
+
+Optionally also demonstrate the pre-fix drift with an old binary:
+
+```bash
+SPLITRAIL_BASELINE_BIN=/path/to/splitrail-3.5.9 \
+  scripts/rewrite-retention-check/run_check.sh
+```
+
+## What it asserts
+
+| # | assertion | guards |
+|---|---|---|
+| R1 | scan of pristine corpus == generator manifest (per-model message counts + 4 token fields) | parser + streaming-line dedup (`local_hash` + token fingerprint) |
+| R2 | scan after a simulated resume/compact == scan before it | **the history-store retention itself — the #200/#204 regression** |
+| R3 | three consecutive scans byte-identical | restart stability of the store |
+| R4 | (only with `SPLITRAIL_BASELINE_BIN`) pre-fix binary drops by exactly the removed usage | demonstrates the bug the fixture pins down |
+
+The synthetic corpus mirrors the parts of Claude Code's format the check
+depends on: each API message written as two streaming lines (distinct line
+`uuid`s, same `message.id`/`requestId`, identical usage), plus skip-cases
+(user lines, `summary`, `file-history-snapshot`, `<synthetic>` api-error).
+Costs are asserted only *within* a binary (R2/R3), never against the
+manifest, so pricing-table changes can't break the check.
+
+## CI
+
+```yaml
+- run: cargo build --release
+- run: scripts/rewrite-retention-check/run_check.sh
+```
+
+Unix (macOS/Linux) with python3 ≥ 3.9 on PATH; Windows untested (WSL works).
+
+## Provenance
+
+Distilled from the validation run posted in #200: all four assertions passed
+against a ~50-day real corpus (3.5.9 vs 3.6.0, frozen-snapshot A/B), with
+3.6.0's main-transcript totals token-exact against an independent
+append-only ingest log. The check fails on 3.5.9 (R2) and passes on 3.6.0.

scripts/rewrite-retention-check/check_retention.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""Assert splitrail's Claude Code rewrite-retention behavior on the synthetic corpus.
+
+Reads scan outputs produced by run_check.sh from --work:
+
+    corpus-manifest.json  drift-manifest.json
+    base.json  post.json  stab-1..3.json
+    base-baseline.json / post-baseline.json   (only with --baseline)
+
+Assertions
+    R1 parse+dedup sanity : base scan == corpus manifest (per-model msgs + 4 token fields)
+    R2 rewrite retention  : post scan == base scan (incl. cost — THE regression guard)
+    R3 restart stability  : stab-1..3 byte-identical
+    R4 drift demo         : baseline binary drops by exactly the removed usage
+                            (convention-agnostic: last_line or sum_lines)
+
+Exit 0 iff all applicable assertions pass.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+TOKEN_KEYS = ("inputTokens", "outputTokens", "cacheCreationTokens", "cacheReadTokens")
+MANIFEST_TO_SR = {
+    "input_tokens": "inputTokens",
+    "output_tokens": "outputTokens",
+    "cache_creation_input_tokens": "cacheCreationTokens",
+    "cache_read_input_tokens": "cacheReadTokens",
+}
+
+
+def cc_by_model(path: Path) -> dict[str, dict]:
+    data = json.loads(path.read_text())
+    out: dict[str, dict] = defaultdict(
+        lambda: {k: 0 for k in ("messageCount", *TOKEN_KEYS)} | {"cost": 0.0})
+    for a in data.get("analyzer_stats", []):
+        if a.get("analyzer_name") != "Claude Code":
+            continue
+        for day in (a.get("daily_stats") or {}).values():
+            for model, ms in (day.get("model_stats") or {}).items():
+                slot = out[model]
+                slot["messageCount"] += int(ms.get("messageCount") or 0)
+                for k in TOKEN_KEYS:
+                    slot[k] += int(ms.get(k) or 0)
+                slot["cost"] += float(ms.get("cost") or 0.0)
+    return dict(out)
+
+
+def int_view(by_model: dict[str, dict]) -> dict[str, dict]:
+    return {m: {k: int(v[k]) for k in ("messageCount", *TOKEN_KEYS)}
+            for m, v in by_model.items()}
+
+
+def diff(a: dict[str, dict], b: dict[str, dict]) -> dict[str, dict]:
+    out: dict[str, dict] = {}
+    for m in sorted(set(a) | set(b)):
+        d = {k: a.get(m, {}).get(k, 0) - b.get(m, {}).get(k, 0)
+             for k in ("messageCount", *TOKEN_KEYS)}
+        if any(d.values()):
+            out[m] = d
+    return out
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--work", type=Path, required=True)
+    ap.add_argument("--baseline", action="store_true")
+    args = ap.parse_args()
+    w = args.work
+
+    corpus = json.loads((w / "corpus-manifest.json").read_text())["expected_by_model"]
+    base = cc_by_model(w / "base.json")
+    post = cc_by_model(w / "post.json")
+
+    results: list[tuple[str, bool, str]] = []
+
+    # R1 — parse + dedup sanity
+    expected = {m: {"messageCount": v["messages"],
+                    **{MANIFEST_TO_SR[k]: v[k] for k in MANIFEST_TO_SR}}
+                for m, v in corpus.items()}
+    got = int_view(base)
+    results.append(("R1 parse+dedup sanity (base == corpus manifest)",
+                    got == expected,
+                    "exact match" if got == expected
+                    else f"got {got} expected {expected}"))
+
+    # R2 — rewrite retention (the regression this fixture exists for)
+    same = int_view(post) == int_view(base) and all(
+        abs(post.get(m, {}).get("cost", 0.0) - base.get(m, {}).get("cost", 0.0)) < 1e-9
+        for m in set(base) | set(post))
+    cost_delta = {m: round(base.get(m, {}).get("cost", 0.0) - post.get(m, {}).get("cost", 0.0), 6)
+                  for m in set(base) | set(post)
+                  if abs(base.get(m, {}).get("cost", 0.0) - post.get(m, {}).get("cost", 0.0)) >= 1e-9}
+    results.append(("R2 rewrite retention (post == base)", same,
+                    "totals unchanged" if same
+                    else f"drifted: {diff(int_view(base), int_view(post))} cost_delta: {cost_delta}"))
+
+    # R3 — restart stability
+    blobs = [(w / f"stab-{i}.json").read_bytes() for i in (1, 2, 3)]
+    ok = all(b == blobs[0] for b in blobs)
+    results.append(("R3 restart stability (3 runs byte-identical)", ok,
+                    "byte-identical" if ok else "outputs differ"))
+
+    # R4 — drift demonstration on a pre-fix baseline binary
+    if args.baseline:
+        drift = json.loads((w / "drift-manifest.json").read_text())["expected_delta_by_model"]
+        b_old = int_view(cc_by_model(w / "base-baseline.json"))
+        p_old = int_view(cc_by_model(w / "post-baseline.json"))
+        drop = diff(b_old, p_old)
+        matched = None
+        for conv, count_key in (("last_line", "messages"), ("sum_lines", "lines")):
+            exp = {m: {"messageCount": v[count_key],
+                       **{MANIFEST_TO_SR[k]: v[conv][k] for k in MANIFEST_TO_SR}}
+                   for m, v in drift.items()}
+            if drop == exp:
+                matched = conv
+                break
+        results.append(("R4 baseline drift demo (old bin drops removed usage)",
+                        matched is not None,
+                        f"matches ({matched})" if matched
+                        else f"drop {drop} vs manifest {drift}"))
+
+    width = max(len(n) for n, _, _ in results)
+    all_ok = True
+    for name, ok, detail in results:
+        all_ok &= ok
+        print(f"{'PASS' if ok else 'FAIL'}  {name:<{width}}  {detail}")
+    print("RESULT:", "ALL PASS" if all_ok else "FAILURES")
+    return 0 if all_ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/rewrite-retention-check/gen_corpus.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Generate a synthetic Claude Code corpus with a manifest of expected totals.
+
+Writes main transcripts (``<slug>/<sessionId>.jsonl``) under ROOT, which
+should be ``<isolated-home>/.claude/projects``.
+
+Realism the check depends on:
+
+- Each assistant API message is written as TWO JSONL lines with distinct line
+  ``uuid``s but identical ``message.id`` / ``requestId`` / ``usage`` —
+  mirroring Claude Code's streaming duplication — so the scan exercises the
+  analyzer's local_hash + token-fingerprint deduplication.
+- Skip-cases that must NOT count toward totals: user lines, a ``summary``
+  line, a ``file-history-snapshot`` line, and one ``<synthetic>`` api-error
+  line per session.
+
+The manifest records expected per-model totals (message counts and the four
+token fields). Cost is intentionally excluded — pricing tables change.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import uuid
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+MODELS = ("claude-opus-4-6", "claude-sonnet-4-20250514")
+
+
+def line(obj: dict) -> str:
+    return json.dumps(obj, separators=(",", ":")) + "\n"
+
+
+def assistant_lines(ts: datetime, sid: str, model: str, usage: dict) -> list[str]:
+    mid = "msg_" + uuid.uuid4().hex[:16]
+    rid = "req_" + uuid.uuid4().hex[:16]
+    out = []
+    for stop in (None, "end_turn"):  # two streaming lines, identical usage
+        out.append(line({
+            "type": "assistant",
+            "timestamp": ts.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
+            "cwd": "/home/user/demo",
+            "sessionId": sid,
+            "uuid": str(uuid.uuid4()),
+            "parentUuid": None,
+            "requestId": rid,
+            "version": "2.1.198",
+            "message": {
+                "id": mid, "type": "message", "role": "assistant",
+                "model": model, "stop_reason": stop,
+                "content": [{"type": "text", "text": "ok"}],
+                "usage": usage,
+            },
+        }))
+    return out
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("root", type=Path, help="projects root to create")
+    ap.add_argument("--manifest", type=Path, required=True)
+    ap.add_argument("--sessions", type=int, default=3)
+    ap.add_argument("--messages", type=int, default=12)
+    args = ap.parse_args()
+
+    proj = args.root / "-home-user-demo"
+    proj.mkdir(parents=True, exist_ok=True)
+    base = datetime(2026, 7, 1, 9, 0, 0, tzinfo=timezone.utc)
+
+    expected: dict[str, dict] = {
+        m: {"messages": 0, "input_tokens": 0, "output_tokens": 0,
+            "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}
+        for m in MODELS
+    }
+
+    for s in range(args.sessions):
+        sid = str(uuid.uuid4())
+        rows: list[str] = []
+        # summary line (skip-case)
+        rows.append(line({"type": "summary", "summary": f"Demo session {s}",
+                          "leafUuid": str(uuid.uuid4())}))
+        # file-history-snapshot line (skip-case)
+        rows.append(line({"type": "file-history-snapshot",
+                          "messageId": str(uuid.uuid4()), "snapshot": {}}))
+        for i in range(args.messages):
+            model = MODELS[i % len(MODELS)]
+            ts = base + timedelta(days=s % 2, hours=s, minutes=7 * i)
+            usage = {
+                "input_tokens": 1000 + 137 * s + 7 * i,
+                "output_tokens": 200 + 3 * i,
+                "cache_read_input_tokens": 5000 + 11 * i,
+                "cache_creation_input_tokens": 300 + i,
+            }
+            # user line before each assistant message (skip-case)
+            rows.append(line({
+                "type": "user", "timestamp": ts.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
+                "sessionId": sid, "uuid": str(uuid.uuid4()),
+                "message": {"role": "user", "content": "do the thing"},
+            }))
+            rows.extend(assistant_lines(ts, sid, model, usage))
+            slot = expected[model]
+            slot["messages"] += 1
+            for k, v in usage.items():
+                slot[k] += v
+        # one <synthetic> api-error line (skip-case)
+        rows.append(line({
+            "type": "assistant", "timestamp": base.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
+            "sessionId": sid, "uuid": str(uuid.uuid4()), "isApiErrorMessage": True,
+            "message": {"role": "assistant", "model": "<synthetic>",
+                        "content": [{"type": "text", "text": "error"}]},
+        }))
+        (proj / f"{sid}.jsonl").write_text("".join(rows), encoding="utf-8")
+
+    args.manifest.parent.mkdir(parents=True, exist_ok=True)
+    args.manifest.write_text(json.dumps({"expected_by_model": expected}, indent=2),
+                             encoding="utf-8")
+    total = sum(v["messages"] for v in expected.values())
+    print(f"corpus: {args.sessions} sessions, {total} assistant messages -> {proj}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/rewrite-retention-check/run_check.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# End-to-end regression check: Claude Code usage must survive session rewrites.
+#
+# Guards the #204 fix (issue #200): resume/compact rewrites session JSONL
+# files in place; totals must not drift because the history store restores
+# retained records.
+#
+# Runs entirely against a synthetic corpus under an isolated $HOME:
+# no real ~/.claude data touched, no config, no upload path.
+#
+# Usage:
+#   scripts/rewrite-retention-check/run_check.sh
+#
+# Env:
+#   SPLITRAIL_BIN           binary under test (default: target/release/splitrail)
+#   SPLITRAIL_BASELINE_BIN  optional pre-fix binary (e.g. 3.5.9) to also
+#                           demonstrate the drift it exhibited (assertion R4)
+#   WORK                    scratch dir (default: mktemp; kept on failure)
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+BIN="${SPLITRAIL_BIN:-$REPO_ROOT/target/release/splitrail}"
+BASE_BIN="${SPLITRAIL_BASELINE_BIN:-}"
+WORK="${WORK:-$(mktemp -d "${TMPDIR:-/tmp}/splitrail-rrc.XXXXXX")}"
+FAKEHOME="$WORK/home"
+
+if [ ! -x "$BIN" ]; then
+  echo "error: splitrail binary not found at $BIN" >&2
+  echo "build it first (cargo build --release) or set SPLITRAIL_BIN" >&2
+  exit 2
+fi
+
+echo "binary under test : $("$BIN" --version)  [$BIN]"
+if [ -n "$BASE_BIN" ]; then
+  echo "baseline binary   : $("$BASE_BIN" --version)  [$BASE_BIN]"
+fi
+echo "work dir          : $WORK"
+
+scan() { # $1=binary $2=outfile
+  # HOME plus explicit XDG overrides: on Linux the history store honors
+  # XDG_STATE_HOME/XDG_DATA_HOME, which would otherwise escape the isolation.
+  HOME="$FAKEHOME" \
+  XDG_STATE_HOME="$FAKEHOME/.local/state" \
+  XDG_DATA_HOME="$FAKEHOME/.local/share" \
+    "$1" stats --pretty > "$2" 2> "$2.err" \
+    || { echo "scan failed; see $2.err" >&2; exit 1; }
+}
+
+python3 "$SCRIPT_DIR/gen_corpus.py" "$FAKEHOME/.claude/projects" \
+  --manifest "$WORK/corpus-manifest.json"
+
+# Baseline scans of the pristine corpus. The bin-under-test scan also
+# populates its history store (under the isolated $HOME) pre-rewrite.
+scan "$BIN" "$WORK/base.json"
+if [ -n "$BASE_BIN" ]; then
+  scan "$BASE_BIN" "$WORK/base-baseline.json"
+fi
+
+# Simulated resume/compact: drop the last 3 assistant message-groups.
+python3 "$SCRIPT_DIR/simulate_rewrite.py" "$FAKEHOME/.claude/projects" \
+  --drop 3 --manifest "$WORK/drift-manifest.json"
+
+# Post-rewrite scans + restart-stability loop.
+scan "$BIN" "$WORK/post.json"
+for i in 1 2 3; do scan "$BIN" "$WORK/stab-$i.json"; done
+if [ -n "$BASE_BIN" ]; then
+  scan "$BASE_BIN" "$WORK/post-baseline.json"
+fi
+
+if python3 "$SCRIPT_DIR/check_retention.py" --work "$WORK" \
+     ${BASE_BIN:+--baseline}; then
+  rm -rf "$WORK"
+else
+  echo "scratch kept for inspection: $WORK" >&2
+  exit 1
+fi