diff --git a/.gitignore b/.gitignore index 8b2374b..2674f30 100644 --- a/.gitignore +++ b/.gitignore @@ -118,8 +118,8 @@ roro-workspace-*.png # On-device voice runtime assets (Silero/ORT wasm) + model weights (whisper STT, Kokoro TTS) — staged # regenerable into the voice sub-package by packages/voice/scripts/stage-voice-assets.mjs. Never committed. packages/voice/public/ -src/brain/eval/latest.json -src/brain/eval/proposalLatest.json +src/core/brain/eval/latest.json +src/core/brain/eval/proposalLatest.json # Voice runtime/model staging (dev-machine only; the app ships WITHOUT voice — packages/voice owns its own staging) public/models/ diff --git a/HANDOFF.md b/HANDOFF.md index c57aa23..43c0214 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -179,14 +179,47 @@ Every turn flows through **one** path — now the typed run state machine under - **W7 (#149):** the **Electron-free `src/core/` split** + the enforced import boundary + the four platform ports + `userData` as a boot parameter; tests reach the platform via `installTestPorts`, not `vi.mock('electron')`. - **W8:** this reconciliation (docs) + the packaged smoke suite consolidation (a parallel track). +- **The floating redesign — resting purity (#153–#160, 2026-07-03 → 07-04).** At rest the cat is ALL there is: + the pill is deleted; input is a hover-summoned bubble (#157) with an Ask·Memory·Project menu + tethered + popovers (#158) — the moat surface (Memory panel) is reachable *in the pet*. Plus the stage + pure tether + positioner + WorkingChip (#156), a titled dismissible AlertCard for run failures (#155), click-through proof + with a live cat-bounds hit-test (#154), and the tethered **SpeechBubble with the truthful memory receipt** + (#159) — the receipt renders ONLY on `run.completed.finalText` in the same run whose recall actually fired + (the truthful-receipt gate). `docs/INTERACTION.md` + smokes reconciled to the new reality (#160); the + executor-facts proposal IPC is exposed only under its flag (#153). +- **Earn the moat (#161, 2026-07-05).** The **codex `finalText` stitch** lights the truthful receipt on the + DEFAULT executor path (not just answers), plus honest copy and the **real-steering guard**: + `npm run verify:memory-steered-real` (opt-in, live qwen2.5:3b + real codex over N treatment/control runs) + first measured **5/5 args.task · 5/5 marker-in-file · 0/5 control** — recalled memory demonstrably steers + real coding work on the default path. +- **First founder dogfooding session (2026-07-05).** Four NAMED failure classes the green suite never met — + **deflect-on-vague** (a vague creative prompt drew a chat deflection, failing ROADMAP §6's + lands-or-clarifies), **feedback-fade** (answers/clarify fade in ≤8s with no reply affordance), + **working-illegibility** (working reads as idle; the cat can fall asleep mid-run), and + **env-overrides-config** (a stale `RORO_WORKDIR` silently outranks the configured project). Loci + status in + `LESSONS.md` (Dogfooding). Class 3's sleep-mid-run half + class 4 → the dogfood-blocker slice; class 2 + (feedback-fade) and class 3's working-pose half are OPEN → the legible-moment slice; class 1 is + fixture-covered in the eval (`vague-initiative`) with the fix deferred. +- **Signal-reconcile slice (2026-07-06).** The eval instrument repaired — post-W7 both live runners crashed + ENOENT writing to stale `src/brain/eval` paths; the path family now lives in `eval/paths.ts` and is + CI-pinned (`paths.test.ts`). The deflect-on-vague class entered the DECIDE fixtures (5 `vague-initiative` + cases, lands-or-clarifies acceptance `{run_agent, clarify}`), and `baseline.json` was refreshed via + `--write-baseline` to the current runner shape (see §4). --- -## 4. The eval scorecard (`src/core/brain/eval/baseline.json`, qwen2.5:3b, temp 0 — updates only via `npm run eval:brain -- --write-baseline`; `latest.json` is the per-run scratch) +## 4. The eval scorecard (`src/core/brain/eval/baseline.json`, qwen2.5:3b; decide streams at temp 0.3, extract at temp 0 — updates only via `npm run eval:brain -- --write-baseline`; `latest.json` is the per-run scratch) - **DECIDE 100%** (22/22) — the clarify trust gate now catches referent-less requests before the model. - **EXTRACT (null-discipline) 100%** (10/10) — the gate holds; no profile poisoning. - **BEHAVIORAL value-quality 40%** (2/5, the original fixture set) — a real 2× gain (#65) **and a measured model ceiling** (the 3B is genuinely weak at describing habits; noun prefs extract cleanly). The guard guarantees no *garbage* is stored — it just sometimes stores nothing. - **W3 expansion (#145) — behavioral fixtures 5→50** (7-tag taxonomy, pairwise+prompt-4-gram disjointness CI-checked), a separate **null-discipline axis**, and the `eval:proposals` runner (live opt-in + a deterministic CI tier). Live **report-only** numbers (baseline untouched): value-quality **74.3%** on the expanded set (the original 5 reproduce 40% exactly), null-discipline **46.7%** (hard negatives are where the 3B invents facts from marker-bearing questions — the gate is the only null discipline). The panel also armed the scorer against **polarity inversion** (an `inverted` mode via `mustAlsoContainOneOf`/`mustNotContainAnyOf`) after it scored "never force push"→"force-pushes" as `ok` — see `LESSONS.md`. +- **Baseline refreshed (2026-07-06, `--write-baseline`)** — the checked-in reference now carries the current + runner shape and the full fixture set (the three bullets above are the historical pre-refresh snapshot): + **DECIDE 96.3%** (26/27 — now includes the 5 `vague-initiative` lands-or-clarifies cases, all 5 ok this + run; the one miss is `run-writetest`→answer, a temperature-0.3 snapshot artifact), **EXTRACT 100%** + (10/10), **BEHAVIORAL value-quality 74.3%** (26/35), **null-discipline 46.7%** (7/15; all 8 misses are + hard-negative `false_fact`s), plus the per-taxonomy breakdown (hard-negative 20% is the weak family; + multi-fact/supersede/marker-less 100%). --- @@ -228,6 +261,15 @@ memory/embeddings). Add new expensive lessons there, not here. 2. **Produce the Developer-ID notarized build:** export `APPLE_TEAM_ID=`, `APPLE_ID`, and app-specific `APPLE_PASSWORD` in the same shell, then run `npm run verify:signing-readiness`, `npm run verify:signing-auth`, `npm run make`, `npm run verify:release-artifact:dmg`, and `npm run verify:release-artifact:signed`; validate install + memory recall on a clean second Mac. 3. **Validate Phase 2 trust on real first turns:** the Memory panel can see/fix/verify/source/forget facts, referent-less requests clarify before dispatch, the README leads with the job/privacy promise, and screen reads show a bounded one-snapshot tell. +> **Update (2026-07-06) — the corrected forward order** (absorbs the first dogfooding session, the floating +> redesign #153–#160, and earn-the-moat #161): **1) dogfood-blocker fixes** (sleep-mid-run / +> env-overrides-config / vision-timeout truthfulness — the sibling slice), **2) signing** (the Developer-ID +> notarized build + clean-Mac gate — item 2 above), **3) the legible-moment slice** (feedback-fade + +> the working pose + the receipt naming the recalled fact), **4) the +> executor-facts pilot** (toward a default-on decision), **5) self-rehearsal**, **6) demo**, **7) the +> stranger magic-moment session** — still the real lever; everything earlier exists to make it credible. +> Deflect-on-vague is measured (the `vague-initiative` eval class) with its fix deferred behind the blockers. + --- ## 9. Founder decisions + open questions diff --git a/LESSONS.md b/LESSONS.md index 66bd875..792a9db 100644 --- a/LESSONS.md +++ b/LESSONS.md @@ -119,6 +119,45 @@ current code — `src/renderer/bootstrap.ts` "cursor movement must NOT keep the on every cursor sample, so the cat could never reach `asleep` (near-zero-idle violated). Gaze must never wake the cat. +## Dogfooding + +- **The first founder dogfooding session (2026-07-05) surfaced four failure classes in one sitting + that a ~1.1k-test green suite and six review rounds never met** — the suite measures what we + encoded, not what a user meets. Each is a NAMED class so fixes and fixtures can reference it: + - **deflect-on-vague** — "create something interesting" drew "Let's brainstorm a fun project idea + together" (`command=answer`): a chat deflection, failing ROADMAP §6's own lands-or-clarifies bar + (do the task or ask a *pertinent* question — `docs/ROADMAP.md` Arc A step 3). Locus: the DECIDE + contract has no vague-initiative guidance (`SYSTEM_PROMPT`, `src/core/brain/index.ts:97`) and the + deterministic clarify gate's regexes don't cover creative-vague (`src/core/brain/clarifyGate.ts:9`). + Status: **fixture-first now, fix later** — 5 `vague-initiative` DECIDE fixtures with a + lands-or-clarifies acceptance set (`src/core/brain/eval/fixtures.ts`) + CI pins (acceptance set + exactly `{run_agent, clarify}`; must not be gate-caught). **Measurement twist (2026-07-06 + baseline refresh):** the current 3B lands-or-clarifies on all 5 paraphrased fixtures — and 30/30 + in a 3×-repeat probe, with and without a memory section — so the live deflection has NOT + reproduced from paraphrase; the fixtures pin the class against regression while the real repro + needs a `RORO_TRACE` capture of an actual deflecting turn promoted into a fixture. *Corollary + law:* **a live failure paraphrased into a fixture may not reproduce — capture the real turn + (trace) before trusting a hand-written golden case to carry it.** + - **feedback-fade** — answers and clarify questions auto-fade in ≤8s with no reply affordance + (`fadeDwellMs` = clamp(words×220ms, 2.5s, 8s) — `src/renderer/reactions/speechBubble.ts:44`); a + clarify question that fades unanswered is a dead end. Status: OPEN — deliberately NOT in the + dogfood-blocker slice; it belongs to the legible-moment slice (persistent answer/clarify bubble + with an inline reply), the next scheduled build item. + - **working-illegibility** — a running executor is barely distinguishable from idle, and the cat + can fall ASLEEP mid-run: energy is computed purely from interaction idleness even while busy + (`src/renderer/character/avatar.ts:632`; 45s/120s thresholds — `src/renderer/character/activity.ts:13`; + no run-lifecycle event pokes the clock — only direct interaction does (petting, + `src/renderer/bootstrap.ts:241`; summoning the ask, `src/renderer/ask/askMachine.ts:43`)). Status: + the sleep-mid-run half is fixed in the sibling dogfood-blocker slice (setBusy wired into floating + runs); a distinct WORKING pose remains open for the legible-moment slice. + - **env-overrides-config** — a stale `RORO_WORKDIR` in the launching shell silently outranks the + user's persisted project choice (`tryResolveWorkdir` checks env first, no tell — + `src/core/orchestrator/workdir.ts:29`). Status: being fixed in the sibling dogfood-blocker slice. + + *Generalizable law:* **every dogfooding finding becomes a NAMED failure class with a file:line + locus and either a fix or a fixture before the session's signal decays — a green suite is evidence + of encoded expectations, not of user-meeting behavior.** + ## Memory / embeddings - **Every cheap substrate stays cheap after the corpus exists — except the embedding vector space, diff --git a/README.md b/README.md index 1897352..7ae20ec 100644 --- a/README.md +++ b/README.md @@ -80,12 +80,12 @@ implementation can be mounted later behind the same interface. flowchart LR user["User task
text by default"] --> renderer["Renderer
cat, captions, controls"] renderer --> main["Electron main
typed IPC"] - main --> memoryRecall["PGlite recall
pgvector (local)"] + main --> memoryRecall["memory2 recall
local index over encrypted files"] memoryRecall --> brain["Local Ollama brain
qwen2.5 decision"] brain --> executor["Executor adapter
Codex or Claude"] executor --> events["Canonical action events"] events --> renderer - events --> memoryWrite["PGlite remember (local)"] + events --> memoryWrite["memory2 remember (local)"] brain --> vision["Local vision
optional screen read"] ``` @@ -93,9 +93,9 @@ flowchart LR | --- | --- | --- | | Electron shell | windowing, IPC, macOS permission checks, floating mode | [`src/main/`](src/main/) | | Character | pixel cat, state machine, lip sync facade | [`src/renderer/character/`](src/renderer/character/) | -| Brain | local Ollama reasoning/vision/embeddings | [`src/brain/`](src/brain/) | -| Memory | encrypted files-as-truth + PGlite-HNSW hybrid recall (local, owner-scoped) | [`src/memory2/`](src/memory2/) | -| Executor | Codex and Claude stream adapters | [`src/executor/`](src/executor/) | +| Brain | local Ollama reasoning/vision/embeddings | [`src/core/brain/`](src/core/brain/) | +| Memory | encrypted files-as-truth + in-memory vector index (local, owner-scoped) | [`src/core/memory2/`](src/core/memory2/) | +| Executor | Codex and Claude stream adapters | [`src/core/executor/`](src/core/executor/) | | Voice | internal on-device VAD + STT + TTS seam (Silero / whisper / Kokoro), hidden behind dev flags and cut from v0 | [`src/renderer/voice/`](src/renderer/voice/) | | Shared contracts | typed IPC, action events, avatar states | [`src/shared/`](src/shared/) | @@ -155,7 +155,7 @@ full prompt, controls, captions, memory panel, and action timeline visible. ## Configuration Roro's default brain and memory paths need **no app-owned cloud/model keys**: -Ollama runs locally, memory is local PGlite + pgvector, and packaged builds +Ollama runs locally, memory is a local encrypted files-as-truth store, and packaged builds store the chosen working repo in `userData/config.json`. A local `.env` (see [`.env.example`](.env.example)) is for development overrides, model tuning, and optional cloud/executor paths: @@ -174,7 +174,7 @@ RORO_WORKDIR=/absolute/path/to/scratch-git-repo ANTHROPIC_API_KEY=... # optional, only for the Claude executor ``` -Memory is local PGlite + pgvector under the app's userData dir (`RORO_DB_DIR` to +Memory is a local encrypted files-as-truth store under the app's userData dir (`RORO_DB_DIR` to override) — no external database. See [`RUN.md`](RUN.md) for the on-device voice dev flags, macOS permissions, and the full live-run checklist. @@ -232,7 +232,7 @@ What is working: - DMG release artifact: macOS CI builds a versioned `.dmg` and verifies it mounts with `Roro.app` - **local Ollama brain** (decide/vision/embeddings) — verified end-to-end against a live daemon -- **local PGlite + pgvector memory** (owner-scoped, survives restarts) +- **local encrypted files-as-truth memory** (owner-scoped, survives restarts) - Codex and Claude executor adapters behind one event stream - typed text path + the on-device voice control core/seam hidden behind dev flags @@ -247,9 +247,9 @@ What needs extra setup or a real device: ```text src/main/ Electron main process and orchestration src/renderer/ UI, character, voice, captions, event wiring -src/brain/ local Ollama decision, vision, embeddings -src/memory2/ local encrypted files-as-truth + PGlite-HNSW memory -src/executor/ Codex and Claude adapters +src/core/brain/ local Ollama decision, vision, embeddings +src/core/memory2/ local encrypted files-as-truth + in-memory vector index +src/core/executor/ Codex and Claude adapters src/shared/ IPC, event, memory, avatar, env contracts RUN.md live setup and integration guide ``` diff --git a/RUN.md b/RUN.md index 3f3509a..c748ceb 100644 --- a/RUN.md +++ b/RUN.md @@ -1,7 +1,7 @@ # Roro — Run & Integration Guide Roro is **local-first**: the brain runs on a local Ollama daemon and memory is an -in-process PGlite + pgvector store. The default brain/memory path needs **no app +in-process encrypted files-as-truth store with a local vector index (no database). The default brain/memory path needs **no app API keys** and makes no cloud model calls. The steps below bring the full app alive on your Mac. ## 1. Local brain — install Ollama + pull core models @@ -17,7 +17,7 @@ ollama pull qwen2.5vl:7b # vision (describeScreen) — needs substantial RA On boot, `main` runs a non-blocking brain preflight: success logs the active model; failure (daemon down / model missing) still renders the window and surfaces a diagnostic — it never silently falls back to the cloud. Verify end-to-end with the -opt-in smoke: `OLLAMA_AVAILABLE=1 npx vitest run src/brain/integration.test.ts` +opt-in smoke: `OLLAMA_AVAILABLE=1 npx vitest run src/core/brain/integration.test.ts` (see [`docs/WS1-OLLAMA-INTEGRATION-TEST.md`](docs/WS1-OLLAMA-INTEGRATION-TEST.md)). ## 2. Optional `.env` — model tuning @@ -39,16 +39,20 @@ ANTHROPIC_API_KEY=... # only if you use the Claude exec ``` > Switching the embed model changes the vector geometry. `OLLAMA_EMBED_DIM` must match -> the model's output (preflight probes it and fails loud on a mismatch), and the memory -> store's `vector(N)` column is fixed at creation — re-embedding is not automatic, so -> move/delete the memory dir when changing embedders. +> the model's output (preflight probes it and fails loud on a mismatch). Memory itself is +> files-as-truth; the `vectors.jsonl` sidecar is a derived, zero-authority cache with an +> (embedModel, dim) identity header — on an embedder change the store FAILS LOUD (identity +> refusal), and the remedy is deleting the derived `index/` subdirectory only (safe: the next +> launch re-embeds from the durable files). Never delete the memory dir itself — it holds the +> durable encrypted memories, not just an index. -## 3. Memory — local PGlite + pgvector (no setup) +## 3. Memory — local encrypted store (no setup) -Memory is an in-process PGlite database with pgvector, stored under the app's userData +Memory is encrypted files-as-truth (JSON + a rebuilt-on-launch in-memory vector index; +embeddings via local nomic-embed-text), stored under the app's userData dir (override with `RORO_DB_DIR`). It is owner-scoped and survives restarts — **no external database, no SQL provisioning, no keys**. A taught fact in one launch is -recalled in the next (proven by `src/main/memorySpine.crosslaunch.test.ts`). +recalled in the next (proven by `src/core/orchestrator/memorySpine.crosslaunch.test.ts`). ## 4. The avatar — a hand-built pixel-art cat (default; no assets needed) @@ -106,8 +110,8 @@ npm start Summon (Cmd+Shift+Space) → type a task in the floating Ask (or the dev prompt box) → the cat thinks (driven by the local brain's content stream) → drives Codex in `RORO_WORKDIR` -(each action narrated + animated) → writes to local PGlite memory. Then ask -*"what did we do?"* for the pgvector recall beat. +(each action narrated + animated) → writes to local encrypted memory. Then ask +*"what did we do?"* for the semantic-recall beat. Roro's brain and memory setup does not need an app-owned cloud key, but the coding executor is your own CLI. Before expecting a coding turn to edit files, authenticate or configure Codex (or diff --git a/docs/COHORT_TRACE_TO_EVAL.md b/docs/COHORT_TRACE_TO_EVAL.md index 111dddb..f553a75 100644 --- a/docs/COHORT_TRACE_TO_EVAL.md +++ b/docs/COHORT_TRACE_TO_EVAL.md @@ -65,7 +65,7 @@ in the local review packet until a future cohort repeats the pattern. After adding redacted fixtures: ```sh -npx vitest run --no-file-parallelism src/brain/eval/fixtures.test.ts src/brain/eval/cohortTraceReview.test.ts +npx vitest run --no-file-parallelism src/core/brain/eval/fixtures.test.ts src/core/brain/eval/cohortTraceReview.test.ts npm run eval:brain ``` diff --git a/docs/PHASE2-TRUST-LOOP.md b/docs/PHASE2-TRUST-LOOP.md index 785bd51..8d28038 100644 --- a/docs/PHASE2-TRUST-LOOP.md +++ b/docs/PHASE2-TRUST-LOOP.md @@ -138,8 +138,8 @@ Accessibility status: Implemented gates: -- `memory2/profileFacts.test.ts`: safe projection, snake/camel source tolerance, owner-scoped fix/verify/source, stale ids, failed replacement keeps old active. -- `memory2/adapter.test.ts`: `profileFacts`, `fixFact`, `verifyFact`, and `factSource`; blank/wrong-owner rejection; fix persists across reopen; embed failure leaves old active. +- `src/core/memory2/profileFacts.test.ts`: safe projection, snake/camel source tolerance, owner-scoped fix/verify/source, stale ids, failed replacement keeps old active. +- `src/core/memory2/facade.test.ts` (successor to the deleted `adapter.test.ts`): `profileFacts`, `fixFact`, `verifyFact`, and `factSource`; blank/wrong-owner rejection; fix persists across reopen; embed failure leaves old active. - `main/ipc.memory.test.ts`: `fixFact`, `verifyFact`, and `factSource` inject owner id MAIN-side; malicious extra owner/key fields are ignored; stale ids do not fall back to another mutation. - `preload` / ambient types: `window.memory` shape matches the actual bridge. - `forgetPanel.test.ts`: fix success updates the row; failed fix/verify/source keeps the row retryable; source renders via `textContent`; forget remains two-step. @@ -149,7 +149,7 @@ Before calling Phase 2 done: ```sh npx tsc --noEmit -p tsconfig.json -npx vitest run --no-file-parallelism src/main/factStore.test.ts src/main/ipc*.test.ts src/memory2/profileFacts.test.ts src/memory2/adapter.test.ts src/memory2/memoryStore*.test.ts src/renderer/memory/forgetPanel.test.ts src/main/memoryContext.test.ts +npx vitest run --no-file-parallelism src/core/orchestrator/factStore.test.ts src/main/ipc*.test.ts src/core/memory2/profileFacts.test.ts src/core/memory2/facade.test.ts src/core/memory2/memoryStore*.test.ts src/renderer/memory/forgetPanel.test.ts src/core/orchestrator/memoryContext.test.ts npm run verify:packaged-onboarding ``` diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 961a884..44f9b79 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -211,12 +211,12 @@ first-turn/memory validation** — otherwise it is deferred (§7 rule 4); C is s **identical prompt + executor/model settings**, the only difference being memory present vs absent — **both roots share identical `owner.json` (same owner identity) + settings, but each `config.json`'s absolute `workdir` points at its *own* clone** (`config.json` stores the workdir — - `src/main/configStore.ts` — so identical configs would aim both arms at the *same* repo and ruin the + `src/core/orchestrator/configStore.ts` — so identical configs would aim both arms at the *same* repo and ruin the diff); **only the memory *content* then differs** (the control's store is empty; `src/main.ts` derives `RORO_DB_DIR` from `userData` when unset, so seed it explicitly). The control's memory root must be **verified memory-empty** — robustly, by capturing the control's **DECIDE input** and confirming it has **no `KNOWN ABOUT THIS USER:` (facts) and no `RELATED PAST CONTEXT:` - (episodic recall) section** (`src/main/memoryContext.ts` builds DECIDE memory from *both* channels, so a + (episodic recall) section** (`src/core/orchestrator/memoryContext.ts` builds DECIDE memory from *both* channels, so a fact-empty root with relevant episodes still contaminates the control); behavior **present with memory, absent without**. Narration merely *echoing* the fact is **not** a pass. Today this is a **manual disposable-repo check**; a **memory-steered packaged @@ -225,12 +225,12 @@ first-turn/memory validation** — otherwise it is deferred (§7 rule 4); C is s `verify:packaged-first-task` doesn't teach/recall). Because the executor is **stochastic** (a single output diff can false-pass/fail even at identical settings), that smoke should assert **deterministically on prompt-capture — at the *correct hop*:** memory flows **memory → the DECIDE - prompt** (`buildDecisionPrompt`, `brain/index.ts`) **→ the generated `decision.args.task` → the + prompt** (`buildDecisionPrompt`, `src/core/brain/index.ts`) **→ the generated `decision.args.task` → the executor** (`orchestrator.ts`); the executor never sees memory verbatim. So capture **(1) the recalled preference in the DECIDE input** *and* **(2) the preference reflected in the generated `args.task`** — *not* "the executor's prompt" (that tests the wrong hop). (**Capture path — opt-in/local only:** product `turnRun` still exposes only events/runEnd (`src/preload.ts`), `buildDecisionPrompt` remains private - (`src/brain/index.ts`), and the debug `brain.decide` bridge **bypasses** orchestrator recall + (`src/core/brain/index.ts`), and the debug `brain.decide` bridge **bypasses** orchestrator recall (`src/main/ipc.ts`). Use the landed trace capture (`RORO_TRACE=1` + `RORO_TRACE_DECIDE=plaintext`) for local proof packets; raw prompt/task traces stay local and are redacted before any committed artifact.) @@ -248,7 +248,7 @@ first-turn/memory validation** — otherwise it is deferred (§7 rule 4); C is s is auditable — it produces an evidence packet:** {build id; both memory-root paths + the empty-control proof; the exact prompt; the event stream; the generated `args.task`; the resulting diff; observer + date}. **Packet format (privacy-safe):** the **raw** packet (full DECIDE prompt — recalled - facts/episodes, `src/main/memoryContext.ts`) stays **local-only, outside the repo, consented, deleted + facts/episodes, `src/core/orchestrator/memoryContext.ts`) stays **local-only, outside the repo, consented, deleted after review, never committed** (unlike traces, which `COHORT_TRACE_TO_EVAL.md` keeps free of memory text, it necessarily contains memory); the **shareable** form carries only the **synthetic marker + whether it reached `args.task`/the diff** — redaction-safe *because* the marker is synthetic, yet still @@ -301,7 +301,7 @@ first-turn/memory validation** — otherwise it is deferred (§7 rule 4); C is s session** — the founder witnesses the reopen directly (the narrow cohort is chosen precisely so every reopen can be watched). A **local-only launch/turn ledger the tester voluntarily exports** would also qualify (consented self-export, never auto-sent), **but it does not exist yet**: today's `RORO_TRACE` - records recall/eval diagnostics, **not** launch/turn/reopen behavior (`src/memory2/tracer.ts`, + records recall/eval diagnostics, **not** launch/turn/reopen behavior (`src/core/memory2/tracer.ts`, `COHORT_TRACE_TO_EVAL.md`), so it is a **tracked prerequisite, not a current capability** — the **prerequisite for `../PUBLIC.md`'s Phase 3 *broaden* gate:** a local-only, consented, tester-exportable log of launch/turn/reopen events; @@ -328,7 +328,15 @@ first-turn/memory validation** — otherwise it is deferred (§7 rule 4); C is s `RORO_TRACE` is **not** an attachment metric — default mode carries no transcripts/facts and only seeds eval fixtures (diagnose DECIDE/EXTRACT/recall *failure mechanisms*); `RORO_TRACE_QUERY=plaintext` is **forbidden for cohort runs** unless a tester opts in and the file stays local. Never silent analytics. - 5. **Founder dogfoods daily** (irreplaceable insight). + 5. **Founder dogfoods daily** (irreplaceable insight). **STARTED 2026-07-05** — the first session + produced four named failure classes (loci + status: `../LESSONS.md` "Dogfooding"): **deflect-on-vague** + (a vague creative prompt drew a chat deflection — failing step 3's own lands-or-clarifies bar; now + fixture-covered in the eval as the `vague-initiative` DECIDE class with a {run_agent, clarify} + acceptance set — the 2026-07-06 baseline measures the current model landing-or-clarifying on the + paraphrased fixtures, so the live deflection still needs a `RORO_TRACE`-captured repro; fix deferred), + **feedback-fade** (OPEN → the legible-moment slice), **working-illegibility** (the sleep-mid-run half → + the dogfood-blocker fixes; a distinct working pose → the legible-moment slice), and + **env-overrides-config** (→ the dogfood-blocker fixes). 6. **Before the first *update* ships — a tracked longevity check, separate from the Arc A release** (there is no build N+1 at first release, so this is *not* a first-release precondition): a **two-build update durability** check — install build N, store a fact, install build N+1 under the *same bundle/team diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 9302ff8..2085f4b 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -8,22 +8,22 @@ memory-persistence, or floating-window UX. ```sh npm run eval:trace-review -- /tmp/roro-cohort/tester-01-first-turn.roro-trace.jsonl --out /tmp/roro-cohort/tester-01.roro-trace-review.md -npx vitest run --no-file-parallelism src/brain/eval/cohortTraceReview.test.ts src/brain/eval/fixtures.test.ts src/brain/eval/score.test.ts +npx vitest run --no-file-parallelism src/core/brain/eval/cohortTraceReview.test.ts src/core/brain/eval/fixtures.test.ts src/core/brain/eval/score.test.ts ``` `eval:trace-review` converts a local `RORO_TRACE=1` JSONL file into a privacy-preserving review packet. It summarizes event counts, extraction stages, fact keys, and recall candidate counts without printing transcripts, recall query text, memory result text, narration, or fact values. The packet is a triage artifact only: promote a case into -`src/brain/eval/fixtures.ts` only after human labeling and redaction, then run `npm run eval:brain` to score the live -brain. Raw traces, raw observer notes, and generated cohort packets stay local. +`src/core/brain/eval/fixtures.ts` only after human labeling and redaction, then run `npm run eval:brain` to score the +live brain. Raw traces, raw observer notes, and generated cohort packets stay local. Run this after changes to: -- `src/memory2/tracer.ts` -- `src/main/orchestrator.ts` extraction tracing -- `src/brain/eval/cohortTraceReview*.ts` -- `src/brain/eval/fixtures*.ts` -- `src/brain/eval/runEval.ts` +- `src/core/memory2/tracer.ts` +- `src/core/orchestrator/orchestrator.ts` extraction tracing +- `src/core/brain/eval/cohortTraceReview*.ts` +- `src/core/brain/eval/fixtures*.ts` +- `src/core/brain/eval/runEval.ts` ## Packaged onboarding smoke @@ -40,8 +40,8 @@ saved project. Run this after changes to: -- `src/main/configStore.ts` -- `src/main/workdir.ts` +- `src/core/orchestrator/configStore.ts` +- `src/core/orchestrator/workdir.ts` - `src/main/ipc.ts` workdir channels - `src/preload.ts` workdir bridge - `src/renderer/bootstrap/workdir*.ts` @@ -72,7 +72,7 @@ bridge and banner recover from a daemon-down state and wire the missing-model pa Run this after changes to: -- `src/main/bootstrapPlan.ts` +- `src/core/orchestrator/bootstrapPlan.ts` - `src/main/ipc.ts` bootstrap/model-pull channels - `src/preload.ts` bootstrap/model-pull bridge exposure - `src/renderer/bootstrap/bootstrapBanner.ts` @@ -114,9 +114,9 @@ not prove real Codex authentication; use the opt-in real-Codex smoke for that lo Run this after changes to: -- `src/executor/resolveBin.ts` -- `src/main/executorReadiness.ts` -- `src/main/orchestrator.ts` `run_agent` dispatch/readiness behavior +- `src/core/executor/resolveBin.ts` +- `src/core/orchestrator/executorReadiness.ts` +- `src/core/orchestrator/orchestrator.ts` `run_agent` dispatch/readiness behavior - `src/main/ipc.ts` executor-readiness channels - `src/preload.ts` product bridge exposure - typed or floating Ask executor/workdir/brain gating @@ -126,7 +126,7 @@ Run this after changes to: ## Memory/keychain health diagnostic ```sh -npx vitest run --no-file-parallelism src/memory2/keyManager.test.ts src/memory2/safeStorageWrapper.test.ts src/main/memoryHealthStatusStore.test.ts src/main/memoryHealthStartup.test.ts src/main/ipc.memory.test.ts src/preload.exposure.test.ts src/renderer/bootstrap/memoryHealthBanner.test.ts src/renderer/bootstrap.typedPrompt.test.ts src/renderer/memory/forgetPanel.test.ts +npx vitest run --no-file-parallelism src/core/memory2/keyManager.test.ts src/core/memory2/safeStorageWrapper.test.ts src/core/orchestrator/memoryHealthStatusStore.test.ts src/core/orchestrator/memoryHealthStartup.test.ts src/main/ipc.memory.test.ts src/preload.exposure.test.ts src/renderer/bootstrap/memoryHealthBanner.test.ts src/renderer/bootstrap.typedPrompt.test.ts src/renderer/memory/forgetPanel.test.ts npx tsc --noEmit -p tsconfig.json ``` @@ -156,13 +156,13 @@ memory-health path instead of calling the synchronous Keychain API that can surf Run this after changes to: -- `src/main/memoryHealth*.ts` +- `src/core/orchestrator/memoryHealth*.ts` - `src/main.ts` memory warmup wiring -- `src/main/siblings.ts` memory module loading +- `src/core/orchestrator/siblings.ts` memory module loading - `src/shared/ipc.ts` memory health channels/types - `src/preload.ts` memory health bridge -- `src/memory2/keyManager.ts` -- `src/memory2/safeStorageWrapper.ts` +- `src/core/memory2/keyManager.ts` +- `src/core/memory2/safeStorageWrapper.ts` - `src/renderer/bootstrap/memoryHealthBanner.ts` - `src/renderer/memory/forgetPanel.ts` - `src/renderer/bootstrap.ts` banner mounting @@ -197,7 +197,7 @@ Run this after changes to: - `src/renderer/bootstrap.ts` Memory panel mounting - `src/main/window.ts` smoke flag injection - `src/main.ts` memory warmup scheduling -- `src/main/memoryWarmupFlag.ts` +- `src/core/orchestrator/memoryWarmupFlag.ts` - `src/renderer/config.ts` smoke flag plumbing - `scripts/v0-deferred-env.mjs` - `src/build/v0DeferredEnv.test.ts` @@ -207,7 +207,7 @@ Run this after changes to: ## Destructive command tripwire ```sh -npx vitest run --no-file-parallelism src/main/destructive.test.ts src/main/orchestrator.destructiveCommand.test.ts src/main/orchestrator.stopSlotRetention.test.ts +npx vitest run --no-file-parallelism src/core/orchestrator/destructive.test.ts src/core/orchestrator/orchestrator.destructiveCommand.test.ts src/core/orchestrator/orchestrator.stopSlotRetention.test.ts npx tsc --noEmit -p tsconfig.json ``` @@ -219,9 +219,9 @@ destructive command, subject to the existing clean-git-tree check. Run this after changes to: -- `src/main/destructive.ts` -- `src/main/orchestrator.ts` dispatch/cancel/slot-retention behavior -- `src/executor/*.ts` command-event mapping +- `src/core/orchestrator/destructive.ts` +- `src/core/orchestrator/orchestrator.ts` dispatch/cancel/slot-retention behavior +- `src/core/executor/*.ts` command-event mapping - `src/renderer/confirm/*` ## Packaged memory persistence smoke @@ -262,9 +262,9 @@ replace the non-founder or clean-Mac signed-build gates. Run this after changes to: - `src/main.ts` startup state initialization -- `src/memory2/index.ts` -- `src/memory2/keyManager.ts` -- `src/memory2/encryptionMode.ts` +- `src/core/memory2/index.ts` +- `src/core/memory2/keyManager.ts` +- `src/core/memory2/encryptionMode.ts` - `src/main/ipc.ts` memory channels - `src/preload.ts` memory bridge - packaged startup/signing/fuse behavior in `forge.config.ts` or `src/build/macSigning.ts` @@ -345,7 +345,7 @@ Fast regression coverage: ```sh npx vitest run --no-file-parallelism src/renderer/bootstrap.typedPrompt.test.ts -npx vitest run --no-file-parallelism src/main/orchestrator.stopSlotRetention.test.ts src/executor/abortKill.test.ts +npx vitest run --no-file-parallelism src/core/orchestrator/orchestrator.stopSlotRetention.test.ts src/core/executor/abortKill.test.ts npm run verify:typed-live-turn ``` diff --git a/docs/WS1-OLLAMA-INTEGRATION-TEST.md b/docs/WS1-OLLAMA-INTEGRATION-TEST.md index 4f396b2..bcede15 100644 --- a/docs/WS1-OLLAMA-INTEGRATION-TEST.md +++ b/docs/WS1-OLLAMA-INTEGRATION-TEST.md @@ -25,11 +25,11 @@ works end-to-end on your machine. ## Automated smoke test (opt-in) -`src/brain/integration.test.ts` hits the live daemon. It's **skipped by default** so normal +`src/core/brain/integration.test.ts` hits the live daemon. It's **skipped by default** so normal `npm test` / CI never blocks on a daemon or multi-GB pulls. Run it explicitly: ```sh -OLLAMA_AVAILABLE=1 npx vitest run src/brain/integration.test.ts +OLLAMA_AVAILABLE=1 npx vitest run src/core/brain/integration.test.ts ``` It verifies: the daemon is reachable + models present (`preflight`), `decide()` streams content and diff --git a/src/core/brain/eval/baseline.json b/src/core/brain/eval/baseline.json index 37a1ff0..89ea8c3 100644 --- a/src/core/brain/eval/baseline.json +++ b/src/core/brain/eval/baseline.json @@ -1,11 +1,12 @@ { "brain": "qwen2.5:3b (local Ollama)", "decide": { - "total": 22, - "ok": 22, - "accuracy": 1, + "total": 27, + "ok": 26, + "accuracy": 0.9629629629629629, "byMode": { - "ok": 22 + "ok": 26, + "wrong_command": 1 } }, "extract": { @@ -17,14 +18,87 @@ } }, "behavioral": { - "total": 5, - "ok": 2, - "accuracy": 0.4, + "total": 35, + "ok": 26, + "accuracy": 0.7428571428571429, "byMode": { + "ok": 26, + "too_thin": 7, "missed_fact": 1, - "too_thin": 1, - "ok": 2, "off_topic": 1 } + }, + "behavioralNull": { + "total": 15, + "ok": 7, + "accuracy": 0.4666666666666667, + "byMode": { + "false_fact": 8, + "ok": 7 + } + }, + "behavioralByTaxonomy": { + "noun-preference": { + "total": 10, + "ok": 7, + "accuracy": 0.7, + "byMode": { + "ok": 7, + "too_thin": 3 + } + }, + "behavioral-habit": { + "total": 12, + "ok": 7, + "accuracy": 0.5833333333333334, + "byMode": { + "missed_fact": 1, + "too_thin": 3, + "ok": 7, + "off_topic": 1 + } + }, + "multi-fact": { + "total": 4, + "ok": 4, + "accuracy": 1, + "byMode": { + "ok": 4 + } + }, + "boolean-collapse": { + "total": 5, + "ok": 4, + "accuracy": 0.8, + "byMode": { + "too_thin": 1, + "ok": 4 + } + }, + "supersede": { + "total": 4, + "ok": 4, + "accuracy": 1, + "byMode": { + "ok": 4 + } + }, + "hard-negative": { + "total": 10, + "ok": 2, + "accuracy": 0.2, + "byMode": { + "false_fact": 8, + "ok": 2 + } + }, + "marker-less": { + "total": 5, + "ok": 5, + "accuracy": 1, + "byMode": { + "ok": 5 + } + } } } diff --git a/src/core/brain/eval/cohortTraceReview.ts b/src/core/brain/eval/cohortTraceReview.ts index aeb34f8..54033a5 100644 --- a/src/core/brain/eval/cohortTraceReview.ts +++ b/src/core/brain/eval/cohortTraceReview.ts @@ -133,7 +133,7 @@ export function renderTraceReviewMarkdown(review: TraceReview): string { lines.push(''); lines.push('- This report does not print recall query text, memory result text, transcripts, narration, or fact values.'); lines.push('- Raw trace files and raw observer notes stay local and must not be committed.'); - lines.push('- Commit only manually redacted, generalized fixtures added to `src/brain/eval/fixtures.ts`.'); + lines.push('- Commit only manually redacted, generalized fixtures added to `src/core/brain/eval/fixtures.ts`.'); if (review.plaintextRecallQueries > 0) { lines.push(`- Warning: ${review.plaintextRecallQueries} recall event(s) used plaintext query mode; redact from observer notes before fixture work.`); } diff --git a/src/core/brain/eval/fixtures.test.ts b/src/core/brain/eval/fixtures.test.ts index 90db319..39f02c8 100644 --- a/src/core/brain/eval/fixtures.test.ts +++ b/src/core/brain/eval/fixtures.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from 'vitest'; import { DECIDE_CASES, EXTRACT_CASES, BEHAVIORAL_EXTRACT_CASES, type BehavioralTaxonomy } from './fixtures'; import { FACT_SYSTEM_PROMPT, isPlausiblePreference } from '../extractFact'; +import { clarifyForReferentlessRequest } from '../clarifyGate'; const COMMANDS = ['run_agent', 'answer', 'capture_screen', 'clarify']; @@ -33,6 +34,35 @@ describe('brain eval fixtures — well-formed golden set', () => { expect(DECIDE_CASES.length).toBeGreaterThanOrEqual(20); }); + it("vague-initiative cases: accept exactly {run_agent, clarify} — a deflection ('answer') can never pass", () => { + // The deflect-on-vague failure class (first dogfooding session, 2026-07-05: "create something + // interesting" → a brainstorm chat instead of acting or asking). PASS per ROADMAP §6's + // lands-or-clarifies is command ∈ {run_agent, clarify}; this pins the acceptance set so the class + // can't silently soften to accept deflection, and that alsoAccept stays well-formed everywhere. + for (const c of DECIDE_CASES) { + for (const a of c.alsoAccept ?? []) { + expect(COMMANDS, `${c.id} alsoAccept contains an unknown command`).toContain(a); + expect(a, `${c.id} alsoAccept must not repeat expect`).not.toBe(c.expect); + } + // Close the untag loophole: widening ANY case's acceptance set requires declaring a failure-class + // tag (whose exact set is then pinned, like vague-initiative below). An untagged alsoAccept would + // soften scoring with no test noticing — the precise quiet drift this pin exists to prevent. + if (c.alsoAccept !== undefined) { + expect(c.tag, `${c.id} has alsoAccept but no failure-class tag — tag it so its set gets pinned`).toBeDefined(); + } + } + const vague = DECIDE_CASES.filter((c) => c.tag === 'vague-initiative'); + expect(vague.length).toBeGreaterThanOrEqual(4); + for (const c of vague) { + const accepted = new Set([c.expect, ...(c.alsoAccept ?? [])]); + expect(accepted, `${c.id} must accept exactly {run_agent, clarify}`).toEqual(new Set(['run_agent', 'clarify'])); + // These must reach the MODEL — same discipline as the marker-less gate pin below: if the + // deterministic clarify gate ever grows a rule covering one, this fails loudly and the case + // must be re-labeled (it would then measure the gate), not silently shift the metric. + expect(clarifyForReferentlessRequest(c.input), `${c.id} is now caught by the clarify gate — re-label it`).toBeNull(); + } + }); + it('extract cases: unique ids, valid expectation, both fact and null exercised', () => { const ids = EXTRACT_CASES.map((c) => c.id); expect(new Set(ids).size).toBe(ids.length); diff --git a/src/core/brain/eval/fixtures.ts b/src/core/brain/eval/fixtures.ts index 055a693..e7c5e3f 100644 --- a/src/core/brain/eval/fixtures.ts +++ b/src/core/brain/eval/fixtures.ts @@ -1,6 +1,6 @@ -// src/brain/eval/fixtures.ts — the GOLDEN turn set the brain eval scores against. +// src/core/brain/eval/fixtures.ts — the GOLDEN turn set the brain eval scores against. // -// Hand-curated to be defensible against the SYSTEM_PROMPT contract (src/brain/index.ts): run_agent is the +// Hand-curated to be defensible against the SYSTEM_PROMPT contract (src/core/brain/index.ts): run_agent is the // default for any coding/file/test/build task; answer is talk-only; capture_screen is ONLY for things // visible on screen but not in the codebase; clarify is for genuinely ambiguous requests with no referent. // Keep expectations UNAMBIGUOUS — the eval measures the model, so a wrong golden answer poisons the metric. @@ -10,10 +10,18 @@ import type { Command, DecideInput } from '../../../shared/brain'; import type { FactExtractInput } from '../extractFact'; import type { ValueContract } from './score'; +/** DECIDE-side failure-class tag (the extract-side analogue is BehavioralTaxonomy). Only tagged failure + * classes carry one; the four plain command families stay untagged. */ +export type DecideTag = 'vague-initiative'; + export interface DecideCase { id: string; input: DecideInput; expect: Command; + /** Commands that ALSO pass, for cases where the golden outcome is a SET (ROADMAP §6's + * lands-or-clarifies: acting or asking a pertinent question both land). Absent → exact match. */ + alsoAccept?: Command[]; + tag?: DecideTag; note?: string; } @@ -57,6 +65,22 @@ export const DECIDE_CASES: DecideCase[] = [ { id: 'clar-thatthing', input: { transcript: 'do that thing we talked about' }, expect: 'clarify' }, { id: 'clar-updateit', input: { transcript: 'update it' }, expect: 'clarify' }, { id: 'clar-color', input: { transcript: 'change the color', }, expect: 'clarify', note: 'of what, to what — no referent' }, + + // vague-initiative — open-ended CREATIVE prompts (the deflect-on-vague class, first founder dogfooding + // session 2026-07-05: "create something interesting" got "Let's brainstorm a fun project idea together"). + // PASS per ROADMAP §6's lands-or-clarifies is command ∈ {run_agent, clarify} — take initiative OR ask one + // pertinent question; command=answer (a brainstorm/chat deflection) FAILS. None of these match the + // deterministic clarify-gate rules (clarifyGate.ts), so they measure the MODEL. MEASURED (2026-07-06 + // baseline refresh): the current 3B lands-or-clarifies on ALL of these — 5/5 in the baseline run and + // 30/30 in a repeat probe (3× each, with and without a memory section) — so the fixtures PIN the class + // against regression rather than reproduce the live failure; the 2026-07-05 deflection has not yet + // reproduced from a paraphrased transcript (the exact live turn context is the missing ingredient — + // promote a RORO_TRACE capture of a real deflecting turn into a fixture when one is next observed). + { id: 'vague-interesting', input: { transcript: 'create something interesting' }, expect: 'run_agent', alsoAccept: ['clarify'], tag: 'vague-initiative', note: 'the live dogfooding deflection' }, + { id: 'vague-cool', input: { transcript: 'make something cool in this repo' }, expect: 'run_agent', alsoAccept: ['clarify'], tag: 'vague-initiative' }, + { id: 'vague-fun', input: { transcript: 'do something fun' }, expect: 'run_agent', alsoAccept: ['clarify'], tag: 'vague-initiative' }, + { id: 'vague-buildme', input: { transcript: 'build me something' }, expect: 'run_agent', alsoAccept: ['clarify'], tag: 'vague-initiative' }, + { id: 'vague-impress', input: { transcript: 'whip up something impressive in this codebase' }, expect: 'run_agent', alsoAccept: ['clarify'], tag: 'vague-initiative' }, ]; export const EXTRACT_CASES: ExtractCase[] = [ diff --git a/src/core/brain/eval/paths.test.ts b/src/core/brain/eval/paths.test.ts new file mode 100644 index 0000000..778468c --- /dev/null +++ b/src/core/brain/eval/paths.test.ts @@ -0,0 +1,33 @@ +import { describe, it, expect } from 'vitest'; +import { existsSync, readFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { EVAL_DIR, evalArtifactPath } from './paths'; + +// The instrument-repair pin (deterministic, CI). After the W7 core-split (src/brain -> src/core/brain) +// both live eval runners kept writing latest/baseline artifacts to the stale 'src/brain/eval' path and +// crashed ENOENT at the end of a full run. These tests pin the path family to the directory the eval +// files ACTUALLY live in, so a future move breaks CI loudly instead of the runners silently. + +describe('eval artifact paths — the runners must aim at the real eval directory', () => { + it('EVAL_DIR resolves (from the repo root) to the directory this test file lives in', () => { + // vitest runs from the repo root — the same cwd the npm eval scripts give the runners. If the eval + // family moves again, __dirname moves with it while EVAL_DIR stays behind: this fails loudly. + expect(resolve(process.cwd(), EVAL_DIR)).toBe(__dirname); + }); + + it('the checked-in baseline.json exists at the resolved baseline path', () => { + expect(existsSync(evalArtifactPath('baseline.json'))).toBe(true); + }); + + it('the runners route ALL artifact paths through evalArtifactPath (no hand-spelled eval dirs)', () => { + // The location pin above protects the CONSTANT; this protects its USAGE — a future artifact write + // hand-spelling a path (the exact way the post-W7 breakage happened) fails here, not at the end of + // a 30-minute live run. Source-level pin, same spirit as the boundary test's import scan. + for (const runner of ['runEval.ts', 'runProposalEval.ts']) { + const src = readFileSync(join(__dirname, runner), 'utf8'); + expect(src, `${runner} must import evalArtifactPath`).toMatch(/evalArtifactPath/); + // Quoted literals only — the repo's header-comment convention legitimately names the file's own path. + expect(src, `${runner} hand-spells an eval dir in code — route it through paths.ts`).not.toMatch(/['"`]src\/(core\/)?brain\/eval/); + } + }); +}); diff --git a/src/core/brain/eval/paths.ts b/src/core/brain/eval/paths.ts new file mode 100644 index 0000000..455e042 --- /dev/null +++ b/src/core/brain/eval/paths.ts @@ -0,0 +1,17 @@ +// src/core/brain/eval/paths.ts — the ONE place the eval runners' artifact directory is spelled. +// +// Both live runners (runEval.ts, runProposalEval.ts) write their latest/baseline artifacts here. After the +// W7 core-split (#149) moved src/brain -> src/core/brain, both runners kept aiming at the OLD +// 'src/brain/eval' and crashed ENOENT at the end of a full (10–30 min) run — the instrument went stale +// silently. The path now lives here in ONE place, based on __dirname (this file lives IN the eval dir), so +// it is correct from ANY invocation cwd and travels with the directory on a future move; paths.test.ts pins +// the repo-relative constant AND that the runners actually route through this module. + +import { join } from 'node:path'; + +/** The eval directory, relative to the repo root (display + the paths.test.ts location pin). */ +export const EVAL_DIR = 'src/core/brain/eval'; + +/** Absolute path of an eval artifact (latest.json / baseline.json / proposalLatest.json / proposalBaseline.json). + * __dirname-based, NOT process.cwd() — a runner invoked from a non-root cwd must not recreate the ENOENT class. */ +export const evalArtifactPath = (filename: string): string => join(__dirname, filename); diff --git a/src/core/brain/eval/proposalFixtures.ts b/src/core/brain/eval/proposalFixtures.ts index 8f87dc3..c74fbf8 100644 --- a/src/core/brain/eval/proposalFixtures.ts +++ b/src/core/brain/eval/proposalFixtures.ts @@ -1,4 +1,4 @@ -// src/brain/eval/proposalFixtures.ts — the proposal eval's DETERMINISTIC tier: two real-shaped +// src/core/brain/eval/proposalFixtures.ts — the proposal eval's DETERMINISTIC tier: two real-shaped // RunDigests built by replaying the executor fixtures (the captured codex v0.139.0 JSONL + the // documented claude 2.1.x sample) through the SAME mappers and digest accumulator production uses. // Nothing here talks to a model: the digests are pure functions of checked-in fixture bytes, so the @@ -54,7 +54,7 @@ export function codexFixtureDigest(): RunDigest { } /** Replay the documented claude stream-json sample through the claude mappers (the same driving loop - * as src/executor/fixtures.test.ts) into a digest accumulator. */ + * as src/core/executor/fixtures.test.ts) into a digest accumulator. */ export function claudeFixtureDigest(): RunDigest { const acc = createDigestAccumulator(); const corr = newClaudeCorrelation(); diff --git a/src/core/brain/eval/runEval.ts b/src/core/brain/eval/runEval.ts index 510f38d..0fb6649 100644 --- a/src/core/brain/eval/runEval.ts +++ b/src/core/brain/eval/runEval.ts @@ -1,4 +1,4 @@ -// src/brain/eval/runEval.ts — the LIVE brain eval runner (opt-in; needs a running Ollama + the model). +// src/core/brain/eval/runEval.ts — the LIVE brain eval runner (opt-in; needs a running Ollama + the model). // // npm run eval:brain # score the local brain (qwen2.5:3b) // @@ -10,8 +10,8 @@ // scoring logic IS unit-tested (score.test.ts). The point: turn "is the 3B brain good enough?" into a number. import { writeFileSync } from 'node:fs'; -import { join } from 'node:path'; import { decide, extractFact, describeBrain } from '../index'; +import { evalArtifactPath } from './paths'; import { DECIDE_CASES, EXTRACT_CASES, BEHAVIORAL_EXTRACT_CASES } from './fixtures'; import { scoreDecision, scoreExtraction, scoreFactValue, summarize, type EvalSummary } from './score'; @@ -40,18 +40,20 @@ async function main(): Promise { console.log('DECIDE — does the brain pick the right command?'); const decideRows: Row[] = []; for (const c of DECIDE_CASES) { + // lands-or-clarifies cases (vague-initiative) accept a SET of commands; plain cases stay exact. + const want = c.alsoAccept ? [c.expect, ...c.alsoAccept].join('|') : c.expect; let mode: string; let got: string; try { const d = await decide(c.input); - mode = scoreDecision(c.expect, d.command); + mode = scoreDecision(c.expect, d.command, c.alsoAccept); got = d.command; } catch (e) { mode = 'bad_json'; // a decide() throw = unparseable/invalid decision (assumes Ollama is reachable) got = `THREW: ${(e as Error).message.slice(0, 70)}`; } - decideRows.push({ id: c.id, expect: c.expect, got, mode }); - console.log(` ${mode === 'ok' ? '✓' : '✗'} ${c.id.padEnd(16)} want=${c.expect.padEnd(14)} got=${got}`); + decideRows.push({ id: c.id, expect: want, got, mode }); + console.log(` ${mode === 'ok' ? '✓' : '✗'} ${c.id.padEnd(18)} want=${want.padEnd(16)} got=${got}`); } console.log('\nEXTRACT — durable fact when expected, null-discipline otherwise?'); @@ -150,11 +152,11 @@ async function main(): Promise { behavioralNull: behavioralNullSummary, behavioralByTaxonomy, }; - const latest = join(process.cwd(), 'src/brain/eval/latest.json'); + const latest = evalArtifactPath('latest.json'); writeFileSync(latest, JSON.stringify(results, null, 2) + '\n'); console.log(`\n[eval] wrote ${latest}`); if (process.argv.includes('--write-baseline')) { - const out = join(process.cwd(), 'src/brain/eval/baseline.json'); + const out = evalArtifactPath('baseline.json'); writeFileSync(out, JSON.stringify(results, null, 2) + '\n'); console.log(`[eval] BASELINE UPDATED: ${out} (explicit --write-baseline)`); } else { diff --git a/src/core/brain/eval/runProposalEval.ts b/src/core/brain/eval/runProposalEval.ts index e23a84e..9e299ca 100644 --- a/src/core/brain/eval/runProposalEval.ts +++ b/src/core/brain/eval/runProposalEval.ts @@ -1,4 +1,4 @@ -// src/brain/eval/runProposalEval.ts — the LIVE executor-proposal eval (docs/plans/executor-facts-pilot.md §7). +// src/core/brain/eval/runProposalEval.ts — the LIVE executor-proposal eval (docs/plans/executor-facts-pilot.md §7). // // npm run eval:proposals # ask the codex CLI (default) // npm run eval:proposals -- --agent claude # ask the claude CLI @@ -26,8 +26,8 @@ // the same lucky-run discipline as runEval.ts. import { writeFileSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; import { BEHAVIORAL_EXTRACT_CASES, type BehavioralCase } from './fixtures'; +import { evalArtifactPath } from './paths'; import { scoreExtraction, scoreFactValue, summarize, type EvalSummary } from './score'; import { codexFixtureDigest, claudeFixtureDigest } from './proposalFixtures'; import { executorProposalSource } from '../../orchestrator/factProposals/proposer'; @@ -166,7 +166,7 @@ async function main(): Promise { console.log(`\n[eval:proposals] VALUE-QUALITY ${pct(valueQuality.accuracy)} (${valueQuality.ok}/${valueQuality.total}) ${JSON.stringify(valueQuality.byMode)}`); try { - const baseline = JSON.parse(readFileSync(join(process.cwd(), 'src/brain/eval/baseline.json'), 'utf8')) as { behavioral?: EvalSummary }; + const baseline = JSON.parse(readFileSync(evalArtifactPath('baseline.json'), 'utf8')) as { behavioral?: EvalSummary }; if (baseline.behavioral) { console.log(`[eval:proposals] vs 3B baseline behavioral value-quality: ${pct(baseline.behavioral.accuracy)} (${baseline.behavioral.ok}/${baseline.behavioral.total})`); } @@ -178,11 +178,11 @@ async function main(): Promise { console.log(`[eval:proposals] axes: ${JSON.stringify(axes)}`); const results = { channel: `${agent} executor CLI`, valueQuality, nullDiscipline, digestTier, byTaxonomy, axes }; - const latest = join(process.cwd(), 'src/brain/eval/proposalLatest.json'); + const latest = evalArtifactPath('proposalLatest.json'); writeFileSync(latest, JSON.stringify(results, null, 2) + '\n'); console.log(`\n[eval:proposals] wrote ${latest}`); if (process.argv.includes('--write-baseline')) { - const out = join(process.cwd(), 'src/brain/eval/proposalBaseline.json'); + const out = evalArtifactPath('proposalBaseline.json'); writeFileSync(out, JSON.stringify(results, null, 2) + '\n'); console.log(`[eval:proposals] BASELINE UPDATED: ${out} (explicit --write-baseline)`); } else { diff --git a/src/core/brain/eval/score.test.ts b/src/core/brain/eval/score.test.ts index 35ff4e0..1ecaa5e 100644 --- a/src/core/brain/eval/score.test.ts +++ b/src/core/brain/eval/score.test.ts @@ -12,6 +12,14 @@ describe('scoreDecision — did the brain pick the right command?', () => { expect(scoreDecision('run_agent', 'answer')).toBe('wrong_command'); expect(scoreDecision('capture_screen', 'run_agent')).toBe('wrong_command'); }); + it("alsoAccept (lands-or-clarifies): any accepted command scores ok; a deflection still fails", () => { + // The vague-initiative class (deflect-on-vague, first dogfooding session 2026-07-05): per ROADMAP §6 + // "lands or clarifies", acting OR asking a pertinent question passes — a chat deflection never does. + expect(scoreDecision('run_agent', 'run_agent', ['clarify'])).toBe('ok'); + expect(scoreDecision('run_agent', 'clarify', ['clarify'])).toBe('ok'); + expect(scoreDecision('run_agent', 'answer', ['clarify'])).toBe('wrong_command'); + expect(scoreDecision('run_agent', 'capture_screen', ['clarify'])).toBe('wrong_command'); + }); }); describe('scoreExtraction — did the extractor honor the durable-fact / null-discipline contract?', () => { diff --git a/src/core/brain/eval/score.ts b/src/core/brain/eval/score.ts index 4cfff81..39990f5 100644 --- a/src/core/brain/eval/score.ts +++ b/src/core/brain/eval/score.ts @@ -1,4 +1,4 @@ -// src/brain/eval/score.ts — the PURE scoring core of the brain eval (no model, CI-runnable). +// src/core/brain/eval/score.ts — the PURE scoring core of the brain eval (no model, CI-runnable). // // The whole local-first thesis rests on qwen2.5:3b reliably (a) picking the right DECIDE command and // (b) honoring extractFact's null-when-unsure discipline. This scores both halves so the live runner @@ -8,9 +8,11 @@ import type { Command } from '../../../shared/brain'; import type { FactCandidate } from '../extractFact'; /** Did the brain pick the right command? (A decide() THROW — unparseable/invalid JSON — is classified - * 'bad_json' by the runner, not here; this scores a successfully-parsed decision.) */ -export function scoreDecision(expected: Command, got: Command): 'ok' | 'wrong_command' { - return expected === got ? 'ok' : 'wrong_command'; + * 'bad_json' by the runner, not here; this scores a successfully-parsed decision.) `alsoAccept` widens + * the golden outcome to a SET for lands-or-clarifies cases (vague-initiative: run_agent OR clarify both + * land; a deflection to answer stays wrong_command). Absent → exact match, as before. */ +export function scoreDecision(expected: Command, got: Command, alsoAccept?: Command[]): 'ok' | 'wrong_command' { + return expected === got || (alsoAccept?.includes(got) ?? false) ? 'ok' : 'wrong_command'; } /** Did the extractor honor the contract? A durable fact must be produced when expected, and the model