From 41077244d523a05e45885a8666d928b359744728 Mon Sep 17 00:00:00 2001 From: Jin Choi Date: Sat, 4 Jul 2026 22:59:50 -0700 Subject: [PATCH 1/9] fix(executor): stitch codex run.completed.finalText from the last agent_message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codex has no single authoritative final-result string (claude uses the SDK m.result); its LAST assistant agent_message IS the answer. runCodex now stitches that onto run.completed as finalText (via attachCodexFinalText), so the truthful memory receipt (speechBubble), the answer caption, and the 'job done' notification light on the DEFAULT codex path — previously structurally dark (finalText only ever set on the claude path). Within-contract: finalText is already an optional field on the run.completed ActionEvent (shared/events.ts) — no union kind added, mapCodexThreadEvent stays PURE (the stitch is stateful-loop-only), no core/electron boundary crossing. LOCKSTEP (the guardrail): the deterministic proposal-eval fixture (proposalFixtures.ts) replays the PURE mapper, so it would NOT auto-reflect the loop stitch — it now replays the shared stitchCodexFinalText so the CI eval stays representative of production. Its test + the stale 'codex emits none' comment are reconciled in the same commit. Reproduce-first: stitchFinalText.test.ts pins last-message-wins, empty→suppressed, never-clobber-existing, pass-through — RED before the helpers existed. tsc clean; stitch + fixture + pure-mapper tests green (25). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Vsq543S6voPTUsGsEqZvnD --- src/core/brain/eval/proposalFixtures.test.ts | 5 +- src/core/brain/eval/proposalFixtures.ts | 17 ++++--- src/core/executor/codex.ts | 27 ++++++++++- src/core/executor/index.ts | 2 +- src/core/executor/stitchFinalText.test.ts | 49 ++++++++++++++++++++ 5 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 src/core/executor/stitchFinalText.test.ts diff --git a/src/core/brain/eval/proposalFixtures.test.ts b/src/core/brain/eval/proposalFixtures.test.ts index d3778c5..21ceae2 100644 --- a/src/core/brain/eval/proposalFixtures.test.ts +++ b/src/core/brain/eval/proposalFixtures.test.ts @@ -11,7 +11,7 @@ import { buildProposalPrompt } from '../../orchestrator/factProposals/prompt'; // canned model replies (grounded admitted; ungrounded/boolean/garbage dropped). describe('proposal eval fixture digests — real-shaped, straight from the production mappers', () => { - it('codex: the captured v0.139.0 run digests to 2 commands, 1 added file, 4 messages, no finalText', () => { + it('codex: the captured v0.139.0 run digests to 2 commands, 1 added file, 4 messages, and the stitched finalText', () => { const d = codexFixtureDigest(); expect(d.agent).toBe('codex'); expect(d.outcome).toBe('completed'); @@ -20,7 +20,8 @@ describe('proposal eval fixture digests — real-shaped, straight from the produ expect(d.commands[1]).toContain('python3 hello.py'); expect(d.files).toEqual([{ path: '/tmp/companion_scratch/hello.py', op: 'add' }]); expect(d.messages).toHaveLength(4); - expect(d.finalText).toBeUndefined(); // codex run.completed carries no finalText — same as production + // finalText is now the LAST agent_message, stitched in the runCodex loop (mirrored here) — same as production. + expect(d.finalText).toBe('Created [hello.py](/tmp/companion_scratch/hello.py:1).\n\nVerified with `python3 hello.py`; it prints:\n\n```text\nhi\n```'); }); it('claude: the documented 2.1.x sample digests to 1 command, 1 message, and a finalText', () => { diff --git a/src/core/brain/eval/proposalFixtures.ts b/src/core/brain/eval/proposalFixtures.ts index 0b83ca1..8f87dc3 100644 --- a/src/core/brain/eval/proposalFixtures.ts +++ b/src/core/brain/eval/proposalFixtures.ts @@ -9,6 +9,7 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { mapCodexThreadEvent, + stitchCodexFinalText, mapClaudeMessage, mapClaudeMessageBlocks, mapClaudeStreamEvent, @@ -29,14 +30,18 @@ const CLAUDE_FIXTURE_TASK = 'make a hello.py script and check it works'; export function codexFixtureDigest(): RunDigest { const lines = readFileSync(join(__dirname, '../../executor/__fixtures__/codex_hello.jsonl'), 'utf8').split('\n'); const acc = createDigestAccumulator(); + // Mirror production exactly: the pure mapper emits run.completed with NO finalText; runCodex's stream loop + // then stitches the LAST agent_message onto it. Replay the same stitch here (stitchCodexFinalText) so the + // deterministic eval stays representative — codex's finalText is now its final assistant message, like claude's. + const events = lines + .map((line) => line.trim()) + .filter((s) => s && s[0] === '{') + .map((s) => mapCodexThreadEvent(JSON.parse(s), 'run_proposal_fixture_codex')) + .filter((ev): ev is NonNullable => ev !== null); let finalText: string | undefined; - for (const line of lines) { - const s = line.trim(); - if (!s || s[0] !== '{') continue; - const ev = mapCodexThreadEvent(JSON.parse(s), 'run_proposal_fixture_codex'); - if (!ev) continue; + for (const ev of stitchCodexFinalText(events)) { acc.see(ev); - if (ev.kind === 'run.completed') finalText = ev.finalText; // codex emits none — stays undefined, like production + if (ev.kind === 'run.completed') finalText = ev.finalText; } return acc.finish({ runId: 'run_proposal_fixture_codex', diff --git a/src/core/executor/codex.ts b/src/core/executor/codex.ts index a4f6613..7192950 100644 --- a/src/core/executor/codex.ts +++ b/src/core/executor/codex.ts @@ -231,6 +231,28 @@ export function codexExecArgs(opts: Pick "" { + if (ev.kind === 'message' && ev.text) last = ev.text; + return attachCodexFinalText(ev, last); + }); +} + export async function* runCodex( opts: ExecutorRunOptions, ): AsyncIterable { @@ -268,6 +290,7 @@ export async function* runCodex( child.on('close', (code, signal) => resolve({ code, signal })); }); let emittedTerminal = false; + let lastMessageText: string | undefined; // the last agent_message → stitched onto run.completed as finalText const rl = createInterface({ input: child.stdout!, crlfDelay: Infinity }); try { @@ -283,8 +306,10 @@ export async function* runCodex( } const mapped = mapCodexThreadEvent(obj, runId); if (mapped) { + if (mapped.kind === 'message' && mapped.text) lastMessageText = mapped.text; if (mapped.kind === 'run.completed' || mapped.kind === 'run.failed') emittedTerminal = true; - yield mapped; + // Streaming mirror of stitchCodexFinalText: attach the last agent_message as run.completed.finalText. + yield attachCodexFinalText(mapped, lastMessageText); } } } catch (e) { diff --git a/src/core/executor/index.ts b/src/core/executor/index.ts index 4714f51..7c8934c 100644 --- a/src/core/executor/index.ts +++ b/src/core/executor/index.ts @@ -9,7 +9,7 @@ import { CodexExecutor } from './codex'; import { ClaudeExecutor } from './claude'; import { ClaudeSdkExecutor } from './claudeSdk'; -export { CodexExecutor, runCodex, mapCodexThreadEvent } from './codex'; +export { CodexExecutor, runCodex, mapCodexThreadEvent, stitchCodexFinalText, attachCodexFinalText } from './codex'; export { ClaudeExecutor, runClaude, diff --git a/src/core/executor/stitchFinalText.test.ts b/src/core/executor/stitchFinalText.test.ts new file mode 100644 index 0000000..51377fe --- /dev/null +++ b/src/core/executor/stitchFinalText.test.ts @@ -0,0 +1,49 @@ +// Reproduce-first (codex finalText): codex has no single authoritative final-result string like claude's +// SDK m.result — its LAST assistant agent_message IS the answer. attachCodexFinalText / stitchCodexFinalText +// stitch that onto run.completed.finalText so the truthful receipt + captions + job notification have it. +import { describe, it, expect } from 'vitest'; +import { attachCodexFinalText, stitchCodexFinalText } from './codex'; +import type { ActionEvent } from '../../shared/events'; + +const msg = (text: string): ActionEvent => ({ kind: 'message', runId: 'r', text, ts: 0 }); +const completed = (finalText?: string): ActionEvent => ({ kind: 'run.completed', runId: 'r', ok: true, ts: 0, ...(finalText ? { finalText } : {}) }); + +describe('attachCodexFinalText', () => { + it('attaches the last agent_message text as finalText on run.completed', () => { + const out = attachCodexFinalText(completed(), 'the answer'); + expect(out.kind).toBe('run.completed'); + expect((out as { finalText?: string }).finalText).toBe('the answer'); + }); + + it('leaves finalText undefined when there was no message (empty answer → receipt stays suppressed)', () => { + expect((attachCodexFinalText(completed(), undefined) as { finalText?: string }).finalText).toBeUndefined(); + expect((attachCodexFinalText(completed(), '') as { finalText?: string }).finalText).toBeUndefined(); + }); + + it('never clobbers an already-set finalText (idempotent — claude/SDK path already carries it)', () => { + expect((attachCodexFinalText(completed('already'), 'other') as { finalText?: string }).finalText).toBe('already'); + }); + + it('passes non-run.completed events through unchanged', () => { + const m = msg('hi'); + expect(attachCodexFinalText(m, 'x')).toBe(m); + }); +}); + +describe('stitchCodexFinalText (batch — the proposal-fixture replay path)', () => { + it('stitches the LAST message text onto the terminal run.completed', () => { + const out = stitchCodexFinalText([msg('first'), msg('the real answer'), completed()]); + const done = out.find((e) => e.kind === 'run.completed') as { finalText?: string }; + expect(done.finalText).toBe('the real answer'); + }); + + it('ignores empty message text (a blank agent_message never becomes the answer)', () => { + const out = stitchCodexFinalText([msg(''), msg('real'), completed()]); + expect((out.find((e) => e.kind === 'run.completed') as { finalText?: string }).finalText).toBe('real'); + }); + + it('a run with no message yields run.completed with no finalText', () => { + const out = stitchCodexFinalText([completed()]); + expect((out[0] as { finalText?: string }).finalText).toBeUndefined(); + }); +}); From 50be67b2dd3101eb50662ebb971ecc7fc78e2c31 Mon Sep 17 00:00:00 2001 From: Jin Choi Date: Sat, 4 Jul 2026 23:02:22 -0700 Subject: [PATCH 2/9] fix(floating): soften the memory receipt to recall-truthful (claims 'checked', not 'used') MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once codex sets finalText (prior commit), the receipt fires on the DEFAULT codex path. But the gate proves only that memory was RECALLED this run (a same-run recall status + run.completed.finalText) — NOT that it shaped the answer. A true application gate is not observable from the ActionEvent stream (would need provenance threaded through decide→executor→event, an invasive frozen-union change). So the honest, renderer-local fix is the COPY: 'used what I remember about your setup' → 'checked what I remember about your setup'. Real steering is proven separately (verify:memory-steered-real / Phase 0: 5/5). Adds a test for the REAL codex event order (message → run.completed{finalText:same}): the bare message earns no receipt; the settled run.completed carries the answer + the receipt. Existing receipt cases stay green (they assert against the exported MEMORY_RECEIPT_TEXT constant, not a literal). FOUNDER-TUNABLE: the exact copy is a taste call — flagged for review. 33 speechBubble tests green; tsc clean. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Vsq543S6voPTUsGsEqZvnD --- src/renderer/reactions/speechBubble.test.ts | 13 +++++++++++++ src/renderer/reactions/speechBubble.ts | 12 ++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/renderer/reactions/speechBubble.test.ts b/src/renderer/reactions/speechBubble.test.ts index e16fd7e..0bb7204 100644 --- a/src/renderer/reactions/speechBubble.test.ts +++ b/src/renderer/reactions/speechBubble.test.ts @@ -207,6 +207,19 @@ describe('SpeechBubble — render + streaming + memory receipt', () => { expect(bubble.debugState().receipt).toBe(MEMORY_RECEIPT_TEXT); }); + it('the REAL codex order (message then run.completed{finalText}) shows the answer once WITH the receipt', () => { + // codex now emits the answer TWICE: as kind:'message' (agent_message), then run.completed carrying the + // SAME text as the stitched finalText. The bubble must end on the answer + the receipt (not the bare + // message which earns no receipt), and the intervening message must not break the recalling-run gate. + applyCaptionEvent(bubble, statusRecalled()); + applyCaptionEvent(bubble, runStarted()); // executor run.started, same runId 'r' + applyCaptionEvent(bubble, { kind: 'message', runId: 'r', text: 'Created hello.py', ts: 0 }); // agent_message (no receipt yet) + expect(bubble.debugState().receipt).toBe(''); // the bare message alone must NOT earn the receipt + applyCaptionEvent(bubble, { kind: 'run.completed', runId: 'r', ok: true, finalText: 'Created hello.py', ts: 0 }); + expect(bubble.debugState().text).toBe('Created hello.py'); + expect(bubble.debugState().receipt).toBe(MEMORY_RECEIPT_TEXT); + }); + it('a bare `message` (planning beat / answer-turn narration) NEVER carries the receipt', () => { applyCaptionEvent(bubble, statusRecalled()); // The orchestrator emits the planning beat as kind:'message' — it must NOT earn the receipt. diff --git a/src/renderer/reactions/speechBubble.ts b/src/renderer/reactions/speechBubble.ts index bb249f9..3addc0b 100644 --- a/src/renderer/reactions/speechBubble.ts +++ b/src/renderer/reactions/speechBubble.ts @@ -1,10 +1,13 @@ // src/renderer/reactions/speechBubble.ts — the cat's SPEECH BUBBLE (S5) + the truthful memory receipt. // // The cat voicing its answer/narration as a tethered #roro-stage island beside the cat, plus founder -// decision 5's strategic payload: a subtle, TRUTHFUL "used what I remember about your setup" line that +// decision 5's strategic payload: a subtle, TRUTHFUL "checked what I remember about your setup" line that // appears ONLY when memory was actually recalled that turn (never on every turn — that would be a // lie that erodes trust; a bare no-payload success shows nothing, matching the locked no-Done-banner -// contract). It is a CaptionSink (the answer text arrives via update('assistant', …), the same seam that +// contract). The copy claims RECALL, not application: the gate can prove memory was consulted this run +// (a same-run recall status), but NOT — from the ActionEvent stream alone — that it shaped the answer, so +// the wording deliberately stops at "checked" to never over-claim (real steering is proven separately by +// verify:memory-steered-real). It is a CaptionSink (the answer text arrives via update('assistant', …), the same seam that // feeds the dev CaptionPanel) AND turn-aware (noteEvent observes the raw stream for the memory-recall beat, // run.started reset, and live message.delta tokens). It is fanned in via a COMPOSITE sink so BOTH the dev // CaptionPanel (unchanged, hidden in floating) AND this bubble receive updates; the bubble mounts in @@ -19,8 +22,9 @@ import { mountIsland, place as placeIsland } from './stage'; // ---- pure bits (unit-tested) ---- -/** The exact, truthful receipt copy (founder-tunable). Rendered as a distinct subtle row under the answer. */ -export const MEMORY_RECEIPT_TEXT = 'used what I remember about your setup'; +/** The exact, truthful receipt copy (founder-tunable). Claims RECALL ("checked"), never application ("used") — + * the gate proves memory was consulted this run, not that it shaped the answer. Rendered as a subtle row. */ +export const MEMORY_RECEIPT_TEXT = 'checked what I remember about your setup'; export function clamp(v: number, lo: number, hi: number): number { return Math.min(Math.max(v, lo), hi); From 1a3294ccbe589d7fa1a897cac7d0a1ff480b686f Mon Sep 17 00:00:00 2001 From: Jin Choi Date: Sat, 4 Jul 2026 23:06:58 -0700 Subject: [PATCH 3/9] test(memory): formalize the real memory-steering proof as an opt-in guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turns the validated in-process experiment into a permanent guard: real qwen2.5:3b decide + real codex executor + a fixed-cipher disk store, measuring recall→args.task→actual-diff, treatment vs control. Renamed to src/core/memory2/memorySteered.live.test.ts, gated on RORO_STEERING_LIVE=1 (collect-but-skip in CI, like crosslaunch.live), run via 'npm run verify:memory-steered-real'. THE TEETH (the guardrail): it now ASSERTS the ROADMAP Arc A go/no-go — treatment steers >=60% on BOTH args.task and the real file diff, control 0 (no false positives) — instead of only console.log. First measured run: 5/5 · 5/5 · 0/5 · 0/5. Without the expect()s an opt-in test is vacuous green, worse than the deterministic fake-smoke it complements. KEEP the faked scripts/smoke-memory-steered.mjs (re-scoped its header): it uniquely covers packaged-app IPC/preload/orchestrator-over-CDP plumbing + the trace-capture path. The two are complementary — plumbing-determinism (fake models, CI-safe) vs real-model semantics (opt-in). Closes the ROADMAP 'real executor honoring the preference in a diff is a SEPARATE unproven check' gap; notes it there. tsc clean · 1096 passed / 12 skipped · eslint 0 errors. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Vsq543S6voPTUsGsEqZvnD --- docs/ROADMAP.md | 8 +- package.json | 1 + scripts/smoke-memory-steered.mjs | 8 +- src/core/memory2/memorySteered.live.test.ts | 151 ++++++++++++++++++++ 4 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 src/core/memory2/memorySteered.live.test.ts diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 8f9581e..f146d11 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -237,7 +237,13 @@ first-turn/memory validation** — otherwise it is deferred (§7 rule 4); C is s Even then, prompt-capture is **necessary, not sufficient** — it proves memory *shaped the task*, not that the executor *honored* it; so keep the **diff-reflects-preference output check** as the real-honoring proof (judged across runs, given - stochasticity), and keep **real Codex/Claude as a separate auth/readiness check**. **The manual check + stochasticity), and keep **real Codex/Claude as a separate auth/readiness check**. + **UPDATE (2026-07-04): the real-honoring check is now AUTOMATED** — `npm run verify:memory-steered-real` + (`src/core/memory2/memorySteered.live.test.ts`, opt-in/non-CI) runs real qwen2.5:3b + real codex over + N treatment/control runs and asserts the recalled marker reaches BOTH `args.task` AND the actual diff + (≥60% treatment, 0 control). First measured run: **5/5 args.task · 5/5 diff · 0/5 control** — the moat + mechanism holds for a well-formed, relevant preference. (The *extraction* hop — conversation→stored fact, + the ~40% behavioral-eval weak link — remains separate and is mitigated by the executor-facts confirm gate.) **The manual check is auditable — it produces an evidence packet:** {build id; both memory-root paths + the empty-control proof; the exact prompt; the event stream; the generated `args.task`; the resulting diff; observer + date}. **Packet format (privacy-safe):** the **raw** packet (full DECIDE prompt — recalled diff --git a/package.json b/package.json index a47e2f2..018924a 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "verify:packaged-sdk-executor": "node scripts/smoke-packaged-sdk-executor.mjs", "verify:packaged-model-setup": "node scripts/smoke-packaged-model-setup.mjs", "verify:memory-steered": "node scripts/smoke-memory-steered.mjs", + "verify:memory-steered-real": "RORO_STEERING_LIVE=1 OLLAMA_AVAILABLE=1 vitest run src/core/memory2/memorySteered.live.test.ts", "verify:release-artifact": "node scripts/verify-release-artifact.mjs --mode default", "verify:release-artifact:dmg": "node scripts/verify-release-artifact.mjs --mode default --require-dmg", "verify:release-artifact:signed": "node scripts/verify-release-artifact.mjs --mode signed", diff --git a/scripts/smoke-memory-steered.mjs b/scripts/smoke-memory-steered.mjs index eee5ff6..fa7516a 100644 --- a/scripts/smoke-memory-steered.mjs +++ b/scripts/smoke-memory-steered.mjs @@ -1,4 +1,10 @@ -// scripts/smoke-memory-steered.mjs — the automated "memory steered the coding work" proof. +// scripts/smoke-memory-steered.mjs — the DETERMINISTIC plumbing + trace-capture proof of memory steering. +// +// This is one of TWO complementary steering proofs. THIS one runs the REAL packaged app end-to-end (preload +// bridge → IPC → orchestrator → RORO_TRACE_DECIDE capture over CDP) with a FAKE brain + FAKE codex, so it is +// deterministic and CI-safe and pins the packaged-app WIRING. Its sibling `npm run verify:memory-steered-real` +// (src/core/memory2/memorySteered.live.test.ts) proves the SEMANTIC half — that a REAL local model + REAL +// codex actually carry a recalled preference into a real diff (opt-in, non-CI). Keep BOTH: plumbing vs models. // // Run AFTER `npm run package`: `npm run verify:memory-steered` (or RORO_PACKAGED_APP=/abs/Roro.app). // diff --git a/src/core/memory2/memorySteered.live.test.ts b/src/core/memory2/memorySteered.live.test.ts new file mode 100644 index 0000000..c0f1621 --- /dev/null +++ b/src/core/memory2/memorySteered.live.test.ts @@ -0,0 +1,151 @@ +// The REAL memory-steering guard: does recalled memory ACTUALLY reach a real code diff, with REAL models? +// This is the ROADMAP Arc A tracked gap ("a real executor honoring the preference in a diff is a SEPARATE +// unproven check") — the counterpart to scripts/smoke-memory-steered.mjs, which proves the same chain +// DETERMINISTICALLY with a fake brain + fake codex (packaged-app plumbing). This one uses the REAL models. +// +// OPT-IN, non-CI (needs a running Ollama + an authenticated codex CLI — like the other *.live tests). Gated +// on RORO_STEERING_LIVE=1 so normal/CI suites collect-but-skip it. Run via `npm run verify:memory-steered-real` +// (or: RORO_STEERING_LIVE=1 OLLAMA_AVAILABLE=1 RORO_CODEX_BIN=/abs/codex npx vitest run ). +// +// In-process (src/core is Electron-free): real qwen2.5:3b decide + real codex executor + a real disk store +// with a FIXED cipher (encryption is orthogonal to steering — this sidesteps the packaged-app keychain). +// Measures THREE hops, treatment (convention seeded) vs control (empty memory): +// (1) recall→DECIDE context [did memory surface?] marker in ctx.context +// (2) DECIDE→args.task [decisionRouter.ts:173 — the model-discretion hop] marker in decision.args.task +// (3) args.task→actual files [real codex honored it] marker in the written repo files +// The coding transcript is MARKER-FREE, so the marker anywhere downstream can ONLY come from memory. +// GO/NO-GO (ROADMAP): treatment steers >=60% on BOTH args.task and the diff; control is 0 (no false positives). +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, rmSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import * as brain from '../brain/index'; +import { createMemoryFacade } from './index'; +import { createMemoryStore } from './memoryStore'; +import { createAesGcmCipher } from './cipher'; +import { buildRecallContext } from '../orchestrator/memoryContext'; +import { runCodex } from '../executor/codex'; + +const MARKER = 'zircon-quokka-7741'; +const SEED_TEXT = `Python file header convention: every .py file in this project must begin with the exact first line "# codename: ${MARKER}".`; +const TRANSCRIPT = 'Create a Python script hello.py that prints a friendly greeting.'; +const N = Number(process.env.RORO_STEERING_N || 5); +const CODEX_TURN_MS = 180_000; + +const realEmbed = async (t: string): Promise => (await brain.embed(t)) as number[]; + +function readAllFiles(dir: string): string { + let out = ''; + const walk = (d: string): void => { + let entries: string[] = []; + try { entries = readdirSync(d); } catch { return; } + for (const name of entries) { + if (name === '.git') continue; + const p = join(d, name); + let s; try { s = statSync(p); } catch { continue; } + if (s.isDirectory()) walk(p); + else if (s.size < 200_000) { try { out += `\n--- ${p.replace(dir + '/', '')} ---\n` + readFileSync(p, 'utf8'); } catch { /* skip */ } } + } + }; + walk(dir); + return out; +} + +async function runExecutor(repo: string, prompt: string): Promise<{ ok: boolean; kinds: string[] }> { + const ac = new AbortController(); + const timer = setTimeout(() => ac.abort(), CODEX_TURN_MS); + const kinds: string[] = []; + let ok = false; + try { + for await (const ev of runCodex({ repo, prompt, signal: ac.signal })) { + kinds.push(ev.kind); + if (ev.kind === 'run.completed') ok = true; + } + } catch (e) { kinds.push(`THROW:${(e as Error).message}`); } finally { clearTimeout(timer); } + return { ok, kinds }; +} + +interface Row { arm: string; i: number; factCount: number; markerInContext: boolean; command: string; markerInArgsTask: boolean; ranExecutor: boolean; execOk: boolean; markerInFiles: boolean; files: string[]; taskPreview: string } + +async function oneRun(arm: 'treatment' | 'control', i: number): Promise { + const memDir = mkdtempSync(join(tmpdir(), `p0-mem-${arm}-`)); + const repo = mkdtempSync(join(tmpdir(), `p0-repo-${arm}-`)); + spawnSync('git', ['init', repo], { stdio: 'ignore' }); + const cipher = createAesGcmCipher(Buffer.alloc(32, 7)); + const dim = (await brain.embed('dimension probe')).length; + const row: Row = { arm, i, factCount: 0, markerInContext: false, command: '-', markerInArgsTask: false, ranExecutor: false, execOk: false, markerInFiles: false, files: [], taskPreview: '' }; + try { + // Launch A: seed the convention (treatment only), then close (persist to disk). + const a = createMemoryFacade(await createMemoryStore({ dir: memDir, embed: realEmbed, dim, cipher })); + if (arm === 'treatment') { + await a.replaceFact({ ownerId: 'me', sessionId: 'A', factKey: 'project.py-header-convention', text: SEED_TEXT, payload: { key: 'project.py-header-convention', value: SEED_TEXT } }); + } + await a.close(); + // Launch B: reopen from disk (real cross-quit), recall against the marker-free coding request. + const b = createMemoryFacade(await createMemoryStore({ dir: memDir, embed: realEmbed, dim, cipher })); + let ctx; + try { + ctx = await buildRecallContext(b, { ownerId: 'me', sessionId: 'B', query: TRANSCRIPT, minSimilarity: 0 }); + } finally { await b.close(); } + const context = ctx.context ?? ''; + row.factCount = ctx.factCount; + row.markerInContext = context.includes(MARKER); + // Hop 2: the REAL brain decides, handed the recalled memory. Does it carry the marker into args.task? + const decision = await brain.decide({ transcript: TRANSCRIPT, memory: context }); + row.command = decision.command; + const task = (decision as { args?: { task?: string } }).args?.task ?? ''; + row.markerInArgsTask = task.includes(MARKER); + row.taskPreview = task.slice(0, 160); + // Hop 3: the REAL executor runs the model's task. Did the marker land in an actual file? + if (decision.command === 'run_agent' && task) { + row.ranExecutor = true; + const exec = await runExecutor(repo, task); + row.execOk = exec.ok; + const filesText = readAllFiles(repo); + row.markerInFiles = filesText.includes(MARKER); + row.files = (filesText.match(/--- (.+?) ---/g) || []).map((m) => m.replace(/--- (.+?) ---/, '$1')); + } + } finally { + rmSync(memDir, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true }); + } + return row; +} + +describe.runIf(process.env.RORO_STEERING_LIVE === '1')('memory-steering (real qwen2.5:3b + real codex)', () => { + it(`recalled memory reaches args.task AND the real diff across ${N} treatment + ${N} control`, async () => { + const rows: Row[] = []; + for (const arm of ['treatment', 'control'] as const) { + for (let i = 0; i < N; i++) { + const r = await oneRun(arm, i); + rows.push(r); + console.log(`[${arm}#${i + 1}] factCount=${r.factCount} ctx=${r.markerInContext} cmd=${r.command} argsTask=${r.markerInArgsTask} ranExec=${r.ranExecutor} execOk=${r.execOk} files=${r.markerInFiles} wrote=[${r.files.join(', ')}]`); + console.log(` task="${r.taskPreview}"`); + } + } + const t = rows.filter((r) => r.arm === 'treatment'); + const c = rows.filter((r) => r.arm === 'control'); + const hit = (rs: Row[], k: keyof Row): string => `${rs.filter((r) => r[k]).length}/${rs.length}`; + console.log('\n===== PHASE 0 SUMMARY ====='); + console.log('TREATMENT (memory seeded):'); + console.log(` (1) recall→context: ${hit(t, 'markerInContext')}`); + console.log(` (2) →args.task: ${hit(t, 'markerInArgsTask')} [decisionRouter.ts:173 — the fragile heart]`); + console.log(` (3) →actual files: ${hit(t, 'markerInFiles')} [real codex honored it]`); + console.log(` run_agent chosen: ${hit(t, 'ranExecutor')}`); + console.log('CONTROL (no memory) — expect ~0:'); + console.log(` (2) →args.task: ${hit(c, 'markerInArgsTask')}`); + console.log(` (3) →actual files: ${hit(c, 'markerInFiles')}`); + console.log('===========================\n'); + + // THE GUARD (ROADMAP Arc A go/no-go): treatment steers >=60% on BOTH args.task and the actual diff; + // control is 0 on both (no false positives). Without these expect()s this opt-in test would be vacuous + // green — worse than the deterministic fake-smoke it complements. First measured run: 5/5 · 5/5 · 0/5 · 0/5. + const count = (rs: Row[], k: keyof Row): number => rs.filter((r) => r[k]).length; + const threshold = Math.ceil(0.6 * N); + expect(count(t, 'markerInArgsTask'), 'treatment: recalled memory reached args.task').toBeGreaterThanOrEqual(threshold); + expect(count(t, 'markerInFiles'), 'treatment: recalled memory reached the REAL diff').toBeGreaterThanOrEqual(threshold); + expect(count(c, 'markerInArgsTask'), 'control (no memory): NO marker in args.task').toBe(0); + expect(count(c, 'markerInFiles'), 'control (no memory): NO marker in the diff').toBe(0); + }, N * 4 * CODEX_TURN_MS + 120_000); +}); From 5dc9595a9aa5e289e00b54a2f8221a96ff5279a0 Mon Sep 17 00:00:00 2001 From: Jin Choi Date: Sat, 4 Jul 2026 23:34:00 -0700 Subject: [PATCH 4/9] =?UTF-8?q?fix(review):=20address=20the=20multi-hat=20?= =?UTF-8?q?panel=20=E2=80=94=20coverage,=20honesty=20consistency,=20doc=20?= =?UTF-8?q?drift?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-hat review (correctness · contract-integrity · truthfulness · test-rigor · ripple): no blockers; both re-run hats (contract, ripple) PASS — finalText stays within the frozen optional field, mapper stays pure, boundary holds, all 7 finalText consumers correct (the codex double-emit is exact parity with claude). Fixes: - test-rigor: the runCodex STREAMING stitch (the production path) had no deterministic CI coverage — add runCodexStitch.test.ts (mocked spawn replays the captured fixture through the real loop, asserts run.completed.finalText). And the 'empty message' test placed the blank FIRST so it did not pin the non-empty guard — add a TRAILING-blank case that fails if the guard is removed (a trailing blank would silently wipe the answer). - truthfulness consistency: the OTHER receipt surface (turnReceipt 'Memory used.') still over-claimed — soften to 'Memory recalled.' so BOTH receipt surfaces claim recall, not application. Two stale 'used what I remember' code docstrings (speechBubble, bootstrap) and three docs (INTERACTION, floating-redesign, VERIFICATION) synced to 'checked'/'recalled'. - codex.ts: 'IS the answer' softened to best-effort-heuristic wording (trailing-non-answer edge). - contract nit: drop the dead attachCodexFinalText re-export from executor/index.ts. - guard: memorySteered.live asserts N>=1 (RORO_STEERING_N=0 would make the threshold vacuous). tsc clean · 1099 passed / 12 skipped · eslint 0 errors. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Vsq543S6voPTUsGsEqZvnD --- docs/INTERACTION.md | 2 +- docs/VERIFICATION.md | 2 +- docs/plans/floating-redesign.md | 2 +- src/core/executor/codex.ts | 6 ++- src/core/executor/index.ts | 2 +- src/core/executor/runCodexStitch.test.ts | 53 +++++++++++++++++++++ src/core/executor/stitchFinalText.test.ts | 7 +++ src/core/memory2/memorySteered.live.test.ts | 2 + src/renderer/bootstrap.ts | 4 +- src/renderer/bootstrap.typedPrompt.test.ts | 4 +- src/renderer/events/turnReceipt.test.ts | 4 +- src/renderer/events/turnReceipt.ts | 4 +- src/renderer/reactions/speechBubble.ts | 2 +- 13 files changed, 80 insertions(+), 14 deletions(-) create mode 100644 src/core/executor/runCodexStitch.test.ts diff --git a/docs/INTERACTION.md b/docs/INTERACTION.md index b2dbbda..a8ce0f1 100644 --- a/docs/INTERACTION.md +++ b/docs/INTERACTION.md @@ -133,7 +133,7 @@ The cat's answer/narration renders in a tethered speech bubble (`reactions/speechBubble.ts`), fed by the same CaptionSink seam that feeds the dev caption panel. When — and only when — memory was actually RECALLED during the turn (a same-run `status` reporting a recalled fact OR episode, `factCount>0 || -episodeCount>0`), the bubble appends a subtle, truthful receipt: "used what I +episodeCount>0`), the bubble appends a subtle, truthful receipt: "checked what I remember about your setup" (`MEMORY_RECEIPT_TEXT`). (It attests recall, not that a specific fact provably shaped the answer — it never claims more than the pipeline can verify.) The gate is strict: the receipt attaches ONLY diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 9f786bb..9302ff8 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -379,7 +379,7 @@ Manual full-window checklist: | 2 | Submit a non-empty prompt with project + brain ready | Start disables, typed text stays visible, Stop enables after readiness accepts the turn, status says `Thinking... click Stop if you need to pause.` | | 3 | Click Stop before `run.started` | Stop says `Stopping...`, status says `Stopping...`, and the turn ends with neutral `Stopped.` copy | | 4 | Click Stop after `run.started` on an active executor task | Stop says `Stopping...`, the executor is aborted, no completed file change lands, and the turn ends with neutral `Stopped.` copy | -| 5 | Let a successful turn finish | Status shows a compact receipt such as `Done. Memory checked.` or `Done. Changed 1 file. Memory used.` | +| 5 | Let a successful turn finish | Status shows a compact receipt such as `Done. Memory checked.` or `Done. Changed 1 file. Memory recalled.` | | 6 | Submit while project selection or local brain readiness blocks dispatch | Start remains enabled, Stop remains disabled, and the draft stays in the input | ## On-screen floating Ask + Stop diff --git a/docs/plans/floating-redesign.md b/docs/plans/floating-redesign.md index a5897f3..63544e0 100644 --- a/docs/plans/floating-redesign.md +++ b/docs/plans/floating-redesign.md @@ -30,7 +30,7 @@ memory/job is the payload. 4. **Modest transparent click-through window** (NOT full-screen). Grow enough for bloom room (~360–480 wide × ~360–440 tall — final size pinned in S1 against the cat sprite + tether math); drag = move the window (existing path, unchanged). The ~80% dead space passes clicks through to the app underneath. -5. **Truthful memory receipt.** On a successful turn, a speech bubble may say "used what I remember about +5. **Truthful memory receipt.** On a successful turn, a speech bubble may say "checked what I remember about your setup" ONLY when memory was actually recalled that turn (a same-run status with a recalled fact or episode; it attests recall, not that a fact provably shaped the answer) — never on every turn (that would be a lie that erodes trust). Bare no-payload success shows nothing (restraint; matches today's no-Done- diff --git a/src/core/executor/codex.ts b/src/core/executor/codex.ts index 7192950..c043fa3 100644 --- a/src/core/executor/codex.ts +++ b/src/core/executor/codex.ts @@ -232,8 +232,10 @@ export function codexExecArgs(opts: Pick ({ child: null as unknown as EventEmitter & { stdout: Readable; stderr: Readable } })); +vi.mock('node:child_process', () => ({ spawn: () => h.child })); + +import { runCodex } from './codex'; + +function fakeChild(stdout: string): EventEmitter & { stdout: Readable; stderr: Readable } { + const child = new EventEmitter() as EventEmitter & { stdout: Readable; stderr: Readable; exitCode: number | null; signalCode: null; kill: () => boolean }; + child.stdout = Readable.from(stdout); + child.stderr = Readable.from(''); + child.exitCode = null; + child.signalCode = null; + child.kill = () => true; + child.stdout.once('end', () => { child.exitCode = 0; queueMicrotask(() => child.emit('close', 0, null)); }); + return child; +} + +async function drain(): Promise { + const events: ActionEvent[] = []; + for await (const ev of runCodex({ repo: '/tmp/scratch', prompt: 'create hello.py' })) events.push(ev); + return events; +} + +describe('runCodex — the streaming finalText stitch (deterministic)', () => { + it('stitches the last agent_message onto run.completed.finalText over the captured v0.139.0 fixture', async () => { + h.child = fakeChild(readFileSync(join(__dirname, '__fixtures__/codex_hello.jsonl'), 'utf8')); + const events = await drain(); + const done = events.find((e) => e.kind === 'run.completed') as { finalText?: string } | undefined; + expect(done).toBeTruthy(); + expect(done!.finalText).toBe('Created [hello.py](/tmp/companion_scratch/hello.py:1).\n\nVerified with `python3 hello.py`; it prints:\n\n```text\nhi\n```'); + }); + + it('leaves finalText undefined when the stream carries no agent_message (receipt stays suppressed)', async () => { + // A minimal stream: thread + turn.completed, no agent_message → no answer → no finalText. + h.child = fakeChild( + '{"type":"thread.started","thread_id":"t"}\n{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}\n', + ); + const events = await drain(); + const done = events.find((e) => e.kind === 'run.completed') as { finalText?: string } | undefined; + expect(done).toBeTruthy(); + expect(done!.finalText).toBeUndefined(); + }); +}); diff --git a/src/core/executor/stitchFinalText.test.ts b/src/core/executor/stitchFinalText.test.ts index 51377fe..e801e25 100644 --- a/src/core/executor/stitchFinalText.test.ts +++ b/src/core/executor/stitchFinalText.test.ts @@ -42,6 +42,13 @@ describe('stitchCodexFinalText (batch — the proposal-fixture replay path)', () expect((out.find((e) => e.kind === 'run.completed') as { finalText?: string }).finalText).toBe('real'); }); + it('a TRAILING blank agent_message does not wipe the real answer (pins the `&& ev.text` guard)', () => { + // Without the guard, last-wins would take the trailing '' and finalText would go undefined → the answer + // would silently vanish from the receipt/caption/notification. This case fails if the guard is removed. + const out = stitchCodexFinalText([msg('the real answer'), msg(''), completed()]); + expect((out.find((e) => e.kind === 'run.completed') as { finalText?: string }).finalText).toBe('the real answer'); + }); + it('a run with no message yields run.completed with no finalText', () => { const out = stitchCodexFinalText([completed()]); expect((out[0] as { finalText?: string }).finalText).toBeUndefined(); diff --git a/src/core/memory2/memorySteered.live.test.ts b/src/core/memory2/memorySteered.live.test.ts index c0f1621..394c596 100644 --- a/src/core/memory2/memorySteered.live.test.ts +++ b/src/core/memory2/memorySteered.live.test.ts @@ -115,6 +115,8 @@ async function oneRun(arm: 'treatment' | 'control', i: number): Promise { describe.runIf(process.env.RORO_STEERING_LIVE === '1')('memory-steering (real qwen2.5:3b + real codex)', () => { it(`recalled memory reaches args.task AND the real diff across ${N} treatment + ${N} control`, async () => { + // Fail loud on a misconfigured sample size: N=0 would make the >=60%/0 threshold trivially true (vacuous). + expect(N, 'RORO_STEERING_N must be >= 1').toBeGreaterThanOrEqual(1); const rows: Row[] = []; for (const arm of ['treatment', 'control'] as const) { for (let i = 0; i < N; i++) { diff --git a/src/renderer/bootstrap.ts b/src/renderer/bootstrap.ts index 84eae67..fe6cd49 100644 --- a/src/renderer/bootstrap.ts +++ b/src/renderer/bootstrap.ts @@ -64,8 +64,8 @@ export async function bootstrap(): Promise { const timeline = new ActionTimeline(); // S5: the cat's SPEECH BUBBLE + truthful memory receipt. A tethered #roro-stage island that voices the // assistant text (the SAME CaptionSink seam that feeds the dev CaptionPanel) and, when memory was actually - // recalled that turn (same-run status + run.completed.finalText), appends a subtle truthful "used what I - // remember…" receipt. FLOATING mode + // recalled that turn (same-run status + run.completed.finalText), appends a subtle truthful "checked what I + // remember…" receipt (claims RECALL, not application). FLOATING mode // only; COMPOSED with the CaptionPanel so ONE subscribeActionEvents feed drives both (the CaptionPanel is // display:none inside #overlay in floating — harmless). In dev mode the bubble is not mounted. const speech = config.floatingWindow diff --git a/src/renderer/bootstrap.typedPrompt.test.ts b/src/renderer/bootstrap.typedPrompt.test.ts index 7d16598..d819faf 100644 --- a/src/renderer/bootstrap.typedPrompt.test.ts +++ b/src/renderer/bootstrap.typedPrompt.test.ts @@ -307,7 +307,7 @@ describe('bootstrap typed prompt Stop lifecycle', () => { h.action({ kind: 'run.completed', runId: 'typed-run', ok: true, finalText: 'done', ts: 4 }); h.runEnd('typed-run'); - expect(h.status.textContent).toBe('Done. Changed 1 file. Memory used.'); + expect(h.status.textContent).toBe('Done. Changed 1 file. Memory recalled.'); expect(h.send.disabled).toBe(false); expect(h.cancel.disabled).toBe(true); expect(h.input.value).toBe(''); @@ -331,7 +331,7 @@ describe('bootstrap typed prompt Stop lifecycle', () => { }); h.action({ kind: 'run.completed', runId: 'typed-run', ok: true, finalText: 'done', ts: 4 }); h.runEnd('typed-run'); - expect(h.status.textContent).toBe('Done. Changed 1 file. Memory used.'); + expect(h.status.textContent).toBe('Done. Changed 1 file. Memory recalled.'); h.input.value = 'what now?'; submit(h.form); diff --git a/src/renderer/events/turnReceipt.test.ts b/src/renderer/events/turnReceipt.test.ts index 26cd92a..ac63bfc 100644 --- a/src/renderer/events/turnReceipt.test.ts +++ b/src/renderer/events/turnReceipt.test.ts @@ -17,7 +17,7 @@ describe('turnReceipt', () => { completed, ]); - expect(receiptForTurnEnd(state)).toEqual({ tone: 'success', text: 'Done. Memory used.' }); + expect(receiptForTurnEnd(state)).toEqual({ tone: 'success', text: 'Done. Memory recalled.' }); }); it('shows memory checked when no saved memory matched', () => { @@ -47,7 +47,7 @@ describe('turnReceipt', () => { completed, ]); - expect(receiptForTurnEnd(state)).toEqual({ tone: 'success', text: 'Done. Changed 2 files. Memory used.' }); + expect(receiptForTurnEnd(state)).toEqual({ tone: 'success', text: 'Done. Changed 2 files. Memory recalled.' }); }); it('keeps stopped turns neutral', () => { diff --git a/src/renderer/events/turnReceipt.ts b/src/renderer/events/turnReceipt.ts index 47dbe07..8c7c814 100644 --- a/src/renderer/events/turnReceipt.ts +++ b/src/renderer/events/turnReceipt.ts @@ -70,7 +70,9 @@ function filePhrase(count: number): string | null { function memoryPhrase(state: TurnReceiptState): string | null { if (!state.memorySeen) return null; - return state.factCount > 0 || state.episodeCount > 0 ? 'Memory used.' : 'Memory checked.'; + // Recall-truthful, matching the SpeechBubble receipt ("checked", not "used"): "recalled" attests that memory + // surfaced something this turn, NOT that it shaped the answer (which the pipeline can't prove from here). + return state.factCount > 0 || state.episodeCount > 0 ? 'Memory recalled.' : 'Memory checked.'; } export function receiptForTurnEnd(state: TurnReceiptState, cancelRequested = false): TurnReceipt { diff --git a/src/renderer/reactions/speechBubble.ts b/src/renderer/reactions/speechBubble.ts index 3addc0b..296eea8 100644 --- a/src/renderer/reactions/speechBubble.ts +++ b/src/renderer/reactions/speechBubble.ts @@ -77,7 +77,7 @@ export function reduceSpeechTurn(state: SpeechTurnState, e: ActionEvent): Speech } /** - * The truthful-receipt gate. The "used what I remember" line shows IFF the event is the turn's DEFINITIVE + * The truthful-receipt gate. The "checked what I remember" line shows IFF the event is the turn's DEFINITIVE * answer — `run.completed.finalText` — AND memory was recalled in the SAME run. It deliberately does NOT * receipt a bare `message`: the orchestrator emits `message` BOTH for the brain's PLANNING beat on executor * turns ("…is planning the task…", before the answer) AND for answer-turn narration, and the two are From 440fac57dc3567a2e0af5100cd88f4325514a3e0 Mon Sep 17 00:00:00 2001 From: Jin Choi Date: Sat, 4 Jul 2026 23:44:04 -0700 Subject: [PATCH 5/9] fix(steering): tighten the live guard to prove the convention is HONORED (codex review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex review caught a truthfulness gap in the guard: the seeded convention is 'every .py file BEGINS with the marker as its first line', but markerInFiles only checked whether the marker appeared ANYWHERE in ANY file — which could pass on a README while hello.py violates the rule, overstating what 'honored the diff' proves. Added markerAsPyHeader() (marker is the FIRST non-blank line of a created .py file) and moved the treatment assertion onto it, so the guard proves the executor actually HONORED the recalled convention. Control still asserts the marker appears nowhere. ROADMAP note reconciled (loose anywhere-run vs the strict per-header guard). No CI impact (opt-in/non-CI, collect-but-skip). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Vsq543S6voPTUsGsEqZvnD --- docs/ROADMAP.md | 7 +-- src/core/memory2/memorySteered.live.test.ts | 57 ++++++++++++++++----- 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index f146d11..961a884 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -240,9 +240,10 @@ first-turn/memory validation** — otherwise it is deferred (§7 rule 4); C is s stochasticity), and keep **real Codex/Claude as a separate auth/readiness check**. **UPDATE (2026-07-04): the real-honoring check is now AUTOMATED** — `npm run verify:memory-steered-real` (`src/core/memory2/memorySteered.live.test.ts`, opt-in/non-CI) runs real qwen2.5:3b + real codex over - N treatment/control runs and asserts the recalled marker reaches BOTH `args.task` AND the actual diff - (≥60% treatment, 0 control). First measured run: **5/5 args.task · 5/5 diff · 0/5 control** — the moat - mechanism holds for a well-formed, relevant preference. (The *extraction* hop — conversation→stored fact, + N treatment/control runs and asserts the recalled marker reaches `args.task` AND that the created `.py` + file HONORS the convention (marker as its header line — not merely present somewhere), ≥60% treatment, + 0 control. First measured run: **5/5 args.task · 5/5 marker-in-file · 0/5 control** — the moat mechanism + holds for a well-formed, relevant preference (the guard's stricter per-header assertion re-measures it). (The *extraction* hop — conversation→stored fact, the ~40% behavioral-eval weak link — remains separate and is mitigated by the executor-facts confirm gate.) **The manual check is auditable — it produces an evidence packet:** {build id; both memory-root paths + the empty-control proof; the exact prompt; the event stream; the generated `args.task`; the resulting diff; observer + diff --git a/src/core/memory2/memorySteered.live.test.ts b/src/core/memory2/memorySteered.live.test.ts index 394c596..f49cc27 100644 --- a/src/core/memory2/memorySteered.live.test.ts +++ b/src/core/memory2/memorySteered.live.test.ts @@ -52,6 +52,32 @@ function readAllFiles(dir: string): string { return out; } +// Faithful "honored" check. The seeded convention is specifically "every .py file BEGINS with the marker as +// its first line", so verify the marker is the FIRST non-blank line of a created .py file — NOT merely present +// somewhere (a README mention or a mid-file line could satisfy 'anywhere' while hello.py violates the rule). +// This makes the treatment assertion prove what it claims (the executor HONORED the recalled convention). +function markerAsPyHeader(dir: string): boolean { + let honored = false; + const walk = (d: string): void => { + let entries: string[] = []; + try { entries = readdirSync(d); } catch { return; } + for (const name of entries) { + if (name === '.git') continue; + const p = join(d, name); + let s; try { s = statSync(p); } catch { continue; } + if (s.isDirectory()) walk(p); + else if (name.endsWith('.py')) { + try { + const firstLine = readFileSync(p, 'utf8').split('\n').find((l) => l.trim().length > 0) ?? ''; + if (firstLine.includes(MARKER)) honored = true; + } catch { /* skip */ } + } + } + }; + walk(dir); + return honored; +} + async function runExecutor(repo: string, prompt: string): Promise<{ ok: boolean; kinds: string[] }> { const ac = new AbortController(); const timer = setTimeout(() => ac.abort(), CODEX_TURN_MS); @@ -66,7 +92,7 @@ async function runExecutor(repo: string, prompt: string): Promise<{ ok: boolean; return { ok, kinds }; } -interface Row { arm: string; i: number; factCount: number; markerInContext: boolean; command: string; markerInArgsTask: boolean; ranExecutor: boolean; execOk: boolean; markerInFiles: boolean; files: string[]; taskPreview: string } +interface Row { arm: string; i: number; factCount: number; markerInContext: boolean; command: string; markerInArgsTask: boolean; ranExecutor: boolean; execOk: boolean; markerInFiles: boolean; markerHonored: boolean; files: string[]; taskPreview: string } async function oneRun(arm: 'treatment' | 'control', i: number): Promise { const memDir = mkdtempSync(join(tmpdir(), `p0-mem-${arm}-`)); @@ -74,7 +100,7 @@ async function oneRun(arm: 'treatment' | 'control', i: number): Promise { spawnSync('git', ['init', repo], { stdio: 'ignore' }); const cipher = createAesGcmCipher(Buffer.alloc(32, 7)); const dim = (await brain.embed('dimension probe')).length; - const row: Row = { arm, i, factCount: 0, markerInContext: false, command: '-', markerInArgsTask: false, ranExecutor: false, execOk: false, markerInFiles: false, files: [], taskPreview: '' }; + const row: Row = { arm, i, factCount: 0, markerInContext: false, command: '-', markerInArgsTask: false, ranExecutor: false, execOk: false, markerInFiles: false, markerHonored: false, files: [], taskPreview: '' }; try { // Launch A: seed the convention (treatment only), then close (persist to disk). const a = createMemoryFacade(await createMemoryStore({ dir: memDir, embed: realEmbed, dim, cipher })); @@ -103,7 +129,8 @@ async function oneRun(arm: 'treatment' | 'control', i: number): Promise { const exec = await runExecutor(repo, task); row.execOk = exec.ok; const filesText = readAllFiles(repo); - row.markerInFiles = filesText.includes(MARKER); + row.markerInFiles = filesText.includes(MARKER); // present anywhere (diagnostic + control false-positive check) + row.markerHonored = markerAsPyHeader(repo); // the CONVENTION honored: marker as the .py header (the real claim) row.files = (filesText.match(/--- (.+?) ---/g) || []).map((m) => m.replace(/--- (.+?) ---/, '$1')); } } finally { @@ -122,31 +149,33 @@ describe.runIf(process.env.RORO_STEERING_LIVE === '1')('memory-steering (real qw for (let i = 0; i < N; i++) { const r = await oneRun(arm, i); rows.push(r); - console.log(`[${arm}#${i + 1}] factCount=${r.factCount} ctx=${r.markerInContext} cmd=${r.command} argsTask=${r.markerInArgsTask} ranExec=${r.ranExecutor} execOk=${r.execOk} files=${r.markerInFiles} wrote=[${r.files.join(', ')}]`); + console.log(`[${arm}#${i + 1}] factCount=${r.factCount} ctx=${r.markerInContext} cmd=${r.command} argsTask=${r.markerInArgsTask} ranExec=${r.ranExecutor} execOk=${r.execOk} anyFile=${r.markerInFiles} honored=${r.markerHonored} wrote=[${r.files.join(', ')}]`); console.log(` task="${r.taskPreview}"`); } } const t = rows.filter((r) => r.arm === 'treatment'); const c = rows.filter((r) => r.arm === 'control'); const hit = (rs: Row[], k: keyof Row): string => `${rs.filter((r) => r[k]).length}/${rs.length}`; - console.log('\n===== PHASE 0 SUMMARY ====='); + console.log('\n===== MEMORY-STEERING SUMMARY ====='); console.log('TREATMENT (memory seeded):'); console.log(` (1) recall→context: ${hit(t, 'markerInContext')}`); - console.log(` (2) →args.task: ${hit(t, 'markerInArgsTask')} [decisionRouter.ts:173 — the fragile heart]`); - console.log(` (3) →actual files: ${hit(t, 'markerInFiles')} [real codex honored it]`); + console.log(` (2) →args.task: ${hit(t, 'markerInArgsTask')} [decisionRouter.ts:173 — the model-discretion hop]`); + console.log(` (3) convention HONORED:${hit(t, 'markerHonored')} [marker is the .py header — codex honored the recalled rule]`); + console.log(` (marker anywhere): ${hit(t, 'markerInFiles')}`); console.log(` run_agent chosen: ${hit(t, 'ranExecutor')}`); - console.log('CONTROL (no memory) — expect ~0:'); + console.log('CONTROL (no memory) — expect 0:'); console.log(` (2) →args.task: ${hit(c, 'markerInArgsTask')}`); - console.log(` (3) →actual files: ${hit(c, 'markerInFiles')}`); - console.log('===========================\n'); + console.log(` (3) marker anywhere: ${hit(c, 'markerInFiles')}`); + console.log('===================================\n'); - // THE GUARD (ROADMAP Arc A go/no-go): treatment steers >=60% on BOTH args.task and the actual diff; - // control is 0 on both (no false positives). Without these expect()s this opt-in test would be vacuous - // green — worse than the deterministic fake-smoke it complements. First measured run: 5/5 · 5/5 · 0/5 · 0/5. + // THE GUARD (ROADMAP Arc A go/no-go): treatment reaches args.task AND HONORS the convention (marker as the + // .py header — not merely present somewhere) on >=60% of runs; control shows the marker NOWHERE (no false + // positives). Without these expect()s this opt-in test would be vacuous green. First measured run (loose + // 'anywhere' check): 5/5 · 5/5 · 0/5 · 0/5; the strict per-file-header assertion is (3) below. const count = (rs: Row[], k: keyof Row): number => rs.filter((r) => r[k]).length; const threshold = Math.ceil(0.6 * N); expect(count(t, 'markerInArgsTask'), 'treatment: recalled memory reached args.task').toBeGreaterThanOrEqual(threshold); - expect(count(t, 'markerInFiles'), 'treatment: recalled memory reached the REAL diff').toBeGreaterThanOrEqual(threshold); + expect(count(t, 'markerHonored'), 'treatment: the created .py file HONORS the recalled convention (marker as header)').toBeGreaterThanOrEqual(threshold); expect(count(c, 'markerInArgsTask'), 'control (no memory): NO marker in args.task').toBe(0); expect(count(c, 'markerInFiles'), 'control (no memory): NO marker in the diff').toBe(0); }, N * 4 * CODEX_TURN_MS + 120_000); From 14009eda790814fd2b40076eed0c00d90423d4ed Mon Sep 17 00:00:00 2001 From: Jin Choi Date: Sat, 4 Jul 2026 23:53:45 -0700 Subject: [PATCH 6/9] fix(steering): exact per-file convention check on hello.py (codex review round 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex re-review sharpened the guard once more: checking the first line INCLUDES the marker on ANY .py file still over-claimed — a Python assignment line (MARKER = '...') or a compliant helper beside a non-compliant hello.py would pass. conventionHonored() now targets hello.py (the file the task creates) and asserts its first non-blank line EQUALS the exact header '# codename: '. This makes markerHonored prove exactly the seeded convention. Opt-in/ non-CI; no CI impact. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Vsq543S6voPTUsGsEqZvnD --- src/core/memory2/memorySteered.live.test.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/core/memory2/memorySteered.live.test.ts b/src/core/memory2/memorySteered.live.test.ts index f49cc27..647c082 100644 --- a/src/core/memory2/memorySteered.live.test.ts +++ b/src/core/memory2/memorySteered.live.test.ts @@ -52,11 +52,13 @@ function readAllFiles(dir: string): string { return out; } -// Faithful "honored" check. The seeded convention is specifically "every .py file BEGINS with the marker as -// its first line", so verify the marker is the FIRST non-blank line of a created .py file — NOT merely present -// somewhere (a README mention or a mid-file line could satisfy 'anywhere' while hello.py violates the rule). -// This makes the treatment assertion prove what it claims (the executor HONORED the recalled convention). -function markerAsPyHeader(dir: string): boolean { +// Faithful "honored" check. The seeded convention requires hello.py (the file the task creates) to BEGIN with +// the EXACT header line `# codename: `. So verify that specific file's first non-blank line EQUALS the +// exact header — not "the marker appears somewhere" (which a `MARKER = '…'` assignment line, a README, or a +// compliant helper beside a non-compliant hello.py could fake). This makes the treatment assertion prove what +// it claims: the executor HONORED the recalled convention on the file it was asked to create. +const CONVENTION_HEADER = `# codename: ${MARKER}`; +function conventionHonored(dir: string): boolean { let honored = false; const walk = (d: string): void => { let entries: string[] = []; @@ -66,10 +68,10 @@ function markerAsPyHeader(dir: string): boolean { const p = join(d, name); let s; try { s = statSync(p); } catch { continue; } if (s.isDirectory()) walk(p); - else if (name.endsWith('.py')) { + else if (name === 'hello.py') { try { const firstLine = readFileSync(p, 'utf8').split('\n').find((l) => l.trim().length > 0) ?? ''; - if (firstLine.includes(MARKER)) honored = true; + if (firstLine.trim() === CONVENTION_HEADER) honored = true; } catch { /* skip */ } } } @@ -130,7 +132,7 @@ async function oneRun(arm: 'treatment' | 'control', i: number): Promise { row.execOk = exec.ok; const filesText = readAllFiles(repo); row.markerInFiles = filesText.includes(MARKER); // present anywhere (diagnostic + control false-positive check) - row.markerHonored = markerAsPyHeader(repo); // the CONVENTION honored: marker as the .py header (the real claim) + row.markerHonored = conventionHonored(repo); // the CONVENTION honored: hello.py's first line IS the exact header row.files = (filesText.match(/--- (.+?) ---/g) || []).map((m) => m.replace(/--- (.+?) ---/, '$1')); } } finally { From 21e13e557d4bd78f8a13eeccf3b7a47da80bd75a Mon Sep 17 00:00:00 2001 From: Jin Choi Date: Sun, 5 Jul 2026 00:01:45 -0700 Subject: [PATCH 7/9] fix(steering): require run success + root hello.py exact-header (codex review round 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex round 3: (P1) markerHonored counted even if codex later run.failed/aborted — now gated on exec.ok (the run.completed ok), per the ROADMAP proof contract. (P2) the recursive hello.py search could match a nested/second compliant file masking a non-compliant root hello.py — conventionHonored now checks the repo-ROOT hello.py directly (codex runs -C repo), exact-header only. Opt-in/non-CI. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Vsq543S6voPTUsGsEqZvnD --- src/core/memory2/memorySteered.live.test.ts | 39 +++++++-------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/src/core/memory2/memorySteered.live.test.ts b/src/core/memory2/memorySteered.live.test.ts index 647c082..be2a694 100644 --- a/src/core/memory2/memorySteered.live.test.ts +++ b/src/core/memory2/memorySteered.live.test.ts @@ -52,32 +52,17 @@ function readAllFiles(dir: string): string { return out; } -// Faithful "honored" check. The seeded convention requires hello.py (the file the task creates) to BEGIN with -// the EXACT header line `# codename: `. So verify that specific file's first non-blank line EQUALS the -// exact header — not "the marker appears somewhere" (which a `MARKER = '…'` assignment line, a README, or a -// compliant helper beside a non-compliant hello.py could fake). This makes the treatment assertion prove what -// it claims: the executor HONORED the recalled convention on the file it was asked to create. +// Faithful "honored" check. The task creates hello.py at the repo ROOT (codex runs `-C repo`), and the seeded +// convention requires it to BEGIN with the EXACT header line `# codename: `. Check exactly THAT file's +// first non-blank line == the exact header — not "the marker appears somewhere" (a `MARKER = '…'` assignment +// line, a README, or a compliant nested/second hello.py could fake it). Caller also requires the run SUCCEEDED +// (run.completed ok, no run.failed) before counting it, per docs/ROADMAP.md's proof contract. const CONVENTION_HEADER = `# codename: ${MARKER}`; -function conventionHonored(dir: string): boolean { - let honored = false; - const walk = (d: string): void => { - let entries: string[] = []; - try { entries = readdirSync(d); } catch { return; } - for (const name of entries) { - if (name === '.git') continue; - const p = join(d, name); - let s; try { s = statSync(p); } catch { continue; } - if (s.isDirectory()) walk(p); - else if (name === 'hello.py') { - try { - const firstLine = readFileSync(p, 'utf8').split('\n').find((l) => l.trim().length > 0) ?? ''; - if (firstLine.trim() === CONVENTION_HEADER) honored = true; - } catch { /* skip */ } - } - } - }; - walk(dir); - return honored; +function conventionHonored(repoRoot: string): boolean { + try { + const firstLine = readFileSync(join(repoRoot, 'hello.py'), 'utf8').split('\n').find((l) => l.trim().length > 0) ?? ''; + return firstLine.trim() === CONVENTION_HEADER; + } catch { return false; } // hello.py absent / unreadable → not honored } async function runExecutor(repo: string, prompt: string): Promise<{ ok: boolean; kinds: string[] }> { @@ -132,7 +117,9 @@ async function oneRun(arm: 'treatment' | 'control', i: number): Promise { row.execOk = exec.ok; const filesText = readAllFiles(repo); row.markerInFiles = filesText.includes(MARKER); // present anywhere (diagnostic + control false-positive check) - row.markerHonored = conventionHonored(repo); // the CONVENTION honored: hello.py's first line IS the exact header + // Honored ONLY if the run SUCCEEDED (run.completed ok, no run.failed/abort) AND root hello.py's first + // line is the exact header — a run that wrote the file then failed is not proof (ROADMAP proof contract). + row.markerHonored = exec.ok && conventionHonored(repo); row.files = (filesText.match(/--- (.+?) ---/g) || []).map((m) => m.replace(/--- (.+?) ---/, '$1')); } } finally { From a2ec249d538328b289ea480a05fa0493088f8845 Mon Sep 17 00:00:00 2001 From: Jin Choi Date: Sun, 5 Jul 2026 00:08:24 -0700 Subject: [PATCH 8/9] fix(steering): exact LITERAL first-line header check (codex review round 4) Codex round 4: skipping leading blanks + trimming let a hello.py starting with a blank line or leading spaces false-pass, looser than 'begin with the exact first line'. Now checks the literal line 0 (only a trailing CR stripped) == the exact header. Maximally faithful to the seeded convention. Opt-in/non-CI. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Vsq543S6voPTUsGsEqZvnD --- src/core/memory2/memorySteered.live.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/memory2/memorySteered.live.test.ts b/src/core/memory2/memorySteered.live.test.ts index be2a694..96305fc 100644 --- a/src/core/memory2/memorySteered.live.test.ts +++ b/src/core/memory2/memorySteered.live.test.ts @@ -60,8 +60,10 @@ function readAllFiles(dir: string): string { const CONVENTION_HEADER = `# codename: ${MARKER}`; function conventionHonored(repoRoot: string): boolean { try { - const firstLine = readFileSync(join(repoRoot, 'hello.py'), 'utf8').split('\n').find((l) => l.trim().length > 0) ?? ''; - return firstLine.trim() === CONVENTION_HEADER; + // The LITERAL first line (line 0, only a trailing \r stripped) must BE the exact header — no skipping a + // leading blank line, no trimming leading spaces, since the convention says "BEGIN with the exact first line". + const firstLine = readFileSync(join(repoRoot, 'hello.py'), 'utf8').split('\n')[0].replace(/\r$/, ''); + return firstLine === CONVENTION_HEADER; } catch { return false; } // hello.py absent / unreadable → not honored } From 24b91149ac3545c92ea6d42a3a80b914f4f928bd Mon Sep 17 00:00:00 2001 From: Jin Choi Date: Sun, 5 Jul 2026 00:14:43 -0700 Subject: [PATCH 9/9] fix(steering): sync stale comment + pin the qwen2.5:3b/nomic models (codex review round 5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 5: (1) the docstring still said 'first non-blank line' after the impl moved to literal line 0 — synced. (2) the 'real qwen2.5:3b' claim was not pinned (brain.decide honors an inherited OLLAMA_MODEL), so verify:memory-steered-real now pins OLLAMA_MODEL=qwen2.5:3b + OLLAMA_EMBED_MODEL=nomic-embed-text so the proof runs the model it claims. Opt-in/non-CI. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Vsq543S6voPTUsGsEqZvnD --- package.json | 2 +- src/core/memory2/memorySteered.live.test.ts | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 018924a..9429c4e 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "verify:packaged-sdk-executor": "node scripts/smoke-packaged-sdk-executor.mjs", "verify:packaged-model-setup": "node scripts/smoke-packaged-model-setup.mjs", "verify:memory-steered": "node scripts/smoke-memory-steered.mjs", - "verify:memory-steered-real": "RORO_STEERING_LIVE=1 OLLAMA_AVAILABLE=1 vitest run src/core/memory2/memorySteered.live.test.ts", + "verify:memory-steered-real": "RORO_STEERING_LIVE=1 OLLAMA_AVAILABLE=1 OLLAMA_MODEL=qwen2.5:3b OLLAMA_EMBED_MODEL=nomic-embed-text vitest run src/core/memory2/memorySteered.live.test.ts", "verify:release-artifact": "node scripts/verify-release-artifact.mjs --mode default", "verify:release-artifact:dmg": "node scripts/verify-release-artifact.mjs --mode default --require-dmg", "verify:release-artifact:signed": "node scripts/verify-release-artifact.mjs --mode signed", diff --git a/src/core/memory2/memorySteered.live.test.ts b/src/core/memory2/memorySteered.live.test.ts index 96305fc..4ea4271 100644 --- a/src/core/memory2/memorySteered.live.test.ts +++ b/src/core/memory2/memorySteered.live.test.ts @@ -53,10 +53,11 @@ function readAllFiles(dir: string): string { } // Faithful "honored" check. The task creates hello.py at the repo ROOT (codex runs `-C repo`), and the seeded -// convention requires it to BEGIN with the EXACT header line `# codename: `. Check exactly THAT file's -// first non-blank line == the exact header — not "the marker appears somewhere" (a `MARKER = '…'` assignment -// line, a README, or a compliant nested/second hello.py could fake it). Caller also requires the run SUCCEEDED -// (run.completed ok, no run.failed) before counting it, per docs/ROADMAP.md's proof contract. +// convention requires it to BEGIN with the EXACT header line `# codename: ` as its FIRST line. Check +// exactly THAT file's LITERAL first line (line 0) == the exact header — not "the marker appears somewhere" (a +// `MARKER = '…'` assignment line, a README, or a compliant nested/second hello.py could fake it) and not a +// blank/indented first line. Caller also requires the run SUCCEEDED (run.completed ok, no run.failed) before +// counting it, per docs/ROADMAP.md's proof contract. const CONVENTION_HEADER = `# codename: ${MARKER}`; function conventionHonored(repoRoot: string): boolean { try {