From 2c67afb01398923d10eeac461d3cc285e10ec7ae Mon Sep 17 00:00:00 2001 From: yuzishu Date: Wed, 8 Jul 2026 14:40:08 +0800 Subject: [PATCH 1/5] feat(eval): collect codex token usage --- .../plans/2026-07-08-codex-token-usage.md | 167 ++++++++++++++++++ .../2026-07-08-codex-token-usage-design.md | 53 ++++++ multi-agent/internal/evalrun/schema.go | 28 +-- multi-agent/internal/evalrun/schema_test.go | 2 + multi-agent/internal/evalrun/writer.go | 15 +- multi-agent/internal/evalrun/writer_test.go | 26 ++- multi-agent/internal/observerstore/schema.sql | 6 +- .../tests/test_metrics_user_promoted.py | 41 +++++ multi-agent/tools/eval/runner/codex_usage.go | 123 +++++++++++++ .../tools/eval/runner/codex_usage_test.go | 78 ++++++++ multi-agent/tools/eval/runner/flags_test.go | 40 +++-- multi-agent/tools/eval/runner/main.go | 10 +- multi-agent/tools/eval/runner/runner.go | 8 + multi-agent/tools/eval/runner/runner_test.go | 45 +++++ multi-agent/tools/eval/runner/writer.go | 10 ++ multi-agent/tools/eval/runner/writer_test.go | 46 ++++- 16 files changed, 650 insertions(+), 48 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-08-codex-token-usage.md create mode 100644 docs/superpowers/specs/2026-07-08-codex-token-usage-design.md create mode 100644 multi-agent/tools/eval/runner/codex_usage.go create mode 100644 multi-agent/tools/eval/runner/codex_usage_test.go diff --git a/docs/superpowers/plans/2026-07-08-codex-token-usage.md b/docs/superpowers/plans/2026-07-08-codex-token-usage.md new file mode 100644 index 00000000..bb00af9d --- /dev/null +++ b/docs/superpowers/plans/2026-07-08-codex-token-usage.md @@ -0,0 +1,167 @@ +# Codex Token Usage Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add per-run Codex CLI input/output token collection to the eval runner, CSV output, SQLite `runs` schema, and existing `TokenUsage` metric path. + +**Architecture:** The runner accepts `--codex-usage-jsonl`, parses a Codex CLI JSONL file into a small `CodexTokenUsage` value, and stores those totals on `RunRow`. CSV and SQLite schemas append `model_input_tokens` and `model_output_tokens`; the Python metric extractor already consumes those column names. + +**Tech Stack:** Go eval runner and evalrun SQLite writer, SQLite schema SQL, Python pytest metric tests. + +--- + +### Task 1: Codex Usage Parser + +**Files:** +- Create: `multi-agent/tools/eval/runner/codex_usage.go` +- Test: `multi-agent/tools/eval/runner/codex_usage_test.go` + +- [ ] **Step 1: Write failing parser tests** + +Add tests that call `ParseCodexUsageJSONL(path)` on JSONL fixtures. Cover summed usage records, alias fields, wrapper objects, ignored non-usage records, malformed JSON, unreadable files, and negative token counts. + +- [ ] **Step 2: Run red test** + +Run: `go test ./tools/eval/runner -run 'TestParseCodexUsageJSONL'` + +Expected: FAIL because `ParseCodexUsageJSONL` and `CodexTokenUsage` do not exist. + +- [ ] **Step 3: Implement minimal parser** + +Implement `CodexTokenUsage`, `ParseCodexUsageJSONL`, and helper functions in `codex_usage.go`. Read line by line, unmarshal to `map[string]any`, find usage maps at top level and under `event`, `response`, and `message`, sum non-negative integer values, and return an error for malformed JSON or negative values. + +- [ ] **Step 4: Run green test** + +Run: `go test ./tools/eval/runner -run 'TestParseCodexUsageJSONL'` + +Expected: PASS. + +### Task 2: Runner CSV Columns + +**Files:** +- Modify: `multi-agent/tools/eval/runner/writer.go` +- Modify: `multi-agent/tools/eval/runner/writer_test.go` + +- [ ] **Step 1: Write failing CSV tests** + +Update frozen column tests to expect appended `model_input_tokens` and `model_output_tokens`. Add a row roundtrip assertion that a `RunRow` with token values serializes those values into the new columns. + +- [ ] **Step 2: Run red test** + +Run: `go test ./tools/eval/runner -run 'TestCSVColumns|TestRunRow_CodexTokenUsage_RoundTripCSV'` + +Expected: FAIL because the columns and fields do not exist. + +- [ ] **Step 3: Implement CSV fields** + +Add `ModelInputTokens int` and `ModelOutputTokens int` to `RunRow`, append column names to `CSVColumns`, and append values in `rowAsCSVRecord`. + +- [ ] **Step 4: Run green test** + +Run: `go test ./tools/eval/runner -run 'TestCSVColumns|TestRunRow_CodexTokenUsage_RoundTripCSV'` + +Expected: PASS. + +### Task 3: Runner Flag and Row Assembly + +**Files:** +- Modify: `multi-agent/tools/eval/runner/main.go` +- Modify: `multi-agent/tools/eval/runner/runner.go` +- Test: `multi-agent/tools/eval/runner/runner_test.go` + +- [ ] **Step 1: Write failing runner test** + +Add a test that supplies `Opts.CodexUsageJSONL` with a JSONL fixture and asserts `res.Row.ModelInputTokens` and `res.Row.ModelOutputTokens` contain the parsed totals. + +- [ ] **Step 2: Run red test** + +Run: `go test ./tools/eval/runner -run 'TestRun_RecordsCodexUsageJSONL'` + +Expected: FAIL because `Opts.CodexUsageJSONL` is not wired. + +- [ ] **Step 3: Implement flag and row wiring** + +Add `CodexUsageJSONL string` to `Opts`, add `--codex-usage-jsonl` in `main.go`, parse usage in `Run` before row assembly, return exit code 2 on parse errors, and stamp totals into `RunRow`. + +- [ ] **Step 4: Run green test** + +Run: `go test ./tools/eval/runner -run 'TestRun_RecordsCodexUsageJSONL'` + +Expected: PASS. + +### Task 4: SQLite Runs Schema + +**Files:** +- Modify: `multi-agent/internal/observerstore/schema.sql` +- Modify: `multi-agent/internal/evalrun/schema.go` +- Modify: `multi-agent/internal/evalrun/writer.go` +- Modify: `multi-agent/internal/evalrun/writer_test.go` +- Test: `multi-agent/internal/evalrun/schema_test.go` + +- [ ] **Step 1: Write failing DB tests** + +Update roundtrip tests to insert and read token counts. Update schema drift expectations to require the two appended columns and detect missing token columns. + +- [ ] **Step 2: Run red test** + +Run: `go test ./internal/evalrun` + +Expected: FAIL because schema, expected columns, insert SQL, and `Schema` do not include token fields. + +- [ ] **Step 3: Implement DB schema changes** + +Append the two integer columns to `schema.sql`, `Schema`, `expectedColumns`, and `insertSQL`. Pass `s.ModelInputTokens` and `s.ModelOutputTokens` to `ExecContext`. + +- [ ] **Step 4: Run green test** + +Run: `go test ./internal/evalrun` + +Expected: PASS. + +### Task 5: Metric Extraction Verification + +**Files:** +- Modify: `multi-agent/tools/eval/metrics/tests/test_metrics_user_promoted.py` + +- [ ] **Step 1: Write failing metric test** + +Add a test that creates a temp DB from a fixture schema with `model_input_tokens` and `model_output_tokens`, inserts two selected runs, and asserts `TokenUsage` returns summed input/output plus count. + +- [ ] **Step 2: Run red test** + +Run: `python -m pytest tests/test_metrics_user_promoted.py` + +Expected: FAIL until the fixture creates the new columns or inserts valid rows with the new schema. + +- [ ] **Step 3: Make metric fixture use new columns** + +Adjust the test fixture setup so the DB has both token columns and the inserted rows have known values. + +- [ ] **Step 4: Run green test** + +Run: `python -m pytest tests/test_metrics_user_promoted.py` + +Expected: PASS. + +### Task 6: Final Verification + +**Files:** +- All changed files above. + +- [ ] **Step 1: Run focused Go tests** + +Run: `go test ./tools/eval/runner ./internal/evalrun` + +Expected: PASS. + +- [ ] **Step 2: Run metric tests** + +Run from `multi-agent/tools/eval/metrics`: `python -m pytest tests/test_metrics_user_promoted.py` + +Expected: PASS. + +- [ ] **Step 3: Inspect worktree** + +Run: `git status --short` + +Expected: only intentional files changed. diff --git a/docs/superpowers/specs/2026-07-08-codex-token-usage-design.md b/docs/superpowers/specs/2026-07-08-codex-token-usage-design.md new file mode 100644 index 00000000..0073bc78 --- /dev/null +++ b/docs/superpowers/specs/2026-07-08-codex-token-usage-design.md @@ -0,0 +1,53 @@ +# Codex Token Usage Design + +## Goal + +Collect per-run Codex CLI input and output token counts in the evaluation runner, so the existing `TokenUsage` metric can report real values from `runs.model_input_tokens` and `runs.model_output_tokens`. + +## Scope + +This change supports Codex CLI only. It does not normalize usage from Claude, OpenHands, E2B, or other agents. If no Codex usage JSONL is supplied, token fields remain zero. + +## Interface + +Add a runner CLI flag and matching `Opts` field: + +```text +--codex-usage-jsonl +``` + +The file is parsed after the workload oracle runs and before `RunRow` assembly. The parser reads Codex CLI JSONL line by line, ignores lines without usage data, and sums usage records. + +Accepted usage keys: + +- input: `input_tokens`, `prompt_tokens` +- output: `output_tokens`, `completion_tokens` + +The parser accepts `usage` objects at the top level and under common wrappers such as `event`, `response`, and `message`. + +## Persistence + +Append two numeric fields to runner CSV: + +- `model_input_tokens` +- `model_output_tokens` + +Append two integer columns to `runs`: + +- `model_input_tokens INTEGER NOT NULL DEFAULT 0` +- `model_output_tokens INTEGER NOT NULL DEFAULT 0` + +The existing Python metric extractor already probes for these two columns and sums them, so no metric API change is needed. + +## Errors + +An unreadable or malformed supplied JSONL file is a preflight-style runner error with exit code 2. Individual JSONL lines without usage are ignored. Negative token counts are rejected. + +## Testing + +Use TDD: + +1. Add parser tests for Codex JSONL usage summing and ignored non-usage lines. +2. Add runner/CSV tests for appended columns and row serialization. +3. Add evalrun DB roundtrip and schema drift tests for the new columns. +4. Add a Python metric test showing `TokenUsage` returns real sums when the columns exist. diff --git a/multi-agent/internal/evalrun/schema.go b/multi-agent/internal/evalrun/schema.go index e840590d..eff0c4af 100644 --- a/multi-agent/internal/evalrun/schema.go +++ b/multi-agent/internal/evalrun/schema.go @@ -18,19 +18,19 @@ import ( // order matches /root/paper_writing/docs/intermediate/08_evaluation_plan_v3.md // lines 256–279 and the DDL in internal/observerstore/schema.sql. type Schema struct { - RunID string // 08:256 PK - WorkloadID string // 08:257 - ClaimID string // 08:258 - ExperimentID string // 08:259 - BaselineOrAblation string // 08:260 - LoomCommit string // 08:261 - AgentserverCommit string // 08:262 - ModelserverCommit string // 08:263 - AppCommit string // 08:264 - MachineTopology string // 08:265 - ContextGroundTruth string // 08:266 - CapabilitySnapshotHash string // 08:267 — interface placeholder - TaskContractHash string // 08:268 — interface placeholder + RunID string // 08:256 PK + WorkloadID string // 08:257 + ClaimID string // 08:258 + ExperimentID string // 08:259 + BaselineOrAblation string // 08:260 + LoomCommit string // 08:261 + AgentserverCommit string // 08:262 + ModelserverCommit string // 08:263 + AppCommit string // 08:264 + MachineTopology string // 08:265 + ContextGroundTruth string // 08:266 + CapabilitySnapshotHash string // 08:267 — interface placeholder + TaskContractHash string // 08:268 — interface placeholder // 08:269 — populated from driver.LastRegistryHash() by the eval // runner. The publisher lives in multi-agent/internal/driver/ // registryhash.go (WT-2-driver-promotion-chain B6 spec §3.3); @@ -47,6 +47,8 @@ type Schema struct { ArtifactHashes []string // 08:277 each MUST match ^[a-f0-9]{64}$ ObserverTracePath string // 08:278 ModelTraceID string // 08:279 + ModelInputTokens int // Codex CLI input token total for this run + ModelOutputTokens int // Codex CLI output token total for this run } // Sentinel errors returned (wrapped via fmt.Errorf("...: %w", sentinel)) diff --git a/multi-agent/internal/evalrun/schema_test.go b/multi-agent/internal/evalrun/schema_test.go index 221463eb..8021b8c8 100644 --- a/multi-agent/internal/evalrun/schema_test.go +++ b/multi-agent/internal/evalrun/schema_test.go @@ -55,6 +55,8 @@ func sampleSchema() Schema { SuccessOracleResult: "pass", FailureCategory: "", HumanInterventionCount: 0, + ModelInputTokens: 0, + ModelOutputTokens: 0, ArtifactHashes: []string{ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", diff --git a/multi-agent/internal/evalrun/writer.go b/multi-agent/internal/evalrun/writer.go index c57cdd97..23804231 100644 --- a/multi-agent/internal/evalrun/writer.go +++ b/multi-agent/internal/evalrun/writer.go @@ -49,7 +49,7 @@ type columnDesc struct { pk int // 0 or N for the Nth PK column } -// expectedColumns is the source of truth: 24 ordered descriptors that +// expectedColumns is the source of truth: 26 ordered descriptors that // match the §3 DDL and the canonical column order. Any future column // change must update this list AND the schema.sql DDL AND the Insert // statement column list below in lock-step. @@ -78,6 +78,8 @@ var expectedColumns = []columnDesc{ {cid: 21, name: "artifact_hashes", sqlType: "TEXT", notNull: true, dflt: sql.NullString{String: "'[]'", Valid: true}, pk: 0}, {cid: 22, name: "observer_trace_path", sqlType: "TEXT", notNull: true, dflt: sql.NullString{String: "''", Valid: true}, pk: 0}, {cid: 23, name: "model_trace_id", sqlType: "TEXT", notNull: true, dflt: sql.NullString{String: "''", Valid: true}, pk: 0}, + {cid: 24, name: "model_input_tokens", sqlType: "INTEGER", notNull: true, dflt: sql.NullString{String: "0", Valid: true}, pk: 0}, + {cid: 25, name: "model_output_tokens", sqlType: "INTEGER", notNull: true, dflt: sql.NullString{String: "0", Valid: true}, pk: 0}, } // insertSQL is a const so Go's vet sql-style checks see it as a literal @@ -92,10 +94,11 @@ const insertSQL = `INSERT INTO runs ( task_contract_hash, dynamic_mcp_registry_hash, selected_context, ground_truth_context, start_time, end_time, success_oracle_result, failure_category, human_intervention_count, artifact_hashes, - observer_trace_path, model_trace_id -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + observer_trace_path, model_trace_id, model_input_tokens, + model_output_tokens +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` -// ColumnNames returns the canonical ordered list of 24 column names +// ColumnNames returns the canonical ordered list of 26 column names // (sourced from the same expectedColumns slice the drift guard uses). // Exported so out-of-package callers (notably cmd/evalrun-export) can // pin their CSV header / SELECT projection to the same SOT, ruling @@ -111,7 +114,7 @@ func ColumnNames() []string { // NewSQLWriter wraps *sql.DB into a Writer that targets the runs table. // Returns ErrSchemaDrift (wrapped) if the runs table is absent or its -// column descriptors do not match the expected 24-column layout. +// column descriptors do not match the expected 26-column layout. func NewSQLWriter(db *sql.DB) (Writer, error) { if err := CheckSchemaDrift(db); err != nil { return nil, err @@ -176,6 +179,8 @@ func (w *SQLWriter) Insert(ctx context.Context, s Schema) error { hashesJSON, s.ObserverTracePath, s.ModelTraceID, + s.ModelInputTokens, + s.ModelOutputTokens, ); err != nil { return fmt.Errorf("evalrun: insert run_id=%s: %w", s.RunID, err) } diff --git a/multi-agent/internal/evalrun/writer_test.go b/multi-agent/internal/evalrun/writer_test.go index b14e1001..6a59849f 100644 --- a/multi-agent/internal/evalrun/writer_test.go +++ b/multi-agent/internal/evalrun/writer_test.go @@ -101,6 +101,8 @@ func TestSchema_AllFieldsRoundtrip(t *testing.T) { s.TaskContractHash = "tch-" + strings.Repeat("0", 60) s.DynamicMCPRegistryHash = "dmr-" + strings.Repeat("0", 60) s.HumanInterventionCount = 3 + s.ModelInputTokens = 12345 + s.ModelOutputTokens = 678 s.FailureCategory = "" if err := w.Insert(context.Background(), s); err != nil { t.Fatalf("Insert: %v", err) @@ -114,14 +116,14 @@ func TestSchema_AllFieldsRoundtrip(t *testing.T) { capHash, contractHash, dynRegHash, selCtx, gtCtx, startStr, endStr, oracle, failCat, artifactJSON, obsTrace, modelTrace string - humanCount int + humanCount, inputTokens, outputTokens int ) if err := row.Scan( &runID, &workloadID, &claimID, &experimentID, &baseline, &loomCommit, &agentCommit, &modelCommit, &appCommit, &machineTopo, &ctxGT, &capHash, &contractHash, &dynRegHash, &selCtx, >Ctx, &startStr, &endStr, &oracle, &failCat, &humanCount, &artifactJSON, - &obsTrace, &modelTrace, + &obsTrace, &modelTrace, &inputTokens, &outputTokens, ); err != nil { t.Fatalf("scan: %v", err) } @@ -154,6 +156,12 @@ func TestSchema_AllFieldsRoundtrip(t *testing.T) { if humanCount != s.HumanInterventionCount { t.Errorf("human_intervention_count: got %d, want %d", humanCount, s.HumanInterventionCount) } + if inputTokens != s.ModelInputTokens { + t.Errorf("model_input_tokens: got %d, want %d", inputTokens, s.ModelInputTokens) + } + if outputTokens != s.ModelOutputTokens { + t.Errorf("model_output_tokens: got %d, want %d", outputTokens, s.ModelOutputTokens) + } // Times: stored as fixed-9-digit-nanosecond UTC for lex-sort // stability (see writer.go formatRFC3339NanoUTC). wantStart := s.StartTime.UTC().Format("2006-01-02T15:04:05.000000000Z07:00") @@ -193,13 +201,13 @@ func TestInsert_Parameterized_SQLInjection(t *testing.T) { if err := w.Insert(context.Background(), s); err != nil { t.Fatalf("Insert: %v", err) } - // Table still exists (24 columns) and row count increases. + // Table still exists (26 columns) and row count increases. var colCount int if err := db.QueryRow("SELECT COUNT(*) FROM pragma_table_info('runs')").Scan(&colCount); err != nil { t.Fatal(err) } - if colCount != 24 { - t.Fatalf("runs table mutated: column count %d (want 24)", colCount) + if colCount != 26 { + t.Fatalf("runs table mutated: column count %d (want 26)", colCount) } var rowCount int if err := db.QueryRow("SELECT COUNT(*) FROM runs").Scan(&rowCount); err != nil { @@ -401,7 +409,9 @@ func TestNewSQLWriter_DetectsSchemaDriftMissingCheck(t *testing.T) { human_intervention_count INTEGER NOT NULL DEFAULT 0, artifact_hashes TEXT NOT NULL DEFAULT '[]', observer_trace_path TEXT NOT NULL DEFAULT '', - model_trace_id TEXT NOT NULL DEFAULT '' + model_trace_id TEXT NOT NULL DEFAULT '', + model_input_tokens INTEGER NOT NULL DEFAULT 0, + model_output_tokens INTEGER NOT NULL DEFAULT 0 )`); err != nil { t.Fatal(err) } @@ -506,6 +516,8 @@ func driftDDL(overrides map[string]string) string { {"artifact_hashes", "TEXT NOT NULL DEFAULT '[]'"}, {"observer_trace_path", "TEXT NOT NULL DEFAULT ''"}, {"model_trace_id", "TEXT NOT NULL DEFAULT ''"}, + {"model_input_tokens", "INTEGER NOT NULL DEFAULT 0"}, + {"model_output_tokens", "INTEGER NOT NULL DEFAULT 0"}, } parts := make([]string, 0, len(cols)) for _, c := range cols { @@ -553,6 +565,8 @@ func driftDDLSwapOrder(a, b string) string { "artifact_hashes TEXT NOT NULL DEFAULT '[]'", "observer_trace_path TEXT NOT NULL DEFAULT ''", "model_trace_id TEXT NOT NULL DEFAULT ''", + "model_input_tokens INTEGER NOT NULL DEFAULT 0", + "model_output_tokens INTEGER NOT NULL DEFAULT 0", } ai, bi := -1, -1 for i, c := range cols { diff --git a/multi-agent/internal/observerstore/schema.sql b/multi-agent/internal/observerstore/schema.sql index d1ed3ea7..76a08ee2 100644 --- a/multi-agent/internal/observerstore/schema.sql +++ b/multi-agent/internal/observerstore/schema.sql @@ -189,7 +189,7 @@ CREATE TABLE IF NOT EXISTS resource_snapshots ( CREATE INDEX IF NOT EXISTS idx_resource_snapshots_latest ON resource_snapshots(workspace_id, created_at); --- WT-1-run-schema: per-run D1 evaluation rows (24 columns matching +-- WT-1-run-schema: per-run D1 evaluation rows (26 columns matching -- /root/paper_writing/docs/intermediate/08_evaluation_plan_v3.md lines 256-279). CREATE TABLE IF NOT EXISTS runs ( run_id TEXT PRIMARY KEY, @@ -215,7 +215,9 @@ CREATE TABLE IF NOT EXISTS runs ( human_intervention_count INTEGER NOT NULL DEFAULT 0, artifact_hashes TEXT NOT NULL DEFAULT '[]', observer_trace_path TEXT NOT NULL DEFAULT '', - model_trace_id TEXT NOT NULL DEFAULT '' + model_trace_id TEXT NOT NULL DEFAULT '', + model_input_tokens INTEGER NOT NULL DEFAULT 0, + model_output_tokens INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_runs_experiment ON runs(experiment_id, workload_id); diff --git a/multi-agent/tools/eval/metrics/tests/test_metrics_user_promoted.py b/multi-agent/tools/eval/metrics/tests/test_metrics_user_promoted.py index 2ef03d4a..a67f820e 100644 --- a/multi-agent/tools/eval/metrics/tests/test_metrics_user_promoted.py +++ b/multi-agent/tools/eval/metrics/tests/test_metrics_user_promoted.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sqlite3 from pathlib import Path from tests._extract_helpers import extract_dict @@ -43,3 +44,43 @@ def test_all_13_user_promoted_null(fixture_3_db: Path) -> None: else: assert val is None, name assert d["notes"].get(name) == "upstream data missing", name + + +def test_token_usage_sums_landed_codex_columns(empty_db: Path) -> None: + conn = sqlite3.connect(str(empty_db)) + conn.executemany( + "INSERT INTO runs (" + " run_id, workload_id, claim_id, experiment_id, baseline_or_ablation," + " loom_commit, agentserver_commit, modelserver_commit, app_commit," + " machine_topology, context_ground_truth, selected_context," + " ground_truth_context, start_time, end_time, success_oracle_result," + " model_input_tokens, model_output_tokens" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [ + ( + "run-token-01", "wl-token", "C41", "E-token", "full_loom", + "loom", "agentserver", "modelserver", "app", + "linux/amd64", "", "driver", "driver", + "2026-07-08T00:00:00Z", "2026-07-08T00:00:10Z", "pass", + 100, 40, + ), + ( + "run-token-02", "wl-token", "C41", "E-token", "full_loom", + "loom", "agentserver", "modelserver", "app", + "linux/amd64", "", "driver", "driver", + "2026-07-08T00:01:00Z", "2026-07-08T00:01:10Z", "pass", + 7, 5, + ), + ], + ) + conn.commit() + conn.close() + + d = extract_dict(empty_db, "user-promoted") + + assert d["metrics"]["TokenUsage"] == { + "input_tokens": 107, + "output_tokens": 45, + "count": 2, + } + assert "TokenUsage" not in d["notes"] diff --git a/multi-agent/tools/eval/runner/codex_usage.go b/multi-agent/tools/eval/runner/codex_usage.go new file mode 100644 index 00000000..4b7878f1 --- /dev/null +++ b/multi-agent/tools/eval/runner/codex_usage.go @@ -0,0 +1,123 @@ +package main + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "os" + "strings" +) + +var ( + ErrCodexUsageJSONLUnreadable = errors.New("eval-runner: --codex-usage-jsonl could not be read") + ErrCodexUsageJSONLInvalid = errors.New("eval-runner: --codex-usage-jsonl invalid") +) + +type CodexTokenUsage struct { + InputTokens int + OutputTokens int +} + +func ParseCodexUsageJSONL(path string) (CodexTokenUsage, error) { + if path == "" { + return CodexTokenUsage{}, nil + } + f, err := os.Open(path) + if err != nil { + return CodexTokenUsage{}, fmt.Errorf("%w: %s: %v", ErrCodexUsageJSONLUnreadable, path, err) + } + defer f.Close() + + var total CodexTokenUsage + sc := bufio.NewScanner(f) + lineNo := 0 + for sc.Scan() { + lineNo++ + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + var obj map[string]any + if err := json.Unmarshal([]byte(line), &obj); err != nil { + return CodexTokenUsage{}, fmt.Errorf("%w: line %d: %v", ErrCodexUsageJSONLInvalid, lineNo, err) + } + u, ok, err := codexUsageFromObject(obj) + if err != nil { + return CodexTokenUsage{}, fmt.Errorf("%w: line %d: %v", ErrCodexUsageJSONLInvalid, lineNo, err) + } + if !ok { + continue + } + total.InputTokens += u.InputTokens + total.OutputTokens += u.OutputTokens + } + if err := sc.Err(); err != nil { + return CodexTokenUsage{}, fmt.Errorf("%w: %s: %v", ErrCodexUsageJSONLUnreadable, path, err) + } + return total, nil +} + +func codexUsageFromObject(obj map[string]any) (CodexTokenUsage, bool, error) { + if usage, ok := mapValue(obj["usage"]); ok { + u, err := codexUsageFromMap(usage) + return u, true, err + } + for _, key := range []string{"event", "response", "message"} { + wrapper, ok := mapValue(obj[key]) + if !ok { + continue + } + if usage, ok := mapValue(wrapper["usage"]); ok { + u, err := codexUsageFromMap(usage) + return u, true, err + } + } + return CodexTokenUsage{}, false, nil +} + +func codexUsageFromMap(m map[string]any) (CodexTokenUsage, error) { + input, err := firstTokenValue(m, "input_tokens", "prompt_tokens") + if err != nil { + return CodexTokenUsage{}, err + } + output, err := firstTokenValue(m, "output_tokens", "completion_tokens") + if err != nil { + return CodexTokenUsage{}, err + } + return CodexTokenUsage{InputTokens: input, OutputTokens: output}, nil +} + +func firstTokenValue(m map[string]any, keys ...string) (int, error) { + for _, key := range keys { + raw, ok := m[key] + if !ok { + continue + } + n, err := tokenInt(raw) + if err != nil { + return 0, fmt.Errorf("%s: %w", key, err) + } + return n, nil + } + return 0, nil +} + +func tokenInt(v any) (int, error) { + f, ok := v.(float64) + if !ok { + return 0, fmt.Errorf("token count is %T, want number", v) + } + if f < 0 { + return 0, errors.New("token count must be non-negative") + } + if f != float64(int(f)) { + return 0, errors.New("token count must be an integer") + } + return int(f), nil +} + +func mapValue(v any) (map[string]any, bool) { + m, ok := v.(map[string]any) + return m, ok +} diff --git a/multi-agent/tools/eval/runner/codex_usage_test.go b/multi-agent/tools/eval/runner/codex_usage_test.go new file mode 100644 index 00000000..4a34f15f --- /dev/null +++ b/multi-agent/tools/eval/runner/codex_usage_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func writeCodexUsageJSONL(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "codex-usage.jsonl") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatalf("write usage jsonl: %v", err) + } + return path +} + +func TestParseCodexUsageJSONL_SumsUsageRecords(t *testing.T) { + t.Parallel() + path := writeCodexUsageJSONL(t, `{"type":"turn.completed","usage":{"input_tokens":10,"output_tokens":4}} +{"type":"debug","message":"ignored"} +{"type":"turn.completed","message":{"usage":{"prompt_tokens":2,"completion_tokens":1}}} +{"type":"response.completed","response":{"usage":{"input_tokens":3,"completion_tokens":5}}} +`) + + got, err := ParseCodexUsageJSONL(path) + if err != nil { + t.Fatalf("ParseCodexUsageJSONL: %v", err) + } + if got.InputTokens != 15 { + t.Fatalf("input tokens = %d, want 15", got.InputTokens) + } + if got.OutputTokens != 10 { + t.Fatalf("output tokens = %d, want 10", got.OutputTokens) + } +} + +func TestParseCodexUsageJSONL_EmptyPathReturnsZero(t *testing.T) { + t.Parallel() + got, err := ParseCodexUsageJSONL("") + if err != nil { + t.Fatalf("empty path should be a no-op, got %v", err) + } + if got != (CodexTokenUsage{}) { + t.Fatalf("empty path usage = %+v, want zero value", got) + } +} + +func TestParseCodexUsageJSONL_RejectsMalformedJSON(t *testing.T) { + t.Parallel() + path := writeCodexUsageJSONL(t, "{\"usage\":{\"input_tokens\":1}\n") + + _, err := ParseCodexUsageJSONL(path) + if !errors.Is(err, ErrCodexUsageJSONLInvalid) { + t.Fatalf("err = %v, want ErrCodexUsageJSONLInvalid", err) + } +} + +func TestParseCodexUsageJSONL_RejectsNegativeTokens(t *testing.T) { + t.Parallel() + path := writeCodexUsageJSONL(t, `{"usage":{"input_tokens":-1,"output_tokens":0}}`+"\n") + + _, err := ParseCodexUsageJSONL(path) + if !errors.Is(err, ErrCodexUsageJSONLInvalid) { + t.Fatalf("err = %v, want ErrCodexUsageJSONLInvalid", err) + } +} + +func TestParseCodexUsageJSONL_UnreadablePath(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "missing.jsonl") + + _, err := ParseCodexUsageJSONL(path) + if !errors.Is(err, ErrCodexUsageJSONLUnreadable) { + t.Fatalf("err = %v, want ErrCodexUsageJSONLUnreadable", err) + } +} diff --git a/multi-agent/tools/eval/runner/flags_test.go b/multi-agent/tools/eval/runner/flags_test.go index e05eb3b5..a51a62f9 100644 --- a/multi-agent/tools/eval/runner/flags_test.go +++ b/multi-agent/tools/eval/runner/flags_test.go @@ -764,12 +764,12 @@ func TestOpts_HasAblationFields(t *testing.T) { } } -// TestCSVColumns_IncludesBaselineOrAblation asserts the new column -// is at the tail of the frozen order. +// TestCSVColumns_IncludesBaselineOrAblation asserts the column keeps its +// original append-only position even when later columns are appended. func TestCSVColumns_IncludesBaselineOrAblation(t *testing.T) { cols := CSVColumns() - if cols[len(cols)-1] != "baseline_or_ablation" { - t.Fatalf("baseline_or_ablation not at CSV tail: %v", cols) + if cols[31] != "baseline_or_ablation" { + t.Fatalf("baseline_or_ablation position drifted: %v", cols) } } @@ -862,12 +862,9 @@ func TestRun_LabelDerivedFromApplied_NoAblation(t *testing.T) { t.Errorf("baseline_or_ablation = %q, want %q", res.Row.BaselineOrAblation, DefaultBaselineName) } rows := readCSV(t, opts.OutCSV) - last := len(rows[0]) - 1 - if rows[0][last] != "baseline_or_ablation" { - t.Fatalf("CSV last column = %q", rows[0][last]) - } - if rows[1][last] != DefaultBaselineName { - t.Errorf("CSV data last cell = %q, want %q", rows[1][last], DefaultBaselineName) + gotLabel := csvCellByName(t, rows, "baseline_or_ablation") + if gotLabel != DefaultBaselineName { + t.Errorf("CSV baseline_or_ablation = %q, want %q", gotLabel, DefaultBaselineName) } } @@ -1148,17 +1145,28 @@ func TestMain_ScrubsParentEnvBeforeParse(t *testing.T) { if _, err := os.Stat(outCSV); err == nil { rows := readCSV(t, outCSV) if len(rows) >= 2 { - last := len(rows[0]) - 1 - if rows[0][last] != "baseline_or_ablation" { - t.Fatalf("CSV last column = %q", rows[0][last]) - } - if rows[1][last] != DefaultBaselineName { - t.Errorf("CSV label = %q, want %q", rows[1][last], DefaultBaselineName) + gotLabel := csvCellByName(t, rows, "baseline_or_ablation") + if gotLabel != DefaultBaselineName { + t.Errorf("CSV label = %q, want %q", gotLabel, DefaultBaselineName) } } } } +func csvCellByName(t *testing.T, rows [][]string, name string) string { + t.Helper() + if len(rows) < 2 { + t.Fatalf("CSV needs header and data row, got %d rows", len(rows)) + } + for i, h := range rows[0] { + if h == name { + return rows[1][i] + } + } + t.Fatalf("CSV column %q missing in %v", name, rows[0]) + return "" +} + func TestMain_BaselineNameCollidesWithFlag_Exit2(t *testing.T) { if testing.Short() { t.Skip("skips end-to-end binary build") diff --git a/multi-agent/tools/eval/runner/main.go b/multi-agent/tools/eval/runner/main.go index e8e02474..cac08aee 100644 --- a/multi-agent/tools/eval/runner/main.go +++ b/multi-agent/tools/eval/runner/main.go @@ -52,13 +52,14 @@ func main() { func runMain(args []string) int { fs := flag.NewFlagSet("run", flag.ContinueOnError) var ( - workload = fs.String("workload", "", "workload id (e.g. cross-device-code-mod)") - workloadDir = fs.String("workload-dir", "multi-agent/tests/eval/workloads", "directory containing /spec.yaml") - stubListen = fs.String("stub-listen", "127.0.0.1:18080", "agentserver-stub --listen address; MUST be loopback") - observerDB = fs.String("observer-db", "", "SQLite DB for run schema; empty = NoopWriter") + workload = fs.String("workload", "", "workload id (e.g. cross-device-code-mod)") + workloadDir = fs.String("workload-dir", "multi-agent/tests/eval/workloads", "directory containing /spec.yaml") + stubListen = fs.String("stub-listen", "127.0.0.1:18080", "agentserver-stub --listen address; MUST be loopback") + observerDB = fs.String("observer-db", "", "SQLite DB for run schema; empty = NoopWriter") codexConfig = fs.String("codex-config", "", "path to codex config.toml (passed through; recorded only)") codexConfigPath = fs.String("codex-config-path", "", "WT-2: filesystem path to a codex config.toml; validated against --codex-config-mode; must resolve under the repo or /tmp") codexConfigMode = fs.String("codex-config-mode", "", "WT-2: \"a\" = local-proxy (experimental_bearer_token) or \"b\" = upstream-direct (env_key=OPENAI_API_KEY); enforces the auth-field / env-var preconditions") + codexUsageJSONL = fs.String("codex-usage-jsonl", "", "path to Codex CLI JSONL usage events; records model_input_tokens/model_output_tokens") runID = fs.String("run-id", "", "explicit run id; default = derived") timeout = fs.Duration("timeout", 0, "override spec.timeout_seconds") outCSV = fs.String("out", "", "output CSV path; required") @@ -102,6 +103,7 @@ func runMain(args []string) int { ObserverDB: *observerDB, CodexConfigPath: recordedCodexPath, CodexConfigMode: *codexConfigMode, + CodexUsageJSONL: *codexUsageJSONL, RunID: *runID, Timeout: *timeout, OutCSV: *outCSV, diff --git a/multi-agent/tools/eval/runner/runner.go b/multi-agent/tools/eval/runner/runner.go index 36e16144..42360522 100644 --- a/multi-agent/tools/eval/runner/runner.go +++ b/multi-agent/tools/eval/runner/runner.go @@ -44,6 +44,7 @@ type Opts struct { StubBin string // path to agentserver-stub binary; empty = auto-build ObserverDB string CodexConfigPath string + CodexUsageJSONL string // CodexConfigMode is the WT-2 dual-path selector ("a" | "b" | ""). // When non-empty (and/or CodexConfigPath is non-empty), the runner // invokes validateCodexConfig at pre-flight; see spec §7.a-.d. @@ -368,6 +369,11 @@ func Run(ctx context.Context, opts Opts) Result { // the CSV write itself is microseconds (PR #53 round 4 P2). finishedAt := time.Now() + codexUsage, err := ParseCodexUsageJSONL(opts.CodexUsageJSONL) + if err != nil { + return preflight(opts, err) + } + // Assemble row. row := RunRow{ RunID: runID, @@ -393,6 +399,8 @@ func Run(ctx context.Context, opts Opts) Result { StubListen: opts.StubListen, TempdirKept: opts.KeepTempdir, BaselineOrAblation: derivedLabel, + ModelInputTokens: codexUsage.InputTokens, + ModelOutputTokens: codexUsage.OutputTokens, } // WT-2-e1e6-probes Edit 6: drain the emitter and merge probe diff --git a/multi-agent/tools/eval/runner/runner_test.go b/multi-agent/tools/eval/runner/runner_test.go index f1e5ccdb..88fa2e60 100644 --- a/multi-agent/tools/eval/runner/runner_test.go +++ b/multi-agent/tools/eval/runner/runner_test.go @@ -168,6 +168,51 @@ func TestRun_CrossDeviceCodeMod_HappyPath_CSVOneLine(t *testing.T) { } } +func TestRun_RecordsCodexUsageJSONL(t *testing.T) { + root := findRepoModuleRoot(t) + withShims(t, commitMetaJSON(), "alice@example.com|alice@example.com") + + usagePath := writeCodexUsageJSONL(t, `{"type":"turn.completed","usage":{"input_tokens":100,"output_tokens":30}} +{"type":"turn.completed","response":{"usage":{"prompt_tokens":7,"completion_tokens":5}}} +`) + outCSV := filepath.Join(t.TempDir(), "run.csv") + res := Run(context.Background(), Opts{ + WorkloadID: "cross-device-code-mod", + WorkloadDir: filepath.Join(root, "tests/eval/workloads"), + StubListen: pickFreePort(t), + StubBin: stubBinaryPath(t), + OutCSV: outCSV, + CodexUsageJSONL: usagePath, + }) + if res.ExitCode != 0 { + t.Fatalf("exit = %d (err=%v); row=%+v", res.ExitCode, res.Err, res.Row) + } + if res.Row.ModelInputTokens != 107 { + t.Fatalf("row input tokens = %d, want 107", res.Row.ModelInputTokens) + } + if res.Row.ModelOutputTokens != 35 { + t.Fatalf("row output tokens = %d, want 35", res.Row.ModelOutputTokens) + } + + rows := readCSV(t, outCSV) + header, data := rows[0], rows[1] + col := func(name string) string { + for i, h := range header { + if h == name { + return data[i] + } + } + t.Fatalf("missing CSV column %s", name) + return "" + } + if col("model_input_tokens") != "107" { + t.Fatalf("csv input tokens = %q", col("model_input_tokens")) + } + if col("model_output_tokens") != "35" { + t.Fatalf("csv output tokens = %q", col("model_output_tokens")) + } +} + // TestCommitMetaRedacted_Email — Security §7(c). Inject a commit_meta JSON // and a git-email shim with named addresses; CSV columns must contain only // the 8-hex SHAs, never the plaintext "@". diff --git a/multi-agent/tools/eval/runner/writer.go b/multi-agent/tools/eval/runner/writer.go index a5360a80..61f2f220 100644 --- a/multi-agent/tools/eval/runner/writer.go +++ b/multi-agent/tools/eval/runner/writer.go @@ -64,6 +64,11 @@ type RunRow struct { ProbeManualSetupStepCount *int ProbeConfigTouchCount *int ProbeNotesJSON string // "{}" when empty; MergeIntoRow always sets this + + // Codex CLI token usage. Zero means either no usage JSONL was + // supplied or Codex reported zero tokens for that side. + ModelInputTokens int + ModelOutputTokens int } // RunWriter is the seam between this worktree and WT-1-run-schema. Skeleton @@ -118,6 +123,9 @@ func CSVColumns() []string { "probe_notes_json", // WT-2-flag-integration §2.4 (append-only): "baseline_or_ablation", + // Codex CLI token usage (append-only): + "model_input_tokens", + "model_output_tokens", } } @@ -157,6 +165,8 @@ func rowAsCSVRecord(r RunRow) []string { nilOrInt(r.ProbeConfigTouchCount), probeNotesOrDefault(r.ProbeNotesJSON), r.BaselineOrAblation, + strconv.Itoa(r.ModelInputTokens), + strconv.Itoa(r.ModelOutputTokens), } } diff --git a/multi-agent/tools/eval/runner/writer_test.go b/multi-agent/tools/eval/runner/writer_test.go index dc6ab112..384886c7 100644 --- a/multi-agent/tools/eval/runner/writer_test.go +++ b/multi-agent/tools/eval/runner/writer_test.go @@ -33,6 +33,8 @@ func TestCSVColumns_FrozenOrder(t *testing.T) { "probe_notes_json", // WT-2-flag-integration §2.4: "baseline_or_ablation", + // Codex CLI token usage (append-only): + "model_input_tokens", "model_output_tokens", }, ",") if got != want { t.Fatalf("CSV columns drifted:\n got %s\n want %s", got, want) @@ -118,8 +120,8 @@ func TestNoopWriter_InsertIsNoop(t *testing.T) { func TestCSVColumns_AppendOnly_WithProbes(t *testing.T) { t.Parallel() cols := CSVColumns() - if len(cols) != 32 { - t.Fatalf("column count = %d, want 32", len(cols)) + if len(cols) != 34 { + t.Fatalf("column count = %d, want 34", len(cols)) } wantProbes := []string{ "probe_task_success_rate", @@ -139,6 +141,46 @@ func TestCSVColumns_AppendOnly_WithProbes(t *testing.T) { } } +func TestRunRow_CodexTokenUsage_RoundTripCSV(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "run.csv") + row := RunRow{ + RunID: "r-token", + WorkloadID: "wl-token", + ModelInputTokens: 1234, + ModelOutputTokens: 567, + } + if err := WriteCSVRow(path, row); err != nil { + t.Fatal(err) + } + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + r := csv.NewReader(f) + rows, err := r.ReadAll() + if err != nil { + t.Fatal(err) + } + header, data := rows[0], rows[1] + col := func(name string) string { + for i, h := range header { + if h == name { + return data[i] + } + } + t.Fatalf("column %s missing", name) + return "" + } + if col("model_input_tokens") != "1234" { + t.Fatalf("model_input_tokens = %q", col("model_input_tokens")) + } + if col("model_output_tokens") != "567" { + t.Fatalf("model_output_tokens = %q", col("model_output_tokens")) + } +} + func TestRunRow_ProbeFields_RoundTripCSV(t *testing.T) { t.Parallel() dir := t.TempDir() From cdfbef0e6c8039027719e6844820778b1fd0fddc Mon Sep 17 00:00:00 2001 From: yuzishu Date: Wed, 8 Jul 2026 14:54:39 +0800 Subject: [PATCH 2/5] fix(eval): export codex token columns --- multi-agent/cmd/evalrun-export/main.go | 90 ++++++++++++--------- multi-agent/cmd/evalrun-export/main_test.go | 21 ++++- 2 files changed, 71 insertions(+), 40 deletions(-) diff --git a/multi-agent/cmd/evalrun-export/main.go b/multi-agent/cmd/evalrun-export/main.go index 64b6aff5..7bdb15d7 100644 --- a/multi-agent/cmd/evalrun-export/main.go +++ b/multi-agent/cmd/evalrun-export/main.go @@ -19,7 +19,9 @@ import ( _ "modernc.org/sqlite" ) -// columnNames is the 24-column header in the order defined by +const runsColumnCount = 26 + +// columnNames is the 26-column header in the order defined by // docs/specs/wt1-run-schema.spec.md §3. The CLI is intentionally // dependency-free with respect to internal/evalrun beyond this list — // it does NOT call into the Writer; it just reads the runs table. @@ -48,6 +50,8 @@ var columnNames = []string{ "artifact_hashes", "observer_trace_path", "model_trace_id", + "model_input_tokens", + "model_output_tokens", } // usageError is exit code 2 — bad arguments. runtimeError is exit 1. @@ -183,7 +187,7 @@ const ( selectRunsFilterSQL = `SELECT ` + columnSelectList + ` FROM runs WHERE experiment_id = ? ORDER BY run_id` ) -// columnSelectList is the 24-column projection in the canonical order. +// columnSelectList is the 26-column projection in the canonical order. // Constant so the SELECT statements above are pure literals; refactors // that touch column order must update this AND the Insert path AND // schema.sql in lock-step. @@ -193,7 +197,8 @@ machine_topology, context_ground_truth, capability_snapshot_hash, task_contract_hash, dynamic_mcp_registry_hash, selected_context, ground_truth_context, start_time, end_time, success_oracle_result, failure_category, human_intervention_count, artifact_hashes, -observer_trace_path, model_trace_id` +observer_trace_path, model_trace_id, model_input_tokens, +model_output_tokens` // queryRuns runs the appropriate SELECT and returns *sql.Rows. func queryRuns(ctx context.Context, db *sql.DB, filterExperiment string) (*sql.Rows, error) { @@ -203,37 +208,40 @@ func queryRuns(ctx context.Context, db *sql.DB, filterExperiment string) (*sql.R return db.QueryContext(ctx, selectRunsFilterSQL, filterExperiment) } -// humanCountIdx is the canonical position of human_intervention_count -// in the 24-column ordering. The CSV path treats this column like the -// others (string round-trip via strconv); the JSONL path emits it as -// a JSON number. -const humanCountIdx = 20 +// Canonical positions of INTEGER columns. The CSV path treats these +// columns like the others (string round-trip via strconv); the JSONL +// path emits them as JSON numbers. +const ( + humanCountIdx = 20 + modelInputTokensIdx = 24 + modelOutputTokensIdx = 25 +) -// runRow holds one scanned row as 24 string cells. human_intervention_count -// is the only non-string column; it's converted to its decimal string -// form in scanOneRow so CSV/JSONL writers can decide whether to quote -// or emit as a number. +// runRow holds one scanned row as string cells. INTEGER columns are +// converted to decimal string form in scanOneRow so CSV/JSONL writers +// can decide whether to quote or emit as numbers. type runRow struct { - values [24]string + values [runsColumnCount]string } func scanOneRow(rows *sql.Rows) (*runRow, error) { var ( r runRow - // int64 (not int) so a row written with - // human_intervention_count > 2^31 on a 32-bit build doesn't - // silently truncate to a negative number. SQLite INTEGER is - // up to 8 bytes signed; the column NOT NULL DEFAULT 0 means - // Scan never sees NULL — drift guard catches the NULL-able - // drift variant separately. - humanCount int64 - // 24 dest pointers in the canonical order. - dest [24]any + // int64 (not int) so a row written with an SQLite INTEGER + // > 2^31 on a 32-bit build doesn't silently truncate. + humanCount, modelInputTokens, modelOutputTokens int64 + // dest pointers in the canonical order. + dest [runsColumnCount]any ) - for i := 0; i < 24; i++ { - if i == humanCountIdx { + for i := 0; i < runsColumnCount; i++ { + switch i { + case humanCountIdx: dest[i] = &humanCount - } else { + case modelInputTokensIdx: + dest[i] = &modelInputTokens + case modelOutputTokensIdx: + dest[i] = &modelOutputTokens + default: dest[i] = &r.values[i] } } @@ -241,6 +249,8 @@ func scanOneRow(rows *sql.Rows) (*runRow, error) { return nil, err } r.values[humanCountIdx] = strconv.FormatInt(humanCount, 10) + r.values[modelInputTokensIdx] = strconv.FormatInt(modelInputTokens, 10) + r.values[modelOutputTokensIdx] = strconv.FormatInt(modelOutputTokens, 10) return &r, nil } @@ -261,7 +271,7 @@ func exportCSV(ctx context.Context, db *sql.DB, w io.Writer, filterExperiment st if err != nil { return fmt.Errorf("scan row: %w", err) } - escaped := make([]string, 24) + escaped := make([]string, runsColumnCount) for i, v := range r.values { escaped[i] = csvEscape(v) } @@ -295,8 +305,8 @@ func csvEscape(s string) string { } // exportJSONL writes one JSON object per line. Key order matches -// columnNames; human_intervention_count is emitted as a number, all -// other fields as strings. +// columnNames; INTEGER columns are emitted as numbers, all other +// fields as strings. func exportJSONL(ctx context.Context, db *sql.DB, w io.Writer, filterExperiment string) error { rows, err := queryRuns(ctx, db, filterExperiment) if err != nil { @@ -324,9 +334,9 @@ func exportJSONL(ctx context.Context, db *sql.DB, w io.Writer, filterExperiment } // orderedJSONLine custom-marshals one row preserving columnNames order -// and emitting human_intervention_count as a JSON number. +// and emitting INTEGER columns as JSON numbers. type orderedJSONLine struct { - values [24]string + values [runsColumnCount]string } func (o orderedJSONLine) MarshalJSON() ([]byte, error) { @@ -344,13 +354,10 @@ func (o orderedJSONLine) MarshalJSON() ([]byte, error) { buf = append(buf, keyJSON...) buf = append(buf, ':') // value - if i == humanCountIdx { - // human_intervention_count: numeric. scanOneRow built this - // via strconv.FormatInt(_, 10) on an int64, so it is always - // a JSON-grammar-conformant decimal integer (sign + digits, - // no whitespace, no exponent). Direct append into the JSON - // stream is therefore safe — no escaping or re-encoding - // pass needed. + if isJSONNumberColumn(i) { + // Numeric columns are built via strconv.FormatInt(_, 10) + // on int64, so they are always JSON-grammar-conformant + // decimal integers. Direct append is safe. buf = append(buf, []byte(o.values[i])...) } else { vJSON, err := json.Marshal(o.values[i]) @@ -363,3 +370,12 @@ func (o orderedJSONLine) MarshalJSON() ([]byte, error) { buf = append(buf, '}') return buf, nil } + +func isJSONNumberColumn(i int) bool { + switch i { + case humanCountIdx, modelInputTokensIdx, modelOutputTokensIdx: + return true + default: + return false + } +} diff --git a/multi-agent/cmd/evalrun-export/main_test.go b/multi-agent/cmd/evalrun-export/main_test.go index 52b39f14..49357c4f 100644 --- a/multi-agent/cmd/evalrun-export/main_test.go +++ b/multi-agent/cmd/evalrun-export/main_test.go @@ -131,6 +131,8 @@ func insertSampleRow(t *testing.T, db *sql.DB, runID, experiment string, mutate "artifact_hashes": "[]", "observer_trace_path": "", "model_trace_id": "", + "model_input_tokens": 0, + "model_output_tokens": 0, } if mutate != nil { mutate(row) @@ -143,7 +145,8 @@ func insertSampleRow(t *testing.T, db *sql.DB, runID, experiment string, mutate "task_contract_hash", "dynamic_mcp_registry_hash", "selected_context", "ground_truth_context", "start_time", "end_time", "success_oracle_result", "failure_category", "human_intervention_count", "artifact_hashes", - "observer_trace_path", "model_trace_id", + "observer_trace_path", "model_trace_id", "model_input_tokens", + "model_output_tokens", } args := make([]any, len(cols)) for i, c := range cols { @@ -192,8 +195,8 @@ func TestExportCSV_EmptyDB_HeaderOnly(t *testing.T) { if err != nil { t.Fatalf("parse csv header: %v (stdout=%q)", err, stdout) } - if len(rec) != 24 { - t.Fatalf("header has %d fields, want 24: %v", len(rec), rec) + if len(rec) != len(columnNames) { + t.Fatalf("header has %d fields, want %d: %v", len(rec), len(columnNames), rec) } for i, want := range columnNames { if rec[i] != want { @@ -430,6 +433,8 @@ func TestExportJSONL_RoundtripValues(t *testing.T) { path, db := freshDB(t) insertSampleRow(t, db, "run-jsonl-A-12345", "E1", func(r map[string]any) { r["human_intervention_count"] = 2 + r["model_input_tokens"] = 123 + r["model_output_tokens"] = 45 r["artifact_hashes"] = `["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]` }) insertSampleRow(t, db, "run-jsonl-B-12345", "E2", nil) @@ -452,6 +457,16 @@ func TestExportJSONL_RoundtripValues(t *testing.T) { } else if _, ok := v.(float64); !ok { t.Fatalf("human_intervention_count must be number, got %T", v) } + if v, ok := raw["model_input_tokens"]; !ok { + t.Fatal("missing model_input_tokens") + } else if got, ok := v.(float64); !ok || got != 123 { + t.Fatalf("model_input_tokens must be number 123, got %T %v", v, v) + } + if v, ok := raw["model_output_tokens"]; !ok { + t.Fatal("missing model_output_tokens") + } else if got, ok := v.(float64); !ok || got != 45 { + t.Fatalf("model_output_tokens must be number 45, got %T %v", v, v) + } // artifact_hashes must be a string (the raw JSON-array storage form). if v, ok := raw["artifact_hashes"]; !ok { t.Fatal("missing artifact_hashes") From 61bc589e53e1d94affde3d42d41dbe3385f4c394 Mon Sep 17 00:00:00 2001 From: yuzishu Date: Wed, 8 Jul 2026 15:09:05 +0800 Subject: [PATCH 3/5] fix(observerstore): migrate runs token columns --- multi-agent/internal/observerstore/store.go | 6 +++ .../internal/observerstore/store_test.go | 44 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/multi-agent/internal/observerstore/store.go b/multi-agent/internal/observerstore/store.go index bf1b7c61..9fb51990 100644 --- a/multi-agent/internal/observerstore/store.go +++ b/multi-agent/internal/observerstore/store.go @@ -214,6 +214,12 @@ func ensureColumns(db *sql.DB) error { if _, err := db.Exec(`ALTER TABLE promotion_audit ADD COLUMN stage_note TEXT NOT NULL DEFAULT ''`); err != nil && !isDuplicateColumn(err) { return err } + if _, err := db.Exec(`ALTER TABLE runs ADD COLUMN model_input_tokens INTEGER NOT NULL DEFAULT 0`); err != nil && !isDuplicateColumn(err) { + return err + } + if _, err := db.Exec(`ALTER TABLE runs ADD COLUMN model_output_tokens INTEGER NOT NULL DEFAULT 0`); err != nil && !isDuplicateColumn(err) { + return err + } return nil } diff --git a/multi-agent/internal/observerstore/store_test.go b/multi-agent/internal/observerstore/store_test.go index 34ca5c2e..ecbe0a7e 100644 --- a/multi-agent/internal/observerstore/store_test.go +++ b/multi-agent/internal/observerstore/store_test.go @@ -3,6 +3,7 @@ package observerstore import ( "bytes" "context" + "database/sql" "encoding/json" "errors" "io" @@ -13,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/yourorg/multi-agent/internal/capability" "github.com/yourorg/multi-agent/internal/observer" + _ "modernc.org/sqlite" ) func testStore(t *testing.T) *SQLiteStore { @@ -72,6 +74,48 @@ func TestSchemaIncludesExternalIdentityColumns(t *testing.T) { require.Contains(t, agentColumns, "external_user_id") } +func TestOpenSQLiteMigratesRunsTokenColumns(t *testing.T) { + path := filepath.Join(t.TempDir(), "observer.db") + db, err := sql.Open("sqlite", path) + require.NoError(t, err) + _, err = db.Exec(`CREATE TABLE runs ( + run_id TEXT PRIMARY KEY, + workload_id TEXT NOT NULL, + claim_id TEXT NOT NULL, + experiment_id TEXT NOT NULL, + baseline_or_ablation TEXT NOT NULL, + loom_commit TEXT NOT NULL, + agentserver_commit TEXT NOT NULL, + modelserver_commit TEXT NOT NULL, + app_commit TEXT NOT NULL, + machine_topology TEXT NOT NULL, + context_ground_truth TEXT NOT NULL, + capability_snapshot_hash TEXT NOT NULL DEFAULT '', + task_contract_hash TEXT NOT NULL DEFAULT '', + dynamic_mcp_registry_hash TEXT NOT NULL DEFAULT '', + selected_context TEXT NOT NULL, + ground_truth_context TEXT NOT NULL, + start_time TEXT NOT NULL, + end_time TEXT NOT NULL, + success_oracle_result TEXT NOT NULL CHECK(success_oracle_result IN ('pass','fail','timeout')), + failure_category TEXT NOT NULL DEFAULT '', + human_intervention_count INTEGER NOT NULL DEFAULT 0, + artifact_hashes TEXT NOT NULL DEFAULT '[]', + observer_trace_path TEXT NOT NULL DEFAULT '', + model_trace_id TEXT NOT NULL DEFAULT '' + )`) + require.NoError(t, err) + require.NoError(t, db.Close()) + + s, err := OpenSQLite(path) + require.NoError(t, err) + defer s.Close() + + cols := tableColumns(t, s, "runs") + require.Contains(t, cols, "model_input_tokens") + require.Contains(t, cols, "model_output_tokens") +} + func TestUpsertAgentRecordsExternalIdentity(t *testing.T) { s, err := Open(filepath.Join(t.TempDir(), "observer.db")) require.NoError(t, err) From 8af4fd2b19622621fa99fc55ddb9e7f8e7ff57b9 Mon Sep 17 00:00:00 2001 From: yuzishu Date: Wed, 8 Jul 2026 15:18:48 +0800 Subject: [PATCH 4/5] fix(eval): tolerate codex usage parse gaps --- multi-agent/tools/eval/runner/codex_usage.go | 1 + .../tools/eval/runner/codex_usage_test.go | 16 ++++++++ multi-agent/tools/eval/runner/runner.go | 3 +- multi-agent/tools/eval/runner/runner_test.go | 37 +++++++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/multi-agent/tools/eval/runner/codex_usage.go b/multi-agent/tools/eval/runner/codex_usage.go index 4b7878f1..69e4e720 100644 --- a/multi-agent/tools/eval/runner/codex_usage.go +++ b/multi-agent/tools/eval/runner/codex_usage.go @@ -31,6 +31,7 @@ func ParseCodexUsageJSONL(path string) (CodexTokenUsage, error) { var total CodexTokenUsage sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64<<10), 8<<20) lineNo := 0 for sc.Scan() { lineNo++ diff --git a/multi-agent/tools/eval/runner/codex_usage_test.go b/multi-agent/tools/eval/runner/codex_usage_test.go index 4a34f15f..7c23653c 100644 --- a/multi-agent/tools/eval/runner/codex_usage_test.go +++ b/multi-agent/tools/eval/runner/codex_usage_test.go @@ -4,6 +4,7 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" ) @@ -47,6 +48,21 @@ func TestParseCodexUsageJSONL_EmptyPathReturnsZero(t *testing.T) { } } +func TestParseCodexUsageJSONL_AllowsLargeNonUsageRecords(t *testing.T) { + t.Parallel() + path := writeCodexUsageJSONL(t, `{"type":"item.completed","aggregated_output":"`+strings.Repeat("x", 128*1024)+`"} +{"type":"turn.completed","usage":{"input_tokens":11,"output_tokens":3}} +`) + + got, err := ParseCodexUsageJSONL(path) + if err != nil { + t.Fatalf("ParseCodexUsageJSONL: %v", err) + } + if got.InputTokens != 11 || got.OutputTokens != 3 { + t.Fatalf("usage = %+v, want input=11 output=3", got) + } +} + func TestParseCodexUsageJSONL_RejectsMalformedJSON(t *testing.T) { t.Parallel() path := writeCodexUsageJSONL(t, "{\"usage\":{\"input_tokens\":1}\n") diff --git a/multi-agent/tools/eval/runner/runner.go b/multi-agent/tools/eval/runner/runner.go index 42360522..e1c5dde7 100644 --- a/multi-agent/tools/eval/runner/runner.go +++ b/multi-agent/tools/eval/runner/runner.go @@ -371,7 +371,8 @@ func Run(ctx context.Context, opts Opts) Result { codexUsage, err := ParseCodexUsageJSONL(opts.CodexUsageJSONL) if err != nil { - return preflight(opts, err) + fmt.Fprintf(opts.Stderr, "eval-runner: warning: %v; recording zero Codex token usage\n", err) + codexUsage = CodexTokenUsage{} } // Assemble row. diff --git a/multi-agent/tools/eval/runner/runner_test.go b/multi-agent/tools/eval/runner/runner_test.go index 88fa2e60..4a44db32 100644 --- a/multi-agent/tools/eval/runner/runner_test.go +++ b/multi-agent/tools/eval/runner/runner_test.go @@ -213,6 +213,43 @@ func TestRun_RecordsCodexUsageJSONL(t *testing.T) { } } +func TestRun_CodexUsageJSONLErrorDoesNotDropCompletedRun(t *testing.T) { + root := findRepoModuleRoot(t) + withShims(t, commitMetaJSON(), "alice@example.com|alice@example.com") + + outCSV := filepath.Join(t.TempDir(), "run.csv") + stderr := discardStderr(t) + res := Run(context.Background(), Opts{ + WorkloadID: "cross-device-code-mod", + WorkloadDir: filepath.Join(root, "tests/eval/workloads"), + StubListen: pickFreePort(t), + StubBin: stubBinaryPath(t), + OutCSV: outCSV, + CodexUsageJSONL: filepath.Join(t.TempDir(), "missing-codex-usage.jsonl"), + Stderr: stderr, + }) + if res.ExitCode != 0 { + t.Fatalf("exit = %d (err=%v); row=%+v", res.ExitCode, res.Err, res.Row) + } + if res.Row.RunID == "" { + t.Fatal("completed run row was dropped") + } + if res.Row.ModelInputTokens != 0 || res.Row.ModelOutputTokens != 0 { + t.Fatalf("usage = input %d output %d, want zeros", res.Row.ModelInputTokens, res.Row.ModelOutputTokens) + } + rows := readCSV(t, outCSV) + if len(rows) != 2 { + t.Fatalf("CSV rows = %d, want 2", len(rows)) + } + b, err := os.ReadFile(stderr.Name()) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(b, []byte("--codex-usage-jsonl")) { + t.Fatalf("stderr missing codex usage warning: %s", b) + } +} + // TestCommitMetaRedacted_Email — Security §7(c). Inject a commit_meta JSON // and a git-email shim with named addresses; CSV columns must contain only // the 8-hex SHAs, never the plaintext "@". From 9903ef41626dd7a2e21355d5b1b410eac01964ea Mon Sep 17 00:00:00 2001 From: yuzishu Date: Wed, 8 Jul 2026 16:09:44 +0800 Subject: [PATCH 5/5] fix(metrics): keep perf fixture compatible with token columns --- .../tools/eval/metrics/tests/test_perf_gated.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/multi-agent/tools/eval/metrics/tests/test_perf_gated.py b/multi-agent/tools/eval/metrics/tests/test_perf_gated.py index f6b42460..8605f43e 100644 --- a/multi-agent/tools/eval/metrics/tests/test_perf_gated.py +++ b/multi-agent/tools/eval/metrics/tests/test_perf_gated.py @@ -48,7 +48,15 @@ def test_extract_10k_rows_completes_under_2s(tmp_path: Path) -> None: 0, "[]", "/t", "", )) conn.executemany( - "INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO runs (" + " run_id, workload_id, claim_id, experiment_id, baseline_or_ablation," + " loom_commit, agentserver_commit, modelserver_commit, app_commit," + " machine_topology, context_ground_truth, capability_snapshot_hash," + " task_contract_hash, dynamic_mcp_registry_hash, selected_context," + " ground_truth_context, start_time, end_time, success_oracle_result," + " failure_category, human_intervention_count, artifact_hashes," + " observer_trace_path, model_trace_id" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", rows, ) conn.commit()