From 3735903ffab35ad8547f3ba1ed527515e9d1553a Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Thu, 13 Aug 2026 23:40:57 +0200 Subject: [PATCH 01/23] docs(trajectories): specify and pin the ACP-session capture-event format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `trajectory/acp_trajectory.jsonl` is produced by the ACP-session capture path and read by the viewer, the judges, the skill evaluator, task verifiers and the review skill — but it had no schema, no version and no written specification. Every consumer hand-rolls its own parser. This writes the format down and makes it machine-checkable, with zero runtime behaviour change: - `src/benchflow/trajectories/schemas/acp-capture-event-v1.schema.json` — Draft 2020-12 schema for one line of the artifact, scoped deliberately to the ACP-session capture events (`_events_to_trajectory` plus the `ACPSession` legacy fallback). `kind` stays an open string because `_canonical_tool_kind` passes agent-supplied values through unchanged; `status` is a closed enum because the serialized value is always `ToolCallStatus(...).value`. `additionalProperties: false` on every event variant, so a field the emitter does not produce today fails the suite. - `docs/trace-interop.md` — the format as emitted, per-event required and optional fields, producers and consumers, the current state of ATIF and OpenTelemetry, and a table of the information losses observable in the existing ACP -> ATIF / ADP conversions. Claims are marked FACT or PROPOSAL; nothing marked PROPOSAL is agreed or implemented. - `tests/trajectories/test_acp_capture_event_schema.py` — conformance suite over two corpora: records generated by driving real `ACPSession` objects through the production capture path and writer, and the ACP event lists already used as exporter inputs by the ATIF and ADP tests. Ten cases pin what the schema must reject. The event-type vocabulary is read out of `_events_to_trajectory` by AST rather than hardcoded, so adding a branch to that function fails the suite until the schema documents it. - `jsonschema` declared as a dev dependency. It was already resolved as a litellm transitive; declaring it keeps the test suite off another package's dependency tree. `uv.lock` gains the two corresponding lines. Scope is narrower than the artifact and the schema says so: oracle-mode rollouts replace the trajectory with an oracle-only list, and session-factory Sessions bypass the ACP-session emitter entirely via `_snapshot_session_trajectory`. Neither is modelled here. No artifact-level contract is defined or implied. No runtime module imports the schema. Nothing under `src/benchflow` changes behaviour. Verified: 25 new tests; tests/trajectories 239 passed; ACP regression (capture, streaming, acp, native usage) 147 passed; ruff check/format clean; `uv lock --check` and `uv sync --locked --extra dev` both pass. --- docs/trace-interop.md | 359 +++++++++++++ pyproject.toml | 5 + .../schemas/acp-capture-event-v1.schema.json | 101 ++++ .../test_acp_capture_event_schema.py | 482 ++++++++++++++++++ uv.lock | 2 + 5 files changed, 949 insertions(+) create mode 100644 docs/trace-interop.md create mode 100644 src/benchflow/trajectories/schemas/acp-capture-event-v1.schema.json create mode 100644 tests/trajectories/test_acp_capture_event_schema.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md new file mode 100644 index 000000000..7932db361 --- /dev/null +++ b/docs/trace-interop.md @@ -0,0 +1,359 @@ +# Trace interoperability + +BenchFlow writes agent execution to disk in several shapes. This document +describes **the ACP-session capture-event format** — the records BenchFlow's +ACP-session emitter writes into `acp_trajectory.jsonl`, which BenchFlow both +produces and consumes — and records the current state of the two interchange +formats it is adjacent to, ATIF and OpenTelemetry. + +Everything below is marked as one of: + +- **FACT** — verified against the referenced implementation and tests when this + document was introduced. Every FACT carries a symbol or `file:line` reference. +- **PROPOSAL** — a suggestion under discussion. Nothing marked PROPOSAL has been + agreed, scheduled, or implemented. + +The companion JSON Schema lives at +[`src/benchflow/trajectories/schemas/acp-capture-event-v1.schema.json`](../src/benchflow/trajectories/schemas/acp-capture-event-v1.schema.json) +and is exercised by `tests/trajectories/test_acp_capture_event_schema.py`. + +--- + +## 1. Two different things are called "ACP" + +**FACT.** The name covers two distinct objects, and only one of them has an +external specification. + +| | ACP, the protocol | The BenchFlow ACP *trajectory* | +|---|---|---| +| What | Agent Client Protocol — a JSON-RPC wire protocol between a client and an agent | `trajectory/acp_trajectory.jsonl`, a BenchFlow artifact | +| Specified by | The `agent-client-protocol` SDK, re-exported in `src/benchflow/acp/types.py` | Nothing. This document specifies its ACP-session records only (§2.1) | +| Shape | Request/response and `session/update` notifications | A flat JSONL event log | +| Produced by | The agent | `benchflow.trajectories._capture`, plus the sources in §2.4 | + +The ACP-session records are **derived from** the protocol, not equal to it. They +are a lossy, flattened projection: `ACPSession.handle_update` consumes protocol +notifications into in-memory state, and `_events_to_trajectory` projects a +subset of that state into the on-disk event records. + +**FACT.** Fields the protocol carries and those records do not include the tool +call's `rawInput`, `rawOutput`, `locations` and `_meta`: `handle_update` reads +only `toolCallId`, `title`, `kind`, `status` and `content`. Verified by driving +a live `ACPSession` with all four present and observing their absence from the +captured events. + +--- + +## 2. The ACP-session capture-event format, as emitted today + +**FACT.** `trajectory/acp_trajectory.jsonl` is JSON Lines: one JSON object per +line, one object per event, in chronological order. There is **no envelope, no +header line, and no version field** — the file is the bare event sequence. + +An empty trajectory is written as a **zero-byte file**, not as a blank line +(`TrajectoryWriter.write_final`). The serializer emits no trailing newline +(`redact_acp_trajectory_jsonl`); some callers append one (`hosted_env.py`), so +readers must tolerate both. + +### 2.1 Scope of the schema + +**FACT.** `TrajectoryWriter` performs **no validation**. It will serialize any +JSON-compatible dict handed to it, including records with an unknown `type` or +no `type` at all — verified by writing both and observing them persisted +unchanged. + +The schema therefore describes **the normative output of the ACP-session capture +emitter** — the records constructed by `_events_to_trajectory` and by the +`ACPSession` legacy fallback in `_capture_session_trajectory` — and **not** the +set of every JSON value `TrajectoryWriter` is technically willing to serialize. +A file that fails the schema is not necessarily a file the writer would have +rejected; it is a file the ACP-session emitter would not have produced. + +**The schema's scope is narrower than the file.** Distinguish between: + +| | Covered by the schema? | +|---|---| +| The **ACP-session capture-event vocabulary** — records built by `_events_to_trajectory` and the `ACPSession` legacy fallback | **Yes.** This is exactly what the schema and the conformance suite pin | +| The **complete `acp_trajectory.jsonl` artifact** — everything that ends up in the file on disk | **No.** Two other sources contribute records; see §2.4 | + +So **validating every line of a final trajectory against this schema is not +equivalent to validating the artifact** — a failing line may come from one of +those other sources rather than be malformed. + +An artifact-level contract does not exist in this repository, and this document +does not create one. Whether `oracle` belongs inside the ACP trajectory contract +or is a separate downstream concern is open question 2 in §6. + +### 2.2 Event types + +**FACT.** `_events_to_trajectory` — the serializer for ACP sessions — emits +exactly **five** event types. + +| `type` | Emitted by | Meaning | +|---|---|---| +| `user_message` | `ACPSession.record_user_prompt` | A prompt handed to the agent | +| `agent_message` | `agent_message_chunk` / `text_update` notifications, via `handle_update` | Agent text visible to the user | +| `agent_thought` | `agent_thought_chunk` / `agent_thought` notifications, via `handle_update` | Agent internal reasoning | +| `tool_call` | `tool_call` / `tool_call_update` notifications, via `handle_update` | One tool invocation and its captured output | +| `agent_timeout` | `ACPSession.record_agent_timeout` | BenchFlow's wall-clock timeout marker — not an ACP notification | + +Consecutive text events of the same type are **merged into one record** before +serialization (`ACPSession._flush_agent_text`), so a single +`agent_message` may span many wire notifications. + +**FACT.** `handle_update` recognizes six `sessionUpdate` values +(`ACPSession._RECOGNIZED_UPDATE_TYPES`) and returns early for anything else. It +does set `_events_active` before that check — so the session stays on the +event-log capture branch — but records no event and fires no snapshot or change +notification. Unknown update types from a future ACP version are therefore +dropped, not persisted. + +### 2.3 Fields per event type + +**FACT.** Every listed field is **always present** on records from the +ACP-session emitter — `_events_to_trajectory` builds these dicts as literals, so +none of them is conditionally omitted. "Required" below means required by the +schema; a missing key means the record did not come from that emitter. + +#### `user_message` · `agent_message` · `agent_thought` + +| Field | Type | Required | Notes | +|---|---|---|---| +| `type` | string | yes | One of the three values | +| `text` | string | yes | The merged text | + +No other field is emitted. + +#### `tool_call` + +| Field | Type | Required | Notes | +|---|---|---|---| +| `type` | `"tool_call"` | yes | | +| `tool_call_id` | string | yes | May be `""` — `handle_update` defaults to the empty string rather than synthesizing an id | +| `kind` | string | yes | **Open vocabulary**, see below | +| `title` | string | yes | May be `""`. For ACP `execute` calls, conventionally the command line | +| `status` | enum | yes | `pending` · `in_progress` · `completed` · `failed` · `cancelled` | +| `content` | array | yes | Captured tool output; `[]` when there was none | + +**FACT — `kind` is not an enum.** `_canonical_tool_kind` +passes the agent-supplied string through unchanged, so production values are not +limited to the vendored `benchflow.acp.types.ToolKind` members +(`other` · `bash` · `search` · `browser` · `read` · `write` · `skill`). Values +documented as production kinds in a code comment at +`tests/integration/agent_judge.py:118-124` include `execute`, `edit`, `delete`, +`move`, `fetch`, `think` and `switch_mode`, none of which are `ToolKind` +members. Independently verified: `handle_update` writes the literal `"tool"` as +the fallback kind when a `tool_call_update` arrives for an id that was never +opened. Constraining `kind` to `ToolKind` in the schema would reject data the +emitter can produce. + +**FACT — `status` *is* closed.** The serialized value is always +`ToolCallStatus(...).value`, and a status the enum cannot parse falls back to +`in_progress`, so no out-of-vocabulary value reaches disk. + +**FACT — `content` blocks are pass-through.** BenchFlow stores the wire value +verbatim and never reshapes it. Two shapes are known to be consumed downstream by +`content_blocks_to_text`: the nested ACP shape +`{"type": "content", "content": {"type": "text", "text": "..."}}` and a flat +`{"text": "..."}` form. The protocol also defines file-edit and terminal block +variants, which BenchFlow persists but does not interpret. The schema keeps +content blocks permissive for this reason. + +#### `agent_timeout` + +| Field | Type | Required | Notes | +|---|---|---|---| +| `type` | `"agent_timeout"` | yes | | +| `reason` | `"wall_clock_timeout"` | yes | Single value, hardcoded by `record_agent_timeout` | +| `timeout_sec` | number | yes | The budget that was exceeded | +| `pending_tool_call_ids` | array of string | yes | Tool calls not in a terminal status when the timeout fired | +| `terminal_trajectory_complete` | boolean | yes | Whether the capture is considered complete anyway | + +### 2.4 Records in the file that the ACP-session emitter did not produce + +**FACT.** Two other sources can put records into `acp_trajectory.jsonl`. Neither +goes through `_events_to_trajectory`, and the schema models neither. Note that +the artifact *path* is the same in every mode, so the same filename can hold +different record shapes depending on how the rollout ran. + +**Session-factory Sessions.** `_snapshot_session_trajectory` duck-types on the +ACP streaming attributes; a `benchflow.agents.protocol.Session` has none, so the +live `on_change` sink writes `session.steps` **unchanged** +(`_capture.py:116-117`), bypassing the ACP-session emitter entirely. Whatever +that plane puts in `steps` is what reaches the streaming file; what survives the +final write depends on the trajectory list the caller passes to +`TrajectoryWriter.write_final`. + +**The `oracle` record — a whole alternative trajectory, not an addition.** In +`oracle_mode` the rollout does not run an agent at all: `_run_oracle` +(`rollout/_setup.py`) builds a **new, oracle-only** trajectory carrying `type`, +`command`, `return_code` and `stdout`, and the rollout **assigns** +`self._trajectory` to that list. ACP rollouts and oracle rollouts are therefore +mutually exclusive today, and **no production path emits a mixed ACP + oracle +artifact**. + +So the file is not a superset of the ACP-session emitter's output; it holds one +of several possible shapes, and no single site defines its full vocabulary. +Whether `oracle` belongs in the ACP trajectory contract or is a separate +downstream concern is an open question (see §6), and this document does not +settle it. + +--- + +## 3. Producers and consumers + +### 3.1 Producers + +**FACT.** + +| Stage | Site | +|---|---| +| Wire → session state | `ACPSession.handle_update` | +| Session state → events | `_events_to_trajectory` | +| Live streaming to disk | `TrajectoryWriter.flush`, wired as `session.on_change` | +| Multi-scene cumulative streaming | `make_trajectory_sink` | +| Final authoritative write | `TrajectoryWriter.write_final`, called from `rollout/_results.py` | +| Session-factory passthrough (§2.4) | `_snapshot_session_trajectory` (`_capture.py:116-117`) | +| Agent-native fallback (Gemini CLI) | `_scrape_agent_trajectory` / `_parse_gemini_trajectory` | +| Provider-evidence repair | `_reconcile_tool_evidence` / `_parse_provider_tool_evidence` | +| Oracle mode — replaces the trajectory (§2.4) | `_run_oracle` | +| Copy into the sandbox for verifiers | `_publish_trajectory_for_verifier` | + +### 3.2 Consumers + +**FACT.** Each consumer below parses the file independently; there is no shared +reader. + +| Consumer | Site | +|---|---| +| Trajectory viewer (`bench eval view`) | `trajectories/viewer.py:161-164, 283` | +| Skill-eval / GEPA export | `skill_eval/gepa_export.py:28-53` | +| In-sandbox judge | `adapters/resources/mcp_atlas_judge.py:27` | +| Task verifiers (declarable input) | `docs/task-standard.md:332, 587` | +| Integration rubric gate | `tests/integration/rubric_checks.py:565-599` | +| Harbor parity check | `tests/integration/check_skillsbench_harbor_parity.py:113-145` | +| Experiment-review skill | `.agents/skills/benchflow-experiment-review/scripts/validate_run_artifacts.py:767` | + +--- + +## 4. State of the adjacent formats + +### 4.1 ATIF + +**FACT.** ATIF (Agent Trajectory Interchange Format) support is one module, +`src/benchflow/trajectories/export_atif.py`, pinned to `ATIF-v1.7` +(`ATIF_SCHEMA_VERSION`). It writes `trainer/atif.json` per scored rollout, from +`_write_trainer_artifact`. + +- **No production reader.** Nothing under `src/` reads it back; the only code + that loads or validates ATIF is test/integration code. +- **No models and no schema.** Documents are assembled as plain `dict` literals; + the specification is prose in the module docstring, pinned against upstream + Harbor models and the ATIF RFC, **neither of which is vendored or checked by + any test**. +- Its validator is `atif_issues` (`tests/integration/scenarios.py:262-290`), + which checks that `schema_version` starts with `ATIF-v1` and that each + `step.source` is in `{user, agent, oracle}`. +- The trajectory viewer explicitly does not read it (`viewer.py:3`: *"No ATIF + conversion."*). + +**FACT.** The same module family also emits ADP (`export_adp.py`, version +`1.3.1`) and the Verifiers/ORS record (`export.py`). All three walk the same ACP +event list, from the same call site, `_write_trainer_artifact`. + +### 4.2 OpenTelemetry + +**FACT.** There is **no OpenTelemetry representation in the codebase**. There is +no OTel dependency in `pyproject.toml`, and no OTLP or `gen_ai.*` handling in +`src/`. + +`src/benchflow/trajectories/otel.py` — recorded in `CHANGELOG.md` under +*Removed*, "Removed the unwired `OTelCollector`" — was deleted as *"a +designed-but-never-wired OTLP receiver … never instantiated, never tested, and +not part of any run path"*. It converted **inbound only** (OTLP/JSON spans into +`LLMExchange` records) and mapped no tool calls, no span status and no +parent/child relationships. It contained no emitter, and no other OTel-named +module appears in the repository history. + +--- + +## 5. Information loss in today's conversions + +**FACT.** All losses below are observable in the current code. They describe +`ACP → ATIF` and `ACP → ADP` as implemented; nothing here is hypothetical. + +| # | What is lost | Where | Effect | +|---|---|---|---| +| 1 | **Tool arguments** | `acp_events_to_atif_steps` emits `"arguments": {}`; `acp_events_to_adp_content` emits `"kwargs": {}` | Exported trajectories record *that* a tool ran, not *with what*. The arguments are absent because the capture path drops `rawInput` (§1), not because the target formats lack a field | +| 2 | **Tool status** | ATIF demotes it into the non-standard `extra`; ADP drops it outright, as its module docstring states | A failed tool call is not machine-distinguishable from a successful one in ADP | +| 3 | **All timestamps** | `ToolCallRecord.started_at` / `.finished_at` exist and are not serialized by `_events_to_trajectory` | No per-event timing survives in any format. Only rollout-level wall clock exists, in `result.json` / `timing.json` | +| 4 | **`agent_timeout` events** | Emitted by `_events_to_trajectory`; no branch handles the type in `acp_events_to_atif_steps`, `acp_events_to_adp_content` or `acp_events_to_messages` | The timeout marker is absent from every exported document. In ADP, where `status` is dropped too (#2), no signal that the rollout timed out survives at all | +| 5 | **Non-text content blocks** | `content_blocks_to_text` skips them | File-edit diffs and terminal output are dropped from ATIF and ADP | +| 6 | **Per-step token usage** | `trajectory_to_atif_record` | Only four trajectory-level totals survive, and they are sourced from the LLM proxy capture (`rollout/_results.py:445-448`), not from ACP | +| 7 | **Agent version** | Hardcoded `"unknown"` by `trajectory_to_atif_record` | Documented in the code as deliberate rather than fabricated | +| 8 | **ACP session token usage** | `ACPSession.usage_snapshots` is routed to `result.json`, never into the trajectory | The trajectory carries no usage at all | +| 9 | **`stop_reason`** | Captured as `ACPSession.stop_reason`, never exported | Why the agent stopped is not in any trajectory format | + +**FACT.** Loss #4 is worth noting on its own: the timeout marker is written to +the trajectory so downstream consumers can see it, and every exporter drops it. + +--- + +## 6. Open questions + +These are unresolved in the repository today; this document does not answer +them. + +1. Should `acp_trajectory.jsonl` carry a `schema_version` field? It has none + today, and the consumers listed in §3.2 parse it without one. +2. Is `oracle` part of the ACP trajectory contract, or a separate downstream + record type that happens to share the file? (§2.4) +3. Should the ATIF validator's `oracle` source be produced by the emitter, or + removed from the validator? (§5) +4. Which definition of `input_tokens` is canonical — the cross-provider + normalized one in `_exchange_token_usage`, or the unnormalized ACP snapshot + in `normalize_acp_usage`? +5. Is OpenTelemetry wanted as an inbound receiver, an outbound emitter, or + both? The only implementation ever written was inbound, and it was removed + deliberately. + +--- + +## 7. Known divergences from this schema + +**FACT.** Files and fixtures in this repository that use the ACP trajectory +filename or shape but do **not** conform to the format described above. They are +**excluded from the conformance corpus**. Their current non-conformance is +recorded here as an observation, not asserted as a contract — a future change +that brings any of them into conformance is an improvement, not a regression. + +| Location | Divergence | +|---|---| +| `.agents/skills/benchflow-experiment-review/evals/files/*/trajectory/acp_trajectory.jsonl` (5 files) | Synthetic eval-harness fixtures that reuse the filename without following the production writer. They use `phase`, `tool`, `args` and `reward` keys, and `type` values `final`, `score`, `stderr`, `timeout` — none of which the writer emits. They pass their own validator, which requires only that `type` be a non-empty string (`validate_run_artifacts.py:232-244`) | +| `tests/test_judge_robustness.py:32` `_tc()` | A nested `{"source": ..., "tool_calls": [...]}` shape, documented in place as the "synthetic/deepagents" form — not ACP | +| `tests/test_judge_robustness.py:38` `_native()` | A `tool_call` with `type`, `tool_call_id`, `kind` and `title` only; the scanner under test reads no other field | +| `tests/test_integration_check_results.py:197` | A `tool_call` with `type`, `kind` and `title` only | +| `tests/test_integration_suite.py:242` | `{"role": "assistant"}` — no `type` | +| `tests/acceptance_live_harness.py:210` and `tests/test_acceptance_live_execution.py` | `{"type": "oracle", ...}` and `{"type": "agent", ...}` records; `oracle` is the oracle-mode type of §2.4; no producer of an `agent`-typed record was found under `src/` | + +The common pattern is that these are **inputs to a consumer under test**, minimal +by design because the consumer reads only a few fields. That is a legitimate +testing practice and not a claim about the file format. + +--- + +## 8. Ideas not yet agreed + +> **PROPOSAL — none of this is approved, scheduled, or implemented.** + +The reconnaissance that produced this document also sketched a canonical +intermediate representation, so that conversion between trace formats would be +`N ↔ 1` rather than `N²`, with each conversion returning an explicit report of +the fields it could not carry. + +That idea is recorded here only so this document is not silently read as an +argument for it. **It has not been reviewed by a maintainer, no design has been +agreed, and nothing in this repository implements it.** Any such work depends on +the open questions in §6 being answered first — in particular whether the capture +path may change at all, since several of the losses in §5 cannot be closed +downstream of it. diff --git a/pyproject.toml b/pyproject.toml index 3232f3b0b..93da41506 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,11 @@ dev = [ # AgentCore's CodeBuild path must reproduce Docker's ignore rules in tests # even when the optional AWS backend is not installed. "pathspec>=0.12", + # Validates the ACP trajectory conformance corpus against + # src/benchflow/trajectories/schemas/acp-capture-event-v1.schema.json. Already + # resolved as a litellm transitive; declared here so the test suite does not + # depend on another package's dependency tree. + "jsonschema>=4.20", "pytest>=9.0.3", "pytest-asyncio>=0.24.0", "ruff>=0.7.0", diff --git a/src/benchflow/trajectories/schemas/acp-capture-event-v1.schema.json b/src/benchflow/trajectories/schemas/acp-capture-event-v1.schema.json new file mode 100644 index 000000000..e528e2b92 --- /dev/null +++ b/src/benchflow/trajectories/schemas/acp-capture-event-v1.schema.json @@ -0,0 +1,101 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://benchflow.ai/schemas/acp-capture-event-v1.schema.json", + "title": "BenchFlow ACP-session capture event (v1)", + "description": "One event produced by the ACP-session capture emitter: the records built by `benchflow.trajectories._capture._events_to_trajectory`, plus the `ACPSession` legacy fallback in `_capture_session_trajectory`, serialized by `TrajectoryWriter` into a line of `trajectory/acp_trajectory.jsonl`. SCOPE IS NARROWER THAN THE FILE — this schema does not describe the complete `acp_trajectory.jsonl` artifact. Two other sources can contribute records to that file and are not modelled here: session-factory Sessions, for which `_snapshot_session_trajectory` returns `session.steps` unchanged and so bypasses the ACP-session emitter entirely; and oracle-mode rollouts, in which the agent is not executed at all — `_run_oracle` (`benchflow.rollout._setup`) produces an oracle-only trajectory, and that list becomes the rollout's trajectory. The same `acp_trajectory.jsonl` artifact path can therefore hold a different record family depending on the rollout mode, and no production path emits a mixed ACP + oracle artifact today. Validating every line of a final trajectory against this schema is therefore NOT equivalent to validating the artifact: a failing line may come from one of those sources rather than be malformed. No document defines the artifact-level contract, and this schema does not create one; see docs/trace-interop.md §2.1. Separately, this describes the emitter and not everything `TrajectoryWriter` will accept: the writer performs no validation and will persist arbitrary dicts. The `v1` in this file's name and `$id` identifies the schema document only — it does NOT correspond to any `schema_version` field in the artifact, which carries none.", + "oneOf": [ + { "$ref": "#/$defs/text_event" }, + { "$ref": "#/$defs/tool_call_event" }, + { "$ref": "#/$defs/agent_timeout_event" } + ], + "$defs": { + "content_block": { + "title": "ACP tool-call content block", + "description": "Pass-through from the ACP wire: BenchFlow stores `session/update.content` verbatim on the tool-call record and never reshapes it, so this stays permissive by design. Two shapes are known to be consumed by `benchflow.trajectories._export_common.content_blocks_to_text`: the nested ACP shape `{\"type\": \"content\", \"content\": {\"type\": \"text\", \"text\": \"...\"}}` and a flat `{\"text\": \"...\"}` form. The ACP protocol additionally defines file-edit and terminal block variants, which BenchFlow persists but does not interpret.", + "type": "object" + }, + + "text_event": { + "title": "Text event — user_message | agent_message | agent_thought", + "description": "Emitted by `_events_to_trajectory` for the three text-bearing event types. Consecutive same-type streamed chunks are merged into one event before serialization (`ACPSession._flush_agent_text`), so a single event may span many wire notifications.", + "type": "object", + "properties": { + "type": { + "enum": ["user_message", "agent_message", "agent_thought"] + }, + "text": { + "type": "string", + "description": "The merged text. MAY be the empty string: `record_user_prompt` records a prompt unconditionally, and the `agent_message_chunk` / `agent_thought_chunk` handlers append a chunk whose text is \"\" (unlike the `text_update` / `agent_thought` shim handlers, which skip empty text). Verified by driving a live session with empty chunks." + } + }, + "required": ["type", "text"], + "additionalProperties": false + }, + + "tool_call_event": { + "title": "Tool-call event", + "description": "One tool call and its captured output, flattened into a single record. All six properties are always written: `_events_to_trajectory` builds this dict literally, so a missing key means the record did not come from the ACP-session capture emitter.", + "type": "object", + "properties": { + "type": { "const": "tool_call" }, + "tool_call_id": { + "type": "string", + "description": "The ACP `toolCallId`. May be the empty string when the agent omitted it — `handle_update` defaults to \"\" rather than synthesizing an id. The ATIF and ADP exporters synthesize their own ids downstream." + }, + "kind": { + "type": "string", + "description": "OPEN string, deliberately not an enum. `_canonical_tool_kind` passes the agent-supplied `kind` through unchanged, so production values are not limited to the vendored `benchflow.acp.types.ToolKind` members. Values observed in production include `execute`, `edit`, `delete`, `move`, `fetch`, `think`, `switch_mode` (none of which are ToolKind members) plus the literal `tool`, which `handle_update` uses as the fallback when a `tool_call_update` arrives for an id that was never opened. Only `skill` is assigned by BenchFlow itself, by the classifier in `benchflow.trajectories.metrics`." + }, + "title": { + "type": "string", + "description": "Human-readable label. Empty string when the agent supplied none. For ACP `execute` calls this is conventionally the command line." + }, + "status": { + "enum": ["pending", "in_progress", "completed", "failed", "cancelled"], + "description": "CLOSED enum: the serialized value is always `ToolCallStatus(...).value`, and `handle_update` falls back to `in_progress` for any status it cannot parse, so no out-of-vocabulary value can reach disk." + }, + "content": { + "type": "array", + "description": "Captured tool output, accumulated across `tool_call_update` notifications. Empty array when the tool produced no content or the call never reached a terminal status.", + "items": { "$ref": "#/$defs/content_block" } + } + }, + "required": ["type", "tool_call_id", "kind", "title", "status", "content"], + "additionalProperties": false + }, + + "agent_timeout_event": { + "title": "Agent timeout marker", + "description": "BenchFlow's own terminal marker, appended by `ACPSession.record_agent_timeout` when the rollout hits its wall-clock limit. It is not an ACP protocol notification. Note that no trajectory exporter currently reads this event (see the loss table in docs/trace-interop.md).", + "type": "object", + "properties": { + "type": { "const": "agent_timeout" }, + "reason": { + "enum": ["wall_clock_timeout"], + "description": "Single-member enum: `record_agent_timeout` is the only producer and hardcodes this value. A second timeout reason would require updating this schema." + }, + "timeout_sec": { + "type": "number", + "description": "The wall-clock budget that was exceeded, in seconds." + }, + "pending_tool_call_ids": { + "type": "array", + "description": "Tool calls that had not reached a terminal status when the timeout fired.", + "items": { "type": "string" } + }, + "terminal_trajectory_complete": { + "type": "boolean", + "description": "Whether the captured trajectory is considered complete despite the timeout." + } + }, + "required": [ + "type", + "reason", + "timeout_sec", + "pending_tool_call_ids", + "terminal_trajectory_complete" + ], + "additionalProperties": false + } + } +} diff --git a/tests/trajectories/test_acp_capture_event_schema.py b/tests/trajectories/test_acp_capture_event_schema.py new file mode 100644 index 000000000..b18c81799 --- /dev/null +++ b/tests/trajectories/test_acp_capture_event_schema.py @@ -0,0 +1,482 @@ +"""Conformance suite for the ACP-session capture-event JSON Schema. + +Pins ``src/benchflow/trajectories/schemas/acp-capture-event-v1.schema.json`` +against what the ACP-session capture emitter actually produces — the records +built by ``_events_to_trajectory`` and by the ``ACPSession`` legacy fallback. +The schema is a description of that emitter, not of everything +``TrajectoryWriter`` is willing to serialize: the writer validates nothing, so +a schema derived from its tolerance would describe nothing at all. + +Its scope is narrower than the file. A complete ``acp_trajectory.jsonl`` may +also carry records from sources this suite does not cover — session-factory +``session.steps`` and the downstream ``oracle`` record — see +``docs/trace-interop.md`` §2.1 and §2.4. + +The suite is falsifiable along two independent axes: + +* **Record shape** — the C1 corpus below. Change the fields a record carries + and validation fails, with no fixture to quietly adjust. +* **Event-type vocabulary** — ``test_schema_covers_exactly_the_emitted_event_types``, + which reads the guards of ``_events_to_trajectory`` via AST. Add a branch to + that function and the comparison against the schema fails. + +Two corpora, both normative: + +* **C1 — emitter-generated.** Real :class:`ACPSession` objects driven through + ``handle_update`` / ``record_user_prompt`` / ``record_agent_timeout``, + captured with the production functions and written to disk with the + production writer. Every line of the resulting file must validate. +* **C2 — existing in-repo fixtures.** The ACP event lists already used as + exporter inputs by the ATIF and ADP export tests, imported rather than copied + so they cannot drift. + +Deliberately NOT validated: the five ``acp_trajectory.jsonl`` files under +``.agents/skills/benchflow-experiment-review/evals/``. They are synthetic +eval-harness fixtures that reuse the filename without following the production +writer, and they are excluded from the conformance corpus rather than pinned — +see the "Known divergences" section of ``docs/trace-interop.md``. +""" + +import ast +import json +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from benchflow.acp.session import ACPSession, ToolCallRecord +from benchflow.acp.types import ToolCallStatus +from benchflow.trajectories import _capture +from benchflow.trajectories._capture import ( + TrajectoryWriter, + _capture_session_trajectory, +) +from tests.trajectories.test_export_adp import _sample_events as _adp_sample_events +from tests.trajectories.test_export_atif import _sample_events as _atif_sample_events + +SCHEMA_PATH = ( + Path(__file__).resolve().parents[2] + / "src" + / "benchflow" + / "trajectories" + / "schemas" + / "acp-capture-event-v1.schema.json" +) + + +def _schema() -> dict: + return json.loads(SCHEMA_PATH.read_text()) + + +@pytest.fixture(scope="module") +def validator() -> Draft202012Validator: + return Draft202012Validator(_schema()) + + +def _assert_valid(validator: Draft202012Validator, events: list[dict]) -> None: + """Fail with the offending event and the concrete reason, not just a bool.""" + for index, event in enumerate(events): + errors = sorted(validator.iter_errors(event), key=lambda e: e.path) + assert not errors, ( + f"event[{index}] type={event.get('type')!r} failed schema:\n" + + "\n".join(f" - {e.json_path}: {e.message}" for e in errors) + + f"\n event = {json.dumps(event, default=str)}" + ) + + +def _round_trip_through_writer(tmp_path: Path, events: list[dict]) -> list[dict]: + """Persist with the production writer, read back what landed on disk. + + Validating the parsed file rather than the in-memory list keeps redaction + and serialization inside the tested path — a redaction that corrupted a + record would surface here. + + Every line is parsed, blanks included: ``splitlines`` already absorbs the + documented trailing newline, so a blank line left anywhere in the payload is + a real defect and must raise here rather than be skipped. + """ + path = tmp_path / "acp_trajectory.jsonl" + TrajectoryWriter(path).write_final(events) + return [json.loads(line) for line in path.read_text().splitlines()] + + +# --------------------------------------------------------------------------- +# The schema document itself +# --------------------------------------------------------------------------- + + +def test_schema_is_a_valid_draft_2020_12_document(): + Draft202012Validator.check_schema(_schema()) + + +def _schema_event_types() -> set[str]: + """Event types the schema declares, gathered across every ``$defs`` branch.""" + declared: set[str] = set() + for definition in _schema()["$defs"].values(): + type_prop = definition.get("properties", {}).get("type") + if not type_prop: + continue + if "const" in type_prop: + declared.add(type_prop["const"]) + declared.update(type_prop.get("enum", [])) + return declared + + +def _is_event_type_lookup(node: ast.expr) -> bool: + """True for the expression ``event["type"]`` exactly.""" + return ( + isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Name) + and node.value.id == "event" + and isinstance(node.slice, ast.Constant) + and node.slice.value == "type" + ) + + +def _emitted_event_types() -> set[str]: + """Event types accepted by ``_events_to_trajectory``'s guards, read from source. + + That function is a filter: an event is serialized if and only if its + ``type`` matches one of its ``event["type"]`` comparisons, so the guard + vocabulary *is* the emitted vocabulary. Only string literals compared + against that exact expression are collected — ``== "x"`` via a single + comparator, ``in ("x", "y")`` via a tuple/list/set of them. + """ + tree = ast.parse(Path(_capture.__file__).read_text()) + function = next( + ( + node + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) + and node.name == "_events_to_trajectory" + ), + None, + ) + assert function is not None, ( + "_events_to_trajectory not found in benchflow.trajectories._capture. The " + "conformance suite derives the emitted event-type vocabulary from its " + "guards; if the function was renamed or moved, update this extraction " + "rather than removing the check." + ) + + emitted: set[str] = set() + for node in ast.walk(function): + if not isinstance(node, ast.Compare) or not _is_event_type_lookup(node.left): + continue + for comparator in node.comparators: + candidates: list[ast.expr] = ( + list(comparator.elts) + if isinstance(comparator, ast.Tuple | ast.List | ast.Set) + else [comparator] + ) + emitted.update( + candidate.value + for candidate in candidates + if isinstance(candidate, ast.Constant) + and isinstance(candidate.value, str) + ) + return emitted + + +def test_schema_covers_exactly_the_emitted_event_types(): + """Schema vocabulary must equal the emitter's, with neither side hardcoded. + + The left side is parsed from the schema document; the right side is read + out of ``benchflow.trajectories._capture._events_to_trajectory`` via AST. + Adding a branch to that function therefore fails this test until the schema + documents the new event type. + + **This guarantee depends on the current shape of those guards.** The + function expresses its output vocabulary as ``event["type"]`` comparisons + against string literals, and the extraction reads exactly that. If it is + refactored so the vocabulary is no longer literal comparisons — a dispatch + table, a module-level constant, a helper predicate — the extraction returns + a different set and this test fails on purpose. There is deliberately no + permissive fallback: a failure here means the extraction needs review, not + that the check should be relaxed. + """ + emitted = _emitted_event_types() + assert emitted, ( + "no event types extracted from _events_to_trajectory. Its guards are no " + 'longer literal `event["type"]` comparisons, so this check can no longer ' + "see the emitted vocabulary — review the extraction above and re-derive " + "it from whatever now expresses that vocabulary." + ) + assert _schema_event_types() == emitted + + +# --------------------------------------------------------------------------- +# C1 — emitter-generated corpus +# --------------------------------------------------------------------------- + + +def _full_session() -> ACPSession: + """One session exercising every emitted event type in one trajectory.""" + session = ACPSession("sess-conformance") + session.record_user_prompt("List the files.") + # Two chunks of the same type merge into a single agent_thought event. + session.handle_update( + { + "sessionUpdate": "agent_thought_chunk", + "content": {"type": "text", "text": "I should "}, + } + ) + session.handle_update( + { + "sessionUpdate": "agent_thought_chunk", + "content": {"type": "text", "text": "run ls."}, + } + ) + session.handle_update( + { + "sessionUpdate": "tool_call", + "toolCallId": "tc1", + "title": "ls -la", + "kind": "execute", + "status": "pending", + } + ) + session.handle_update( + { + "sessionUpdate": "tool_call_update", + "toolCallId": "tc1", + "status": "completed", + "content": [ + {"type": "content", "content": {"type": "text", "text": "README.md"}} + ], + } + ) + session.handle_update( + { + "sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": "One file: README.md."}, + } + ) + session.record_agent_timeout( + timeout_sec=1.5, + pending_tool_call_ids=["tc9"], + terminal_trajectory_complete=False, + ) + return session + + +def test_c1_full_session_validates(validator, tmp_path): + events = _round_trip_through_writer( + tmp_path, _capture_session_trajectory(_full_session()) + ) + _assert_valid(validator, events) + assert {event["type"] for event in events} == { + "user_message", + "agent_thought", + "tool_call", + "agent_message", + "agent_timeout", + } + + +def test_c1_openclaw_shim_updates_validate(validator, tmp_path): + """``text_update`` / ``agent_thought`` are whole-text shim variants.""" + session = ACPSession("sess-shim") + session.record_user_prompt("go") + session.handle_update({"sessionUpdate": "text_update", "text": "shim message"}) + session.handle_update({"sessionUpdate": "agent_thought", "text": "shim thought"}) + events = _round_trip_through_writer(tmp_path, _capture_session_trajectory(session)) + _assert_valid(validator, events) + + +def test_c1_tool_call_update_for_unopened_id_validates(validator, tmp_path): + """The fallback record carries ``kind: "tool"`` and empty title/content. + + ``"tool"`` is not a ``ToolKind`` member — this case is why the schema keeps + ``kind`` an open string. + """ + session = ACPSession("sess-orphan") + session.handle_update( + { + "sessionUpdate": "tool_call_update", + "toolCallId": "orphan", + "status": "failed", + } + ) + events = _round_trip_through_writer(tmp_path, _capture_session_trajectory(session)) + _assert_valid(validator, events) + assert events[0]["kind"] == "tool" + assert events[0]["title"] == "" + assert events[0]["content"] == [] + + +def test_c1_legacy_capture_path_validates(validator, tmp_path): + """Sessions with no event log fall back to flat tool_calls + message.""" + session = ACPSession("sess-legacy") + record = ToolCallRecord("legacy1", "grep foo", "search") + record.update_status( + ToolCallStatus.COMPLETED, + [{"type": "content", "content": {"type": "text", "text": "hit"}}], + ) + session.tool_calls.append(record) + session.message_chunks.append("done") + events = _round_trip_through_writer(tmp_path, _capture_session_trajectory(session)) + _assert_valid(validator, events) + assert [event["type"] for event in events] == ["tool_call", "agent_message"] + + +@pytest.mark.parametrize("status", [s.value for s in ToolCallStatus]) +def test_c1_every_tool_call_status_validates(validator, tmp_path, status): + session = ACPSession(f"sess-{status}") + session.handle_update( + { + "sessionUpdate": "tool_call", + "toolCallId": "tc1", + "title": "t", + "kind": "read", + "status": "pending", + } + ) + session.handle_update( + {"sessionUpdate": "tool_call_update", "toolCallId": "tc1", "status": status} + ) + events = _round_trip_through_writer(tmp_path, _capture_session_trajectory(session)) + _assert_valid(validator, events) + assert events[0]["status"] == status + + +def test_c1_empty_text_events_validate(validator, tmp_path): + """Empty text is emitted, not filtered — the schema must accept it. + + ``record_user_prompt`` records unconditionally, and the chunk handlers + append a chunk whose text is ``""`` (the ``text_update`` / ``agent_thought`` + shim handlers do skip empty text, which is why this differs by path). + """ + session = ACPSession("sess-empty-text") + session.record_user_prompt("") + session.handle_update( + { + "sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": ""}, + } + ) + events = _round_trip_through_writer(tmp_path, _capture_session_trajectory(session)) + _assert_valid(validator, events) + assert [event["text"] for event in events] == ["", ""] + + +def test_c1_empty_session_writes_no_lines(tmp_path): + """An empty trajectory is an empty file, not a blank line.""" + events = _capture_session_trajectory(ACPSession("sess-empty")) + assert events == [] + path = tmp_path / "acp_trajectory.jsonl" + TrajectoryWriter(path).write_final(events) + assert path.read_text() == "" + + +# --------------------------------------------------------------------------- +# C2 — existing in-repo fixtures +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "sample", + [ + pytest.param(_atif_sample_events, id="test_export_atif._sample_events"), + pytest.param(_adp_sample_events, id="test_export_adp._sample_events"), + ], +) +def test_c2_exporter_input_fixtures_validate(validator, sample): + _assert_valid(validator, sample()) + + +# --------------------------------------------------------------------------- +# Falsifiability — the schema must reject, or it documents nothing +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("event", "why"), + [ + pytest.param( + {"type": "not_a_real_type", "text": "x"}, + "unknown event type", + id="unknown-type", + ), + pytest.param({"text": "x"}, "no discriminator", id="missing-type"), + pytest.param( + {"type": "user_message"}, "text is required", id="text-event-missing-text" + ), + pytest.param( + {"type": "agent_message", "text": "x", "timestamp": 1}, + "additionalProperties is false — timestamps are not emitted today", + id="text-event-extra-field", + ), + pytest.param( + { + "type": "tool_call", + "tool_call_id": "t", + "kind": "read", + "title": "t", + "status": "completed", + }, + "content is required", + id="tool-call-missing-content", + ), + pytest.param( + { + "type": "tool_call", + "tool_call_id": "t", + "kind": "read", + "title": "t", + "status": "finished", + "content": [], + }, + "status is a closed enum", + id="tool-call-bad-status", + ), + pytest.param( + { + "type": "tool_call", + "tool_call_id": "t", + "kind": "read", + "title": "t", + "status": "completed", + "content": [], + "rawInput": {"command": "ls"}, + }, + "rawInput is dropped by ACP-session capture and must not appear", + id="tool-call-rawinput", + ), + pytest.param( + { + "type": "agent_timeout", + "reason": "wall_clock_timeout", + "timeout_sec": 1.0, + "pending_tool_call_ids": [], + }, + "terminal_trajectory_complete is required", + id="timeout-missing-field", + ), + pytest.param( + { + "type": "agent_timeout", + "reason": "some_other_reason", + "timeout_sec": 1.0, + "pending_tool_call_ids": [], + "terminal_trajectory_complete": True, + }, + "reason is a single-member enum today", + id="timeout-unknown-reason", + ), + pytest.param( + { + "type": "tool_call", + "tool_call_id": "t", + "kind": "read", + "title": "t", + "status": "completed", + "content": {"type": "content"}, + }, + "content must be an array", + id="tool-call-content-not-array", + ), + ], +) +def test_schema_rejects(validator, event, why): + assert not validator.is_valid(event), f"schema should reject: {why}" diff --git a/uv.lock b/uv.lock index 87a5e1c5d..acddf21b9 100644 --- a/uv.lock +++ b/uv.lock @@ -357,6 +357,7 @@ deepagents = [ { name = "langchain-openai" }, ] dev = [ + { name = "jsonschema" }, { name = "pathspec" }, { name = "pre-commit" }, { name = "pytest" }, @@ -406,6 +407,7 @@ requires-dist = [ { name = "google-cloud-aiplatform", specifier = ">=1.133.0,<2.0" }, { name = "google-genai", marker = "extra == 'judge'", specifier = ">=1.0" }, { name = "httpx", specifier = ">=0.27.0" }, + { name = "jsonschema", marker = "extra == 'dev'", specifier = ">=4.20" }, { name = "langchain-openai", marker = "extra == 'deepagents'", specifier = ">=0.2" }, { name = "litellm", extras = ["proxy"], specifier = "==1.91.0" }, { name = "modal", marker = "extra == 'sandbox-modal'", specifier = ">=0.73" }, From 57cddaa27f54097cac0f2890f762273e693f2867 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Fri, 14 Aug 2026 10:08:59 +0200 Subject: [PATCH 02/23] test(trajectories): pin ACP -> ATIF preservation and loss invariants test_export_atif.py pins the shape ATIF export produces. This pins which information survives the conversion and which does not, so a future change to the converter either keeps the property or fails a test that says in one line what changed. Adds tests/trajectories/test_atif_preservation.py: 10 preservation invariants, 12 loss characterizations, 2 producer-boundary tests. No runtime module and no public format is touched, and no dependency is added. Two choices make the suite falsifiable rather than decorative. Losses are asserted with sentinel values absent from the whole serialized document, not with structural checks on the one field a test happened to look at. And both sides of the oracle-source divergence are read from source via AST -- the converter's event-type branches and the validator's accepted `source` set -- so changing either fails a test instead of silently invalidating the premise. The load-bearing producer-boundary test drives a real ACPSession with a tool_call update carrying rawInput, rawOutput, locations and _meta and shows that those fields are absent before the ACP -> ATIF converter runs. Real rollout evidence independently shows non-empty tool arguments in provider capture for calls whose ATIF arguments are {}. Together, these observations place the loss upstream of the ATIF converter rather than inside it. docs/trace-interop.md gains a per-field table (5.1), one previously unrecorded loss -- the agent_thought join is irreversible, so one thought containing a blank line is indistinguishable from two events -- and a section recording what two real gemini rollouts showed (5.2), including that the proxy capture holds the tool arguments the ATIF document records as {}. --- docs/trace-interop.md | 125 ++++ tests/trajectories/test_atif_preservation.py | 590 +++++++++++++++++++ 2 files changed, 715 insertions(+) create mode 100644 tests/trajectories/test_atif_preservation.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md index 7932db361..5ac5b7265 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -293,9 +293,134 @@ module appears in the repository history. | 7 | **Agent version** | Hardcoded `"unknown"` by `trajectory_to_atif_record` | Documented in the code as deliberate rather than fabricated | | 8 | **ACP session token usage** | `ACPSession.usage_snapshots` is routed to `result.json`, never into the trajectory | The trajectory carries no usage at all | | 9 | **`stop_reason`** | Captured as `ACPSession.stop_reason`, never exported | Why the agent stopped is not in any trajectory format | +| 10 | **`agent_thought` boundaries** | `ThoughtBuffer.take` joins buffered thoughts with a blank line | One thought containing a blank line and two consecutive thought events produce the same `reasoning_content`; the number of thought events is not recoverable. Reachable in production — `_parse_gemini_trajectory` appends one event per entry of a message's `thoughts` list | **FACT.** Loss #4 is worth noting on its own: the timeout marker is written to the trajectory so downstream consumers can see it, and every exporter drops it. +It is not lost at rollout level — `trajectory_summary.event_type_counts` in +`result.json` counts every event type including `agent_timeout` +(`_utils/result_metadata.py`), and the `agent_timeout_info` diagnostic records +it separately. The loss is confined to the exported trajectory documents. + +### 5.1 `ACP → ATIF`, field by field + +**FACT.** Every row below is asserted by +`tests/trajectories/test_atif_preservation.py` against records built through +the production capture path. This section describes what the conversion does +today; it proposes nothing. + +Four classes are used, and the distinction between the last two is where a fix +would have to land: + +- **preserved** — reaches the ATIF document unchanged. +- **normalized** — reaches it relocated or reshaped, so a consumer must know + BenchFlow's convention to read it. +- **dropped** — the capture events carry it and the converter discards it. + Fixable in `export_atif.py` alone. +- **unsupported** — the capture events never carried it. `export_atif.py` + cannot fix these; the loss is upstream, at `handle_update` or in + `_events_to_trajectory`. + +| ACP capture field / event | In ATIF today | Class | +|---|---|---| +| `user_message.text`, `agent_message.text` | `step.message`, verbatim | preserved | +| text-empty message or thought event | no step emitted | dropped | +| `agent_thought.text` | joined into the next agent step's `reasoning_content` | normalized (loss #10) | +| `tool_call.tool_call_id` | `tool_calls[].tool_call_id`; synthesized `call_{n}` when empty | preserved | +| `tool_call.kind` | `tool_calls[].function_name`; `"tool"` when empty | normalized — an ACP kind is a category, not a function name | +| tool arguments | `"arguments": {}`, always | unsupported — see §1 | +| `tool_call.content`, text blocks | `observation.results[].content` | preserved | +| `tool_call.content`, non-text blocks | absent | dropped (loss #5) | +| `tool_call.status`, `tool_call.title` | `tool_calls[].extra`, stringified | normalized (loss #2) — `extra` is ATIF-v1.7 and non-standard | +| `rawInput` / `rawOutput` / `locations` / `_meta` | absent | unsupported — dropped at the wire, never captured | +| `ToolCallRecord.started_at` / `.finished_at` | absent | unsupported — never serialized (loss #3) | +| event order | `step_id`, dense from 1 | preserved | +| per-step token usage | no `metrics` on any step | unsupported (loss #6) | +| `ACPSession.usage_snapshots`, `stop_reason` | absent | dropped (losses #8, #9) | +| `agent_timeout` | absent | dropped (loss #4) | +| unknown event types, non-dict entries | skipped silently, no gap in `step_id` | dropped, deliberate | +| `oracle` record | `source: "agent"`, `message: "[oracle: ]"` | normalized — **ambiguous**, see below | +| agent version | `"unknown"` | unsupported, deliberate (loss #7) | +| `prompts` argument | leading `user` steps | addition — not an ACP event | + +**FACT — the `oracle` source is a live divergence.** The in-repo validator +(`tests/integration/scenarios.py`) accepts `source` in +`{"user", "agent", "oracle"}`; the emitter produces only `user` and `agent`, +rendering oracle activity as an `agent` step with an `[oracle: …]` prefix. The +produced set is therefore a strict subset of the accepted set, and a consumer +cannot tell an oracle step from an agent step except by matching that string. +Whether the emitter should produce the third value or the validator should drop +it is open question 3 below; this document does not settle it, and the test +suite asserts the divergence rather than either resolution. + +**FACT — an ATIF document is not a function of the capture file alone.** +`prompts` adds steps with no corresponding ACP event, and text-empty events add +none, so the event list cannot be reconstructed from the document. Any future +round-trip claim has to account for both. + +### 5.2 Confirmed against real rollouts + +**FACT.** Two rollouts of the `gemini` agent were run through the production +path — docker sandbox, ACP transport, the standard artifact writers — and their +`acp_trajectory.jsonl`, `trainer/atif.json` and `result.json` were inspected by +hand. Everything §5.1 describes held. This section records only what those +artifacts showed. + +In both rollouts: + +- every `tool_call` capture record carried exactly `type`, `tool_call_id`, + `kind`, `title`, `status` and `content` — the six fields §5.1 predicts, and + nothing else; +- every `tool_call_id` reached the ATIF document unchanged; +- textual tool output reached `observation.results[].content`; +- `kind` became `function_name`, with observed values `execute`, `read` and + `think` — none of them `ToolKind` members, corroborating §2.3; +- `title` and `status` appeared only inside `tool_calls[].extra`; +- every `arguments` was `{}`; +- no ISO-8601 value appeared in either artifact; +- `rawInput`, `rawOutput`, `locations` and `_meta` appeared in neither the ACP + capture nor the ATIF document. + +**FACT — the tool inputs existed and were dropped, not unavailable.** In the +same rollouts the proxy capture (`llm_trajectory.jsonl`) carried non-empty +tool-call `arguments` payloads — in one case keyed `command` and `description`, +holding the shell command the agent ran — while the ATIF document recorded `{}` +for that same call. The inputs were therefore present in traffic BenchFlow +already captures, and absent from the exported trajectory. + +This is **not** evidence that the ACP `rawInput` family was on the wire. Those +four fields occurred nowhere in the captured artifacts, the proxy capture +included, which is expected because that capture is not ACP. Their absence from +the ACP capture is consistent with `handle_update` never reading them (§1), but +no observation in this repository shows them arriving. + +**FACT — the prompt-derived step duplicates the first user message.** Both ATIF +documents opened with two `user` steps carrying identical text: one built from +the `prompts` argument, one from the captured `user_message` event. A consumer +counting user turns from an ATIF document over-counts by one. + +**FACT — `agent_timeout` is written to the capture and exported nowhere.** A +rollout driven into its wall-clock budget recorded + +```json +{"type": "agent_timeout", "reason": "wall_clock_timeout", "timeout_sec": 90.0, + "pending_tool_call_ids": [], "terminal_trajectory_complete": true} +``` + +in `acp_trajectory.jsonl`, and `result.json` counted it under +`trajectory_summary.event_type_counts.agent_timeout`. The ATIF document for that +same rollout carried no representation of it — confirming loss #4 end to end, +including that the signal survives at rollout level (§5, note after the table). + +Note the shape observed: the budget expired between tool calls, so the pending +list was empty and the capture was marked terminally complete. The other branch +— a timeout with tool calls still in flight — is covered by the test suite but +was not observed in a real rollout. + +**Not exercised by these rollouts**, and still resting on code reading plus the +synthetic tests: the non-text content-block loss (#5), because neither agent +emitted a file-edit or terminal block; and the `oracle` source divergence, +because neither rollout ran in oracle mode. --- diff --git a/tests/trajectories/test_atif_preservation.py b/tests/trajectories/test_atif_preservation.py new file mode 100644 index 000000000..704c8a680 --- /dev/null +++ b/tests/trajectories/test_atif_preservation.py @@ -0,0 +1,590 @@ +"""Preservation and loss characterization for the ``ACP → ATIF`` conversion. + +``tests/trajectories/test_export_atif.py`` pins the *shape* ATIF export +produces. This suite pins something different and complementary: which +information survives the conversion, and which does not. Every claim in the +loss table of ``docs/trace-interop.md`` §5 that concerns ATIF has an executable +assertion here, so a future change to the converter either keeps the property +or fails a test that says in one line what changed. + +Four classes are used throughout, and the distinction between the last two is +the point of the suite: + +* **preserved** — the value reaches the ATIF document unchanged. +* **normalized** — it reaches the document, but relocated or reshaped, so a + consumer must know BenchFlow's convention to read it. +* **dropped** — the ACP capture events carry it and the converter discards it. + Fixable in ``export_atif.py`` alone. +* **unsupported** — the capture events never carried it in the first place. + ``export_atif.py`` cannot fix these; the loss is upstream, at the ACP wire + boundary (:meth:`ACPSession.handle_update`) or in + ``_events_to_trajectory``. + +Losses are asserted with sentinel values rather than with structural checks +wherever possible: a sentinel absent from ``json.dumps(document)`` is a claim +about the whole document, not about the one field the test happened to look +at, and it stays true if the converter later moves data around. + +Scope: the ``ACP-session capture events`` → ATIF path only. ADP and +Verifiers/ORS share ``_export_common`` and most of these properties, but they +are deliberately out of scope here; nothing in this file needs to change to +cover them later. + +No runtime module is imported for its side effects and nothing here writes +outside ``tmp_path``. +""" + +from __future__ import annotations + +import ast +import json +import re +from pathlib import Path +from typing import Any + +from benchflow.acp.session import ACPSession +from benchflow.acp.types import StopReason +from benchflow.trajectories import export_atif +from benchflow.trajectories._capture import _capture_session_trajectory +from benchflow.trajectories.export_atif import ( + acp_events_to_atif_steps, + trajectory_to_atif_record, + write_rollout_atif_json, +) +from tests.integration.scenarios import atif_issues + +# Sentinels. Distinctive enough that finding one anywhere in a serialized +# document is unambiguous evidence the value survived, and finding none is +# evidence it did not. +RAW_INPUT = "SENTINEL-rawInput-8f21" +RAW_OUTPUT = "SENTINEL-rawOutput-4c07" +LOCATION = "/SENTINEL-locations-1b93/main.py" +META = "SENTINEL-meta-77de" +PENDING_TOOL_CALL_ID = "SENTINEL-pending-tc-9f13" +TIMEOUT_SEC = 1337.75 +NON_TEXT_BLOCK = "SENTINEL-diff-newText-a55c" +USAGE_TOKENS = 424242 + +# `datetime.now()` renders as `2026-08-14T06:43:29.280301`; the capture path +# builds such values on every ToolCallRecord, so their absence downstream is +# the observable form of "timestamps are not exported". +ISO_DATETIME = re.compile(r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}") + + +# --------------------------------------------------------------------------- +# Corpora — built through the production capture path, not hand-written +# --------------------------------------------------------------------------- + + +def _rich_session() -> ACPSession: + """One session carrying every emitted event type plus the dropped extras. + + The ``tool_call`` updates deliberately include the four ACP protocol + fields BenchFlow does not read (``rawInput``, ``rawOutput``, ``locations``, + ``_meta``) so the boundary tests below can assert where they stop. + """ + session = ACPSession("sess-a2") + session.record_user_prompt("List the files.") + session.handle_update( + { + "sessionUpdate": "agent_thought_chunk", + "content": {"type": "text", "text": "I should run ls."}, + } + ) + session.handle_update( + { + "sessionUpdate": "tool_call", + "toolCallId": "tc1", + "title": "ls -la", + "kind": "execute", + "status": "pending", + "rawInput": {"command": RAW_INPUT}, + "locations": [{"path": LOCATION}], + "_meta": {"trace": META}, + } + ) + session.handle_update( + { + "sessionUpdate": "tool_call_update", + "toolCallId": "tc1", + "status": "completed", + "rawOutput": {"stdout": RAW_OUTPUT}, + "content": [ + {"type": "content", "content": {"type": "text", "text": "README.md"}} + ], + } + ) + session.handle_update( + { + "sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": "One file: README.md."}, + } + ) + # Session-level state that lives beside the event log and is never exported. + session.stop_reason = next(iter(StopReason)) + session.usage_snapshots.append({"input_tokens": USAGE_TOKENS}) + session.record_agent_timeout( + timeout_sec=TIMEOUT_SEC, + pending_tool_call_ids=[PENDING_TOOL_CALL_ID], + terminal_trajectory_complete=False, + ) + return session + + +def _rich_events() -> list[dict[str, Any]]: + return _capture_session_trajectory(_rich_session()) + + +PROMPTS = ["Solve the task.", "Then stop."] + + +def _rich_document() -> dict[str, Any]: + return trajectory_to_atif_record( + session_id="sess-a2", + agent_name="claude-code", + events=_rich_events(), + prompts=PROMPTS, + ) + + +def _walk_keys(obj: Any): + """Yield every mapping key in a nested document.""" + if isinstance(obj, dict): + for key, value in obj.items(): + yield key + yield from _walk_keys(value) + elif isinstance(obj, list): + for item in obj: + yield from _walk_keys(item) + + +def _tool_calls(document: dict[str, Any]) -> list[dict[str, Any]]: + return [call for step in document["steps"] for call in step.get("tool_calls", [])] + + +# --------------------------------------------------------------------------- +# Source introspection — keeps the loss claims tied to the code, not to a copy +# --------------------------------------------------------------------------- + + +def _converter_handled_event_types() -> set[str]: + """Event types ``acp_events_to_atif_steps`` branches on, read via AST. + + Mirrors the technique in ``test_acp_capture_event_schema.py``: asserting + against the source rather than against a hand-maintained list means adding + a branch to the converter fails a test here instead of silently + invalidating this suite's premise. + """ + tree = ast.parse(Path(export_atif.__file__).read_text()) + function = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == "acp_events_to_atif_steps" + ) + handled: set[str] = set() + for node in ast.walk(function): + if not isinstance(node, ast.Compare): + continue + if not (isinstance(node.left, ast.Name) and node.left.id == "etype"): + continue + # ast.Compare keeps ops and comparators in lockstep, so strict zipping + # is a free assertion that the node is well-formed. + for op, comparator in zip(node.ops, node.comparators, strict=True): + if ( + isinstance(op, ast.Eq) + and isinstance(comparator, ast.Constant) + and isinstance(comparator.value, str) + ): + handled.add(comparator.value) + return handled + + +def _validator_valid_sources() -> set[str]: + """The ``source`` allowlist inside ``tests.integration.scenarios.atif_issues``. + + A local variable rather than a module constant, so it is read from source. + Extracting it keeps the emitter and its only in-repo consumer from drifting + apart without a test noticing. + """ + import tests.integration.scenarios as scenarios + + tree = ast.parse(Path(scenarios.__file__).read_text()) + function = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == "atif_issues" + ) + for node in ast.walk(function): + if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Set): + continue + if any( + isinstance(target, ast.Name) and target.id == "valid_sources" + for target in node.targets + ): + return { + element.value + for element in node.value.elts + if isinstance(element, ast.Constant) + } + raise AssertionError( + "atif_issues no longer assigns a literal `valid_sources` set; re-derive " + "the consumer's accepted sources from whatever replaced it." + ) + + +# --------------------------------------------------------------------------- +# Preservation invariants — these must survive any future refactor +# --------------------------------------------------------------------------- + + +def test_message_text_survives_verbatim(): + document = _rich_document() + messages = [step.get("message", "") for step in document["steps"]] + assert "List the files." in messages + assert "One file: README.md." in messages + + +def test_every_prompt_becomes_a_leading_user_step(): + document = _rich_document() + leading = document["steps"][: len(PROMPTS)] + assert [step["message"] for step in leading] == PROMPTS + assert {step["source"] for step in leading} == {"user"} + + +def test_tool_call_identity_is_preserved(): + (call,) = _tool_calls(_rich_document()) + assert call["tool_call_id"] == "tc1" + + +def test_textual_tool_output_reaches_the_observation(): + document = _rich_document() + contents = [ + result["content"] + for step in document["steps"] + for result in step.get("observation", {}).get("results", []) + ] + assert contents == ["README.md"] + + +def test_no_thought_text_is_lost(): + document = _rich_document() + reasoning = " ".join( + step.get("reasoning_content", "") for step in document["steps"] + ) + assert "I should run ls." in reasoning + + +def test_tool_kind_becomes_function_name_with_tool_fallback(): + (call,) = _tool_calls(_rich_document()) + assert call["function_name"] == "execute" + + # `handle_update` writes the literal "tool" as the kind when an update + # arrives for an id that was never opened; the converter's own fallback + # covers the case where even that is missing. + (step,) = acp_events_to_atif_steps( + [{"type": "tool_call", "tool_call_id": "x", "kind": "", "content": []}] + ) + assert step["tool_calls"][0]["function_name"] == "tool" + + +def test_tool_status_and_title_survive_somewhere(): + """Location-agnostic on purpose — that they survive is the invariant.""" + (call,) = _tool_calls(_rich_document()) + serialized = json.dumps(call) + assert "completed" in serialized + assert "ls -la" in serialized + + +def test_step_ids_are_dense_and_match_total_steps(): + document = _rich_document() + assert [step["step_id"] for step in document["steps"]] == list( + range(1, len(document["steps"]) + 1) + ) + assert document["final_metrics"]["total_steps"] == len(document["steps"]) + + +def test_observation_results_resolve_within_their_own_step(): + """The one ATIF spec constraint the converter must not break. + + ``source_call_id`` resolves against the *same* step's ``tool_calls``, so a + trajectory with several calls is the case that would catch a converter + that started resolving trajectory-wide. + """ + events = [ + { + "type": "tool_call", + "tool_call_id": tool_call_id, + "kind": "execute", + "title": f"cmd {index}", + "status": "completed", + "content": [{"text": f"out {index}"}], + } + for index, tool_call_id in enumerate(["tc1", "", "tc3", ""]) + ] + steps = acp_events_to_atif_steps(events) + assert len(steps) == len(events) + seen: set[str] = set() + for step in steps: + ids = {call["tool_call_id"] for call in step["tool_calls"]} + for result in step["observation"]["results"]: + assert result["source_call_id"] in ids + seen |= ids + # Synthesized ids must not collide with each other or with real ones. + assert len(seen) == len(events) + + +def test_emitter_output_satisfies_the_in_repo_atif_validator(tmp_path): + """The only consumer of ATIF inside this repository must accept the output.""" + write_rollout_atif_json( + tmp_path, + session_id="sess-a2", + agent_name="claude-code", + prompts=PROMPTS, + trajectory=_rich_events(), + ) + assert atif_issues(tmp_path) == [] + + +# --------------------------------------------------------------------------- +# Loss characterization — pins today's behaviour so a change is deliberate +# --------------------------------------------------------------------------- + + +def test_tool_arguments_are_always_empty(): + """Loss #1. Empty even when the source event carries extra structure. + + If the capture layer is ever enriched (open question 3 in + ``docs/trace-interop.md``), this test fails and points at the converter as + the place that still has to be taught to read the new field. + """ + for call in _tool_calls(_rich_document()): + assert call["arguments"] == {} + + (step,) = acp_events_to_atif_steps( + [ + { + "type": "tool_call", + "tool_call_id": "tc1", + "kind": "execute", + "title": "ls", + "status": "completed", + "content": [], + "rawInput": {"command": RAW_INPUT}, + } + ] + ) + assert step["tool_calls"][0]["arguments"] == {} + assert RAW_INPUT not in json.dumps(step) + + +def test_tool_status_and_title_live_in_the_non_standard_extra(): + """Loss #2, the normalized half of ``test_tool_status_and_title_survive``. + + ``extra`` was added in ATIF-v1.7; a consumer reading only standard fields + sees neither the status nor the command. + """ + (call,) = _tool_calls(_rich_document()) + assert call["extra"] == {"title": "ls -la", "status": "completed"} + assert "status" not in {key for key in call if key != "extra"} + + +def test_no_timestamp_reaches_the_document(): + """Loss #3. ``ToolCallRecord`` stamps ``started_at``/``finished_at``. + + Those attributes exist on the record the capture path built above, and + nothing serializes them, so no ISO-8601 value can appear downstream. + """ + serialized = json.dumps(_rich_document()) + assert not ISO_DATETIME.search(serialized) + + +def test_agent_timeout_leaves_no_trace_in_the_document(): + """Loss #4. The marker is captured, then dropped by the converter. + + Asserted against the whole serialized document, not against the step list, + so a future partial rendering of the timeout would still be visible here. + """ + events = _rich_events() + assert any(event["type"] == "agent_timeout" for event in events), ( + "the corpus no longer contains an agent_timeout event; this test would " + "pass vacuously" + ) + serialized = json.dumps(_rich_document()) + assert "wall_clock_timeout" not in serialized + assert PENDING_TOOL_CALL_ID not in serialized + assert str(TIMEOUT_SEC) not in serialized + + +def test_converter_handles_exactly_the_event_types_it_documents(): + """The structural counterpart of the loss above. + + Reads the converter's own branches. ``agent_timeout`` is emitted by + ``_events_to_trajectory`` and deliberately absent here; ``oracle`` is + handled although the ACP-session emitter never produces it (§2.4). + """ + assert _converter_handled_event_types() == { + "user_message", + "agent_thought", + "agent_message", + "tool_call", + "oracle", + } + + +def test_non_text_content_blocks_are_dropped(): + """Loss #5. ``content_blocks_to_text`` renders text blocks and nothing else.""" + (step,) = acp_events_to_atif_steps( + [ + { + "type": "tool_call", + "tool_call_id": "tc1", + "kind": "edit", + "title": "patch main.py", + "status": "completed", + "content": [ + { + "type": "diff", + "path": "main.py", + "oldText": "a", + "newText": NON_TEXT_BLOCK, + } + ], + } + ] + ) + assert "observation" not in step + assert NON_TEXT_BLOCK not in json.dumps(step) + + +def test_no_per_step_metrics_and_no_session_level_usage_or_stop_reason(): + """Losses #6, #8 and #9, asserted over the whole document. + + Per-step ``metrics`` is a valid ATIF field on agent steps and is never + emitted; ``ACPSession.usage_snapshots`` and ``stop_reason`` were populated + on the session that produced this corpus and reach nothing. + """ + document = _rich_document() + assert all("metrics" not in step for step in document["steps"]) + keys = set(_walk_keys(document)) + assert "metrics" not in keys + assert "stop_reason" not in keys + serialized = json.dumps(document) + assert str(USAGE_TOKENS) not in serialized + assert next(iter(StopReason)).value not in serialized + + +def test_agent_version_is_declared_unknown_rather_than_fabricated(): + """Loss #7, documented in the converter as deliberate.""" + assert _rich_document()["agent"]["version"] == "unknown" + + +def test_oracle_events_never_produce_an_oracle_source(): + """The emitter renders oracle activity as an ``agent`` step. + + The in-repo validator accepts ``source: "oracle"``; nothing produces it. + Whether that third value should be produced or removed is open question 3 + in ``docs/trace-interop.md`` — this test states the divergence rather than + resolving it, and fails if either side moves. + """ + steps = acp_events_to_atif_steps( + [ + {"type": "user_message", "text": "go"}, + {"type": "agent_message", "text": "ok"}, + {"type": "oracle", "command": "bash run.sh"}, + ] + ) + produced = {step["source"] for step in steps} + accepted = _validator_valid_sources() + + assert produced <= accepted + assert "oracle" in accepted + assert "oracle" not in produced + assert steps[-1] == { + "step_id": 3, + "source": "agent", + "message": "[oracle: bash run.sh]", + } + + +def test_empty_text_events_produce_no_step(): + """Step count is not a function of event count.""" + assert ( + acp_events_to_atif_steps( + [ + {"type": "user_message", "text": ""}, + {"type": "agent_message", "text": ""}, + {"type": "agent_thought", "text": ""}, + ] + ) + == [] + ) + + +def test_unknown_and_malformed_events_do_not_perturb_step_numbering(): + """Dropping is silent and leaves no gap in ``step_id``.""" + steps = acp_events_to_atif_steps( + [ + {"type": "user_message", "text": "first"}, + {"type": "a_future_event_type", "text": "ignored"}, + "not-a-dict", + {"type": "agent_timeout", "reason": "wall_clock_timeout"}, + {"type": "agent_message", "text": "second"}, + ] + ) + assert [step["step_id"] for step in steps] == [1, 2] + assert [step["message"] for step in steps] == ["first", "second"] + + +def test_consecutive_thoughts_are_indistinguishable_from_one_joined_thought(): + """The ``\\n\\n`` join is not reversible. + + ``ThoughtBuffer`` joins buffered thoughts with a blank line, so a single + thought that already contains one produces the same ``reasoning_content`` + as two separate events. Reachable in production: the Gemini scrape path + (``_capture._parse_gemini_trajectory``) appends one ``agent_thought`` event + per entry of a message's ``thoughts`` list, so consecutive thought events + are a shape the capture layer really emits. + """ + two_events = acp_events_to_atif_steps( + [ + {"type": "agent_thought", "text": "first"}, + {"type": "agent_thought", "text": "second"}, + ] + ) + one_event = acp_events_to_atif_steps( + [{"type": "agent_thought", "text": "first\n\nsecond"}] + ) + assert two_events == one_event + + +# --------------------------------------------------------------------------- +# Producer boundary — where the tool-argument loss actually happens +# --------------------------------------------------------------------------- + + +def test_raw_input_family_never_reaches_the_capture_events(): + """``handle_update`` reads five fields and drops the rest. + + This is the load-bearing test of the suite: it places the tool-argument + loss at the ACP wire boundary rather than in the converter, which is what + makes ``arguments: {}`` unfixable in ``export_atif.py`` alone. + """ + events = _rich_events() + (tool_call,) = [event for event in events if event["type"] == "tool_call"] + assert set(tool_call) == { + "type", + "tool_call_id", + "kind", + "title", + "status", + "content", + } + serialized = json.dumps(events) + for sentinel in (RAW_INPUT, RAW_OUTPUT, LOCATION, META): + assert sentinel not in serialized + + +def test_raw_input_family_never_reaches_the_atif_document(): + serialized = json.dumps(_rich_document()) + for sentinel in (RAW_INPUT, RAW_OUTPUT, LOCATION, META): + assert sentinel not in serialized From 174053d147143bd5f8ac7b805e12ad4b86374454 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Sat, 15 Aug 2026 09:43:21 +0200 Subject: [PATCH 03/23] feat(trajectories): provisional canonical Trace IR v0, unwired MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four trace-shaped representations already exist here — the ACP-session capture events, ATIF, ADP and the Verifiers/ORS record — and all three exporters walk the same ACP event list from the same call site. The cost of that is not the edge count; it is that each edge answers the same questions privately, and the answers already diverge: for one `tool_call` event ATIF emits `"arguments": {}` while ADP emits `"kwargs": {}`, ATIF keeps the tool status in a non-standard `extra` while ADP drops it, both join thought boundaries irreversibly, and neither represents `agent_timeout` at all. This adds a canonical hub so each format becomes one edge against a written contract, and so the information loss is a typed value rather than a comment in a module docstring. The direction is not approved. The four open questions in docs/trace-interop.md §6 have had no maintainer answer, so this takes a provisional position on the first one and implements it in isolation, where it can be reviewed as code and reverted by deleting two files. - `src/benchflow/trajectories/ir.py` — the IR types, the loss model (`LossReport` / `LossRecord` over an unsupported/dropped/normalized/ synthesized taxonomy, the same one §5.1 already uses), and `validate_trace`, which returns one string per invariant violation rather than raising. The rule the module is built on is that the IR is a pragmatic superset of what BenchFlow can observe, not a model of what an agent trace could contain. Three consequences are load-bearing: optional values are tri-state (a value / `None` "this source never carried it" / an empty value "carried and empty"); a `None` that is not covered by a loss record makes the trace invalid, so absence is declared and never silent; and normalization is non-destructive — `source_type` keeps the source's own type string next to the normalized `kind`, and `name_semantics` records that an ACP `kind` is a category rather than a function name. The IR deliberately cannot fabricate an agent version, a synthetic tool-call id, a timestamp, or an OTel span id. Those are target-side obligations and belong in converters, which record them as SYNTHESIZED. - `tests/trajectories/test_trace_ir.py` — 25 tests. Each invariant is exercised with a violating trace and a clean one, so a rule that stopped firing fails here instead of passing silently. The IR's tool-status vocabulary is checked against the ACP `ToolCallStatus` enum and its event kinds against the vocabulary `_events_to_trajectory` actually emits, read from source by AST — the mechanism the Slice A conformance suite already uses, so adding a branch to the capture path fails this suite until the IR accounts for it. Two tests pin the isolation claim rather than the design: no module under `src/benchflow` imports the IR, and the IR imports no benchflow module. The first is the executable form of "zero runtime behaviour change"; wiring the IR into a run path has to update it deliberately. - `docs/trace-interop.md` §8 — replaces the "ideas not yet agreed" placeholder with the design: why a hub, the four alternatives considered and why each was not taken, the field classes (supported today / optional / needs enrichment / must not be invented), the planned ACP→IR and IR→ATIF mappings, an OTel sketch marked unverified because no version of the GenAI conventions is vendored here, the invariants, and an explicit list of what a review can still reject. §6 gains a note that question 1 now has a provisional, unagreed position and that questions 2-5 are untouched. The worked example in §8.4 is generated from the models and compared against the document by a test, so it cannot drift. Not included, deliberately: `ACP → IR`, `ATIF ↔ IR`, any OTel work, any wiring into a run path, any on-disk artifact. The representation is meant to be reviewed before anything depends on it. No existing format, exporter, artifact or code path changes. No new dependency — pydantic is already a runtime dependency and `uv.lock` is untouched. Verified: 25 new tests; tests/trajectories 288 passed; the trajectory, capture, streaming and ACP regression lane 453 passed; `ruff check .` and `ruff format --check` clean; `ty check src/` clean. --- docs/trace-interop.md | 337 ++++++++++++++- src/benchflow/trajectories/ir.py | 532 ++++++++++++++++++++++++ tests/trajectories/test_trace_ir.py | 618 ++++++++++++++++++++++++++++ 3 files changed, 1475 insertions(+), 12 deletions(-) create mode 100644 src/benchflow/trajectories/ir.py create mode 100644 tests/trajectories/test_trace_ir.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md index 5ac5b7265..2168a1cb8 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -442,6 +442,11 @@ them. both? The only implementation ever written was inbound, and it was removed deliberately. +**Still open.** §8 takes a *provisional* position on question 1 — a canonical +hub — and implements it in isolation so it can be reviewed as code. That is a +proposal, not an answer: no maintainer has agreed it, nothing depends on it, and +questions 2–5 are untouched by it. + --- ## 7. Known divergences from this schema @@ -467,18 +472,326 @@ testing practice and not a claim about the file format. --- -## 8. Ideas not yet agreed +## 8. Canonical Trace IR v0 — provisional + +> **PROPOSAL — not approved.** This section describes a design decision taken +> *without* maintainer sign-off, so the task could continue while the questions +> in §6 stayed open. It is implemented in +> [`src/benchflow/trajectories/ir.py`](../src/benchflow/trajectories/ir.py) so it +> can be reviewed as code rather than as a sketch. **Nothing imports it, nothing +> writes it to disk, and no existing format, artifact or code path changes +> because it exists** — a property pinned by a test +> (`test_no_runtime_module_imports_the_ir`). Deleting the module and its test +> returns the tree to its previous behaviour. +> +> Statements about the *current code* below remain FACT, with references, as +> everywhere else in this document. Statements about the IR are PROPOSAL. + +### 8.1 Why a hub, and why not pairwise converters + +**FACT.** Four trace-shaped representations already exist in this repository — +the ACP-session capture events (§2), ATIF (§4.1), ADP and the Verifiers/ORS +record — and all three exporters walk the same ACP event list from the same call +site, `_write_trainer_artifact`. OpenTelemetry (§4.2) would be a fifth. + +Pairwise conversion costs `N*(N-1)` directed edges, but the cost that matters is +not the edge count. It is that **each edge answers the same questions +privately**, and in this codebase those answers already diverge: + +**FACT.** For the identical ACP `tool_call` event, ATIF emits +`"arguments": {}` while ADP emits `"kwargs": {}`; ATIF preserves the tool status +in a non-standard `extra` while ADP drops it (§5, losses #1–#2). Both join +thought boundaries irreversibly through the same `ThoughtBuffer` (loss #10), and +neither represents `agent_timeout` at all (loss #4) — three independent +decisions about the same event, taken three times, recorded nowhere. + +A hub turns each format into one edge against a written contract, and — the +actual point — makes the loss a **typed value** rather than a comment in a +module docstring. See `LossReport` / `LossRecord` in the module. + +The alternative shapes considered and not taken: + +| Option | Why not | +|---|---| +| **Direct pairwise converters** | What exists today. Each new format multiplies the decisions above, and none of them is recorded anywhere a consumer can read | +| **Promote ATIF to the hub** | ATIF has no reader, no models and no vendored schema in this repository (§4.1); it cannot represent `agent_timeout` or per-event timestamps, so it would bake today's losses into the hub | +| **Promote the ACP capture events to the hub** | It is a lossy projection of the protocol by construction (§1), has no artifact-level contract (§2.1), and its file can hold non-ACP records (§2.4) | +| **Extend the capture format instead** | Changes the on-disk artifact that the consumers in §3.2 already parse — a compatibility decision, and precisely open question 3 | + +### 8.2 What the IR carries, and on what evidence -> **PROPOSAL — none of this is approved, scheduled, or implemented.** +**PROPOSAL.** The rule the module is built on: *the IR is a pragmatic superset +of what BenchFlow can observe, not a model of what an agent trace could contain*. +Every field is in one of four states, and the distinction is deliberate: -The reconnaissance that produced this document also sketched a canonical -intermediate representation, so that conversion between trace formats would be -`N ↔ 1` rather than `N²`, with each conversion returning an explicit report of -the fields it could not carry. +| Class | Meaning | Fields | +|---|---|---| +| **Supported today** | Some source in this repository carries the value now | event order · event kind · role · text · reasoning · tool call id / kind / title / status · content blocks · outcome status · trace-level usage · session id · agent + model name | +| **Optional** | Carried when the source has it, absent without loss when it does not | trace id · provider · reward · error category · stop reason · per-event usage | +| **Needs enrichment first** | The slot exists because the value demonstrably exists *upstream* of the capture, and is dropped before disk | tool `arguments` (the ACP `rawInput` family, §1) · per-event `started_at` / `finished_at` (`ToolCallRecord` tracks both, loss #3) | +| **Must not be invented** | The IR deliberately has no way to fabricate these | agent version (`"unknown"` is ATIF's requirement, not a fact) · synthetic tool-call ids (`call_{n}`, `call_NNNNNN`) · timestamps for sources that carry none · OTel span/trace ids | + +Three design choices carry most of the weight: + +1. **Tri-state optionality.** A value, `None` ("this source never carried it"), + and an empty value ("carried, and empty") are three different facts. + `arguments={}` and `arguments=None` are the case that matters: every + ACP-derived tool call is the second, and both ATIF and ADP serialize the + first, which is why their documents read as though every tool was called with + no arguments. +2. **Absence must be declared.** A `None` argument map without a matching + `LossRecord` is an *invalid* trace (`validate_trace`, invariant 7). This is + what makes the loss report a contract instead of documentation. +3. **Normalization is never destructive.** `TraceEvent.source_type` keeps the + source's own type string next to the normalized `kind`, and + `ToolCall.name_semantics` records that an ACP `kind` is a category rather + than a function name — the normalization §5.1 currently performs silently. + +### 8.3 Planned mapping + +**PROPOSAL — none of these converters is implemented.** Slice B is the IR +alone; `ACP → IR` and `ATIF ↔ IR` are deliberately not in it, so the +representation can be reviewed before anything depends on it. + +#### ACP-session capture events → IR + +| Capture event (§2.2) | IR | Loss recorded | +|---|---|---| +| `user_message` | `kind=user_message`, `role=user`, `text` | — | +| `agent_message` | `kind=agent_message`, `role=agent`, `text` | — | +| `agent_thought` | `kind=agent_reasoning`, `reasoning_segments=[text]` | none — the segment list is what preserves loss #10 | +| `tool_call` | `kind=tool_call` + `ToolCall`; `kind`→`name` with `name_semantics="acp_kind"` | `arguments` → UNSUPPORTED (#1); `started_at`/`finished_at` → UNSUPPORTED (#3) | +| `tool_call.content`, text block | `ContentBlock(kind=text)` | — | +| `tool_call.content`, other blocks | `ContentBlock(kind=opaque, raw=…)` | none — carrying the block verbatim is how loss #5 stops being a loss | +| `agent_timeout` | `kind=timeout`, `outcome="wall_clock_timeout"`, the rest in `extensions` | none — loss #4 becomes representable | +| `oracle` record (§2.4) | `kind=oracle`, `role=oracle` | none — no `[oracle: …]` string prefix, so a consumer need not string-match | +| unrecognized `type` | `kind=unknown` with `source_type` verbatim | — | + +#### IR → ATIF + +| IR | ATIF | Loss to record | +|---|---|---| +| `events[].text` | `step.message` | — | +| `reasoning` / `reasoning_segments` | `reasoning_content` (joined) | NORMALIZED — the segmentation does not survive | +| `tool_call.name` | `function_name` | NORMALIZED — an ACP kind is not a function name | +| `tool_call.call_id` empty or absent | synthesized `call_{n}` | SYNTHESIZED | +| `tool_call.status` / `title` | `tool_calls[].extra` | NORMALIZED | +| `agent.agent_version` absent | `"unknown"` | SYNTHESIZED | +| `kind=timeout` | *(no ATIF representation)* | DROPPED | +| opaque content blocks | *(no ATIF representation)* | DROPPED | + +#### IR ↔ OpenTelemetry + +**PROPOSAL, and unverified.** §4.2 is FACT — there is no OTel dependency and no +`gen_ai.*` handling in this repository — so the mapping below is a sketch +against the OpenTelemetry GenAI semantic conventions **as remembered, not as +vendored**: no version of that specification is pinned in this repo, nothing +validates against it, and the conventions have been changing. It is written down +to show the hub is reachable from OTel, not as a design to implement as-is. + +| IR | OTel sketch | +|---|---| +| `CanonicalTrace` | one root span per rollout | +| `trace_id` | the OTel trace id — `None` until a source provides one; never invented | +| `events[]` ordering | child span order / span events | +| `tool_call` | a child span; `name` → `gen_ai.tool.name`, `call_id` → `gen_ai.tool.call.id` | +| `usage` | `gen_ai.usage.input_tokens` / `.output_tokens` | +| `agent.model` | `gen_ai.request.model` | +| `outcome.status` | span status | +| `extensions` | span attributes, verbatim | + +Which direction is wanted — receiver, emitter, or both — is **open question 5** +and stays open. The IR is symmetric on purpose so the answer does not invalidate +it. + +### 8.4 A worked example + +**FACT.** The document below is produced by the models in `ir.py` and compared +against this block by `test_the_documented_example_matches_the_models`, so it +cannot drift from what the code emits. + +It shows the four properties the design exists for: an unavailable value that is +absent *and* declared (`arguments`), a thought whose boundaries survive next to +the joined form, a non-text content block carried as `opaque` instead of +skipped, and a timeout that is representable at all — with its source-specific +fields in `extensions` rather than as four new IR fields. + + +```json +{ + "ir_version": "bf-trace-ir-v0", + "session_id": "rollout-7f3a", + "agent": { + "agent_name": "gemini", + "model": "gemini-2.5-flash" + }, + "events": [ + { + "index": 0, + "kind": "user_message", + "source_type": "user_message", + "role": "user", + "text": "Count the rows in data.csv", + "provenance": { + "source_format": "acp-capture-v1", + "producer": "_events_to_trajectory" + }, + "extensions": {} + }, + { + "index": 1, + "kind": "tool_call", + "source_type": "tool_call", + "role": "agent", + "reasoning": "Check the file first.\n\nThen count.", + "reasoning_segments": [ + "Check the file first.", + "Then count." + ], + "tool_call": { + "call_id": "tc_1", + "name": "execute", + "name_semantics": "acp_kind", + "title": "wc -l data.csv", + "status": "completed", + "content": [ + { + "kind": "text", + "text": "42 data.csv", + "raw": { + "type": "content", + "content": { + "type": "text", + "text": "42 data.csv" + } + } + }, + { + "kind": "opaque", + "raw": { + "type": "diff", + "path": "/w/data.csv", + "oldText": "a", + "newText": "b" + } + } + ] + }, + "provenance": { + "source_format": "acp-capture-v1", + "producer": "_events_to_trajectory" + }, + "extensions": {} + }, + { + "index": 2, + "kind": "timeout", + "source_type": "agent_timeout", + "outcome": "wall_clock_timeout", + "provenance": { + "source_format": "acp-capture-v1", + "producer": "_events_to_trajectory" + }, + "extensions": { + "timeout_sec": 90.0, + "pending_tool_call_ids": [], + "terminal_trajectory_complete": true + } + } + ], + "usage": { + "input_tokens": 1180, + "output_tokens": 96, + "total_tokens": 1276, + "source": "llm_proxy_normalized" + }, + "outcome": { + "status": "timeout" + }, + "provenance": { + "source_format": "acp-capture-v1", + "producer": "_events_to_trajectory" + }, + "extensions": {}, + "losses": { + "direction": "acp->ir", + "ir_version": "bf-trace-ir-v0", + "records": [ + { + "field": "events[1].tool_call.arguments", + "loss_class": "unsupported", + "detail": "ACPSession.handle_update reads five fields and rawInput is not one of them, so no ACP-derived tool call carries arguments.", + "doc_ref": "§5 loss #1" + }, + { + "field": "events[1].tool_call.started_at", + "loss_class": "unsupported", + "detail": "ToolCallRecord tracks started_at/finished_at in memory and _events_to_trajectory serializes neither.", + "doc_ref": "§5 loss #3" + } + ] + } +} +``` -That idea is recorded here only so this document is not silently read as an -argument for it. **It has not been reviewed by a maintainer, no design has been -agreed, and nothing in this repository implements it.** Any such work depends on -the open questions in §6 being answered first — in particular whether the capture -path may change at all, since several of the losses in §5 cannot be closed -downstream of it. +### 8.5 Invariants + +**PROPOSAL.** Checked by `validate_trace`, which returns one string per +violation rather than raising, and asserted in +`tests/trajectories/test_trace_ir.py` with both a violating and a clean trace +for each: + +1. `ir_version` equals `bf-trace-ir-v0` — v0 defines no migration path. +2. `events[i].index == i` — dense and ordered, so a dropped event leaves a hole + rather than disappearing. +3. `kind == tool_call` if and only if a `tool_call` payload is present. +4. A `text` block carries text; an `opaque` block carries `raw`. An opaque block + with no payload would be a dropped block claiming preservation. +5. When both are present, `"\n\n".join(reasoning_segments) == reasoning` — the + segments are a strictly richer encoding of the same value, never a second + divergent one. +6. An `agent_reasoning` event carries reasoning in one of the two fields. +7. **Every `arguments is None` has a matching loss record.** Absence is declared, + never silent. +8. A `user_message` is attributed to `user` or to nobody. Agent-side attribution + is genuinely ambiguous today (§5.1, the `oracle` divergence) and the IR does + not pretend to settle it. + +Two further properties are pinned by tests rather than by `validate_trace`: the +IR's tool-status vocabulary is a superset of the ACP `ToolCallStatus` enum, read +off the enum itself; and every event type `_events_to_trajectory` emits maps to +an `EventKind`, with the producer's vocabulary read from source by AST — the +same mechanism the Slice A conformance suite uses (§2.2). + +### 8.6 What is provisional, and what a review can still change + +Everything in §8 is provisional. In particular, a review can reject any of the +following without any other work having to be undone, because nothing depends on +them: + +- **the hub itself** — if direct converters are preferred (open question 1), the + module is deleted and the loss taxonomy survives as documentation; +- **the tri-state / declared-absence contract** — the strongest opinion here, + and the one most likely to feel heavy in a converter; +- **`name_semantics` and `reasoning_segments`** — both exist to preserve a + distinction the current exporters discard; if that distinction is not wanted, + both fields go and losses #10 and the `kind`→`function_name` normalization + stay as they are; +- **`extensions` as the escape hatch** — the alternative is a field per source + quirk, which is how a pragmatic superset becomes a spec; +- **`TraceUsage.source`** — it exists only because open question 4 is open. If a + canonical definition of `input_tokens` is chosen, the field can go; +- **the version string and the absence of migration machinery**; +- **the module's location** (`benchflow.trajectories.ir`) and every name in it. + +What a review cannot change by rejecting the IR: the losses in §5 are properties +of the current code, not of this proposal, and they remain whatever happens +here. + +### 8.7 Status + +- **Implemented:** the IR types, the loss model, the invariants, the validation + suite and this section. +- **Not implemented, deliberately:** `ACP → IR`, `ATIF ↔ IR`, any OTel work, any + wiring into a run path, any on-disk artifact. +- **Unchanged:** every existing format, exporter, artifact and code path. diff --git a/src/benchflow/trajectories/ir.py b/src/benchflow/trajectories/ir.py new file mode 100644 index 000000000..a6716cb3c --- /dev/null +++ b/src/benchflow/trajectories/ir.py @@ -0,0 +1,532 @@ +"""Canonical Trace IR — the provisional hub representation for trace interop. + +> **PROVISIONAL — v0.** This module is a proposal implemented in code so it can +> be reviewed as code. No maintainer has approved this direction; see +> `docs/trace-interop.md` §8. Nothing imports it, nothing writes it to disk, and +> no existing format changes because it exists. It is deliberately reversible: +> deleting this file and its test restores the tree to its previous behaviour. + +## Why a hub at all + +BenchFlow has four trace-shaped representations in play — the ACP-session +capture events (`acp_trajectory.jsonl`), ATIF (`trainer/atif.json`), ADP +(`trainer/adp.jsonl`) and the Verifiers/ORS record — and OpenTelemetry is the +obvious fifth. Pairwise converters cost ``N*(N-1)`` edges and, worse, give every +edge its own private answer to the same questions: what happens to a tool call +with no arguments, how a thought boundary is preserved, whether a timeout is +representable. Those answers already diverge today (`docs/trace-interop.md` §5). + +A hub makes each format's conversion one edge to a written contract, and makes +the information loss a *value* rather than a comment — see :class:`LossReport`. + +## The rule this module is built on + +**The IR is a pragmatic superset of what BenchFlow can actually observe, not a +model of what an agent trace could theoretically contain.** Every field below +exists because some source in this repository carries the value today, or +because an adjacent format has a required slot for it. Where a value is not +observable, the IR carries ``None`` *and requires a matching loss record* — +absence is declared, never silent (see :func:`validate_trace`). + +Consequently the IR does **not** invent: tool arguments (the ACP capture path +never reads ``rawInput``), per-event timestamps for sources that carry none, +agent versions, synthetic tool-call ids, or OTel span/trace ids. Those are +target-side concerns and belong in converters, which record them as +:attr:`LossClass.SYNTHESIZED`. + +## Tri-state fields + +For an optional value, the IR distinguishes three states, and converters must +preserve the distinction: + +- a value — observed in the source; +- ``None`` — not available from this source (a loss record says why); +- an empty value — observed *and* empty (``{}``, ``""``, ``[]``). + +``arguments={}`` ("the source captured an empty argument map") and +``arguments=None`` ("the source never carried arguments") are different facts. +Today every ACP-derived tool call is the second; ATIF and ADP both serialize the +first, which is why their documents read as though the agent called every tool +with no arguments. +""" + +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +# ``v0`` is a statement, not a placeholder: the shape is unreviewed and carries +# no compatibility promise. There is deliberately no migration machinery — a v1 +# would be a new constant and a converter, not a mutation of this one. Named so +# it can never be confused with ``ATIF-v1.7`` or ADP's ``1.3.1``. +TRACE_IR_VERSION = "bf-trace-ir-v0" + + +class LossClass(StrEnum): + """Why a value is not in the IR (or not in a target document). + + The distinction between the first two is the one that decides *where a fix + would have to land*, and it is the same taxonomy `docs/trace-interop.md` + §5.1 already uses for `ACP -> ATIF`. + """ + + UNSUPPORTED = "unsupported" + """The source never carried the value. Not fixable in a converter.""" + + DROPPED = "dropped" + """The source carried it and the conversion discarded it. Fixable here.""" + + NORMALIZED = "normalized" + """Carried, but relocated or reshaped — readable only with convention + knowledge (an ACP ``kind`` landing in an ATIF ``function_name`` slot).""" + + SYNTHESIZED = "synthesized" + """The *target* required a value the source did not have, and the converter + produced one. Recorded so a fabricated value is never mistaken for an + observed one (ATIF's ``call_{n}`` ids, ADP's ``call_NNNNNN``).""" + + +class LossRecord(BaseModel): + """One declared information loss, addressed to a field.""" + + model_config = ConfigDict(extra="forbid") + + field: str + """Dotted path in IR terms, e.g. ``events[3].tool_call.arguments``. The + path addresses the IR, not the source document, so one vocabulary covers + every direction.""" + + loss_class: LossClass + detail: str + """Why, in one sentence, naming the responsible symbol where possible.""" + + doc_ref: str | None = None + """Anchor into ``docs/trace-interop.md`` (e.g. ``"§5 loss #1"``).""" + + +class LossReport(BaseModel): + """The explicit, typed result of one conversion. + + A conversion returns a trace *and* this. A converter that carries nothing + across still produces a report; an empty report is a claim ("nothing was + lost"), not a default. + """ + + model_config = ConfigDict(extra="forbid") + + direction: str + """``"acp->ir"``, ``"ir->atif"``, … — free text, one edge of the hub.""" + + ir_version: str = TRACE_IR_VERSION + records: list[LossRecord] = Field(default_factory=list) + + def add( + self, + field: str, + loss_class: LossClass, + detail: str, + doc_ref: str | None = None, + ) -> None: + self.records.append( + LossRecord( + field=field, loss_class=loss_class, detail=detail, doc_ref=doc_ref + ) + ) + + def by_class(self, loss_class: LossClass) -> list[LossRecord]: + return [r for r in self.records if r.loss_class is loss_class] + + def for_field(self, field: str) -> list[LossRecord]: + return [r for r in self.records if r.field == field] + + @property + def lossless(self) -> bool: + """True only when the conversion declared no loss of any class.""" + return not self.records + + +class Provenance(BaseModel): + """Where the values in a trace or event came from. + + Kept per-event as well as per-trace because a single `acp_trajectory.jsonl` + can hold records from more than one producer (`docs/trace-interop.md` §2.4), + so a trace-level answer would be wrong for some of its own events. + """ + + model_config = ConfigDict(extra="forbid") + + source_format: str + """``"acp-capture-v1"``, ``"atif"``, ``"adp"``, ``"otel"``, ``"oracle"``…""" + + producer: str | None = None + """The emitting symbol when known (``"_events_to_trajectory"``).""" + + captured_at: datetime | None = None + + +class Role(StrEnum): + """Who a step is attributable to. + + Exactly the four values some source in this repository already + distinguishes: ATIF's validator accepts ``user``/``agent``/``oracle``, ADP + uses ``user``/``environment``. No ``system`` member — no producer here emits + one, and inventing it would be inventing semantics. + """ + + USER = "user" + AGENT = "agent" + ENVIRONMENT = "environment" + ORACLE = "oracle" + + +class EventKind(StrEnum): + """What an event *is*, normalized across sources. + + ``UNKNOWN`` is load-bearing: the ACP trajectory has an open type vocabulary + (`docs/trace-interop.md` §2.4, §7) and today every exporter silently skips + what it does not recognize. An unrecognized record becomes ``UNKNOWN`` with + :attr:`TraceEvent.source_type` holding the original string, so it survives + conversion instead of vanishing. + """ + + USER_MESSAGE = "user_message" + AGENT_MESSAGE = "agent_message" + AGENT_REASONING = "agent_reasoning" + TOOL_CALL = "tool_call" + TIMEOUT = "timeout" + ORACLE = "oracle" + UNKNOWN = "unknown" + + +class ToolStatus(StrEnum): + """Tool-call lifecycle status. + + Mirrors the ACP ``ToolCallStatus`` vocabulary, plus ``UNKNOWN`` for sources + that carry no status at all (ADP drops it outright — §5 loss #2). The + superset relationship is pinned by a test rather than by an import, so the + IR does not take a runtime dependency on the ACP layer it is supposed to be + neutral about. + """ + + PENDING = "pending" + IN_PROGRESS = "in_progress" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + UNKNOWN = "unknown" + + +class ContentBlockKind(StrEnum): + """Two kinds, because two are all BenchFlow interprets today. + + ``content_blocks_to_text`` renders text blocks and skips everything else, + which is §5 loss #5. ``OPAQUE`` is how the IR stops that from being a loss: + the block is not understood, but it is carried. + """ + + TEXT = "text" + OPAQUE = "opaque" + + +class ContentBlock(BaseModel): + """One block of captured tool output. + + ``raw`` holds the source block verbatim and is required for ``OPAQUE``. It + is also kept for ``TEXT`` when available, so a round-trip can reproduce the + original block rather than a re-serialization of its text. + """ + + model_config = ConfigDict(extra="forbid") + + kind: ContentBlockKind + text: str | None = None + raw: dict[str, Any] | None = None + + +class ToolCall(BaseModel): + """One tool invocation and its captured result.""" + + model_config = ConfigDict(extra="forbid") + + call_id: str | None = None + """The id as observed. ``""`` is a real captured value (``handle_update`` + defaults to it); ``None`` means the source carried no id field at all. + Converters that need a unique id synthesize one and record + :attr:`LossClass.SYNTHESIZED` — the IR does not.""" + + name: str | None = None + """The tool's name *as the source labels it*.""" + + name_semantics: str | None = None + """What :attr:`name` actually is — ``"acp_kind"``, ``"function_name"`` or + ``"span_name"``. An ACP ``kind`` is a category (``execute``, ``read``), not + a function name, and ATIF today puts it in a ``function_name`` slot. Naming + the semantics keeps the IR from laundering that normalization into a fact.""" + + title: str | None = None + """Human-readable label. For ACP ``execute`` calls, conventionally the + command line — which is the only place the command survives today.""" + + status: ToolStatus | None = None + + arguments: dict[str, Any] | None = None + """``None`` = the source never carried arguments (every ACP-derived call + today); ``{}`` = the source carried an empty argument map. See the module + docstring; :func:`validate_trace` requires a loss record for the ``None`` + case.""" + + content: list[ContentBlock] = Field(default_factory=list) + started_at: datetime | None = None + finished_at: datetime | None = None + """``ToolCallRecord`` tracks both in memory and ``_events_to_trajectory`` + serializes neither (§5 loss #3), so both are ``None`` for anything read off + disk today. The slots exist because the values demonstrably exist upstream, + not because a target format wants them.""" + + +class TraceUsage(BaseModel): + """Token accounting, with the definition it was computed under. + + Named :class:`TraceUsage` rather than ``TokenUsage`` to stay distinct from + ``benchflow.trajectories.types.TokenUsage``, which is the proxy-capture + dataclass. + + ``source`` exists because open question 4 in `docs/trace-interop.md` §6 is + unanswered: ``_exchange_token_usage`` (cross-provider normalized, cache + folded into input) and ``normalize_acp_usage`` (the raw ACP snapshot) do not + mean the same thing. The IR refuses to pick one and instead records which + was used — a consumer can then compare like with like. Documented values: + ``"llm_proxy_normalized"``, ``"acp_session_snapshot"``. + """ + + model_config = ConfigDict(extra="forbid") + + input_tokens: int | None = None + output_tokens: int | None = None + cache_read_tokens: int | None = None + cache_creation_tokens: int | None = None + reasoning_tokens: int | None = None + total_tokens: int | None = None + source: str | None = None + + +class TraceEvent(BaseModel): + """One ordered step of a trace.""" + + model_config = ConfigDict(extra="forbid") + + index: int + """Position in the source sequence, dense from 0. Ordering is the one + property every format in play preserves, so the IR makes it structural.""" + + kind: EventKind + + source_type: str | None = None + """The source's own type string, verbatim (``"agent_thought"``, + ``"oracle"``, an unrecognized future value). Kept next to the normalized + :attr:`kind` so normalization is never destructive.""" + + role: Role | None = None + + text: str | None = None + """User-visible text. ``""`` is meaningful: text-empty events exist and are + dropped by both exporters today (§5.1).""" + + reasoning: str | None = None + """Internal reasoning, kept in its own field rather than merged into + :attr:`text` — ATIF and ADP both have a separate ``reasoning_content``.""" + + reasoning_segments: list[str] | None = None + """The individual thought events, when the source had boundaries. + + ``ThoughtBuffer.take`` joins thoughts with a blank line, so a thought + containing a blank line and two consecutive thoughts serialize identically + and the event count is unrecoverable (§5 loss #10). The segment list is how + the IR keeps the boundary that the join destroys; :func:`validate_trace` + checks the two stay consistent.""" + + tool_call: ToolCall | None = None + + started_at: datetime | None = None + finished_at: datetime | None = None + + outcome: str | None = None + """Terminal signal carried by the event itself — e.g. the + ``wall_clock_timeout`` reason of an ``agent_timeout`` record, which no + exporter represents today (§5 loss #4).""" + + usage: TraceUsage | None = None + """Per-event usage. ``None`` for everything ACP-derived: ACP events carry no + usage and ``usage_snapshots`` is routed to ``result.json`` instead (§5 + losses #6, #8).""" + + provenance: Provenance + extensions: dict[str, Any] = Field(default_factory=dict) + """Source fields with no IR home, carried verbatim so a conversion is not + forced to choose between dropping a value and growing the IR for it. Values + must be JSON-serializable.""" + + +class ModelInfo(BaseModel): + """Agent / model identity, as observed. + + ``agent_version`` is ``None`` when unknown. It is *not* ``"unknown"``: + ``trajectory_to_atif_record`` hardcodes that string because ATIF requires + the field, which is a target-side obligation and belongs in that converter, + recorded as :attr:`LossClass.SYNTHESIZED`. + """ + + model_config = ConfigDict(extra="forbid") + + agent_name: str | None = None + agent_version: str | None = None + model: str | None = None + provider: str | None = None + + +class OutcomeStatus(StrEnum): + COMPLETED = "completed" + FAILED = "failed" + TIMEOUT = "timeout" + CANCELLED = "cancelled" + UNKNOWN = "unknown" + + +class TraceOutcome(BaseModel): + """How the run ended. + + ``stop_reason`` is captured on ``ACPSession`` and exported nowhere (§5 loss + #9); ``reward`` and ``error_category`` live in ``result.json``, outside the + trajectory. All optional — a trace built from the capture file alone has + none of them, and that is a declarable loss rather than a reason to fake a + value. + """ + + model_config = ConfigDict(extra="forbid") + + status: OutcomeStatus | None = None + stop_reason: str | None = None + reward: float | None = None + error_category: str | None = None + + +class CanonicalTrace(BaseModel): + """One agent run, in the hub representation.""" + + model_config = ConfigDict(extra="forbid") + + ir_version: str = TRACE_IR_VERSION + trace_id: str | None = None + """``None`` until a source supplies one. No OTel-shaped id is invented.""" + + session_id: str | None = None + agent: ModelInfo = Field(default_factory=ModelInfo) + started_at: datetime | None = None + finished_at: datetime | None = None + events: list[TraceEvent] = Field(default_factory=list) + usage: TraceUsage | None = None + outcome: TraceOutcome | None = None + provenance: Provenance + extensions: dict[str, Any] = Field(default_factory=dict) + + losses: LossReport | None = None + """The report from the conversion that produced this trace. Attached rather + than returned alongside so a trace cannot be passed around separated from + the record of what building it cost.""" + + +def validate_trace(trace: CanonicalTrace) -> list[str]: + """Check the IR invariants, returning one string per violation. + + Returns a list rather than raising: a caller checking a batch wants every + problem, and a test asserting an invariant wants to name it. An empty list + means every invariant below holds. + + The invariants, and why each one is here: + + 1. **Version.** ``ir_version`` equals :data:`TRACE_IR_VERSION`. v0 has no + migration path, so a mismatch is an error rather than an upgrade. + 2. **Dense ordering.** ``events[i].index == i``. Ordering is the property + every format preserves; making it structural means a converter cannot + silently drop an event without leaving a hole. + 3. **Kind/payload agreement.** ``kind is TOOL_CALL`` iff ``tool_call`` is + set. Without this the IR would have two ways to say the same thing. + 4. **Content-block integrity.** A ``TEXT`` block carries ``text``; an + ``OPAQUE`` block carries ``raw``. An opaque block with no payload is a + block that was dropped while claiming to have been preserved. + 5. **Reasoning consistency.** When both ``reasoning`` and + ``reasoning_segments`` are set, joining the segments with a blank line + reproduces ``reasoning`` exactly — the join `ThoughtBuffer` performs. The + segments are then a strictly richer encoding of the same value, not a + second, divergent one. + 6. **Reasoning presence.** An ``AGENT_REASONING`` event carries reasoning in + one of the two fields. + 7. **No silent absence.** Every ``arguments is None`` has a matching loss + record at ``events[i].tool_call.arguments``. This is the invariant that + makes the loss report a contract instead of documentation: a converter + that quietly fails to carry arguments produces an invalid trace. + 8. **Role coherence.** A ``USER_MESSAGE`` is attributed to ``USER`` or to + nobody. The IR checks only this direction; agent-side attribution is + genuinely ambiguous today (§5, the ``oracle`` divergence) and the IR does + not pretend to settle it. + """ + issues: list[str] = [] + + if trace.ir_version != TRACE_IR_VERSION: + issues.append( + f"ir_version {trace.ir_version!r} != {TRACE_IR_VERSION!r}; " + "v0 defines no migration" + ) + + declared_losses = {r.field for r in (trace.losses.records if trace.losses else [])} + + for position, event in enumerate(trace.events): + where = f"events[{position}]" + + if event.index != position: + issues.append(f"{where}.index is {event.index}, expected {position}") + + is_tool = event.kind is EventKind.TOOL_CALL + if is_tool and event.tool_call is None: + issues.append(f"{where} is a tool_call event with no tool_call payload") + if not is_tool and event.tool_call is not None: + issues.append( + f"{where} carries a tool_call payload but kind is {event.kind.value}" + ) + + for block_position, block in enumerate( + event.tool_call.content if event.tool_call else [] + ): + block_where = f"{where}.tool_call.content[{block_position}]" + if block.kind is ContentBlockKind.TEXT and block.text is None: + issues.append(f"{block_where} is text but carries no text") + if block.kind is ContentBlockKind.OPAQUE and block.raw is None: + issues.append(f"{block_where} is opaque but carries no raw block") + + if event.reasoning is not None and event.reasoning_segments is not None: + joined = "\n\n".join(event.reasoning_segments) + if joined != event.reasoning: + issues.append(f"{where}.reasoning_segments do not join to .reasoning") + if event.kind is EventKind.AGENT_REASONING and not ( + event.reasoning or event.reasoning_segments + ): + issues.append(f"{where} is agent_reasoning but carries no reasoning") + + if event.tool_call is not None and event.tool_call.arguments is None: + field = f"{where}.tool_call.arguments" + if field not in declared_losses: + issues.append( + f"{field} is None with no loss record; absence must be declared" + ) + + if ( + event.kind is EventKind.USER_MESSAGE + and event.role is not None + and event.role is not Role.USER + ): + issues.append(f"{where} is a user_message attributed to {event.role.value}") + + return issues diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py new file mode 100644 index 000000000..14cb2b544 --- /dev/null +++ b/tests/trajectories/test_trace_ir.py @@ -0,0 +1,618 @@ +"""Validation suite for the provisional Canonical Trace IR (Slice B). + +The IR (:mod:`benchflow.trajectories.ir`) is a *proposal implemented as code*. +Nothing imports it and nothing writes it to disk, so the usual safety net — a +downstream consumer breaking — does not exist for it. This suite is that net: +it pins the invariants the module documents, the vocabularies it must stay +aligned with, and the isolation claim that makes it reversible. + +Four groups, in the order they matter: + +* **Invariants** — one test per rule in :func:`validate_trace`, each with a + violating trace *and* a clean one, so a rule that stopped firing fails here + rather than passing silently. +* **Vocabulary alignment** — the IR's tool-status and event-kind vocabularies + are checked against the real producer (``_events_to_trajectory``) and the real + ACP enum, both read from source. Adding a branch to the capture path fails + this suite until the IR accounts for it, exactly as the Slice A conformance + suite does for the schema. +* **Isolation** — nothing under ``src/benchflow`` imports the IR. This is the + executable form of "zero runtime behaviour change"; if a future PR wires the + IR into a run path, this test is the one that must be deliberately updated. +* **The worked example** — the JSON in ``docs/trace-interop.md`` §8 is built + here in code and compared to the block in the document, so the documented + example cannot drift from what the models actually produce. + +Nothing here writes outside ``tmp_path`` and no runtime module is imported for +its side effects. +""" + +from __future__ import annotations + +import ast +import json +import re +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from benchflow.acp.types import ToolCallStatus +from benchflow.trajectories import ir as ir_module +from benchflow.trajectories.ir import ( + TRACE_IR_VERSION, + CanonicalTrace, + ContentBlock, + ContentBlockKind, + EventKind, + LossClass, + LossReport, + ModelInfo, + OutcomeStatus, + Provenance, + Role, + ToolCall, + ToolStatus, + TraceEvent, + TraceOutcome, + TraceUsage, + validate_trace, +) +from tests.trajectories.test_acp_capture_event_schema import _emitted_event_types + +REPO_ROOT = Path(__file__).resolve().parents[2] +SRC_ROOT = REPO_ROOT / "src" / "benchflow" +DOCS_PATH = REPO_ROOT / "docs" / "trace-interop.md" + +ACP_PROVENANCE = Provenance( + source_format="acp-capture-v1", producer="_events_to_trajectory" +) + + +def _event(index: int, kind: EventKind, **kwargs) -> TraceEvent: + """A minimally-populated event, so each test states only what it is about.""" + return TraceEvent(index=index, kind=kind, provenance=ACP_PROVENANCE, **kwargs) + + +def _trace(*events: TraceEvent, losses: LossReport | None = None) -> CanonicalTrace: + return CanonicalTrace(provenance=ACP_PROVENANCE, events=list(events), losses=losses) + + +def _tool_call(**kwargs) -> ToolCall: + """A tool call whose ``arguments`` absence is already declared elsewhere.""" + kwargs.setdefault("arguments", {}) + return ToolCall(**kwargs) + + +# --------------------------------------------------------------------------- +# Versioning +# --------------------------------------------------------------------------- + + +def test_version_constant_is_explicitly_v0(): + """v0 is a claim about stability, so the string is pinned, not derived.""" + assert TRACE_IR_VERSION == "bf-trace-ir-v0" + assert CanonicalTrace(provenance=ACP_PROVENANCE).ir_version == TRACE_IR_VERSION + assert LossReport(direction="acp->ir").ir_version == TRACE_IR_VERSION + + +def test_mismatched_ir_version_is_an_invariant_violation(): + trace = _trace() + trace.ir_version = "bf-trace-ir-v1" + issues = validate_trace(trace) + assert any("ir_version" in issue for issue in issues), issues + + +# --------------------------------------------------------------------------- +# Invariants +# --------------------------------------------------------------------------- + + +def test_dense_ordering_is_required(): + clean = _trace( + _event(0, EventKind.AGENT_MESSAGE, text="a"), + _event(1, EventKind.AGENT_MESSAGE, text="b"), + ) + assert validate_trace(clean) == [] + + holed = _trace( + _event(0, EventKind.AGENT_MESSAGE, text="a"), + _event(2, EventKind.AGENT_MESSAGE, text="b"), + ) + assert any("index is 2, expected 1" in issue for issue in validate_trace(holed)) + + +def test_tool_call_kind_and_payload_must_agree_both_ways(): + missing_payload = _trace(_event(0, EventKind.TOOL_CALL)) + assert any( + "no tool_call payload" in issue for issue in validate_trace(missing_payload) + ) + + stray_payload = _trace( + _event(0, EventKind.AGENT_MESSAGE, text="hi", tool_call=_tool_call()) + ) + assert any( + "carries a tool_call payload" in issue + for issue in validate_trace(stray_payload) + ) + + agreed = _trace(_event(0, EventKind.TOOL_CALL, tool_call=_tool_call())) + assert validate_trace(agreed) == [] + + +def test_content_blocks_must_carry_their_payload(): + """An opaque block with no ``raw`` is a dropped block claiming preservation.""" + empty_text = _trace( + _event( + 0, + EventKind.TOOL_CALL, + tool_call=_tool_call(content=[ContentBlock(kind=ContentBlockKind.TEXT)]), + ) + ) + assert any("is text but carries no text" in i for i in validate_trace(empty_text)) + + empty_opaque = _trace( + _event( + 0, + EventKind.TOOL_CALL, + tool_call=_tool_call(content=[ContentBlock(kind=ContentBlockKind.OPAQUE)]), + ) + ) + assert any( + "is opaque but carries no raw block" in i for i in validate_trace(empty_opaque) + ) + + carried = _trace( + _event( + 0, + EventKind.TOOL_CALL, + tool_call=_tool_call( + content=[ + ContentBlock(kind=ContentBlockKind.TEXT, text=""), + ContentBlock( + kind=ContentBlockKind.OPAQUE, + raw={"type": "diff", "oldText": "a", "newText": "b"}, + ), + ] + ), + ) + ) + assert validate_trace(carried) == [] + + +def test_a_text_block_may_be_empty_but_not_absent(): + """``""`` is an observed value; ``None`` means the block was not carried.""" + present = ContentBlock(kind=ContentBlockKind.TEXT, text="") + assert present.text == "" + assert ( + validate_trace( + _trace( + _event(0, EventKind.TOOL_CALL, tool_call=_tool_call(content=[present])) + ) + ) + == [] + ) + + +def test_reasoning_segments_must_join_to_reasoning(): + """The join is ``ThoughtBuffer``'s; the segments are the boundary it destroys.""" + consistent = _trace( + _event( + 0, + EventKind.AGENT_REASONING, + reasoning="first\n\nsecond", + reasoning_segments=["first", "second"], + ) + ) + assert validate_trace(consistent) == [] + + # The §5 loss #10 shape: one thought that already contains a blank line. + # Same joined string, different segmentation — and the IR keeps them apart. + single = _trace( + _event( + 0, + EventKind.AGENT_REASONING, + reasoning="first\n\nsecond", + reasoning_segments=["first\n\nsecond"], + ) + ) + assert validate_trace(single) == [] + assert consistent.events[0].reasoning == single.events[0].reasoning + assert ( + consistent.events[0].reasoning_segments != single.events[0].reasoning_segments + ) + + divergent = _trace( + _event( + 0, + EventKind.AGENT_REASONING, + reasoning="first\n\nsecond", + reasoning_segments=["totally", "different"], + ) + ) + assert any("do not join" in issue for issue in validate_trace(divergent)) + + +def test_a_reasoning_event_must_carry_reasoning(): + assert any( + "carries no reasoning" in issue + for issue in validate_trace(_trace(_event(0, EventKind.AGENT_REASONING))) + ) + assert ( + validate_trace( + _trace(_event(0, EventKind.AGENT_REASONING, reasoning_segments=["only"])) + ) + == [] + ) + + +def test_absent_arguments_require_a_declared_loss(): + """The invariant that makes the loss report a contract rather than a comment.""" + undeclared = _trace( + _event(0, EventKind.TOOL_CALL, tool_call=ToolCall(arguments=None)) + ) + issues = validate_trace(undeclared) + assert any("absence must be declared" in issue for issue in issues), issues + + losses = LossReport(direction="acp->ir") + losses.add( + "events[0].tool_call.arguments", + LossClass.UNSUPPORTED, + "handle_update never reads rawInput", + "§5 loss #1", + ) + declared = _trace( + _event(0, EventKind.TOOL_CALL, tool_call=ToolCall(arguments=None)), + losses=losses, + ) + assert validate_trace(declared) == [] + + +def test_empty_arguments_are_not_absent_arguments(): + """``{}`` needs no loss record: it is an observation, not a gap.""" + captured_empty = _trace( + _event(0, EventKind.TOOL_CALL, tool_call=ToolCall(arguments={})) + ) + assert validate_trace(captured_empty) == [] + assert captured_empty.events[0].tool_call.arguments == {} + + +def test_a_user_message_is_not_attributed_to_the_agent(): + assert any( + "attributed to agent" in issue + for issue in validate_trace( + _trace(_event(0, EventKind.USER_MESSAGE, text="hi", role=Role.AGENT)) + ) + ) + assert ( + validate_trace( + _trace(_event(0, EventKind.USER_MESSAGE, text="hi", role=Role.USER)) + ) + == [] + ) + assert validate_trace(_trace(_event(0, EventKind.USER_MESSAGE, text="hi"))) == [] + + +def test_validate_trace_reports_every_violation_not_just_the_first(): + trace = _trace( + _event(5, EventKind.TOOL_CALL), + _event(9, EventKind.AGENT_REASONING), + ) + issues = validate_trace(trace) + assert len(issues) >= 4, issues + + +# --------------------------------------------------------------------------- +# Strictness and round-tripping +# --------------------------------------------------------------------------- + + +def test_unknown_fields_are_rejected_and_extensions_are_the_escape_hatch(): + """``extra="forbid"`` makes a typo a failure instead of a silent no-op.""" + with pytest.raises(ValidationError): + TraceEvent( + index=0, + kind=EventKind.AGENT_MESSAGE, + provenance=ACP_PROVENANCE, + reasonning="typo", + ) + + carried = _event( + 0, + EventKind.TIMEOUT, + outcome="wall_clock_timeout", + extensions={"timeout_sec": 90.0, "pending_tool_call_ids": []}, + ) + assert carried.extensions["timeout_sec"] == 90.0 + + +def test_a_trace_round_trips_through_json(): + original = _example_trace() + restored = CanonicalTrace.model_validate(json.loads(original.model_dump_json())) + assert restored == original + assert validate_trace(restored) == [] + + +def test_unknown_source_types_survive_as_unknown_rather_than_vanishing(): + """Today every exporter skips what it does not recognize (§5.1, last rows).""" + event = _event(0, EventKind.UNKNOWN, source_type="some_future_acp_event") + assert validate_trace(_trace(event)) == [] + assert event.source_type == "some_future_acp_event" + + +# --------------------------------------------------------------------------- +# The loss model +# --------------------------------------------------------------------------- + + +def test_loss_report_is_empty_only_as_a_claim(): + report = LossReport(direction="acp->ir") + assert report.lossless + + report.add("events[0].tool_call.arguments", LossClass.UNSUPPORTED, "no rawInput") + report.add("agent.agent_version", LossClass.SYNTHESIZED, "ATIF requires a version") + + assert not report.lossless + assert len(report.by_class(LossClass.UNSUPPORTED)) == 1 + assert len(report.by_class(LossClass.DROPPED)) == 0 + assert report.for_field("agent.agent_version")[0].loss_class is ( + LossClass.SYNTHESIZED + ) + + +def test_every_loss_class_of_the_documented_taxonomy_exists(): + """The four classes are the ones §5.1 already uses; renaming one is a change.""" + assert {c.value for c in LossClass} == { + "unsupported", + "dropped", + "normalized", + "synthesized", + } + + +# --------------------------------------------------------------------------- +# Vocabulary alignment with the real producer +# --------------------------------------------------------------------------- + + +def test_ir_tool_status_is_a_superset_of_the_acp_status_vocabulary(): + """Read off the ACP enum, so widening it fails here instead of silently.""" + acp_values = {status.value for status in ToolCallStatus} + ir_values = {status.value for status in ToolStatus} + assert acp_values <= ir_values, sorted(acp_values - ir_values) + assert "unknown" in ir_values, "sources that carry no status need a value" + + +def test_every_emitted_capture_event_type_maps_to_an_ir_kind(): + """The producer's vocabulary, read from ``_events_to_trajectory`` by AST. + + Mirrors the Slice A conformance check: adding a branch to the capture path + fails this test until the IR says what that event becomes. + """ + mapping = { + "user_message": EventKind.USER_MESSAGE, + "agent_message": EventKind.AGENT_MESSAGE, + "agent_thought": EventKind.AGENT_REASONING, + "tool_call": EventKind.TOOL_CALL, + "agent_timeout": EventKind.TIMEOUT, + } + emitted = _emitted_event_types() + assert emitted, "AST extraction found no event types; the walker is broken" + + unmapped = emitted - set(mapping) + assert not unmapped, ( + f"_events_to_trajectory emits {sorted(unmapped)}, which the IR does not " + "map to an EventKind. Add the mapping here and to docs/trace-interop.md " + "§8 rather than deleting this check." + ) + assert set(mapping.values()) <= set(EventKind) + + +def test_the_oracle_record_has_its_own_kind_and_role(): + """§2.4's alternative trajectory is representable without being called agent. + + ``acp_events_to_atif_steps`` renders it as an ``agent`` step prefixed + ``[oracle: …]``, which a consumer can only undo by string matching. The IR + keeps the distinction the ATIF validator already accepts. + """ + assert EventKind.ORACLE.value == "oracle" + assert Role.ORACLE.value == "oracle" + + +# --------------------------------------------------------------------------- +# Isolation — the executable form of "zero runtime behaviour change" +# --------------------------------------------------------------------------- + + +def _imports_the_ir(path: Path) -> bool: + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + if any(alias.name == "benchflow.trajectories.ir" for alias in node.names): + return True + elif isinstance(node, ast.ImportFrom): + module = node.module or "" + if module == "benchflow.trajectories.ir" or (node.level and module == "ir"): + return True + if module in ("benchflow.trajectories", "") and any( + alias.name == "ir" for alias in node.names + ): + return True + return False + + +def test_no_runtime_module_imports_the_ir(): + """Slice B is reversible: deleting ``ir.py`` cannot break a run path. + + When the IR is deliberately wired into production, this test is the one to + update — and updating it is the moment the reversibility claim ends. + """ + importers = [ + path.relative_to(REPO_ROOT).as_posix() + for path in sorted(SRC_ROOT.rglob("*.py")) + if _imports_the_ir(path) + ] + assert importers == [], importers + + +def test_the_ir_module_imports_no_benchflow_runtime_module(): + """The hub must not depend on the formats it is supposed to be neutral about.""" + tree = ast.parse(Path(ir_module.__file__).read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add(node.module or "") + assert not [name for name in imported if name.startswith("benchflow")], sorted( + imported + ) + + +# --------------------------------------------------------------------------- +# The worked example, shared with the documentation +# --------------------------------------------------------------------------- + + +def _example_trace() -> CanonicalTrace: + """The three-event trace documented in ``docs/trace-interop.md`` §8.4. + + Chosen to exercise the parts of the design that are actually load-bearing: + a tool call whose arguments are unavailable (declared, not silent), a + thought whose boundaries are kept alongside the joined form, a non-text + content block carried as opaque, and a timeout whose source-specific fields + ride in ``extensions`` instead of growing the IR a field each. + """ + losses = LossReport(direction="acp->ir") + losses.add( + "events[1].tool_call.arguments", + LossClass.UNSUPPORTED, + "ACPSession.handle_update reads five fields and rawInput is not one of " + "them, so no ACP-derived tool call carries arguments.", + "§5 loss #1", + ) + losses.add( + "events[1].tool_call.started_at", + LossClass.UNSUPPORTED, + "ToolCallRecord tracks started_at/finished_at in memory and " + "_events_to_trajectory serializes neither.", + "§5 loss #3", + ) + + return CanonicalTrace( + session_id="rollout-7f3a", + agent=ModelInfo(agent_name="gemini", model="gemini-2.5-flash"), + provenance=ACP_PROVENANCE, + events=[ + TraceEvent( + index=0, + kind=EventKind.USER_MESSAGE, + source_type="user_message", + role=Role.USER, + text="Count the rows in data.csv", + provenance=ACP_PROVENANCE, + ), + TraceEvent( + index=1, + kind=EventKind.TOOL_CALL, + source_type="tool_call", + role=Role.AGENT, + reasoning="Check the file first.\n\nThen count.", + reasoning_segments=["Check the file first.", "Then count."], + tool_call=ToolCall( + call_id="tc_1", + name="execute", + name_semantics="acp_kind", + title="wc -l data.csv", + status=ToolStatus.COMPLETED, + arguments=None, + content=[ + ContentBlock( + kind=ContentBlockKind.TEXT, + text="42 data.csv", + raw={ + "type": "content", + "content": {"type": "text", "text": "42 data.csv"}, + }, + ), + ContentBlock( + kind=ContentBlockKind.OPAQUE, + raw={ + "type": "diff", + "path": "/w/data.csv", + "oldText": "a", + "newText": "b", + }, + ), + ], + ), + provenance=ACP_PROVENANCE, + ), + TraceEvent( + index=2, + kind=EventKind.TIMEOUT, + source_type="agent_timeout", + outcome="wall_clock_timeout", + extensions={ + "timeout_sec": 90.0, + "pending_tool_call_ids": [], + "terminal_trajectory_complete": True, + }, + provenance=ACP_PROVENANCE, + ), + ], + usage=TraceUsage( + input_tokens=1180, + output_tokens=96, + total_tokens=1276, + source="llm_proxy_normalized", + ), + outcome=TraceOutcome(status=OutcomeStatus.TIMEOUT), + losses=losses, + ) + + +def test_the_example_trace_satisfies_every_invariant(): + assert validate_trace(_example_trace()) == [] + + +def test_the_example_demonstrates_what_the_ir_is_for(): + """Read as a spec: these four properties are the reason the module exists.""" + trace = _example_trace() + tool_event = trace.events[1] + + # 1. An unavailable value is absent *and* declared. + assert tool_event.tool_call.arguments is None + assert trace.losses.for_field("events[1].tool_call.arguments") + + # 2. A non-text block is carried instead of skipped (§5 loss #5). + opaque = [ + b for b in tool_event.tool_call.content if b.kind is ContentBlockKind.OPAQUE + ] + assert opaque and opaque[0].raw["newText"] == "b" + + # 3. Thought boundaries survive the join (§5 loss #10). + assert tool_event.reasoning_segments == ["Check the file first.", "Then count."] + assert "\n\n".join(tool_event.reasoning_segments) == tool_event.reasoning + + # 4. The timeout is representable at all (§5 loss #4). + assert trace.events[2].kind is EventKind.TIMEOUT + assert trace.events[2].extensions["timeout_sec"] == 90.0 + + +def _documented_example() -> dict: + """The JSON block tagged ```` in the interop document.""" + text = DOCS_PATH.read_text(encoding="utf-8") + match = re.search(r"\s*```json\n(.*?)\n```", text, re.S) + assert match, ( + "docs/trace-interop.md no longer contains an JSON " + "block. The documented example is generated from the models; regenerate " + "it rather than deleting this check." + ) + return json.loads(match.group(1)) + + +def test_the_documented_example_matches_the_models(): + """A doc example that cannot drift, because the drift fails a test.""" + produced = _example_trace().model_dump(mode="json", exclude_none=True) + assert produced == _documented_example() From 91fa30e1a410c830c28ec8ad03ca7fd38568fe61 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Sat, 15 Aug 2026 10:01:42 +0200 Subject: [PATCH 04/23] feat(trajectories): convert ACP capture events to the canonical IR, unwired MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice B proposed a hub and a contract: every `None` in the IR is covered by a `LossRecord`, and a conversion's cost is a typed value rather than a comment in a docstring. This is the first real edge, and its job is as much to stress that contract as to convert. `acp_events_to_ir` reads the event list of `trajectory/acp_trajectory.jsonl` — the ACP-session capture vocabulary Slice A pinned, plus the oracle and unknown record families that share the file — and returns one `CanonicalTrace` carrying its own report. It reads no other artifact: `result.json`, `timing.json` and the proxy capture are separate, so a value that lives only there is a declared loss rather than a silent enrichment. What the conversion keeps that the existing exporters lose: - `agent_timeout` becomes an event with its fields in `extensions`, and sets the trace outcome. Every exporter drops it today (§5 loss #4). - Non-text content blocks are carried as `opaque` with the block verbatim, instead of being skipped by `content_blocks_to_text` (loss #5). - Thought boundaries survive: one capture record is one reasoning segment, and nothing is joined, so the ambiguity `ThoughtBuffer` creates never arises (loss #10). A thought whose own text contains a blank line stays one segment, because splitting it would invent a boundary the source does not have. - `""` and absent stay distinguishable everywhere; text-empty events are kept rather than dropped. - The `oracle` record keeps its own kind and role instead of becoming an agent step prefixed `[oracle: …]` that a consumer can only undo by string matching. - An unrecognized `type` becomes `unknown` with the record carried verbatim, instead of being skipped silently. What it refuses to invent: arguments, timestamps, tool-call ids, agent version. It also does not prepend the `prompts` argument as leading user events, which both existing exporters do — §5.2 showed the cost, an ATIF document that opens with two identical `user` steps so user turns over-count by one. Those steps are not ACP events; a target that wants them adds them at its own edge as SYNTHESIZED. Loss addressing distinguishes three shapes. `events[i].…` is a field of one IR event, and is what `validate_trace` matches, so the per-call `arguments` records use it. `source[i]` is an input entry that produced no IR event, which cannot be addressed as `events[i]` because that index belongs to a different event once an entry is skipped. Systemic losses — timestamps, per-event usage, agent version, stop reason — are declared once each under an unindexed `events[].…` path. That last choice is what makes the contract affordable, and it is measured rather than asserted: the report is `n_tool_calls + 5` records and does not grow with trace length. On the two real gemini rollouts of §5.2 it is 7 records (H1, 2 tool calls) and 6 (H2, 1 tool call and a real wall-clock timeout). - `tests/trajectories/test_ir_from_acp.py` — 35 tests. Preservation is checked against events produced by driving a real `ACPSession` through the production capture path, reusing the Slice A2 fixture rather than hand-written dicts that would only prove the converter agrees with itself. The report is asserted as a complete set, so an undeclared loss and a spurious one both fail. One test removes a declared loss from a converted trace and shows the trace becomes invalid — the Slice B contract, demonstrated end to end. Two tests pin the volume property, including that doubling the non-tool events does not change the report at all. The block classifier is pinned against `content_blocks_to_text` so the IR and every existing consumer cannot disagree about what counts as text output. - `tests/trajectories/test_trace_ir.py` — the isolation test is restated at the boundary that now matters. Slice B asserted that nothing under `src/benchflow` imports the IR; the converter necessarily does, so `ir.py` and `ir_from_acp.py` are declared a closed family and the test asserts nothing outside it imports either. The guarantee is unchanged in substance and the test is strictly stronger: a new converter has to join the family explicitly, and a stale name in that list now fails a test of its own. - `docs/trace-interop.md` — §8.3 gains the implemented mapping table with a class per row and the measured loss counts; §8.7 records what is now implemented and restates the isolation property; §8.8 records what writing the first converter showed about the declared-absence rule, including the part of it most likely to be revised in review. No runtime module imports either module, no capture path or exporter changes, no on-disk format changes, and no new dependency. Verified: 35 new tests; tests/trajectories 324 passed; the trajectory, capture, streaming and ACP regression lane 489 passed; `ruff check .`, `ruff format --check` and `ty check src/` clean. Two real rollouts converted and cross-checked against their source files by hand. --- docs/trace-interop.md | 94 +++- src/benchflow/trajectories/ir_from_acp.py | 513 ++++++++++++++++++ tests/trajectories/test_ir_from_acp.py | 624 ++++++++++++++++++++++ tests/trajectories/test_trace_ir.py | 75 ++- 4 files changed, 1267 insertions(+), 39 deletions(-) create mode 100644 src/benchflow/trajectories/ir_from_acp.py create mode 100644 tests/trajectories/test_ir_from_acp.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md index 2168a1cb8..ca81edf90 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -549,23 +549,58 @@ Three design choices carry most of the weight: ### 8.3 Planned mapping -**PROPOSAL — none of these converters is implemented.** Slice B is the IR -alone; `ACP → IR` and `ATIF ↔ IR` are deliberately not in it, so the -representation can be reviewed before anything depends on it. +**`ACP → IR` is implemented** (Slice C, +[`src/benchflow/trajectories/ir_from_acp.py`](../src/benchflow/trajectories/ir_from_acp.py)), +still unwired and still provisional. `ATIF ↔ IR` and the OTel direction are not. #### ACP-session capture events → IR -| Capture event (§2.2) | IR | Loss recorded | -|---|---|---| -| `user_message` | `kind=user_message`, `role=user`, `text` | — | -| `agent_message` | `kind=agent_message`, `role=agent`, `text` | — | -| `agent_thought` | `kind=agent_reasoning`, `reasoning_segments=[text]` | none — the segment list is what preserves loss #10 | -| `tool_call` | `kind=tool_call` + `ToolCall`; `kind`→`name` with `name_semantics="acp_kind"` | `arguments` → UNSUPPORTED (#1); `started_at`/`finished_at` → UNSUPPORTED (#3) | -| `tool_call.content`, text block | `ContentBlock(kind=text)` | — | -| `tool_call.content`, other blocks | `ContentBlock(kind=opaque, raw=…)` | none — carrying the block verbatim is how loss #5 stops being a loss | -| `agent_timeout` | `kind=timeout`, `outcome="wall_clock_timeout"`, the rest in `extensions` | none — loss #4 becomes representable | -| `oracle` record (§2.4) | `kind=oracle`, `role=oracle` | none — no `[oracle: …]` string prefix, so a consumer need not string-match | -| unrecognized `type` | `kind=unknown` with `source_type` verbatim | — | +**FACT** for the implemented rows — asserted by +`tests/trajectories/test_ir_from_acp.py` against events produced by driving a +real `ACPSession` through the production capture path. + +| Capture field / event (§2.2) | IR | Class | Loss recorded | +|---|---|---|---| +| event order | `events[].index`, dense from 0 | preserved | — | +| `type` | `kind` **and** `source_type` verbatim | preserved | — | +| `user_message.text` | `text`, `role=user` | preserved | — | +| `agent_message.text` | `text`, `role=agent` | preserved | — | +| text-empty message (`""`) | `text=""`, event kept | preserved | — (both exporters drop the event) | +| `agent_thought.text` | `reasoning` **and** `reasoning_segments=[text]` | preserved | — the segment list is what avoids loss #10 | +| `tool_call.tool_call_id` | `tool_call.call_id`, `""` kept as `""` | preserved | — (ATIF/ADP synthesize here) | +| `tool_call.kind` | `tool_call.name` + `name_semantics="acp_kind"` | preserved | — the semantics is recorded instead of assumed | +| `tool_call.title` | `tool_call.title` | preserved | — | +| `tool_call.status` | `tool_call.status` | preserved | — | +| `tool_call.content`, text blocks | `ContentBlock(kind=text)` + `raw` | preserved | — | +| `tool_call.content`, other blocks | `ContentBlock(kind=opaque, raw=…)` | preserved | — carrying the block verbatim is how loss #5 stops being a loss | +| tool arguments | `arguments=None` | unsupported | `events[i].tool_call.arguments`, per call (#1) | +| `ToolCallRecord.started_at`/`.finished_at` | `None` | unsupported | `events[].tool_call.*`, once (#3) | +| `agent_timeout` | `kind=timeout`, `outcome=reason`, rest in `extensions`; trace `outcome.status=timeout` | preserved | — loss #4 becomes representable | +| `oracle` record (§2.4) | `kind=oracle`, `role=oracle`, fields in `extensions` | preserved | — no `[oracle: …]` prefix, so no string matching | +| unrecognized `type` | `kind=unknown`, `source_type` verbatim, record in `extensions` | preserved | — every exporter skips these today | +| unrecognized extra field on a known record | `extensions` | preserved | — | +| per-event usage | `None` | unsupported | `events[].usage`, once (#6, #8) | +| agent version | `None` | unsupported | `agent.agent_version`, once (#7) | +| `stop_reason` | `None` | unsupported | `outcome.stop_reason`, once (#9) | +| non-object entry in the list | *(no IR event)* | dropped | `source[i]` | +| non-string where a string is expected | coerced with `str()` | normalized | `events[i].` | +| status outside the ACP vocabulary | `unknown`, original in `extensions.source_status` | normalized | `events[i].tool_call.status` | + +Two rows are deliberately absent. The converter does **not** prepend the +`prompts` argument as leading `user` events, though `acp_events_to_atif_steps` +and `acp_events_to_adp_content` both do: those steps are not ACP events, and +§5.2 records what they cost — an ATIF document that opens with two identical +`user` steps, so user turns over-count by one. A target that wants them adds +them at its own edge as `SYNTHESIZED`. And it reads no other artifact: +`result.json`, `timing.json` and the proxy capture are not consulted, so a value +that lives only there is a declared loss rather than a silent enrichment. + +**FACT — measured on two real rollouts.** The `gemini` rollouts of §5.2, +converted through this path: H1 (5 events, 2 tool calls) produces **7** loss +records, H2 (4 events, 1 tool call, one real wall-clock timeout) produces **6**. +All `unsupported`. In both, 5 records are systemic and the rest are the +per-call `arguments`, so the report is `n_tool_calls + 5` and does not grow with +trace length. #### IR → ATIF @@ -791,7 +826,32 @@ here. ### 8.7 Status - **Implemented:** the IR types, the loss model, the invariants, the validation - suite and this section. -- **Not implemented, deliberately:** `ACP → IR`, `ATIF ↔ IR`, any OTel work, any - wiring into a run path, any on-disk artifact. + suite, this section, and the `ACP → IR` converter with its loss report. +- **Not implemented, deliberately:** `ATIF ↔ IR`, any OTel work, any wiring into + a run path, any on-disk artifact, any capture-layer enrichment. - **Unchanged:** every existing format, exporter, artifact and code path. + +The isolation property is unchanged in substance and restated at a new +boundary: `ir.py` and `ir_from_acp.py` form a closed family that may import each +other, and `test_only_the_ir_family_imports_the_ir` asserts that nothing else in +`src/benchflow` imports either. The converter itself imports one benchflow +module — the IR — and reads the capture format as data. + +### 8.8 What the first converter showed + +**FACT.** Writing `ACP → IR` was also the first stress test of §8.2's +declared-absence rule, and two things came out of it that a design document +could not have settled: + +- **The rule is affordable, because the report is bounded by tool calls rather + than by trace length.** Systemic absences — timestamps, per-event usage, agent + version, stop reason — are declared once each under an unindexed + `events[].…` path; only `arguments`, which `validate_trace` requires per + event, scales. A 50-tool-call trace declares 55 records that carry one + sentence of distinct information between them, which is why the converter + ships `loss_summary`. +- **The per-event requirement is the part to review.** It is what makes an + undeclared absence a test failure rather than a habit, and it is also the + reason 50 records say the same thing. An `events[*].…` wildcard would collapse + them at the cost of making a single missed call invisible. That trade is open; + §8.6 already lists this contract as the most likely thing to change. diff --git a/src/benchflow/trajectories/ir_from_acp.py b/src/benchflow/trajectories/ir_from_acp.py new file mode 100644 index 000000000..00dea4691 --- /dev/null +++ b/src/benchflow/trajectories/ir_from_acp.py @@ -0,0 +1,513 @@ +"""ACP-session capture events → canonical Trace IR (Slice C). + +> **PROVISIONAL.** Companion to :mod:`benchflow.trajectories.ir`, which is +> itself an unapproved proposal (`docs/trace-interop.md` §8). Nothing imports +> this module, nothing writes its output to disk, and no capture path, exporter +> or artifact changes because it exists. + +This is the first real edge of the hub, and its job is as much to *stress the +contract* as to convert: every `None` the IR carries has to be justified by a +`LossRecord`, and this converter is where that stops being a design statement. + +## What it converts + +The event list as written to `trajectory/acp_trajectory.jsonl` — the ACP-session +capture vocabulary pinned by Slice A's schema, plus the two other record +families §2.4 documents as sharing that file. It reads the events and nothing +else: `result.json`, `timing.json` and the proxy capture are separate artifacts +and are not consulted, so a value that only exists there is a declared loss +rather than a silent enrichment. + +## What it deliberately does not do + +**It does not prepend `prompts` as leading user events.** `acp_events_to_atif_steps` +and `acp_events_to_adp_content` both do, and §5.2 records the consequence: an +ATIF document opens with two identical `user` steps — one from the `prompts` +argument, one from the captured `user_message` event — so a consumer counting +user turns over-counts by one. Those steps are not ACP events. If a target +format wants them, its own converter adds them and records +:attr:`LossClass.SYNTHESIZED`; inventing them here would put the defect in the +hub, where every format would inherit it. + +It also does not synthesize tool-call ids, timestamps, an agent version, or +arguments. Those are target-side obligations (§8.2). + +## Loss addressing + +Two path shapes appear in the report, and the difference is load-bearing: + +- ``events[i].…`` — a field of the IR event at index *i*. This is what + :func:`~benchflow.trajectories.ir.validate_trace` matches against, so the + per-call `arguments` records must use it. +- ``source[i]`` — an entry of the *input* list that produced no IR event at all. + It cannot be addressed as ``events[i]``, because that index belongs to a + different event once an entry is skipped. + +Systemic losses — the ones that hold for every event of a kind rather than for +one event — are declared **once** with an unindexed ``events[].…`` path. Writing +them per event would multiply the report by the trace length while adding no +information; see §8.6 and the volume test in the suite. + +## What the report does and does not claim + +Losses are declared against **the documented emitter contract** (§2.3), not +against arbitrary input. `_events_to_trajectory` writes all six tool-call fields +as literals, so a record missing one did not come from that emitter; the IR +carries ``None`` for it and declares nothing, because "the ACP-session emitter +does not produce this value" would be a false statement about a record it did +not produce. The one exception is ``arguments``, which the emitter *never* +produces for any record and which :func:`validate_trace` therefore requires to +be declared every time. + +A value this converter reshapes — a coerced non-string, an unmappable status — +is always declared, whatever the record's provenance. Reshaping is this +module's own act, and the point of the contract is that it owns up to it. +""" + +from __future__ import annotations + +from typing import Any + +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlock, + ContentBlockKind, + EventKind, + LossClass, + LossRecord, + LossReport, + ModelInfo, + OutcomeStatus, + Provenance, + Role, + ToolCall, + ToolStatus, + TraceEvent, + TraceOutcome, +) + +LOSS_DIRECTION = "acp->ir" + +# Source labels for per-event provenance. A single ``acp_trajectory.jsonl`` can +# hold records from more than one producer (§2.4), which is why the IR carries +# provenance per event and not only per trace. +ACP_CAPTURE_SOURCE = "acp-capture-v1" +"""The ACP-session capture vocabulary — exactly what Slice A's schema pins.""" + +ORACLE_SOURCE = "oracle" +"""`_run_oracle`'s alternative trajectory. Not an ACP-session record.""" + +UNKNOWN_SOURCE = "acp-trajectory-unknown" +"""In the file, outside every vocabulary this repository documents.""" + +ACP_TRAJECTORY_SOURCE = "acp-trajectory" +"""Trace level: names the artifact family, since its events may disagree.""" + +_ACP_CAPTURE_PRODUCER = "_events_to_trajectory" + +# Two of the three text-bearing capture types. ``agent_thought`` has its own +# branch below: it becomes reasoning rather than text, and carries the segment +# list that keeps its boundaries. +_TEXT_EVENTS: dict[str, tuple[EventKind, Role]] = { + "user_message": (EventKind.USER_MESSAGE, Role.USER), + "agent_message": (EventKind.AGENT_MESSAGE, Role.AGENT), +} + +_STATUS_BY_VALUE = {status.value: status for status in ToolStatus} + +# Keys the emitter writes on each record. Anything else on a recognized record +# is carried into ``extensions`` rather than dropped. +_TOOL_CALL_KEYS = frozenset( + {"type", "tool_call_id", "kind", "title", "status", "content"} +) +_TEXT_KEYS = frozenset({"type", "text"}) +_TIMEOUT_KEYS = frozenset({"type", "reason"}) +_ORACLE_KEYS = frozenset({"type"}) + + +def _str_field( + raw: dict[str, Any], key: str, field: str, losses: LossReport +) -> str | None: + """Read a string-typed source field, coercing and declaring if it is not one. + + The emitter only ever writes strings here, so the coercion path is + unreachable from `_events_to_trajectory`. It exists because this converter + accepts the *file*, and §7 lists fixtures in this repository that use the + ACP filename with other shapes. Coercing silently would be exactly the kind + of undeclared normalization the IR exists to stop. + + ``None`` when the key is absent or explicitly null — absent and empty stay + distinguishable, which is the tri-state rule (§8.2). + """ + if key not in raw: + return None + value = raw[key] + if value is None or isinstance(value, str): + return value + losses.add( + field, + LossClass.NORMALIZED, + f"{key} is {type(value).__name__}, not a string; coerced with str()", + ) + return str(value) + + +def _extras(raw: dict[str, Any], known: frozenset[str]) -> dict[str, Any]: + """Every field of *raw* the mapping below does not place explicitly. + + Carrying them verbatim is what keeps a record that is *almost* a known + shape — an extra key from a future capture-layer change — from being + silently truncated to the fields this converter happens to know. + """ + return {key: value for key, value in raw.items() if key not in known} + + +def _content_block_to_ir(block: Any) -> ContentBlock | None: + """Classify one ACP content block, mirroring ``content_blocks_to_text``. + + That helper renders the two text shapes it recognizes — the nested ACP + ``{"type": "content", "content": {"type": "text", "text": …}}`` and the flat + ``{"text": …}`` — and skips everything else, which is §5 loss #5. Here the + same two shapes become :attr:`ContentBlockKind.TEXT` and *everything else + becomes* :attr:`ContentBlockKind.OPAQUE` with the block kept verbatim, so + the skip stops being a loss. + + ``None`` for a block that cannot be represented at all (a non-object entry); + the caller declares that as a loss. + """ + if not isinstance(block, dict): + return None + inner = block.get("content") + if isinstance(inner, dict): + inner = inner.get("text") + text = block.get("text") or inner + if text: + return ContentBlock(kind=ContentBlockKind.TEXT, text=str(text), raw=block) + return ContentBlock(kind=ContentBlockKind.OPAQUE, raw=block) + + +def _tool_call_to_ir( + raw: dict[str, Any], index: int, losses: LossReport +) -> tuple[ToolCall, dict[str, Any]]: + """Build the IR tool call, and return the extras to hang on its event.""" + extras = _extras(raw, _TOOL_CALL_KEYS) + + raw_status = raw.get("status") + status = _STATUS_BY_VALUE.get(str(raw_status)) if raw_status is not None else None + if raw_status is not None and status is None: + # Unreachable from the emitter — the serialized value is always + # ``ToolCallStatus(...).value`` — but reachable from a hand-written or + # future record. Keep the original addressable instead of only in prose. + status = ToolStatus.UNKNOWN + extras["source_status"] = raw_status + losses.add( + f"events[{index}].tool_call.status", + LossClass.NORMALIZED, + f"status {raw_status!r} is outside the ACP ToolCallStatus vocabulary; " + "mapped to unknown, original kept in extensions.source_status", + "§2.3", + ) + + content: list[ContentBlock] = [] + raw_content = raw.get("content") + if isinstance(raw_content, str): + # ``content_blocks_to_text`` accepts a bare string; mirror it rather + # than treating the whole field as unrepresentable. + content.append(ContentBlock(kind=ContentBlockKind.TEXT, text=raw_content)) + elif isinstance(raw_content, list): + for position, block in enumerate(raw_content): + converted = _content_block_to_ir(block) + if converted is None: + losses.add( + f"events[{index}].tool_call.content", + LossClass.DROPPED, + f"source content block {position} is {type(block).__name__}, " + "not a JSON object, and has no IR representation", + ) + continue + content.append(converted) + elif raw_content is not None: + losses.add( + f"events[{index}].tool_call.content", + LossClass.DROPPED, + f"content is {type(raw_content).__name__}; the emitter writes a list " + "and content_blocks_to_text also accepts a string, so neither this " + "converter nor any existing consumer can read it", + ) + + tool_call = ToolCall( + call_id=_str_field( + raw, "tool_call_id", f"events[{index}].tool_call.call_id", losses + ), + name=_str_field(raw, "kind", f"events[{index}].tool_call.name", losses), + # An ACP ``kind`` is a category (``execute``, ``read``), not a function + # name. Recording that is what keeps the IR from laundering the + # normalization ATIF performs when it lands the value in + # ``function_name`` (§5.1). + name_semantics="acp_kind" if "kind" in raw else None, + title=_str_field(raw, "title", f"events[{index}].tool_call.title", losses), + status=status, + # Never ``{}``: the capture path does not read ``rawInput``, so this is + # "not carried", not "carried and empty" (§8.2). + arguments=None, + content=content, + ) + losses.add( + f"events[{index}].tool_call.arguments", + LossClass.UNSUPPORTED, + "ACPSession.handle_update reads toolCallId/title/kind/status/content and " + "drops rawInput, so no ACP-derived tool call carries arguments", + "§5 loss #1", + ) + return tool_call, extras + + +def _declare_systemic_losses(losses: LossReport, *, had_tool_call: bool) -> None: + """Losses that hold for the whole conversion, declared once each. + + Every one of these is a row of the §5 table that no per-event record could + make more precise: the capture events carry the value nowhere, so an + indexed path would repeat one fact once per event. + """ + if had_tool_call: + losses.add( + "events[].tool_call.started_at", + LossClass.UNSUPPORTED, + "ToolCallRecord tracks started_at/finished_at in memory and " + "_events_to_trajectory serializes neither", + "§5 loss #3", + ) + losses.add( + "events[].tool_call.finished_at", + LossClass.UNSUPPORTED, + "ToolCallRecord tracks started_at/finished_at in memory and " + "_events_to_trajectory serializes neither", + "§5 loss #3", + ) + losses.add( + "events[].usage", + LossClass.UNSUPPORTED, + "ACP capture events carry no usage; ACPSession.usage_snapshots is routed " + "to result.json and never into the trajectory", + "§5 losses #6, #8", + ) + losses.add( + "agent.agent_version", + LossClass.UNSUPPORTED, + 'BenchFlow does not track agent binary versions; ATIF\'s "unknown" is a ' + "target-side obligation, not an observation", + "§5 loss #7", + ) + losses.add( + "outcome.stop_reason", + LossClass.UNSUPPORTED, + "ACPSession.stop_reason is captured on the session and exported nowhere", + "§5 loss #9", + ) + + +def acp_events_to_ir( + events: list[dict[str, Any]], + *, + session_id: str | None = None, + agent_name: str | None = None, + model: str | None = None, +) -> CanonicalTrace: + """Convert captured ACP trajectory events into one :class:`CanonicalTrace`. + + *session_id*, *agent_name* and *model* are metadata the caller already has + from elsewhere (``result.json``, the rollout config); they are not read out + of the events, because the events do not carry them. Everything else comes + from *events* alone. + + The returned trace carries its own :class:`LossReport` on + :attr:`CanonicalTrace.losses`, and satisfies + :func:`~benchflow.trajectories.ir.validate_trace` for any input — including + input the Slice A schema would reject. + + Ordering is preserved exactly. An entry that produces no IR event (a + non-object entry in the list) is declared under a ``source[i]`` path and + leaves no hole: IR indices stay dense, which is invariant 2. + """ + losses = LossReport(direction=LOSS_DIRECTION) + ir_events: list[TraceEvent] = [] + had_tool_call = False + timed_out = False + + for source_position, raw in enumerate(events): + if not isinstance(raw, dict): + losses.add( + f"source[{source_position}]", + LossClass.DROPPED, + f"entry is {type(raw).__name__}, not a JSON object; the IR has no " + "representation for it", + ) + continue + + index = len(ir_events) + etype = raw.get("type") + etype_str = str(etype) if isinstance(etype, str) else None + + if etype_str in _TEXT_EVENTS: + kind, role = _TEXT_EVENTS[etype_str] + ir_events.append( + TraceEvent( + index=index, + kind=kind, + source_type=etype_str, + role=role, + # ``""`` is preserved: it is an observed value, and both + # exporters drop text-empty events today (§5.1). + text=_str_field(raw, "text", f"events[{index}].text", losses), + provenance=Provenance( + source_format=ACP_CAPTURE_SOURCE, + producer=_ACP_CAPTURE_PRODUCER, + ), + extensions=_extras(raw, _TEXT_KEYS), + ) + ) + elif etype_str == "agent_thought": + text = _str_field(raw, "text", f"events[{index}].reasoning", losses) + ir_events.append( + TraceEvent( + index=index, + kind=EventKind.AGENT_REASONING, + source_type=etype_str, + role=Role.AGENT, + reasoning=text, + # One capture record is one thought. Keeping the segment + # list means the boundary survives; ``ThoughtBuffer`` joins + # thoughts with a blank line and makes the count + # unrecoverable (§5 loss #10). Nothing is joined here, so + # the loss is avoided rather than reproduced — and a thought + # whose own text contains a blank line stays one segment, + # because splitting it would invent a boundary. + reasoning_segments=[text] if text is not None else None, + provenance=Provenance( + source_format=ACP_CAPTURE_SOURCE, + producer=_ACP_CAPTURE_PRODUCER, + ), + extensions=_extras(raw, _TEXT_KEYS), + ) + ) + elif etype_str == "tool_call": + had_tool_call = True + tool_call, extras = _tool_call_to_ir(raw, index, losses) + ir_events.append( + TraceEvent( + index=index, + kind=EventKind.TOOL_CALL, + source_type=etype_str, + role=Role.AGENT, + tool_call=tool_call, + provenance=Provenance( + source_format=ACP_CAPTURE_SOURCE, + producer=_ACP_CAPTURE_PRODUCER, + ), + extensions=extras, + ) + ) + elif etype_str == "agent_timeout": + timed_out = True + ir_events.append( + TraceEvent( + index=index, + kind=EventKind.TIMEOUT, + source_type=etype_str, + # No role: this is BenchFlow's own marker, not an agent + # action and not an ACP notification. Attributing it to the + # agent would be inventing semantics. + outcome=_str_field( + raw, "reason", f"events[{index}].outcome", losses + ), + provenance=Provenance( + source_format=ACP_CAPTURE_SOURCE, + producer="record_agent_timeout", + ), + # timeout_sec / pending_tool_call_ids / + # terminal_trajectory_complete ride here rather than as three + # IR fields no other source would ever populate. + extensions=_extras(raw, _TIMEOUT_KEYS), + ) + ) + elif etype_str == "oracle": + ir_events.append( + TraceEvent( + index=index, + kind=EventKind.ORACLE, + source_type=etype_str, + role=Role.ORACLE, + provenance=Provenance( + source_format=ORACLE_SOURCE, producer="_run_oracle" + ), + # command / return_code / stdout, verbatim. The ATIF + # exporter renders these as an agent step prefixed + # "[oracle: …]", which a consumer can only undo by string + # matching (§5.1); here the record stays itself. + extensions=_extras(raw, _ORACLE_KEYS), + ) + ) + else: + ir_events.append( + TraceEvent( + index=index, + kind=EventKind.UNKNOWN, + # Verbatim, including ``None`` for a record with no ``type``. + source_type=etype_str, + provenance=Provenance(source_format=UNKNOWN_SOURCE), + # Nothing is lost: the whole record is carried. Every + # exporter in the tree skips these silently today (§5.1). + extensions=dict(raw), + ) + ) + + _declare_systemic_losses(losses, had_tool_call=had_tool_call) + + return CanonicalTrace( + session_id=session_id, + agent=ModelInfo(agent_name=agent_name, model=model), + events=ir_events, + # A timeout marker in the stream is an observation about how the run + # ended. Every other outcome — pass, fail, reward — lives in + # ``result.json``, which this converter does not read. + outcome=TraceOutcome(status=OutcomeStatus.TIMEOUT) if timed_out else None, + provenance=Provenance(source_format=ACP_TRAJECTORY_SOURCE), + losses=losses, + ) + + +def loss_summary(report: LossReport) -> dict[str, int]: + """Count a report's records by class — for logs, reviews and eyeballing. + + A conversion's cost should be readable without printing every record; a + 50-tool-call trace declares 50 `arguments` losses that say the same thing. + """ + summary: dict[str, int] = {} + for record in report.records: + summary[record.loss_class.value] = summary.get(record.loss_class.value, 0) + 1 + return summary + + +def _is_per_event(field: str) -> bool: + """True for a record addressed to one event or one source entry. + + ``events[]…`` is the unindexed, systemic form and is deliberately not + per-event, so the prefix test has to run before the general one. + """ + if field.startswith("events[]"): + return False + return field.startswith(("events[", "source[")) + + +def systemic_losses(report: LossReport) -> list[LossRecord]: + """The records declared once for the whole trace rather than per event. + + The complement — :func:`per_event_losses` — is the part that grows with the + trace, and is what makes the report's size worth watching. + """ + return [record for record in report.records if not _is_per_event(record.field)] + + +def per_event_losses(report: LossReport) -> list[LossRecord]: + """The records addressed to a single event or a single source entry.""" + return [record for record in report.records if _is_per_event(record.field)] diff --git a/tests/trajectories/test_ir_from_acp.py b/tests/trajectories/test_ir_from_acp.py new file mode 100644 index 000000000..1d92dbe10 --- /dev/null +++ b/tests/trajectories/test_ir_from_acp.py @@ -0,0 +1,624 @@ +"""Conversion suite for ``ACP capture events → canonical Trace IR`` (Slice C). + +Slice B asserted a contract: every ``None`` in the IR is covered by a +``LossRecord``, and a conversion's cost is a value rather than a comment. This +suite is where that contract meets a real converter, so it is written to be +able to *fail* the design, not only the code: + +* **Preservation** is checked field by field against events produced by the + production capture path (``_rich_events`` from the Slice A2 suite drives a + real :class:`ACPSession` through ``handle_update``), not against hand-written + dicts that happen to match the converter. +* **The loss report is asserted as a complete set**, not with membership + checks. An undeclared loss and a spurious one both fail. +* **The volume test** measures how the report scales, because "declare every + absence" is only a workable contract if the report stays readable on a real + trace. It pins the property that matters: the report grows with tool calls, + not with trace length. +* **Robustness** is exercised with input the Slice A schema would reject — + §7 lists fixtures in this repository that use the ACP filename with other + shapes, so a converter that only accepts conformant input would be a + converter for a file that does not always exist. + +Nothing here writes to disk and no runtime module is imported for its side +effects. +""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path +from typing import Any + +from benchflow.trajectories import ir_from_acp +from benchflow.trajectories._export_common import content_blocks_to_text +from benchflow.trajectories.ir import ( + ContentBlockKind, + EventKind, + LossClass, + OutcomeStatus, + Role, + ToolStatus, + validate_trace, +) +from benchflow.trajectories.ir_from_acp import ( + ACP_CAPTURE_SOURCE, + ACP_TRAJECTORY_SOURCE, + LOSS_DIRECTION, + ORACLE_SOURCE, + UNKNOWN_SOURCE, + acp_events_to_ir, + loss_summary, + per_event_losses, + systemic_losses, +) +from tests.trajectories.test_acp_capture_event_schema import _emitted_event_types +from tests.trajectories.test_atif_preservation import ( + PENDING_TOOL_CALL_ID, + TIMEOUT_SEC, + _rich_events, +) + +# The systemic records every conversion of a tool-bearing trace declares. Held +# as a literal so adding one is a deliberate edit to this suite. +SYSTEMIC_FIELDS = { + "events[].tool_call.started_at", + "events[].tool_call.finished_at", + "events[].usage", + "agent.agent_version", + "outcome.stop_reason", +} + + +def _fields(trace) -> set[str]: + return {record.field for record in trace.losses.records} + + +def test_the_converter_depends_on_the_ir_and_nothing_else_in_benchflow(): + """Slice C must not drag the hub into the rest of the tree. + + ``tests/trajectories/test_trace_ir.py`` checks the other direction — that + nothing outside the family imports the IR. This one checks that the family + itself stays a leaf: the converter reads the capture format as *data*, and + reaching into ``_capture`` or an exporter would couple the hub to the very + modules it is meant to sit between. + """ + tree = ast.parse(Path(ir_from_acp.__file__).read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add(node.module or "") + benchflow_imports = {name for name in imported if name.startswith("benchflow")} + assert benchflow_imports == {"benchflow.trajectories.ir"}, sorted(benchflow_imports) + + +# --------------------------------------------------------------------------- +# 1. Field-by-field preservation, from real captured events +# --------------------------------------------------------------------------- + + +def test_every_emitted_event_type_becomes_a_typed_ir_event(): + """No event the capture path emits may land in ``UNKNOWN``. + + The producer's vocabulary is read out of ``_events_to_trajectory`` by AST + (the Slice A mechanism), so adding a branch there fails this test until the + converter handles it — rather than silently degrading it to ``unknown``. + """ + events = [{"type": etype} for etype in sorted(_emitted_event_types())] + trace = acp_events_to_ir(events) + unknown = [e.source_type for e in trace.events if e.kind is EventKind.UNKNOWN] + assert unknown == [], unknown + + +def test_text_events_preserve_role_text_and_source_type(): + events = [ + {"type": "user_message", "text": "List the files."}, + {"type": "agent_message", "text": "Done."}, + ] + trace = acp_events_to_ir(events) + + user, agent = trace.events + assert (user.kind, user.role, user.text) == ( + EventKind.USER_MESSAGE, + Role.USER, + "List the files.", + ) + assert (agent.kind, agent.role, agent.text) == ( + EventKind.AGENT_MESSAGE, + Role.AGENT, + "Done.", + ) + assert [e.source_type for e in trace.events] == ["user_message", "agent_message"] + assert all( + e.provenance.source_format == ACP_CAPTURE_SOURCE + and e.provenance.producer == "_events_to_trajectory" + for e in trace.events + ) + assert trace.provenance.source_format == ACP_TRAJECTORY_SOURCE + + +def test_empty_text_is_preserved_rather_than_dropped(): + """Both exporters drop text-empty events (§5.1); ``""`` is an observation.""" + trace = acp_events_to_ir([{"type": "agent_message", "text": ""}]) + assert len(trace.events) == 1 + assert trace.events[0].text == "" + assert trace.events[0].text is not None + + +def test_a_real_captured_trace_converts_field_by_field(): + events = _rich_events() + trace = acp_events_to_ir( + events, session_id="sess-a2", agent_name="claude-code", model="claude-sonnet-5" + ) + + assert [e.source_type for e in trace.events] == [e["type"] for e in events] + assert trace.session_id == "sess-a2" + assert trace.agent.agent_name == "claude-code" + assert trace.agent.model == "claude-sonnet-5" + # Never fabricated, even though ATIF requires the field. + assert trace.agent.agent_version is None + + source_tool = next(e for e in events if e["type"] == "tool_call") + ir_tool = next(e for e in trace.events if e.kind is EventKind.TOOL_CALL) + assert ir_tool.tool_call.call_id == source_tool["tool_call_id"] + assert ir_tool.tool_call.name == source_tool["kind"] + assert ir_tool.tool_call.title == source_tool["title"] + assert ir_tool.tool_call.status.value == source_tool["status"] + assert ir_tool.role is Role.AGENT + # The ACP kind is a category, not a function name — and the IR says so. + assert ir_tool.tool_call.name_semantics == "acp_kind" + + assert validate_trace(trace) == [] + + +# --------------------------------------------------------------------------- +# 2. Ordering +# --------------------------------------------------------------------------- + + +def test_ordering_and_dense_indices_are_preserved(): + events = [{"type": "agent_message", "text": f"m{n}"} for n in range(6)] + events.insert(3, {"type": "tool_call", "tool_call_id": "x", "kind": "read"}) + trace = acp_events_to_ir(events) + + assert [e.index for e in trace.events] == list(range(len(events))) + assert [e.source_type for e in trace.events] == [e["type"] for e in events] + + +def test_an_unrepresentable_entry_leaves_no_hole(): + """A skipped source entry is declared under ``source[i]``, not ``events[i]``. + + The distinction matters: index 1 of the IR belongs to the event that + followed the skipped entry, so addressing the loss as ``events[1]`` would + blame a different event. + """ + events: list[Any] = [ + {"type": "agent_message", "text": "a"}, + "not an object", + {"type": "agent_message", "text": "b"}, + ] + trace = acp_events_to_ir(events) + + assert [e.index for e in trace.events] == [0, 1] + assert [e.text for e in trace.events] == ["a", "b"] + dropped = trace.losses.for_field("source[1]") + assert len(dropped) == 1 + assert dropped[0].loss_class is LossClass.DROPPED + assert validate_trace(trace) == [] + + +# --------------------------------------------------------------------------- +# 3. Tri-state arguments +# --------------------------------------------------------------------------- + + +def test_arguments_are_none_never_empty_and_always_declared(): + """`{}` would claim the agent called the tool with no arguments (§8.2).""" + trace = acp_events_to_ir( + [ + {"type": "tool_call", "tool_call_id": "a", "kind": "execute"}, + {"type": "tool_call", "tool_call_id": "b", "kind": "read"}, + ] + ) + for position, event in enumerate(trace.events): + assert event.tool_call.arguments is None + assert event.tool_call.arguments != {} + declared = trace.losses.for_field(f"events[{position}].tool_call.arguments") + assert len(declared) == 1 + assert declared[0].loss_class is LossClass.UNSUPPORTED + assert declared[0].doc_ref == "§5 loss #1" + assert validate_trace(trace) == [] + + +def test_the_serialized_trace_never_carries_an_empty_argument_map(): + """A whole-document check, in the A2 style: no `"arguments": {}` anywhere.""" + document = acp_events_to_ir(_rich_events()).model_dump(mode="json") + assert '"arguments": {}' not in json.dumps(document, indent=1) + + +# --------------------------------------------------------------------------- +# 4. Timeout +# --------------------------------------------------------------------------- + + +def test_the_timeout_marker_survives_with_its_fields(): + """§5 loss #4: every exporter drops this event. The IR keeps it whole.""" + events = _rich_events() + trace = acp_events_to_ir(events) + + timeout = next(e for e in trace.events if e.kind is EventKind.TIMEOUT) + assert timeout.source_type == "agent_timeout" + assert timeout.outcome == "wall_clock_timeout" + assert timeout.extensions["timeout_sec"] == TIMEOUT_SEC + assert timeout.extensions["pending_tool_call_ids"] == [PENDING_TOOL_CALL_ID] + assert timeout.extensions["terminal_trajectory_complete"] is False + # BenchFlow's own marker: not an agent action, so no role is invented. + assert timeout.role is None + assert trace.outcome.status is OutcomeStatus.TIMEOUT + + +def test_a_trace_without_a_timeout_claims_no_outcome(): + """The capture events say nothing about pass/fail; that lives in result.json.""" + trace = acp_events_to_ir([{"type": "agent_message", "text": "done"}]) + assert trace.outcome is None + + +# --------------------------------------------------------------------------- +# 5. Reasoning boundaries +# --------------------------------------------------------------------------- + + +def test_consecutive_thoughts_keep_their_boundaries(): + """The loss the ATIF/ADP path cannot avoid (#10) is avoided by not joining.""" + events = [ + {"type": "agent_thought", "text": "first"}, + {"type": "agent_thought", "text": "second"}, + ] + trace = acp_events_to_ir(events) + + assert [e.kind for e in trace.events] == [EventKind.AGENT_REASONING] * 2 + assert [e.reasoning_segments for e in trace.events] == [["first"], ["second"]] + assert validate_trace(trace) == [] + + +def test_a_thought_containing_a_blank_line_stays_one_segment(): + """Splitting it would invent a boundary the source does not have. + + This is the exact ambiguity §5 loss #10 describes: after ``ThoughtBuffer`` + joins, one thought with a blank line and two thoughts are indistinguishable. + The converter refuses to guess in either direction. + """ + trace = acp_events_to_ir([{"type": "agent_thought", "text": "one\n\ntwo"}]) + event = trace.events[0] + assert event.reasoning_segments == ["one\n\ntwo"] + assert event.reasoning == "one\n\ntwo" + assert validate_trace(trace) == [] + + +def test_reasoning_is_not_merged_into_text(): + trace = acp_events_to_ir([{"type": "agent_thought", "text": "thinking"}]) + assert trace.events[0].text is None + assert trace.events[0].reasoning == "thinking" + + +# --------------------------------------------------------------------------- +# 6. Unknown, oracle and non-conformant records +# --------------------------------------------------------------------------- + + +def test_unknown_event_types_are_carried_not_dropped(): + """Every exporter skips these silently today (§5.1, last rows).""" + raw = {"type": "some_future_event", "payload": {"a": 1}, "n": 3} + trace = acp_events_to_ir([raw]) + + event = trace.events[0] + assert event.kind is EventKind.UNKNOWN + assert event.source_type == "some_future_event" + assert event.extensions == raw + assert event.provenance.source_format == UNKNOWN_SOURCE + assert validate_trace(trace) == [] + + +def test_a_record_without_a_type_is_still_carried(): + trace = acp_events_to_ir([{"role": "assistant"}]) + assert trace.events[0].kind is EventKind.UNKNOWN + assert trace.events[0].source_type is None + assert trace.events[0].extensions == {"role": "assistant"} + + +def test_the_oracle_record_keeps_its_identity(): + """ATIF renders it as an agent step prefixed ``[oracle: …]`` (§5.1).""" + raw = {"type": "oracle", "command": "solve.sh", "return_code": 0, "stdout": "ok"} + trace = acp_events_to_ir([raw]) + + event = trace.events[0] + assert event.kind is EventKind.ORACLE + assert event.role is Role.ORACLE + assert event.provenance.source_format == ORACLE_SOURCE + assert event.extensions == { + "command": "solve.sh", + "return_code": 0, + "stdout": "ok", + } + assert "[oracle:" not in json.dumps(trace.model_dump(mode="json")) + + +def test_extra_fields_on_a_known_record_are_carried_into_extensions(): + """A capture-layer change that adds a field must not be truncated away.""" + trace = acp_events_to_ir( + [{"type": "agent_message", "text": "hi", "future_field": 42}] + ) + assert trace.events[0].extensions == {"future_field": 42} + assert trace.events[0].text == "hi" + + +def test_non_string_values_are_coerced_and_the_coercion_is_declared(): + """Unreachable from the emitter, reachable from the file (§7).""" + trace = acp_events_to_ir([{"type": "agent_message", "text": 42}]) + assert trace.events[0].text == "42" + declared = trace.losses.for_field("events[0].text") + assert len(declared) == 1 + assert declared[0].loss_class is LossClass.NORMALIZED + + +def test_an_out_of_vocabulary_status_is_mapped_and_kept_recoverable(): + trace = acp_events_to_ir( + [{"type": "tool_call", "tool_call_id": "a", "status": "exploded"}] + ) + event = trace.events[0] + assert event.tool_call.status is ToolStatus.UNKNOWN + assert event.extensions["source_status"] == "exploded" + assert ( + trace.losses.for_field("events[0].tool_call.status")[0].loss_class + is LossClass.NORMALIZED + ) + + +# --------------------------------------------------------------------------- +# 7. Tool status, title, id and content +# --------------------------------------------------------------------------- + + +def test_empty_tool_id_and_title_are_preserved_as_empty_not_synthesized(): + """ATIF and ADP both synthesize ids here; the IR records what happened.""" + trace = acp_events_to_ir( + [ + { + "type": "tool_call", + "tool_call_id": "", + "kind": "tool", + "title": "", + "status": "in_progress", + "content": [], + } + ] + ) + call = trace.events[0].tool_call + assert call.call_id == "" + assert call.title == "" + assert call.status is ToolStatus.IN_PROGRESS + assert call.content == [] + assert not any("call_" in record.field for record in trace.losses.records) + + +def test_absent_tool_fields_stay_none_and_are_not_confused_with_empty(): + trace = acp_events_to_ir([{"type": "tool_call"}]) + call = trace.events[0].tool_call + assert call.call_id is None + assert call.title is None + assert call.name is None + assert call.name_semantics is None + assert call.status is None + + +def test_non_text_content_blocks_are_carried_as_opaque(): + """§5 loss #5: ``content_blocks_to_text`` skips these; the IR keeps them.""" + diff_block = {"type": "diff", "path": "/w/a.py", "oldText": "a", "newText": "b"} + trace = acp_events_to_ir( + [ + { + "type": "tool_call", + "tool_call_id": "t1", + "kind": "edit", + "status": "completed", + "content": [ + {"type": "content", "content": {"type": "text", "text": "ok"}}, + diff_block, + ], + } + ] + ) + blocks = trace.events[0].tool_call.content + assert [b.kind for b in blocks] == [ContentBlockKind.TEXT, ContentBlockKind.OPAQUE] + assert blocks[0].text == "ok" + assert blocks[1].raw == diff_block + # The verbatim block reached the serialized trace, not just the object. + assert "newText" in json.dumps(trace.model_dump(mode="json")) + assert validate_trace(trace) == [] + + +def test_text_blocks_agree_with_the_shared_renderer(): + """Pins the classifier against ``content_blocks_to_text`` rather than a copy. + + The two must recognize the same blocks as text, or the IR and every + existing consumer would disagree about what a tool produced. + """ + content = [ + {"type": "content", "content": {"type": "text", "text": "first"}}, + {"text": "flat form"}, + {"type": "diff", "oldText": "a", "newText": "b"}, + {"type": "content", "content": {"type": "image", "data": "…"}}, + ] + trace = acp_events_to_ir( + [{"type": "tool_call", "tool_call_id": "t", "content": content}] + ) + blocks = trace.events[0].tool_call.content + rendered = "\n".join(b.text for b in blocks if b.kind is ContentBlockKind.TEXT) + assert rendered == content_blocks_to_text(content) + assert sum(1 for b in blocks if b.kind is ContentBlockKind.OPAQUE) == 2 + + +def test_a_non_object_content_block_is_declared_dropped(): + trace = acp_events_to_ir( + [{"type": "tool_call", "tool_call_id": "t", "content": ["bare string"]}] + ) + assert trace.events[0].tool_call.content == [] + declared = trace.losses.for_field("events[0].tool_call.content") + assert len(declared) == 1 + assert declared[0].loss_class is LossClass.DROPPED + + +def test_a_string_content_field_is_read_like_the_shared_renderer_reads_it(): + trace = acp_events_to_ir( + [{"type": "tool_call", "tool_call_id": "t", "content": "plain output"}] + ) + blocks = trace.events[0].tool_call.content + assert [b.kind for b in blocks] == [ContentBlockKind.TEXT] + assert blocks[0].text == content_blocks_to_text("plain output") + + +# --------------------------------------------------------------------------- +# 8. Loss report completeness +# --------------------------------------------------------------------------- + + +def test_the_report_for_a_real_trace_is_exactly_this_set(): + """Asserted as a set: an undeclared loss and a spurious one both fail.""" + trace = acp_events_to_ir(_rich_events()) + tool_index = next(e.index for e in trace.events if e.kind is EventKind.TOOL_CALL) + assert _fields(trace) == SYSTEMIC_FIELDS | { + f"events[{tool_index}].tool_call.arguments" + } + assert trace.losses.direction == LOSS_DIRECTION + assert loss_summary(trace.losses) == {"unsupported": 6} + + +def test_every_record_names_a_symbol_or_a_document_section(): + """A loss whose detail does not say *why* is a loss nobody can act on.""" + trace = acp_events_to_ir(_rich_events()) + for record in trace.losses.records: + assert record.detail.strip() + assert record.doc_ref or any( + token in record.detail + for token in ("handle_update", "ToolCallRecord", "usage_snapshots", "ATIF") + ), record + + +def test_a_trace_with_no_tool_calls_declares_no_tool_losses(): + trace = acp_events_to_ir([{"type": "agent_message", "text": "hi"}]) + assert _fields(trace) == { + "events[].usage", + "agent.agent_version", + "outcome.stop_reason", + } + + +def test_an_empty_event_list_still_produces_a_report(): + """An empty report would be a claim that nothing was lost.""" + trace = acp_events_to_ir([]) + assert trace.events == [] + assert not trace.losses.lossless + assert validate_trace(trace) == [] + + +# --------------------------------------------------------------------------- +# 9 & 10. The contract itself +# --------------------------------------------------------------------------- + + +def test_validate_trace_is_green_on_every_shape_this_suite_exercises(): + shapes: list[list[Any]] = [ + [], + _rich_events(), + [{"type": "oracle", "command": "x"}], + [{"type": "mystery"}, {"no": "type"}, "not an object", 7], + [{"type": "tool_call"}], + [{"type": "agent_thought", "text": "a\n\nb"}], + [{"type": "tool_call", "tool_call_id": "t", "content": [1, 2, 3]}], + ] + for events in shapes: + trace = acp_events_to_ir(events) + assert validate_trace(trace) == [], (events, validate_trace(trace)) + + +def test_removing_one_declared_loss_makes_the_trace_invalid(): + """The Slice B contract, demonstrated end to end on a real conversion. + + This is what stops a future converter from quietly failing to carry + arguments: the absence is only legal while it is declared. + """ + trace = acp_events_to_ir(_rich_events()) + assert validate_trace(trace) == [] + + field = next(f for f in _fields(trace) if f.endswith(".tool_call.arguments")) + trace.losses.records = [r for r in trace.losses.records if r.field != field] + + issues = validate_trace(trace) + assert any("absence must be declared" in issue for issue in issues), issues + + +def test_dropping_the_whole_report_invalidates_a_tool_bearing_trace(): + trace = acp_events_to_ir(_rich_events()) + trace.losses = None + assert validate_trace(trace) != [] + + +# --------------------------------------------------------------------------- +# 11. Volume and ergonomics +# --------------------------------------------------------------------------- + + +def _synthetic_trace_events(tool_calls: int, chatter: int) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [{"type": "user_message", "text": "go"}] + for n in range(tool_calls): + events.append({"type": "agent_thought", "text": f"thinking {n}"}) + events.append( + { + "type": "tool_call", + "tool_call_id": f"call_{n}", + "kind": "execute", + "title": f"cmd {n}", + "status": "completed", + "content": [ + {"type": "content", "content": {"type": "text", "text": "out"}} + ], + } + ) + for n in range(chatter): + events.append({"type": "agent_message", "text": f"message {n}"}) + return events + + +def test_the_report_grows_with_tool_calls_not_with_trace_length(): + """The ergonomics property that decides whether the contract is workable. + + "Declare every absence" is only affordable if the report is bounded by the + thing the absences are about. Doubling the chatter must not change it. + """ + lean = acp_events_to_ir(_synthetic_trace_events(tool_calls=40, chatter=5)) + chatty = acp_events_to_ir(_synthetic_trace_events(tool_calls=40, chatter=200)) + + assert len(lean.losses.records) == len(chatty.losses.records) + assert len(systemic_losses(lean.losses)) == len(SYSTEMIC_FIELDS) + assert len(per_event_losses(lean.losses)) == 40 + assert len(lean.losses.records) == 40 + len(SYSTEMIC_FIELDS) + + +def test_a_realistic_trace_keeps_the_report_readable(): + """One record per tool call, and a summary that fits on one line.""" + events = _synthetic_trace_events(tool_calls=50, chatter=20) + trace = acp_events_to_ir(events) + + assert len(trace.events) == len(events) + assert loss_summary(trace.losses) == {"unsupported": 55} + # Every per-event record says the same thing about a different call, which + # is why `loss_summary` exists — the detail is worth reading once. + details = {r.detail for r in per_event_losses(trace.losses)} + assert len(details) == 1 + + report_bytes = len(trace.losses.model_dump_json()) + trace_bytes = len(trace.model_dump_json()) + assert report_bytes < trace_bytes diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py index 14cb2b544..ed3dc37b3 100644 --- a/tests/trajectories/test_trace_ir.py +++ b/tests/trajectories/test_trace_ir.py @@ -16,9 +16,10 @@ ACP enum, both read from source. Adding a branch to the capture path fails this suite until the IR accounts for it, exactly as the Slice A conformance suite does for the schema. -* **Isolation** — nothing under ``src/benchflow`` imports the IR. This is the - executable form of "zero runtime behaviour change"; if a future PR wires the - IR into a run path, this test is the one that must be deliberately updated. +* **Isolation** — nothing outside the IR module family imports the IR. This is + the executable form of "zero runtime behaviour change"; if a future PR wires + the IR into a run path, this test is the one that must be deliberately + updated. * **The worked example** — the JSON in ``docs/trace-interop.md`` §8 is built here in code and compared to the block in the document, so the documented example cannot drift from what the models actually produce. @@ -424,35 +425,65 @@ def test_the_oracle_record_has_its_own_kind_and_role(): # --------------------------------------------------------------------------- -def _imports_the_ir(path: Path) -> bool: +IR_FAMILY = ("ir", "ir_from_acp") +"""The unwired modules of the proposal: the IR and its converters. + +They may import each other — a converter that could not import the IR would be +useless — but nothing else may import them. Growing this tuple is how a new +converter joins the family; it is not a way to let a run path in. +""" + +IR_FAMILY_PATHS = {f"src/benchflow/trajectories/{name}.py" for name in IR_FAMILY} + + +def _imported_ir_modules(path: Path) -> set[str]: + """Which members of :data:`IR_FAMILY` *path* imports, read by AST.""" tree = ast.parse(path.read_text(encoding="utf-8")) + found: set[str] = set() for node in ast.walk(tree): if isinstance(node, ast.Import): - if any(alias.name == "benchflow.trajectories.ir" for alias in node.names): - return True + for alias in node.names: + for name in IR_FAMILY: + if alias.name == f"benchflow.trajectories.{name}": + found.add(name) elif isinstance(node, ast.ImportFrom): module = node.module or "" - if module == "benchflow.trajectories.ir" or (node.level and module == "ir"): - return True - if module in ("benchflow.trajectories", "") and any( - alias.name == "ir" for alias in node.names - ): - return True - return False - - -def test_no_runtime_module_imports_the_ir(): - """Slice B is reversible: deleting ``ir.py`` cannot break a run path. + for name in IR_FAMILY: + if module == f"benchflow.trajectories.{name}" or ( + node.level and module == name + ): + found.add(name) + if module in ("benchflow.trajectories", ""): + found.update( + alias.name for alias in node.names if alias.name in IR_FAMILY + ) + return found + + +def test_only_the_ir_family_imports_the_ir(): + """The proposal is reversible: deleting it cannot break a run path. + + Slice B stated this as "nothing under ``src/benchflow`` imports the IR". + Slice C adds `ir_from_acp`, which necessarily imports it, so the property + is restated at the boundary that actually matters — the family is closed, + and no module outside it may reach in. When the IR is deliberately wired into production, this test is the one to - update — and updating it is the moment the reversibility claim ends. + update, and updating it is the moment the reversibility claim ends. """ - importers = [ + importers = { path.relative_to(REPO_ROOT).as_posix() for path in sorted(SRC_ROOT.rglob("*.py")) - if _imports_the_ir(path) - ] - assert importers == [], importers + if _imported_ir_modules(path) + } + outside = importers - IR_FAMILY_PATHS + assert outside == set(), sorted(outside) + + +def test_every_ir_family_module_exists(): + """A stale name in :data:`IR_FAMILY` would silently weaken the test above.""" + missing = [path for path in IR_FAMILY_PATHS if not (REPO_ROOT / path).exists()] + assert missing == [], missing def test_the_ir_module_imports_no_benchflow_runtime_module(): From 48dbbff3ce3e216fada2dfad9fae94538bfab7f6 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Sat, 15 Aug 2026 10:31:00 +0200 Subject: [PATCH 05/23] fix(trajectories): keep nulls in the canonical Trace IR encoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by reading a converted real rollout by hand, not by the test suite, which was green: §8.4 published its worked example with `exclude_none=True`, so `arguments` was absent from the document while the loss report kept a record addressing `events[1].tool_call.arguments`. The declaration that legalizes the absence pointed at a key no reader of that document could find. The suite could not see it because both sides of the comparison used the same non-canonical encoding, so it was self-consistent. `None` in this IR is a positive statement — the source did not carry this field — and every one of them is paired with a `LossRecord` that addresses the field by path. Dropping the key makes the address dangle and collapses "we looked and it was not there" into "this version has no such field". Both encodings re-validate to an equal pydantic model, so the model layer cannot enforce this; the rule is stated in the `ir.py` docstring and enforced by tests. - `ir.py` — declares the canonical encoding: nulls retained, `exclude_none=True` is not a valid encoding of a Trace IR document. No serializer is added: there is no on-disk artifact yet, and a writer would anticipate an interface this proposal has not earned. Adds the corollary that a record names the outermost absent node, so a conversion with no usage declares `usage`, not `usage.input_tokens`. - `CanonicalTrace.outcome` is no longer optional. Applying the new guard immediately found a second instance of the same class: `outcome.stop_reason` is a loss every ACP conversion declares, and it could not resolve in any trace that did not time out, because the section itself was null. It is now always present with `None` fields, like `agent` — which is exactly why `agent.agent_version` resolved and `outcome.stop_reason` did not. - `docs/trace-interop.md` — §8.2 gains the encoding rule as a fourth design choice, §8.4 is regenerated in the canonical encoding so `arguments: null` and its loss record are visible in the same document, §8.5 records the new test-pinned property, and §8.8 records what the human end-to-end pass showed, including the limit below. - Tests — `test_every_concrete_loss_path_resolves_in_the_canonical_encoding` (IR) and `test_the_canonical_document_shows_null_arguments_beside_their_loss_record` plus `test_every_concrete_loss_path_of_a_converted_trace_resolves` (converter). Each asserts in the same test that the discarded encoding *fails* to resolve those paths, so none of them can pass for both encodings at once. Known limit, recorded rather than fixed: the invariant forces a converter to declare an absence, it cannot stop one from writing `arguments: {}` instead of `null`. Such a trace is valid. Closing that would mean the IR taking a position on what an empty map means for each source, which the tri-state rule deliberately leaves to the converter. Verified: tests/trajectories 328 passed; the trajectory, capture, streaming and ACP regression lane 493 passed; `ruff check .`, `ruff format --check` and `ty check src/` clean. Human end-to-end H1-H4 run against real captured rollouts, all PASS. --- docs/trace-interop.md | 127 ++++++++++++++++++---- src/benchflow/trajectories/ir.py | 58 +++++++++- src/benchflow/trajectories/ir_from_acp.py | 9 +- tests/trajectories/test_ir_from_acp.py | 91 +++++++++++++++- tests/trajectories/test_trace_ir.py | 93 +++++++++++++++- 5 files changed, 342 insertions(+), 36 deletions(-) diff --git a/docs/trace-interop.md b/docs/trace-interop.md index ca81edf90..bc456255e 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -546,6 +546,23 @@ Three design choices carry most of the weight: source's own type string next to the normalized `kind`, and `ToolCall.name_semantics` records that an ACP `kind` is a category rather than a function name — the normalization §5.1 currently performs silently. +4. **The canonical JSON encoding keeps the nulls.** A trace serializes with + `model_dump(mode="json")`; **`exclude_none=True` is not a valid encoding of a + Trace IR document.** `None` is a positive statement — *the source did not + carry this* — and the `LossRecord` that legalizes it addresses the field **by + path**. Drop the key and that address stops resolving, so the declaration + becomes unverifiable inside the document that carries it. Both encodings + re-validate to an equal pydantic model, which is exactly why the rule lives + in a test rather than in the type; the audience of an interchange format + reads the JSON. The corollary is that a record names the outermost absent + node — a conversion with no usage declares `usage`, not `usage.input_tokens` + — and that sections every conversion has an opinion about (`agent`, + `outcome`) are always present, with `None` fields inside them. The normative + statement lives in the `ir.py` module docstring. + + *This rule was written after the fact: §8.4 was first published with + `exclude_none=True`, and a human end-to-end read of a converted rollout found + the loss records pointing at keys the document did not contain.* ### 8.3 Planned mapping @@ -643,23 +660,32 @@ it. **FACT.** The document below is produced by the models in `ir.py` and compared against this block by `test_the_documented_example_matches_the_models`, so it -cannot drift from what the code emits. +cannot drift from what the code emits. It is written in the **canonical +encoding** — nulls retained (§8.2). An earlier revision published it with +`exclude_none=True`, which dropped `arguments` from the document while the loss +report kept addressing it; the two together are the point of the example, so +they are now shown together. It shows the four properties the design exists for: an unavailable value that is -absent *and* declared (`arguments`), a thought whose boundaries survive next to -the joined form, a non-text content block carried as `opaque` instead of -skipped, and a timeout that is representable at all — with its source-specific -fields in `extensions` rather than as four new IR fields. +`null` *and* declared (`arguments`, beside its `LossRecord`), a thought whose +boundaries survive next to the joined form, a non-text content block carried as +`opaque` instead of skipped, and a timeout that is representable at all — with +its source-specific fields in `extensions` rather than as four new IR fields. ```json { "ir_version": "bf-trace-ir-v0", + "trace_id": null, "session_id": "rollout-7f3a", "agent": { "agent_name": "gemini", - "model": "gemini-2.5-flash" + "agent_version": null, + "model": "gemini-2.5-flash", + "provider": null }, + "started_at": null, + "finished_at": null, "events": [ { "index": 0, @@ -667,9 +693,17 @@ fields in `extensions` rather than as four new IR fields. "source_type": "user_message", "role": "user", "text": "Count the rows in data.csv", + "reasoning": null, + "reasoning_segments": null, + "tool_call": null, + "started_at": null, + "finished_at": null, + "outcome": null, + "usage": null, "provenance": { "source_format": "acp-capture-v1", - "producer": "_events_to_trajectory" + "producer": "_events_to_trajectory", + "captured_at": null }, "extensions": {} }, @@ -678,6 +712,7 @@ fields in `extensions` rather than as four new IR fields. "kind": "tool_call", "source_type": "tool_call", "role": "agent", + "text": null, "reasoning": "Check the file first.\n\nThen count.", "reasoning_segments": [ "Check the file first.", @@ -689,6 +724,7 @@ fields in `extensions` rather than as four new IR fields. "name_semantics": "acp_kind", "title": "wc -l data.csv", "status": "completed", + "arguments": null, "content": [ { "kind": "text", @@ -703,6 +739,7 @@ fields in `extensions` rather than as four new IR fields. }, { "kind": "opaque", + "text": null, "raw": { "type": "diff", "path": "/w/data.csv", @@ -710,11 +747,18 @@ fields in `extensions` rather than as four new IR fields. "newText": "b" } } - ] + ], + "started_at": null, + "finished_at": null }, + "started_at": null, + "finished_at": null, + "outcome": null, + "usage": null, "provenance": { "source_format": "acp-capture-v1", - "producer": "_events_to_trajectory" + "producer": "_events_to_trajectory", + "captured_at": null }, "extensions": {} }, @@ -722,10 +766,19 @@ fields in `extensions` rather than as four new IR fields. "index": 2, "kind": "timeout", "source_type": "agent_timeout", + "role": null, + "text": null, + "reasoning": null, + "reasoning_segments": null, + "tool_call": null, + "started_at": null, + "finished_at": null, "outcome": "wall_clock_timeout", + "usage": null, "provenance": { "source_format": "acp-capture-v1", - "producer": "_events_to_trajectory" + "producer": "_events_to_trajectory", + "captured_at": null }, "extensions": { "timeout_sec": 90.0, @@ -737,15 +790,22 @@ fields in `extensions` rather than as four new IR fields. "usage": { "input_tokens": 1180, "output_tokens": 96, + "cache_read_tokens": null, + "cache_creation_tokens": null, + "reasoning_tokens": null, "total_tokens": 1276, "source": "llm_proxy_normalized" }, "outcome": { - "status": "timeout" + "status": "timeout", + "stop_reason": null, + "reward": null, + "error_category": null }, "provenance": { "source_format": "acp-capture-v1", - "producer": "_events_to_trajectory" + "producer": "_events_to_trajectory", + "captured_at": null }, "extensions": {}, "losses": { @@ -756,19 +816,24 @@ fields in `extensions` rather than as four new IR fields. "field": "events[1].tool_call.arguments", "loss_class": "unsupported", "detail": "ACPSession.handle_update reads five fields and rawInput is not one of them, so no ACP-derived tool call carries arguments.", - "doc_ref": "§5 loss #1" + "doc_ref": "\u00a75 loss #1" }, { "field": "events[1].tool_call.started_at", "loss_class": "unsupported", "detail": "ToolCallRecord tracks started_at/finished_at in memory and _events_to_trajectory serializes neither.", - "doc_ref": "§5 loss #3" + "doc_ref": "\u00a75 loss #3" } ] } } ``` +Read it beside the `losses.records` at the bottom: every `LossRecord.field` is a +path into this same document, and following it lands on a key that is present +and `null`. That resolvability is what `exclude_none` broke, and what +`test_every_concrete_loss_path_resolves_in_the_canonical_encoding` now guards. + ### 8.5 Invariants **PROPOSAL.** Checked by `validate_trace`, which returns one string per @@ -792,11 +857,15 @@ for each: is genuinely ambiguous today (§5.1, the `oracle` divergence) and the IR does not pretend to settle it. -Two further properties are pinned by tests rather than by `validate_trace`: the -IR's tool-status vocabulary is a superset of the ACP `ToolCallStatus` enum, read -off the enum itself; and every event type `_events_to_trajectory` emits maps to +Three further properties are pinned by tests rather than by `validate_trace`: +the IR's tool-status vocabulary is a superset of the ACP `ToolCallStatus` enum, +read off the enum itself; every event type `_events_to_trajectory` emits maps to an `EventKind`, with the producer's vocabulary read from source by AST — the -same mechanism the Slice A conformance suite uses (§2.2). +same mechanism the Slice A conformance suite uses (§2.2); and **every concrete +`LossRecord.field` resolves to a key that is present in the canonical encoding** +(§8.2, choice 4). The last one asserts in the same test that the discarded +`exclude_none` encoding *fails* to resolve those paths, so it cannot pass for +both encodings at once. ### 8.6 What is provisional, and what a review can still change @@ -839,9 +908,9 @@ module — the IR — and reads the capture format as data. ### 8.8 What the first converter showed -**FACT.** Writing `ACP → IR` was also the first stress test of §8.2's -declared-absence rule, and two things came out of it that a design document -could not have settled: +**FACT.** Writing `ACP → IR` — and then reading a converted real rollout by hand +— was the first stress test of §8.2's declared-absence rule. Three things came +out of it that a design document could not have settled: - **The rule is affordable, because the report is bounded by tool calls rather than by trace length.** Systemic absences — timestamps, per-event usage, agent @@ -855,3 +924,19 @@ could not have settled: reason 50 records say the same thing. An `events[*].…` wildcard would collapse them at the cost of making a single missed call invisible. That trade is open; §8.6 already lists this contract as the most likely thing to change. +- **A loss report addressed by path constrains the encoding, which the design + had not noticed.** Reading a converted rollout by hand — not running the test + suite, which was green and self-consistent — showed the §8.4 example published + with `exclude_none=True`, its loss records pointing at keys the document did + not contain. The rule in §8.2 choice 4 and the guard in §8.5 are the result. + Applying that guard immediately found a second instance of the same class: + `outcome.stop_reason` could not resolve in any trace that did not time out, + because the section itself was `None`, which is why `outcome` is now always + present. + +**Worth stating plainly, because it is a limit and not a fix.** The invariant +forces a converter to *declare* an absence; it cannot stop one from writing +`arguments: {}` instead of `null`. A trace with a fabricated empty argument map +and no loss record is valid. Verified by hand. Closing that would mean the IR +taking a position on what an empty map means for each source, which §8.2's +tri-state rule deliberately leaves to the converter. diff --git a/src/benchflow/trajectories/ir.py b/src/benchflow/trajectories/ir.py index a6716cb3c..154bde985 100644 --- a/src/benchflow/trajectories/ir.py +++ b/src/benchflow/trajectories/ir.py @@ -48,6 +48,37 @@ Today every ACP-derived tool call is the second; ATIF and ADP both serialize the first, which is why their documents read as though the agent called every tool with no arguments. + +## Canonical JSON encoding + +A trace serializes with **every null retained** — ``model_dump(mode="json")`` or +``model_dump_json()``. **``exclude_none=True`` is not a valid encoding of a +Trace IR document.** + +This is a semantic rule, not a formatting preference. ``None`` here is a +positive statement — *the source did not carry this field* — and every such +statement is paired with a :class:`LossRecord` that addresses the field **by +path**. Drop the key and the record points at something a reader of the document +cannot find: the declaration that makes the absence legal becomes unverifiable +inside the very document that carries it, and "we looked and it was not there" +becomes indistinguishable from "this version has no such field". + +A pydantic consumer is unaffected either way — both encodings re-validate to an +equal model — but the audience of an interchange format reads the JSON, and it +is the JSON that has to be self-describing. + +No dedicated serializer ships with this module. There is no on-disk artifact +yet, and providing a writer would anticipate an interface this proposal has not +earned. The rule is enforced by +``test_every_concrete_loss_path_resolves_in_the_canonical_encoding`` rather than +by a function, so a future writer inherits it instead of redefining it. + +**Corollary — address the outermost absent node.** A record may only name a path +that resolves, so when a whole section is missing the record names the section, +not a field inside it: a conversion with no usage at all declares ``usage``, not +``usage.input_tokens``. Sections that every conversion has an opinion about — +:attr:`CanonicalTrace.agent`, :attr:`CanonicalTrace.outcome` — are therefore +always present, with ``None`` fields inside them. """ from __future__ import annotations @@ -400,9 +431,12 @@ class TraceOutcome(BaseModel): ``stop_reason`` is captured on ``ACPSession`` and exported nowhere (§5 loss #9); ``reward`` and ``error_category`` live in ``result.json``, outside the - trajectory. All optional — a trace built from the capture file alone has - none of them, and that is a declarable loss rather than a reason to fake a - value. + trajectory. All fields optional — a trace built from the capture file alone + has none of them, and that is a declarable loss rather than a reason to fake + a value. + + The *section* is not optional on :class:`CanonicalTrace`, unlike its fields; + see :attr:`CanonicalTrace.outcome`. """ model_config = ConfigDict(extra="forbid") @@ -428,7 +462,19 @@ class CanonicalTrace(BaseModel): finished_at: datetime | None = None events: list[TraceEvent] = Field(default_factory=list) usage: TraceUsage | None = None - outcome: TraceOutcome | None = None + """``None`` when no source measured usage at all. A conversion that has none + declares the loss against ``usage``, the outermost absent node — see the + addressing rule in the module docstring.""" + + outcome: TraceOutcome = Field(default_factory=TraceOutcome) + """Always present, like :attr:`agent`, even when every field inside it is + ``None``. + + It was briefly optional, and that made ``outcome.stop_reason`` — a loss + every ACP conversion declares — unresolvable in any trace that did not time + out, because the parent object was ``null``. A section that loss records + address by path has to exist for the path to land.""" + provenance: Provenance extensions: dict[str, Any] = Field(default_factory=dict) @@ -467,7 +513,9 @@ def validate_trace(trace: CanonicalTrace) -> list[str]: 7. **No silent absence.** Every ``arguments is None`` has a matching loss record at ``events[i].tool_call.arguments``. This is the invariant that makes the loss report a contract instead of documentation: a converter - that quietly fails to carry arguments produces an invalid trace. + that quietly fails to carry arguments produces an invalid trace. Note + that the record addresses the field *by path*, which is why the canonical + encoding has to keep that path resolvable — see the module docstring. 8. **Role coherence.** A ``USER_MESSAGE`` is attributed to ``USER`` or to nobody. The IR checks only this direction; agent-side attribution is genuinely ambiguous today (§5, the ``oracle`` divergence) and the IR does diff --git a/src/benchflow/trajectories/ir_from_acp.py b/src/benchflow/trajectories/ir_from_acp.py index 00dea4691..f90134a4d 100644 --- a/src/benchflow/trajectories/ir_from_acp.py +++ b/src/benchflow/trajectories/ir_from_acp.py @@ -469,8 +469,13 @@ def acp_events_to_ir( events=ir_events, # A timeout marker in the stream is an observation about how the run # ended. Every other outcome — pass, fail, reward — lives in - # ``result.json``, which this converter does not read. - outcome=TraceOutcome(status=OutcomeStatus.TIMEOUT) if timed_out else None, + # ``result.json``, which this converter does not read, so the status + # stays ``None`` rather than being guessed. The section itself is always + # present: ``outcome.stop_reason`` is a loss this converter always + # declares, and a record cannot address a path through a null parent. + outcome=TraceOutcome( + status=OutcomeStatus.TIMEOUT if timed_out else None, + ), provenance=Provenance(source_format=ACP_TRAJECTORY_SOURCE), losses=losses, ) diff --git a/tests/trajectories/test_ir_from_acp.py b/tests/trajectories/test_ir_from_acp.py index 1d92dbe10..240f7d39d 100644 --- a/tests/trajectories/test_ir_from_acp.py +++ b/tests/trajectories/test_ir_from_acp.py @@ -59,6 +59,7 @@ TIMEOUT_SEC, _rich_events, ) +from tests.trajectories.test_trace_ir import resolve_ir_path # The systemic records every conversion of a tool-bearing trace declares. Held # as a literal so adding one is a deliberate edit to this suite. @@ -239,6 +240,76 @@ def test_the_serialized_trace_never_carries_an_empty_argument_map(): assert '"arguments": {}' not in json.dumps(document, indent=1) +def test_the_canonical_document_shows_null_arguments_beside_their_loss_record(): + """The three facts that have to hold together, checked together. + + Found by hand during the Slice C human E2E: the E2E procedure claimed no + ``arguments`` key should appear at all, which inverted the property. What + must hold is that the key **is** there, explicitly null, and that the loss + record declaring it resolves to that exact key in the canonical encoding. + + Written as one test because the three facts are only meaningful together: + a null with no record is an undeclared absence, a record with no key is an + unverifiable declaration, and ``{}`` is a fabrication. + """ + trace = acp_events_to_ir(_rich_events()) + document = trace.model_dump(mode="json") + serialized = json.dumps(document, indent=1) + + tool_events = [e for e in document["events"] if e["kind"] == "tool_call"] + assert tool_events, "the fixture must contain a tool call" + + for event in tool_events: + # 1. the key is present and explicitly null … + assert "arguments" in event["tool_call"] + assert event["tool_call"]["arguments"] is None + # 2. … and its declaration resolves to that key. + field = f"events[{event['index']}].tool_call.arguments" + assert trace.losses.for_field(field), field + resolved, value = resolve_ir_path(document, field) + assert resolved and value is None, (field, resolved, value) + + # 3. never the fabricated form ATIF and ADP ship. + assert '"arguments": {}' not in serialized + assert '"arguments": null' in serialized + + # The encoding this replaced satisfies none of it. + lean = trace.model_dump(mode="json", exclude_none=True) + assert all( + "arguments" not in e["tool_call"] + for e in lean["events"] + if e["kind"] == "tool_call" + ) + assert not resolve_ir_path( + lean, f"events[{tool_events[0]['index']}].tool_call.arguments" + )[0] + + +def test_every_concrete_loss_path_of_a_converted_trace_resolves(): + """The IR-level encoding rule, applied to real converter output. + + ``test_trace_ir.py`` pins it on the documented example; this pins it on + every shape this suite converts, so a future converter that invents a path + for a field it did not emit fails here. + """ + shapes: list[list[Any]] = [ + _rich_events(), + [{"type": "tool_call", "tool_call_id": "t", "content": ["bare"]}], + [{"type": "agent_message", "text": 42}], + [{"type": "tool_call", "status": "exploded"}], + ] + for events in shapes: + trace = acp_events_to_ir(events) + document = trace.model_dump(mode="json") + for record in trace.losses.records: + if record.field.startswith("events[]") or record.field.startswith( + "source[" + ): + continue # systemic and source-side paths address no IR field + resolved, _ = resolve_ir_path(document, record.field) + assert resolved, (record.field, events) + + # --------------------------------------------------------------------------- # 4. Timeout # --------------------------------------------------------------------------- @@ -260,10 +331,24 @@ def test_the_timeout_marker_survives_with_its_fields(): assert trace.outcome.status is OutcomeStatus.TIMEOUT -def test_a_trace_without_a_timeout_claims_no_outcome(): - """The capture events say nothing about pass/fail; that lives in result.json.""" +def test_a_trace_without_a_timeout_claims_no_outcome_status(): + """The capture events say nothing about pass/fail; that lives in result.json. + + The outcome *section* is still present with every field ``None``: this + converter always declares ``outcome.stop_reason`` as a loss, and that path + could not resolve through a null parent. + """ trace = acp_events_to_ir([{"type": "agent_message", "text": "done"}]) - assert trace.outcome is None + assert trace.outcome.status is None + assert trace.outcome.stop_reason is None + document = trace.model_dump(mode="json") + assert document["outcome"] == { + "status": None, + "stop_reason": None, + "reward": None, + "error_category": None, + } + assert resolve_ir_path(document, "outcome.stop_reason") == (True, None) # --------------------------------------------------------------------------- diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py index ed3dc37b3..8f72dfe46 100644 --- a/tests/trajectories/test_trace_ir.py +++ b/tests/trajectories/test_trace_ir.py @@ -20,9 +20,12 @@ the executable form of "zero runtime behaviour change"; if a future PR wires the IR into a run path, this test is the one that must be deliberately updated. -* **The worked example** — the JSON in ``docs/trace-interop.md`` §8 is built - here in code and compared to the block in the document, so the documented - example cannot drift from what the models actually produce. +* **The worked example and the canonical encoding** — the JSON in + ``docs/trace-interop.md`` §8 is built here in code and compared to the block + in the document, so the documented example cannot drift from what the models + actually produce. The encoding it is written in is itself pinned: nulls are + retained, because a dropped key leaves the loss record that declares it + pointing at nothing. Nothing here writes outside ``tmp_path`` and no runtime module is imported for its side effects. @@ -34,6 +37,7 @@ import json import re from pathlib import Path +from typing import Any import pytest from pydantic import ValidationError @@ -631,6 +635,80 @@ def test_the_example_demonstrates_what_the_ir_is_for(): assert trace.events[2].extensions["timeout_sec"] == 90.0 +def resolve_ir_path(document: dict, path: str) -> tuple[bool, Any]: + """Walk a `LossRecord.field` path through a serialized trace. + + Returns ``(resolved, value)``. ``resolved`` is False as soon as a segment + names a key the document does not contain — which is the whole point: a + loss record that addresses a missing key is a declaration nobody can check. + + Shared with the Slice C suite, so both directions of the hub check the same + property with the same walker. + """ + node: Any = document + for part in re.findall(r"[^.\[\]]+|\[\d+\]", path): + if part.startswith("["): + index = int(part[1:-1]) + if not isinstance(node, list) or index >= len(node): + return False, None + node = node[index] + else: + if not isinstance(node, dict) or part not in node: + return False, None + node = node[part] + return True, node + + +def test_every_concrete_loss_path_resolves_in_the_canonical_encoding(): + """The canonical encoding keeps every declared absence addressable. + + Guards a real defect: the §8.4 example was published with + ``exclude_none=True``, which drops ``arguments`` entirely while the loss + report kept pointing at it — the declaration that makes the absence legal + addressed a key no reader of that document could find. + + The second half is what makes this a guard rather than a tautology: it + asserts the discarded encoding *fails*, so the test cannot pass for both. + """ + trace = _example_trace() + canonical = trace.model_dump(mode="json") + + concrete = [ + record.field + for record in trace.losses.records + if record.field.startswith("events[") + and not record.field.startswith("events[]") + ] + assert concrete, "the example must declare at least one concrete-path loss" + + unresolved = [ + field for field in concrete if not resolve_ir_path(canonical, field)[0] + ] + assert unresolved == [], unresolved + + # Every one of them addresses a field that is present and explicitly null — + # the positive statement "the source did not carry this". + for field in concrete: + resolved, value = resolve_ir_path(canonical, field) + assert resolved and value is None, (field, value) + + # And the encoding this replaced does not resolve them. + lean = trace.model_dump(mode="json", exclude_none=True) + assert all(not resolve_ir_path(lean, field)[0] for field in concrete), ( + "exclude_none must not resolve these paths; if it does, this guard has " + "stopped discriminating between the two encodings" + ) + + +def test_the_canonical_encoding_round_trips_like_the_model(): + """Keeping the nulls costs nothing in fidelity — it only adds legibility.""" + trace = _example_trace() + canonical = trace.model_dump(mode="json") + restored = CanonicalTrace.model_validate(json.loads(json.dumps(canonical))) + assert restored == trace + assert validate_trace(restored) == [] + + def _documented_example() -> dict: """The JSON block tagged ```` in the interop document.""" text = DOCS_PATH.read_text(encoding="utf-8") @@ -644,6 +722,11 @@ def _documented_example() -> dict: def test_the_documented_example_matches_the_models(): - """A doc example that cannot drift, because the drift fails a test.""" - produced = _example_trace().model_dump(mode="json", exclude_none=True) + """A doc example that cannot drift, because the drift fails a test. + + Serialized in the canonical encoding — nulls retained. The example is the + one place a reviewer sees what a trace looks like, so it must not be the + one place that shows an encoding the IR does not accept. + """ + produced = _example_trace().model_dump(mode="json") assert produced == _documented_example() From a41c2161d97ac01652220ab303b330883dae3e26 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Sat, 15 Aug 2026 10:49:05 +0200 Subject: [PATCH 06/23] feat(trajectories): declare which document a LossRecord path addresses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preparation for the first outbound converter, and a gap the inbound one had already papered over. `LossRecord.field` was documented as a path in IR terms, but not every record is about an IR node. Slice C already needed an escape and invented a `source[i]` string convention in a module docstring, recognized by `startswith` in two guards. The ATIF edge makes the same gap unavoidable in the other direction: a prompt-derived step, a `message: ""` the format requires, a `total_steps` the document computes about itself — none of these has an IR antecedent, and giving them an invented IR path would produce an address that does not resolve. Adds `PathSpace` — `hub` · `source` · `target` — and `LossRecord.space`, defaulting to `hub` so every record written before this field existed keeps its meaning. `field` is the path *inside* its space and carries no prefix repeating it: the space is a property of the record, never inferred from the string. Two records may legitimately hold the identical path in different spaces, which is exactly the `acp -> ir` case where a skipped input entry and the IR event that inherited its index are both `events[3]`. Three spaces cover every direction and a new format adds none, because every edge has the IR on exactly one side and therefore exactly one non-hub space: an inbound edge can talk about its source, an outbound one about its target. Only `hub` records compose across edges — the IR is the output of an inbound conversion and the input of an outbound one, so `events[1].tool_call.arguments` denotes the same field in both reports and the records join on it. `source` and `target` records are terminal by construction. No unified report is introduced; composition happens at read time over `(direction, field, space)`. The same asymmetry settles which side owns a report, now stated explicitly in the module docstring: a trace is built exactly once so an inbound conversion may attach its report to it, while a trace may be converted to many targets so an outbound conversion returns its report and leaves `trace.losses` untouched. - `validate_trace` invariant 7 now requires a *hub* record. A `target` record holding the same path addresses another document and declares nothing about this trace; previously the string alone would have satisfied it. - Both canonical-path guards filter on `space is HUB` instead of testing string prefixes. `_is_per_event` classifies by space and takes a record rather than a string. - `LossReport` gains `by_space`, and `for_field` takes the space — the space is part of the address, so it is not defaulted away silently. - `ir_from_acp` migrates its one `source[i]` record to `space=SOURCE, field=events[i]`: the path is now the one the input actually uses, since the space no longer has to be spelled into the string. Five new tests pin the contract, including that no space is inferred from the string in either direction: a `source`/`target` record with a hub-shaped path does not satisfy the declared-absence invariant, and a hub record with a source-shaped path does. `docs/trace-interop.md` §8.2 gains the spaces and the report-ownership rule as a fifth design choice; §8.4 is regenerated so the example carries the new field. Additive: no existing path changes meaning, no runtime module imports either IR module, and no format, exporter or artifact changes. Verified: tests/trajectories 334 passed; the wider trajectory/capture/ACP lane 466 passed; `ruff check .`, `ruff format --check` and `ty check src/` clean. --- docs/trace-interop.md | 26 +++++ src/benchflow/trajectories/ir.py | 123 +++++++++++++++++++--- src/benchflow/trajectories/ir_from_acp.py | 35 +++--- tests/trajectories/test_ir_from_acp.py | 23 ++-- tests/trajectories/test_trace_ir.py | 118 ++++++++++++++++++++- 5 files changed, 289 insertions(+), 36 deletions(-) diff --git a/docs/trace-interop.md b/docs/trace-interop.md index bc456255e..d472f6a24 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -563,6 +563,30 @@ Three design choices carry most of the weight: *This rule was written after the fact: §8.4 was first published with `exclude_none=True`, and a human end-to-end read of a converted rollout found the loss records pointing at keys the document did not contain.* +5. **A loss record declares which document its path addresses.** Not every + record is about an IR node: an inbound edge can read an input element that + becomes no IR node at all, and an outbound edge can emit a value the IR never + held — one its target format requires, or one supplied by the conversion + context. `LossRecord.space` is `hub` · `source` · `target`, defaulting to + `hub`, and `field` is the path *inside* that space with no prefix repeating + it. Three spaces cover every direction, because **every edge has the IR on + exactly one side** and therefore exactly one non-hub space: OTel will add + none. + + Only `hub` records compose across edges — the IR is the output of an inbound + conversion and the input of an outbound one, so `events[1].tool_call.arguments` + denotes the same field in both reports and the records join on it: `acp -> ir` + declares it `unsupported`, `ir -> atif` declares it `synthesized`, and read + together they are the whole history of that field along the pipeline. + `source` and `target` records are terminal by construction. There is no + unified report; composition happens at read time over + `(direction, field, space)`. + + **Which side owns the report** follows from the same asymmetry. A trace is + built exactly once, so an inbound conversion may attach its report to the + trace (`CanonicalTrace.losses`). A trace may be converted to many targets, so + an outbound conversion returns its report alongside its document and leaves + `trace.losses` untouched. ### 8.3 Planned mapping @@ -814,12 +838,14 @@ its source-specific fields in `extensions` rather than as four new IR fields. "records": [ { "field": "events[1].tool_call.arguments", + "space": "hub", "loss_class": "unsupported", "detail": "ACPSession.handle_update reads five fields and rawInput is not one of them, so no ACP-derived tool call carries arguments.", "doc_ref": "\u00a75 loss #1" }, { "field": "events[1].tool_call.started_at", + "space": "hub", "loss_class": "unsupported", "detail": "ToolCallRecord tracks started_at/finished_at in memory and _events_to_trajectory serializes neither.", "doc_ref": "\u00a75 loss #3" diff --git a/src/benchflow/trajectories/ir.py b/src/benchflow/trajectories/ir.py index 154bde985..7f4888355 100644 --- a/src/benchflow/trajectories/ir.py +++ b/src/benchflow/trajectories/ir.py @@ -79,6 +79,45 @@ ``usage.input_tokens``. Sections that every conversion has an opinion about — :attr:`CanonicalTrace.agent`, :attr:`CanonicalTrace.outcome` — are therefore always present, with ``None`` fields inside them. + +Both rules apply to :attr:`PathSpace.HUB` records only; see below. + +## Path spaces + +Not every record is about a node of the IR. An inbound edge can read an input +element that becomes no IR node at all, and an outbound edge can emit a value +the IR never held — a field its target format requires, or one supplied by the +conversion context rather than by the trace. Neither has an IR path, and +inventing one would produce an address that does not resolve. + +:class:`PathSpace` states which document a record's path addresses, so the space +is a property of the record rather than something inferred from the string. See +that class for the three values and why three is enough for any format. + +Two consequences worth stating plainly: + +- **Only ``HUB`` records compose across edges.** The IR is the output of an + inbound conversion and the input of an outbound one, so ``events[1].tool_call.arguments`` + denotes the same field in both reports and the two records join on it: the + ACP edge declares it ``UNSUPPORTED`` (the source never carried arguments), the + ATIF edge declares it ``SYNTHESIZED`` (the target required a value). Read + together they are the whole history of one field along the pipeline. +- **``SOURCE`` and ``TARGET`` records are terminal.** They name objects in + documents that only one edge ever sees, so joining them across edges would be + meaningless. + +## Which side owns the report + +A report belongs to a *conversion*, not to a document — but for one direction +the distinction collapses, and that is why :attr:`CanonicalTrace.losses` exists: + +- **Inbound** (``X -> IR``): a trace is built exactly once, by one conversion, + so its report may be attached to it. A trace separated from the record of what + building it cost is a trace whose absences cannot be checked. +- **Outbound** (``IR -> Y``): one trace may be converted to ATIF, to OTel and to + ADP, so there are *N* reports and none of them describes how the trace came to + exist. An outbound converter therefore **returns** its report alongside its + document and leaves ``trace.losses`` untouched. """ from __future__ import annotations @@ -120,15 +159,50 @@ class LossClass(StrEnum): observed one (ATIF's ``call_{n}`` ids, ADP's ``call_NNNNNN``).""" +class PathSpace(StrEnum): + """Which document a :attr:`LossRecord.field` path addresses. + + Every conversion has the IR on exactly one side, so it has exactly one + non-hub space: an inbound edge (``X -> IR``) can talk about its *source*, an + outbound edge (``IR -> Y``) about its *target*. Three spaces therefore cover + every direction, and a new format adds none. + + The space is a property of the record, never of the string: ``field`` is the + path *inside* its space and carries no prefix announcing which one. Two + records may legitimately hold the identical path in different spaces — for + ``acp -> ir``, source ``events[3]`` and hub ``events[3]`` are different + objects whenever an earlier entry produced no IR event. + """ + + HUB = "hub" + """A node of the canonical IR. **The only composable space**: the IR is the + output of an inbound edge and the input of an outbound one, so the same path + means the same thing in both reports and their records join on it.""" + + SOURCE = "source" + """An element of an inbound edge's input with no corresponding IR node.""" + + TARGET = "target" + """A value an outbound edge produced with no IR antecedent — required by the + target format, or supplied by the conversion context rather than by the + trace.""" + + class LossRecord(BaseModel): """One declared information loss, addressed to a field.""" model_config = ConfigDict(extra="forbid") field: str - """Dotted path in IR terms, e.g. ``events[3].tool_call.arguments``. The - path addresses the IR, not the source document, so one vocabulary covers - every direction.""" + """Dotted path **within** :attr:`space`, e.g. ``events[3].tool_call.arguments``. + + For :attr:`PathSpace.HUB` — the default, and the only one the resolvability + guard checks — this addresses the canonical IR, so one vocabulary covers + every direction and records from different edges join on it.""" + + space: PathSpace = PathSpace.HUB + """Which document :attr:`field` addresses. Defaults to the hub, so every + record written before this field existed keeps its meaning.""" loss_class: LossClass detail: str @@ -160,18 +234,34 @@ def add( loss_class: LossClass, detail: str, doc_ref: str | None = None, + space: PathSpace = PathSpace.HUB, ) -> None: self.records.append( LossRecord( - field=field, loss_class=loss_class, detail=detail, doc_ref=doc_ref + field=field, + space=space, + loss_class=loss_class, + detail=detail, + doc_ref=doc_ref, ) ) def by_class(self, loss_class: LossClass) -> list[LossRecord]: return [r for r in self.records if r.loss_class is loss_class] - def for_field(self, field: str) -> list[LossRecord]: - return [r for r in self.records if r.field == field] + def by_space(self, space: PathSpace) -> list[LossRecord]: + return [r for r in self.records if r.space is space] + + def for_field( + self, field: str, space: PathSpace = PathSpace.HUB + ) -> list[LossRecord]: + """Records addressing *field* in *space*. + + The space is part of the address: the same string in two spaces names + two different objects, so it is not defaulted away silently — the + default is the hub because that is the composable space. + """ + return [r for r in self.records if r.field == field and r.space is space] @property def lossless(self) -> bool: @@ -510,12 +600,15 @@ def validate_trace(trace: CanonicalTrace) -> list[str]: second, divergent one. 6. **Reasoning presence.** An ``AGENT_REASONING`` event carries reasoning in one of the two fields. - 7. **No silent absence.** Every ``arguments is None`` has a matching loss - record at ``events[i].tool_call.arguments``. This is the invariant that - makes the loss report a contract instead of documentation: a converter - that quietly fails to carry arguments produces an invalid trace. Note - that the record addresses the field *by path*, which is why the canonical - encoding has to keep that path resolvable — see the module docstring. + 7. **No silent absence.** Every ``arguments is None`` has a matching + :attr:`PathSpace.HUB` loss record at ``events[i].tool_call.arguments``. + This is the invariant that makes the loss report a contract instead of + documentation: a converter that quietly fails to carry arguments produces + an invalid trace. The space is checked, not guessed from the string — a + ``TARGET`` record that happens to hold the same path addresses another + document and does not declare anything about this one. And because the + record addresses the field *by path*, the canonical encoding has to keep + that path resolvable; see the module docstring. 8. **Role coherence.** A ``USER_MESSAGE`` is attributed to ``USER`` or to nobody. The IR checks only this direction; agent-side attribution is genuinely ambiguous today (§5, the ``oracle`` divergence) and the IR does @@ -529,7 +622,11 @@ def validate_trace(trace: CanonicalTrace) -> list[str]: "v0 defines no migration" ) - declared_losses = {r.field for r in (trace.losses.records if trace.losses else [])} + declared_losses = { + record.field + for record in (trace.losses.records if trace.losses else []) + if record.space is PathSpace.HUB + } for position, event in enumerate(trace.events): where = f"events[{position}]" diff --git a/src/benchflow/trajectories/ir_from_acp.py b/src/benchflow/trajectories/ir_from_acp.py index f90134a4d..06e58dd80 100644 --- a/src/benchflow/trajectories/ir_from_acp.py +++ b/src/benchflow/trajectories/ir_from_acp.py @@ -34,14 +34,16 @@ ## Loss addressing -Two path shapes appear in the report, and the difference is load-bearing: +Records come in two path spaces, and the difference is load-bearing: -- ``events[i].…`` — a field of the IR event at index *i*. This is what +- :attr:`~benchflow.trajectories.ir.PathSpace.HUB`, ``events[i].…`` — a field of + the IR event at index *i*. This is what :func:`~benchflow.trajectories.ir.validate_trace` matches against, so the per-call `arguments` records must use it. -- ``source[i]`` — an entry of the *input* list that produced no IR event at all. - It cannot be addressed as ``events[i]``, because that index belongs to a - different event once an entry is skipped. +- :attr:`~benchflow.trajectories.ir.PathSpace.SOURCE`, ``events[i]`` — an entry + of the *input* list that produced no IR event at all. It cannot be a hub path, + because that index belongs to a different event once an entry is skipped; the + space is what distinguishes the two, not the string. Systemic losses — the ones that hold for every event of a kind rather than for one event — are declared **once** with an unindexed ``events[].…`` path. Writing @@ -78,6 +80,7 @@ LossReport, ModelInfo, OutcomeStatus, + PathSpace, Provenance, Role, ToolCall, @@ -337,10 +340,11 @@ def acp_events_to_ir( for source_position, raw in enumerate(events): if not isinstance(raw, dict): losses.add( - f"source[{source_position}]", + f"events[{source_position}]", LossClass.DROPPED, f"entry is {type(raw).__name__}, not a JSON object; the IR has no " "representation for it", + space=PathSpace.SOURCE, ) continue @@ -493,15 +497,20 @@ def loss_summary(report: LossReport) -> dict[str, int]: return summary -def _is_per_event(field: str) -> bool: +def _is_per_event(record: LossRecord) -> bool: """True for a record addressed to one event or one source entry. - ``events[]…`` is the unindexed, systemic form and is deliberately not - per-event, so the prefix test has to run before the general one. + Within the hub space, ``events[]…`` is the unindexed *systemic* form and + ``events[i]…`` the per-event one, so that prefix test runs first. Source + records are per-entry by construction. """ - if field.startswith("events[]"): + if record.space is PathSpace.SOURCE: + return True + if record.space is not PathSpace.HUB: return False - return field.startswith(("events[", "source[")) + if record.field.startswith("events[]"): + return False + return record.field.startswith("events[") def systemic_losses(report: LossReport) -> list[LossRecord]: @@ -510,9 +519,9 @@ def systemic_losses(report: LossReport) -> list[LossRecord]: The complement — :func:`per_event_losses` — is the part that grows with the trace, and is what makes the report's size worth watching. """ - return [record for record in report.records if not _is_per_event(record.field)] + return [record for record in report.records if not _is_per_event(record)] def per_event_losses(report: LossReport) -> list[LossRecord]: """The records addressed to a single event or a single source entry.""" - return [record for record in report.records if _is_per_event(record.field)] + return [record for record in report.records if _is_per_event(record)] diff --git a/tests/trajectories/test_ir_from_acp.py b/tests/trajectories/test_ir_from_acp.py index 240f7d39d..fa6f54d05 100644 --- a/tests/trajectories/test_ir_from_acp.py +++ b/tests/trajectories/test_ir_from_acp.py @@ -38,6 +38,7 @@ EventKind, LossClass, OutcomeStatus, + PathSpace, Role, ToolStatus, validate_trace, @@ -190,11 +191,12 @@ def test_ordering_and_dense_indices_are_preserved(): def test_an_unrepresentable_entry_leaves_no_hole(): - """A skipped source entry is declared under ``source[i]``, not ``events[i]``. + """A skipped source entry is declared in the SOURCE space, not the hub. - The distinction matters: index 1 of the IR belongs to the event that - followed the skipped entry, so addressing the loss as ``events[1]`` would - blame a different event. + The distinction matters and the space is what carries it: index 1 of the IR + belongs to the event that followed the skipped entry, so a *hub* record at + ``events[1]`` would blame a different event. The identical path in the + source space names the input entry instead. """ events: list[Any] = [ {"type": "agent_message", "text": "a"}, @@ -205,9 +207,12 @@ def test_an_unrepresentable_entry_leaves_no_hole(): assert [e.index for e in trace.events] == [0, 1] assert [e.text for e in trace.events] == ["a", "b"] - dropped = trace.losses.for_field("source[1]") + + dropped = trace.losses.for_field("events[1]", PathSpace.SOURCE) assert len(dropped) == 1 assert dropped[0].loss_class is LossClass.DROPPED + # …and nothing was declared about the hub event that now holds index 1. + assert trace.losses.for_field("events[1]") == [] assert validate_trace(trace) == [] @@ -302,10 +307,10 @@ def test_every_concrete_loss_path_of_a_converted_trace_resolves(): trace = acp_events_to_ir(events) document = trace.model_dump(mode="json") for record in trace.losses.records: - if record.field.startswith("events[]") or record.field.startswith( - "source[" - ): - continue # systemic and source-side paths address no IR field + if record.space is not PathSpace.HUB: + continue # addresses another document entirely + if record.field.startswith("events[]"): + continue # the unindexed systemic form, by convention resolved, _ = resolve_ir_path(document, record.field) assert resolved, (record.field, events) diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py index 8f72dfe46..9f3c156fe 100644 --- a/tests/trajectories/test_trace_ir.py +++ b/tests/trajectories/test_trace_ir.py @@ -51,9 +51,11 @@ ContentBlockKind, EventKind, LossClass, + LossRecord, LossReport, ModelInfo, OutcomeStatus, + PathSpace, Provenance, Role, ToolCall, @@ -365,6 +367,119 @@ def test_loss_report_is_empty_only_as_a_claim(): ) +# --------------------------------------------------------------------------- +# Path spaces +# --------------------------------------------------------------------------- + + +def test_the_three_path_spaces_are_the_documented_ones(): + """Three spaces cover every direction; a new format must add none.""" + assert {space.value for space in PathSpace} == {"hub", "source", "target"} + + +def test_hub_is_the_default_so_older_records_keep_their_meaning(): + """The field is additive: a record written without a space is a hub record.""" + record = LossRecord( + field="events[0].tool_call.arguments", + loss_class=LossClass.UNSUPPORTED, + detail="no rawInput", + ) + assert record.space is PathSpace.HUB + report = LossReport(direction="acp->ir") + report.add("agent.agent_version", LossClass.SYNTHESIZED, "ATIF requires it") + assert report.records[0].space is PathSpace.HUB + + +def test_the_same_path_in_two_spaces_names_two_different_objects(): + """`for_field` therefore takes the space, and does not default it away.""" + report = LossReport(direction="ir->atif") + report.add("events[0]", LossClass.DROPPED, "hub event dropped") + report.add( + "events[0]", LossClass.SYNTHESIZED, "target step", space=PathSpace.TARGET + ) + + assert len(report.records) == 2 + assert len(report.for_field("events[0]")) == 1 + assert report.for_field("events[0]")[0].loss_class is LossClass.DROPPED + assert ( + report.for_field("events[0]", PathSpace.TARGET)[0].loss_class + is LossClass.SYNTHESIZED + ) + assert len(report.by_space(PathSpace.HUB)) == 1 + assert len(report.by_space(PathSpace.TARGET)) == 1 + + +def test_only_hub_records_satisfy_the_declared_absence_invariant(): + """The space is checked, never inferred from the string. + + A `TARGET` record holding the identical path addresses another document, so + it declares nothing about this trace — and a `SOURCE` one likewise. If the + invariant sniffed the string, both would wrongly satisfy it. + """ + for space in (PathSpace.SOURCE, PathSpace.TARGET): + losses = LossReport(direction="ir->atif") + losses.add( + "events[0].tool_call.arguments", + LossClass.SYNTHESIZED, + "same path, different document", + space=space, + ) + trace = _trace( + _event(0, EventKind.TOOL_CALL, tool_call=ToolCall(arguments=None)), + losses=losses, + ) + issues = validate_trace(trace) + assert any("absence must be declared" in issue for issue in issues), ( + space, + issues, + ) + + hub = LossReport(direction="acp->ir") + hub.add("events[0].tool_call.arguments", LossClass.UNSUPPORTED, "no rawInput") + assert ( + validate_trace( + _trace( + _event(0, EventKind.TOOL_CALL, tool_call=ToolCall(arguments=None)), + losses=hub, + ) + ) + == [] + ) + + +def test_non_hub_records_are_not_read_as_ir_paths(): + """A source or target path may be unresolvable in the IR without penalty. + + Both of these address documents the IR does not contain, so the canonical + resolvability guard must not look at them at all. + """ + losses = LossReport(direction="ir->atif") + losses.add("events[7]", LossClass.DROPPED, "input entry", space=PathSpace.SOURCE) + losses.add( + "final_metrics.total_steps", + LossClass.SYNTHESIZED, + "target-only value", + space=PathSpace.TARGET, + ) + trace = _trace(_event(0, EventKind.AGENT_MESSAGE, text="hi"), losses=losses) + + assert validate_trace(trace) == [] + + canonical = trace.model_dump(mode="json") + for record in trace.losses.records: + assert record.space is not PathSpace.HUB + # Unresolvable as an IR path, and that is precisely why it is not one. + assert not resolve_ir_path(canonical, record.field)[0] + + +def test_the_space_survives_the_canonical_encoding(): + report = LossReport(direction="ir->atif") + report.add("steps[0]", LossClass.SYNTHESIZED, "prompt step", space=PathSpace.TARGET) + document = report.model_dump(mode="json") + assert document["records"][0]["space"] == "target" + assert LossReport.model_validate(document).records[0].space is PathSpace.TARGET + + def test_every_loss_class_of_the_documented_taxonomy_exists(): """The four classes are the ones §5.1 already uses; renaming one is a change.""" assert {c.value for c in LossClass} == { @@ -676,7 +791,8 @@ def test_every_concrete_loss_path_resolves_in_the_canonical_encoding(): concrete = [ record.field for record in trace.losses.records - if record.field.startswith("events[") + if record.space is PathSpace.HUB + and record.field.startswith("events[") and not record.field.startswith("events[]") ] assert concrete, "the example must declare at least one concrete-path loss" From 4b734fdde4bd775eaa53b4c025918d9e56adb25b Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Sat, 15 Aug 2026 10:58:56 +0200 Subject: [PATCH 07/23] feat(trajectories): convert the canonical IR to ATIF, unwired MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first outbound edge, and the first converter that has to fabricate. Where `ACP -> IR` stress-tested whether every absence could be declared, this one stress-tests the other half of the taxonomy: ATIF requires values the IR does not carry, so `SYNTHESIZED` stops being a decorative enum member. `export_atif.py` is untouched and remains the only writer of `trainer/atif.json`. Nothing imports the new module. The claim it is built to support, and the reason the slice is worth anything: ir_to_atif(acp_events_to_ir(events), prompts=P) == trajectory_to_atif_record(events=events, prompts=P) Parity with the existing direct exporter, on the same inputs, for the document. A hub that lost anything the direct path preserved would fail that equality. Asserted over events driven through the production capture path plus nine further shapes, and confirmed against the two real gemini rollouts of §5.2: for both, the document produced through the hub is identical to the `trainer/atif.json` those rollouts actually wrote, back when they ran. Parity is about the document, not the report — the direct exporter produces no report, which is the difference this whole proposal is about. **One deliberate deviation, enumerated by a test rather than left in a diff.** `acp_events_to_atif_steps` renders an oracle record as a `source: "agent"` step prefixed `[oracle: …]`, recoverable only by string matching; §5.1 records this as a live divergence, since the in-repo validator already accepts `source: "oracle"` while no emitter produces one. The IR carries the role, so this edge emits `source: "oracle"` with the command as the message. A test runs both paths over a trajectory holding every capture event type plus an oracle record and asserts that step is the only one that differs. Seven fabrications, each recorded where it is invented: `agent.version` and `agent.name`, a tool-call id, a `function_name`, `arguments`, `steps[].message`, and `final_metrics.total_steps` — plus the prompt-derived user steps. The first five are hub-space records at the IR field whose absence forced them; the last three have no IR antecedent and are target-space. `arguments` keeps its `{}`. It is what ATIF requires and what the direct exporter writes, and departing from it would trade a real compatibility property for a cosmetic one. What changes is that it is no longer silent: the fabrication is declared at the same hub path the ACP edge declared `UNSUPPORTED`, so reading the two reports together says the source never had arguments and the target demanded them anyway. A tool call carrying real arguments — including a genuinely captured `{}` — passes through and is declared nothing. Losses are declared only for what this edge actually loses given the trace in hand. An ACP-derived trace has no per-event timestamps or usage, and the inbound report already declared those as `UNSUPPORTED`; repeating them here as `DROPPED` would double-count one fact and misdescribe an edge that loses nothing it was given. - `src/benchflow/trajectories/ir_to_atif.py` — returns `(document, report)` rather than attaching, because a trace may be converted to many targets and none of those conversions describes how the trace was built. It imports one benchflow module, the IR: `ATIF_SCHEMA_VERSION` is redefined and pinned equal to `export_atif`'s by a test, and the `ThoughtBuffer` join is reimplemented and pinned the same way, so the hub does not depend on export plumbing. - `tests/trajectories/test_ir_to_atif.py` — 35 tests, organized around parity. Also pins that an outbound conversion leaves the input trace byte-identical, that two conversions of one trace are independent, that every hub record of the outbound report resolves in the canonical encoding while target records do not, and that a hand-built trace converts without going through ACP. - `test_only_the_ir_family_imports_the_ir` gains `ir_to_atif`. That test failing first was the mechanism working: a new converter joins the family explicitly. - `docs/trace-interop.md` — §8.3 gains the implemented mapping with a class per row and the measured counts, §8.7 the status, §8.9 what the first outbound converter settled. Measured on the real rollouts: H1 produces 12 outbound records (6 synthesized, 4 dropped, 2 normalized; 9 hub, 3 target), H2 produces 14, the extra drops being the timeout event, its extensions and the trace outcome. Known gap, recorded not fixed: `TraceUsage` has no cost field, so `final_metrics.total_cost_usd` cannot be produced through the hub. Neither real rollout carries one, so parity is unaffected there; closing it means adding a field to the IR. Verified: 35 new tests; tests/trajectories 369 passed; the trajectory, capture, streaming and ACP regression lane 534 passed; `ruff check .`, `ruff format --check` and `ty check src/` clean. --- docs/trace-interop.md | 130 ++++- src/benchflow/trajectories/ir_to_atif.py | 473 +++++++++++++++++ tests/trajectories/test_ir_to_atif.py | 644 +++++++++++++++++++++++ tests/trajectories/test_trace_ir.py | 2 +- 4 files changed, 1227 insertions(+), 22 deletions(-) create mode 100644 src/benchflow/trajectories/ir_to_atif.py create mode 100644 tests/trajectories/test_ir_to_atif.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md index d472f6a24..c019563e1 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -590,9 +590,12 @@ Three design choices carry most of the weight: ### 8.3 Planned mapping -**`ACP → IR` is implemented** (Slice C, -[`src/benchflow/trajectories/ir_from_acp.py`](../src/benchflow/trajectories/ir_from_acp.py)), -still unwired and still provisional. `ATIF ↔ IR` and the OTel direction are not. +**`ACP → IR`** (Slice C, +[`ir_from_acp.py`](../src/benchflow/trajectories/ir_from_acp.py)) **and +`IR → ATIF`** (Slice D, +[`ir_to_atif.py`](../src/benchflow/trajectories/ir_to_atif.py)) **are +implemented**, both unwired and still provisional. `ATIF → IR` and the OTel +direction are not. #### ACP-session capture events → IR @@ -645,16 +648,77 @@ trace length. #### IR → ATIF -| IR | ATIF | Loss to record | -|---|---|---| -| `events[].text` | `step.message` | — | -| `reasoning` / `reasoning_segments` | `reasoning_content` (joined) | NORMALIZED — the segmentation does not survive | -| `tool_call.name` | `function_name` | NORMALIZED — an ACP kind is not a function name | -| `tool_call.call_id` empty or absent | synthesized `call_{n}` | SYNTHESIZED | -| `tool_call.status` / `title` | `tool_calls[].extra` | NORMALIZED | -| `agent.agent_version` absent | `"unknown"` | SYNTHESIZED | -| `kind=timeout` | *(no ATIF representation)* | DROPPED | -| opaque content blocks | *(no ATIF representation)* | DROPPED | +**Implemented** (Slice D, +[`src/benchflow/trajectories/ir_to_atif.py`](../src/benchflow/trajectories/ir_to_atif.py)), +unwired: `export_atif.py` is untouched and still the only writer of +`trainer/atif.json`. + +**FACT — the hub reproduces the direct exporter.** For any conformant capture +input, `ir_to_atif(acp_events_to_ir(events), prompts=P)` produces the same +document as `trajectory_to_atif_record(events=events, prompts=P)`, with one +deliberate exception (oracle, below). Asserted by +`tests/trajectories/test_ir_to_atif.py` over events driven through the +production capture path and nine further shapes, and confirmed against the two +real rollouts of §5.2: for both, the document produced through the hub is +identical to the `trainer/atif.json` those rollouts actually wrote. + +This is the evidence that the IR is sufficient for this format — a hub that lost +something the direct path carried would fail that equality. + +| IR | ATIF | Class | Loss recorded | +|---|---|---|---| +| `events[].text` | `step.message` | preserved | — | +| `reasoning` / `reasoning_segments` | `reasoning_content`, joined by blank line | normalized | `events[].reasoning_segments`, once (#10) | +| `events[].index` | `step_id`, dense from 1 over emitted steps | normalized | `events[].index`, once | +| `tool_call.name` | `function_name` | preserved | — the ACP-kind semantics is what is lost, below | +| `tool_call.name_semantics` | *(no slot)* | dropped | `events[].tool_call.name_semantics`, once | +| `tool_call.call_id` empty or absent | `call_{n}` | **synthesized** | `events[i].tool_call.call_id` | +| `tool_call.name` empty or absent | `"tool"` | **synthesized** | `events[i].tool_call.name` | +| `tool_call.arguments = None` | `{}` | **synthesized** | `events[i].tool_call.arguments`, per call (#1) | +| `tool_call.arguments` present | passed through | preserved | — nothing declared | +| `tool_call.status` / `title` | `tool_calls[].extra`, stringified | normalized | — (shape unchanged from the direct path) | +| text content blocks | `observation.results[].content` | preserved | — | +| opaque content blocks | *(no slot)* | dropped | `events[i].tool_call.content` (#5) | +| `kind=timeout` | *(no slot)* | dropped | `events[i]` (#4) | +| `kind=unknown` | *(no slot)* | dropped | `events[i]` | +| text-empty event | *(no step)* | dropped | `events[i]` | +| `kind=oracle` | `source: "oracle"`, command as `message` | normalized | `events[i].extensions` — **deviation, see below** | +| `agent.agent_name` absent | `"unknown"` | **synthesized** | `agent.agent_name` | +| `agent.agent_version` absent | `"unknown"` | **synthesized** | `agent.agent_version` (#7) | +| `usage.input/output/cache_read` | `final_metrics.total_prompt/completion/cached_tokens` | preserved | — | +| `usage.cache_creation_tokens`, `.reasoning_tokens`, `.total_tokens`, `.source` | *(no slot)* | dropped | `usage.` | +| `outcome.*` | *(no slot)* | dropped | `outcome`, once | +| `events[].provenance`, `.source_type`, `.extensions` | *(no slot)* | dropped | once each | +| *(the `prompts` argument)* | leading `user` steps | **synthesized** | `steps[i]`, **target space** | +| *(none)* | `steps[].message` on tool / flushed-thought steps | **synthesized** | `steps[].message`, **target space**, once | +| *(none)* | `final_metrics.total_steps` | **synthesized** | `final_metrics.total_steps`, **target space** | + +**FACT — the one deliberate deviation is `oracle`.** `acp_events_to_atif_steps` +renders an oracle record as a `source: "agent"` step prefixed `[oracle: …]`, +recoverable only by string matching; §5.1 records this as a live divergence, +since the in-repo validator already accepts `source: "oracle"` while no emitter +produces it. The IR carries the role, so this edge emits `source: "oracle"` with +the command as the message and no prefix. A test asserts that this is the *only* +step that differs on a trajectory containing every capture event type plus an +oracle record. + +**FACT — measured on the same two real rollouts.** H1 produces **12** outbound +records (6 synthesized, 4 dropped, 2 normalized; 9 hub, 3 target); H2, whose +trajectory contains a real wall-clock timeout, produces **14** (5 synthesized, +7 dropped, 2 normalized), the extra drops being the timeout event, its +`extensions` and the trace `outcome`. + +**FACT — the two reports compose.** For H1, `events[2].tool_call.arguments` +carries `unsupported` in the `acp -> ir` report and `synthesized` in the +`ir -> atif` one: the source never had arguments and the target demanded them +anyway, joined on one hub path (§8.2, choice 5). + +**Known gap: the IR cannot express cost.** `TraceUsage` has no cost field, so +`final_metrics.total_cost_usd` — which the direct exporter accepts as an +argument — cannot be produced through the hub. Neither real rollout carries one +(agent-native runs have no proxy cost), so parity is unaffected there, but this +is a hub gap rather than a converter one and closing it means adding a field to +the IR. #### IR ↔ OpenTelemetry @@ -920,17 +984,22 @@ here. ### 8.7 Status -- **Implemented:** the IR types, the loss model, the invariants, the validation - suite, this section, and the `ACP → IR` converter with its loss report. -- **Not implemented, deliberately:** `ATIF ↔ IR`, any OTel work, any wiring into +- **Implemented:** the IR types, the loss model with its path spaces, the + invariants, the validation suite, this section, the `ACP → IR` converter and + the `IR → ATIF` converter, each with its loss report. +- **Not implemented, deliberately:** `ATIF → IR`, any OTel work, any wiring into a run path, any on-disk artifact, any capture-layer enrichment. - **Unchanged:** every existing format, exporter, artifact and code path. + `export_atif.py` in particular is untouched and remains the only writer of + `trainer/atif.json`. -The isolation property is unchanged in substance and restated at a new -boundary: `ir.py` and `ir_from_acp.py` form a closed family that may import each -other, and `test_only_the_ir_family_imports_the_ir` asserts that nothing else in -`src/benchflow` imports either. The converter itself imports one benchflow -module — the IR — and reads the capture format as data. +The isolation property is unchanged in substance and restated at a new boundary: +`ir.py`, `ir_from_acp.py` and `ir_to_atif.py` form a closed family that may +import each other, and `test_only_the_ir_family_imports_the_ir` asserts that +nothing else in `src/benchflow` imports any of them. Each converter imports one +benchflow module — the IR — and reads or writes its own format as data; in +particular `ir_to_atif` does not import `export_atif`, and pins the shared schema +version by test instead. ### 8.8 What the first converter showed @@ -966,3 +1035,22 @@ forces a converter to *declare* an absence; it cannot stop one from writing and no loss record is valid. Verified by hand. Closing that would mean the IR taking a position on what an empty map means for each source, which §8.2's tri-state rule deliberately leaves to the converter. + +### 8.9 What the first outbound converter showed + +**FACT.** `IR → ATIF` was the first edge that had to *fabricate* rather than +declare an absence, and it settled three things the inbound edge could not: + +- **`SYNTHESIZED` earns its place.** Seven values ATIF requires and the IR does + not carry are now produced and recorded — an agent version, a tool-call id, a + function name, an empty argument map, an empty step message, a step count, and + the prompt-derived steps. Without the class, each would be an invented value + indistinguishable from an observed one, which is exactly how the `{}` in + today's documents reads. +- **The hub is sufficient for this format, demonstrably.** Parity with the + direct exporter holds byte-for-byte on real captured rollouts, so the + round trip through the IR costs nothing that ATIF was previously getting. +- **The report's ownership had to be settled**, and target-only values forced + `PathSpace` into existence (§8.2, choice 5). A prompt-derived step has no IR + antecedent at all, so it cannot be addressed in the hub vocabulary — and + before this edge, `LossRecord` had no way to say so. diff --git a/src/benchflow/trajectories/ir_to_atif.py b/src/benchflow/trajectories/ir_to_atif.py new file mode 100644 index 000000000..b60a70e18 --- /dev/null +++ b/src/benchflow/trajectories/ir_to_atif.py @@ -0,0 +1,473 @@ +"""Canonical Trace IR → ATIF (Slice D) — the first outbound edge. + +> **PROVISIONAL.** Companion to :mod:`benchflow.trajectories.ir`, itself an +> unapproved proposal (`docs/trace-interop.md` §8). Nothing imports this module, +> nothing writes its output to disk, and `export_atif.py` is untouched and still +> the only writer of `trainer/atif.json`. + +Where :mod:`benchflow.trajectories.ir_from_acp` stress-tested the +declared-absence rule, this edge stress-tests the other half of the taxonomy: +**ATIF requires values the IR does not have**, so this is the first converter +that must fabricate, and every fabrication is recorded as +:attr:`~benchflow.trajectories.ir.LossClass.SYNTHESIZED`. + +## The claim this module is built to support + +`ir_to_atif(acp_events_to_ir(events), prompts=P)` produces **the same document** +as `trajectory_to_atif_record(events=events, prompts=P)` — the existing direct +exporter — for any conformant capture input, with one deliberate exception +(oracle, below). If the hub lost something the direct path preserved, that +equality would fail, so the parity test is the strongest available evidence that +the IR is sufficient for this format. + +Parity is against the *document*, not the report: the direct exporter produces +no report, which is the difference this whole proposal is about. + +## Deliberate deviation from the direct exporter + +**`oracle` steps.** `acp_events_to_atif_steps` renders an oracle record as a +`source: "agent"` step whose message is prefixed `[oracle: …]`, so a consumer +can only recover the distinction by string matching — §5.1 records this as a +live divergence, since the in-repo validator already accepts +`source: "oracle"` while no emitter produces it. The IR carries +:attr:`~benchflow.trajectories.ir.Role.ORACLE`, so this converter emits +`source: "oracle"` and puts the command in `message` without the prefix. That is +the first thing the hub buys that the direct path could not, and it is +enumerated by a test rather than left as a silent difference. + +## What ATIF forces + +Seven values ATIF requires and the IR does not carry, each recorded where it is +invented: + +- `agent.version`, and `agent.name` when the trace has none — hub space, at the + IR field whose absence forced it; +- a tool-call id when the IR's is empty, and a `function_name` when its name is; +- `arguments`, always — see below; +- `steps[].message` on steps that carry only a tool call or a flushed thought, + and `final_metrics.total_steps` — target space, since no IR field corresponds; +- the leading `user` steps built from *prompts*, which are not trace data at all + but an argument of this conversion — target space. + +## `arguments` + +The document still receives `{}`. It is what ATIF requires and what the direct +exporter writes, and departing from it would trade a real compatibility property +for a cosmetic one. What changes is that it is no longer **silent**: every +fabricated `{}` is declared `SYNTHESIZED` at the hub path +``events[i].tool_call.arguments`` — the same path the ACP edge declared +``UNSUPPORTED``. Read together, the two reports say the source never had +arguments and the target demanded them anyway. + +A tool call whose IR `arguments` is a real mapping — including a genuinely +captured `{}` — passes through and is declared nothing. The two cases are +indistinguishable in the ATIF document, which is a limit of ATIF; they are +distinguishable in the report, which is the point. +""" + +from __future__ import annotations + +from typing import Any + +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlockKind, + EventKind, + LossClass, + LossReport, + PathSpace, + TraceEvent, +) + +LOSS_DIRECTION = "ir->atif" + +ATIF_SCHEMA_VERSION = "ATIF-v1.7" +"""Deliberately not imported from ``export_atif``. + +The hub must not depend on the exporters it sits between; a test pins this +constant equal to that module's, so drift fails instead of going unnoticed. +""" + +# Steps carrying only a tool call or a flushed thought get an empty message, +# because ATIF steps always have one. Declared once, not per step. +_EMPTY_MESSAGE_FIELD = "steps[].message" + + +def _reasoning_texts(event: TraceEvent) -> list[str]: + """The thought texts this event contributes, boundaries first. + + ``reasoning_segments`` is the richer encoding and is preferred; ``reasoning`` + is the fallback for a trace built without segments. Empty strings are + dropped, matching the direct exporter's ``if text:`` guard. + """ + if event.reasoning_segments is not None: + return [text for text in event.reasoning_segments if text] + return [event.reasoning] if event.reasoning else [] + + +def _observation_text(event: TraceEvent) -> str: + """Text output of a tool call, joined as ``content_blocks_to_text`` joins it. + + Opaque blocks contribute nothing — ATIF has no slot for them — which is why + the caller declares them dropped. + """ + if event.tool_call is None: + return "" + return "\n".join( + block.text or "" + for block in event.tool_call.content + if block.kind is ContentBlockKind.TEXT + ) + + +def ir_to_atif( + trace: CanonicalTrace, + *, + prompts: list[str] | None = None, +) -> tuple[dict[str, Any], LossReport]: + """Build one ATIF document from a canonical trace. + + *prompts* are the user-facing prompts handed to the agent before any event + was captured. They are **not** trace data — `acp_events_to_ir` deliberately + does not invent events for them (§5.2: doing so is what makes an ATIF + document open with two identical `user` steps) — so they enter here, at the + edge whose format wants them, and every step they produce is declared + ``SYNTHESIZED`` in the target space. + + Returns the document **and** its report. The report is returned rather than + attached because a trace may be converted to many targets and none of those + conversions describes how the trace was built; ``trace`` is not modified. + + Raises ``ValueError`` for a trace with no representable event, mirroring the + direct exporter: ATIF requires at least one step and fabricating an empty one + would be inventing. + """ + losses = LossReport(direction=LOSS_DIRECTION) + steps: list[dict[str, Any]] = [] + pending_thoughts: list[str] = [] + empty_message_declared = False + + def append_step(source: str, body: dict[str, Any]) -> None: + steps.append({"step_id": len(steps) + 1, "source": source, **body}) + + def take_reasoning() -> str | None: + """Join buffered thoughts with a blank line, then clear. + + The join is ``ThoughtBuffer``'s, reimplemented rather than imported so + the hub does not depend on export plumbing; a test pins the two equal. + """ + if not pending_thoughts: + return None + joined = "\n\n".join(pending_thoughts) + pending_thoughts.clear() + return joined + + def declare_empty_message() -> None: + nonlocal empty_message_declared + if empty_message_declared: + return + empty_message_declared = True + losses.add( + _EMPTY_MESSAGE_FIELD, + LossClass.SYNTHESIZED, + "ATIF steps always carry a message; a step holding only a tool call " + "or a flushed thought has no IR text to put there", + space=PathSpace.TARGET, + ) + + def flush_thoughts() -> None: + reasoning = take_reasoning() + if reasoning: + declare_empty_message() + append_step("agent", {"message": "", "reasoning_content": reasoning}) + + for prompt in prompts or []: + if not prompt: + continue + append_step("user", {"message": str(prompt)}) + losses.add( + f"steps[{len(steps) - 1}]", + LossClass.SYNTHESIZED, + "step built from the prompts argument, not from any IR event; the " + "trace carries no antecedent for it", + space=PathSpace.TARGET, + ) + + for event in trace.events: + where = f"events[{event.index}]" + + if event.kind is EventKind.USER_MESSAGE: + if event.text: + flush_thoughts() + append_step("user", {"message": event.text}) + else: + losses.add( + where, + LossClass.DROPPED, + "text-empty event; ATIF has no step for it and the direct " + "exporter drops it too", + ) + + elif event.kind is EventKind.AGENT_REASONING: + texts = _reasoning_texts(event) + if texts: + pending_thoughts.extend(texts) + else: + losses.add(where, LossClass.DROPPED, "reasoning event with no text") + + elif event.kind is EventKind.AGENT_MESSAGE: + if event.text: + body: dict[str, Any] = {"message": event.text} + reasoning = take_reasoning() + if reasoning: + body["reasoning_content"] = reasoning + append_step("agent", body) + else: + losses.add( + where, + LossClass.DROPPED, + "text-empty event; ATIF has no step for it and the direct " + "exporter drops it too", + ) + + elif event.kind is EventKind.TOOL_CALL and event.tool_call is not None: + call = event.tool_call + # Computed before the step is appended, so the fallback id matches + # the step_id the step is about to receive — the direct exporter's + # convention, which ATIF resolves within one step anyway. + call_id = call.call_id or f"call_{len(steps) + 1}" + if not call.call_id: + losses.add( + f"{where}.tool_call.call_id", + LossClass.SYNTHESIZED, + f"ATIF needs an id to bind the observation to; the IR carries " + f"{call.call_id!r}, so {call_id!r} was generated", + ) + function_name = call.name or "tool" + if not call.name: + losses.add( + f"{where}.tool_call.name", + LossClass.SYNTHESIZED, + "ATIF requires function_name; the IR carries no tool name, " + 'so "tool" was generated', + ) + + tool_call: dict[str, Any] = { + "tool_call_id": call_id, + "function_name": function_name, + "arguments": {}, + } + if call.arguments is None: + losses.add( + f"{where}.tool_call.arguments", + LossClass.SYNTHESIZED, + "the IR carries no arguments for this call and ATIF requires " + "the field, so an empty mapping was written; it is not an " + "observation that the tool was called with none", + "§5 loss 1", + ) + else: + tool_call["arguments"] = call.arguments + + extra: dict[str, str] = {} + if call.title: + extra["title"] = str(call.title) + if call.status: + extra["status"] = call.status.value + if extra: + tool_call["extra"] = extra + + declare_empty_message() + tool_body: dict[str, Any] = {"message": "", "tool_calls": [tool_call]} + reasoning = take_reasoning() + if reasoning: + tool_body["reasoning_content"] = reasoning + result_text = _observation_text(event) + if result_text: + tool_body["observation"] = { + "results": [{"source_call_id": call_id, "content": result_text}] + } + append_step("agent", tool_body) + + if any(block.kind is ContentBlockKind.OPAQUE for block in call.content): + losses.add( + f"{where}.tool_call.content", + LossClass.DROPPED, + "non-text content blocks have no ATIF representation; the IR " + "carries them verbatim and this edge cannot", + "§5 loss 5", + ) + + elif event.kind is EventKind.ORACLE: + command = str(event.extensions.get("command") or "oracle") + append_step("oracle", {"message": command}) + losses.add( + f"{where}.extensions", + LossClass.NORMALIZED, + "the oracle command becomes the step message; return_code and " + "stdout have no ATIF slot", + ) + + else: + losses.add( + where, + LossClass.DROPPED, + f"{event.kind.value} events have no ATIF representation" + + ( + "; the timeout marker is absent from every exported document" + if event.kind is EventKind.TIMEOUT + else "" + ), + "§5 loss 4" if event.kind is EventKind.TIMEOUT else None, + ) + + flush_thoughts() + + if not steps: + raise ValueError("ATIF requires at least one step; trace is empty") + + agent: dict[str, Any] = { + "name": trace.agent.agent_name or "unknown", + "version": trace.agent.agent_version or "unknown", + } + if not trace.agent.agent_name: + losses.add( + "agent.agent_name", + LossClass.SYNTHESIZED, + 'ATIF requires agent.name; the trace carries none, so "unknown" was ' + "written", + ) + if not trace.agent.agent_version: + losses.add( + "agent.agent_version", + LossClass.SYNTHESIZED, + "ATIF requires agent.version; BenchFlow does not track agent binary " + 'versions, so "unknown" was written', + "§5 loss 7", + ) + if trace.agent.model: + agent["model_name"] = trace.agent.model + + final_metrics: dict[str, Any] = {"total_steps": len(steps)} + losses.add( + "final_metrics.total_steps", + LossClass.SYNTHESIZED, + "computed from the produced document; no IR field corresponds to it", + space=PathSpace.TARGET, + ) + if trace.usage is not None: + for source_field, atif_field in ( + ("input_tokens", "total_prompt_tokens"), + ("output_tokens", "total_completion_tokens"), + ("cache_read_tokens", "total_cached_tokens"), + ): + value = getattr(trace.usage, source_field) + if value is not None: + final_metrics[atif_field] = value + for unmapped in ( + "cache_creation_tokens", + "reasoning_tokens", + "total_tokens", + "source", + ): + if getattr(trace.usage, unmapped) is not None: + losses.add( + f"usage.{unmapped}", + LossClass.DROPPED, + "ATIF final_metrics has no slot for it", + ) + + _declare_systemic_losses(trace, losses) + + record: dict[str, Any] = {"schema_version": ATIF_SCHEMA_VERSION} + if trace.session_id: + record["session_id"] = trace.session_id + record["agent"] = agent + record["steps"] = steps + record["final_metrics"] = final_metrics + return record, losses + + +def _declare_systemic_losses(trace: CanonicalTrace, losses: LossReport) -> None: + """Losses that hold for the conversion rather than for one event. + + Declared **only when the trace actually carries the value**. An ACP-derived + trace has no per-event timestamps or usage, and the inbound report already + declared those absences as ``UNSUPPORTED``; repeating them here as + ``DROPPED`` would double-count one fact and misdescribe this edge, which + loses nothing it was given. + """ + events = trace.events + if not events: + return + + losses.add( + "events[].index", + LossClass.NORMALIZED, + "ATIF step_id is dense from 1 over the steps actually emitted, so an " + "event that produces no step shifts every later number; event identity " + "does not survive", + ) + losses.add( + "events[].provenance", + LossClass.DROPPED, + "ATIF records no per-step provenance", + ) + if any(event.source_type for event in events): + losses.add( + "events[].source_type", + LossClass.DROPPED, + "the source's own type string has no ATIF slot; only the normalized " + "kind survives, through the step shape", + ) + if any(event.extensions and event.kind is not EventKind.ORACLE for event in events): + losses.add( + "events[].extensions", + LossClass.DROPPED, + "carried verbatim by the IR, no ATIF slot", + ) + if any(event.tool_call and event.tool_call.name_semantics for event in events): + losses.add( + "events[].tool_call.name_semantics", + LossClass.DROPPED, + "ATIF has one function_name slot and no way to say that the value in " + "it is an ACP kind rather than a function name", + ) + if any(event.reasoning_segments for event in events): + losses.add( + "events[].reasoning_segments", + LossClass.NORMALIZED, + "thoughts are joined with a blank line into reasoning_content, so " + "the boundaries the IR preserved are not recoverable", + "§5 loss 10", + ) + if any(event.usage for event in events): + losses.add( + "events[].usage", + LossClass.DROPPED, + "ATIF per-step metrics are not emitted by this converter", + ) + if any(event.started_at or event.finished_at for event in events) or any( + event.tool_call and (event.tool_call.started_at or event.tool_call.finished_at) + for event in events + ): + losses.add( + "events[].started_at", + LossClass.DROPPED, + "ATIF steps carry no timestamps", + ) + if trace.outcome and any( + value is not None + for value in ( + trace.outcome.status, + trace.outcome.stop_reason, + trace.outcome.reward, + trace.outcome.error_category, + ) + ): + losses.add( + "outcome", + LossClass.DROPPED, + "ATIF has no run-outcome section; reward and error category live in " + "result.json for the direct path too", + ) diff --git a/tests/trajectories/test_ir_to_atif.py b/tests/trajectories/test_ir_to_atif.py new file mode 100644 index 000000000..20da68ec1 --- /dev/null +++ b/tests/trajectories/test_ir_to_atif.py @@ -0,0 +1,644 @@ +"""Conversion suite for ``canonical Trace IR → ATIF`` (Slice D). + +The inbound edge tested whether every absence could be declared. This one tests +the opposite half of the taxonomy: ATIF *requires* values the IR does not carry, +so this is the first converter that fabricates, and `SYNTHESIZED` stops being a +decorative enum member. + +The suite is organized around one claim: + + ir_to_atif(acp_events_to_ir(events), prompts=P) + == + trajectory_to_atif_record(events=events, prompts=P) + +Parity with the existing direct exporter, on the same inputs, for the document — +not the report, since the direct exporter produces none. If the hub lost +anything the direct path preserved, that equality fails. Every deviation is +enumerated in one test rather than left to be discovered in a diff. + +Nothing here writes to disk and `export_atif.py` is imported read-only, as the +oracle to compare against. +""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path +from typing import Any + +import pytest + +from benchflow.trajectories import export_atif +from benchflow.trajectories import ir_to_atif as ir_to_atif_module +from benchflow.trajectories._export_common import ThoughtBuffer +from benchflow.trajectories.export_atif import trajectory_to_atif_record +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlock, + ContentBlockKind, + EventKind, + LossClass, + ModelInfo, + PathSpace, + Provenance, + Role, + ToolCall, + ToolStatus, + TraceEvent, + TraceUsage, + validate_trace, +) +from benchflow.trajectories.ir_from_acp import acp_events_to_ir +from benchflow.trajectories.ir_to_atif import ( + ATIF_SCHEMA_VERSION, + LOSS_DIRECTION, + ir_to_atif, +) +from tests.trajectories.test_atif_preservation import _rich_events +from tests.trajectories.test_trace_ir import resolve_ir_path + +PROMPTS = ["Solve the task.", "Then stop."] + + +def _both_paths( + events: list[dict[str, Any]], + *, + prompts: list[str] | None = None, + session_id: str = "sess-d", + agent_name: str = "claude-code", + model: str | None = "claude-sonnet-5", +) -> tuple[dict[str, Any], dict[str, Any]]: + """Run the direct exporter and the hub over the same input.""" + direct = trajectory_to_atif_record( + session_id=session_id, + agent_name=agent_name, + events=events, + prompts=prompts, + model=model, + ) + trace = acp_events_to_ir( + events, session_id=session_id, agent_name=agent_name, model=model + ) + through_hub, _ = ir_to_atif(trace, prompts=prompts) + return direct, through_hub + + +def _fields(report, space: PathSpace = PathSpace.HUB) -> set[str]: + return {r.field for r in report.records if r.space is space} + + +# --------------------------------------------------------------------------- +# Parity with the direct exporter +# --------------------------------------------------------------------------- + + +def test_the_hub_reproduces_the_direct_exporter_on_a_real_captured_trace(): + """The load-bearing test of this slice. + + Events produced by driving a real `ACPSession` through the production + capture path, then converted both ways. Byte-for-byte equality of the two + documents is what says the IR did not lose anything ATIF was getting. + """ + direct, through_hub = _both_paths(_rich_events(), prompts=PROMPTS) + assert through_hub == direct + + +@pytest.mark.parametrize( + "events", + [ + pytest.param( + [{"type": "agent_message", "text": "only a message"}], id="message" + ), + pytest.param( + [ + {"type": "user_message", "text": "u"}, + {"type": "agent_thought", "text": "t"}, + {"type": "agent_message", "text": "a"}, + ], + id="thought-then-message", + ), + pytest.param( + [ + {"type": "agent_thought", "text": "one"}, + {"type": "agent_thought", "text": "two"}, + {"type": "agent_message", "text": "a"}, + ], + id="consecutive-thoughts-joined", + ), + pytest.param( + [{"type": "agent_thought", "text": "trailing"}], id="trailing-thought-flush" + ), + pytest.param( + [ + { + "type": "tool_call", + "tool_call_id": "", + "kind": "", + "title": "", + "status": "completed", + "content": [], + } + ], + id="empty-tool-fields-synthesized", + ), + pytest.param( + [ + {"type": "agent_thought", "text": "why"}, + { + "type": "tool_call", + "tool_call_id": "t1", + "kind": "execute", + "title": "ls", + "status": "failed", + "content": [ + {"type": "content", "content": {"type": "text", "text": "out"}} + ], + }, + ], + id="tool-with-reasoning-and-observation", + ), + pytest.param( + [ + {"type": "user_message", "text": ""}, + {"type": "agent_message", "text": ""}, + {"type": "agent_message", "text": "kept"}, + ], + id="text-empty-events-dropped-by-both", + ), + pytest.param( + [ + {"type": "agent_message", "text": "a"}, + { + "type": "agent_timeout", + "reason": "wall_clock_timeout", + "timeout_sec": 1.0, + "pending_tool_call_ids": [], + "terminal_trajectory_complete": True, + }, + ], + id="timeout-dropped-by-both", + ), + pytest.param( + [{"type": "mystery", "payload": 1}, {"type": "agent_message", "text": "a"}], + id="unknown-dropped-by-both", + ), + ], +) +def test_parity_holds_shape_by_shape(events): + direct, through_hub = _both_paths(events, prompts=PROMPTS) + assert through_hub == direct + + +def test_parity_holds_without_prompts(): + direct, through_hub = _both_paths(_rich_events()) + assert through_hub == direct + + +def test_parity_holds_when_the_trace_has_no_session_id_or_model(): + direct, through_hub = _both_paths( + _rich_events(), prompts=PROMPTS, session_id="", agent_name="", model=None + ) + assert through_hub == direct + assert through_hub["agent"] == {"name": "unknown", "version": "unknown"} + assert "session_id" not in through_hub + + +def test_both_paths_refuse_a_trajectory_with_no_representable_step(): + """ATIF requires one step; fabricating an empty one would be inventing.""" + empty_events: list[dict[str, Any]] = [] + with pytest.raises(ValueError): + trajectory_to_atif_record( + session_id="s", agent_name="a", events=empty_events, prompts=None + ) + with pytest.raises(ValueError): + ir_to_atif(acp_events_to_ir(empty_events)) + + +def test_the_schema_version_matches_the_direct_exporter(): + """Redefined rather than imported, so the equality is asserted not assumed.""" + assert ATIF_SCHEMA_VERSION == export_atif.ATIF_SCHEMA_VERSION + + +def test_the_thought_join_matches_the_shared_buffer(): + """The hub reimplements ``ThoughtBuffer``'s join; the two must agree.""" + buffer = ThoughtBuffer() + for text in ("one", "two", "three"): + buffer.push(text) + expected = buffer.take() + + events = [{"type": "agent_thought", "text": t} for t in ("one", "two", "three")] + events.append({"type": "agent_message", "text": "done"}) + document, _ = ir_to_atif(acp_events_to_ir(events)) + assert document["steps"][0]["reasoning_content"] == expected + + +# --------------------------------------------------------------------------- +# The one deliberate deviation +# --------------------------------------------------------------------------- + + +def test_oracle_becomes_its_own_source_instead_of_a_prefixed_agent_step(): + """The single enumerated divergence from the direct exporter. + + `acp_events_to_atif_steps` renders oracle activity as an `agent` step whose + message is prefixed ``[oracle: …]``, recoverable only by string matching + (§5.1). The in-repo validator already accepts ``source: "oracle"``, and the + IR carries the role, so the hub emits it. + """ + events = [{"type": "oracle", "command": "solve.sh", "return_code": 0}] + direct, through_hub = _both_paths(events, prompts=None) + + assert direct["steps"][0] == { + "step_id": 1, + "source": "agent", + "message": "[oracle: solve.sh]", + } + assert through_hub["steps"][0] == { + "step_id": 1, + "source": "oracle", + "message": "solve.sh", + } + assert through_hub != direct + # Everything except that step is still identical. + assert through_hub["agent"] == direct["agent"] + assert through_hub["final_metrics"] == direct["final_metrics"] + + +def test_the_oracle_deviation_is_the_only_one_on_conformant_input(): + """Enumerated: any *other* divergence has to fail a test, not be discovered. + + Runs both paths over a trajectory containing every capture event type plus + an oracle record, and asserts the documents differ in exactly the oracle + step and nothing else. + """ + events = [*_rich_events(), {"type": "oracle", "command": "check.sh"}] + direct, through_hub = _both_paths(events, prompts=PROMPTS) + + assert len(direct["steps"]) == len(through_hub["steps"]) + differing = [ + (a, b) + for a, b in zip(direct["steps"], through_hub["steps"], strict=True) + if a != b + ] + assert len(differing) == 1, differing + assert differing[0][0]["source"] == "agent" + assert differing[0][1]["source"] == "oracle" + + +def test_the_produced_document_passes_the_in_repo_atif_validator(tmp_path): + """Including the oracle source, which that validator already accepts.""" + from tests.integration.scenarios import atif_issues + + events = [*_rich_events(), {"type": "oracle", "command": "check.sh"}] + document, _ = ir_to_atif( + acp_events_to_ir(events, session_id="s", agent_name="a"), prompts=PROMPTS + ) + rollout = tmp_path / "rollout" + (rollout / "trainer").mkdir(parents=True) + (rollout / "trainer" / "atif.json").write_text(json.dumps(document)) + assert atif_issues(rollout) == [] + + +# --------------------------------------------------------------------------- +# SYNTHESIZED — the class this slice exists to stress +# --------------------------------------------------------------------------- + + +def test_every_fabricated_value_is_declared_synthesized(): + """The four hub-space fabrications, on one trace that forces all of them.""" + events = [ + { + "type": "tool_call", + "tool_call_id": "", + "kind": "", + "title": "", + "status": "completed", + "content": [], + } + ] + trace = acp_events_to_ir(events) # no agent_name, no version + document, report = ir_to_atif(trace) + + synthesized = { + r.field + for r in report.by_class(LossClass.SYNTHESIZED) + if r.space is PathSpace.HUB + } + assert synthesized == { + "events[0].tool_call.call_id", + "events[0].tool_call.name", + "events[0].tool_call.arguments", + "agent.agent_name", + "agent.agent_version", + } + + call = document["steps"][0]["tool_calls"][0] + assert call["tool_call_id"] == "call_1" + assert call["function_name"] == "tool" + assert call["arguments"] == {} + assert document["agent"] == {"name": "unknown", "version": "unknown"} + + +def test_arguments_are_declared_synthesized_only_when_the_ir_carried_none(): + """`{}` in the document either way; the report is what tells them apart.""" + absent = acp_events_to_ir([{"type": "tool_call", "tool_call_id": "t"}]) + document, report = ir_to_atif(absent) + assert document["steps"][0]["tool_calls"][0]["arguments"] == {} + assert report.for_field("events[0].tool_call.arguments") + + captured = acp_events_to_ir([{"type": "tool_call", "tool_call_id": "t"}]) + captured.events[0].tool_call.arguments = {} + document, report = ir_to_atif(captured) + assert document["steps"][0]["tool_calls"][0]["arguments"] == {} + assert report.for_field("events[0].tool_call.arguments") == [] + + real = acp_events_to_ir([{"type": "tool_call", "tool_call_id": "t"}]) + real.events[0].tool_call.arguments = {"command": "ls"} + document, report = ir_to_atif(real) + assert document["steps"][0]["tool_calls"][0]["arguments"] == {"command": "ls"} + assert report.for_field("events[0].tool_call.arguments") == [] + + +def test_the_arguments_story_composes_across_the_two_edges(): + """The property the hub exists for, on one field and one path. + + The ACP edge says the source never carried arguments; the ATIF edge says the + target demanded them anyway. Same hub path, two reports, one history. + """ + trace = acp_events_to_ir(_rich_events()) + _, outbound = ir_to_atif(trace) + + field = next( + r.field + for r in trace.losses.records + if r.field.endswith(".tool_call.arguments") + ) + inbound_record = trace.losses.for_field(field)[0] + outbound_record = outbound.for_field(field)[0] + + assert inbound_record.loss_class is LossClass.UNSUPPORTED + assert outbound_record.loss_class is LossClass.SYNTHESIZED + assert inbound_record.space is outbound_record.space is PathSpace.HUB + assert trace.losses.direction == "acp->ir" + assert outbound.direction == LOSS_DIRECTION + + +# --------------------------------------------------------------------------- +# TARGET space +# --------------------------------------------------------------------------- + + +def test_target_only_values_are_declared_in_the_target_space(): + trace = acp_events_to_ir(_rich_events(), agent_name="a") + document, report = ir_to_atif(trace, prompts=PROMPTS) + + assert _fields(report, PathSpace.TARGET) == { + "steps[0]", + "steps[1]", + "steps[].message", + "final_metrics.total_steps", + } + assert all( + r.loss_class is LossClass.SYNTHESIZED for r in report.by_space(PathSpace.TARGET) + ) + # The prompt steps they name are really there. + assert document["steps"][0]["message"] == PROMPTS[0] + assert document["steps"][1]["message"] == PROMPTS[1] + + +def test_no_prompt_steps_means_no_prompt_records(): + _, report = ir_to_atif(acp_events_to_ir(_rich_events())) + assert not [ + r for r in report.by_space(PathSpace.TARGET) if r.field.startswith("steps[0") + ] + + +def test_target_records_are_not_read_as_ir_paths(): + """They address the ATIF document, which the IR does not contain.""" + trace = acp_events_to_ir(_rich_events()) + _, report = ir_to_atif(trace, prompts=PROMPTS) + canonical = trace.model_dump(mode="json") + for record in report.by_space(PathSpace.TARGET): + assert not resolve_ir_path(canonical, record.field)[0] + + +def test_every_hub_record_of_the_outbound_report_resolves_in_the_trace(): + """The same guard as the inbound edge, applied to the outbound report.""" + for events in ( + _rich_events(), + [*_rich_events(), {"type": "oracle", "command": "x"}], + [{"type": "agent_message", "text": ""}, {"type": "agent_message", "text": "a"}], + ): + trace = acp_events_to_ir(events) + _, report = ir_to_atif(trace, prompts=PROMPTS) + canonical = trace.model_dump(mode="json") + for record in report.records: + if record.space is not PathSpace.HUB: + continue + if record.field.startswith("events[]"): + continue + assert resolve_ir_path(canonical, record.field)[0], record.field + + +# --------------------------------------------------------------------------- +# Report ownership +# --------------------------------------------------------------------------- + + +def test_an_outbound_conversion_leaves_the_input_trace_untouched(): + """A trace may be converted to many targets; none of them describes it.""" + trace = acp_events_to_ir(_rich_events(), agent_name="a") + before = trace.model_dump_json() + inbound_records = len(trace.losses.records) + + document, report = ir_to_atif(trace, prompts=PROMPTS) + + assert trace.model_dump_json() == before + assert len(trace.losses.records) == inbound_records + assert trace.losses.direction == "acp->ir" + assert report is not trace.losses + assert validate_trace(trace) == [] + assert document["schema_version"] == ATIF_SCHEMA_VERSION + + +def test_two_outbound_conversions_of_one_trace_are_independent(): + trace = acp_events_to_ir(_rich_events()) + _, first = ir_to_atif(trace, prompts=PROMPTS) + _, second = ir_to_atif(trace) + assert first is not second + assert len(first.records) > len(second.records) # the prompt steps + assert trace.losses.direction == "acp->ir" + + +# --------------------------------------------------------------------------- +# Losses this edge really has +# --------------------------------------------------------------------------- + + +def test_opaque_content_blocks_are_declared_dropped(): + """§5 loss 5 reappears here: the IR carries them, ATIF has no slot.""" + trace = acp_events_to_ir( + [ + { + "type": "tool_call", + "tool_call_id": "t", + "kind": "edit", + "status": "completed", + "content": [ + {"type": "content", "content": {"type": "text", "text": "ok"}}, + {"type": "diff", "oldText": "a", "newText": "b"}, + ], + } + ] + ) + document, report = ir_to_atif(trace) + + assert "newText" not in json.dumps(document) + dropped = report.for_field("events[0].tool_call.content") + assert len(dropped) == 1 + assert dropped[0].loss_class is LossClass.DROPPED + + +def test_the_timeout_is_dropped_here_and_says_so(): + """The hub preserved it; this edge cannot, and that is the honest result.""" + trace = acp_events_to_ir( + [ + {"type": "agent_message", "text": "a"}, + { + "type": "agent_timeout", + "reason": "wall_clock_timeout", + "timeout_sec": 1.0, + "pending_tool_call_ids": [], + "terminal_trajectory_complete": True, + }, + ] + ) + document, report = ir_to_atif(trace) + + assert "wall_clock_timeout" not in json.dumps(document) + dropped = report.for_field("events[1]") + assert dropped and dropped[0].loss_class is LossClass.DROPPED + assert dropped[0].doc_ref == "§5 loss 4" + + +def test_reasoning_boundaries_are_declared_normalized(): + trace = acp_events_to_ir( + [ + {"type": "agent_thought", "text": "one"}, + {"type": "agent_thought", "text": "two"}, + {"type": "agent_message", "text": "a"}, + ] + ) + document, report = ir_to_atif(trace) + assert document["steps"][0]["reasoning_content"] == "one\n\ntwo" + normalized = report.for_field("events[].reasoning_segments") + assert normalized and normalized[0].loss_class is LossClass.NORMALIZED + + +def test_usage_maps_three_fields_and_declares_the_rest_dropped(): + trace = acp_events_to_ir(_rich_events(), agent_name="a") + trace.usage = TraceUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_creation_tokens=3, + total_tokens=125, + source="llm_proxy_normalized", + ) + document, report = ir_to_atif(trace) + + assert document["final_metrics"]["total_prompt_tokens"] == 100 + assert document["final_metrics"]["total_completion_tokens"] == 20 + assert document["final_metrics"]["total_cached_tokens"] == 5 + assert _fields(report) >= { + "usage.cache_creation_tokens", + "usage.total_tokens", + "usage.source", + } + assert "reasoning_tokens" not in json.dumps(document) + + +def test_a_trace_without_usage_declares_no_usage_losses(): + """This edge loses nothing it was never given; the inbound report said that.""" + _, report = ir_to_atif(acp_events_to_ir(_rich_events())) + assert not [r for r in report.records if r.field.startswith("usage")] + + +def test_systemic_losses_are_declared_once_and_only_when_they_apply(): + trace = acp_events_to_ir(_rich_events(), agent_name="a") + _, report = ir_to_atif(trace) + hub = _fields(report) + + assert "events[].index" in hub + assert "events[].provenance" in hub + assert "events[].source_type" in hub + assert "events[].tool_call.name_semantics" in hub + assert "events[].reasoning_segments" in hub + # No per-event usage or timestamps in an ACP-derived trace, so no claim. + assert "events[].usage" not in hub + assert "events[].started_at" not in hub + + +# --------------------------------------------------------------------------- +# Isolation +# --------------------------------------------------------------------------- + + +def test_the_converter_imports_only_the_ir(): + """The hub must not depend on the exporters it sits between. + + In particular it does not import ``export_atif``: the schema version is + redefined and pinned by a test instead, so the family stays a leaf. + """ + tree = ast.parse(Path(ir_to_atif_module.__file__).read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add(node.module or "") + benchflow_imports = {name for name in imported if name.startswith("benchflow")} + assert benchflow_imports == {"benchflow.trajectories.ir"}, sorted(benchflow_imports) + + +def test_a_hand_built_trace_converts_without_going_through_acp(): + """The hub is not a wrapper around the ACP path.""" + provenance = Provenance(source_format="hand-built") + trace = CanonicalTrace( + provenance=provenance, + agent=ModelInfo(agent_name="agent-x", agent_version="1.2.3", model="m"), + events=[ + TraceEvent( + index=0, + kind=EventKind.USER_MESSAGE, + role=Role.USER, + text="hello", + provenance=provenance, + ), + TraceEvent( + index=1, + kind=EventKind.TOOL_CALL, + role=Role.AGENT, + tool_call=ToolCall( + call_id="c1", + name="search", + arguments={"q": "x"}, + status=ToolStatus.COMPLETED, + content=[ContentBlock(kind=ContentBlockKind.TEXT, text="found")], + ), + provenance=provenance, + ), + ], + ) + document, report = ir_to_atif(trace) + + assert document["agent"] == { + "name": "agent-x", + "version": "1.2.3", + "model_name": "m", + } + assert document["steps"][1]["tool_calls"][0]["arguments"] == {"q": "x"} + # A real version and real arguments mean nothing was fabricated for them. + assert "agent.agent_version" not in _fields(report) + assert "events[1].tool_call.arguments" not in _fields(report) diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py index 9f3c156fe..871440317 100644 --- a/tests/trajectories/test_trace_ir.py +++ b/tests/trajectories/test_trace_ir.py @@ -544,7 +544,7 @@ def test_the_oracle_record_has_its_own_kind_and_role(): # --------------------------------------------------------------------------- -IR_FAMILY = ("ir", "ir_from_acp") +IR_FAMILY = ("ir", "ir_from_acp", "ir_to_atif") """The unwired modules of the proposal: the IR and its converters. They may import each other — a converter that could not import the IR would be From 91cbf95edd8c65b63c9a4ffc553b28bd596f081c Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Sat, 15 Aug 2026 11:34:36 +0200 Subject: [PATCH 08/23] fix(trajectories): carry cost through the hub, and declare every outbound loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two holes in Slice D's central claim, found by auditing it rather than by a test. Both are about the same thing: the outbound edge was describing a trace shaped like the one ACP happens to produce, not the trace it actually received. ## Cost `trajectory_to_atif_record` accepts `total_cost_usd` and writes it to `final_metrics`. The IR had no cost field, so a document produced through the hub could not carry it — the parity claim held only for traces without one. `TraceUsage` gains `cost_usd` and `price_source`, mapped to `final_metrics.total_cost_usd` (preserved) and to a declared `DROPPED` (ATIF has no slot). Cost sits with the token counters because that is where every BenchFlow object already puts it — `agent_result`, `TaskTelemetry` and ATIF's own `final_metrics` — and because it is derived from them. `price_source` is to `cost_usd` what `source` is to the counters: BenchFlow computes no prices of its own, it imports a number from the model gateway's log, so a cost without the table that produced it is not comparable. The writing path is real: `providers/litellm_logging.py:618-623` sums the callback log's per-entry `cost` into `Trajectory.metadata["cost_usd"]`, it surfaces as `AgentResult.cost_usd`, and `rollout/_results.py:448` hands it to the ATIF writer. Stated precisely, because the audit turned up a nuance worth recording: **no rollout artifact available here has ever carried a non-null cost** — including four whose `usage_source` is `provider_response`, which went through the proxy but whose gateway log carried no per-entry cost. So the field's production is established by reading the code, not by observation, and parity with a cost is asserted on synthetic input given to both paths. Per-call cost stays out of scope: it exists in the proxy capture, and modelling it is a larger question this proposal does not open. ## Conditional outbound losses The rule was right and the coverage was not. Probing with a trace that really carried the values showed six kinds of information the IR can hold, ATIF cannot represent, and the report did not mention: trace `trace_id` / `started_at` / `finished_at` / `provenance` / `extensions`, `agent.provider`, per-event `outcome`, tool-call timestamps, the `raw` block behind a rendered text block, and a `role` that disagrees with the source ATIF derives from the event kind. Two were also addressing bugs rather than gaps: a tool-call timestamp was covered by a condition that emitted a record at `events[].started_at`, blaming the event for a value belonging to its tool call, and `finished_at` was never named at all. Timestamps are now four separate claims, each addressed where the value lives. `role` is declared per event and only when it disagrees with the implied source — for anything ACP produces the two always agree, so an ACP trace declares nothing for it, which is the rule working. ## Pinning the rule Two tests convert the same shape twice: once through the ACP edge, where the fields are absent and nothing is declared, and once hand-built with every field present, where the complete `DROPPED` set is asserted as an equality. A third test derives the field list from the IR models themselves and requires each one to have a disposition here — mapped, normalized, declared, container, or representation. Adding a field to the IR that ATIF cannot represent now fails the suite until its fate is decided, which is what makes the first two tests stay honest as the IR grows. ## Verification H1 and H2 parity is unchanged: for both, the document produced through the hub is still identical to the `trainer/atif.json` the rollout really wrote. Their outbound reports grow from 12 to 14 and from 14 to 16 records, the two additions being the trace-level `provenance` and the `raw` behind each text block — both values those traces really carry and ATIF really drops. 45 tests in the ATIF suite; tests/trajectories 379 passed; the wider lane 544 passed; `ruff check .`, `ruff format --check` and `ty check src/` clean. One intermittent failure was seen once in `tests/test_acp.py::TestIdleTimeoutDiagnostics` — a wall-clock assertion that passes alone and on re-run, in a module that imports nothing from the IR family. --- docs/trace-interop.md | 62 +++- src/benchflow/trajectories/ir.py | 13 + src/benchflow/trajectories/ir_to_atif.py | 136 ++++++-- tests/trajectories/test_ir_to_atif.py | 404 +++++++++++++++++++++++ 4 files changed, 581 insertions(+), 34 deletions(-) diff --git a/docs/trace-interop.md b/docs/trace-interop.md index c019563e1..3bc5c9adf 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -686,9 +686,16 @@ something the direct path carried would fail that equality. | `agent.agent_name` absent | `"unknown"` | **synthesized** | `agent.agent_name` | | `agent.agent_version` absent | `"unknown"` | **synthesized** | `agent.agent_version` (#7) | | `usage.input/output/cache_read` | `final_metrics.total_prompt/completion/cached_tokens` | preserved | — | -| `usage.cache_creation_tokens`, `.reasoning_tokens`, `.total_tokens`, `.source` | *(no slot)* | dropped | `usage.` | +| `usage.cost_usd` | `final_metrics.total_cost_usd` | preserved | — | +| `usage.cache_creation_tokens`, `.reasoning_tokens`, `.total_tokens`, `.source`, `.price_source` | *(no slot)* | dropped | `usage.` | | `outcome.*` | *(no slot)* | dropped | `outcome`, once | -| `events[].provenance`, `.source_type`, `.extensions` | *(no slot)* | dropped | once each | +| `trace_id`, `started_at`, `finished_at`, `provenance`, `extensions` | *(no slot)* | dropped | once each | +| `agent.provider` | *(no slot)* | dropped | `agent.provider` | +| `events[].provenance`, `.source_type`, `.extensions`, `.outcome`, `.usage` | *(no slot)* | dropped | once each | +| `events[].started_at` / `.finished_at` | *(no slot)* | dropped | once each | +| `events[].tool_call.started_at` / `.finished_at` | *(no slot)* | dropped | once each, **under the tool call** | +| `events[].tool_call.content[].raw` on text blocks | *(only the rendered text survives)* | dropped | once | +| `events[].role`, when it disagrees with the source the kind implies | *(no slot)* | dropped | `events[i].role`, per event | | *(the `prompts` argument)* | leading `user` steps | **synthesized** | `steps[i]`, **target space** | | *(none)* | `steps[].message` on tool / flushed-thought steps | **synthesized** | `steps[].message`, **target space**, once | | *(none)* | `final_metrics.total_steps` | **synthesized** | `final_metrics.total_steps`, **target space** | @@ -702,10 +709,20 @@ the command as the message and no prefix. A test asserts that this is the *only* step that differs on a trajectory containing every capture event type plus an oracle record. -**FACT — measured on the same two real rollouts.** H1 produces **12** outbound -records (6 synthesized, 4 dropped, 2 normalized; 9 hub, 3 target); H2, whose -trajectory contains a real wall-clock timeout, produces **14** (5 synthesized, -7 dropped, 2 normalized), the extra drops being the timeout event, its +**FACT — the outbound report describes the trace it received, not ACP's +habits.** A loss is declared only when the IR actually carries the value. An +ACP-derived trace has no per-event timestamps or usage, and the inbound report +already declared those absences `unsupported`; re-declaring them here would +count one fact twice and misdescribe an edge that loses nothing it was given. +The same conversion run over a trace that *does* carry them declares every one. +Both halves are asserted, the second as a complete set, and a companion test +derives the field list from the IR models themselves — so a field added to the +IR that ATIF cannot represent fails the suite until its fate is decided. + +**FACT — measured on the same two real rollouts.** H1 produces **14** outbound +records (6 synthesized, 6 dropped, 2 normalized; 11 hub, 3 target); H2, whose +trajectory contains a real wall-clock timeout, produces **16** (5 synthesized, +9 dropped, 2 normalized), the extra drops being the timeout event, its `extensions` and the trace `outcome`. **FACT — the two reports compose.** For H1, `events[2].tool_call.arguments` @@ -713,12 +730,29 @@ carries `unsupported` in the `acp -> ir` report and `synthesized` in the `ir -> atif` one: the source never had arguments and the target demanded them anyway, joined on one hub path (§8.2, choice 5). -**Known gap: the IR cannot express cost.** `TraceUsage` has no cost field, so -`final_metrics.total_cost_usd` — which the direct exporter accepts as an -argument — cannot be produced through the hub. Neither real rollout carries one -(agent-native runs have no proxy cost), so parity is unaffected there, but this -is a hub gap rather than a converter one and closing it means adding a field to -the IR. +**Cost.** `TraceUsage` carries `cost_usd` and `price_source`, so +`final_metrics.total_cost_usd` survives the hub. + +**FACT — the writing path exists.** The LiteLLM callback log's per-entry `cost` +is summed into `Trajectory.metadata["cost_usd"]` +(`providers/litellm_logging.py:618-623`), surfaces as `AgentResult.cost_usd` +with `price_source: "litellm"` (`extract_usage_from_trajectory`), and is handed +to the ATIF writer by `rollout/_results.py:448`. The agent-native ACP path sets +it to `None` explicitly (`rollout/__init__.py:1737`). + +**FACT — and it has not been observed producing a value here.** Every rollout +artifact on the machine this was developed on carries `cost_usd: null`, +including four whose `usage_source` is `provider_response` — they went through +the proxy, and their gateway log simply carried no per-entry cost, so +`price_source` stayed `None` too. So the field's *production* is established by +reading the code, not by observation, and parity with a cost is asserted on +synthetic input on both sides. Without the field, though, the hub would be +unable to carry a value the direct exporter's own API accepts — a gap in the +contract regardless of how often it is exercised. + +`price_source` has no ATIF slot and is declared dropped: BenchFlow computes no +prices of its own, so a cost without the table that produced it is not +comparable. Per-call cost stays out of scope. #### IR ↔ OpenTelemetry @@ -882,7 +916,9 @@ its source-specific fields in `extensions` rather than as four new IR fields. "cache_creation_tokens": null, "reasoning_tokens": null, "total_tokens": 1276, - "source": "llm_proxy_normalized" + "source": "llm_proxy_normalized", + "cost_usd": null, + "price_source": null }, "outcome": { "status": "timeout", diff --git a/src/benchflow/trajectories/ir.py b/src/benchflow/trajectories/ir.py index 7f4888355..42c82f15e 100644 --- a/src/benchflow/trajectories/ir.py +++ b/src/benchflow/trajectories/ir.py @@ -421,6 +421,17 @@ class TraceUsage(BaseModel): mean the same thing. The IR refuses to pick one and instead records which was used — a consumer can then compare like with like. Documented values: ``"llm_proxy_normalized"``, ``"acp_session_snapshot"``. + + Cost sits here rather than in its own section because that is where every + BenchFlow object already puts it — ``agent_result``, ``TaskTelemetry`` and + ATIF's ``final_metrics`` all carry it beside the token totals — and because + it is derived from them. ``price_source`` is to ``cost_usd`` what ``source`` + is to the counters: BenchFlow computes no prices of its own, it imports a + number from the model gateway's log, so a cost without the table that + produced it is not comparable. Runtime values: ``"litellm"`` or ``None``. + + Deliberately **trace-level only**. Per-call cost exists in the proxy capture + and modelling it is a larger question this proposal does not open. """ model_config = ConfigDict(extra="forbid") @@ -432,6 +443,8 @@ class TraceUsage(BaseModel): reasoning_tokens: int | None = None total_tokens: int | None = None source: str | None = None + cost_usd: float | None = None + price_source: str | None = None class TraceEvent(BaseModel): diff --git a/src/benchflow/trajectories/ir_to_atif.py b/src/benchflow/trajectories/ir_to_atif.py index b60a70e18..00c29817c 100644 --- a/src/benchflow/trajectories/ir_to_atif.py +++ b/src/benchflow/trajectories/ir_to_atif.py @@ -76,11 +76,24 @@ LossClass, LossReport, PathSpace, + Role, TraceEvent, ) LOSS_DIRECTION = "ir->atif" +# The ATIF `source` is derived from the event kind, not from the IR role, and +# for anything an ACP capture produces the two agree. They can disagree in a +# trace from another source, and then the role is information this edge cannot +# carry — declared per event, because it is a fact about that one event. +_IMPLIED_ROLE = { + EventKind.USER_MESSAGE: Role.USER, + EventKind.AGENT_MESSAGE: Role.AGENT, + EventKind.AGENT_REASONING: Role.AGENT, + EventKind.TOOL_CALL: Role.AGENT, + EventKind.ORACLE: Role.ORACLE, +} + ATIF_SCHEMA_VERSION = "ATIF-v1.7" """Deliberately not imported from ``export_atif``. @@ -196,6 +209,20 @@ def flush_thoughts() -> None: for event in trace.events: where = f"events[{event.index}]" + implied_role = _IMPLIED_ROLE.get(event.kind) + if ( + event.role is not None + and implied_role is not None + and event.role is not implied_role + ): + losses.add( + f"{where}.role", + LossClass.DROPPED, + f"the IR attributes this event to {event.role.value!r}, but ATIF " + f"derives step source from the event kind and will write " + f"{implied_role.value!r}; the disagreement has no ATIF slot", + ) + if event.kind is EventKind.USER_MESSAGE: if event.text: flush_thoughts() @@ -360,6 +387,7 @@ def flush_thoughts() -> None: ("input_tokens", "total_prompt_tokens"), ("output_tokens", "total_completion_tokens"), ("cache_read_tokens", "total_cached_tokens"), + ("cost_usd", "total_cost_usd"), ): value = getattr(trace.usage, source_field) if value is not None: @@ -369,6 +397,7 @@ def flush_thoughts() -> None: "reasoning_tokens", "total_tokens", "source", + "price_source", ): if getattr(trace.usage, unmapped) is not None: losses.add( @@ -391,13 +420,62 @@ def flush_thoughts() -> None: def _declare_systemic_losses(trace: CanonicalTrace, losses: LossReport) -> None: """Losses that hold for the conversion rather than for one event. - Declared **only when the trace actually carries the value**. An ACP-derived - trace has no per-event timestamps or usage, and the inbound report already - declared those absences as ``UNSUPPORTED``; repeating them here as - ``DROPPED`` would double-count one fact and misdescribe this edge, which - loses nothing it was given. + Declared **only when the trace actually carries the value**, which is the + rule this edge is built on: *the outbound report describes what is lost from + the trace it received, not what an ACP-derived trace typically lacks*. An + ACP trace has no per-event timestamps or usage, and the inbound report + already declared those absences ``UNSUPPORTED``; repeating them here as + ``DROPPED`` would double-count one fact and misdescribe an edge that loses + nothing it was given. A trace from a richer source carries them, and then + every one is declared. + + Two IR fields are deliberately never declared, because they describe the + representation rather than the run: ``ir_version`` and ``losses``. Every + other field of every IR model is accounted for here or in the event walk, + and a test derives that list from the models themselves so a new field + cannot be added without a disposition. """ events = trace.events + + if trace.trace_id: + losses.add("trace_id", LossClass.DROPPED, "ATIF documents carry no trace id") + for field in ("started_at", "finished_at"): + if getattr(trace, field) is not None: + losses.add( + field, + LossClass.DROPPED, + "ATIF has no run-level timestamps; wall clock lives in " + "timing.json for the direct path too", + ) + losses.add( + "provenance", + LossClass.DROPPED, + "ATIF records no provenance for the document it describes", + ) + if trace.extensions: + losses.add("extensions", LossClass.DROPPED, "carried by the IR, no ATIF slot") + if trace.agent.provider: + losses.add( + "agent.provider", + LossClass.DROPPED, + "ATIF's agent block is name/version/model_name only", + ) + if trace.outcome and any( + value is not None + for value in ( + trace.outcome.status, + trace.outcome.stop_reason, + trace.outcome.reward, + trace.outcome.error_category, + ) + ): + losses.add( + "outcome", + LossClass.DROPPED, + "ATIF has no run-outcome section; reward and error category live in " + "result.json for the direct path too", + ) + if not events: return @@ -447,27 +525,43 @@ def _declare_systemic_losses(trace: CanonicalTrace, losses: LossReport) -> None: LossClass.DROPPED, "ATIF per-step metrics are not emitted by this converter", ) - if any(event.started_at or event.finished_at for event in events) or any( - event.tool_call and (event.tool_call.started_at or event.tool_call.finished_at) - for event in events - ): + if any(event.outcome and event.kind is not EventKind.TIMEOUT for event in events): losses.add( - "events[].started_at", + "events[].outcome", LossClass.DROPPED, - "ATIF steps carry no timestamps", + "ATIF steps carry no per-step outcome; a timeout event is dropped " + "whole and declared at its own index instead", ) - if trace.outcome and any( - value is not None - for value in ( - trace.outcome.status, - trace.outcome.stop_reason, - trace.outcome.reward, - trace.outcome.error_category, + for field in ("started_at", "finished_at"): + if any(getattr(event, field) is not None for event in events): + losses.add( + f"events[].{field}", + LossClass.DROPPED, + "ATIF steps carry no timestamps", + ) + # Addressed under the tool call, never under the event: they are + # different values, and blaming the event would misdescribe which one + # was dropped. + if any( + event.tool_call and getattr(event.tool_call, field) is not None + for event in events + ): + losses.add( + f"events[].tool_call.{field}", + LossClass.DROPPED, + "ATIF tool calls carry no timestamps", + ) + if any( + event.tool_call + and any( + block.kind is ContentBlockKind.TEXT and block.raw is not None + for block in event.tool_call.content ) + for event in events ): losses.add( - "outcome", + "events[].tool_call.content[].raw", LossClass.DROPPED, - "ATIF has no run-outcome section; reward and error category live in " - "result.json for the direct path too", + "only the rendered text of a block reaches observation; the source " + "block the IR kept verbatim does not", ) diff --git a/tests/trajectories/test_ir_to_atif.py b/tests/trajectories/test_ir_to_atif.py index 20da68ec1..f001ee76e 100644 --- a/tests/trajectories/test_ir_to_atif.py +++ b/tests/trajectories/test_ir_to_atif.py @@ -24,6 +24,7 @@ import ast import json +from datetime import datetime from pathlib import Path from typing import Any @@ -46,6 +47,7 @@ ToolCall, ToolStatus, TraceEvent, + TraceOutcome, TraceUsage, validate_trace, ) @@ -585,6 +587,408 @@ def test_systemic_losses_are_declared_once_and_only_when_they_apply(): # --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# The rule: outbound loss depends on the IR received, not on ACP's habits +# --------------------------------------------------------------------------- + +# Fields the IR can carry that ATIF cannot represent. Asserted as a complete +# set against a trace that really carries all of them, so a value silently +# dropped by this edge fails here. +RICH_DROPPED = { + "trace_id", + "started_at", + "finished_at", + "provenance", + "extensions", + "agent.provider", + "outcome", + "events[].provenance", + "events[].source_type", + "events[].extensions", + "events[].outcome", + "events[].usage", + "events[].started_at", + "events[].finished_at", + "events[].tool_call.started_at", + "events[].tool_call.finished_at", + "events[].tool_call.name_semantics", + "events[].tool_call.content[].raw", + "events[1].role", + "usage.cache_creation_tokens", + "usage.reasoning_tokens", + "usage.total_tokens", + "usage.source", + "usage.price_source", +} + +T0 = datetime(2026, 8, 15, 10, 0, 0) +T1 = datetime(2026, 8, 15, 10, 0, 5) + + +def _same_shape_from_acp() -> CanonicalTrace: + """A user message, a thought and a tool call — through the ACP edge.""" + return acp_events_to_ir( + [ + {"type": "user_message", "text": "u"}, + {"type": "agent_thought", "text": "why"}, + { + "type": "tool_call", + "tool_call_id": "t", + "kind": "execute", + "title": "ls", + "status": "completed", + "content": [ + {"type": "content", "content": {"type": "text", "text": "out"}} + ], + }, + ], + session_id="s", + agent_name="a", + model="m", + ) + + +def _same_shape_rich() -> CanonicalTrace: + """The same shape, with every field the ACP edge leaves empty filled in.""" + provenance = Provenance(source_format="rich", producer="probe", captured_at=T0) + return CanonicalTrace( + trace_id="trace-abc", + session_id="s", + agent=ModelInfo( + agent_name="a", agent_version="9.9", model="m", provider="anthropic" + ), + started_at=T0, + finished_at=T1, + provenance=provenance, + extensions={"run": "x"}, + events=[ + TraceEvent( + index=0, + kind=EventKind.USER_MESSAGE, + source_type="user_message", + role=Role.USER, + text="u", + started_at=T0, + finished_at=T1, + outcome="ok", + usage=TraceUsage(input_tokens=1), + extensions={"seq": 1}, + provenance=provenance, + ), + TraceEvent( + index=1, + kind=EventKind.TOOL_CALL, + source_type="tool_call", + # Disagrees with the source ATIF will derive from the kind. + role=Role.ENVIRONMENT, + tool_call=ToolCall( + call_id="t", + name="execute", + name_semantics="acp_kind", + title="ls", + status=ToolStatus.COMPLETED, + arguments={"cmd": "ls"}, + started_at=T0, + finished_at=T1, + content=[ + ContentBlock( + kind=ContentBlockKind.TEXT, + text="out", + raw={"type": "content", "meta": "kept"}, + ) + ], + ), + started_at=T0, + finished_at=T1, + usage=TraceUsage(output_tokens=2), + provenance=provenance, + ), + ], + usage=TraceUsage( + input_tokens=1, + output_tokens=2, + cache_read_tokens=3, + cache_creation_tokens=4, + reasoning_tokens=5, + total_tokens=6, + source="llm_proxy_normalized", + cost_usd=0.25, + price_source="litellm", + ), + outcome=TraceOutcome(reward=1.0), + ) + + +def test_an_acp_shaped_trace_declares_none_of_the_richness_it_never_had(): + """Half one of the rule. The inbound report already owns those absences.""" + _, report = ir_to_atif(_same_shape_from_acp()) + declared = _fields(report) + assert declared & RICH_DROPPED == { + # The two an ACP trace really does carry. + "events[].provenance", + "events[].source_type", + "events[].tool_call.name_semantics", + "events[].tool_call.content[].raw", + "provenance", + }, sorted(declared & RICH_DROPPED) + + +def test_a_rich_trace_declares_every_field_atif_cannot_represent(): + """Half two, asserted as a complete set. + + A value the IR carries and ATIF drops must appear here. Adding a field to + the IR that this edge cannot represent fails this test until it is + declared — which is the point. + """ + document, report = ir_to_atif(_same_shape_rich()) + dropped = { + r.field for r in report.by_class(LossClass.DROPPED) if r.space is PathSpace.HUB + } + assert dropped == RICH_DROPPED, { + "missing": sorted(RICH_DROPPED - dropped), + "unexpected": sorted(dropped - RICH_DROPPED), + } + + # None of it reached the document. + text = json.dumps(document) + for probe in ("trace-abc", "anthropic", "2026-08-15T10:00:00", "kept", '"reward"'): + assert probe not in text, probe + + +def test_the_role_disagreement_is_declared_at_the_event_that_has_it(): + """Per event, because it is a fact about one event and not about the trace.""" + document, report = ir_to_atif(_same_shape_rich()) + assert document["steps"][1]["source"] == "agent" + record = report.for_field("events[1].role")[0] + assert record.loss_class is LossClass.DROPPED + assert "environment" in record.detail + # The event whose role agrees with its kind declares nothing. + assert report.for_field("events[0].role") == [] + + +def test_tool_call_timestamps_are_addressed_under_the_tool_call(): + """A path that blamed the event would misdescribe which value was dropped.""" + _, report = ir_to_atif(_same_shape_rich()) + declared = _fields(report) + assert "events[].tool_call.started_at" in declared + assert "events[].tool_call.finished_at" in declared + assert "events[].started_at" in declared + assert "events[].finished_at" in declared + + +def test_only_the_timestamp_actually_present_is_declared(): + """Each timestamp is its own claim, not a single "timestamps" bucket.""" + provenance = Provenance(source_format="probe") + trace = CanonicalTrace( + provenance=provenance, + events=[ + TraceEvent( + index=0, + kind=EventKind.AGENT_MESSAGE, + text="a", + finished_at=T1, + provenance=provenance, + ) + ], + ) + declared = _fields(ir_to_atif(trace)[1]) + assert "events[].finished_at" in declared + assert "events[].started_at" not in declared + + +# Every field of every IR model, and what this edge does with it. A field +# missing from these tables fails `test_every_ir_field_has_a_disposition`, +# which is how a new IR field cannot be added without deciding its fate here. +FIELD_DISPOSITION: dict[str, dict[str, str]] = { + "CanonicalTrace": { + "ir_version": "representation", + "losses": "representation", + "trace_id": "declared", + "session_id": "mapped", + "agent": "container", + "started_at": "declared", + "finished_at": "declared", + "events": "container", + "usage": "container", + "outcome": "declared", + "provenance": "declared", + "extensions": "declared", + }, + "TraceEvent": { + "index": "normalized", + "kind": "mapped", + "source_type": "declared", + "role": "declared", + "text": "mapped", + "reasoning": "mapped", + "reasoning_segments": "normalized", + "tool_call": "container", + "started_at": "declared", + "finished_at": "declared", + "outcome": "declared", + "usage": "declared", + "provenance": "declared", + "extensions": "declared", + }, + "ToolCall": { + "call_id": "mapped", + "name": "mapped", + "name_semantics": "declared", + "title": "mapped", + "status": "mapped", + "arguments": "mapped", + "content": "container", + "started_at": "declared", + "finished_at": "declared", + }, + "ModelInfo": { + "agent_name": "mapped", + "agent_version": "mapped", + "model": "mapped", + "provider": "declared", + }, + "TraceUsage": { + "input_tokens": "mapped", + "output_tokens": "mapped", + "cache_read_tokens": "mapped", + "cost_usd": "mapped", + "cache_creation_tokens": "declared", + "reasoning_tokens": "declared", + "total_tokens": "declared", + "source": "declared", + "price_source": "declared", + }, + "TraceOutcome": { + "status": "declared-with-outcome", + "stop_reason": "declared-with-outcome", + "reward": "declared-with-outcome", + "error_category": "declared-with-outcome", + }, + "ContentBlock": { + "kind": "mapped", + "text": "mapped", + "raw": "declared", + }, +} + + +def test_every_ir_field_has_a_disposition_at_this_edge(): + """Read off the models, so a new IR field cannot slip through undecided. + + ``mapped`` reaches ATIF, ``normalized`` reaches it reshaped, ``declared`` + is dropped *and* recorded, ``container`` holds fields covered by their own + entry, and ``representation`` describes the IR rather than the run. + """ + models = { + "CanonicalTrace": CanonicalTrace, + "TraceEvent": TraceEvent, + "ToolCall": ToolCall, + "ModelInfo": ModelInfo, + "TraceUsage": TraceUsage, + "TraceOutcome": TraceOutcome, + "ContentBlock": ContentBlock, + } + for name, model in models.items(): + assert set(model.model_fields) == set(FIELD_DISPOSITION[name]), { + "model": name, + "undecided": sorted(set(model.model_fields) - set(FIELD_DISPOSITION[name])), + "stale": sorted(set(FIELD_DISPOSITION[name]) - set(model.model_fields)), + } + + +def test_every_declared_disposition_really_produces_a_record(): + """The table is not decoration: each `declared` field must fire on a trace + that carries it.""" + _, report = ir_to_atif(_same_shape_rich()) + declared = _fields(report) + expected = set() + for model_name, fields_ in FIELD_DISPOSITION.items(): + for field_name, disposition in fields_.items(): + if disposition != "declared": + continue + if model_name == "CanonicalTrace": + expected.add(field_name) + elif model_name == "TraceEvent": + expected.add(f"events[].{field_name}") + elif model_name == "ToolCall": + expected.add(f"events[].tool_call.{field_name}") + elif model_name == "ModelInfo": + expected.add(f"agent.{field_name}") + elif model_name == "TraceUsage": + expected.add(f"usage.{field_name}") + elif model_name == "ContentBlock": + expected.add(f"events[].tool_call.content[].{field_name}") + # `role` is declared per event rather than once, so it is addressed by index. + expected.discard("events[].role") + missing = expected - declared + assert not missing, sorted(missing) + + +# --------------------------------------------------------------------------- +# Cost +# --------------------------------------------------------------------------- + + +def test_cost_reaches_atif_and_is_not_declared_lost(): + trace = acp_events_to_ir(_rich_events(), agent_name="a") + trace.usage = TraceUsage( + input_tokens=100, output_tokens=20, cost_usd=1.25, price_source="litellm" + ) + document, report = ir_to_atif(trace) + + assert document["final_metrics"]["total_cost_usd"] == 1.25 + assert report.for_field("usage.cost_usd") == [] + # The table that produced the number has no ATIF slot, and says so. + priced = report.for_field("usage.price_source") + assert priced and priced[0].loss_class is LossClass.DROPPED + + +def test_no_cost_means_no_invented_total_cost_usd(): + trace = acp_events_to_ir(_rich_events(), agent_name="a") + trace.usage = TraceUsage(input_tokens=100, output_tokens=20) + document, report = ir_to_atif(trace) + + assert "total_cost_usd" not in document["final_metrics"] + assert report.for_field("usage.cost_usd") == [] + assert report.for_field("usage.price_source") == [] + + +def test_cost_survives_the_full_pipeline_with_parity(): + """The case H1/H2 cannot exercise: a proxy-backed run carries a cost. + + Both paths are given the same cost, so this is parity for the LiteLLM path + the two real rollouts (agent-native, cost `None`) never reach. + """ + events = _rich_events() + direct = trajectory_to_atif_record( + session_id="s", + agent_name="a", + events=events, + prompts=PROMPTS, + model="m", + total_prompt_tokens=100, + total_completion_tokens=20, + total_cached_tokens=5, + total_cost_usd=1.25, + ) + trace = acp_events_to_ir(events, session_id="s", agent_name="a", model="m") + trace.usage = TraceUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cost_usd=1.25, + source="llm_proxy_normalized", + ) + through_hub, report = ir_to_atif(trace, prompts=PROMPTS) + + assert through_hub == direct + assert through_hub["final_metrics"]["total_cost_usd"] == 1.25 + # `source` is the only usage field lost here; cost is not. + assert {r.field for r in report.records if r.field.startswith("usage.")} == { + "usage.source" + } + + def test_the_converter_imports_only_the_ir(): """The hub must not depend on the exporters it sits between. From 4ec9aaea91ddbd61cebd1a043b253474530f6880 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Sun, 16 Aug 2026 09:31:39 +0200 Subject: [PATCH 09/23] feat(trajectories): read ATIF documents back into the canonical IR, unwired MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the ATIF pair: `ir_to_atif` writes the document, `atif_to_ir` reads one back. Nothing imports either, `export_atif.py` is untouched, and it is still the only writer of `trainer/atif.json`. The module is built on one rule — read what the document says, never what it probably meant. An ATIF document is the output of a lossy conversion and several of its values were fabricated by the converter that wrote it: `agent.version` is the literal "unknown" whenever BenchFlow had none, `arguments` is `{}` for every ACP-derived call, `message` is `""` on a step carrying only a tool call. Reading those back as absent would be guessing which values its counterpart invented, and would make a round trip look better than it is. So every value is taken verbatim. The consequence is the point rather than a wart: a fabricated value returns indistinguishable from an observed one. `arguments: {}` reads back as an observed empty argument map, which the IR's tri-state contract cannot separate from a real one. The information was not lost, it was replaced by a false statement of the same shape — and only the two loss reports carry the truth. What no ATIF document holds — trace id, timestamps at any level, run outcome, provider, per-event provenance — is declared UNSUPPORTED, not DROPPED: the value is absent from the source, not discarded here, and that is the distinction which decides where a fix would land. Every DROPPED record in this direction addresses the source document instead, in the source path space. Two shapes are deliberately not undone. A step with both `message` and `reasoning_content` stays one event, because the blank-line join that produced it is not injective (§5 loss #10) and splitting it would invent a boundary. A step with several tool calls becomes one event per call, since the IR models one per event; no writer here emits that, another producer can. 54 tests, over documents from both writers — the direct exporter and the hub — plus malformed input a document read off disk can actually have. --- src/benchflow/trajectories/ir_from_atif.py | 795 ++++++++++++++++++ tests/trajectories/test_ir_from_atif.py | 918 +++++++++++++++++++++ tests/trajectories/test_trace_ir.py | 2 +- 3 files changed, 1714 insertions(+), 1 deletion(-) create mode 100644 src/benchflow/trajectories/ir_from_atif.py create mode 100644 tests/trajectories/test_ir_from_atif.py diff --git a/src/benchflow/trajectories/ir_from_atif.py b/src/benchflow/trajectories/ir_from_atif.py new file mode 100644 index 000000000..ac85ca226 --- /dev/null +++ b/src/benchflow/trajectories/ir_from_atif.py @@ -0,0 +1,795 @@ +"""ATIF → canonical Trace IR — the inbound ATIF edge. + +> **PROVISIONAL.** Companion to :mod:`benchflow.trajectories.ir`, itself an +> unapproved proposal (`docs/trace-interop.md` §8). Nothing imports this module, +> nothing writes its output to disk, and `export_atif.py` is untouched and still +> the only writer of `trainer/atif.json`. + +This closes the ATIF pair: :mod:`benchflow.trajectories.ir_to_atif` writes the +document, this reads one back. Together they make the round trip +`ACP → IR → ATIF → IR′` measurable, which is the question +:mod:`benchflow.trajectories.ir_round_trip` exists to answer — *how much of a +trace survives a trip through the interchange format?* + +## The rule this module is built on + +**Read what the document says, never what it probably meant.** + +An ATIF document is the output of a lossy conversion, and several of its values +were fabricated by the converter that wrote it: `agent.version` is the literal +`"unknown"` whenever BenchFlow had no version, `arguments` is `{}` for every +ACP-derived call, `message` is `""` on any step carrying only a tool call. +Reading those back as "absent" would make the round trip look better than it is +— the converter would be *guessing* which values were real, and a guess that +happens to be right is still a guess. + +So this edge takes every value verbatim. `"unknown"` becomes the agent version +`"unknown"`; `{}` becomes an observed empty argument map; `""` becomes observed +empty text. The consequence is the most interesting thing the round trip +measures, and it is not a subtraction: + +**A fabricated value returns indistinguishable from an observed one.** On the +way out, `ir_to_atif` declares `arguments` ``SYNTHESIZED`` — "the target +required this and the source never had it". On the way back, the document says +`{}` and nothing marks it as invented, so the reconstructed trace states, with +the full authority of the IR's tri-state contract, that the tool was observed to +be called with no arguments. The information did not merely disappear; it was +*replaced by a false statement of the same shape*. Only the pair of loss reports +carries the truth, and only for the edges that had one. + +That is a fact about ATIF, not a defect of this converter, and it is the reason +the round trip is worth measuring rather than asserting. + +## What has no ATIF antecedent + +ATIF carries no trace id, no timestamps at any level, no run outcome, no +provider, and no per-event provenance. Each is declared ``UNSUPPORTED`` — the +source never carried it — exactly as the ACP edge declares what the capture +events never carried. ``DROPPED`` would be a lie: there is nothing in the +document to drop. + +## Fusion, and why no boundary is invented + +`ir_to_atif` folds a run of reasoning events into the *next* agent step's +`reasoning_content`, joined by a blank line. Reading back, one step is one +event: a step with both `message` and `reasoning_content` becomes one +``AGENT_MESSAGE`` carrying both, not a reasoning event followed by a message +event. Splitting it would invent a boundary the document does not contain — the +join is not injective, which is `§5 loss #10` — so the fusion is reported as +what it is: fewer events out than in. + +The one place a step legitimately becomes more than one event is a step with +several `tool_calls`. The IR models one tool call per event (invariant 3), so an +*n*-call step becomes *n* events. `trajectory_to_atif_record` never writes such +a step; a document from another producer can. + +## Step metrics are carried, not interpreted + +ATIF allows `metrics` on an agent step. This repository never writes one, and no +ATIF schema is vendored here to read its vocabulary against, so a step's +`metrics` object is carried verbatim into ``extensions`` rather than mapped onto +:class:`~benchflow.trajectories.ir.TraceUsage`. Mapping it would be asserting a +field correspondence nobody has checked. `final_metrics` *is* mapped: this +repository writes it, and `ir_to_atif` pins the four keys. +""" + +from __future__ import annotations + +from typing import Any + +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlock, + ContentBlockKind, + EventKind, + LossClass, + LossReport, + ModelInfo, + PathSpace, + Provenance, + Role, + ToolCall, + ToolStatus, + TraceEvent, + TraceOutcome, + TraceUsage, +) + +LOSS_DIRECTION = "atif->ir" + +ATIF_SOURCE = "atif" +"""Trace- and event-level provenance for anything read out of an ATIF document.""" + +_STATUS_BY_VALUE = {status.value: status for status in ToolStatus} + +# ``source`` values ``ir_to_atif`` and ``export_atif`` emit. A step whose source +# is outside this map becomes an ``UNKNOWN`` event with the string kept, rather +# than being dropped the way every exporter drops what it does not recognize. +_ROLE_BY_SOURCE: dict[str, Role] = { + "user": Role.USER, + "agent": Role.AGENT, + "oracle": Role.ORACLE, +} + +# Keys the two writers in this repository produce. Anything else on a recognized +# object rides into ``extensions`` verbatim instead of being discarded. +_DOCUMENT_KEYS = frozenset( + {"schema_version", "session_id", "agent", "steps", "final_metrics"} +) +_AGENT_KEYS = frozenset({"name", "version", "model_name"}) +_STEP_KEYS = frozenset( + { + "step_id", + "source", + "message", + "reasoning_content", + "tool_calls", + "observation", + "metrics", + } +) +_TOOL_CALL_KEYS = frozenset({"tool_call_id", "function_name", "arguments", "extra"}) + +# ``final_metrics`` keys with an IR home, in the direction ``ir_to_atif`` writes +# them. Inverting the same tuple in both modules is what keeps the pair honest. +_USAGE_FIELDS: tuple[tuple[str, str], ...] = ( + ("total_prompt_tokens", "input_tokens"), + ("total_completion_tokens", "output_tokens"), + ("total_cached_tokens", "cache_read_tokens"), + ("total_cost_usd", "cost_usd"), +) + +# IR usage fields no ATIF document carries. +_USAGE_UNSUPPORTED: tuple[str, ...] = ( + "cache_creation_tokens", + "reasoning_tokens", + "total_tokens", + "source", + "price_source", +) + + +def _extras(raw: dict[str, Any], known: frozenset[str]) -> dict[str, Any]: + """Keys of *raw* outside *known*, carried verbatim.""" + return {key: value for key, value in raw.items() if key not in known} + + +def _coerce_str( + raw: dict[str, Any], + key: str, + field: str, + losses: LossReport, +) -> str | None: + """Read a string field, declaring any coercion this module performs. + + Absence returns ``None`` and declares nothing: the field simply is not in + the document, and the systemic records cover what ATIF never carries. A + non-string *present* value is this module's own reshaping act, so it is + declared — the same rule :mod:`ir_from_acp` follows. + """ + if key not in raw: + return None + value = raw[key] + if isinstance(value, str): + return value + losses.add( + field, + LossClass.NORMALIZED, + f"document carries {type(value).__name__} where ATIF specifies a string; " + "coerced with str()", + ) + return str(value) + + +def _int_or_none(value: Any) -> int | None: + """An int, or ``None`` for anything that is not one. + + ``bool`` is excluded on purpose: it is an ``int`` subclass in Python, and a + token count of ``True`` is a malformed document, not the number 1. + """ + if isinstance(value, bool) or not isinstance(value, int): + return None + return value + + +def _float_or_none(value: Any) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) + + +def _content_blocks( + results: list[Any], + call_id: str | None, + where: str, + losses: LossReport, +) -> list[ContentBlock]: + """Observation results belonging to *call_id*, as IR content blocks. + + Every block is ``TEXT`` with ``raw=None``: ATIF stores rendered content, so + the source block `ir_to_atif` declared dropped is genuinely not there to + reconstruct. + """ + blocks: list[ContentBlock] = [] + for position, result in enumerate(results): + if not isinstance(result, dict) or result.get("source_call_id") != call_id: + continue + content = result.get("content") + if not isinstance(content, str): + losses.add( + f"{where}.tool_call.content[{len(blocks)}]", + LossClass.NORMALIZED, + f"observation.results[{position}].content is " + f"{type(content).__name__}, not a string; coerced with str()", + ) + content = str(content) + blocks.append(ContentBlock(kind=ContentBlockKind.TEXT, text=content)) + return blocks + + +def _tool_call_to_ir( + raw: dict[str, Any], + where: str, + losses: LossReport, +) -> tuple[ToolCall, dict[str, Any]]: + """One ATIF ``tool_calls`` entry as an IR tool call, and its unmapped keys. + + :class:`~benchflow.trajectories.ir.ToolCall` has no ``extensions`` of its + own, so the leftovers are returned for the caller to put on the event, under + a ``tool_call`` key — merging them into the event's own extensions would let + a step key and a tool-call key of the same name collide silently. + + ``arguments`` is taken verbatim, including the ``{}`` that `ir_to_atif` + writes for every ACP-derived call. This is the module's central rule and its + most consequential application: reading ``{}`` back as ``None`` would + reconstruct the ACP-side truth by guessing, and would make the round trip + appear to preserve a distinction the document does not carry. + """ + extra = raw.get("extra") + extra_map = extra if isinstance(extra, dict) else {} + + status: ToolStatus | None = None + extras = _extras(raw, _TOOL_CALL_KEYS) + raw_status = extra_map.get("status") + if raw_status is not None: + status = _STATUS_BY_VALUE.get(str(raw_status)) + if status is None: + status = ToolStatus.UNKNOWN + extras["source_status"] = raw_status + losses.add( + f"{where}.tool_call.status", + LossClass.NORMALIZED, + f"status {raw_status!r} is outside the IR vocabulary; recorded as " + "unknown with the original kept in extensions", + ) + + arguments = raw.get("arguments") + if "arguments" not in raw: + # ATIF requires the field; a document without it is non-conformant, and + # invariant 7 requires the resulting ``None`` to be declared. + losses.add( + f"{where}.tool_call.arguments", + LossClass.UNSUPPORTED, + "ATIF requires tool_calls[].arguments and this document omits it, so " + "the call carries no arguments to read", + ) + arguments = None + elif not isinstance(arguments, dict): + losses.add( + f"{where}.tool_call.arguments", + LossClass.UNSUPPORTED, + f"arguments is {type(arguments).__name__}, not an object; the IR " + "models arguments as a mapping and cannot carry this value", + ) + extras["source_arguments"] = arguments + arguments = None + + unmapped_extra = { + key: value for key, value in extra_map.items() if key not in ("title", "status") + } + if unmapped_extra: + extras["extra"] = unmapped_extra + + title = extra_map.get("title") + tool_call = ToolCall( + call_id=_coerce_str(raw, "tool_call_id", f"{where}.tool_call.call_id", losses), + name=_coerce_str(raw, "function_name", f"{where}.tool_call.name", losses), + # What the document says it is. The ACP edge writes ``"acp_kind"`` for + # the same value, and the two disagreeing is exactly the normalization + # `ir_to_atif` declared on the way out: ATIF has one slot and no way to + # say the string in it is a category rather than a function name. + name_semantics="function_name" if "function_name" in raw else None, + title=str(title) if title is not None else None, + status=status, + arguments=arguments, + ) + return tool_call, extras + + +def atif_to_ir(document: dict[str, Any]) -> CanonicalTrace: + """Convert one ATIF document into a :class:`CanonicalTrace`. + + Everything comes from *document*: no rollout artifact is consulted, so a + value that lives only in `result.json` or `timing.json` is a declared loss + rather than a silent enrichment — the same discipline + :func:`~benchflow.trajectories.ir_from_acp.acp_events_to_ir` follows. + + The returned trace carries its own :class:`LossReport` on + :attr:`~benchflow.trajectories.ir.CanonicalTrace.losses` and satisfies + :func:`~benchflow.trajectories.ir.validate_trace` for any input, including + documents ATIF's own validator would reject. + + Raises ``TypeError`` for a non-mapping document. There is no useful partial + reading of something that is not an ATIF document at all, and returning an + empty trace would claim a conversion happened. + """ + if not isinstance(document, dict): + raise TypeError( + f"ATIF document must be a mapping, got {type(document).__name__}" + ) + + losses = LossReport(direction=LOSS_DIRECTION) + ir_events: list[TraceEvent] = [] + + raw_steps = document.get("steps") + steps = raw_steps if isinstance(raw_steps, list) else [] + if raw_steps is not None and not isinstance(raw_steps, list): + losses.add( + "steps", + LossClass.DROPPED, + f"steps is {type(raw_steps).__name__}, not a list; no event could be " + "read from it", + space=PathSpace.SOURCE, + ) + + had_tool_call = False + for position, raw_step in enumerate(steps): + if not isinstance(raw_step, dict): + losses.add( + f"steps[{position}]", + LossClass.DROPPED, + f"step is {type(raw_step).__name__}, not a JSON object; the IR has " + "no representation for it", + space=PathSpace.SOURCE, + ) + continue + events = _step_to_events(raw_step, len(ir_events), losses) + had_tool_call = had_tool_call or any( + event.kind is EventKind.TOOL_CALL for event in events + ) + ir_events.extend(events) + + usage = _read_usage(document, losses) + agent = _read_agent(document, losses) + _declare_systemic_losses(losses, had_tool_call=had_tool_call, has_usage=usage) + + extensions = _extras(document, _DOCUMENT_KEYS) + schema_version = document.get("schema_version") + if schema_version is not None: + # Kept because it identifies the dialect this trace was read from, and + # the IR has no field for a source format's own version. + extensions["schema_version"] = schema_version + + return CanonicalTrace( + session_id=_coerce_str(document, "session_id", "session_id", losses), + agent=agent, + events=ir_events, + usage=usage, + # Every field stays ``None``: ATIF has no outcome section, so there is + # nothing to read. The section itself is present because + # ``outcome.stop_reason`` is declared below and a record cannot address + # a path through a null parent. + outcome=TraceOutcome(), + provenance=Provenance(source_format=ATIF_SOURCE), + extensions=extensions, + losses=losses, + ) + + +def _step_to_events( + raw_step: dict[str, Any], + first_index: int, + losses: LossReport, +) -> list[TraceEvent]: + """One ATIF step as one IR event — or *n* for an *n*-tool-call step.""" + where = f"events[{first_index}]" + source = raw_step.get("source") + source_str = source if isinstance(source, str) else None + role = _ROLE_BY_SOURCE.get(source_str or "") + + message = _coerce_str(raw_step, "message", f"{where}.text", losses) + reasoning = _coerce_str(raw_step, "reasoning_content", f"{where}.reasoning", losses) + + extensions = _extras(raw_step, _STEP_KEYS) + if "step_id" in raw_step: + # Carried, not mapped onto ``index``: invariant 2 makes the IR index a + # dense position, and a document whose step_ids are sparse or restarted + # would otherwise be silently renumbered with no trace of the original. + extensions["step_id"] = raw_step["step_id"] + if "metrics" in raw_step: + extensions["metrics"] = raw_step["metrics"] + losses.add( + f"{where}.usage", + LossClass.NORMALIZED, + "step metrics are carried verbatim into extensions; no ATIF schema is " + "vendored here to map their vocabulary onto TraceUsage", + ) + + raw_calls = raw_step.get("tool_calls") + calls = ( + [call for call in raw_calls if isinstance(call, dict)] + if isinstance(raw_calls, list) + else [] + ) + if isinstance(raw_calls, list) and len(calls) != len(raw_calls): + losses.add( + f"steps[{first_index}].tool_calls", + LossClass.DROPPED, + "the step carries tool_calls entries that are not JSON objects", + space=PathSpace.SOURCE, + ) + + observation = raw_step.get("observation") + results = ( + observation.get("results") + if isinstance(observation, dict) + and isinstance(observation.get("results"), list) + else [] + ) + + if source_str is not None and role is None: + # Outside the vocabulary this repository writes. Kept whole rather than + # forced into a role the document does not support. + return [ + TraceEvent( + index=first_index, + kind=EventKind.UNKNOWN, + source_type=source_str, + text=message, + reasoning=reasoning, + provenance=Provenance(source_format=ATIF_SOURCE), + extensions={**extensions, **_unread(raw_step)}, + ) + ] + + if calls: + return _tool_call_events( + calls, + results, + first_index=first_index, + source_str=source_str, + role=role, + message=message, + reasoning=reasoning, + extensions=extensions, + losses=losses, + ) + + if results: + # An observation with nothing to attach it to: the IR models tool output + # under a tool call, so it rides verbatim rather than being dropped. + extensions["observation"] = observation + losses.add( + f"{where}.tool_call", + LossClass.NORMALIZED, + "the step carries an observation but no tool call; the IR models " + "content blocks under a tool call, so it is kept in extensions", + ) + + kind = _kind_for_step(role, message, reasoning) + return [ + TraceEvent( + index=first_index, + kind=kind, + source_type=source_str, + role=role, + text=message, + reasoning=reasoning, + # One step is one segment. `ThoughtBuffer` joined an unknown number + # of thoughts with a blank line and the join is not injective, so + # splitting on blank lines would invent boundaries — §5 loss #10 is + # exactly this, and it is not recoverable here. + reasoning_segments=[reasoning] if reasoning is not None else None, + provenance=Provenance(source_format=ATIF_SOURCE), + extensions=extensions, + ) + ] + + +def _kind_for_step( + role: Role | None, message: str | None, reasoning: str | None +) -> EventKind: + """The event kind a non-tool step maps to. + + A step with reasoning and no message is what `ir_to_atif` writes when it + flushes buffered thoughts, so it reads back as reasoning. A step with both + is one event carrying both — the fusion is reported, not undone. + """ + if role is Role.USER: + return EventKind.USER_MESSAGE + if role is Role.ORACLE: + return EventKind.ORACLE + if reasoning is not None and not message: + return EventKind.AGENT_REASONING + return EventKind.AGENT_MESSAGE + + +def _tool_call_events( + calls: list[dict[str, Any]], + results: list[Any], + *, + first_index: int, + source_str: str | None, + role: Role | None, + message: str | None, + reasoning: str | None, + extensions: dict[str, Any], + losses: LossReport, +) -> list[TraceEvent]: + """The events for a step carrying tool calls, one per call. + + Text, reasoning and the step's extra keys ride on the **first** event only. + Copying them onto each event of a multi-call step would multiply one + observation into several; the alternative — dropping them — would lose the + step's message. A multi-call step is declared normalized, since one source + object became several IR events. + """ + if len(calls) > 1: + losses.add( + f"steps[{first_index}]", + LossClass.NORMALIZED, + f"the step carries {len(calls)} tool calls and the IR models one per " + "event, so it became that many events; the grouping is not preserved", + space=PathSpace.SOURCE, + ) + + events: list[TraceEvent] = [] + for offset, raw_call in enumerate(calls): + index = first_index + offset + where = f"events[{index}]" + tool_call, call_extras = _tool_call_to_ir(raw_call, where, losses) + tool_call.content = _content_blocks(results, tool_call.call_id, where, losses) + + event_extensions = dict(extensions) if offset == 0 else {} + if call_extras: + event_extensions["tool_call"] = call_extras + events.append( + TraceEvent( + index=index, + kind=EventKind.TOOL_CALL, + source_type=source_str, + role=role, + text=message if offset == 0 else None, + reasoning=reasoning if offset == 0 else None, + reasoning_segments=( + [reasoning] if offset == 0 and reasoning is not None else None + ), + tool_call=tool_call, + provenance=Provenance(source_format=ATIF_SOURCE), + extensions=event_extensions, + ) + ) + + # Results addressing no call in this step. ATIF resolves ``source_call_id`` + # within one step, so these are malformed rather than cross-step references; + # they ride on the first event so they stay in the trace. + known_ids = { + event.tool_call.call_id for event in events if event.tool_call is not None + } + unmatched = [ + result + for result in results + if not (isinstance(result, dict) and result.get("source_call_id") in known_ids) + ] + if unmatched and events: + events[0].extensions["unmatched_observation_results"] = unmatched + losses.add( + f"steps[{first_index}].observation.results", + LossClass.NORMALIZED, + "results whose source_call_id matches no tool call in the step; kept " + "verbatim in extensions rather than attached to a call they do not " + "belong to", + space=PathSpace.SOURCE, + ) + return events + + +def _unread(raw_step: dict[str, Any]) -> dict[str, Any]: + """The recognized-but-unmapped keys of an unknown-source step. + + An unknown step keeps its whole body, since this converter cannot say which + parts it understood. + """ + return { + key: raw_step[key] for key in ("tool_calls", "observation") if key in raw_step + } + + +def _read_agent(document: dict[str, Any], losses: LossReport) -> ModelInfo: + """The agent block, verbatim. + + ``"unknown"`` is **not** translated back to ``None``. `ir_to_atif` writes + that literal whenever the trace had no name or version, but a document can + equally carry it as an observed value, and this converter has no way to tell + the two apart. Guessing would be the one thing this module refuses to do — + see the module docstring. + """ + raw_agent = document.get("agent") + if not isinstance(raw_agent, dict): + if raw_agent is not None: + losses.add( + "agent", + LossClass.DROPPED, + f"agent is {type(raw_agent).__name__}, not an object", + space=PathSpace.SOURCE, + ) + return ModelInfo() + return ModelInfo( + agent_name=_coerce_str(raw_agent, "name", "agent.agent_name", losses), + agent_version=_coerce_str(raw_agent, "version", "agent.agent_version", losses), + model=_coerce_str(raw_agent, "model_name", "agent.model", losses), + ) + + +def _read_usage(document: dict[str, Any], losses: LossReport) -> TraceUsage | None: + """Trace usage from ``final_metrics``, or ``None`` when it carries none. + + ``total_steps`` is a property of the document, not of the run: `ir_to_atif` + declares it ``SYNTHESIZED`` in the target space on the way out, and it is + declared dropped in the source space here. The two records are the same fact + from the two sides of the edge. + """ + raw_metrics = document.get("final_metrics") + if not isinstance(raw_metrics, dict): + if raw_metrics is not None: + losses.add( + "final_metrics", + LossClass.DROPPED, + f"final_metrics is {type(raw_metrics).__name__}, not an object", + space=PathSpace.SOURCE, + ) + return None + + if "total_steps" in raw_metrics: + losses.add( + "final_metrics.total_steps", + LossClass.DROPPED, + "a count of the document's own steps; the IR carries no step count " + "and deriving one would restate the event list", + space=PathSpace.SOURCE, + ) + + values: dict[str, Any] = {} + for atif_field, ir_field in _USAGE_FIELDS: + if atif_field not in raw_metrics: + continue + raw_value = raw_metrics[atif_field] + value = ( + _float_or_none(raw_value) + if ir_field == "cost_usd" + else _int_or_none(raw_value) + ) + if value is None: + losses.add( + f"usage.{ir_field}", + LossClass.DROPPED, + f"final_metrics.{atif_field} is {type(raw_value).__name__}, which " + "is not a number the IR can carry for this field", + space=PathSpace.SOURCE, + ) + continue + values[ir_field] = value + + unmapped = ( + set(raw_metrics) - {field for field, _ in _USAGE_FIELDS} - {"total_steps"} + ) + if unmapped: + losses.add( + "final_metrics", + LossClass.DROPPED, + "final_metrics keys with no IR field: " + ", ".join(sorted(unmapped)), + space=PathSpace.SOURCE, + ) + + if not values: + return None + return TraceUsage(**values) + + +def _declare_systemic_losses( + losses: LossReport, + *, + had_tool_call: bool, + has_usage: TraceUsage | None, +) -> None: + """What no ATIF document carries, declared once each. + + All ``UNSUPPORTED``: the values are absent from the source, not discarded by + this conversion. That is the same distinction `ir_from_acp` draws, and it is + what makes the two inbound reports comparable — a reader can ask which of + two source formats carries more, and the answer is in the class rather than + in the prose. + + Declared unconditionally, because they hold for every ATIF document rather + than for the one in hand. The outbound rule is the opposite (declare only + what *this* trace actually loses) and the asymmetry is deliberate: an + inbound report describes a format's ceiling, an outbound one describes a + conversion's cost. + """ + losses.add( + "trace_id", + LossClass.UNSUPPORTED, + "ATIF documents carry no trace id", + ) + for field in ("started_at", "finished_at"): + losses.add( + field, + LossClass.UNSUPPORTED, + "ATIF has no run-level timestamps; wall clock lives in timing.json, " + "which this converter does not read", + ) + losses.add( + "agent.provider", + LossClass.UNSUPPORTED, + "ATIF's agent block is name/version/model_name only", + ) + losses.add( + "outcome", + LossClass.UNSUPPORTED, + "ATIF has no run-outcome section; status, stop_reason, reward and error " + "category live in result.json, which this converter does not read", + "§5 loss #9", + ) + losses.add( + "events[].started_at", + LossClass.UNSUPPORTED, + "ATIF steps carry no timestamps", + ) + losses.add( + "events[].finished_at", + LossClass.UNSUPPORTED, + "ATIF steps carry no timestamps", + ) + losses.add( + "events[].usage", + LossClass.UNSUPPORTED, + "ATIF per-step metrics are not written by any producer in this " + "repository, and an unrecognized metrics object is carried verbatim " + "rather than interpreted", + ) + losses.add( + "events[].outcome", + LossClass.UNSUPPORTED, + "ATIF steps carry no per-step outcome; a timeout event does not survive " + "the outbound edge at all", + "§5 loss #4", + ) + if had_tool_call: + for field in ("started_at", "finished_at"): + losses.add( + f"events[].tool_call.{field}", + LossClass.UNSUPPORTED, + "ATIF tool calls carry no timestamps", + "§5 loss #3", + ) + losses.add( + "events[].tool_call.content[].raw", + LossClass.UNSUPPORTED, + "ATIF stores rendered observation text; the source content block is " + "not in the document to reconstruct", + "§5 loss #5", + ) + if has_usage is None: + losses.add( + "usage", + LossClass.UNSUPPORTED, + "the document carries no final_metrics this converter can read", + ) + else: + for field in _USAGE_UNSUPPORTED: + if getattr(has_usage, field) is None: + losses.add( + f"usage.{field}", + LossClass.UNSUPPORTED, + "ATIF final_metrics has no slot for it", + ) diff --git a/tests/trajectories/test_ir_from_atif.py b/tests/trajectories/test_ir_from_atif.py new file mode 100644 index 000000000..7027a7dd8 --- /dev/null +++ b/tests/trajectories/test_ir_from_atif.py @@ -0,0 +1,918 @@ +"""Conversion suite for ``ATIF → canonical Trace IR`` — the inbound ATIF edge. + +The two edges before this one had an easy source of truth: `ACP → IR` could be +checked against events a real `ACPSession` emitted, and `IR → ATIF` against the +document the direct exporter writes. This edge reads a document that is *itself* +the output of a lossy conversion, and its whole discipline is one rule: + + read what the document says, never what it probably meant. + +Most of what follows tests that rule at the places where breaking it would be +tempting and would make the round trip look better than it is — `arguments: {}`, +`agent.version: "unknown"`, `message: ""`, and the thought boundaries the +outbound join destroyed. + +Nothing here writes to disk. `export_atif.py` is imported read-only, as the +producer whose documents this edge has to be able to read. +""" + +from __future__ import annotations + +import ast +from pathlib import Path +from typing import Any + +import pytest + +from benchflow.trajectories import ir_from_atif as ir_from_atif_module +from benchflow.trajectories.export_atif import trajectory_to_atif_record +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlock, + ContentBlockKind, + EventKind, + LossClass, + ModelInfo, + PathSpace, + Role, + ToolCall, + ToolStatus, + TraceEvent, + TraceOutcome, + TraceUsage, + validate_trace, +) +from benchflow.trajectories.ir_from_acp import acp_events_to_ir +from benchflow.trajectories.ir_from_atif import ( + ATIF_SOURCE, + LOSS_DIRECTION, + atif_to_ir, +) +from benchflow.trajectories.ir_to_atif import ir_to_atif +from tests.trajectories.test_atif_preservation import _rich_events +from tests.trajectories.test_trace_ir import resolve_ir_path + +PROMPTS = ["Solve the task.", "Then stop."] + + +def _rich_document(**kwargs: Any) -> dict[str, Any]: + """An ATIF document from the direct exporter, over real captured events.""" + return trajectory_to_atif_record( + session_id="sess-e", + agent_name="claude-code", + events=_rich_events(), + prompts=PROMPTS, + model="claude-sonnet-5", + **kwargs, + ) + + +def _hub_document(**kwargs: Any) -> dict[str, Any]: + """The same document, produced through the hub instead.""" + trace = acp_events_to_ir( + _rich_events(), + session_id="sess-e", + agent_name="claude-code", + model="claude-sonnet-5", + ) + document, _ = ir_to_atif(trace, prompts=PROMPTS, **kwargs) + return document + + +def _minimal(**overrides: Any) -> dict[str, Any]: + document: dict[str, Any] = { + "schema_version": "ATIF-v1.7", + "agent": {"name": "a", "version": "1"}, + "steps": [{"step_id": 1, "source": "user", "message": "hi"}], + "final_metrics": {"total_steps": 1}, + } + document.update(overrides) + return document + + +def _fields(trace: CanonicalTrace, loss_class: LossClass) -> set[str]: + return { + record.field + for record in trace.losses.records + if record.loss_class is loss_class + } + + +# --------------------------------------------------------------------------- +# The documents this repository actually produces +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("document_factory", [_rich_document, _hub_document]) +def test_a_real_document_reads_back_into_a_valid_trace(document_factory): + """Both writers' output is readable, and satisfies every IR invariant.""" + trace = atif_to_ir(document_factory()) + assert validate_trace(trace) == [] + assert trace.provenance.source_format == ATIF_SOURCE + assert trace.losses.direction == LOSS_DIRECTION + + +def test_the_two_writers_produce_the_same_trace(): + """Parity, read from the other side. + + Slice D asserts the hub and the direct exporter write the same document. + Reading both back has to give the same trace — a weaker claim implied by + the first, and worth pinning here because this suite would otherwise never + notice if that parity broke underneath it. + """ + direct = atif_to_ir(_rich_document()).model_dump(mode="json") + hub = atif_to_ir(_hub_document()).model_dump(mode="json") + assert direct == hub + + +def test_step_order_is_preserved_and_indices_are_dense(): + trace = atif_to_ir(_rich_document()) + assert [event.index for event in trace.events] == list(range(len(trace.events))) + + +def test_every_step_becomes_an_event(): + """No step is silently skipped — the failure mode every exporter has today.""" + document = _rich_document() + trace = atif_to_ir(document) + assert len(trace.events) == len(document["steps"]) + + +# --------------------------------------------------------------------------- +# The rule: verbatim, never inferred +# --------------------------------------------------------------------------- + + +def test_empty_arguments_are_read_as_observed_not_as_absent(): + """The central rule, at the field that matters most. + + `ir_to_atif` writes ``{}`` for a call whose IR ``arguments`` is ``None`` and + declares it SYNTHESIZED. Nothing in the document marks it as invented, so + reading it back as ``None`` would be this converter guessing which values + its counterpart fabricated. It reads ``{}``. + + That is the fabrication the round trip measures: ``None`` ("never observed") + becomes ``{}`` ("observed empty"), and the tri-state contract cannot tell. + """ + trace = atif_to_ir(_rich_document()) + calls = [event.tool_call for event in trace.events if event.tool_call] + assert calls, "fixture must contain a tool call" + assert all(call.arguments == {} for call in calls) + assert all(call.arguments is not None for call in calls) + # And nothing is declared, because nothing was lost *here*. + assert not [ + record + for record in trace.losses.records + if record.field.endswith("tool_call.arguments") + ] + + +def test_unknown_agent_version_is_read_verbatim(): + """``"unknown"`` is the exporter's filler and also a legal observed value. + + Translating it back to ``None`` would recover the ACP-side truth by + guessing, and would silently corrupt a document that really did carry an + agent named ``unknown``. + """ + trace = atif_to_ir(_rich_document()) + assert trace.agent.agent_version == "unknown" + + +def test_empty_message_is_read_as_observed_empty_text(): + """ATIF requires a message on every step; tool steps get ``""``. + + The IR distinguishes ``None`` from ``""``, and this edge cannot tell which + empty string was observed and which was filler — so it reads what is there. + """ + trace = atif_to_ir(_rich_document()) + tool_events = [event for event in trace.events if event.kind is EventKind.TOOL_CALL] + assert tool_events + assert all(event.text == "" for event in tool_events) + + +def test_a_fused_step_stays_one_event(): + """A step with message *and* reasoning_content is one event, not two. + + The outbound edge folded a run of reasoning events into the next agent + step, joined by a blank line. The join is not injective (§5 loss #10), so + splitting it here would invent boundaries. The fusion is reported by the + event count, not undone. + """ + document = _minimal( + steps=[ + { + "step_id": 1, + "source": "agent", + "message": "done", + "reasoning_content": "first\n\nsecond", + } + ] + ) + trace = atif_to_ir(document) + assert len(trace.events) == 1 + event = trace.events[0] + assert event.kind is EventKind.AGENT_MESSAGE + assert event.text == "done" + assert event.reasoning == "first\n\nsecond" + assert event.reasoning_segments == ["first\n\nsecond"] + assert validate_trace(trace) == [] + + +def test_a_flushed_thought_step_reads_as_reasoning(): + """message ``""`` plus reasoning_content is what a flushed thought looks + like, and it comes back as a reasoning event.""" + document = _minimal( + steps=[ + {"step_id": 1, "source": "agent", "message": "", "reasoning_content": "t"} + ] + ) + event = atif_to_ir(document).events[0] + assert event.kind is EventKind.AGENT_REASONING + assert event.reasoning == "t" + + +def test_no_prompt_step_is_recognized_as_synthetic(): + """The leading `user` steps built from *prompts* read as user messages. + + They are not trace data — `ir_to_atif` declares each one SYNTHESIZED in the + target space — but the document does not say so, and this edge does not + guess. The document's first two user steps carry the same text and both + become events, which is the over-count §5.2 measured, now visible from the + other side. + """ + trace = atif_to_ir(_rich_document()) + user_texts = [ + event.text for event in trace.events if event.kind is EventKind.USER_MESSAGE + ] + assert user_texts[: len(PROMPTS)] == PROMPTS + + +# --------------------------------------------------------------------------- +# Step shapes +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("source", "kind", "role"), + [ + ("user", EventKind.USER_MESSAGE, Role.USER), + ("agent", EventKind.AGENT_MESSAGE, Role.AGENT), + ("oracle", EventKind.ORACLE, Role.ORACLE), + ], +) +def test_step_source_maps_to_kind_and_role(source, kind, role): + document = _minimal(steps=[{"step_id": 1, "source": source, "message": "m"}]) + event = atif_to_ir(document).events[0] + assert event.kind is kind + assert event.role is role + assert event.source_type == source + + +def test_an_unknown_step_source_survives_whole(): + """Every exporter in the tree drops what it does not recognize. This does + not: the kind becomes UNKNOWN and the source string is kept.""" + document = _minimal(steps=[{"step_id": 1, "source": "environment", "message": "m"}]) + event = atif_to_ir(document).events[0] + assert event.kind is EventKind.UNKNOWN + assert event.source_type == "environment" + assert event.text == "m" + assert event.role is None + + +def test_tool_call_fields_are_read_into_the_ir(): + document = _minimal( + steps=[ + { + "step_id": 1, + "source": "agent", + "message": "", + "tool_calls": [ + { + "tool_call_id": "c1", + "function_name": "execute", + "arguments": {"cmd": "ls"}, + "extra": {"title": "list files", "status": "completed"}, + } + ], + "observation": { + "results": [{"source_call_id": "c1", "content": "a\nb"}] + }, + } + ] + ) + event = atif_to_ir(document).events[0] + assert event.kind is EventKind.TOOL_CALL + call = event.tool_call + assert call.call_id == "c1" + assert call.name == "execute" + assert call.title == "list files" + assert call.status is ToolStatus.COMPLETED + assert call.arguments == {"cmd": "ls"} + assert call.content == [ContentBlock(kind=ContentBlockKind.TEXT, text="a\nb")] + + +def test_function_name_semantics_are_recorded_as_the_document_states_them(): + """ATIF calls the slot ``function_name`` and the IR records that, even + though an ACP-derived document holds a *kind* in it. + + This is the normalization `ir_to_atif` declared on the way out, seen from + the other side: the semantics the ACP edge recorded (``acp_kind``) is not + in the document, so it cannot come back. + """ + trace = atif_to_ir(_rich_document()) + calls = [event.tool_call for event in trace.events if event.tool_call] + assert all(call.name_semantics == "function_name" for call in calls) + + +def test_a_multi_call_step_becomes_one_event_per_call(): + """The IR models one tool call per event, so an n-call step is n events. + + No writer in this repository emits such a step; another producer can, and + the grouping it had is declared lost rather than silently flattened. + """ + document = _minimal( + steps=[ + { + "step_id": 1, + "source": "agent", + "message": "", + "tool_calls": [ + {"tool_call_id": "c1", "function_name": "a", "arguments": {}}, + {"tool_call_id": "c2", "function_name": "b", "arguments": {}}, + ], + "observation": { + "results": [ + {"source_call_id": "c2", "content": "second"}, + {"source_call_id": "c1", "content": "first"}, + ] + }, + } + ] + ) + trace = atif_to_ir(document) + assert [event.index for event in trace.events] == [0, 1] + assert [event.tool_call.call_id for event in trace.events] == ["c1", "c2"] + # Results are matched by id, not by position. + assert trace.events[0].tool_call.content[0].text == "first" + assert trace.events[1].tool_call.content[0].text == "second" + assert validate_trace(trace) == [] + normalized = [ + record + for record in trace.losses.records + if record.loss_class is LossClass.NORMALIZED + and record.space is PathSpace.SOURCE + and record.field == "steps[0]" + ] + assert len(normalized) == 1 + + +def test_a_result_matching_no_call_is_kept_in_extensions(): + document = _minimal( + steps=[ + { + "step_id": 1, + "source": "agent", + "message": "", + "tool_calls": [ + {"tool_call_id": "c1", "function_name": "a", "arguments": {}} + ], + "observation": { + "results": [{"source_call_id": "other", "content": "orphan"}] + }, + } + ] + ) + trace = atif_to_ir(document) + event = trace.events[0] + assert event.tool_call.content == [] + assert event.extensions["unmatched_observation_results"] == [ + {"source_call_id": "other", "content": "orphan"} + ] + + +def test_an_observation_without_a_tool_call_is_kept_verbatim(): + document = _minimal( + steps=[ + { + "step_id": 1, + "source": "agent", + "message": "m", + "observation": {"results": [{"source_call_id": "x", "content": "c"}]}, + } + ] + ) + trace = atif_to_ir(document) + event = trace.events[0] + assert event.kind is EventKind.AGENT_MESSAGE + assert event.extensions["observation"]["results"][0]["content"] == "c" + assert "events[0].tool_call" in _fields(trace, LossClass.NORMALIZED) + + +def test_step_id_is_carried_rather_than_mapped_onto_index(): + """Invariant 2 makes ``index`` a dense position, so a document with sparse + or restarted step_ids would be renumbered with no record of the original.""" + document = _minimal( + steps=[ + {"step_id": 7, "source": "user", "message": "a"}, + {"step_id": 9, "source": "user", "message": "b"}, + ] + ) + trace = atif_to_ir(document) + assert [event.index for event in trace.events] == [0, 1] + assert [event.extensions["step_id"] for event in trace.events] == [7, 9] + + +def test_step_metrics_are_carried_without_being_interpreted(): + """No ATIF schema is vendored here, so mapping the vocabulary onto + TraceUsage would assert a correspondence nobody checked.""" + document = _minimal( + steps=[ + { + "step_id": 1, + "source": "agent", + "message": "m", + "metrics": {"tokens": 12}, + } + ] + ) + trace = atif_to_ir(document) + assert trace.events[0].usage is None + assert trace.events[0].extensions["metrics"] == {"tokens": 12} + assert "events[0].usage" in _fields(trace, LossClass.NORMALIZED) + + +# --------------------------------------------------------------------------- +# Vocabulary and coercion — this module's own reshaping, always declared +# --------------------------------------------------------------------------- + + +def test_a_status_outside_the_vocabulary_becomes_unknown_and_is_kept(): + document = _minimal( + steps=[ + { + "step_id": 1, + "source": "agent", + "message": "", + "tool_calls": [ + { + "tool_call_id": "c", + "function_name": "f", + "arguments": {}, + "extra": {"status": "reticulating"}, + } + ], + } + ] + ) + trace = atif_to_ir(document) + event = trace.events[0] + assert event.tool_call.status is ToolStatus.UNKNOWN + # The original is kept: normalizing to ``unknown`` must not destroy what the + # document said, or a future vocabulary could not be reconstructed. + assert event.extensions["tool_call"]["source_status"] == "reticulating" + assert "events[0].tool_call.status" in _fields(trace, LossClass.NORMALIZED) + + +def test_unmapped_tool_call_keys_are_kept_under_their_own_extension_key(): + """``ToolCall`` has no extensions of its own, and merging its leftovers into + the event's would let a step key and a tool-call key collide.""" + document = _minimal( + steps=[ + { + "step_id": 1, + "source": "agent", + "message": "", + "latency_ms": "step", + "tool_calls": [ + { + "tool_call_id": "c", + "function_name": "f", + "arguments": {}, + "latency_ms": "call", + "extra": {"title": "t", "vendor_field": 1}, + } + ], + } + ] + ) + extensions = atif_to_ir(document).events[0].extensions + assert extensions["latency_ms"] == "step" + assert extensions["tool_call"]["latency_ms"] == "call" + assert extensions["tool_call"]["extra"] == {"vendor_field": 1} + + +def test_a_non_string_message_is_coerced_and_declared(): + document = _minimal(steps=[{"step_id": 1, "source": "user", "message": 42}]) + trace = atif_to_ir(document) + assert trace.events[0].text == "42" + assert "events[0].text" in _fields(trace, LossClass.NORMALIZED) + + +def test_missing_arguments_are_declared_so_the_invariant_holds(): + """A non-conformant document with no ``arguments`` produces ``None``, and + invariant 7 requires that absence to be declared.""" + document = _minimal( + steps=[ + { + "step_id": 1, + "source": "agent", + "message": "", + "tool_calls": [{"tool_call_id": "c", "function_name": "f"}], + } + ] + ) + trace = atif_to_ir(document) + assert trace.events[0].tool_call.arguments is None + assert "events[0].tool_call.arguments" in _fields(trace, LossClass.UNSUPPORTED) + assert validate_trace(trace) == [] + + +def test_non_object_arguments_are_kept_and_declared(): + document = _minimal( + steps=[ + { + "step_id": 1, + "source": "agent", + "message": "", + "tool_calls": [ + {"tool_call_id": "c", "function_name": "f", "arguments": "ls -la"} + ], + } + ] + ) + trace = atif_to_ir(document) + event = trace.events[0] + assert event.tool_call.arguments is None + # Not modelled, but not thrown away either. + assert event.extensions["tool_call"]["source_arguments"] == "ls -la" + assert "events[0].tool_call.arguments" in _fields(trace, LossClass.UNSUPPORTED) + assert validate_trace(trace) == [] + + +# --------------------------------------------------------------------------- +# Malformed input — a document read off disk is not a document we wrote +# --------------------------------------------------------------------------- + + +def test_a_non_mapping_document_is_rejected(): + with pytest.raises(TypeError): + atif_to_ir(["not", "a", "document"]) + + +def test_a_non_object_step_is_declared_in_the_source_space(): + document = _minimal( + steps=[{"step_id": 1, "source": "user", "message": "a"}, "junk"] + ) + trace = atif_to_ir(document) + assert len(trace.events) == 1 + dropped = [ + record + for record in trace.losses.records + if record.space is PathSpace.SOURCE and record.field == "steps[1]" + ] + assert len(dropped) == 1 + assert validate_trace(trace) == [] + + +def test_steps_of_the_wrong_type_leave_an_empty_but_valid_trace(): + trace = atif_to_ir(_minimal(steps={"not": "a list"})) + assert trace.events == [] + assert validate_trace(trace) == [] + assert "steps" in { + record.field + for record in trace.losses.records + if record.space is PathSpace.SOURCE + } + + +def test_a_document_with_no_steps_is_a_valid_empty_trace(): + trace = atif_to_ir({"schema_version": "ATIF-v1.7"}) + assert trace.events == [] + assert validate_trace(trace) == [] + + +# --------------------------------------------------------------------------- +# final_metrics +# --------------------------------------------------------------------------- + + +def test_final_metrics_map_onto_trace_usage(): + document = _minimal( + final_metrics={ + "total_steps": 1, + "total_prompt_tokens": 10, + "total_completion_tokens": 2, + "total_cached_tokens": 3, + "total_cost_usd": 0.5, + } + ) + usage = atif_to_ir(document).usage + assert usage.input_tokens == 10 + assert usage.output_tokens == 2 + assert usage.cache_read_tokens == 3 + assert usage.cost_usd == 0.5 + + +def test_total_steps_is_declared_dropped_in_the_source_space(): + """The mirror of `ir_to_atif`'s target-space SYNTHESIZED record: a count of + the document's own steps is not a property of the run.""" + trace = atif_to_ir(_minimal()) + assert "final_metrics.total_steps" in { + record.field + for record in trace.losses.records + if record.space is PathSpace.SOURCE and record.loss_class is LossClass.DROPPED + } + + +def test_usage_fields_atif_cannot_carry_are_declared_unsupported(): + document = _minimal(final_metrics={"total_steps": 1, "total_prompt_tokens": 1}) + trace = atif_to_ir(document) + unsupported = _fields(trace, LossClass.UNSUPPORTED) + for field in ( + "usage.cache_creation_tokens", + "usage.reasoning_tokens", + "usage.total_tokens", + "usage.source", + "usage.price_source", + ): + assert field in unsupported + + +def test_a_document_with_no_readable_metrics_declares_the_outermost_node(): + """The addressing rule: name the section, not a field inside it.""" + trace = atif_to_ir(_minimal(final_metrics={"total_steps": 1})) + assert trace.usage is None + assert "usage" in _fields(trace, LossClass.UNSUPPORTED) + + +def test_unknown_final_metrics_keys_are_declared(): + trace = atif_to_ir(_minimal(final_metrics={"total_steps": 1, "total_reward": 1.0})) + records = [ + record + for record in trace.losses.records + if record.field == "final_metrics" and record.space is PathSpace.SOURCE + ] + assert len(records) == 1 + assert "total_reward" in records[0].detail + + +def test_a_non_numeric_token_count_is_declared_rather_than_coerced(): + trace = atif_to_ir( + _minimal(final_metrics={"total_steps": 1, "total_prompt_tokens": "many"}) + ) + assert trace.usage is None + assert "usage.input_tokens" in { + record.field + for record in trace.losses.records + if record.space is PathSpace.SOURCE + } + + +# --------------------------------------------------------------------------- +# Unmapped document content is carried, not dropped +# --------------------------------------------------------------------------- + + +def test_schema_version_and_unknown_document_keys_ride_in_extensions(): + trace = atif_to_ir(_minimal(dialect="harbor-1.4")) + assert trace.extensions["schema_version"] == "ATIF-v1.7" + assert trace.extensions["dialect"] == "harbor-1.4" + + +def test_unknown_step_keys_ride_in_event_extensions(): + document = _minimal( + steps=[{"step_id": 1, "source": "user", "message": "m", "latency_ms": 12}] + ) + assert atif_to_ir(document).events[0].extensions["latency_ms"] == 12 + + +# --------------------------------------------------------------------------- +# What ATIF never carries — UNSUPPORTED, never DROPPED +# --------------------------------------------------------------------------- + + +ALWAYS_UNSUPPORTED = ( + "trace_id", + "started_at", + "finished_at", + "agent.provider", + "outcome", + "events[].started_at", + "events[].finished_at", + "events[].usage", + "events[].outcome", +) + + +@pytest.mark.parametrize("field", ALWAYS_UNSUPPORTED) +def test_what_atif_never_carries_is_declared_unsupported(field): + """``UNSUPPORTED`` and ``DROPPED`` decide where a fix would have to land. + + Nothing in an ATIF document holds these, so calling them dropped would + blame this converter for an absence it inherited — and would make the two + inbound reports incomparable, since `ACP → IR` draws the same line. + """ + trace = atif_to_ir(_rich_document()) + assert field in _fields(trace, LossClass.UNSUPPORTED) + + +def test_the_inbound_report_declares_nothing_dropped_about_the_hub(): + """Every DROPPED record addresses the source document, not the IR. + + This edge cannot drop an IR field: it is the thing building the IR. What it + drops are parts of the document with no IR home, and those are source-space + records by construction. + """ + trace = atif_to_ir(_rich_document()) + dropped = [ + record + for record in trace.losses.records + if record.loss_class is LossClass.DROPPED + ] + assert dropped + assert all(record.space is PathSpace.SOURCE for record in dropped) + + +def test_tool_call_timestamps_are_only_declared_when_a_call_exists(): + """Declaring a per-call absence for a trace with no calls would describe a + conversion that never happened.""" + with_calls = atif_to_ir(_rich_document()) + without = atif_to_ir(_minimal()) + assert "events[].tool_call.started_at" in _fields(with_calls, LossClass.UNSUPPORTED) + assert "events[].tool_call.started_at" not in _fields( + without, LossClass.UNSUPPORTED + ) + + +def test_every_hub_loss_path_resolves_in_the_trace_it_describes(): + """A record must address something a reader of the document can find. + + The same guard Slice B applies to the canonical encoding, applied to the + report this edge attaches: an unresolvable path is a declaration nobody can + check. Unindexed ``events[].…`` paths are systemic and exempt, since they + name a field of every event rather than one node. + """ + trace = atif_to_ir(_rich_document()) + document = trace.model_dump(mode="json") + for record in trace.losses.records: + if record.space is not PathSpace.HUB or record.field.startswith("events[]"): + continue + resolved, _ = resolve_ir_path(document, record.field) + assert resolved, record.field + + +# --------------------------------------------------------------------------- +# Field coverage — read off the models, so a new IR field cannot slip through +# --------------------------------------------------------------------------- + + +FIELD_DISPOSITION: dict[str, dict[str, str]] = { + "CanonicalTrace": { + "ir_version": "representation", + "trace_id": "declared", + "session_id": "read", + "agent": "container", + "started_at": "declared", + "finished_at": "declared", + "events": "container", + "usage": "read", + "outcome": "container", + "provenance": "representation", + "extensions": "read", + "losses": "representation", + }, + "TraceEvent": { + "index": "read", + "kind": "read", + "source_type": "read", + "role": "read", + "text": "read", + "reasoning": "read", + "reasoning_segments": "read", + "tool_call": "container", + "started_at": "declared", + "finished_at": "declared", + "outcome": "declared", + "usage": "declared", + "provenance": "representation", + "extensions": "read", + }, + "ToolCall": { + "call_id": "read", + "name": "read", + "name_semantics": "read", + "title": "read", + "status": "read", + "arguments": "read", + "content": "container", + "started_at": "declared", + "finished_at": "declared", + }, + "ModelInfo": { + "agent_name": "read", + "agent_version": "read", + "model": "read", + "provider": "declared", + }, + "TraceUsage": { + "input_tokens": "read", + "output_tokens": "read", + "cache_read_tokens": "read", + "cost_usd": "read", + "cache_creation_tokens": "declared", + "reasoning_tokens": "declared", + "total_tokens": "declared", + "source": "declared", + "price_source": "declared", + }, + "TraceOutcome": { + "status": "declared-as-section", + "stop_reason": "declared-as-section", + "reward": "declared-as-section", + "error_category": "declared-as-section", + }, + "ContentBlock": { + "kind": "read", + "text": "read", + "raw": "declared", + }, +} + + +def test_every_ir_field_has_a_disposition_at_this_edge(): + """``read`` is populated from the document, ``declared`` is absent from ATIF + and recorded, ``container`` holds fields covered by their own entry, + ``representation`` describes the IR rather than the run, and + ``declared-as-section`` is covered by the single ``outcome`` record — the + addressing rule says to name the outermost absent node. + """ + models = { + "CanonicalTrace": CanonicalTrace, + "TraceEvent": TraceEvent, + "ToolCall": ToolCall, + "ModelInfo": ModelInfo, + "TraceUsage": TraceUsage, + "TraceOutcome": TraceOutcome, + "ContentBlock": ContentBlock, + } + for name, model in models.items(): + assert set(model.model_fields) == set(FIELD_DISPOSITION[name]), { + "model": name, + "undecided": sorted(set(model.model_fields) - set(FIELD_DISPOSITION[name])), + "stale": sorted(set(FIELD_DISPOSITION[name]) - set(model.model_fields)), + } + + +def test_every_declared_field_really_produces_a_record(): + """The table is a claim about behaviour, not a comment. + + Every ``declared`` field must appear in the report of a document rich + enough to reach it — with the per-call ones only when the trace has a call, + which is the conditional the previous test pins. + """ + trace = atif_to_ir(_rich_document(total_prompt_tokens=5)) + declared = { + record.field for record in trace.losses.records if record.space is PathSpace.HUB + } + expected = { + "trace_id", + "started_at", + "finished_at", + "agent.provider", + "outcome", + "events[].started_at", + "events[].finished_at", + "events[].outcome", + "events[].usage", + "events[].tool_call.started_at", + "events[].tool_call.finished_at", + "events[].tool_call.content[].raw", + "usage.cache_creation_tokens", + "usage.reasoning_tokens", + "usage.total_tokens", + "usage.source", + "usage.price_source", + } + assert expected <= declared, sorted(expected - declared) + + +# --------------------------------------------------------------------------- +# Isolation +# --------------------------------------------------------------------------- + + +def test_this_module_imports_only_the_ir_from_benchflow(): + """The converter depends on the hub and on nothing else in the package. + + An edge that imported the ACP layer or an exporter would make the hub's + neutrality a matter of convention; here it is a property of the import + graph. + """ + tree = ast.parse(Path(ir_from_atif_module.__file__).read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add(node.module or "") + benchflow_imports = {name for name in imported if name.startswith("benchflow")} + assert benchflow_imports == {"benchflow.trajectories.ir"} diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py index 871440317..e0d06ebe5 100644 --- a/tests/trajectories/test_trace_ir.py +++ b/tests/trajectories/test_trace_ir.py @@ -544,7 +544,7 @@ def test_the_oracle_record_has_its_own_kind_and_role(): # --------------------------------------------------------------------------- -IR_FAMILY = ("ir", "ir_from_acp", "ir_to_atif") +IR_FAMILY = ("ir", "ir_from_acp", "ir_to_atif", "ir_from_atif") """The unwired modules of the proposal: the IR and its converters. They may import each other — a converter that could not import the IR would be From 92cbeaeb5a3b26c5c3341a7dd53ca4823e4c1093 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Sun, 16 Aug 2026 09:39:07 +0200 Subject: [PATCH 10/23] feat(trajectories): measure what survives ACP -> IR -> ATIF -> IR, unwired MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice D proved the hub reproduces the direct exporter's document. With both ATIF edges in place the loop closes, and this answers the harder question: how much of a trace is still there after a trip through the interchange format. It is a measurement, not an assertion. `compare_traces` reads the two traces and never the loss reports, so a converter that lost something without declaring it is caught rather than confirmed. No percentage, because one number would merge two unrelated things. The report crosses an observed axis — preserved / transformed / lost / fabricated, from the comparison alone — with a declared one: whether ATIF has a slot at all. A loss with a slot is a gap in our own edge and fixable; a loss without one is a cost of the format. The declared half is a table with one entry per IR field, and a test derives the field list from the models so a new field cannot reach the round trip undecided. Comparison is by canonical path rather than by event position: after a conversion that fuses and drops events there is no recoverable correspondence between event i and event j, and inventing an alignment would be guessing. Measured on the two real rollouts of docs §5.2, machine measurement only: - Nothing representable is lost, on either. Every value the loop drops is dropped because ATIF has nowhere to put it, so there is no gap in our edges to close and the remaining loss belongs to the format. A test pins this, and a failure there names a converter bug. - The trace comes back with MORE values than it left with — 46 in, 56 out for H1 — while having lost information. Four fields are fabricated every run, and two of them matter: `agent.agent_version` and `tool_call.arguments` leave declared SYNTHESIZED and return unmarked, so the reconstructed trace asserts they were observed. The information was not lost so much as overwritten with a plausible value of the same shape, and only the pair of reports still carries the truth. - A timeout costs six fields. H2 differs from H1 only by ending in one, and that single fact takes the event, its reason, the run status and three extension fields with it — §5 loss #4, measured rather than asserted. Docs: §8.3 gains the ATIF -> IR mapping table, §8.10 is the measurement, §8.7 is updated. The doc for the converter that landed in the previous commit is here rather than there, so the two tables could be written against each other. 31 tests. Still nothing imported by a run path; `export_atif.py` untouched and still the only writer of trainer/atif.json. --- docs/trace-interop.md | 197 ++++++- src/benchflow/trajectories/ir_round_trip.py | 503 ++++++++++++++++++ tests/trajectories/test_ir_round_trip.py | 550 ++++++++++++++++++++ tests/trajectories/test_trace_ir.py | 8 +- 4 files changed, 1241 insertions(+), 17 deletions(-) create mode 100644 src/benchflow/trajectories/ir_round_trip.py create mode 100644 tests/trajectories/test_ir_round_trip.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md index 3bc5c9adf..a54a2c6b0 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -590,12 +590,15 @@ Three design choices carry most of the weight: ### 8.3 Planned mapping -**`ACP → IR`** (Slice C, -[`ir_from_acp.py`](../src/benchflow/trajectories/ir_from_acp.py)) **and -`IR → ATIF`** (Slice D, -[`ir_to_atif.py`](../src/benchflow/trajectories/ir_to_atif.py)) **are -implemented**, both unwired and still provisional. `ATIF → IR` and the OTel -direction are not. +**`ACP → IR`** ([`ir_from_acp.py`](../src/benchflow/trajectories/ir_from_acp.py)), +**`IR → ATIF`** ([`ir_to_atif.py`](../src/benchflow/trajectories/ir_to_atif.py)) +and **`ATIF → IR`** +([`ir_from_atif.py`](../src/benchflow/trajectories/ir_from_atif.py)) **are +implemented**, all unwired and still provisional. The OTel direction is not, and +stays blocked on open question 2. + +With both ATIF edges in place the loop `ACP → IR → ATIF → IR′` closes, and §8.10 +reports what it measures. #### ACP-session capture events → IR @@ -778,6 +781,67 @@ Which direction is wanted — receiver, emitter, or both — is **open question and stays open. The IR is symmetric on purpose so the answer does not invalidate it. +#### ATIF → IR + +**Implemented** ([`ir_from_atif.py`](../src/benchflow/trajectories/ir_from_atif.py)), +unwired. Asserted by `tests/trajectories/test_ir_from_atif.py` against documents +from *both* writers — the direct exporter and the hub — plus malformed input. + +This edge reads a document that is itself the output of a lossy conversion, and +it is built on one rule: **read what the document says, never what it probably +meant.** Several values in an ATIF document were fabricated by the converter that +wrote it, and nothing in the document marks them as such. Reading them back as +absences would be guessing which ones were invented — a guess that happens to be +right is still a guess, and it would make the round trip below report a +preservation that did not happen. + +| ATIF | IR | Class | Loss recorded | +|---|---|---|---| +| `session_id` | `session_id` | preserved | — | +| `schema_version` | `extensions.schema_version` | preserved | — no IR field names a source format's version | +| unknown document key | `extensions` | preserved | — | +| `agent.name` / `.version` / `.model_name` | `agent.agent_name` / `.agent_version` / `.model` | preserved | — **including the literal `"unknown"`**, see below | +| step order | `events[].index`, dense from 0 | preserved | — | +| `step.source` | `role` **and** `source_type` verbatim | preserved | — | +| `step.source` outside the vocabulary | `kind=unknown`, `source_type` verbatim | preserved | — every exporter drops these today | +| `step.message` | `text`, **including `""`** | preserved | — | +| `step.reasoning_content` | `reasoning` + `reasoning_segments=[joined]` | normalized | — one step is one segment; the boundaries are already gone | +| `step_id` | `events[].extensions.step_id` | preserved | — not mapped onto `index`, which invariant 2 makes a dense position | +| `tool_calls[].tool_call_id` | `tool_call.call_id` | preserved | — | +| `tool_calls[].function_name` | `tool_call.name` + `name_semantics="function_name"` | preserved | — the ACP-kind semantics is not in the document to recover | +| `tool_calls[].arguments` | `arguments`, **verbatim including `{}`** | preserved | — see below | +| `tool_calls[].extra.title` / `.status` | `title` / `status` | preserved | — | +| `.extra.status` outside the vocabulary | `unknown`, original in `extensions` | normalized | `events[i].tool_call.status` | +| `observation.results[].content` | `ContentBlock(kind=text)`, `raw=None` | preserved | — the source block is not in the document | +| a result matching no call in the step | `extensions.unmatched_observation_results` | normalized | `steps[i].observation.results`, **source space** | +| `step.metrics` | `extensions.metrics`, uninterpreted | normalized | `events[i].usage` — no ATIF schema is vendored here to map it | +| a step with *n* tool calls | *n* IR events | normalized | `steps[i]`, **source space** | +| `final_metrics.total_prompt/completion/cached_tokens`, `total_cost_usd` | `usage.input/output/cache_read_tokens`, `.cost_usd` | preserved | — | +| `final_metrics.total_steps` | *(no IR field)* | dropped | `final_metrics.total_steps`, **source space** | +| other `final_metrics` keys | *(no IR field)* | dropped | `final_metrics`, **source space** | +| a non-object step | *(no IR event)* | dropped | `steps[i]`, **source space** | +| non-string where ATIF specifies a string | coerced with `str()` | normalized | the field, hub space | +| *(none)* | `trace_id`, `started_at`, `finished_at`, `agent.provider`, `outcome`, `events[].started_at`/`.finished_at`/`.outcome`/`.usage`, `events[].tool_call.started_at`/`.finished_at`/`.content[].raw`, the five unmapped `usage` fields | **unsupported** | one record each | + +**Everything ATIF does not carry is `UNSUPPORTED`, never `DROPPED`.** There is +nothing in the document to drop, and the distinction is what says where a fix +would have to land — the same line `ACP → IR` draws, which is what makes the two +inbound reports comparable. Every `DROPPED` record in this direction addresses +the source document, in the source path space. + +**Three values are deliberately read as observed, and this is the whole point of +the edge.** `agent.version: "unknown"` is what `ir_to_atif` writes when the trace +has no version *and* a legal observed value; `arguments: {}` is what it writes +for every ACP-derived call; `message: ""` is what it writes on a step carrying +only a tool call. All three come back as observations. §8.10 measures what that +costs. + +**One shape is deliberately not undone.** A step with both `message` and +`reasoning_content` becomes **one** event carrying both, not a reasoning event +followed by a message event: the blank-line join that produced it is not +injective (§5 loss #10), so splitting it would invent a boundary. The fusion is +reported by the event count instead. + ### 8.4 A worked example **FACT.** The document below is produced by the models in `ir.py` and compared @@ -1021,21 +1085,23 @@ here. ### 8.7 Status - **Implemented:** the IR types, the loss model with its path spaces, the - invariants, the validation suite, this section, the `ACP → IR` converter and - the `IR → ATIF` converter, each with its loss report. -- **Not implemented, deliberately:** `ATIF → IR`, any OTel work, any wiring into - a run path, any on-disk artifact, any capture-layer enrichment. + invariants, the validation suite, this section, all three converters — + `ACP → IR`, `IR → ATIF`, `ATIF → IR` — each with its loss report, and the + round-trip measurement over them (§8.10). +- **Not implemented, deliberately:** any OTel work, any wiring into a run path, + any on-disk artifact, any capture-layer enrichment. - **Unchanged:** every existing format, exporter, artifact and code path. `export_atif.py` in particular is untouched and remains the only writer of `trainer/atif.json`. The isolation property is unchanged in substance and restated at a new boundary: -`ir.py`, `ir_from_acp.py` and `ir_to_atif.py` form a closed family that may -import each other, and `test_only_the_ir_family_imports_the_ir` asserts that -nothing else in `src/benchflow` imports any of them. Each converter imports one -benchflow module — the IR — and reads or writes its own format as data; in -particular `ir_to_atif` does not import `export_atif`, and pins the shared schema -version by test instead. +`ir.py`, `ir_from_acp.py`, `ir_to_atif.py`, `ir_from_atif.py` and +`ir_round_trip.py` form a closed family that may import each other, and +`test_only_the_ir_family_imports_the_ir` asserts that nothing else in +`src/benchflow` imports any of them. Each converter imports one benchflow module +— the IR — and reads or writes its own format as data; in particular +`ir_to_atif` does not import `export_atif`, and pins the shared schema version by +test instead. ### 8.8 What the first converter showed @@ -1090,3 +1156,102 @@ declare an absence, and it settled three things the inbound edge could not: `PathSpace` into existence (§8.2, choice 5). A prompt-derived step has no IR antecedent at all, so it cannot be addressed in the hub vocabulary — and before this edge, `LossRecord` had no way to say so. + +### 8.10 What the round trip measures + +With both ATIF edges implemented the loop `ACP → IR → ATIF → IR′` closes, and +the question it answers is a harder one than parity: **how much of a trace is +still there after a trip through the interchange format?** +[`ir_round_trip.py`](../src/benchflow/trajectories/ir_round_trip.py) answers it +as a measurement. It compares the two *traces* and never reads the loss reports, +so a converter that lost something without declaring it is caught rather than +confirmed. + +#### Why not a percentage + +A single number would merge two things that have nothing to do with each other, +so the report crosses an **observed** axis with a **declared** one: + +| observed (`RoundTripOutcome`) | meaning | +|---|---| +| `preserved` | same values, same count | +| `transformed` | values on both sides, not the same ones — `matched=0` means *replaced* | +| `lost` | values in, none out | +| `fabricated` | none in, values out | + +| declared (`Representability`) | meaning | +|---|---| +| `representable` | ATIF has a slot — a loss here is a gap in **our** edge, and fixable | +| `non_representable` | ATIF has no slot — a loss here is a cost of the **format** | + +The declared half is a table with one entry per IR field, and a test derives the +field list from the models, so a new IR field cannot reach the round trip +without a disposition. Comparison is by canonical path — `events[0].text` and +`events[7].text` are one path with two values — because after a conversion that +fuses and drops events there is no recoverable correspondence between event *i* +and event *j*, and inventing an alignment would be the same guessing §8.3's +inbound edge refuses. + +#### Measured on the two real rollouts of §5.2 + +**FACT — machine measurement, not a human E2E.** The rollouts themselves are +real and were verified by hand when they were captured; the loop below was run +over their `acp_trajectory.jsonl` by the harness. `with prompts` reproduces what +a real export does, and its document is byte-identical to the `trainer/atif.json` +each rollout actually wrote. + +| | H1 fields | H2 fields | H1 values | H2 values | +|---|---|---|---|---| +| `preserved` | 15 | 15 | 35 of 46 | 25 of 37 | +| `transformed` | 5 | 5 | | | +| `lost` (fixable) | **0** | **0** | | | +| `non_representable` | 1 | 6 | | | +| `fabricated` | 4 | 4 | | | + +H1 is a 5-event tool-use rollout; H2 is 4 events ending in a real wall-clock +timeout. Three results, in the order they matter: + +- **Nothing representable is lost.** Every value the loop drops is dropped + because ATIF has nowhere to put it. There is no gap in our own edges to close, + so the remaining loss is a property of the format and not of this + implementation. `test_nothing_representable_is_lost_on_a_captured_rollout` + pins it, and a failure there would name a converter bug. +- **The trace comes back with *more* values than it left with** — 46 in, 56 out + for H1 — while having lost information. Four fields are fabricated on every + run: `agent.agent_version` (`"unknown"`), `events[].tool_call.arguments` + (`{}`), `events[].extensions.step_id`, and `extensions.schema_version`. The + first two are the ones that matter: on the way out `ir_to_atif` declares both + `SYNTHESIZED`, and on the way back nothing in the document marks them as + invented, so the reconstructed trace asserts — with the full authority of the + tri-state contract of §8.2 — that the agent's version was observed to be + `"unknown"` and that every tool was observed to be called with no arguments. + **The information was not lost so much as overwritten with a plausible value + of the same shape.** A consumer reading only the second trace cannot tell. + With `prompts` the same laundering happens one level up: steps that are not + trace data at all return as captured user messages, and the event count grows + by exactly the number of prompts. +- **A timeout costs six fields.** H2 differs from H1 only in ending in a + timeout, and that single fact takes with it `events[].outcome`, + `outcome.status`, the three `agent_timeout` extension fields and the event + itself — §5 loss #4, measured rather than asserted. The IR made all of it + representable; the trip through ATIF makes the run look like one that simply + ended. + +`transformed` with `matched=0` is worth reading as its own category: +`events[].source_type` and `events[].tool_call.name_semantics` are not degraded, +they are **replaced** — ATIF's step `source` and its `function_name` slot sit +down where the ACP type string and the `acp_kind` semantics used to be. + +#### What this does and does not argue + +It argues for the hub in the one way an assertion cannot: the pair of loss +reports carries exactly the facts the second trace has lost the ability to +state, and the measurement shows there is no third place to get them from. It +does **not** argue that the round trip is safe, or that ATIF should be used as a +storage format — the opposite, if anything. + +**Not measured, and not claimed:** oracle rollouts and non-text content blocks +appear only in constructed test input, never in a captured rollout here; no +artifact on hand carries a non-null cost, so `usage.cost_usd` round-trips in +tests only; and the loop has been run over two rollouts from one agent, which is +a demonstration, not a survey. diff --git a/src/benchflow/trajectories/ir_round_trip.py b/src/benchflow/trajectories/ir_round_trip.py new file mode 100644 index 000000000..c8a9ef5dd --- /dev/null +++ b/src/benchflow/trajectories/ir_round_trip.py @@ -0,0 +1,503 @@ +"""Round-trip measurement: how much of a trace survives `ACP → IR → ATIF → IR′`. + +> **PROVISIONAL.** Companion to :mod:`benchflow.trajectories.ir`, itself an +> unapproved proposal (`docs/trace-interop.md` §8). Nothing imports this module +> and no artifact changes because it exists. + +Slice D established that the hub *reproduces* the direct ATIF exporter — the +document that comes out is the one that came out before. That answers "is the IR +sufficient to write ATIF?" and says nothing about the more useful question: + +**how much of the trace is still there after a trip through the format?** + +This module answers it as a measurement rather than an assertion. It runs the +loop, compares the trace that went in against the trace that came back, and +reports the difference per field. + +## Why the answer is not a percentage + +A single number would hide the two things worth knowing. The comparison +therefore separates *what happened to a value* from *whether ATIF could have +carried it*: + +- :class:`RoundTripOutcome` is **observed** — it comes from comparing the two + traces and nothing else. +- :class:`Representability` is **declared**, in :data:`ATIF_REPRESENTABILITY`, + one entry per IR field. + +Crossing them is what makes the report actionable. A value that is gone *and* +has no ATIF slot is a cost of the format: no converter can fix it, and only a +format change or a side channel would. A value that is gone and ATIF *does* have +a slot for it is a gap in our own edge — a bug report, not a fact of life. Both +read as "lost" in a percentage, and they have nothing to do with each other. + +## The fourth outcome + +The loop does not only subtract. :attr:`RoundTripOutcome.FABRICATED` marks a +field that had no value going in and has one coming back, and it is the finding +this measurement exists to surface: + +`ir_to_atif` writes `arguments: {}` for a call the IR says has *no* arguments, +and declares it ``SYNTHESIZED``. `atif_to_ir` reads the document as written — +correctly, since nothing in it marks the value as invented — and produces a +trace asserting the call was observed with an empty argument map. The tri-state +contract that made `None` mean "the source never carried this" is intact in both +traces; it is the *round trip* that turns one state into the other. + +The information was not lost so much as **overwritten with a plausible value of +the same shape**. A consumer reading only the second trace cannot tell. That is +the strongest available argument for the hub — not that it converts, but that it +carries a report the format cannot. + +## What is deliberately not compared + +``ir_version``, ``losses`` and ``provenance`` (trace- and event-level) describe +the *representation*, not the run: a trace read from ATIF is correctly labelled +as coming from ATIF, and calling that a loss would count a true statement as +damage. Slice D excludes the same fields from its outbound coverage rule, for +the same reason. +""" + +from __future__ import annotations + +from enum import StrEnum +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from benchflow.trajectories.ir import CanonicalTrace, LossReport, TraceUsage +from benchflow.trajectories.ir_from_acp import acp_events_to_ir +from benchflow.trajectories.ir_from_atif import atif_to_ir +from benchflow.trajectories.ir_to_atif import ir_to_atif + + +class RoundTripOutcome(StrEnum): + """What the comparison observed for one field. Measured, never declared.""" + + PRESERVED = "preserved" + """Same values, same number of them, in the same order.""" + + TRANSFORMED = "transformed" + """Values on both sides, but not the same ones — reshaped, renumbered, + fused, or partially carried.""" + + LOST = "lost" + """Values going in, none coming back.""" + + FABRICATED = "fabricated" + """No values going in, values coming back. The round trip invented them.""" + + +class Representability(StrEnum): + """Whether ATIF has anywhere to put a field. Declared, never measured.""" + + REPRESENTABLE = "representable" + """ATIF has a slot. A loss here is a gap in a converter, and fixable.""" + + NOT_IN_ATIF = "non_representable" + """ATIF has no slot at all. A loss here is a cost of the format.""" + + +# One entry per IR field, keyed by canonical path (list indices collapsed to +# ``[]``). Lookup is longest-prefix, so ``extensions`` covers every key under it. +# +# This table is the declared half of the measurement and the place to argue with +# it. A test derives the field list from the IR models and fails when a field has +# no entry, so the IR cannot grow a field that silently escapes the round trip. +ATIF_REPRESENTABILITY: dict[str, Representability] = { + # -- trace level --------------------------------------------------------- + "trace_id": Representability.NOT_IN_ATIF, + "session_id": Representability.REPRESENTABLE, + "started_at": Representability.NOT_IN_ATIF, + "finished_at": Representability.NOT_IN_ATIF, + "extensions": Representability.NOT_IN_ATIF, + "agent.agent_name": Representability.REPRESENTABLE, + "agent.agent_version": Representability.REPRESENTABLE, + "agent.model": Representability.REPRESENTABLE, + "agent.provider": Representability.NOT_IN_ATIF, + # ATIF's final_metrics has four of the nine usage fields. + "usage.input_tokens": Representability.REPRESENTABLE, + "usage.output_tokens": Representability.REPRESENTABLE, + "usage.cache_read_tokens": Representability.REPRESENTABLE, + "usage.cost_usd": Representability.REPRESENTABLE, + "usage.cache_creation_tokens": Representability.NOT_IN_ATIF, + "usage.reasoning_tokens": Representability.NOT_IN_ATIF, + "usage.total_tokens": Representability.NOT_IN_ATIF, + "usage.source": Representability.NOT_IN_ATIF, + "usage.price_source": Representability.NOT_IN_ATIF, + "outcome.status": Representability.NOT_IN_ATIF, + "outcome.stop_reason": Representability.NOT_IN_ATIF, + "outcome.reward": Representability.NOT_IN_ATIF, + "outcome.error_category": Representability.NOT_IN_ATIF, + # -- event level --------------------------------------------------------- + # step_id is dense over emitted steps, so the number survives but the + # identity does not; that shows up as TRANSFORMED whenever an event is + # dropped ahead of it. + "events[].index": Representability.REPRESENTABLE, + # The step shape encodes user/agent/oracle and a tool call. It has no shape + # for a timeout or an unrecognized record: those kinds are unrepresentable + # values of a representable field, which the per-value counts show. + "events[].kind": Representability.REPRESENTABLE, + "events[].role": Representability.REPRESENTABLE, + "events[].text": Representability.REPRESENTABLE, + "events[].reasoning": Representability.REPRESENTABLE, + "events[].source_type": Representability.NOT_IN_ATIF, + "events[].reasoning_segments": Representability.NOT_IN_ATIF, + "events[].started_at": Representability.NOT_IN_ATIF, + "events[].finished_at": Representability.NOT_IN_ATIF, + "events[].outcome": Representability.NOT_IN_ATIF, + "events[].extensions": Representability.NOT_IN_ATIF, + # ATIF *does* have a per-step metrics slot; `ir_to_atif` writes none. So a + # loss here is ours to close, unlike the timestamps beside it. + "events[].usage": Representability.REPRESENTABLE, + "events[].tool_call.call_id": Representability.REPRESENTABLE, + "events[].tool_call.name": Representability.REPRESENTABLE, + "events[].tool_call.title": Representability.REPRESENTABLE, + "events[].tool_call.status": Representability.REPRESENTABLE, + "events[].tool_call.arguments": Representability.REPRESENTABLE, + "events[].tool_call.name_semantics": Representability.NOT_IN_ATIF, + "events[].tool_call.started_at": Representability.NOT_IN_ATIF, + "events[].tool_call.finished_at": Representability.NOT_IN_ATIF, + "events[].tool_call.content[].kind": Representability.REPRESENTABLE, + "events[].tool_call.content[].text": Representability.REPRESENTABLE, + "events[].tool_call.content[].raw": Representability.NOT_IN_ATIF, +} + +# Fields describing the representation rather than the run. See the module +# docstring: comparing them would count a true statement as damage. +EXCLUDED_PATHS: frozenset[str] = frozenset( + {"ir_version", "losses", "provenance", "events[].provenance"} +) + +# Carried verbatim from a source that has no schema here, so their internal +# shape is not a set of IR fields. Compared whole: a source block either comes +# back or it does not, and reporting three findings because it happened to have +# three keys would weight one loss by the shape of the data. +OPAQUE_PATHS: frozenset[str] = frozenset({"events[].tool_call.content[].raw"}) + +# Fields whose default *is* the empty container. An empty value at one of these +# is the absence of values, not an observed empty one, so it is not counted — +# unlike ``arguments``, which defaults to ``None`` and where ``{}`` is the +# observation the fabrication finding turns on. +DEFAULT_EMPTY_PATHS: frozenset[str] = frozenset( + {"events", "extensions", "events[].extensions", "events[].tool_call.content"} +) + + +class FieldComparison(BaseModel): + """One canonical field path, before and after the round trip.""" + + model_config = ConfigDict(extra="forbid") + + path: str + outcome: RoundTripOutcome + representability: Representability + + before_count: int + """How many non-null values the input trace held at this path.""" + + after_count: int + matched_count: int + """How many of the input's values came back unchanged, counted as a + multiset — order-insensitive, so a renumbering that keeps every value is + visible as a preserved set rather than as total loss. + + Read it together with :attr:`outcome`. ``TRANSFORMED`` with ``matched=0`` + means *replaced*, not partially carried: ``events[].source_type`` has a + value on both sides and not one in common, because ATIF's step `source` sat + down where the ACP type string used to be. + + For a positional field the multiset can match while the events do not + correspond at all — ``events[].index`` is dense on both sides, so it matches + whenever the two traces are the same length, and says nothing about whether + event *i* is the same event. Event correspondence is what + :attr:`RoundTripReport.kinds_before` / ``kinds_after`` are for.""" + + sample_before: Any = None + """One representative dropped or changed value, for reading the report.""" + + sample_after: Any = None + + @property + def is_loss(self) -> bool: + return self.outcome in (RoundTripOutcome.LOST, RoundTripOutcome.TRANSFORMED) + + +class RoundTripReport(BaseModel): + """The measured difference between a trace and its round trip.""" + + model_config = ConfigDict(extra="forbid") + + comparisons: list[FieldComparison] = Field(default_factory=list) + events_before: int = 0 + events_after: int = 0 + kinds_before: dict[str, int] = Field(default_factory=dict) + kinds_after: dict[str, int] = Field(default_factory=dict) + + def by_outcome(self, outcome: RoundTripOutcome) -> list[FieldComparison]: + return [c for c in self.comparisons if c.outcome is outcome] + + def summary(self) -> dict[str, int]: + """Field counts per outcome, with lost split by representability. + + The split is the point of the report: ``lost`` is what a converter could + still recover, ``non_representable`` is what the format cannot hold. + """ + counts = { + RoundTripOutcome.PRESERVED.value: 0, + RoundTripOutcome.TRANSFORMED.value: 0, + "lost": 0, + "non_representable": 0, + RoundTripOutcome.FABRICATED.value: 0, + } + for comparison in self.comparisons: + if comparison.outcome is RoundTripOutcome.LOST: + key = ( + "non_representable" + if comparison.representability is Representability.NOT_IN_ATIF + else "lost" + ) + else: + key = comparison.outcome.value + counts[key] += 1 + return counts + + def value_summary(self) -> dict[str, int]: + """The same measurement counted in values rather than in fields. + + A field-level count treats ``events[].text`` as one thing whether the + trace has two events or two hundred. This one weights by how much data + each field actually held. + """ + total = sum(c.before_count for c in self.comparisons) + matched = sum(c.matched_count for c in self.comparisons) + return { + "values_before": total, + "values_after": sum(c.after_count for c in self.comparisons), + "values_preserved": matched, + "values_not_preserved": total - matched, + } + + +def _canonical_leaves(node: Any, path: str = "") -> list[tuple[str, Any]]: + """Every leaf of a dumped trace as ``(canonical_path, value)``. + + List indices collapse to ``[]``, so ``events[0].text`` and ``events[7].text`` + are the same path with two values. That is what makes the comparison + independent of event alignment — which is not recoverable after a conversion + that fuses and drops events, and guessing at it would be the same kind of + invention this whole slice refuses. + + Empty containers are leaves: ``{}`` is a value, and the difference between + ``None`` and ``{}`` is the one the fabrication finding rests on — except at + the paths whose default is the empty container, which + :func:`_values_by_path` filters out. + + Paths in :data:`OPAQUE_PATHS` are leaves whatever they contain. + """ + if path in OPAQUE_PATHS: + return [(path, node)] + if isinstance(node, dict) and node: + leaves: list[tuple[str, Any]] = [] + for key, value in node.items(): + child = f"{path}.{key}" if path else key + leaves.extend(_canonical_leaves(value, child)) + return leaves + if isinstance(node, list) and node: + leaves = [] + for item in node: + leaves.extend(_canonical_leaves(item, f"{path}[]")) + return leaves + return [(path, node)] + + +def _is_excluded(path: str) -> bool: + return any( + path == excluded or path.startswith(f"{excluded}.") + for excluded in EXCLUDED_PATHS + ) + + +def _values_by_path(trace: CanonicalTrace) -> dict[str, list[Any]]: + """Non-null values of *trace*, grouped by canonical path.""" + dumped = trace.model_dump(mode="json") + grouped: dict[str, list[Any]] = {} + for path, value in _canonical_leaves(dumped): + if value is None or _is_excluded(path): + continue + if path in DEFAULT_EMPTY_PATHS and value in ({}, []): + continue + grouped.setdefault(path, []).append(value) + return grouped + + +def _lookup_candidates(path: str): + """*path*, then its parents, each also tried without a trailing ``[]``. + + A list of scalars dumps to ``events[].reasoning_segments[]`` while the table + names the field itself, so the bare form has to be tried too — otherwise the + entry would be missed and the field would silently default to + unrepresentable. + """ + parts = path.split(".") + for cut in range(len(parts), 0, -1): + prefix = ".".join(parts[:cut]) + yield prefix + if prefix.endswith("[]"): + yield prefix[:-2] + + +def declared_entry_for(path: str) -> str | None: + """The table entry governing *path*, or ``None`` when nothing does. + + Separate from :func:`representability_of` because "no entry" and "an entry + saying unrepresentable" are different states, and only the first is a gap in + the table. The suite asserts there are none. + """ + for candidate in _lookup_candidates(path): + if candidate in ATIF_REPRESENTABILITY: + return candidate + return None + + +def representability_of(path: str) -> Representability: + """The declared representability of *path*, by longest matching prefix. + + Unknown paths — every key a trace carries inside ``extensions`` — inherit + from their parent, which is why the table needs no entry per extension key. + Anything with no entry at all is treated as unrepresentable: a field ATIF was + never shown to carry should not be credited as carriable by default. + """ + entry = declared_entry_for(path) + return ATIF_REPRESENTABILITY[entry] if entry else Representability.NOT_IN_ATIF + + +def _multiset_overlap(before: list[Any], after: list[Any]) -> int: + """How many of *before*'s values appear in *after*, counted with duplicates. + + Values are compared by their JSON dump, since a trace dumps to plain JSON + types and dicts are not hashable. + """ + remaining = list(after) + matched = 0 + for value in before: + for position, candidate in enumerate(remaining): + if candidate == value: + remaining.pop(position) + matched += 1 + break + return matched + + +def compare_traces(before: CanonicalTrace, after: CanonicalTrace) -> RoundTripReport: + """Compare a trace with its round trip, field by canonical field. + + Neither argument is modified and neither report is read: this is a + measurement of the *documents*, deliberately independent of what the + converters declared. A converter that lost something without declaring it + shows up here all the same, which is the property that makes this worth + running against the loss reports rather than instead of them. + """ + before_values = _values_by_path(before) + after_values = _values_by_path(after) + + comparisons: list[FieldComparison] = [] + for path in sorted(set(before_values) | set(after_values)): + mine = before_values.get(path, []) + theirs = after_values.get(path, []) + matched = _multiset_overlap(mine, theirs) + + if not mine: + outcome = RoundTripOutcome.FABRICATED + elif not theirs: + outcome = RoundTripOutcome.LOST + elif matched == len(mine) == len(theirs): + outcome = RoundTripOutcome.PRESERVED + else: + outcome = RoundTripOutcome.TRANSFORMED + + comparisons.append( + FieldComparison( + path=path, + outcome=outcome, + representability=representability_of(path), + before_count=len(mine), + after_count=len(theirs), + matched_count=matched, + sample_before=mine[0] if mine else None, + sample_after=theirs[0] if theirs else None, + ) + ) + + return RoundTripReport( + comparisons=comparisons, + events_before=len(before.events), + events_after=len(after.events), + kinds_before=_kind_counts(before), + kinds_after=_kind_counts(after), + ) + + +def _kind_counts(trace: CanonicalTrace) -> dict[str, int]: + counts: dict[str, int] = {} + for event in trace.events: + counts[event.kind.value] = counts.get(event.kind.value, 0) + 1 + return counts + + +class RoundTrip(BaseModel): + """One full `ACP → IR → ATIF → IR′` loop and everything it produced.""" + + model_config = ConfigDict(extra="forbid") + + before: CanonicalTrace + document: dict[str, Any] + after: CanonicalTrace + outbound: LossReport + """`IR → ATIF`, returned by the outbound edge.""" + + report: RoundTripReport + + @property + def inbound(self) -> LossReport | None: + """`ATIF → IR`, carried on the reconstructed trace.""" + return self.after.losses + + +def round_trip_through_atif( + events: list[dict[str, Any]], + *, + prompts: list[str] | None = None, + session_id: str | None = None, + agent_name: str | None = None, + model: str | None = None, + usage: TraceUsage | None = None, +) -> RoundTrip: + """Run captured ACP events through the hub, out to ATIF, and back. + + *prompts* are passed to the outbound edge exactly as a real export does. + They are worth including in at least one measurement: they are not trace + data, they become `user` steps declared ``SYNTHESIZED`` in the target space, + and they come back indistinguishable from captured user messages — the same + laundering as `arguments`, one level up. + + *usage* is the token accounting a real export passes to + `trajectory_to_atif_record`, which reads it from `result.json`. The capture + events carry none, so without it the four representable usage fields never + enter the measurement at all — the loop would report on a trace poorer than + the one a rollout actually produces. + """ + before = acp_events_to_ir( + events, session_id=session_id, agent_name=agent_name, model=model + ) + if usage is not None: + before = before.model_copy(update={"usage": usage}) + document, outbound = ir_to_atif(before, prompts=prompts) + after = atif_to_ir(document) + return RoundTrip( + before=before, + document=document, + after=after, + outbound=outbound, + report=compare_traces(before, after), + ) diff --git a/tests/trajectories/test_ir_round_trip.py b/tests/trajectories/test_ir_round_trip.py new file mode 100644 index 000000000..81c77096f --- /dev/null +++ b/tests/trajectories/test_ir_round_trip.py @@ -0,0 +1,550 @@ +"""Round-trip measurement suite — `ACP → IR → ATIF → IR′`. + +Slice D proved the hub reproduces the direct exporter's document. This suite is +about the question that raises: **how much of the trace is still there +afterwards?** + +Two properties are worth stating up front, because they are what keeps the +measurement from being a self-fulfilling one: + +- The comparison reads **the two traces**, never the loss reports. A converter + that lost something without declaring it shows up here anyway. +- The measurement is by canonical *path*, not by event position. After a + conversion that fuses and drops events there is no recoverable correspondence + between event *i* and event *j*, and inventing an alignment would be the same + guessing this whole slice refuses to do. + +The suite also pins the two findings the loop produced on real rollouts, so that +a change in either converter has to confront them: `arguments` comes back +fabricated, and a timeout does not come back at all. +""" + +from __future__ import annotations + +from types import UnionType +from typing import Any, Union, get_args, get_origin + +import pytest +from pydantic import BaseModel + +from benchflow.trajectories.ir import ( + CanonicalTrace, + EventKind, + Provenance, + ToolCall, + TraceEvent, + TraceUsage, + validate_trace, +) +from benchflow.trajectories.ir_from_acp import acp_events_to_ir +from benchflow.trajectories.ir_round_trip import ( + ATIF_REPRESENTABILITY, + DEFAULT_EMPTY_PATHS, + EXCLUDED_PATHS, + OPAQUE_PATHS, + Representability, + RoundTripOutcome, + compare_traces, + declared_entry_for, + representability_of, + round_trip_through_atif, +) +from tests.trajectories.test_atif_preservation import _rich_events + +PROMPTS = ["Solve the task.", "Then stop."] + + +def _trip(**kwargs: Any): + return round_trip_through_atif( + _rich_events(), + session_id="sess-f", + agent_name="claude-code", + model="claude-sonnet-5", + **kwargs, + ) + + +def _outcome(report, path: str) -> RoundTripOutcome: + for comparison in report.comparisons: + if comparison.path == path: + return comparison.outcome + raise AssertionError( + f"{path} not in the report: {[c.path for c in report.comparisons]}" + ) + + +# --------------------------------------------------------------------------- +# The loop itself +# --------------------------------------------------------------------------- + + +def test_the_loop_produces_a_valid_trace_at_both_ends(): + trip = _trip(prompts=PROMPTS) + assert validate_trace(trip.before) == [] + assert validate_trace(trip.after) == [] + assert trip.outbound.direction == "ir->atif" + assert trip.inbound.direction == "atif->ir" + + +def test_comparing_a_trace_with_itself_finds_nothing(): + """The measurement's zero point. + + Without this, a comparator that reported everything as preserved — or as + lost — would still pass every other test in this file. + """ + trace = acp_events_to_ir(_rich_events(), session_id="s", agent_name="a") + report = compare_traces(trace, trace) + assert report.summary() == { + "preserved": len(report.comparisons), + "transformed": 0, + "lost": 0, + "non_representable": 0, + "fabricated": 0, + } + values = report.value_summary() + assert values["values_preserved"] == values["values_before"] + + +def test_a_removed_field_is_reported_lost_and_a_new_one_fabricated(): + """The comparator's two directions, on a difference built by hand.""" + before = CanonicalTrace( + trace_id="t-1", + provenance=Provenance(source_format="test"), + ) + after = CanonicalTrace( + session_id="s-1", + provenance=Provenance(source_format="test"), + ) + report = compare_traces(before, after) + assert _outcome(report, "trace_id") is RoundTripOutcome.LOST + assert _outcome(report, "session_id") is RoundTripOutcome.FABRICATED + + +def test_a_changed_value_is_transformed_not_lost(): + before = CanonicalTrace(session_id="a", provenance=Provenance(source_format="test")) + after = CanonicalTrace(session_id="b", provenance=Provenance(source_format="test")) + report = compare_traces(before, after) + comparison = next(c for c in report.comparisons if c.path == "session_id") + assert comparison.outcome is RoundTripOutcome.TRANSFORMED + assert comparison.matched_count == 0 + assert (comparison.before_count, comparison.after_count) == (1, 1) + + +def test_values_are_matched_as_a_multiset_not_by_position(): + """Two events swapping places is not a loss. + + Positional comparison would report every field of both as transformed, + which would drown the real findings in noise the moment an event is + dropped ahead of another. + """ + + def _trace(texts: list[str]) -> CanonicalTrace: + return CanonicalTrace( + provenance=Provenance(source_format="test"), + events=[ + TraceEvent( + index=index, + kind=EventKind.USER_MESSAGE, + text=text, + provenance=Provenance(source_format="test"), + ) + for index, text in enumerate(texts) + ], + ) + + report = compare_traces(_trace(["a", "b"]), _trace(["b", "a"])) + assert _outcome(report, "events[].text") is RoundTripOutcome.PRESERVED + + +# --------------------------------------------------------------------------- +# The two findings, on events from the production capture path +# --------------------------------------------------------------------------- + + +def test_tool_arguments_come_back_fabricated(): + """The finding this measurement exists to surface. + + `None` — "the source never carried arguments" — leaves as `{}` because ATIF + requires the field, and returns as `{}` — "observed with an empty argument + map". Two different facts, one document, and no way to tell them apart + without the reports. + """ + trip = _trip(prompts=PROMPTS) + comparison = next( + c for c in trip.report.comparisons if c.path == "events[].tool_call.arguments" + ) + assert comparison.outcome is RoundTripOutcome.FABRICATED + assert comparison.before_count == 0 + assert comparison.after_count > 0 + assert comparison.sample_after == {} + + # The outbound edge did say so — and the inbound one cannot, which is the + # asymmetry that makes the report worth carrying. + assert [ + record + for record in trip.outbound.records + if record.field.endswith("tool_call.arguments") + ] + assert not [ + record + for record in trip.inbound.records + if record.field.endswith("tool_call.arguments") + ] + + +def test_a_timeout_does_not_survive_the_loop(): + """§5 loss #4, measured end to end rather than asserted. + + The IR made the timeout representable; ATIF has no slot for it, so the + event, its reason and the run-level status all go, and the trace that comes + back describes a run that simply ended. + """ + trip = _trip(prompts=PROMPTS) + assert trip.report.kinds_before["timeout"] >= 1 + assert "timeout" not in trip.report.kinds_after + assert _outcome(trip.report, "events[].outcome") is RoundTripOutcome.LOST + assert _outcome(trip.report, "outcome.status") is RoundTripOutcome.LOST + for path in ("events[].outcome", "outcome.status"): + assert representability_of(path) is Representability.NOT_IN_ATIF + + +def test_reasoning_survives_but_its_boundaries_do_not_have_to(): + """The IR keeps thought boundaries; ATIF joins them with a blank line. + + A single thought round-trips intact, which is why this is stated as a + boundary property rather than a loss: the join only becomes irreversible + once there are two thoughts to join. + """ + events = [ + {"type": "agent_thought", "text": "first"}, + {"type": "agent_thought", "text": "second"}, + {"type": "agent_message", "text": "done"}, + ] + trip = round_trip_through_atif(events, session_id="s", agent_name="a") + segments_before = [ + event.reasoning_segments + for event in trip.before.events + if event.reasoning_segments + ] + segments_after = [ + event.reasoning_segments + for event in trip.after.events + if event.reasoning_segments + ] + assert segments_before == [["first"], ["second"]] + assert segments_after == [["first\n\nsecond"]] + + +def test_the_prompts_argument_returns_as_captured_user_messages(): + """The same laundering as `arguments`, one level up. + + *prompts* are not trace data. They become `user` steps declared SYNTHESIZED + in the target space, and they read back as ordinary user messages — so the + loop adds events that no capture ever produced. + """ + with_prompts = _trip(prompts=PROMPTS) + without = _trip() + assert with_prompts.report.events_after - without.report.events_after == len( + PROMPTS + ) + assert with_prompts.report.kinds_after["user_message"] == ( + without.report.kinds_after["user_message"] + len(PROMPTS) + ) + + +def test_nothing_representable_is_lost_on_a_captured_rollout(): + """The result that makes the table worth having. + + Everything the loop drops from these events is dropped because ATIF has no + slot for it. Nothing representable goes missing — so there is no gap in our + own edges to close, and every remaining loss is a property of the format. + + If this ever fails, the failure names a converter bug rather than a fact + about ATIF. + """ + for trip in (_trip(prompts=PROMPTS), _trip()): + fixable = [ + comparison + for comparison in trip.report.comparisons + if comparison.outcome is RoundTripOutcome.LOST + and comparison.representability is Representability.REPRESENTABLE + ] + assert fixable == [], [c.path for c in fixable] + assert trip.report.summary()["lost"] == 0 + + +def test_usage_only_enters_the_measurement_when_it_is_supplied(): + """The capture events carry no usage, so without it the four representable + usage fields are never exercised at all — the loop would be reporting on a + trace poorer than a real rollout's.""" + without = _trip(prompts=PROMPTS) + with_usage = _trip( + prompts=PROMPTS, usage=TraceUsage(input_tokens=10, output_tokens=2) + ) + assert not [c for c in without.report.comparisons if c.path.startswith("usage.")] + assert ( + _outcome(with_usage.report, "usage.input_tokens") is RoundTripOutcome.PRESERVED + ) + + +# --------------------------------------------------------------------------- +# What the comparison deliberately does not look at +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("path", sorted(EXCLUDED_PATHS)) +def test_representation_fields_are_not_compared(path): + """A trace read from ATIF is correctly labelled as coming from ATIF. + + Counting that as damage would report a true statement as a loss, and + `provenance` would be the single largest "loss" in every measurement. + """ + trip = _trip(prompts=PROMPTS) + assert not [ + comparison + for comparison in trip.report.comparisons + if comparison.path == path or comparison.path.startswith(f"{path}.") + ] + + +def test_an_opaque_block_is_compared_whole(): + """A source block has no schema here, so its keys are not IR fields. + + Walking into it would report three findings for one loss whenever the block + happened to have three keys — weighting a single fact by the shape of the + data it described. + """ + trip = _trip(prompts=PROMPTS) + paths = {comparison.path for comparison in trip.report.comparisons} + assert "events[].tool_call.content[].raw" in paths + assert not [ + path for path in paths if path.startswith("events[].tool_call.content[].raw.") + ] + + +def test_a_default_empty_container_is_not_counted_as_a_value(): + """``extensions: {}`` is the absence of extensions, not an observed empty + mapping — unlike ``arguments: {}``, where the distinction is the finding.""" + before = CanonicalTrace( + provenance=Provenance(source_format="test"), + events=[ + TraceEvent( + index=0, + kind=EventKind.USER_MESSAGE, + text="hi", + provenance=Provenance(source_format="test"), + ) + ], + ) + after = before.model_copy(deep=True) + report = compare_traces(before, after) + assert "events[].extensions" not in {c.path for c in report.comparisons} + assert "extensions" not in {c.path for c in report.comparisons} + + +def test_arguments_are_not_treated_as_a_default_empty_container(): + """The exception that makes the rule above safe.""" + assert "events[].tool_call.arguments" not in DEFAULT_EMPTY_PATHS + trace = CanonicalTrace( + provenance=Provenance(source_format="test"), + events=[ + TraceEvent( + index=0, + kind=EventKind.TOOL_CALL, + tool_call=ToolCall(call_id="c", arguments={}), + provenance=Provenance(source_format="test"), + ) + ], + ) + report = compare_traces(trace, trace) + comparison = next( + c for c in report.comparisons if c.path == "events[].tool_call.arguments" + ) + assert comparison.before_count == 1 + + +# --------------------------------------------------------------------------- +# The declared half — read off the models, so no field escapes silently +# --------------------------------------------------------------------------- + + +def _nested_model(annotation: Any) -> tuple[type[BaseModel] | None, bool]: + """The model an annotation ultimately names, and whether it is a list of it. + + ``ToolCall | None`` and ``list[TraceEvent]`` both wrap a model, and only the + second contributes ``[]`` to the path — conflating them would produce + ``events[].tool_call[].arguments``, a path no dumped trace can contain. + """ + origin = get_origin(annotation) + if origin is list: + nested, _ = _nested_model(get_args(annotation)[0]) + return nested, nested is not None + if origin in (Union, UnionType): + for argument in get_args(annotation): + if argument is type(None): + continue + nested, is_list = _nested_model(argument) + if nested is not None: + return nested, is_list + return None, False + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return annotation, False + return None, False + + +def _model_paths(model: type[BaseModel], prefix: str = "") -> set[str]: + """Every canonical field path reachable from *model*. + + Nested models are walked; a list of models contributes ``[]`` to the path, + exactly as the harness produces it from a dumped trace. + """ + paths: set[str] = set() + for name, field in model.model_fields.items(): + path = f"{prefix}.{name}" if prefix else name + nested, is_list = _nested_model(field.annotation) + if nested is not None and nested is not model: + paths |= _model_paths(nested, f"{path}[]" if is_list else path) + else: + paths.add(path) + return paths + + +def test_every_ir_field_has_a_declared_representability(): + """A new IR field cannot slip through the round trip undecided. + + The measurement is only as honest as its declared half: a field with no + entry defaults to unrepresentable, which would quietly credit the format + with a limit it may not have. + """ + undecided = sorted( + path + for path in _model_paths(CanonicalTrace) + if not any( + path == excluded or path.startswith(f"{excluded}.") + for excluded in EXCLUDED_PATHS + ) + and declared_entry_for(path) is None + ) + assert undecided == [] + + +def test_no_stale_entry_in_the_table(): + """The complement: an entry naming a field the IR no longer has would make + the table look more complete than it is. + + An entry may name a section rather than a leaf — ``events[].usage`` governs + all nine of its fields, because ATIF's decision is the same for every one of + them — so a valid entry is a leaf path or a prefix of one. + """ + known = _model_paths(CanonicalTrace) + prefixes = { + ".".join(path.split(".")[:cut]) + for path in known + for cut in range(1, len(path.split(".")) + 1) + } + stale = sorted(path for path in ATIF_REPRESENTABILITY if path not in prefixes) + assert stale == [] + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + ("events[].extensions.step_id", Representability.NOT_IN_ATIF), + ("extensions.schema_version", Representability.NOT_IN_ATIF), + ("events[].reasoning_segments[]", Representability.NOT_IN_ATIF), + ("events[].tool_call.arguments", Representability.REPRESENTABLE), + ("agent.agent_name", Representability.REPRESENTABLE), + ("nothing.like.this", Representability.NOT_IN_ATIF), + ], +) +def test_representability_lookup_walks_up_to_the_nearest_entry(path, expected): + assert representability_of(path) is expected + + +def test_a_representable_loss_would_be_reported_separately_from_a_format_one(): + """The distinction the summary rests on, forced rather than observed. + + `events[].usage` is representable — ATIF has a per-step metrics slot that + `ir_to_atif` does not write — so a trace carrying per-event usage loses + something *we* could carry. It must not be counted with the timestamps + beside it, which no ATIF document could hold. + """ + before = CanonicalTrace( + provenance=Provenance(source_format="test"), + events=[ + TraceEvent( + index=0, + kind=EventKind.AGENT_MESSAGE, + text="m", + usage=TraceUsage(input_tokens=1), + started_at=None, + provenance=Provenance(source_format="test"), + ) + ], + ) + after = CanonicalTrace( + provenance=Provenance(source_format="test"), + events=[ + TraceEvent( + index=0, + kind=EventKind.AGENT_MESSAGE, + text="m", + provenance=Provenance(source_format="test"), + ) + ], + ) + summary = compare_traces(before, after).summary() + assert summary["lost"] == 1 + assert summary["non_representable"] == 0 + + +def test_the_summary_accounts_for_every_comparison(): + trip = _trip(prompts=PROMPTS) + assert sum(trip.report.summary().values()) == len(trip.report.comparisons) + + +def test_the_value_summary_never_claims_more_survived_than_arrived(): + trip = _trip(prompts=PROMPTS) + values = trip.report.value_summary() + assert values["values_preserved"] <= values["values_before"] + assert ( + values["values_preserved"] + values["values_not_preserved"] + == values["values_before"] + ) + + +# --------------------------------------------------------------------------- +# Isolation +# --------------------------------------------------------------------------- + + +def test_the_harness_reads_traces_and_not_reports(): + """`compare_traces` must not consult the loss reports it sits beside. + + A comparison derived from the declarations would confirm the converters + against themselves: a loss nobody declared would be invisible, which is the + one failure this measurement exists to catch. + """ + trace = acp_events_to_ir(_rich_events(), session_id="s", agent_name="a") + stripped = trace.model_copy(update={"losses": None}) + assert compare_traces(trace, trace).summary() == ( + compare_traces(stripped, stripped).summary() + ) + + +def test_a_text_block_and_its_source_block_are_declared_apart(): + """The rendered text is representable and the block that produced it is not. + + They sit at neighbouring paths, so a single entry covering ``content[]`` + would credit ATIF with carrying the source block — which is §5 loss #5, + the one the IR was built to stop being silent. + """ + assert ( + ATIF_REPRESENTABILITY["events[].tool_call.content[].text"] + is Representability.REPRESENTABLE + ) + assert ( + ATIF_REPRESENTABILITY["events[].tool_call.content[].raw"] + is Representability.NOT_IN_ATIF + ) + assert "events[].tool_call.content[].raw" in OPAQUE_PATHS diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py index e0d06ebe5..651a98661 100644 --- a/tests/trajectories/test_trace_ir.py +++ b/tests/trajectories/test_trace_ir.py @@ -544,7 +544,13 @@ def test_the_oracle_record_has_its_own_kind_and_role(): # --------------------------------------------------------------------------- -IR_FAMILY = ("ir", "ir_from_acp", "ir_to_atif", "ir_from_atif") +IR_FAMILY = ( + "ir", + "ir_from_acp", + "ir_to_atif", + "ir_from_atif", + "ir_round_trip", +) """The unwired modules of the proposal: the IR and its converters. They may import each other — a converter that could not import the IR would be From a36b4bd1468ee13122272da51b7c9493903c8604 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Sun, 16 Aug 2026 10:04:26 +0200 Subject: [PATCH 11/23] fix(trajectories): stop the ATIF reader inventing values on malformed input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defects found by adversarially probing the edge with documents it does not write itself. None of them touches a conformant document — the H1/H2 numbers are unchanged — but each one broke the rule the module is built on, which is exactly the rule that makes the round-trip measurement worth anything. - A JSON `null` in a string slot became the literal `"None"`. `message: null` produced a four-character string the document never contained, and `reasoning_content: null` produced a segment list holding it. Invariant 5 was satisfied throughout, which is why the suite could not see it. Null and absent now both read as no value. - A step with no usable `source` was attributed to the agent. Missing, non-string and out-of-vocabulary sources all became `agent_message` — not a harmless default, since that is the kind a consumer counts as a model turn. They become UNKNOWN, as `ir_from_acp` already does for an unrecognized record. - A non-string `source` was discarded. It cannot be the IR's `source_type`, but it was the only thing the document said about the step's origin, so it is kept in extensions with a record. - A `tool_calls` or `observation` this converter cannot read vanished with no record in any path space. Unreadable here is not the same as not present; both are now kept verbatim and declared in the source space. Also closes a hole in the measurement itself. `lost` counts fields the table calls representable that did not survive, and the table is written by hand — so a field wrongly marked unrepresentable would move a real, fixable loss into the format's column and leave `lost` at zero for free. Two tests make the table falsifiable: on a trace populating every IR field `lost` is 2 (`ir_to_atif` writes no per-step metrics, so per-event usage is lost through a slot ATIF has), and no field the table calls unrepresentable comes back with any value intact. `lost = 0` on the captured rollouts is now a result rather than a definition. `arguments` joins `content[].raw` as an opaque path: both hold a mapping whose keys come from a tool rather than from the IR, and without it the field's path depended on its own contents. Docs: the timeout costs five fields and the event carrying them, not six — the sixth entry in H2's column is the source content block, which H1 loses too. 20 tests added, all of them failing on the code before this commit. --- docs/trace-interop.md | 28 +++- src/benchflow/trajectories/ir_from_atif.py | 92 ++++++++--- src/benchflow/trajectories/ir_round_trip.py | 10 +- tests/trajectories/test_ir_from_atif.py | 113 +++++++++++++ tests/trajectories/test_ir_round_trip.py | 171 +++++++++++++++++++- 5 files changed, 383 insertions(+), 31 deletions(-) diff --git a/docs/trace-interop.md b/docs/trace-interop.md index a54a2c6b0..d4d6ad61a 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -1211,11 +1211,20 @@ each rollout actually wrote. H1 is a 5-event tool-use rollout; H2 is 4 events ending in a real wall-clock timeout. Three results, in the order they matter: -- **Nothing representable is lost.** Every value the loop drops is dropped - because ATIF has nowhere to put it. There is no gap in our own edges to close, - so the remaining loss is a property of the format and not of this +- **Within the declared field mapping, nothing representable is lost.** Every + value the loop drops from these rollouts is dropped because ATIF has nowhere + to put it, so the remaining loss is a property of the format and not of this implementation. `test_nothing_representable_is_lost_on_a_captured_rollout` pins it, and a failure there would name a converter bug. + + **This is a result, not a definition**, and the distinction matters because + the table declaring what is representable is written by hand: a field wrongly + marked unrepresentable would move a real, fixable loss into the format's + column and shrink `lost` to zero for free. Two tests close that. On a trace + populating *every* IR field, `lost` is **2** — `ir_to_atif` writes no per-step + metrics, so per-event usage is lost through a slot ATIF actually has — and no + field the table calls unrepresentable comes back with any value intact. So + `lost = 0` on these rollouts says something about them, not about the table. - **The trace comes back with *more* values than it left with** — 46 in, 56 out for H1 — while having lost information. Four fields are fabricated on every run: `agent.agent_version` (`"unknown"`), `events[].tool_call.arguments` @@ -1230,12 +1239,15 @@ timeout. Three results, in the order they matter: With `prompts` the same laundering happens one level up: steps that are not trace data at all return as captured user messages, and the event count grows by exactly the number of prompts. -- **A timeout costs six fields.** H2 differs from H1 only in ending in a - timeout, and that single fact takes with it `events[].outcome`, - `outcome.status`, the three `agent_timeout` extension fields and the event - itself — §5 loss #4, measured rather than asserted. The IR made all of it +- **A timeout costs five fields and the event carrying them.** H2 differs from + H1 only in ending in a timeout, and that single fact takes with it + `events[].outcome` (`"wall_clock_timeout"`), `outcome.status` (`"timeout"`), + and the three extension fields the marker carried — `timeout_sec`, + `pending_tool_call_ids`, `terminal_trajectory_complete` — along with the event + itself. §5 loss #4, measured rather than asserted. The IR made all of it representable; the trip through ATIF makes the run look like one that simply - ended. + ended. (H1's single unrepresentable field, the source content block, is the + sixth entry in H2's column and is not a cost of the timeout.) `transformed` with `matched=0` is worth reading as its own category: `events[].source_type` and `events[].tool_call.name_semantics` are not degraded, diff --git a/src/benchflow/trajectories/ir_from_atif.py b/src/benchflow/trajectories/ir_from_atif.py index ac85ca226..3980bef1f 100644 --- a/src/benchflow/trajectories/ir_from_atif.py +++ b/src/benchflow/trajectories/ir_from_atif.py @@ -40,6 +40,15 @@ That is a fact about ATIF, not a defect of this converter, and it is the reason the round trip is worth measuring rather than asserting. +The rule cuts the other way too, at the places a document can be malformed. +A JSON ``null`` in a string slot reads as *no value*, not as the four-character +string ``"None"``. A step whose ``source`` is missing, or is not a string, or is +outside the vocabulary becomes an ``UNKNOWN`` event rather than being attributed +to the agent — the document does not say whose step it is, so neither does the +trace. And a `tool_calls` or `observation` this converter cannot read is kept +verbatim in ``extensions`` with a record, because "unreadable here" is not the +same as "not present". + ## What has no ATIF antecedent ATIF carries no trace id, no timestamps at any level, no run outcome, no @@ -170,6 +179,15 @@ def _coerce_str( if key not in raw: return None value = raw[key] + if value is None: + # JSON null in a string slot. Coercing it would put the literal + # ``"None"`` in the trace — a four-character string the document never + # contained, and the exact kind of invention this edge exists to avoid. + # Absent and null both mean "no value to read", which is what the IR's + # ``None`` says; the difference between them is not representable here + # and neither is worth a record, since ATIF requires the field either + # way and a document without it is non-conformant, not lossy. + return None if isinstance(value, str): return value losses.add( @@ -395,7 +413,7 @@ def _step_to_events( where = f"events[{first_index}]" source = raw_step.get("source") source_str = source if isinstance(source, str) else None - role = _ROLE_BY_SOURCE.get(source_str or "") + role = _ROLE_BY_SOURCE.get(source_str) if source_str is not None else None message = _coerce_str(raw_step, "message", f"{where}.text", losses) reasoning = _coerce_str(raw_step, "reasoning_content", f"{where}.reasoning", losses) @@ -416,30 +434,64 @@ def _step_to_events( ) raw_calls = raw_step.get("tool_calls") - calls = ( - [call for call in raw_calls if isinstance(call, dict)] - if isinstance(raw_calls, list) - else [] - ) - if isinstance(raw_calls, list) and len(calls) != len(raw_calls): + calls: list[dict[str, Any]] = [] + if isinstance(raw_calls, list): + calls = [call for call in raw_calls if isinstance(call, dict)] + if len(calls) != len(raw_calls): + losses.add( + f"steps[{first_index}].tool_calls", + LossClass.DROPPED, + "the step carries tool_calls entries that are not JSON objects", + space=PathSpace.SOURCE, + ) + elif raw_calls is not None: + # Not a list at all. Kept verbatim rather than discarded: this converter + # cannot read it, which is not the same as it not being there. + extensions["tool_calls"] = raw_calls losses.add( f"steps[{first_index}].tool_calls", - LossClass.DROPPED, - "the step carries tool_calls entries that are not JSON objects", + LossClass.NORMALIZED, + f"tool_calls is {type(raw_calls).__name__}, not a list; kept verbatim " + "in extensions since no tool call could be read from it", space=PathSpace.SOURCE, ) observation = raw_step.get("observation") - results = ( - observation.get("results") - if isinstance(observation, dict) - and isinstance(observation.get("results"), list) - else [] - ) + results: list[Any] = [] + unreadable_observation = False + if isinstance(observation, dict): + raw_results = observation.get("results") + if isinstance(raw_results, list): + results = raw_results + elif raw_results is not None: + unreadable_observation = True + elif observation is not None: + unreadable_observation = True + if unreadable_observation: + extensions["observation"] = observation + losses.add( + f"steps[{first_index}].observation", + LossClass.NORMALIZED, + "observation does not have the results list this converter reads; " + "kept verbatim in extensions", + space=PathSpace.SOURCE, + ) - if source_str is not None and role is None: - # Outside the vocabulary this repository writes. Kept whole rather than - # forced into a role the document does not support. + if role is None: + # Nothing in the document says whose step this is: the source is + # missing, not a string, or outside the vocabulary. Inferring ``agent`` + # would be the converter asserting something the document does not — + # `ir_from_acp` maps an unrecognized record to UNKNOWN for the same + # reason. The whole body is kept, since this edge cannot say which parts + # of an unrecognized step it understood. + if source is not None and source_str is None: + extensions["source"] = source + losses.add( + f"{where}.source_type", + LossClass.NORMALIZED, + f"source is {type(source).__name__}, not a string; the IR records " + "no source type and the original is kept in extensions", + ) return [ TraceEvent( index=first_index, @@ -501,6 +553,10 @@ def _kind_for_step( ) -> EventKind: """The event kind a non-tool step maps to. + Only reached for a step whose ``source`` is in the vocabulary; anything else + became ``UNKNOWN`` before this point, rather than being attributed to the + agent by default. + A step with reasoning and no message is what `ir_to_atif` writes when it flushes buffered thoughts, so it reads back as reasoning. A step with both is one event carrying both — the fusion is reported, not undone. diff --git a/src/benchflow/trajectories/ir_round_trip.py b/src/benchflow/trajectories/ir_round_trip.py index c8a9ef5dd..4a55e528e 100644 --- a/src/benchflow/trajectories/ir_round_trip.py +++ b/src/benchflow/trajectories/ir_round_trip.py @@ -173,7 +173,15 @@ class Representability(StrEnum): # shape is not a set of IR fields. Compared whole: a source block either comes # back or it does not, and reporting three findings because it happened to have # three keys would weight one loss by the shape of the data. -OPAQUE_PATHS: frozenset[str] = frozenset({"events[].tool_call.content[].raw"}) +OPAQUE_PATHS: frozenset[str] = frozenset( + {"events[].tool_call.content[].raw", "events[].tool_call.arguments"} +) +"""Both hold a mapping whose keys come from a tool or an agent, not from the IR. + +``arguments`` is here for the same reason ``raw`` is, and for one more: without +it the field's path would depend on its contents — ``arguments.cmd`` when a call +carries arguments, ``arguments`` when it carries ``{}`` — so the same IR field +would be counted in different places depending on the data.""" # Fields whose default *is* the empty container. An empty value at one of these # is the absence of values, not an observed empty one, so it is not counted — diff --git a/tests/trajectories/test_ir_from_atif.py b/tests/trajectories/test_ir_from_atif.py index 7027a7dd8..8cfbdb916 100644 --- a/tests/trajectories/test_ir_from_atif.py +++ b/tests/trajectories/test_ir_from_atif.py @@ -548,11 +548,124 @@ def test_non_object_arguments_are_kept_and_declared(): assert validate_trace(trace) == [] +# --------------------------------------------------------------------------- +# null, absent and empty are three different things +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("step", "expected"), + [ + ({"step_id": 1, "source": "user", "message": None}, None), + ({"step_id": 1, "source": "user"}, None), + ({"step_id": 1, "source": "user", "message": ""}, ""), + ({"step_id": 1, "source": "user", "message": "m"}, "m"), + ], +) +def test_null_absent_and_empty_text_stay_distinct(step, expected): + """A JSON ``null`` is not a value, and it is certainly not ``"None"``. + + Found by an adversarial probe, not by the suite: the coercion path turned + ``message: null`` into the four-character string ``"None"`` — a value the + document never contained, produced by the converter whose one rule is not to + invent. ``""`` stays ``""``, because that one *is* observed. + """ + trace = atif_to_ir(_minimal(steps=[step])) + assert trace.events[0].text == expected + + +def test_a_null_agent_field_does_not_become_a_string(): + trace = atif_to_ir( + _minimal(agent={"name": None, "version": "1"}, steps=[{"source": "user"}]) + ) + assert trace.agent.agent_name is None + assert trace.agent.agent_version == "1" + + +def test_null_reasoning_produces_no_segments(): + """``reasoning_segments=["None"]`` would also have satisfied invariant 5, + which is why the invariants alone could not catch this.""" + trace = atif_to_ir( + _minimal(steps=[{"source": "agent", "message": "m", "reasoning_content": None}]) + ) + event = trace.events[0] + assert event.reasoning is None + assert event.reasoning_segments is None + + +# --------------------------------------------------------------------------- +# Attribution the document does not license +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "step", + [ + {"step_id": 1, "message": "m"}, + {"step_id": 1, "source": None, "message": "m"}, + {"step_id": 1, "source": 123, "message": "m"}, + {"step_id": 1, "source": "environment", "message": "m"}, + ], +) +def test_a_step_with_no_usable_source_is_unknown_not_agent(step): + """Nothing in these steps says whose turn it is. + + Attributing them to the agent would be the converter asserting what the + document does not — and `agent_message` is not a harmless default: it is the + kind a consumer counts as a model turn. `ir_from_acp` maps an unrecognized + record to UNKNOWN for the same reason. + + Found by an adversarial probe; before it, a step with no `source` at all + read back as an agent message. + """ + event = atif_to_ir(_minimal(steps=[step])).events[0] + assert event.kind is EventKind.UNKNOWN + assert event.role is None + + +def test_a_non_string_source_is_kept_rather_than_discarded(): + """It cannot be the IR's ``source_type``, which is a string — but dropping + it would lose the only thing the document said about the step's origin.""" + trace = atif_to_ir(_minimal(steps=[{"step_id": 1, "source": 123, "message": "m"}])) + event = trace.events[0] + assert event.source_type is None + assert event.extensions["source"] == 123 + assert "events[0].source_type" in _fields(trace, LossClass.NORMALIZED) + + # --------------------------------------------------------------------------- # Malformed input — a document read off disk is not a document we wrote # --------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("key", "value"), + [ + ("tool_calls", {"not": "a list"}), + ("tool_calls", "nonsense"), + ("observation", "nonsense"), + ("observation", {"results": {"not": "a list"}}), + ], +) +def test_a_structure_this_converter_cannot_read_is_kept_and_declared(key, value): + """ "Unreadable here" is not "not present". + + Found by an adversarial probe: all four of these used to vanish with no + record in any path space, which is precisely the silent drop the loss + contract exists to make impossible. + """ + trace = atif_to_ir( + _minimal(steps=[{"step_id": 1, "source": "agent", "message": "m", key: value}]) + ) + event = trace.events[0] + assert event.extensions[key] == value + assert [ + record + for record in trace.losses.records + if record.space is PathSpace.SOURCE and record.field.endswith(key) + ] + + def test_a_non_mapping_document_is_rejected(): with pytest.raises(TypeError): atif_to_ir(["not", "a", "document"]) diff --git a/tests/trajectories/test_ir_round_trip.py b/tests/trajectories/test_ir_round_trip.py index 81c77096f..fe5a963b2 100644 --- a/tests/trajectories/test_ir_round_trip.py +++ b/tests/trajectories/test_ir_round_trip.py @@ -21,6 +21,7 @@ from __future__ import annotations +from datetime import datetime from types import UnionType from typing import Any, Union, get_args, get_origin @@ -29,14 +30,22 @@ from benchflow.trajectories.ir import ( CanonicalTrace, + ContentBlock, + ContentBlockKind, EventKind, + ModelInfo, + OutcomeStatus, Provenance, + Role, ToolCall, + ToolStatus, TraceEvent, + TraceOutcome, TraceUsage, validate_trace, ) from benchflow.trajectories.ir_from_acp import acp_events_to_ir +from benchflow.trajectories.ir_from_atif import atif_to_ir from benchflow.trajectories.ir_round_trip import ( ATIF_REPRESENTABILITY, DEFAULT_EMPTY_PATHS, @@ -49,6 +58,7 @@ representability_of, round_trip_through_atif, ) +from benchflow.trajectories.ir_to_atif import ir_to_atif from tests.trajectories.test_atif_preservation import _rich_events PROMPTS = ["Solve the task.", "Then stop."] @@ -202,10 +212,27 @@ def test_a_timeout_does_not_survive_the_loop(): trip = _trip(prompts=PROMPTS) assert trip.report.kinds_before["timeout"] >= 1 assert "timeout" not in trip.report.kinds_after - assert _outcome(trip.report, "events[].outcome") is RoundTripOutcome.LOST - assert _outcome(trip.report, "outcome.status") is RoundTripOutcome.LOST - for path in ("events[].outcome", "outcome.status"): - assert representability_of(path) is Representability.NOT_IN_ATIF + + # Each field the marker carried, named rather than counted: a count would + # still pass if one of these came back and another vanished. Matched by + # prefix because a list-valued extension dumps to a path ending in ``[]`` + # only while the list is non-empty, and the fixture's is. + cost = ( + "events[].outcome", + "outcome.status", + "events[].extensions.timeout_sec", + "events[].extensions.pending_tool_call_ids", + "events[].extensions.terminal_trajectory_complete", + ) + for path in cost: + matching = [ + comparison + for comparison in trip.report.comparisons + if comparison.path == path or comparison.path == f"{path}[]" + ] + assert len(matching) == 1, (path, [c.path for c in trip.report.comparisons]) + assert matching[0].outcome is RoundTripOutcome.LOST, path + assert matching[0].representability is Representability.NOT_IN_ATIF, path def test_reasoning_survives_but_its_boundaries_do_not_have_to(): @@ -498,6 +525,142 @@ def test_a_representable_loss_would_be_reported_separately_from_a_format_one(): assert summary["non_representable"] == 0 +def _maximal_trace() -> CanonicalTrace: + """A trace with a value in every IR field, so the table can be falsified. + + Built by hand rather than captured: no rollout populates per-event usage, + opaque content blocks or a run outcome at once, and the point of this + fixture is to exercise the fields a real rollout leaves empty. + """ + when = datetime(2026, 1, 1, 12, 0, 0) + return CanonicalTrace( + trace_id="trace-1", + session_id="sess-1", + agent=ModelInfo( + agent_name="claude-code", + agent_version="2.1.0", + model="claude-sonnet-5", + provider="anthropic", + ), + started_at=when, + finished_at=when, + events=[ + TraceEvent( + index=0, + kind=EventKind.USER_MESSAGE, + source_type="user_message", + role=Role.USER, + text="do the thing", + provenance=Provenance(source_format="probe"), + ), + TraceEvent( + index=1, + kind=EventKind.TOOL_CALL, + source_type="tool_call", + role=Role.AGENT, + text="calling", + tool_call=ToolCall( + call_id="call-1", + name="execute", + name_semantics="acp_kind", + title="ls -la", + status=ToolStatus.COMPLETED, + arguments={"cmd": "ls -la"}, + content=[ + ContentBlock( + kind=ContentBlockKind.TEXT, + text="a\nb", + raw={"type": "text", "text": "a\nb"}, + ), + ContentBlock( + kind=ContentBlockKind.OPAQUE, + raw={"type": "image", "data": "..."}, + ), + ], + started_at=when, + finished_at=when, + ), + started_at=when, + finished_at=when, + outcome="ok", + usage=TraceUsage(input_tokens=5, output_tokens=1), + provenance=Provenance(source_format="probe"), + extensions={"vendor": "x"}, + ), + TraceEvent( + index=2, + kind=EventKind.AGENT_MESSAGE, + source_type="agent_message", + role=Role.AGENT, + text="done", + provenance=Provenance(source_format="probe"), + ), + ], + usage=TraceUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=3, + cache_creation_tokens=4, + reasoning_tokens=5, + total_tokens=132, + source="llm_proxy_normalized", + cost_usd=0.25, + price_source="litellm", + ), + outcome=TraceOutcome( + status=OutcomeStatus.COMPLETED, stop_reason="end_turn", reward=1.0 + ), + provenance=Provenance(source_format="probe"), + extensions={"run": "probe"}, + ) + + +def _maximal_report(): + before = _maximal_trace() + document, _ = ir_to_atif(before) + return compare_traces(before, atif_to_ir(document)) + + +def test_lost_is_not_zero_by_construction(): + """The check that keeps `lost == 0` on a real rollout from being a tautology. + + `lost` counts fields the table calls representable that did not survive. If + the table were merely pessimistic — everything marked unrepresentable — the + category would be empty on every input and `lost == 0` would mean nothing. + + On a trace that populates every field it is **not** empty: `ir_to_atif` + writes no per-step metrics, so per-event usage is lost through a slot ATIF + actually has. That is a gap in our own edge, and it is exactly the kind of + finding the split exists to keep visible. + """ + summary = _maximal_report().summary() + assert summary["lost"] >= 1 + lost_paths = { + comparison.path + for comparison in _maximal_report().comparisons + if comparison.outcome is RoundTripOutcome.LOST + and comparison.representability is Representability.REPRESENTABLE + } + assert lost_paths == {"events[].usage.input_tokens", "events[].usage.output_tokens"} + + +def test_the_table_never_calls_a_surviving_field_unrepresentable(): + """The table must not be pessimistic in our favour. + + A field wrongly marked `non_representable` would move a real, fixable loss + into the "cost of the format" column and quietly shrink `lost`. So: on a + trace that populates everything, no field the table calls unrepresentable + may come back with any of its values intact. + """ + survived = { + comparison.path: comparison.matched_count + for comparison in _maximal_report().comparisons + if comparison.representability is Representability.NOT_IN_ATIF + and comparison.matched_count > 0 + } + assert survived == {} + + def test_the_summary_accounts_for_every_comparison(): trip = _trip(prompts=PROMPTS) assert sum(trip.report.summary().values()) == len(trip.report.comparisons) From 56ecab6f9f85f9807e78a179fa4a3b587282d3b5 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Sun, 16 Aug 2026 11:01:10 +0200 Subject: [PATCH 12/23] docs(trace-interop): record the human verification of the round trip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §8.10 described its own numbers as a machine measurement and not a human E2E. That was accurate when written and is no longer: the loop's output has since been checked against the raw artifacts by hand on both rollouts — parity, the conversion read in both directions, every fabricated value confirmed present in the ATIF document and absent from the capture, every unrepresentable one confirmed the other way round, and the negative control. The declared limits are unchanged: oracle rollouts and non-text content blocks remain test-only, no artifact here carries a cost, the representability table is checked against our converters rather than against a vendored ATIF schema, and two rollouts from one agent stay a demonstration rather than a survey. --- docs/trace-interop.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/trace-interop.md b/docs/trace-interop.md index d4d6ad61a..b3e168b8e 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -1194,11 +1194,18 @@ inbound edge refuses. #### Measured on the two real rollouts of §5.2 -**FACT — machine measurement, not a human E2E.** The rollouts themselves are -real and were verified by hand when they were captured; the loop below was run -over their `acp_trajectory.jsonl` by the harness. `with prompts` reproduces what -a real export does, and its document is byte-identical to the `trainer/atif.json` -each rollout actually wrote. +**FACT — measured by the harness, and verified by hand.** The rollouts are real +captures; the loop was run over their `acp_trajectory.jsonl`, and its output was +then checked against the raw artifacts by a person: document parity, the +conversion read step by step in both directions, each fabricated value confirmed +present in `trainer/atif.json` and absent from the capture, each unrepresentable +value confirmed absent from the document and present in the capture, and a +negative control that corrupts a value and confirms the comparison notices. +`with prompts` reproduces what a real export does, and its document is +byte-identical to the `trainer/atif.json` each rollout actually wrote. + +The limits of that verification are in the closing paragraph of this section and +are not narrowed by it. | | H1 fields | H2 fields | H1 values | H2 values | |---|---|---|---|---| From 796e436dd4710331a04d47bd1fc22487408d9322 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Tue, 18 Aug 2026 19:58:13 +0200 Subject: [PATCH 13/23] feat(trajectories): read OTLP/JSON spans into the canonical IR, unwired MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the inbound OpenTelemetry edge, plus the AnyValue decoder it is built on. Inbound only: no `IR -> OTel` emitter, no wiring into a run path, no on-disk artifact, and no existing format, exporter or artifact changes because it exists. `uv.lock` is untouched. §4.2 is FACT — this repository has no OTel code and no OTel dependency — so the mapping is written against the two artifacts `uv.lock` already pins rather than against recollection: `opentelemetry-proto` 1.41.1 for the wire shape, `opentelemetry-semantic-conventions` 0.62b1 for the `gen_ai.*` vocabulary. Both versions are recorded in code, both wheel hashes were checked against the lock, and neither package is imported — the edge reads JSON dictionaries. Reading them settled four things a mapping written from memory gets wrong, and the collector deleted in e84795d7 got three of them wrong: `intValue` is a JSON string in the canonical encoding; an `AnyValue` writes `""`, `0` and `false` explicitly, so its `stringValue or intValue or ...` read observed values as absent; `gen_ai.usage.total_tokens` does not exist at 0.62b1 and the cache counters are spelled `cache_read.input_tokens`; enums serialize as member names by default and as integers under a flag, and both parse. The rule the edge is built on: a span is evidence of an operation, not a statement about an agent. Exactly one span shape becomes a typed IR kind — the `execute_tool` operation the pinned vocabulary defines as that — and every other span becomes UNKNOWN with its whole content carried in `extensions.otel`. So of the IR's seven EventKind members this edge reaches two. For four of the eight `gen_ai.operation.name` values there is no IR candidate at all; for `chat`, `generate_content` and `text_completion` there is one, and the blocker is checkable rather than a matter of taste — filling it means reading `gen_ai.input.messages`, whose structure the pinned package defines by reference to a JSON schema it does not ship. Order is preserved and never sorted by time: siblings overlap, a batch may omit a parent, and two spans can share a start instant. The `parentSpanId` edge set is preserved per span instead, together with the envelope coordinates, so the `resourceSpans`/`scopeSpans` partition survives flattening — two spans batched separately under an equal `scope` stay distinguishable. Identifiers are carried exactly as written and never re-encoded: a 32-character hex trace id fed to the pinned JSON parser is accepted as base64 and silently yields 24 bytes, so hex and base64 are not reliably distinguishable and normalizing would make identity depend on a heuristic. Two losses are structural and both are declared: nanoseconds do not fit a `datetime`, and OTLP models `name`, the timestamps, `parentSpanId`, `kind` and the `dropped*Count`s as scalars with no presence, so absent and default are the same document. A third is a distinction worth keeping — decoding `AnyValue` wrappers into a map preserves the values and loses the wire form, which is `normalized`, not `preserved`. Trace-level `started_at`, `finished_at` and `usage` are left empty on purpose. The class says why: NORMALIZED when the spans carry the information per span and aggregating it would assume the payload holds the whole run, UNSUPPORTED when there was nothing to aggregate. An OTLP export request is a batch. 129 tests. The contract guard derives the IR field list from the models and requires every field to be filled or declared **per instance** — per event, per content block — with two exemptions that are properties of the IR rather than of a converter. It found two real undeclared absences while the edge was being written, and a structural review of the finished slice found four more, one of them in the guard itself: it had been field-level, so a field the fixture happens to fill passed on every payload. 33 targeted mutations, all caught. The suite's fixture is not hand-written: it is `MessageToJson` output from the pinned library, so its encoding is the library's rather than this suite's, and `e2e-f/regenerate_fixture.py` rebuilds it from a throwaway venv and diffs. HUMAN E2E VERIFIED 2026-08-18, F1-F6 all PASS, recorded in §8.11 together with the limits it does not extend past: the fixture's encoding is authoritative and its content is constructed; no OTLP payload from a real agent has ever been read, because nothing in BenchFlow emits spans; the mapping is checked against two pinned packages, not the specification; the GenAI conventions are still `_incubating`; there is no `IR -> OTel`; and there is therefore no OTel round trip and none is measured. `0 dropped` on the fixture is a property of that payload, not of the edge — two fully conformant inputs do produce DROPPED records. --- docs/trace-interop.md | 511 ++++- src/benchflow/trajectories/_otlp_anyvalue.py | 203 ++ src/benchflow/trajectories/ir_from_otel.py | 1333 +++++++++++ tests/trajectories/test_ir_from_otel.py | 2122 ++++++++++++++++++ tests/trajectories/test_trace_ir.py | 2 + 5 files changed, 4133 insertions(+), 38 deletions(-) create mode 100644 src/benchflow/trajectories/_otlp_anyvalue.py create mode 100644 src/benchflow/trajectories/ir_from_otel.py create mode 100644 tests/trajectories/test_ir_from_otel.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md index b3e168b8e..54debc4c1 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -275,6 +275,33 @@ not part of any run path"*. It converted **inbound only** (OTLP/JSON spans into parent/child relationships. It contained no emitter, and no other OTel-named module appears in the repository history. +**What *is* pinned, and why it matters.** `uv.lock` resolves +`opentelemetry-proto` **1.41.1**, `opentelemetry-sdk` **1.41.1** and +`opentelemetry-semantic-conventions` **0.62b1**, transitively, as dependencies of +`daytona` — the `sandbox-daytona` extra (`pyproject.toml:69-75`). None of them is +a BenchFlow dependency, none is installed in the default dev environment, and +nothing in `src/` imports them. They matter for one reason: they are artifacts +this repository already names, at a version, with a hash. So a question about the +OTLP wire shape or about a `gen_ai.*` attribute name has a *checkable* answer +here, instead of an answer from recollection. §8.3's OTel edge is written against +those two artifacts and records both versions in code. + +Two facts worth stating up front, because both bound what any OTel mapping can +claim: + +- **The GenAI conventions are experimental even at the pinned version.** The + package puts every `gen_ai.*` name under `opentelemetry.semconv._incubating`, + and several are already marked deprecated in 0.62b1 — `gen_ai.system`, + `gen_ai.usage.prompt_tokens`, `gen_ai.usage.completion_tokens`, + `gen_ai.prompt`, `gen_ai.completion`. +- **The deleted collector's attribute names do not match it.** `otel.py` read + `gen_ai.usage.total_tokens`, which does not exist at 0.62b1, and + `gen_ai.usage.cache_read_input_tokens` / `…cache_creation_input_tokens`, which + are spelled `gen_ai.usage.cache_read.input_tokens` and + `gen_ai.usage.cache_creation.input_tokens`. Its `_parse_attributes` also read + `stringValue or intValue or doubleValue or boolValue`, which turns an observed + `""`, `0` or `false` into "no value". + --- ## 5. Information loss in today's conversions @@ -440,7 +467,11 @@ them. in `normalize_acp_usage`? 5. Is OpenTelemetry wanted as an inbound receiver, an outbound emitter, or both? The only implementation ever written was inbound, and it was removed - deliberately. + deliberately. **Partly acted on, not answered:** §8.3 implements the inbound + half (`OTel → IR`) because it is useful whichever way the question is + resolved — it lets BenchFlow *read* traces from instrumented agents without + committing to emitting any. The emitter is not written, and §8.11 lists the + decisions it would depend on. **Still open.** §8 takes a *provisional* position on question 1 — a canonical hub — and implements it in isolation so it can be reviewed as code. That is a @@ -588,17 +619,19 @@ Three design choices carry most of the weight: an outbound conversion returns its report alongside its document and leaves `trace.losses` untouched. -### 8.3 Planned mapping +### 8.3 Implemented mapping **`ACP → IR`** ([`ir_from_acp.py`](../src/benchflow/trajectories/ir_from_acp.py)), -**`IR → ATIF`** ([`ir_to_atif.py`](../src/benchflow/trajectories/ir_to_atif.py)) -and **`ATIF → IR`** -([`ir_from_atif.py`](../src/benchflow/trajectories/ir_from_atif.py)) **are -implemented**, all unwired and still provisional. The OTel direction is not, and -stays blocked on open question 2. +**`IR → ATIF`** ([`ir_to_atif.py`](../src/benchflow/trajectories/ir_to_atif.py)), +**`ATIF → IR`** ([`ir_from_atif.py`](../src/benchflow/trajectories/ir_from_atif.py)) +and **`OTLP/JSON → IR`** +([`ir_from_otel.py`](../src/benchflow/trajectories/ir_from_otel.py)) **are +implemented**, all unwired and still provisional. `IR → OTel` is not, and is +deliberately not sketched — §8.11 lists the decisions it depends on. With both ATIF edges in place the loop `ACP → IR → ATIF → IR′` closes, and §8.10 -reports what it measures. +reports what it measures. The OTel edge is inbound only, so no loop closes +through it and §8.11 reports what it showed instead. #### ACP-session capture events → IR @@ -757,30 +790,6 @@ contract regardless of how often it is exercised. prices of its own, so a cost without the table that produced it is not comparable. Per-call cost stays out of scope. -#### IR ↔ OpenTelemetry - -**PROPOSAL, and unverified.** §4.2 is FACT — there is no OTel dependency and no -`gen_ai.*` handling in this repository — so the mapping below is a sketch -against the OpenTelemetry GenAI semantic conventions **as remembered, not as -vendored**: no version of that specification is pinned in this repo, nothing -validates against it, and the conventions have been changing. It is written down -to show the hub is reachable from OTel, not as a design to implement as-is. - -| IR | OTel sketch | -|---|---| -| `CanonicalTrace` | one root span per rollout | -| `trace_id` | the OTel trace id — `None` until a source provides one; never invented | -| `events[]` ordering | child span order / span events | -| `tool_call` | a child span; `name` → `gen_ai.tool.name`, `call_id` → `gen_ai.tool.call.id` | -| `usage` | `gen_ai.usage.input_tokens` / `.output_tokens` | -| `agent.model` | `gen_ai.request.model` | -| `outcome.status` | span status | -| `extensions` | span attributes, verbatim | - -Which direction is wanted — receiver, emitter, or both — is **open question 5** -and stays open. The IR is symmetric on purpose so the answer does not invalidate -it. - #### ATIF → IR **Implemented** ([`ir_from_atif.py`](../src/benchflow/trajectories/ir_from_atif.py)), @@ -842,6 +851,115 @@ followed by a message event: the blank-line join that produced it is not injective (§5 loss #10), so splitting it would invent a boundary. The fusion is reported by the event count instead. +#### OTLP/JSON → IR + +**Implemented** ([`ir_from_otel.py`](../src/benchflow/trajectories/ir_from_otel.py)), +unwired, inbound only. Asserted by `tests/trajectories/test_ir_from_otel.py` +against a payload produced by `opentelemetry-proto` 1.41.1 itself — see §8.11 — +plus documents no conformant producer writes. + +The mapping is written against the two artifacts §4.2 names, at the versions +`uv.lock` pins, and the module records both in `OTLP_PROTO_VERSION` and +`SEMCONV_VERSION`. It takes **no OTel dependency**: it reads JSON dictionaries, +so `uv.lock` is untouched. Nothing here is written against the published +specification text, which is not vendored. + +The rule the edge is built on: **a span is evidence of an operation, not a +statement about an agent.** OTLP is a general tracing format; it says what was +instrumented, when, and under what identifiers, and it does not say who spoke or +what a turn was. So exactly one span shape maps onto a typed IR kind — the one +the pinned vocabulary defines as a tool execution — and everything else becomes +`UNKNOWN` with its whole content carried. What a plausible reading would have +added instead is in §8.11 as an open decision. + +| OTLP | IR | Class | Loss recorded | +|---|---|---|---| +| span order in the payload | `events[].index`, dense from 0 | preserved | — **never sorted by time**; see below | +| `span.traceId` | `trace_id`, **verbatim, never re-encoded** | preserved | — | +| `span.spanId` / `.parentSpanId` / `.traceState` / `.flags` / `.kind` | `events[].extensions.otel.span`, verbatim | preserved | — the IR models no parent link | +| `span.name` | `source_type` | preserved *(value)* | `events[].source_type` — the **value** is carried verbatim; what is unsupported is the absent-versus-empty *distinction*, because the field has no presence | +| `span.attributes[]` | `extensions.otel.attributes`, decoded to a map | **normalized** | `events[].extensions` — values are preserved, the wire form is not; see below | +| `span.events[]` / `span.links[]` | `extensions.otel.span`, verbatim | preserved | — | +| `span.status` | `extensions.otel.span.status`, verbatim | normalized | `events[i].outcome` — the IR slot is free text with no vocabulary a status code maps into | +| `span.startTimeUnixNano` / `.endTimeUnixNano` | `events[].started_at` / `.finished_at` | preserved | — unless not a whole microsecond, then normalized (see below) | +| `resource` / `scope`, with schema URLs | `extensions.otel`, per event | preserved | — per event, not per trace: one payload mixes them | +| envelope position (`resourceSpans[i].scopeSpans[j].spans[k]`) | `extensions.otel.envelope` | preserved | — absent when the caller did not read the span out of a payload | +| `scope.name` | `events[].provenance.producer` | preserved | — | +| `span.dropped*Count > 0` | *(nothing to carry)* | **unsupported** | `events[i].extensions` — the SDK discarded it before export | +| `gen_ai.operation.name == "execute_tool"` | `kind = tool_call` | preserved | — the only typed mapping | +| any other span | `kind = unknown`, everything carried | preserved | — | +| `gen_ai.tool.name` | `tool_call.name` + `name_semantics="gen_ai.tool.name"` | preserved | — | +| `gen_ai.tool.call.id` | `tool_call.call_id` | preserved | `events[i].tool_call.call_id` when absent — **no id is synthesized** | +| `gen_ai.tool.call.arguments` (kvlist) | `tool_call.arguments`, **including `{}`** | preserved | — | +| `gen_ai.tool.call.arguments` (JSON string) | *(kept in the attribute map)* | normalized | `events[i].tool_call.arguments` — **not parsed** | +| `gen_ai.tool.call.result` (string / object) | `ContentBlock` `text` / `opaque` | preserved | `events[i].tool_call.content[0].raw` for a text block (the attribute *is* the text) · `…content[0].text` for an object block (the document holds no rendering) · `…tool_call.content` when the attribute is absent | +| the span's own extent | `tool_call.started_at` / `.finished_at` | preserved | `events[i].tool_call.started_at` / `.finished_at` when the span carries no readable instant, or when nanoseconds are truncated — the first inbound edge that can *fill* these (§5 loss #3), and it declares them at their own path when it cannot | +| `gen_ai.usage.input_tokens` / `.output_tokens` / `.cache_read.input_tokens` / `.cache_creation.input_tokens` | `events[].usage.*`, `source="otel_gen_ai_usage"` | preserved | — | +| `gen_ai.usage.prompt_tokens` / `.completion_tokens` | the same fields | normalized | `events[i].usage.*` — read from a spelling the pinned package marks deprecated | +| `gen_ai.agent.name` / `.version`, `gen_ai.request.model`, `gen_ai.provider.name` | `agent.*` | preserved | — | +| `gen_ai.system` | `agent.provider` | normalized | `agent.provider` — deprecated spelling | +| spans disagreeing on an `agent.*` value | the first is kept | dropped | `agent.` | +| `gen_ai.input.messages` / `.output.messages` / `gen_ai.prompt` / `.completion` | `extensions.otel.attributes` | normalized | `events[i].text` — **not read as text** | +| *(none)* | `session_id`, `started_at`, `finished_at`, `outcome`, `usage`, `events[].role`, `.reasoning`, `.reasoning_segments`, `.source_type` (presence), `events[].tool_call.title`, the usage fields with no attribute | **unsupported** / **normalized** | one record each; see below | +| a non-object entry at any envelope level | *(no span)* | dropped | `resourceSpans[i]…`, **source space** | +| a non-string `traceId` | *(not read)* | dropped | `spans[i].traceId`, **source space** | + +**Order is preserved; causality is not inferred from it.** Document order is +kept exactly and spans are never sorted by start time — siblings overlap, a +batch may omit a parent, and two spans can share a start instant. The real +structure is the `parentSpanId` edge set, and it is preserved per span. Because +the IR has no parent field, it lives in `extensions`; adding one is a change to +the hub and therefore a maintainer's decision (§8.11). + +**Two losses are structural and unavoidable, and both are declared.** +`datetime` resolves to microseconds while OTLP counts nanoseconds, so a +timestamp that is not a whole microsecond is truncated in the IR field — the +exact integer stays in `extensions.otel.span`, so the value survives, but the +canonical field no longer holds it. And OTLP models `name`, the timestamps, +`parentSpanId`, `kind` and the `dropped*Count`s as protobuf scalars **without +presence**, so an unset field and one holding the default are the same document: +the IR's absent-versus-empty distinction is simply not recoverable for them. + +**Semantic preservation is not wire preservation, and the table says which one +it means.** Attribute values are decoded out of their `AnyValue` wrappers into a +plain map, and the canonical protobuf JSON mapping writes an `int64` as a +*string* while much of the ecosystem writes it as a number. `{"intValue": "7"}` +and `{"intValue": 7}` are therefore the same `7` afterwards, and the wrapper +type is gone with them. No value is lost, so `attributes_raw` is *not* kept — +that copy exists for the cases where a value would be, listed above — but the +two payloads are indistinguishable, and an `IR → OTel` emitter could not +reproduce either byte-for-byte. Declared once per conversion at +`events[].extensions`, because it is a property of the decoding rather than of +any one span. + +**The envelope partition is preserved as coordinates, not as structure.** One +payload nests spans under `resourceSpans[] → scopeSpans[] → spans[]`, and +reading it into one list of events flattens that: two spans the producer batched +under *different* `ScopeSpans` objects that happen to carry an equal `scope` +would otherwise become indistinguishable. `extensions.otel.envelope` keeps the +three indices, so the grouping stays reconstructible without the IR growing a +concept of an envelope. Spans a caller assembled itself carry no coordinates — +there is no envelope position to record, and inventing one would claim a payload +that never existed. + +**Trace-level absences carry the class that says where a fix would land.** +`outcome`, `session_id` and `events[].role` are `UNSUPPORTED` — OTLP has nothing +to read. `started_at`, `finished_at` and `usage` are `NORMALIZED` when spans +carry the information per span: the values are preserved on the events, and +deriving a run extent or a token total from them would assume this payload holds +the whole run, which a batch does not promise. When no span carries them at all, +the same fields become `UNSUPPORTED` instead. Two different reasons for the same +empty field, kept apart. + +#### IR → OpenTelemetry + +**Not implemented, and deliberately not sketched further.** The inbound edge +above establishes what OTLP can be read *as*; writing spans raises a different +and larger set of questions — what a span tree for a BenchFlow rollout should +look like, which ids to mint, which encoding to write them in, and whether +emitting `gen_ai.*` attributes commits the project to an `_incubating` +vocabulary. Those are in §8.11. + ### 8.4 A worked example **FACT.** The document below is produced by the models in `ir.py` and compared @@ -1085,18 +1203,19 @@ here. ### 8.7 Status - **Implemented:** the IR types, the loss model with its path spaces, the - invariants, the validation suite, this section, all three converters — - `ACP → IR`, `IR → ATIF`, `ATIF → IR` — each with its loss report, and the - round-trip measurement over them (§8.10). -- **Not implemented, deliberately:** any OTel work, any wiring into a run path, + invariants, the validation suite, this section, four converters — `ACP → IR`, + `IR → ATIF`, `ATIF → IR`, `OTLP/JSON → IR` — each with its loss report, and + the round-trip measurement over the ATIF pair (§8.10). +- **Not implemented, deliberately:** `IR → OTel`, any wiring into a run path, any on-disk artifact, any capture-layer enrichment. - **Unchanged:** every existing format, exporter, artifact and code path. `export_atif.py` in particular is untouched and remains the only writer of `trainer/atif.json`. The isolation property is unchanged in substance and restated at a new boundary: -`ir.py`, `ir_from_acp.py`, `ir_to_atif.py`, `ir_from_atif.py` and -`ir_round_trip.py` form a closed family that may import each other, and +`ir.py`, `ir_from_acp.py`, `ir_to_atif.py`, `ir_from_atif.py`, +`ir_from_otel.py`, `_otlp_anyvalue.py` and `ir_round_trip.py` form a closed +family that may import each other, and `test_only_the_ir_family_imports_the_ir` asserts that nothing else in `src/benchflow` imports any of them. Each converter imports one benchflow module — the IR — and reads or writes its own format as data; in particular @@ -1274,3 +1393,319 @@ appear only in constructed test input, never in a captured rollout here; no artifact on hand carries a non-null cost, so `usage.cost_usd` round-trips in tests only; and the loop has been run over two rollouts from one agent, which is a demonstration, not a survey. + +### 8.11 What the OTel edge showed + +#### Where its ground truth comes from + +**FACT.** The three edges before this one could be checked against a producer in +this repository. There is no OpenTelemetry producer here at all (§4.2), so the +alternative to guessing was the lock file. The suite's fixture is the output of +`google.protobuf.json_format.MessageToJson` over an `ExportTraceServiceRequest` +built with `opentelemetry-proto` **1.41.1** — the version `uv.lock` pins, whose +wheel hash matches the lock entry — and every `gen_ai.*` constant in the module +is copied from `opentelemetry-semantic-conventions` **0.62b1**, likewise pinned. +Neither package is imported, neither is added as a dependency, and `uv.lock` is +untouched. + +Reading them settled four things that a mapping written from memory gets wrong, +and the deleted `OTelCollector` got three of them wrong: + +- `intValue` is a JSON **string** in the canonical mapping (`"1204"`), not a + number, and `startTimeUnixNano` likewise. Both spellings are accepted on + parse, so a reader has to handle either. +- An `AnyValue` writes its member explicitly even when the value is falsy: + `{"stringValue": ""}`, `{"boolValue": false}`, `{"doubleValue": 0.0}`. The + deleted collector's `stringValue or intValue or doubleValue or boolValue` read + all three as "no value". +- `gen_ai.usage.total_tokens` **does not exist** at 0.62b1, and the cache + counters are spelled `gen_ai.usage.cache_read.input_tokens` / + `gen_ai.usage.cache_creation.input_tokens` — one dot away from the collector's + spelling, and never a match. +- Enum fields serialize as member **names** by default (`"STATUS_CODE_ERROR"`) + and as integers under `use_integers_for_enums`. Both are accepted on parse, so + both are recognized here. + +#### The finding that decides how identity is handled + +`traceId` and `spanId` are `bytes` in the proto, and the protobuf canonical JSON +mapping encodes bytes as **base64** — a 16-byte trace id becomes 24 characters +ending in `==`. Much of the ecosystem writes lowercase hex instead. The two are +**not reliably distinguishable**, and this is checkable rather than a worry: +feeding the 32-character hex string `4bf92f35…` to the pinned JSON parser is +accepted *as base64* and yields **24 bytes**, silently. + +So the edge carries identifiers **exactly as written** and re-encodes nothing. +Any normalization would make identity depend on a heuristic, and a heuristic +that is right most of the time is the worst possible property for an identifier. + +#### Two limits that are properties of OTLP, not of this converter + +- **Absent and default are the same document.** `name`, both timestamps, + `parentSpanId`, `kind` and the `dropped*Count`s are protobuf scalars *without + presence* — verified against the pinned descriptors — so a producer that never + set the field and one that set it to the default write identical JSON. The + IR's absent/`None`/observed-empty tri-state (§8.2) is therefore real on this + edge only for the fields OTLP models with presence, and the limit is declared + once per conversion rather than left to a reader to discover. +- **Nanoseconds do not fit a `datetime`.** OTLP counts nanoseconds; `datetime` + resolves to microseconds. A span ending at `…1900000375` loses 375 ns from the + IR field. The exact integer stays in `extensions.otel.span`, so the value is + preserved — but the canonical field no longer holds it, which is a + normalization and is recorded as one. + +#### `UNSUPPORTED` and `NORMALIZED` for the same empty field + +The trace-level `started_at`, `finished_at` and `usage` are empty after every +OTel conversion, and the class says *why*, which is where a fix would land: + +- when spans carry timestamps or token counters, the records are `NORMALIZED` — + the information is in the trace, per event, and deriving a run extent or a + token total from it would assume the payload holds the whole run. **An OTLP + export request is a batch**: it may carry several traces, it need not contain + the root span, and one trace may arrive across several requests. +- when no span carries them, the same fields are `UNSUPPORTED` — there was + nothing to aggregate. + +The same reasoning keeps this edge from mapping a span status onto +`outcome.status` or onto `ToolStatus`: a status is per span, and choosing one +span's status as the run's needs a root the payload does not promise. + +#### What the report costs + +**MEASURED**, on the producer-derived fixture — five spans: an `invoke_agent` +root, a `chat` child with usage, two `execute_tool` children, and a plain HTTP +span. + +| | count | +|---|---| +| records total | 31 | +| `unsupported` | 19 | +| `normalized` | 12 | +| `dropped` | **0 — see below** | +| `synthesized` | **0 — see below** | +| systemic (declared once) | 19 | +| per-event | 12 | +| envelope (source space) | 0 | + +**The two zeros are not the same kind of zero, and neither should be read as a +property of the edge.** + +`synthesized` is **structurally unreachable**: the class does not appear +anywhere in `ir_from_otel.py`, and it could not. It means "the *target* required +a value the source did not have", and on an inbound edge the target is the +canonical IR, whose only required fields are `provenance`, an event's `index` +and `kind`, and a content block's `kind` — each derived from the input rather +than invented. There is no slot this edge could be forced to fill. `ACP → IR` +and `ATIF → IR` have zero sites for the same reason; `ir_to_atif`, which is +outbound, has eight. + +`dropped` is **a property of this payload**, not of the edge. Four inputs reach +a `DROPPED` site, and two of them are fully conformant OTLP *and* fully +conformant semantic conventions: + +| input | validity | record | +|---|---|---| +| two spans with different `gen_ai.request.model` | valid OTLP, valid semconv | `agent.model` | +| `gen_ai.usage.input_tokens` and the deprecated `gen_ai.usage.prompt_tokens` disagreeing | valid OTLP, both names in 0.62b1 | `events[i].usage.input_tokens` | +| a token counter carried as a `doubleValue` or `bytesValue` | valid OTLP, violates semconv | `events[i].usage.` | +| a non-list `scopeSpans`, a non-object span entry, a non-string `traceId` | wire-invalid | source-space records | + +The fixture is a single-model trace with no deprecated spellings, so it reaches +none of them. Reading its `0` as "this edge never drops anything" would be +exactly the inference the loss model exists to prevent. + +Repeating the same five spans *k* times gives **31, 43, 67, 139** records for 5, +10, 20 and 50 spans — `19 + 2.4n` for this payload's span mix. The systemic half +is constant; only the per-span half grows, which is the same affordability +property `ACP → IR` has (§8.8). + +#### The edge reaches two of the seven event kinds + +**FACT.** `gen_ai.operation.name` has eight values at `SEMCONV_VERSION`, read +out of the pinned wheel: `chat`, `generate_content`, `text_completion`, +`embeddings`, `retrieval`, `create_agent`, `invoke_agent`, `execute_tool`. Only +the last becomes a typed IR kind. Everything else — including a span with no +`gen_ai` attribute at all — becomes `UNKNOWN`, so of the IR's seven +`EventKind` members this edge can emit exactly **`tool_call` and `unknown`**. +`user_message`, `agent_message`, `agent_reasoning`, `timeout` and `oracle` are +unreachable from OTLP. + +That is worth stating plainly, because it means **an OTel-derived trace is +structurally poorer in the hub than an ACP-derived one**, and a consumer that +assumes otherwise will be wrong. + +`UNKNOWN` does not mean nothing was read. A `chat` span still fills +`agent.model`, `events[].usage.*`, both timestamps, the decoded attribute map +and the whole span in `extensions`; what `UNKNOWN` withholds is the assertion of +an *event type*. The reason splits in two: + +- for `invoke_agent`, `create_agent`, `embeddings` and `retrieval` there is **no + candidate**: the IR has no member that means any of them, so `UNKNOWN` is + forced rather than chosen; +- for `chat`, `generate_content` and `text_completion` there *is* a candidate — + `agent_message` — and the blocker is checkable rather than a matter of taste. + Filling it means reading `gen_ai.input.messages` / `gen_ai.output.messages`, + whose structure the pinned package defines by reference to a JSON schema it + does not ship: the wheel contains 129 entries, none of them a `.json` file. + Mapping them would be writing against a document nobody in this repository + can check. + +#### What the guards caught, and what caught the guard + +**FACT.** Three rounds, each finding something the round before could not. + +**The contract guard, first version.** It derives the IR field list from the +models and requires every field to be filled or declared. It found two **real +undeclared absences** in the first working converter: +`events[].reasoning_segments` and `events[].usage.cache_creation_tokens`. The +second produced a distinction worth keeping — a usage field the edge *can* fill +but this payload lacks reads differently from one no OTLP payload can carry, and +the records now say which. + +**Twenty targeted mutations, twenty caught** — sorting spans by time, +attributing every span to the agent, synthesizing a tool-call id, parsing a +serialized arguments string, deriving the run extent, summing usage, normalizing +a hex id, reproducing the deleted collector's falsy-value bug, among others. Two +were green on the first run: one mutation was a no-op and was rewritten, and the +other exposed a **real gap in the suite** — nothing asserted that +`gen_ai.conversation.id` is not read as a session id, so a converter that read +it stayed green. That test exists now. + +**A structural review of the finished slice found four more, and one of them was +in the guard itself.** All four were invisible to a suite that was green +throughout, which is the same lesson `ATIF → IR` produced (§8.10) in a different +place. + +- **The guard was field-level, not instance-level.** It asked whether a path was + filled *somewhere* or declared *somewhere* in the trace. So a field the + fixture happens to fill was satisfied on every payload — including one where + it is empty and nothing declares it. `events[i].tool_call.started_at` was + exactly that: an `execute_tool` span with no readable instant left both + tool-call timestamps `None` with no record at any path beneath `tool_call`, + while `ACP → IR` and `ATIF → IR` both declare those two paths + unconditionally. The guard now checks **one instance at a time** — per event, + per content block — with two exemptions that are properties of the IR rather + than of a converter: a `tool_call` payload that invariant 3 *forbids* on a + non-tool event, and a field covered by a record on an outer node. +- **The content blocks were outside the contract entirely.** The walker did not + descend into `list[ContentBlock]`, so `content[].kind`, `.text` and `.raw` + were never asked about. A text block read from a string result has no `raw` + and an object block has no rendered `text`; neither absence is structural, and + both are now declared at their own concrete path — the same path + `ATIF → IR` uses for `content[].raw`. +- **The envelope partition was being flattened silently.** Two spans batched + under *different* `ScopeSpans` objects with an equal `scope` became + indistinguishable once the payload was read into one list. `§8.3` now carries + the three coordinates in `extensions.otel.envelope`. +- **"Faithful" decoding was described as preserving the attributes.** It + preserves the *values*; the wire form goes. `{"intValue": "7"}` and + `{"intValue": 7}` are the same `7` afterwards. The table row is now + `normalized`, with a record to match. + +The point worth carrying forward: **a guard derived from the models still +encodes a choice about what counts as an absence**, and that choice is not +checked by the guard. Deriving the field list from code removed one class of +blind spot and left another. + +#### Open maintainer decisions + +None of these is answered by the code, and each is a place where two readings +are semantically plausible. They are listed rather than decided, because +deciding one here would put the decision in the hub where every format inherits +it. + +1. **Should the hub model causal structure?** OTLP's span tree is the first + source with real parentage. Today it is preserved verbatim in + `extensions.otel.span`; a `parent` field on `TraceEvent` would make it + canonical, and would oblige every other edge to have a position on it. +2. **Which identifier encoding is canonical?** Hex or base64 — and should this + edge normalize to it, given the two are not distinguishable with certainty? +3. **Is `gen_ai.conversation.id` a BenchFlow `session_id`?** The pinned package + calls it "a conversation (session, thread)", which is close enough to be + tempting and not close enough to be a fact. +4. **Should spans other than `execute_tool` map to typed kinds?** A `chat` span + plausibly corresponds to an agent turn. Doing it needs a position on + `gen_ai.input.messages` / `gen_ai.output.messages`, whose structure the + pinned package defines by reference to a JSON schema it does not ship. +5. **Should a serialized `gen_ai.tool.call.arguments` string be deserialized?** + The pinned text puts that obligation on the instrumentation. A reader that + does it anyway is deciding that a string which happens to be JSON was meant + as structure. +6. **Should a span status map onto `ToolStatus` or `OutcomeStatus`?** See above: + plausible per span, unfounded for a run. +7. **Is depending on `_incubating` conventions acceptable?** Every `gen_ai.*` + name is experimental at 0.62b1 and several are already deprecated. Copying + the constants (as here) keeps the lock file untouched but freezes a snapshot; + importing `opentelemetry-semantic-conventions` would track it and would make + the package a real dependency. +8. **Is an `IR → OTel` emitter wanted at all?** Open question 5, still open for + the outbound half. It needs answers to 1, 2 and 4 before it can be written + without inventing a span tree for BenchFlow rollouts. + +#### Human verification + +**HUMAN E2E VERIFIED, 2026-08-18.** The procedure is `e2e-f/PROCEDURE.md`, and +every step was carried out and judged by a person rather than by the suite: + +| | what was verified | +|---|---| +| F1 | fixture provenance and encoding, regenerated against the pinned `opentelemetry-proto==1.41.1` wheel with a hash matching `uv.lock` | +| F2 | the mapping span by span — identity, parentage, ordering, tool calls, verbatim status retention, timestamp truncation, no trace-level aggregation, no conversation-to-session inference | +| F3 | all 31 loss records read individually; every per-event path resolved | +| F4 | the tri-state `AnyValue` behaviour | +| F4b | the `resourceSpans`/`scopeSpans` partition surviving as coordinates | +| F5 | every negative control biting as intended | +| F5b | the mutation harness: every defined mutation applied and caught, no `MISSED`, no `SKIP`, exit 0, followed by a green `tests/trajectories` run | +| F6 | the IR family still unwired — no external importers, no `opentelemetry` dependency, `uv.lock` unchanged | + +**The verification found two defects, both in the procedure and neither in the +converter.** F3's scaling table and F5's resolvability check were still stating +the counts a pre-review version of the edge produced. Both are corrected, and +both criteria are now written as properties — a constant systemic half and a +constant per-span rate; zero unresolved paths in the canonical encoding and more +than zero under `exclude_none=True` — rather than as numbers that go stale the +next time a record is added. That is the reusable part: **a count in a procedure +is an expectation with a shelf life.** + +Nothing in the converter changed as a result, and no test was weakened to make a +step pass. + +#### Not claimed + +The verification above is bounded by the following. None of them is a formality; +each one names something a reader could otherwise reasonably assume, and none is +narrowed by the fact that a person signed the steps off. + +- **Human verification does not extend past those limits.** What was checked is + that this edge does what §8.3 says it does, on the payloads named above. It is + not evidence about payloads nobody has seen. +- **No OTLP payload emitted by a real agent has ever been read, including + during the human verification.** Nothing in BenchFlow emits spans, so there is + no rollout to check against — F1 verifies the fixture against the *library*, + which is the strongest available substitute and is not the same thing. The + fixture is + the output of `opentelemetry-proto` 1.41.1 itself, which makes its **encoding** + authoritative — base64 ids, `intValue` as a string, enums as member names, + defaults written as absence — and its **content** a construction: the five + spans, their attributes and their shape were chosen by hand, not observed. +- **There is no `IR → OTel` emitter.** This edge is inbound only. Nothing in + this repository writes an OTLP span, and the outbound direction is not + designed, sketched or estimated here. +- **There is no OTel round trip, and none is measured.** §8.10's + `ACP → IR → ATIF → IR′` loop closes because both ATIF edges exist. No loop + closes through OTel, so there is no preservation figure for this edge and none + is implied by the record counts above. +- **`0 dropped` is a measurement of the fixture, not a property of the edge.** + See the table above: two fully conformant inputs produce `DROPPED` records. +- **The mapping is checked against the pinned artifacts, not against the + specification.** Where the OpenTelemetry specification says something those + two packages do not express — the OTLP/JSON hex-identifier convention is the + clearest case — this edge implements nothing and says so. +- **The GenAI semantic conventions are still `_incubating` at the pinned + version**, and several attributes read here are already deprecated in it. A + later version can move the ground under every `gen_ai.*` constant, and no + amount of verification against 0.62b1 prevents that. +- **Nothing is wired.** `ir_from_otel.py` and `_otlp_anyvalue.py` join the + closed family of §8.7; no run path imports either, and no artifact changes + because they exist. diff --git a/src/benchflow/trajectories/_otlp_anyvalue.py b/src/benchflow/trajectories/_otlp_anyvalue.py new file mode 100644 index 000000000..07914c932 --- /dev/null +++ b/src/benchflow/trajectories/_otlp_anyvalue.py @@ -0,0 +1,203 @@ +"""OTLP ``AnyValue`` / ``KeyValue`` decoding — no IR, no loss model, no OTel. + +Split out of :mod:`benchflow.trajectories.ir_from_otel` unchanged. It is the one +part of that edge with no dependency in either direction: it turns the protobuf +JSON attribute wrappers into plain Python values and reports what a plain map +could not hold, and it knows nothing about the canonical IR, about loss records +or about semantic conventions. Keeping it separate makes the edge's remaining +code about mapping rather than about parsing, and leaves the decoder reusable by +a future outbound edge. + +**Faithful is not the same as invertible.** :func:`decode_attributes` reports +``faithful=False`` when a *value* would be lost or collapsed — a duplicate key, +an empty ``AnyValue``, a type this reader cannot name. It says nothing about the +wire *spelling*: the canonical protobuf JSON mapping writes an ``int64`` as a +string, much of the ecosystem writes it as a number, and both decode to the same +Python ``int``. The wrapper type and that spelling are gone from the map either +way. Callers that care declare it; see `docs/trace-interop.md` §8.11. + +Part of the unwired IR family (`docs/trace-interop.md` §8.7): nothing outside it +may import this module. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import field as dataclass_field +from typing import Any, cast + +ANY_VALUE_MEMBERS = frozenset( + { + "stringValue", + "boolValue", + "intValue", + "doubleValue", + "arrayValue", + "kvlistValue", + "bytesValue", + } +) +NON_FINITE_DOUBLES = {"NaN": float("nan"), "Infinity": float("inf")} + + +@dataclass +class Attributes: + """A decoded attribute map, plus whether decoding was faithful.""" + + values: dict[str, Any] = dataclass_field(default_factory=dict) + faithful: bool = True + notes: list[str] = dataclass_field(default_factory=list) + + def text(self, key: str) -> str | None: + """The value at *key* when it is a string, else ``None``.""" + value = self.values.get(key) + return value if isinstance(value, str) else None + + +def decode_any_value(raw: Any) -> tuple[bool, Any]: + """Decode one OTLP ``AnyValue`` wrapper. + + Returns ``(decoded, value)``. ``decoded`` is ``False`` whenever the result + would not be a faithful representation, which the caller turns into a + declared loss *and* into keeping the original list. The false cases are: + + - not an object, or more than one oneof member set (a oneof holds one); + - an **empty** ``AnyValue`` (``{}``) — protobuf's "no member set", which a + plain map cannot distinguish from a key whose value is absent; + - a member name this reader does not know (a future ``AnyValue`` type); + - a malformed payload for a known member — ``intValue`` that is not an + integer, ``arrayValue``/``kvlistValue`` that is not the expected shape, or + a nested value that is itself not decodable. + + ``bytesValue`` decodes to the base64 text verbatim rather than to bytes: the + map has to stay JSON-serializable, and re-encoding is the one thing this + module does not do. It is reported as unfaithful because in the map it is + then indistinguishable from a string attribute. + """ + if not isinstance(raw, dict): + return False, None + members = [key for key in raw if key in ANY_VALUE_MEMBERS] + if len(members) != 1 or len(raw) != len(members): + # Zero members is the empty AnyValue; more than one is not a oneof; an + # extra unknown key is a member this reader cannot name. + return False, None + + member = members[0] + value = raw[member] + + if member == "stringValue": + return (True, value) if isinstance(value, str) else (False, None) + if member == "boolValue": + return (True, value) if isinstance(value, bool) else (False, None) + if member == "bytesValue": + return (False, value) if isinstance(value, str) else (False, None) + if member == "intValue": + # int64 is a JSON string in the canonical mapping and a JSON number in + # much of the wild; both are accepted by the pinned parser. + if isinstance(value, bool): + return False, None + if isinstance(value, int): + return True, value + if isinstance(value, str): + try: + return True, int(value) + except ValueError: + return False, None + return False, None + if member == "doubleValue": + if isinstance(value, bool): + return False, None + if isinstance(value, (int, float)): + return True, float(value) + if isinstance(value, str): + # The canonical mapping spells the non-finite doubles as strings. + if value in NON_FINITE_DOUBLES: + return True, NON_FINITE_DOUBLES[value] + if value.startswith("-") and value[1:] in NON_FINITE_DOUBLES: + return True, -NON_FINITE_DOUBLES[value[1:]] + return False, None + return False, None + if member == "arrayValue": + if not isinstance(value, dict): + return False, None + items = value.get("values", []) + if not isinstance(items, list): + return False, None + decoded: list[Any] = [] + for item in items: + ok, inner = decode_any_value(item) + if not ok: + return False, None + decoded.append(inner) + return True, decoded + # kvlistValue + if not isinstance(value, dict): + return False, None + pairs = value.get("values", []) + if not isinstance(pairs, list): + return False, None + mapping: dict[str, Any] = {} + for pair in pairs: + if not isinstance(pair, dict): + return False, None + key = pair.get("key") + if not isinstance(key, str) or key in mapping: + return False, None + ok, inner = decode_any_value(pair.get("value")) + if not ok: + return False, None + mapping[key] = inner + return True, mapping + + +def decode_attributes(raw: Any) -> Attributes: + """Decode an OTLP ``KeyValue`` list into a map, tracking what that cost. + + OTLP models attributes as a *list*, so duplicate keys are representable and + a map cannot hold them. That, and every case :func:`decode_any_value` + refuses, sets :attr:`Attributes.faithful` to ``False`` — which is the + caller's signal to keep the original list beside the map and declare the + collapse. + """ + result = Attributes() + if raw is None: + return result + if not isinstance(raw, list): + result.faithful = False + result.notes.append( + f"attributes is {type(raw).__name__}, not a list of KeyValue objects" + ) + return result + + for position, entry in enumerate(raw): + if not isinstance(entry, dict): + result.faithful = False + result.notes.append(f"attributes[{position}] is not a JSON object") + continue + # ``cast`` only: ``isinstance`` above is the real check, but the + # narrowed element type is not assignable to an invariant dict. + pair = cast(dict[str, Any], entry) + key = pair.get("key") + if not isinstance(key, str): + result.faithful = False + result.notes.append(f"attributes[{position}] has no string key") + continue + if key in result.values: + result.faithful = False + result.notes.append( + f"attributes[{position}] repeats key {key!r}; a map keeps one" + ) + if "value" not in pair: + result.faithful = False + result.notes.append(f"attributes[{position}] ({key!r}) carries no value") + result.values[key] = None + continue + ok, value = decode_any_value(pair["value"]) + if not ok: + result.faithful = False + result.notes.append( + f"attributes[{position}] ({key!r}) is an AnyValue this reader " + "cannot represent faithfully in a map" + ) + result.values[key] = value + return result diff --git a/src/benchflow/trajectories/ir_from_otel.py b/src/benchflow/trajectories/ir_from_otel.py new file mode 100644 index 000000000..1a33a438e --- /dev/null +++ b/src/benchflow/trajectories/ir_from_otel.py @@ -0,0 +1,1333 @@ +"""OTLP/JSON spans → canonical Trace IR — the inbound OpenTelemetry edge. + +> **PROVISIONAL.** Companion to :mod:`benchflow.trajectories.ir`, itself an +> unapproved proposal (`docs/trace-interop.md` §8). Nothing imports this module, +> nothing writes its output to disk, no run path changes because it exists, and +> **no `IR → OTel` emitter ships with it.** This is one direction of one edge. + +`§4.2` is FACT: this repository has no OpenTelemetry code and no OTel runtime +dependency. What it *does* have is a lock file, and that is what this module is +built against rather than recollection. + +## What is pinned, and what that buys + +``uv.lock`` resolves ``opentelemetry-proto==1.41.1`` and +``opentelemetry-semantic-conventions==0.62b1`` as transitive dependencies of +``daytona`` (the ``sandbox-daytona`` extra). Neither is a dependency of this +module — nothing here imports ``opentelemetry`` and the lock is untouched — but +both are *artifacts this repository already names, at a version, with a hash*. +So the two questions an OTel reader has to answer have checkable answers: + +- **the wire shape** — field names, JSON names and types — comes from the + ``ExportTraceServiceRequest`` descriptor in ``opentelemetry-proto`` 1.41.1; +- **the attribute vocabulary** — every ``gen_ai.*`` name below — comes from + ``opentelemetry.semconv._incubating.attributes.gen_ai_attributes`` in + ``opentelemetry-semantic-conventions`` 0.62b1. + +Both versions are recorded in :data:`OTLP_PROTO_VERSION` and +:data:`SEMCONV_VERSION` so a reader can re-derive every constant here. The +mapping is *not* written against the published specification text, which is not +vendored: where the specification says something the pinned artifacts do not, +this module does nothing and says so. + +**The GenAI conventions are `_incubating` even at the pinned version** — the +package puts them under that namespace, which is its own statement that the +names are not stable. Several are already marked deprecated in 0.62b1 +(``gen_ai.system``, ``gen_ai.usage.prompt_tokens``, ``gen_ai.prompt``, +``gen_ai.completion``). This module reads the deprecated spellings, because the +pinned package documents the replacement relationship itself, and declares each +such read :attr:`~benchflow.trajectories.ir.LossClass.NORMALIZED`. + +## The rule this module is built on + +**A span is evidence of an operation, not a statement about an agent.** + +OTLP is a general tracing format. A payload of spans says what was instrumented, +when, and under what identifiers; it does not say who spoke, what a turn was, or +how a conversation was structured. So this edge maps exactly one span shape onto +a typed IR kind — a span whose ``gen_ai.operation.name`` is ``execute_tool``, +which the pinned vocabulary defines with that meaning — and every other span +becomes :attr:`~benchflow.trajectories.ir.EventKind.UNKNOWN` with its whole +content carried verbatim. + +That is deliberately less than a plausible reading would allow. A ``chat`` span +*probably* corresponds to an agent turn; ``gen_ai.input.messages`` *probably* +holds the user text. Both are guesses about intent, both would put invented +agent semantics into the hub where every other format would inherit them, and +the second one additionally depends on a JSON schema the pinned package +references by relative path and does not ship. They are listed in +`docs/trace-interop.md` §8.11 as open maintainer decisions, not implemented. + +## Order is order; it is not causality + +The IR event list is ordered and dense (invariant 2), and this edge preserves +**document order** — the order the spans appear in the payload — and nothing +else. Spans are *not* sorted by start time. + +Sorting would be the converter asserting a causal or logical sequence that OTLP +does not carry: sibling spans overlap, a partial batch may omit a parent, and +two spans with the same start instant have no defined order at all. The real +structure is the ``parentSpanId`` edge set, and it is preserved exactly, per +span, in ``extensions.otel.span`` — together with ``spanId``, ``traceId``, +``traceState``, ``flags`` and both timestamps. Nothing that expresses causality +is dropped, reordered, or turned into an ordering claim. + +**The IR models no parent link**, so parentage lives in ``extensions`` rather +than in a canonical field. Adding one would be a change to the hub, which is a +maintainer's decision and not this slice's; §8.11 records it as such. + +## Batches, and why the entry points are shaped as they are + +An OTLP export request is a *batch*. It may carry spans from several traces, it +need not contain a trace's root span, and the same trace may arrive across +several requests. "One payload is one run" is therefore false, and an API that +implied it would be wrong at the boundary rather than merely imprecise. + +So reading is split into the three things that actually happen: + +- :func:`otlp_json_spans` walks the envelope and returns the spans it found, + each with its resource and scope context, plus a report addressed to the + *payload* — an envelope is not a trace, so its defects are not a trace's + losses, and that report is returned rather than attached. +- :func:`group_spans_by_trace_id` groups by trace id, in first-appearance order. +- :func:`otel_spans_to_ir` converts one group into one + :class:`~benchflow.trajectories.ir.CanonicalTrace` with its report attached, + which is the inbound convention `ir.py` describes. + +:func:`otlp_json_to_ir` composes the three for callers that want a payload in +and traces out. + +## What is carried but not canonicalized + +Everything OTel-specific with no stable IR meaning rides in +``extensions.otel``: the span verbatim (minus its attributes), the decoded +attribute map, the resource and scope with their schema URLs. That is the IR's +own contract for ``extensions`` — "source fields with no IR home, carried +verbatim so a conversion is not forced to choose between dropping a value and +growing the IR for it". + +Attributes are decoded from OTLP's ``KeyValue``/``AnyValue`` wrappers into a +plain map because a map is what a consumer wants. Decoding can lose something — +duplicate keys, an empty ``AnyValue``, a value type this reader does not know — +and whenever it does, the original list is kept beside the map as +``attributes_raw`` and the collapse is declared. The map is a convenience; the +list is the record. + +## Identifiers are never re-encoded + +``traceId`` and ``spanId`` are ``bytes`` in the proto, and the protobuf canonical +JSON mapping encodes bytes as base64 — verified against the pinned package, +which round-trips a 16-byte id as a 24-character base64 string. Much of the OTel +ecosystem writes them as lowercase hex instead, and the two are **not reliably +distinguishable**: feeding a 32-character hex id to the pinned JSON parser is +accepted as base64 and silently yields 24 bytes. + +This module therefore carries the id **exactly as written**, as a string, and +re-encodes nothing. Which encoding a BenchFlow trace id should canonically use +is a maintainer's decision (§8.11); guessing it here would make identity depend +on a heuristic. + +## What OTLP/JSON cannot tell this reader + +Protobuf scalar fields without presence — ``name``, ``startTimeUnixNano``, +``parentSpanId``, ``kind``, ``droppedAttributesCount`` — serialize to nothing +when they hold the default, so **absent and default are the same document**. +Verified against the pinned descriptors: ``has_presence`` is ``False`` for all +of them. The IR's tri-state rule (value / ``None`` / observed-empty) is real +here only for fields OTLP models with presence, and the limit is declared once +per conversion rather than left implicit. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any + +from benchflow.trajectories._otlp_anyvalue import ( + Attributes, + decode_attributes, +) +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlock, + ContentBlockKind, + EventKind, + LossClass, + LossReport, + ModelInfo, + PathSpace, + Provenance, + ToolCall, + TraceEvent, + TraceOutcome, + TraceUsage, +) + +LOSS_DIRECTION = "otel->ir" + +OTEL_SOURCE = "otel" +"""Per-event provenance: the value the IR's :class:`Provenance` already names.""" + +OTLP_JSON_SOURCE = "otlp-json" +"""Trace-level provenance: the wire encoding actually read, since a trace's +spans may come from different instrumentation scopes.""" + +OTLP_PROTO_VERSION = "1.41.1" +"""``opentelemetry-proto`` version whose descriptors define the wire shape below. + +Pinned in this repository's ``uv.lock`` (transitively, via ``daytona``). Not +imported: this module reads JSON dictionaries and takes no OTel dependency. +""" + +SEMCONV_VERSION = "0.62b1" +"""``opentelemetry-semantic-conventions`` version every ``GEN_AI_*`` name below +was copied from, likewise pinned in ``uv.lock`` and likewise not imported. + +The GenAI group is ``_incubating`` at this version — the package's own statement +that the names are unstable. +""" + +USAGE_SOURCE = "otel_gen_ai_usage" +"""Value written to :attr:`~benchflow.trajectories.ir.TraceUsage.source`. + +Counters read off ``gen_ai.usage.*`` are not the same measurement as BenchFlow's +``llm_proxy_normalized`` or ``acp_session_snapshot``; naming the definition is +how the IR keeps a consumer from comparing unlike numbers. +""" + +TOOL_NAME_SEMANTICS = "gen_ai.tool.name" +"""What a tool call's :attr:`~benchflow.trajectories.ir.ToolCall.name` is when +it came from this edge — neither an ACP ``kind`` nor an ATIF ``function_name``. +""" + +# --- semantic-convention attribute names, copied from the pinned package ------ +# Every constant below is the literal value of the same-named symbol in +# ``opentelemetry.semconv._incubating.attributes.gen_ai_attributes`` at +# ``SEMCONV_VERSION``. The comments record what that package says about them. + +GEN_AI_OPERATION_NAME = "gen_ai.operation.name" +GEN_AI_AGENT_NAME = "gen_ai.agent.name" +GEN_AI_AGENT_VERSION = "gen_ai.agent.version" +GEN_AI_PROVIDER_NAME = "gen_ai.provider.name" +GEN_AI_SYSTEM = "gen_ai.system" # deprecated: replaced by gen_ai.provider.name +GEN_AI_REQUEST_MODEL = "gen_ai.request.model" +GEN_AI_RESPONSE_MODEL = "gen_ai.response.model" +GEN_AI_CONVERSATION_ID = "gen_ai.conversation.id" + +GEN_AI_TOOL_NAME = "gen_ai.tool.name" +GEN_AI_TOOL_CALL_ID = "gen_ai.tool.call.id" +GEN_AI_TOOL_CALL_ARGUMENTS = "gen_ai.tool.call.arguments" +GEN_AI_TOOL_CALL_RESULT = "gen_ai.tool.call.result" + +GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens" +GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" +GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read.input_tokens" +GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = "gen_ai.usage.cache_creation.input_tokens" +GEN_AI_USAGE_PROMPT_TOKENS = "gen_ai.usage.prompt_tokens" # deprecated → input +GEN_AI_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens" # deprecated → output + +GEN_AI_INPUT_MESSAGES = "gen_ai.input.messages" +GEN_AI_OUTPUT_MESSAGES = "gen_ai.output.messages" +GEN_AI_PROMPT = "gen_ai.prompt" # deprecated: removed, no replacement +GEN_AI_COMPLETION = "gen_ai.completion" # deprecated: removed, no replacement + +OPERATION_EXECUTE_TOOL = "execute_tool" +"""``GenAiOperationNameValues.EXECUTE_TOOL`` — the one operation this edge maps +onto a typed IR kind, because the pinned vocabulary defines it as exactly that. +""" + +_CONTENT_ATTRIBUTES: tuple[str, ...] = ( + GEN_AI_INPUT_MESSAGES, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROMPT, + GEN_AI_COMPLETION, +) +"""Attributes that carry conversation text. None is mapped onto ``text``; see +the module docstring and §8.11.""" + +# ``final_metrics``-style pairing, but for spans: the IR usage field each pinned +# attribute feeds, and whether reading it is a deprecated spelling. +_USAGE_ATTRIBUTES: tuple[tuple[str, str, str | None], ...] = ( + (GEN_AI_USAGE_INPUT_TOKENS, "input_tokens", None), + (GEN_AI_USAGE_OUTPUT_TOKENS, "output_tokens", None), + (GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, "cache_read_tokens", None), + (GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, "cache_creation_tokens", None), + (GEN_AI_USAGE_PROMPT_TOKENS, "input_tokens", GEN_AI_USAGE_INPUT_TOKENS), + (GEN_AI_USAGE_COMPLETION_TOKENS, "output_tokens", GEN_AI_USAGE_OUTPUT_TOKENS), +) + +_USAGE_UNSUPPORTED: tuple[str, ...] = ( + "total_tokens", + "reasoning_tokens", + "cost_usd", + "price_source", +) +"""IR usage fields with no attribute in the pinned GenAI vocabulary. + +``gen_ai.usage.total_tokens`` is worth naming explicitly: the deleted +``OTelCollector`` (§4.2) read it, and it does not exist in ``SEMCONV_VERSION``. +""" + +_USAGE_PREFERRED: tuple[tuple[str, str], ...] = ( + ("input_tokens", GEN_AI_USAGE_INPUT_TOKENS), + ("output_tokens", GEN_AI_USAGE_OUTPUT_TOKENS), + ("cache_read_tokens", GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS), + ("cache_creation_tokens", GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS), +) +"""IR usage fields this edge *can* fill, and the attribute each one reads. + +Distinct from :data:`_USAGE_UNSUPPORTED`, and the difference is the point: a +field here that stays empty is a value this payload did not carry, while one +there is a value no OTLP payload can carry. Both are declared; only the second +is a property of the format. +""" + +# ``AnyValue`` oneof members, by their protobuf JSON names. +_EPOCH = datetime(1970, 1, 1, tzinfo=UTC) + +_SPAN_ATTRIBUTES_KEY = "attributes" + + +@dataclass(frozen=True) +class OtlpSpan: + """One span with the envelope context it was found under. + + A plain frozen dataclass rather than a model: nothing here validates, + coerces or copies its input. The dictionaries are the caller's, verbatim, + which is what lets :func:`otel_spans_to_ir` promise it carried them + unchanged. + + ``resource`` and ``scope`` are kept per span, not per conversion, because + one payload legitimately mixes them and a trace-level answer would be wrong + for some of its own spans — the same reason the IR carries + :class:`~benchflow.trajectories.ir.Provenance` per event. + + The three positions record **where in the envelope the span was found**. + Flattening ``resourceSpans[] → scopeSpans[] → spans[]`` into one list + otherwise destroys the partition: two spans under *different* ``ScopeSpans`` + objects that happen to carry an equal ``scope`` become indistinguishable, + and the payload said they were separately batched. Carrying the coordinates + keeps that recoverable without the IR growing a concept of an envelope. + + They are ``None`` when the caller built the span itself rather than reading + it out of a payload — then there is no partition to preserve, and inventing + coordinates would claim an envelope that never existed. + """ + + span: dict[str, Any] + resource: dict[str, Any] | None = None + resource_schema_url: str | None = None + scope: dict[str, Any] | None = None + scope_schema_url: str | None = None + resource_index: int | None = None + scope_index: int | None = None + span_index: int | None = None + + +# --------------------------------------------------------------------------- +# Envelope +# --------------------------------------------------------------------------- + + +def otlp_json_spans(payload: dict[str, Any]) -> tuple[list[OtlpSpan], LossReport]: + """Flatten one OTLP/JSON export request into spans plus envelope context. + + The structure walked is ``resourceSpans[] → scopeSpans[] → spans[]``, the + JSON names of ``ExportTraceServiceRequest`` at :data:`OTLP_PROTO_VERSION`. + Nothing is interpreted here: a span is returned as the dictionary it was. + + The returned :class:`~benchflow.trajectories.ir.LossReport` is addressed to + the **payload**, in :attr:`~benchflow.trajectories.ir.PathSpace.SOURCE`, and + is *returned* rather than attached to anything. A malformed + ``resourceSpans`` entry is not any trace's loss — the spans it would have + contained are unidentifiable, so there is no trace to attach it to. This is + the one place an inbound edge in this family hands a report back, and the + reason is that an envelope is not a trace. + + Raises ``TypeError`` for a non-mapping payload: there is no partial reading + of something that is not an export request, and returning an empty list + would claim a payload was read. + """ + if not isinstance(payload, dict): + raise TypeError(f"OTLP payload must be a mapping, got {type(payload).__name__}") + + losses = LossReport(direction=LOSS_DIRECTION) + spans: list[OtlpSpan] = [] + + for resource_position, resource_spans in enumerate( + _entries(payload, "resourceSpans", "resourceSpans", losses) + ): + where_resource = f"resourceSpans[{resource_position}]" + if not isinstance(resource_spans, dict): + _not_an_object(losses, where_resource, resource_spans) + continue + resource = resource_spans.get("resource") + resource = resource if isinstance(resource, dict) else None + resource_schema_url = _schema_url(resource_spans) + + for scope_position, scope_spans in enumerate( + _entries( + resource_spans, + "scopeSpans", + f"{where_resource}.scopeSpans", + losses, + ) + ): + where_scope = f"{where_resource}.scopeSpans[{scope_position}]" + if not isinstance(scope_spans, dict): + _not_an_object(losses, where_scope, scope_spans) + continue + scope = scope_spans.get("scope") + scope = scope if isinstance(scope, dict) else None + + for span_position, span in enumerate( + _entries(scope_spans, "spans", f"{where_scope}.spans", losses) + ): + if not isinstance(span, dict): + _not_an_object( + losses, f"{where_scope}.spans[{span_position}]", span + ) + continue + spans.append( + OtlpSpan( + span=span, + resource=resource, + resource_schema_url=resource_schema_url, + scope=scope, + scope_schema_url=_schema_url(scope_spans), + resource_index=resource_position, + scope_index=scope_position, + span_index=span_position, + ) + ) + + return spans, losses + + +def _entries( + holder: dict[str, Any], key: str, where: str, losses: LossReport +) -> list[Any]: + """The list at *key*, declaring the case where it is present but not one.""" + raw = holder.get(key) + if isinstance(raw, list): + return raw + if raw is not None: + losses.add( + where, + LossClass.DROPPED, + f"{key} is {type(raw).__name__}, not a list; nothing could be read from it", + space=PathSpace.SOURCE, + ) + return [] + + +def _not_an_object(losses: LossReport, where: str, value: Any) -> None: + losses.add( + where, + LossClass.DROPPED, + f"entry is {type(value).__name__}, not a JSON object; it carries no span " + "this reader can address", + space=PathSpace.SOURCE, + ) + + +def _schema_url(holder: dict[str, Any]) -> str | None: + value = holder.get("schemaUrl") + return value if isinstance(value, str) else None + + +def group_spans_by_trace_id( + spans: list[OtlpSpan], +) -> list[tuple[str | None, list[OtlpSpan]]]: + """Group *spans* by trace id, preserving first-appearance order. + + The key is the ``traceId`` string **exactly as written** — see the module + docstring on identifier encodings — or ``None`` for a span that carries no + string trace id. Spans with no id are grouped together rather than merged + into an identified trace: they may belong to it, and "may" is not a fact + this reader gets to record. + + Order is preserved twice over: groups appear in the order their first span + appeared, and each group's spans keep their relative document order. + """ + groups: dict[str | None, list[OtlpSpan]] = {} + for entry in spans: + raw = entry.span.get("traceId") + key = raw if isinstance(raw, str) else None + groups.setdefault(key, []).append(entry) + return list(groups.items()) + + +# --------------------------------------------------------------------------- +# Timestamps +# --------------------------------------------------------------------------- + + +def _timestamp(span: dict[str, Any], key: str) -> tuple[datetime | None, str | None]: + """Read a ``fixed64`` nanosecond field as a datetime, plus what it cost. + + Returns ``(value, note)``; *note* is ``None`` when nothing needs declaring. + + Two losses are real here and neither is avoidable: + + - **Absent is default.** ``startTimeUnixNano`` has no presence in the proto + (verified against the pinned descriptors), so a producer that never set it + and one that set it to 0 write the same document. Both read as ``None``. + - **Nanoseconds do not fit.** ``datetime`` resolves to microseconds, so a + timestamp that is not a whole number of microseconds is truncated. The + exact integer stays in ``extensions.otel.span``, so the value is not lost + — but the IR field no longer holds it, and that is a normalization. + """ + if key not in span: + return None, ( + f"{key} is absent; OTLP models it as a fixed64 with no presence, so " + "an unset field and a 0 are the same document and no instant can be " + "read from either" + ) + raw = span[key] + if isinstance(raw, bool) or not isinstance(raw, (int, str)): + return None, ( + f"{key} is {type(raw).__name__}; OTLP encodes fixed64 as a JSON " + "string or number" + ) + try: + nanos = int(raw) + except ValueError: + return None, f"{key} is {raw!r}, which is not an integer nanosecond count" + if nanos < 0: + return None, ( + f"{key} is negative and fixed64 is unsigned, so no conformant " + "producer wrote it" + ) + if nanos == 0: + return None, ( + f"{key} is 0, the protobuf default for a field with no presence; the " + "document cannot say whether the producer set it" + ) + micros, remainder = divmod(nanos, 1000) + value = _EPOCH + timedelta(microseconds=micros) + if remainder: + return value, ( + f"{key} is {nanos} ns and datetime resolves to microseconds, so " + f"{remainder} ns is truncated; the exact integer is kept in " + "extensions.otel.span" + ) + return value, None + + +# --------------------------------------------------------------------------- +# One trace +# --------------------------------------------------------------------------- + + +def otel_spans_to_ir( + spans: list[OtlpSpan], + *, + session_id: str | None = None, +) -> CanonicalTrace: + """Convert one group of OTLP spans into a :class:`CanonicalTrace`. + + *spans* is normally one trace's worth — see :func:`group_spans_by_trace_id` + — but nothing requires it: a list mixing trace ids converts, and the + disagreement is declared rather than silently resolved. + + *session_id* is context the caller has from elsewhere, exactly as in + :func:`~benchflow.trajectories.ir_from_acp.acp_events_to_ir`. It is not read + from the spans: ``gen_ai.conversation.id`` is a candidate and is preserved + in the attribute map, but whether a conversation id *is* a BenchFlow session + id is a maintainer's decision (§8.11), and answering it here would put the + answer in the hub. + + One span becomes exactly one event, in document order, so indices stay dense + (invariant 2) and no span is skipped. The returned trace carries its own + report on :attr:`~benchflow.trajectories.ir.CanonicalTrace.losses` and + satisfies :func:`~benchflow.trajectories.ir.validate_trace` for any input, + including payloads no conformant producer would write. + """ + losses = LossReport(direction=LOSS_DIRECTION) + events: list[TraceEvent] = [] + agent = _AgentAccumulator() + + for entry in spans: + events.append(_span_to_event(entry, len(events), agent, losses)) + + trace_id = _trace_id(spans, losses) + _declare_trace_losses( + losses, + events=events, + session_id=session_id, + has_tool_call=any(event.kind is EventKind.TOOL_CALL for event in events), + ) + agent.declare(losses) + + extensions: dict[str, Any] = {} + other_ids = _distinct_trace_ids(spans) + if len(other_ids) > 1: + extensions["otel"] = {"trace_ids": other_ids} + + return CanonicalTrace( + trace_id=trace_id, + session_id=session_id, + agent=agent.model_info(), + events=events, + # Never derived. Every span's own extent is preserved on its event and + # in extensions; computing a run's extent from them would be this + # converter asserting that the payload holds the whole run, which a + # batch does not promise. Declared below. + started_at=None, + finished_at=None, + usage=None, + # Always present, never populated: OTLP has no run-outcome concept, and + # per-span status is preserved per event. A record addresses this + # section by path, so the section has to exist. + outcome=TraceOutcome(), + provenance=Provenance(source_format=OTLP_JSON_SOURCE), + extensions=extensions, + losses=losses, + ) + + +def otlp_json_to_ir( + payload: dict[str, Any], +) -> tuple[list[CanonicalTrace], LossReport]: + """Read one OTLP/JSON export request into one trace per trace id. + + Returns the traces — each with its own report attached — and the + **envelope** report from :func:`otlp_json_spans`, which belongs to the + payload rather than to any trace. + + A payload carrying no readable span yields an empty list. That is a claim, + not a failure: the envelope report says what was in the way. + """ + spans, envelope = otlp_json_spans(payload) + traces = [otel_spans_to_ir(group) for _, group in group_spans_by_trace_id(spans)] + return traces, envelope + + +class _AgentAccumulator: + """Trace-level agent identity, gathered from span attributes. + + OTel puts identity on spans; the IR puts it on the trace. The first + observed value for each field wins, in document order, and a *different* + later value is kept and declared rather than overwritten silently — a + payload whose spans disagree about the model is saying something, and + picking one quietly would erase it. + """ + + _FIELDS = ("agent_name", "agent_version", "model", "provider") + + def __init__(self) -> None: + self.values: dict[str, str] = {} + self.conflicts: dict[str, list[str]] = {} + self.deprecated: set[str] = set() + + def offer(self, field: str, value: str | None, *, deprecated: bool = False) -> None: + if value is None: + return + if field not in self.values: + self.values[field] = value + if deprecated: + self.deprecated.add(field) + return + if self.values[field] != value: + self.conflicts.setdefault(field, []).append(value) + return + if not deprecated: + # A later span agreed, using the current spelling. The value did not + # come only from the deprecated one after all, so the normalization + # record would be false. + self.deprecated.discard(field) + + def model_info(self) -> ModelInfo: + return ModelInfo(**self.values) + + def declare(self, losses: LossReport) -> None: + for field in self._FIELDS: + if field in self.values: + continue + losses.add( + f"agent.{field}", + LossClass.UNSUPPORTED, + "no span in this trace carries the attribute this field reads " + f"(semantic conventions {SEMCONV_VERSION})", + ) + if "provider" in self.deprecated: + losses.add( + "agent.provider", + LossClass.NORMALIZED, + f"read from the deprecated {GEN_AI_SYSTEM!r}; the pinned package " + f"marks it replaced by {GEN_AI_PROVIDER_NAME!r}", + ) + for field, extra in sorted(self.conflicts.items()): + losses.add( + f"agent.{field}", + LossClass.DROPPED, + f"spans disagree: the first value is kept and {extra!r} " + "is not representable in a trace-level agent block", + ) + + +def _trace_id(spans: list[OtlpSpan], losses: LossReport) -> str | None: + """The trace id, verbatim, or ``None`` with the reason declared.""" + ids = _distinct_trace_ids(spans) + non_string = [ + position + for position, entry in enumerate(spans) + if "traceId" in entry.span and not isinstance(entry.span["traceId"], str) + ] + for position in non_string: + losses.add( + f"spans[{position}].traceId", + LossClass.DROPPED, + "traceId is not a string; OTLP/JSON encodes the id as text and this " + "reader does not re-encode, so it cannot be read", + space=PathSpace.SOURCE, + ) + if not ids: + losses.add( + "trace_id", + LossClass.UNSUPPORTED, + "no span in this group carries a string traceId", + ) + return None + if len(ids) > 1: + losses.add( + "trace_id", + LossClass.NORMALIZED, + f"the group spans {len(ids)} trace ids; the first is kept and all of " + "them are listed in extensions.otel.trace_ids", + ) + return ids[0] + + +def _distinct_trace_ids(spans: list[OtlpSpan]) -> list[str]: + seen: list[str] = [] + for entry in spans: + raw = entry.span.get("traceId") + if isinstance(raw, str) and raw not in seen: + seen.append(raw) + return seen + + +def _span_to_event( + entry: OtlpSpan, + index: int, + agent: _AgentAccumulator, + losses: LossReport, +) -> TraceEvent: + """One span as one IR event.""" + span = entry.span + where = f"events[{index}]" + attributes = decode_attributes(span.get(_SPAN_ATTRIBUTES_KEY)) + + agent.offer("agent_name", attributes.text(GEN_AI_AGENT_NAME)) + agent.offer("agent_version", attributes.text(GEN_AI_AGENT_VERSION)) + agent.offer("model", attributes.text(GEN_AI_REQUEST_MODEL)) + provider = attributes.text(GEN_AI_PROVIDER_NAME) + if provider is not None: + agent.offer("provider", provider) + else: + agent.offer("provider", attributes.text(GEN_AI_SYSTEM), deprecated=True) + + started_at, started_note = _timestamp(span, "startTimeUnixNano") + finished_at, finished_note = _timestamp(span, "endTimeUnixNano") + for note, field in ((started_note, "started_at"), (finished_note, "finished_at")): + if note is not None: + losses.add(f"{where}.{field}", LossClass.NORMALIZED, note) + + if not attributes.faithful: + losses.add( + f"{where}.extensions", + LossClass.NORMALIZED, + "the attribute list does not fit a map without loss (" + + "; ".join(attributes.notes) + + "); the original list is kept as extensions.otel.attributes_raw", + ) + _declare_dropped_counts(span, where, losses) + + status = span.get("status") + if _has_status(status): + losses.add( + f"{where}.outcome", + LossClass.NORMALIZED, + "the span carries a status; the IR outcome slot is free text with no " + "vocabulary an OTLP status code maps into, so the status object is " + "kept verbatim in extensions.otel.span.status", + ) + + present_content = [key for key in _CONTENT_ATTRIBUTES if key in attributes.values] + if present_content: + losses.add( + f"{where}.text", + LossClass.NORMALIZED, + "conversation content is carried in extensions.otel.attributes (" + + ", ".join(present_content) + + ") rather than in text; the pinned package defines the message " + "structure by reference to a JSON schema it does not ship", + ) + + usage = _read_usage(attributes, where, losses) + tool_call = None + kind = EventKind.UNKNOWN + if attributes.text(GEN_AI_OPERATION_NAME) == OPERATION_EXECUTE_TOOL: + kind = EventKind.TOOL_CALL + tool_call = _read_tool_call( + attributes, + where=where, + started_at=started_at, + finished_at=finished_at, + timestamp_notes={ + "started_at": started_note, + "finished_at": finished_note, + }, + has_status=_has_status(status), + losses=losses, + ) + + name = span.get("name") + return TraceEvent( + index=index, + kind=kind, + # The span name, verbatim. It is the only thing a non-GenAI span says + # about what it is, and ``gen_ai.operation.name`` — the typed answer + # when there is one — stays readable in the attribute map. + source_type=name if isinstance(name, str) else None, + # Never set. OTLP attributes nothing to a speaker, and the IR's Role + # members are the ones some source in this repository distinguishes. + role=None, + text=None, + reasoning=None, + tool_call=tool_call, + started_at=started_at, + finished_at=finished_at, + outcome=None, + usage=usage, + provenance=Provenance( + source_format=OTEL_SOURCE, + producer=_scope_name(entry), + ), + extensions={"otel": _otel_extensions(entry, attributes)}, + ) + + +def _scope_name(entry: OtlpSpan) -> str | None: + """The instrumentation scope's name — the closest OTLP has to an emitter.""" + if entry.scope is None: + return None + name = entry.scope.get("name") + return name if isinstance(name, str) else None + + +def _has_status(status: Any) -> bool: + """True when the span carries a status other than the unset default. + + ``STATUS_CODE_UNSET`` is 0, the protobuf default, so a status object holding + only that says nothing a conformant producer had to write. Both the enum + name and the integer are accepted: the pinned JSON mapping emits names by + default and integers under ``use_integers_for_enums``, and parses either. + """ + if not isinstance(status, dict): + return False + code = status.get("code") + if code in (None, 0, "STATUS_CODE_UNSET"): + return bool(status.get("message")) + return True + + +def _declare_dropped_counts( + span: dict[str, Any], where: str, losses: LossReport +) -> None: + """Declare what the *producer* dropped before this reader ever saw it. + + OTLP's ``dropped*Count`` fields are the SDK stating that it discarded data + to stay inside a limit. That is the cleanest possible + :attr:`~benchflow.trajectories.ir.LossClass.UNSUPPORTED`: the value never + reached the document, so no converter can recover it and the fix is a + producer-side limit, not code here. + """ + for key, what in ( + ("droppedAttributesCount", "attributes"), + ("droppedEventsCount", "span events"), + ("droppedLinksCount", "links"), + ): + count = span.get(key) + if isinstance(count, bool) or not isinstance(count, int) or count <= 0: + continue + losses.add( + f"{where}.extensions", + LossClass.UNSUPPORTED, + f"the producer reports {key}={count}: that many {what} were " + "discarded by the emitting SDK and are not in the document", + ) + + +def _otel_extensions(entry: OtlpSpan, attributes: Attributes) -> dict[str, Any]: + """Everything OTel-specific, carried verbatim under one key. + + The span goes in whole, minus its attribute list, so a field this reader has + never heard of — a future ``Span`` member — rides along instead of being + truncated to what the mapping happens to know. The decoded attribute map + sits beside it, and the original list joins them whenever decoding was not + faithful. + + ``envelope`` holds the span's coordinates in the payload it was read from. + That is what keeps the ``resourceSpans``/``scopeSpans`` partition from being + silently flattened: two spans whose ``resource`` and ``scope`` objects are + equal but which the producer batched separately differ here and nowhere + else. Absent when the caller did not read the span out of a payload. + """ + carried: dict[str, Any] = { + "span": { + key: value + for key, value in entry.span.items() + if key != _SPAN_ATTRIBUTES_KEY + }, + "attributes": attributes.values, + } + if not attributes.faithful: + carried["attributes_raw"] = entry.span.get(_SPAN_ATTRIBUTES_KEY) + envelope = { + name: index + for name, index in ( + ("resource_spans_index", entry.resource_index), + ("scope_spans_index", entry.scope_index), + ("span_index", entry.span_index), + ) + if index is not None + } + if envelope: + carried["envelope"] = envelope + if entry.resource is not None: + carried["resource"] = entry.resource + if entry.resource_schema_url is not None: + carried["resource_schema_url"] = entry.resource_schema_url + if entry.scope is not None: + carried["scope"] = entry.scope + if entry.scope_schema_url is not None: + carried["scope_schema_url"] = entry.scope_schema_url + return carried + + +def _read_usage( + attributes: Attributes, where: str, losses: LossReport +) -> TraceUsage | None: + """Per-span token counters from the pinned ``gen_ai.usage.*`` attributes. + + Reading a deprecated spelling is declared: the pinned package states the + replacement itself, so following it is supported rather than assumed, but a + consumer should still know which name the number came from. A non-integer + value is declared and dropped rather than coerced — a token count that is + not a count is a malformed document, not a number to salvage. + """ + values: dict[str, Any] = {} + for attribute, ir_field, replaced_by in _USAGE_ATTRIBUTES: + if attribute not in attributes.values: + continue + raw = attributes.values[attribute] + if isinstance(raw, bool) or not isinstance(raw, int): + losses.add( + f"{where}.usage.{ir_field}", + LossClass.DROPPED, + f"{attribute} is {type(raw).__name__}, not an integer token count", + ) + continue + if ir_field in values: + # Both the current and the deprecated spelling are present. The + # current one is read first by the order of _USAGE_ATTRIBUTES. + if values[ir_field] != raw: + losses.add( + f"{where}.usage.{ir_field}", + LossClass.DROPPED, + f"{attribute} disagrees with the preferred attribute already " + f"read for this field ({raw} vs {values[ir_field]}); the " + "preferred one is kept", + ) + continue + values[ir_field] = raw + if replaced_by is not None: + losses.add( + f"{where}.usage.{ir_field}", + LossClass.NORMALIZED, + f"read from the deprecated {attribute!r}; the pinned package " + f"marks it replaced by {replaced_by!r}", + ) + if not values: + return None + return TraceUsage(source=USAGE_SOURCE, **values) + + +def _read_tool_call( + attributes: Attributes, + *, + where: str, + started_at: datetime | None, + finished_at: datetime | None, + timestamp_notes: dict[str, str | None], + has_status: bool, + losses: LossReport, +) -> ToolCall: + """The tool call of an ``execute_tool`` span. + + The span's own extent is the tool call's: an ``execute_tool`` span *is* the + execution, so its start and end are not an inference. They are the only + timestamps in this family that an inbound edge has ever been able to fill — + both ACP and ATIF declare them unsupported outright. + + *Being* fillable is not the same as being filled, and the difference is + exactly where a silent absence hides. ``timestamp_notes`` carries whatever + :func:`_timestamp` had to say about each instant, and the same note is + declared a second time against the tool call's own path: both fields hold + that one value, so a reader checking ``events[i].tool_call.started_at`` + must find the declaration there rather than one level up. The two ACP and + ATIF edges declare these paths unconditionally; this one declares them + whenever it could not fill them, which is the same contract for a source + that sometimes can. + + Nothing is synthesized. No id is invented when ``gen_ai.tool.call.id`` is + absent, no name is taken from the span name (the recommended span name is + ``execute_tool {name}``, so reading it as the tool name would be reading a + convention as a value), and no status is derived from the span's. + """ + for field, note in timestamp_notes.items(): + if note is None: + continue + losses.add( + f"{where}.tool_call.{field}", + LossClass.NORMALIZED, + f"{note}; the tool call reads the same instant as its span, so the " + "same thing happened to this field", + ) + + call_id = attributes.text(GEN_AI_TOOL_CALL_ID) + if call_id is None: + losses.add( + f"{where}.tool_call.call_id", + LossClass.UNSUPPORTED, + f"the span carries no string {GEN_AI_TOOL_CALL_ID!r}; an id is not " + "invented here, because a synthesized id is indistinguishable from " + "an observed one once written", + ) + + name = attributes.text(GEN_AI_TOOL_NAME) + if name is None: + losses.add( + f"{where}.tool_call.name", + LossClass.UNSUPPORTED, + f"the span carries no string {GEN_AI_TOOL_NAME!r}; the span name is " + "not read as a substitute, since the recommended span name is " + f"{OPERATION_EXECUTE_TOOL!r} followed by the tool name", + ) + losses.add( + f"{where}.tool_call.name_semantics", + LossClass.UNSUPPORTED, + "there is no name, so there is nothing to say about what kind of " + "name it is; the field is not filled with the attribute this edge " + "would have read", + ) + + arguments = _read_arguments(attributes, where, losses) + content = _read_result(attributes, where, losses) + + losses.add( + f"{where}.tool_call.status", + LossClass.NORMALIZED if has_status else LossClass.UNSUPPORTED, + ( + "the span's status is kept in extensions.otel.span.status; whether an " + "OTLP status code maps onto ToolStatus is a semantic question this " + "edge does not answer" + ) + if has_status + else ( + "the span carries no status, and the pinned vocabulary has no tool " + "lifecycle attribute" + ), + ) + + return ToolCall( + call_id=call_id, + name=name, + name_semantics=TOOL_NAME_SEMANTICS if name is not None else None, + # No pinned attribute is a human-readable label for the call. The ACP + # edge fills this from a title the capture path records; OTel has none. + title=None, + status=None, + arguments=arguments, + content=content, + started_at=started_at, + finished_at=finished_at, + ) + + +def _read_arguments( + attributes: Attributes, where: str, losses: LossReport +) -> dict[str, Any] | None: + """``gen_ai.tool.call.arguments``, and every way it can fail to be a map. + + The pinned package says the attribute "is expected to be an object" and MAY + be recorded as a JSON string where structured attributes are unsupported. A + ``kvlistValue`` therefore decodes straight into + :attr:`~benchflow.trajectories.ir.ToolCall.arguments`; **a JSON string is + not parsed.** Parsing would be this converter deciding that a string that + happens to be JSON was meant as structure, and the pinned text puts that + obligation on the instrumentation, not on the reader. The string stays in + the attribute map and the case is declared (§8.11). + + Every branch declares something, because invariant 7 requires an + ``arguments is None`` to be paired with a hub record at exactly this path — + the one invariant that makes the loss report a contract. + """ + field = f"{where}.tool_call.arguments" + if GEN_AI_TOOL_CALL_ARGUMENTS not in attributes.values: + losses.add( + field, + LossClass.UNSUPPORTED, + f"the span carries no {GEN_AI_TOOL_CALL_ARGUMENTS!r}", + ) + return None + raw = attributes.values[GEN_AI_TOOL_CALL_ARGUMENTS] + if isinstance(raw, dict): + # Including ``{}``: an empty kvlist is an observed empty argument map, + # which the IR distinguishes from "the source carried no arguments". + return raw + losses.add( + field, + LossClass.NORMALIZED, + f"{GEN_AI_TOOL_CALL_ARGUMENTS} is " + f"{'a serialized string' if isinstance(raw, str) else type(raw).__name__}" + ", not an object; it is kept verbatim in extensions.otel.attributes and " + "is not parsed into a mapping here", + ) + return None + + +def _read_result( + attributes: Attributes, where: str, losses: LossReport +) -> list[ContentBlock]: + """``gen_ai.tool.call.result`` as content blocks. + + A string result is the rendered output and becomes a ``TEXT`` block. An + object result becomes an ``OPAQUE`` block carrying it verbatim, which is + exactly what that kind exists for. Anything else — a bare number, a list — + has no block shape it fits, and wrapping it would invent one, so it stays in + the attribute map with a record. + + Both block kinds leave one of :class:`ContentBlock`'s two payload fields + empty, and each is declared at its own concrete path. A ``TEXT`` block has + no ``raw`` because the attribute value *is* the text — there was never a + separate source block — and an ``OPAQUE`` block has no ``text`` because the + document holds an object and rendering it would be this converter writing + the string. Neither absence is structural, so neither is left to inference. + """ + if GEN_AI_TOOL_CALL_RESULT not in attributes.values: + losses.add( + f"{where}.tool_call.content", + LossClass.UNSUPPORTED, + f"the span carries no {GEN_AI_TOOL_CALL_RESULT!r}, so there is no " + "captured output to represent as content blocks", + ) + return [] + raw = attributes.values[GEN_AI_TOOL_CALL_RESULT] + if isinstance(raw, str): + losses.add( + f"{where}.tool_call.content[0].raw", + LossClass.UNSUPPORTED, + f"{GEN_AI_TOOL_CALL_RESULT} is the rendered text itself; OTLP has no " + "separate source block behind it to carry", + ) + return [ContentBlock(kind=ContentBlockKind.TEXT, text=raw)] + if isinstance(raw, dict): + losses.add( + f"{where}.tool_call.content[0].text", + LossClass.UNSUPPORTED, + f"{GEN_AI_TOOL_CALL_RESULT} is an object and the document carries no " + "rendering of it; producing one here would be this converter writing " + "the text rather than reading it", + ) + return [ContentBlock(kind=ContentBlockKind.OPAQUE, raw=raw)] + losses.add( + f"{where}.tool_call.content", + LossClass.NORMALIZED, + f"{GEN_AI_TOOL_CALL_RESULT} is {type(raw).__name__}; a content block is " + "text or an object, and the value is kept in extensions.otel.attributes " + "rather than wrapped in a shape the source did not have", + ) + return [] + + +def _declare_trace_losses( + losses: LossReport, + *, + events: list[TraceEvent], + session_id: str | None, + has_tool_call: bool, +) -> None: + """The records that describe the conversion rather than one span. + + Two kinds are mixed here, and the classes keep them apart: + + - **``UNSUPPORTED``** — OTLP carries nothing for the field. A run outcome, a + session id, a speaker: no attribute in the pinned vocabulary expresses + them, so no converter could fill them. + - **``NORMALIZED``** — OTLP carries the information, *per span*, and the IR + wants it per trace. A run's temporal extent and its total token usage are + both of that kind. Deriving them would mean assuming this payload holds + the whole run, which a batch does not promise; the per-span values are + preserved, and the aggregate is simply not computed. + + The difference is the one that says where a fix would land — the same line + `ir_from_acp` and `ir_from_atif` draw. + """ + if session_id is None: + losses.add( + "session_id", + LossClass.UNSUPPORTED, + f"no span attribute is a BenchFlow session id; {GEN_AI_CONVERSATION_ID!r} " + "is the candidate and is preserved in the attribute map, but reading " + "it as one would settle a mapping this edge does not own", + ) + + has_span_times = any( + event.started_at is not None or event.finished_at is not None + for event in events + ) + for field in ("started_at", "finished_at"): + if has_span_times: + losses.add( + field, + LossClass.NORMALIZED, + "OTLP has no run-level timestamp; each span's own extent is " + "preserved on its event, and deriving a run extent from them " + "would assume this payload holds the whole trace", + ) + else: + losses.add( + field, + LossClass.UNSUPPORTED, + "no span in this group carries a readable timestamp", + ) + + losses.add( + "outcome", + LossClass.UNSUPPORTED, + "OTLP has no run-outcome section; a span status is per span and is " + "preserved on each event, and reward and error category have no " + "attribute in the pinned vocabulary", + ) + + with_usage = [event for event in events if event.usage is not None] + if with_usage: + losses.add( + "usage", + LossClass.NORMALIZED, + "token counters are per span and are preserved on their events; " + "summing them would be an aggregation this converter does not " + "perform, and a batch need not hold every span of the run", + ) + if len(with_usage) != len(events): + # Declared once for the whole conversion rather than per bare span: + # the fact is "not every event has usage", and repeating it per + # event would multiply one sentence by the trace length. + losses.add( + "events[].usage", + LossClass.UNSUPPORTED, + f"{len(events) - len(with_usage)} of {len(events)} spans carry no " + f"{GEN_AI_USAGE_INPUT_TOKENS!r}-family attribute, so their events " + "carry no usage at all", + ) + for field in _USAGE_UNSUPPORTED: + losses.add( + f"events[].usage.{field}", + LossClass.UNSUPPORTED, + f"no attribute in the pinned GenAI vocabulary ({SEMCONV_VERSION}) " + "carries it", + ) + for field, attribute in _USAGE_PREFERRED: + # Declared unless *every* usage-bearing event filled it. "Some spans + # have it" leaves the others empty, and that absence needs the same + # declaration as "no span has it" — with a detail that says which. + filled = [ + event for event in with_usage if getattr(event.usage, field) is not None + ] + if len(filled) == len(with_usage): + continue + losses.add( + f"events[].usage.{field}", + LossClass.UNSUPPORTED, + f"{len(with_usage) - len(filled)} of {len(with_usage)} spans with " + f"usage carry no {attribute!r}; the attribute is readable, this " + "payload simply does not have it there", + ) + else: + losses.add( + "usage", + LossClass.UNSUPPORTED, + f"no span carries a {GEN_AI_USAGE_INPUT_TOKENS!r}-family attribute", + ) + losses.add( + "events[].usage", + LossClass.UNSUPPORTED, + "no span carries token counters", + ) + + losses.add( + "events[].role", + LossClass.UNSUPPORTED, + "OTLP attributes a span to an instrumentation scope, not to a speaker; " + "the IR's roles are the ones a source in this repository distinguishes", + ) + losses.add( + "events[].text", + LossClass.UNSUPPORTED, + "no span becomes user-visible text: the attributes that carry " + f"conversation content ({', '.join(_CONTENT_ATTRIBUTES)}) are defined by " + "reference to a JSON schema the pinned package does not ship, so they " + "are carried in extensions and not read. A per-event record names the " + "ones a given span actually had", + ) + losses.add( + "events[].outcome", + LossClass.UNSUPPORTED, + "the IR outcome slot is free text with no vocabulary an OTLP status code " + "maps into, so no span fills it; a per-event record names the spans that " + "carried a status worth mapping", + ) + for field in ("reasoning", "reasoning_segments"): + losses.add( + f"events[].{field}", + LossClass.UNSUPPORTED, + f"the pinned GenAI vocabulary ({SEMCONV_VERSION}) has no attribute for " + "reasoning content, so there are neither thoughts nor boundaries " + "between them to carry", + ) + if any(event.extensions["otel"]["attributes"] for event in events): + # Faithful decoding means no *value* was lost or collapsed. It does not + # mean the wire form survives: the canonical protobuf JSON mapping + # writes an int64 as a string and much of the ecosystem writes it as a + # number, so ``{"intValue": "7"}`` and ``{"intValue": 7}`` both decode + # to ``7`` and become indistinguishable, and the AnyValue wrapper type + # goes the same way. Semantic preservation, wire normalization — + # declared rather than implied, and declared once, because it is a + # property of the decoding and not of any one span. + losses.add( + "events[].extensions", + LossClass.NORMALIZED, + "attribute values are decoded out of their AnyValue wrappers into a " + "JSON map, so the wrapper type and the int64 spelling ('7' versus 7) " + "are not recoverable from it; the values are preserved, their wire " + "form is not. The original list is kept beside the map only when a " + "value itself would have been lost, and a per-event record says so", + ) + losses.add( + "events[].source_type", + LossClass.UNSUPPORTED, + "OTLP models the span name as a scalar with no presence, so an absent " + "name and an empty one are the same document and the IR's " + "absent-versus-empty distinction cannot be recovered", + ) + if has_tool_call: + losses.add( + "events[].tool_call.title", + LossClass.UNSUPPORTED, + "the pinned vocabulary has no human-readable label for a tool call", + ) diff --git a/tests/trajectories/test_ir_from_otel.py b/tests/trajectories/test_ir_from_otel.py new file mode 100644 index 000000000..60702b358 --- /dev/null +++ b/tests/trajectories/test_ir_from_otel.py @@ -0,0 +1,2122 @@ +"""Conversion suite for ``OTLP/JSON → canonical Trace IR`` — the inbound OTel edge. + +The three edges before this one could be checked against a producer in this +repository: `ACP → IR` against events a real `ACPSession` emitted, `IR → ATIF` +against the document the direct exporter writes, `ATIF → IR` against both. There +is no OpenTelemetry producer here at all (`docs/trace-interop.md` §4.2), so this +suite gets its ground truth from the two libraries the repository's ``uv.lock`` +already pins: + +- ``opentelemetry-proto==1.41.1`` — the wire shape; +- ``opentelemetry-semantic-conventions==0.62b1`` — the ``gen_ai.*`` vocabulary. + +:data:`PRODUCER_PAYLOAD_JSON` below is **not hand-written**. It is the output of +``google.protobuf.json_format.MessageToJson`` over an ``ExportTraceServiceRequest`` +built with ``opentelemetry-proto`` at the pinned version, so every encoding +choice in it — base64 identifiers, ``intValue`` as a JSON *string*, enum members +as names, absent fields where the value is the protobuf default — is the +library's, not this suite's. The generator is reproduced in +``_PRODUCER_PAYLOAD_RECIPE`` so anyone can rebuild it. Neither package is +imported here or added as a dependency. + +What the rest of the suite is for: + +- the **contract guards** — that no IR field quietly stops being filled and + quietly stops being declared, that every declared path resolves in the + canonical encoding, and that removing a declaration makes the guard fail; +- the **anti-invention** tests — no role, no synthesized id, no derived run + extent, no time sorting, no parsing of a serialized argument string; +- the **adversarial** cases — payloads no conformant producer writes, which is + where the ATIF edge found four real defects that the conformant path never + touched. + +Nothing here writes to disk and nothing imports a run path. +""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path +from typing import Any + +import pytest + +from benchflow.trajectories import _otlp_anyvalue as anyvalue_module +from benchflow.trajectories import ir_from_otel as ir_from_otel_module +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlockKind, + EventKind, + LossClass, + LossRecord, + LossReport, + PathSpace, + ToolCall, + TraceUsage, + validate_trace, +) +from benchflow.trajectories.ir_from_otel import ( + GEN_AI_OPERATION_NAME, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_ID, + GEN_AI_TOOL_NAME, + LOSS_DIRECTION, + OPERATION_EXECUTE_TOOL, + OTEL_SOURCE, + OTLP_JSON_SOURCE, + OTLP_PROTO_VERSION, + SEMCONV_VERSION, + USAGE_SOURCE, + OtlpSpan, + group_spans_by_trace_id, + otel_spans_to_ir, + otlp_json_spans, + otlp_json_to_ir, +) +from tests.trajectories.test_trace_ir import resolve_ir_path + +_PRODUCER_PAYLOAD_RECIPE = """ +# pip install opentelemetry-proto==1.41.1 (the version uv.lock pins) +import json +from google.protobuf.json_format import MessageToJson +from opentelemetry.proto.collector.trace.v1 import trace_service_pb2 as ts +from opentelemetry.proto.common.v1 import common_pb2 as c +from opentelemetry.proto.resource.v1 import resource_pb2 as r +from opentelemetry.proto.trace.v1 import trace_pb2 as t +# ... build one ResourceSpans holding five spans: an invoke_agent root, a chat +# child carrying usage and gen_ai.input.messages, two execute_tool children (one +# OK with arguments and a result, one ERROR with neither), and a plain HTTP +# client span with no gen_ai attribute at all ... +print(MessageToJson(ts.ExportTraceServiceRequest(resource_spans=[...]))) +""" + +PRODUCER_PAYLOAD_JSON = r""" +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "benchflow-rollout" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "0.6.10" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "benchflow.agent", + "version": "0.1.0" + }, + "spans": [ + { + "traceId": "S/kvNXezTaajzpKdDg5HNg==", + "spanId": "APBnqgupArc=", + "name": "invoke_agent solver", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1755500000000000000", + "endTimeUnixNano": "1755500004200000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "solver" + } + }, + { + "key": "gen_ai.agent.version", + "value": { + "stringValue": "2.1.0" + } + }, + { + "key": "gen_ai.provider.name", + "value": { + "stringValue": "anthropic" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "conv-7731" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "S/kvNXezTaajzpKdDg5HNg==", + "spanId": "APBnqgupArg=", + "parentSpanId": "APBnqgupArc=", + "name": "chat claude-sonnet-5", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1755500000010000000", + "endTimeUnixNano": "1755500001900000375", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chat" + } + }, + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "claude-sonnet-5" + } + }, + { + "key": "gen_ai.response.model", + "value": { + "stringValue": "claude-sonnet-5-20260101" + } + }, + { + "key": "gen_ai.response.finish_reasons", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "tool_use" + } + ] + } + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "1204" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "87" + } + }, + { + "key": "gen_ai.usage.cache_read.input_tokens", + "value": { + "intValue": "1024" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"role\":\"user\",\"parts\":[{\"type\":\"text\",\"content\":\"list the files\"}]}]" + } + } + ], + "events": [ + { + "timeUnixNano": "1755500001200000000", + "name": "first_token" + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "S/kvNXezTaajzpKdDg5HNg==", + "spanId": "APBnqgupArk=", + "parentSpanId": "APBnqgupArc=", + "name": "execute_tool read_file", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1755500001950000000", + "endTimeUnixNano": "1755500002100000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "read_file" + } + }, + { + "key": "gen_ai.tool.type", + "value": { + "stringValue": "function" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "toolu_01A" + } + }, + { + "key": "gen_ai.tool.call.arguments", + "value": { + "kvlistValue": { + "values": [ + { + "key": "path", + "value": { + "stringValue": "/repo/README.md" + } + }, + { + "key": "limit", + "value": { + "intValue": "200" + } + } + ] + } + } + }, + { + "key": "gen_ai.tool.call.result", + "value": { + "stringValue": "# benchflow\n" + } + } + ], + "droppedAttributesCount": 2, + "links": [ + { + "traceId": "S/kvNXezTaajzpKdDg5HNg==", + "spanId": "APBnqgupArg=" + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "S/kvNXezTaajzpKdDg5HNg==", + "spanId": "APBnqgupAro=", + "parentSpanId": "APBnqgupArc=", + "name": "execute_tool write_file", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1755500002200000000", + "endTimeUnixNano": "1755500002260000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "write_file" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "toolu_01B" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "PermissionError" + } + } + ], + "status": { + "message": "read-only filesystem", + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "S/kvNXezTaajzpKdDg5HNg==", + "spanId": "APBnqgupArs=", + "parentSpanId": "APBnqgupArg=", + "name": "POST", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1755500000012000000", + "endTimeUnixNano": "1755500001890000000", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "url.full", + "value": { + "stringValue": "https://api.anthropic.com/v1/messages" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ] + } + ], + "schemaUrl": "https://opentelemetry.io/schemas/1.40.0" + } + ], + "schemaUrl": "https://opentelemetry.io/schemas/1.40.0" + } + ] +} +""" + +TRACE_ID = "S/kvNXezTaajzpKdDg5HNg==" +"""The fixture's trace id, base64 — which is what the pinned library writes for +a 16-byte id, and the reason this edge never re-encodes one.""" + +ROOT_SPAN_ID = "APBnqgupArc=" +CHAT_SPAN_ID = "APBnqgupArg=" + + +def payload() -> dict[str, Any]: + """A fresh copy of the producer-derived payload.""" + return json.loads(PRODUCER_PAYLOAD_JSON) + + +def only_trace(document: dict[str, Any] | None = None) -> CanonicalTrace: + traces, envelope = otlp_json_to_ir(document if document is not None else payload()) + assert envelope.lossless, envelope.records + assert len(traces) == 1 + return traces[0] + + +# --------------------------------------------------------------------------- +# Payload builders for the cases a real producer will not write +# --------------------------------------------------------------------------- + + +def S(value: str) -> dict[str, Any]: + return {"stringValue": value} + + +def I(value: int | str) -> dict[str, Any]: # noqa: E743 - mirrors the OTLP name + return {"intValue": value} + + +def attrs(*pairs: tuple[str, Any]) -> list[dict[str, Any]]: + return [{"key": key, "value": value} for key, value in pairs] + + +def span(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "traceId": TRACE_ID, + "spanId": ROOT_SPAN_ID, + "name": "span", + "startTimeUnixNano": "1755500000000000000", + "endTimeUnixNano": "1755500001000000000", + } + base.update(overrides) + return base + + +def tool_span(**overrides: Any) -> dict[str, Any]: + attributes = overrides.pop("attributes", []) + return span( + name="execute_tool read_file", + attributes=attrs((GEN_AI_OPERATION_NAME, S(OPERATION_EXECUTE_TOOL))) + + attributes, + **overrides, + ) + + +def tool_span_without_timestamps(**overrides: Any) -> dict[str, Any]: + """An ``execute_tool`` span a producer wrote with no readable instants. + + The degenerate shape the field-level contract guard used to walk straight + past: the fixture fills ``tool_call.started_at``, so the guard was satisfied + for that path on every payload, including this one where it is empty. + """ + raw = tool_span(**overrides) + raw.pop("startTimeUnixNano", None) + raw.pop("endTimeUnixNano", None) + return raw + + +def wrap(*spans: dict[str, Any], **envelope: Any) -> dict[str, Any]: + scope_spans: dict[str, Any] = {"spans": list(spans)} + scope_spans.update(envelope.pop("scope_spans", {})) + resource_spans: dict[str, Any] = {"scopeSpans": [scope_spans]} + resource_spans.update(envelope.pop("resource_spans", {})) + assert not envelope, envelope + return {"resourceSpans": [resource_spans]} + + +def convert(*spans: dict[str, Any]) -> CanonicalTrace: + """One trace from bare spans, with no envelope in the way.""" + return otel_spans_to_ir([OtlpSpan(span=one) for one in spans]) + + +def fields(trace: CanonicalTrace, loss_class: LossClass | None = None) -> set[str]: + return { + record.field + for record in trace.losses.records + if record.space is PathSpace.HUB + and (loss_class is None or record.loss_class is loss_class) + } + + +def record_for(trace: CanonicalTrace, field: str) -> LossRecord: + matching = trace.losses.for_field(field) + assert len(matching) == 1, (field, matching) + return matching[0] + + +# --------------------------------------------------------------------------- +# The producer-derived payload +# --------------------------------------------------------------------------- + + +def test_the_pinned_versions_are_the_ones_this_suite_reads(): + """The constants naming the evidence are not decoration. + + Every ``gen_ai.*`` string and every wire-shape assumption in the module + under test was copied from these two versions. If the module's idea of what + it was written against drifts from this suite's, the mapping stops being + checkable against anything. + """ + assert OTLP_PROTO_VERSION == "1.41.1" + assert SEMCONV_VERSION == "0.62b1" + assert "opentelemetry-proto==1.41.1" in _PRODUCER_PAYLOAD_RECIPE + + +def test_the_attribute_names_are_the_pinned_spellings(): + """Spelled out once, because a near-miss reads exactly like a hit. + + Each right-hand side is the literal value of the same-named constant in + ``opentelemetry.semconv._incubating.attributes.gen_ai_attributes`` at + :data:`SEMCONV_VERSION`. The deleted `OTelCollector` is the cautionary case: + it read ``gen_ai.usage.cache_read_input_tokens``, and the attribute is + ``gen_ai.usage.cache_read.input_tokens`` — one dot apart, and never a match. + """ + assert GEN_AI_OPERATION_NAME == "gen_ai.operation.name" + assert GEN_AI_TOOL_NAME == "gen_ai.tool.name" + assert GEN_AI_TOOL_CALL_ID == "gen_ai.tool.call.id" + assert GEN_AI_TOOL_CALL_ARGUMENTS == "gen_ai.tool.call.arguments" + assert OPERATION_EXECUTE_TOOL == "execute_tool" + assert ir_from_otel_module.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS == ( + "gen_ai.usage.cache_read.input_tokens" + ) + assert ir_from_otel_module.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS == ( + "gen_ai.usage.cache_creation.input_tokens" + ) + assert not hasattr(ir_from_otel_module, "GEN_AI_USAGE_TOTAL_TOKENS") + + +def test_the_fixture_is_shaped_the_way_the_pinned_library_writes(): + """Guards the fixture itself against being 'tidied' into a hand-written one. + + Each assertion is an encoding choice ``MessageToJson`` made and a + hand-written payload would probably get wrong — and each is one this edge + has to handle: a base64 id it must not re-encode, an int64 as a JSON string, + an enum as its member name, and a default-valued field written as absence. + """ + document = payload() + spans = document["resourceSpans"][0]["scopeSpans"][0]["spans"] + assert len(spans) == 5 + + assert spans[0]["traceId"] == TRACE_ID + assert spans[0]["traceId"].endswith("==") # base64 of 16 bytes, not 32 hex + + chat = spans[1] + usage = {pair["key"]: pair["value"] for pair in chat["attributes"]} + assert usage["gen_ai.usage.input_tokens"] == {"intValue": "1204"} + assert isinstance(chat["startTimeUnixNano"], str) + assert chat["kind"] == "SPAN_KIND_CLIENT" + assert spans[2]["status"] == {"code": "STATUS_CODE_OK"} + + # The root span has no parent, and a zero-valued ``parentSpanId`` is absent + # rather than empty — the presence limit this edge declares. + assert "parentSpanId" not in spans[0] + assert spans[1]["parentSpanId"] == ROOT_SPAN_ID + + +def test_a_producer_payload_reads_into_one_valid_trace(): + trace = only_trace() + assert validate_trace(trace) == [] + assert trace.losses is not None + assert trace.losses.direction == LOSS_DIRECTION + assert trace.provenance.source_format == OTLP_JSON_SOURCE + assert len(trace.events) == 5 + assert all(event.provenance.source_format == OTEL_SOURCE for event in trace.events) + assert all(event.provenance.producer == "benchflow.agent" for event in trace.events) + + +def test_only_execute_tool_becomes_a_typed_event(): + """The mapping's whole reach, asserted as a set rather than described. + + ``invoke_agent``, ``chat`` and a plain HTTP span are all ``UNKNOWN``. That + is not a gap this suite is waiting to close — it is the edge refusing to + read agent semantics into spans that do not carry them, and the assertion + exists so widening it is a deliberate edit. + """ + trace = only_trace() + assert [event.kind for event in trace.events] == [ + EventKind.UNKNOWN, + EventKind.UNKNOWN, + EventKind.TOOL_CALL, + EventKind.TOOL_CALL, + EventKind.UNKNOWN, + ] + assert [event.source_type for event in trace.events] == [ + "invoke_agent solver", + "chat claude-sonnet-5", + "execute_tool read_file", + "execute_tool write_file", + "POST", + ] + + +def test_identity_and_parentage_survive_verbatim(): + """The property the IR has no field for, and therefore the one at risk. + + The IR models no parent link, so the span graph lives in ``extensions``. + Preserved is preserved: every id is the payload's own string, byte for byte, + and the edge set is reconstructible from the trace alone. + """ + document = payload() + source_spans = document["resourceSpans"][0]["scopeSpans"][0]["spans"] + trace = only_trace(document) + + assert trace.trace_id == TRACE_ID + for event, source in zip(trace.events, source_spans, strict=True): + carried = event.extensions["otel"]["span"] + assert carried["spanId"] == source["spanId"] + assert carried.get("parentSpanId") == source.get("parentSpanId") + assert carried["traceId"] == source["traceId"] + + edges = { + event.extensions["otel"]["span"]["spanId"]: event.extensions["otel"][ + "span" + ].get("parentSpanId") + for event in trace.events + } + assert edges[ROOT_SPAN_ID] is None + assert edges[CHAT_SPAN_ID] == ROOT_SPAN_ID + assert sum(1 for parent in edges.values() if parent == ROOT_SPAN_ID) == 3 + + +def test_links_and_span_events_are_carried(): + """Two more structures with no IR home, and both explicitly in scope.""" + trace = only_trace() + chat = trace.events[1].extensions["otel"]["span"] + assert chat["events"] == [ + {"timeUnixNano": "1755500001200000000", "name": "first_token"} + ] + tool = trace.events[2].extensions["otel"]["span"] + assert tool["links"] == [{"traceId": TRACE_ID, "spanId": CHAT_SPAN_ID}] + + +def test_two_structurally_distinct_scope_spans_stay_distinguishable(): + """F-1: flattening the envelope must not flatten the partition. + + The producer batched these two spans under *separate* ``ScopeSpans`` + objects. Their ``scope`` payloads are equal, so once the envelope is + flattened into one list of spans, nothing except the coordinates says they + were ever apart — and "the producer grouped these separately" is a fact + about the payload, not noise. + """ + payload = { + "resourceSpans": [ + { + "resource": {"attributes": []}, + "scopeSpans": [ + {"scope": {"name": "s"}, "spans": [span(name="a")]}, + {"scope": {"name": "s"}, "spans": [span(name="b")]}, + ], + } + ] + } + trace = only_trace(payload) + envelopes = [event.extensions["otel"]["envelope"] for event in trace.events] + assert envelopes == [ + {"resource_spans_index": 0, "scope_spans_index": 0, "span_index": 0}, + {"resource_spans_index": 0, "scope_spans_index": 1, "span_index": 0}, + ] + # The scopes themselves are indistinguishable, which is exactly why the + # coordinates have to carry the difference. + assert ( + trace.events[0].extensions["otel"]["scope"] + == trace.events[1].extensions["otel"]["scope"] + ) + + +def test_two_spans_in_one_scope_spans_are_also_distinguishable(): + """The complement: same group, different position within it.""" + payload = wrap(span(name="a"), span(name="b")) + trace = only_trace(payload) + assert [ + event.extensions["otel"]["envelope"]["span_index"] for event in trace.events + ] == [0, 1] + assert { + event.extensions["otel"]["envelope"]["scope_spans_index"] + for event in trace.events + } == {0} + + +def test_a_span_the_caller_built_carries_no_envelope_coordinates(): + """No payload, no partition — and no invented coordinates. + + ``otel_spans_to_ir`` accepts spans a caller assembled itself. Writing + ``span_index: 0`` for those would claim an envelope position that never + existed, which is the same class of invention as a synthesized id. + """ + trace = convert(span()) + assert "envelope" not in trace.events[0].extensions["otel"] + + +def test_resource_and_scope_ride_with_every_event(): + """Per event, not per trace — one payload legitimately mixes them.""" + trace = only_trace() + for event in trace.events: + carried = event.extensions["otel"] + assert carried["scope"] == {"name": "benchflow.agent", "version": "0.1.0"} + assert carried["scope_schema_url"] == "https://opentelemetry.io/schemas/1.40.0" + assert carried["resource_schema_url"] == ( + "https://opentelemetry.io/schemas/1.40.0" + ) + assert carried["resource"]["attributes"][0]["key"] == "service.name" + + +def test_the_tool_call_reads_every_pinned_attribute(): + trace = only_trace() + call = trace.events[2].tool_call + assert call == ToolCall( + call_id="toolu_01A", + name="read_file", + name_semantics=GEN_AI_TOOL_NAME, + title=None, + status=None, + arguments={"path": "/repo/README.md", "limit": 200}, + content=call.content, + started_at=trace.events[2].started_at, + finished_at=trace.events[2].finished_at, + ) + assert [(block.kind, block.text) for block in call.content] == [ + (ContentBlockKind.TEXT, "# benchflow\n") + ] + # An observed argument map is not a declared absence. + assert trace.losses.for_field("events[2].tool_call.arguments") == [] + + +def test_a_tool_span_is_the_only_source_that_has_ever_timed_a_tool_call(): + """`§5 loss #3` closed from one side. + + ACP tracks the two instants in memory and serializes neither; ATIF has no + slot at all. An ``execute_tool`` span *is* the execution, so its extent is + the call's without inference — the first inbound edge in this family that + can fill these fields. + """ + trace = only_trace() + call = trace.events[2].tool_call + assert call.started_at is not None and call.finished_at is not None + assert call.started_at < call.finished_at + + +def test_usage_is_per_span_and_says_which_definition_it_used(): + trace = only_trace() + assert trace.events[1].usage == TraceUsage( + input_tokens=1204, + output_tokens=87, + cache_read_tokens=1024, + source=USAGE_SOURCE, + ) + assert all(event.usage is None for event in trace.events if event.index != 1) + # And the run total is not computed from it. + assert trace.usage is None + assert record_for(trace, "usage").loss_class is LossClass.NORMALIZED + + +def test_agent_identity_is_gathered_from_the_spans_that_carry_it(): + trace = only_trace() + assert trace.agent.agent_name == "solver" + assert trace.agent.agent_version == "2.1.0" + assert trace.agent.model == "claude-sonnet-5" + assert trace.agent.provider == "anthropic" + assert not [field for field in fields(trace) if field.startswith("agent.")] + + +def test_a_conversation_id_is_not_read_as_a_session_id(): + """The plausible mapping this edge declines to make. + + ``gen_ai.conversation.id`` is "a conversation (session, thread)" in the + pinned package, which is close enough that reading it into ``session_id`` + would look right and would quietly settle a question nobody asked. The + value is preserved; the mapping is left to a maintainer (§8.11). + + Found by mutation: making this edge read the attribute left the suite green + until this test existed. + """ + trace = only_trace() + assert trace.session_id is None + record = record_for(trace, "session_id") + assert record.loss_class is LossClass.UNSUPPORTED + assert "gen_ai.conversation.id" in record.detail + assert ( + trace.events[0].extensions["otel"]["attributes"]["gen_ai.conversation.id"] + == "conv-7731" + ) + + +def test_a_caller_supplied_session_id_is_used_and_declares_nothing(): + """The context an inbound edge is allowed to take: from its caller.""" + spans, _ = otlp_json_spans(payload()) + trace = otel_spans_to_ir(spans, session_id="rollout-42") + assert trace.session_id == "rollout-42" + assert trace.losses is not None + assert trace.losses.for_field("session_id") == [] + + +def test_sub_microsecond_precision_is_declared_and_kept(): + """The loss only a real OTLP timestamp produces. + + ``datetime`` resolves to microseconds and OTLP counts nanoseconds, so the + IR field cannot hold the value. Declaring it is not enough on its own — + what makes it a preserved value rather than a lost one is that the exact + integer is still in the document. + """ + trace = only_trace() + record = record_for(trace, "events[1].finished_at") + assert record.loss_class is LossClass.NORMALIZED + assert "375 ns is truncated" in record.detail + assert trace.events[1].extensions["otel"]["span"]["endTimeUnixNano"] == ( + "1755500001900000375" + ) + assert trace.events[1].finished_at is not None + assert trace.events[1].finished_at.microsecond == 900000 + + +def test_the_producer_declaring_its_own_drop_is_unsupported_not_dropped(): + """``droppedAttributesCount`` is loss that happened before this reader. + + The class is the whole point: no converter can recover an attribute the + emitting SDK discarded, so the fix is a producer-side limit rather than + code here. Calling it ``DROPPED`` would point the reader at this file. + """ + trace = only_trace() + records = [ + record + for record in trace.losses.for_field("events[2].extensions") + if "droppedAttributesCount" in record.detail + ] + assert len(records) == 1 + assert records[0].loss_class is LossClass.UNSUPPORTED + + +# --------------------------------------------------------------------------- +# Ordering, and the causality it is not +# --------------------------------------------------------------------------- + + +def test_document_order_is_preserved_and_time_order_is_not_imposed(): + """The explicit instruction, made a property. + + A payload whose spans are out of chronological order stays out of order. + Sorting would be the converter claiming a sequence OTLP does not carry — + siblings overlap, and the real structure is the parent edge set, which is + preserved regardless. + """ + late = span( + spanId="AAAAAAAAAAE=", name="late", startTimeUnixNano="1755500009000000000" + ) + early = span( + spanId="AAAAAAAAAAI=", name="early", startTimeUnixNano="1755500001000000000" + ) + trace = convert(late, early) + + assert [event.source_type for event in trace.events] == ["late", "early"] + assert [event.index for event in trace.events] == [0, 1] + assert trace.events[0].started_at > trace.events[1].started_at + + +def test_an_unreadable_span_leaves_no_hole_in_the_index(): + """Invariant 2 under an envelope that carries something unreadable.""" + document = wrap(span(name="a"), "not a span", span(name="b")) + traces, envelope = otlp_json_to_ir(document) + assert [record.field for record in envelope.records] == [ + "resourceSpans[0].scopeSpans[0].spans[1]" + ] + assert envelope.records[0].space is PathSpace.SOURCE + assert len(traces) == 1 + assert [event.index for event in traces[0].events] == [0, 1] + assert [event.source_type for event in traces[0].events] == ["a", "b"] + + +def test_the_run_extent_is_never_derived_from_the_spans(): + """Even when every span is timed, the trace-level fields stay empty.""" + trace = convert(span(name="a"), span(name="b")) + assert trace.started_at is None and trace.finished_at is None + assert all(event.started_at is not None for event in trace.events) + for field in ("started_at", "finished_at"): + assert record_for(trace, field).loss_class is LossClass.NORMALIZED + + +def test_with_no_readable_timestamp_the_run_extent_is_unsupported_instead(): + """The class tracks the reason, not the outcome. + + Both cases leave the field ``None``. ``NORMALIZED`` says the information + exists per span and was not aggregated; ``UNSUPPORTED`` says there was + nothing to aggregate. Collapsing them would lose where a fix would land. + """ + trace = convert( + {"traceId": TRACE_ID, "spanId": ROOT_SPAN_ID, "name": "untimed"}, + ) + for field in ("started_at", "finished_at"): + assert record_for(trace, field).loss_class is LossClass.UNSUPPORTED + + +# --------------------------------------------------------------------------- +# What this edge refuses to invent +# --------------------------------------------------------------------------- + + +def test_no_event_is_ever_attributed_to_a_speaker(): + trace = only_trace() + assert all(event.role is None for event in trace.events) + assert record_for(trace, "events[].role").loss_class is LossClass.UNSUPPORTED + + +def test_no_conversation_text_is_read_out_of_message_attributes(): + """`gen_ai.input.messages` is carried, not interpreted. + + The pinned package defines the message structure by reference to a JSON + schema it does not ship, so reading it into ``text`` would be a mapping + against a document nobody in this repository can check. + """ + trace = only_trace() + assert all(event.text is None for event in trace.events) + assert all(event.reasoning is None for event in trace.events) + record = record_for(trace, "events[1].text") + assert record.loss_class is LossClass.NORMALIZED + assert "gen_ai.input.messages" in record.detail + assert ( + trace.events[1].extensions["otel"]["attributes"]["gen_ai.input.messages"] + == '[{"role":"user","parts":[{"type":"text","content":"list the files"}]}]' + ) + + +def test_nothing_is_synthesized_on_an_inbound_edge(): + """An inbound edge has no target to satisfy, so it fabricates nothing.""" + trace = only_trace() + assert trace.losses.by_class(LossClass.SYNTHESIZED) == [] + + +def test_a_tool_span_with_no_id_and_no_name_gets_neither(): + trace = convert(tool_span()) + call = trace.events[0].tool_call + assert call is not None + assert call.call_id is None and call.name is None + assert call.name_semantics is None + assert ( + record_for(trace, "events[0].tool_call.call_id").loss_class + is LossClass.UNSUPPORTED + ) + record = record_for(trace, "events[0].tool_call.name") + assert record.loss_class is LossClass.UNSUPPORTED + # And the span name is not quietly used instead. + assert "execute_tool read_file" not in str(call) + + +def test_a_serialized_argument_string_is_not_parsed(): + """The pinned text puts deserialization on the instrumentation, not here. + + Parsing would be the converter deciding a string that happens to be JSON + was meant as structure. The string is preserved; the refusal is declared; + invariant 7 is satisfied by that same record. + """ + trace = convert( + tool_span( + attributes=attrs((GEN_AI_TOOL_CALL_ARGUMENTS, S('{"path": "/tmp/x"}'))) + ) + ) + call = trace.events[0].tool_call + assert call is not None and call.arguments is None + record = record_for(trace, "events[0].tool_call.arguments") + assert record.loss_class is LossClass.NORMALIZED + assert "serialized string" in record.detail + assert ( + trace.events[0].extensions["otel"]["attributes"][GEN_AI_TOOL_CALL_ARGUMENTS] + == '{"path": "/tmp/x"}' + ) + assert validate_trace(trace) == [] + + +def test_an_observed_empty_argument_map_is_not_an_absence(): + """The tri-state rule at the one place the IR makes it an invariant.""" + trace = convert( + tool_span( + attributes=attrs( + (GEN_AI_TOOL_CALL_ARGUMENTS, {"kvlistValue": {"values": []}}) + ) + ) + ) + call = trace.events[0].tool_call + assert call is not None + assert call.arguments == {} + assert trace.losses.for_field("events[0].tool_call.arguments") == [] + assert validate_trace(trace) == [] + + +def test_no_trace_id_is_invented(): + trace = convert({"spanId": ROOT_SPAN_ID, "name": "orphan"}) + assert trace.trace_id is None + assert record_for(trace, "trace_id").loss_class is LossClass.UNSUPPORTED + + +def test_identifiers_are_never_re_encoded(): + """Hex and base64 ids are not distinguishable, so neither is normalized. + + The pinned JSON parser accepts a 32-character hex trace id *as base64* and + yields 24 bytes from it, so a reader that decided which encoding it was + looking at would be guessing. Both survive as written. + """ + hex_id = "4bf92f3577b34da6a3ce929d0e0e4736" + trace = convert(span(traceId=hex_id)) + assert trace.trace_id == hex_id + assert trace.events[0].extensions["otel"]["span"]["traceId"] == hex_id + + base64_trace = convert(span()) + assert base64_trace.trace_id == TRACE_ID + + +# --------------------------------------------------------------------------- +# Batches +# --------------------------------------------------------------------------- + + +def test_one_payload_can_hold_several_traces(): + other = "AAAAAAAAAAAAAAAAAAAAAA==" + document = wrap( + span(name="a"), + span(traceId=other, name="b"), + span(name="c"), + ) + traces, envelope = otlp_json_to_ir(document) + assert envelope.lossless + assert [trace.trace_id for trace in traces] == [TRACE_ID, other] + assert [event.source_type for event in traces[0].events] == ["a", "c"] + assert [event.source_type for event in traces[1].events] == ["b"] + + +def test_spans_with_no_trace_id_group_together_rather_than_joining_one(): + """ "May belong to that trace" is not a fact this reader gets to record.""" + groups = group_spans_by_trace_id( + [ + OtlpSpan(span=span(name="a")), + OtlpSpan(span={"name": "b"}), + OtlpSpan(span=span(name="c")), + ] + ) + assert [key for key, _ in groups] == [TRACE_ID, None] + assert [len(members) for _, members in groups] == [2, 1] + + +def test_a_mixed_group_keeps_the_first_id_and_lists_them_all(): + other = "AAAAAAAAAAAAAAAAAAAAAA==" + trace = convert(span(name="a"), span(traceId=other, name="b")) + assert trace.trace_id == TRACE_ID + assert trace.extensions["otel"]["trace_ids"] == [TRACE_ID, other] + assert record_for(trace, "trace_id").loss_class is LossClass.NORMALIZED + + +def test_a_non_string_trace_id_is_reported_against_the_source(): + trace = convert(span(traceId=17)) + assert trace.trace_id is None + source = trace.losses.for_field("spans[0].traceId", PathSpace.SOURCE) + assert len(source) == 1 and source[0].loss_class is LossClass.DROPPED + + +def test_an_envelope_report_belongs_to_the_payload_not_to_a_trace(): + """The one asymmetry in this family, asserted so it stays deliberate.""" + document = {"resourceSpans": [{"scopeSpans": "not a list"}, 7]} + spans, envelope = otlp_json_spans(document) + assert spans == [] + assert {record.field for record in envelope.records} == { + "resourceSpans[0].scopeSpans", + "resourceSpans[1]", + } + assert all(record.space is PathSpace.SOURCE for record in envelope.records) + traces, _ = otlp_json_to_ir(document) + assert traces == [] + + +def test_an_empty_payload_reads_as_no_traces_rather_than_an_empty_one(): + traces, envelope = otlp_json_to_ir({}) + assert traces == [] + assert envelope.lossless + + +def test_a_non_mapping_payload_is_refused(): + with pytest.raises(TypeError): + otlp_json_spans([]) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("document", "expected"), + [ + ({"resourceSpans": {}}, {"resourceSpans"}), + ({"resourceSpans": [1]}, {"resourceSpans[0]"}), + ( + {"resourceSpans": [{"scopeSpans": [{"spans": 3}]}]}, + {"resourceSpans[0].scopeSpans[0].spans"}, + ), + ( + {"resourceSpans": [{"scopeSpans": [None]}]}, + {"resourceSpans[0].scopeSpans[0]"}, + ), + ], +) +def test_every_envelope_level_declares_what_it_could_not_read(document, expected): + spans, envelope = otlp_json_spans(document) + assert spans == [] + assert {record.field for record in envelope.records} == expected + + +# --------------------------------------------------------------------------- +# AnyValue: the three states, and the ones a map cannot hold +# --------------------------------------------------------------------------- + + +def test_the_three_states_of_an_attribute_value_are_kept_apart(): + """Present-and-empty, present-but-typeless, and absent. + + ``{"stringValue": ""}`` is an observed empty string. ``{}`` is protobuf's + "no oneof member set". A ``KeyValue`` with no ``value`` key is a third + thing again. A plain map can hold the first and collapses the other two to + ``None`` — so the map is not the record: the original list is kept beside + it and the collapse is declared. + """ + trace = convert( + span( + attributes=[ + {"key": "empty_string", "value": {"stringValue": ""}}, + {"key": "typeless", "value": {}}, + {"key": "valueless"}, + ] + ) + ) + carried = trace.events[0].extensions["otel"] + assert carried["attributes"] == { + "empty_string": "", + "typeless": None, + "valueless": None, + } + assert carried["attributes_raw"] == [ + {"key": "empty_string", "value": {"stringValue": ""}}, + {"key": "typeless", "value": {}}, + {"key": "valueless"}, + ] + record = record_for(trace, "events[0].extensions") + assert record.loss_class is LossClass.NORMALIZED + assert "typeless" in record.detail and "valueless" in record.detail + + +def test_faithful_decoding_is_semantic_not_wire_invertible(): + """F-2: the two int64 spellings collapse, and that is declared. + + The canonical protobuf JSON mapping writes an ``int64`` as a *string*; much + of the ecosystem writes it as a number. Both are the value ``7``, so nothing + semantic is lost and ``attributes_raw`` is correctly not kept — but the two + payloads are indistinguishable afterwards, so the edge must not be described + as preserving the wire form. + """ + as_string = convert(span(attributes=attrs(("k", I("7"))))) + as_number = convert(span(attributes=attrs(("k", I(7))))) + carried = [trace.events[0].extensions["otel"] for trace in (as_string, as_number)] + assert carried[0]["attributes"] == carried[1]["attributes"] == {"k": 7} + assert all("attributes_raw" not in one for one in carried) + + for trace in (as_string, as_number): + record = record_for(trace, "events[].extensions") + assert record.loss_class is LossClass.NORMALIZED + assert "wire form is not" in record.detail + + +def test_the_wire_normalization_is_declared_once_not_per_span(): + """A property of the decoding, so declaring it per span would be noise. + + §8.6's affordability argument: a record that repeats one sentence per event + multiplies the report by the trace length while adding nothing. + """ + one = convert(span(attributes=attrs(("k", S("v"))))) + many = convert(*[span(attributes=attrs(("k", S("v")))) for _ in range(10)]) + assert len(one.losses.for_field("events[].extensions")) == 1 + assert len(many.losses.for_field("events[].extensions")) == 1 + + +def test_a_span_with_no_attributes_declares_no_wire_normalization(): + """Nothing was decoded, so there is nothing to say about how.""" + trace = convert(span()) + assert trace.losses.for_field("events[].extensions") == [] + + +def test_a_faithful_attribute_list_keeps_no_raw_copy(): + """The complement, so the guard above is not satisfied by always copying.""" + trace = convert(span(attributes=attrs(("a", S("x")), ("b", I("2"))))) + carried = trace.events[0].extensions["otel"] + assert carried["attributes"] == {"a": "x", "b": 2} + assert "attributes_raw" not in carried + assert trace.losses.for_field("events[0].extensions") == [] + + +def test_duplicate_attribute_keys_cannot_be_silently_deduplicated(): + """OTLP models attributes as a list, so duplicates are representable.""" + trace = convert(span(attributes=attrs(("k", S("first")), ("k", S("second"))))) + carried = trace.events[0].extensions["otel"] + assert carried["attributes"] == {"k": "second"} + assert len(carried["attributes_raw"]) == 2 + assert "repeats key" in record_for(trace, "events[0].extensions").detail + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ({"stringValue": "x"}, "x"), + ({"boolValue": False}, False), + ({"intValue": "42"}, 42), + ({"intValue": 42}, 42), + ({"intValue": "-1"}, -1), + ({"doubleValue": 0.0}, 0.0), + ({"doubleValue": 2}, 2.0), + ( + {"arrayValue": {"values": [{"intValue": "1"}, {"stringValue": "a"}]}}, + [1, "a"], + ), + ({"arrayValue": {"values": []}}, []), + ( + {"kvlistValue": {"values": [{"key": "k", "value": {"boolValue": True}}]}}, + {"k": True}, + ), + ], +) +def test_every_any_value_member_this_reader_knows(value, expected): + """Falsy values included — the deleted `OTelCollector` collapsed them. + + Its ``_parse_attributes`` read + ``stringValue or intValue or doubleValue or boolValue``, which turns ``""``, + ``0`` and ``False`` into "no value". They are values. + """ + trace = convert(span(attributes=attrs(("k", value)))) + carried = trace.events[0].extensions["otel"] + assert carried["attributes"]["k"] == expected + assert "attributes_raw" not in carried + + +@pytest.mark.parametrize( + "spelling", + ["NaN", "Infinity", "-Infinity"], +) +def test_the_non_finite_doubles_are_read_from_their_json_spelling(spelling): + trace = convert(span(attributes=attrs(("k", {"doubleValue": spelling})))) + value = trace.events[0].extensions["otel"]["attributes"]["k"] + assert isinstance(value, float) + assert (value != value) if spelling == "NaN" else abs(value) == float("inf") + + +@pytest.mark.parametrize( + "value", + [ + {"bytesValue": "AQI="}, + {"stringValue": "x", "intValue": "1"}, + {"futureValue": 1}, + {"intValue": "not a number"}, + {"intValue": True}, + {"doubleValue": "3.5"}, + {"arrayValue": {"values": [{"futureValue": 1}]}}, + {"arrayValue": {"values": "no"}}, + {"kvlistValue": {"values": [{"key": "k"}]}}, + { + "kvlistValue": { + "values": [ + {"key": "k", "value": {"stringValue": "a"}}, + {"key": "k", "value": {"stringValue": "b"}}, + ] + } + }, + "not an object", + ], +) +def test_an_any_value_a_map_cannot_hold_keeps_the_list_and_says_so(value): + """Every refusal is declared; none of them loses the payload.""" + trace = convert(span(attributes=attrs(("k", value)))) + carried = trace.events[0].extensions["otel"] + assert carried["attributes_raw"] == [{"key": "k", "value": value}] + assert record_for(trace, "events[0].extensions").loss_class is LossClass.NORMALIZED + assert validate_trace(trace) == [] + + +def test_a_malformed_attribute_list_is_declared_rather_than_ignored(): + trace = convert(span(attributes={"k": "v"})) + carried = trace.events[0].extensions["otel"] + assert carried["attributes"] == {} + assert carried["attributes_raw"] == {"k": "v"} + assert "not a list" in record_for(trace, "events[0].extensions").detail + + +# --------------------------------------------------------------------------- +# Timestamps +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("value", "marker"), + [ + (None, "is absent"), + ("0", "protobuf default"), + (0, "protobuf default"), + ("-1", "unsigned"), + ("later", "not an integer"), + (True, "bool"), + ([], "list"), + ], +) +def test_an_unreadable_timestamp_is_declared_and_never_guessed(value, marker): + raw = span() + if value is None: + raw.pop("startTimeUnixNano") + else: + raw["startTimeUnixNano"] = value + trace = convert(raw) + assert trace.events[0].started_at is None + record = record_for(trace, "events[0].started_at") + assert record.loss_class is LossClass.NORMALIZED + assert marker in record.detail + + +def test_a_whole_microsecond_timestamp_declares_nothing(): + trace = convert(span()) + assert trace.losses.for_field("events[0].started_at") == [] + assert trace.events[0].started_at is not None + assert trace.events[0].started_at.tzinfo is not None + + +def test_a_tool_calls_own_timestamps_are_declared_at_their_own_path(): + """F-3: both slots hold the value, so both absences must be addressable. + + A record at ``events[i].started_at`` does not answer for + ``events[i].tool_call.started_at``: they are different paths, a reader + checking the second one finds nothing, and "the parent said something" is + not how the loss report is addressed. `ir_from_acp` and `ir_from_atif` both + declare these two paths unconditionally; this edge declares them whenever it + could not fill them. + """ + trace = convert(tool_span_without_timestamps()) + call = trace.events[0].tool_call + assert call is not None + assert call.started_at is None and call.finished_at is None + for field in ("started_at", "finished_at"): + record = record_for(trace, f"events[0].tool_call.{field}") + assert record.loss_class is LossClass.NORMALIZED + assert "the same instant as its span" in record.detail + # and the event-level path is still declared in its own right + assert trace.losses.for_field(f"events[0].{field}") + + +def test_a_filled_tool_call_timestamp_declares_nothing(): + """The complement — otherwise the record above would be unconditional.""" + trace = convert(tool_span()) + call = trace.events[0].tool_call + assert call.started_at is not None and call.finished_at is not None + assert trace.losses.for_field("events[0].tool_call.started_at") == [] + assert trace.losses.for_field("events[0].tool_call.finished_at") == [] + + +def test_the_tool_call_inherits_the_truncation_record_too(): + """The 375 ns leave both fields, so both say so.""" + trace = convert(tool_span(endTimeUnixNano="1755500001000000375")) + record = record_for(trace, "events[0].tool_call.finished_at") + assert record.loss_class is LossClass.NORMALIZED + assert "375 ns is truncated" in record.detail + assert trace.events[0].tool_call.finished_at == trace.events[0].finished_at + + +# --------------------------------------------------------------------------- +# Status, results, usage +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "status", + [ + {"code": "STATUS_CODE_ERROR", "message": "boom"}, + {"code": 2}, + {"code": "STATUS_CODE_OK"}, + {"code": 1}, + {"message": "note"}, + ], +) +def test_a_span_status_is_carried_and_declared_but_never_mapped(status): + """Both enum spellings the pinned mapping produces are recognized.""" + trace = convert(span(status=status)) + assert trace.events[0].outcome is None + assert trace.events[0].extensions["otel"]["span"]["status"] == status + assert record_for(trace, "events[0].outcome").loss_class is LossClass.NORMALIZED + + +@pytest.mark.parametrize( + "status", [None, {}, {"code": 0}, {"code": "STATUS_CODE_UNSET"}] +) +def test_an_unset_status_declares_nothing(status): + """``STATUS_CODE_UNSET`` is the protobuf default: the span said nothing.""" + raw = span() + if status is not None: + raw["status"] = status + trace = convert(raw) + assert trace.losses.for_field("events[0].outcome") == [] + + +def test_a_tool_calls_status_records_which_question_was_left_open(): + with_status = convert(tool_span(status={"code": "STATUS_CODE_ERROR"})) + without = convert(tool_span()) + assert ( + record_for(with_status, "events[0].tool_call.status").loss_class + is LossClass.NORMALIZED + ) + assert ( + record_for(without, "events[0].tool_call.status").loss_class + is LossClass.UNSUPPORTED + ) + assert with_status.events[0].tool_call is not None + assert with_status.events[0].tool_call.status is None + + +def test_an_object_result_becomes_an_opaque_block(): + result = {"kvlistValue": {"values": [{"key": "rows", "value": {"intValue": "3"}}]}} + trace = convert(tool_span(attributes=attrs(("gen_ai.tool.call.result", result)))) + call = trace.events[0].tool_call + assert call is not None + assert [(block.kind, block.raw) for block in call.content] == [ + (ContentBlockKind.OPAQUE, {"rows": 3}) + ] + assert validate_trace(trace) == [] + + +def test_a_result_that_is_neither_text_nor_an_object_is_not_wrapped(): + trace = convert( + tool_span( + attributes=attrs( + ("gen_ai.tool.call.result", {"arrayValue": {"values": [S("a")]}}) + ) + ) + ) + call = trace.events[0].tool_call + assert call is not None and call.content == [] + record = record_for(trace, "events[0].tool_call.content") + assert record.loss_class is LossClass.NORMALIZED + assert trace.events[0].extensions["otel"]["attributes"][ + "gen_ai.tool.call.result" + ] == ["a"] + + +def test_a_deprecated_usage_spelling_is_read_and_the_reading_is_declared(): + """The pinned package states the replacement, so following it is supported. + + Declaring it anyway is what lets a consumer tell a ``prompt_tokens`` + document from an ``input_tokens`` one after the fact. + """ + trace = convert(span(attributes=attrs(("gen_ai.usage.prompt_tokens", I("11"))))) + assert trace.events[0].usage is not None + assert trace.events[0].usage.input_tokens == 11 + record = record_for(trace, "events[0].usage.input_tokens") + assert record.loss_class is LossClass.NORMALIZED + assert "gen_ai.usage.prompt_tokens" in record.detail + + +def test_the_preferred_spelling_wins_and_the_disagreement_is_declared(): + trace = convert( + span( + attributes=attrs( + ("gen_ai.usage.input_tokens", I("11")), + ("gen_ai.usage.prompt_tokens", I("99")), + ) + ) + ) + assert trace.events[0].usage is not None + assert trace.events[0].usage.input_tokens == 11 + classes = { + record.loss_class + for record in trace.losses.for_field("events[0].usage.input_tokens") + } + assert classes == {LossClass.DROPPED} + + +def test_a_token_count_that_is_not_a_count_is_declared_rather_than_coerced(): + trace = convert(span(attributes=attrs(("gen_ai.usage.input_tokens", S("many"))))) + assert trace.events[0].usage is None + assert ( + record_for(trace, "events[0].usage.input_tokens").loss_class + is LossClass.DROPPED + ) + + +def test_gen_ai_usage_total_tokens_is_not_in_the_pinned_vocabulary(): + """A concrete correction of the deleted collector, kept as a test. + + ``otel.py`` read ``gen_ai.usage.total_tokens``; no such attribute exists at + :data:`SEMCONV_VERSION`, so a document carrying it is carried, not read. + """ + trace = convert(span(attributes=attrs(("gen_ai.usage.total_tokens", I("300"))))) + assert trace.events[0].usage is None + assert ( + trace.events[0].extensions["otel"]["attributes"]["gen_ai.usage.total_tokens"] + == 300 + ) + + +def test_the_deprecated_system_attribute_fills_provider_and_says_so(): + trace = convert(span(attributes=attrs(("gen_ai.system", S("anthropic"))))) + assert trace.agent.provider == "anthropic" + record = record_for(trace, "agent.provider") + assert record.loss_class is LossClass.NORMALIZED + assert "gen_ai.system" in record.detail + + +def test_the_current_provider_attribute_is_preferred_over_the_deprecated_one(): + trace = convert( + span( + attributes=attrs( + ("gen_ai.system", S("legacy")), + ("gen_ai.provider.name", S("anthropic")), + ) + ) + ) + assert trace.agent.provider == "anthropic" + assert trace.losses.for_field("agent.provider") == [] + + +def test_a_later_span_using_the_current_spelling_clears_the_deprecation(): + """The record has to describe where the value came from, not where it could. + + The first span offers the value through the deprecated attribute; a later + span offers the same value through the current one. Keeping the + ``NORMALIZED`` record would say the trace's provider is only readable with + convention knowledge, which by then is false. + """ + trace = convert( + span(attributes=attrs(("gen_ai.system", S("anthropic")))), + span(attributes=attrs(("gen_ai.provider.name", S("anthropic")))), + ) + assert trace.agent.provider == "anthropic" + assert trace.losses.for_field("agent.provider") == [] + + +def test_spans_disagreeing_about_the_agent_declare_it(): + trace = convert( + span(attributes=attrs(("gen_ai.request.model", S("a")))), + span(attributes=attrs(("gen_ai.request.model", S("b")))), + ) + assert trace.agent.model == "a" + record = record_for(trace, "agent.model") + assert record.loss_class is LossClass.DROPPED + assert "'b'" in record.detail + + +def test_a_repeated_identical_agent_value_is_not_a_conflict(): + trace = convert( + span(attributes=attrs(("gen_ai.request.model", S("a")))), + span(attributes=attrs(("gen_ai.request.model", S("a")))), + ) + assert trace.losses.for_field("agent.model") == [] + + +def test_a_payload_with_no_gen_ai_attribute_still_converts(): + """The general-tracing case: nothing is a tool call, nothing is lost.""" + trace = convert( + span(name="GET", attributes=attrs(("http.request.method", S("GET")))) + ) + assert validate_trace(trace) == [] + assert trace.events[0].kind is EventKind.UNKNOWN + assert trace.events[0].extensions["otel"]["attributes"] == { + "http.request.method": "GET" + } + assert record_for(trace, "events[].usage").loss_class is LossClass.UNSUPPORTED + + +def test_a_span_with_no_name_reads_as_no_source_type(): + trace = convert({"traceId": TRACE_ID, "spanId": ROOT_SPAN_ID}) + assert trace.events[0].source_type is None + record = record_for(trace, "events[].source_type") + assert record.loss_class is LossClass.UNSUPPORTED + assert "no presence" in record.detail + + +# --------------------------------------------------------------------------- +# Contract guards — the tests that make an undeclared loss fail the suite +# --------------------------------------------------------------------------- + + +def _nested_models(annotation: Any) -> list[type]: + """The pydantic models reachable from a field annotation.""" + from pydantic import BaseModel + + found: list[type] = [] + stack = [annotation] + while stack: + current = stack.pop() + if isinstance(current, type) and issubclass(current, BaseModel): + found.append(current) + continue + stack.extend(getattr(current, "__args__", ()) or ()) + return found + + +def _is_list_of_models(annotation: Any) -> bool: + """True for ``list[SomeModel]`` — the shape that needs an ``[]`` in its path.""" + import typing + + if typing.get_origin(annotation) is list: + return bool(_nested_models(annotation)) + return any(_is_list_of_models(argument) for argument in typing.get_args(annotation)) + + +def _model_paths( + model: type, prefix: str = "", skip: frozenset[str] = frozenset() +) -> set[str]: + """Every field of *model*, as the systemic loss path that would address it. + + Derived from the models rather than listed by hand, so a field added to the + IR shows up here without anyone remembering to add it — which is the only + reason the guards below are worth having. + + A ``list[Model]`` field contributes an ``[]`` segment, so ``events`` yields + ``events[].text`` and ``tool_call.content`` yields + ``tool_call.content[].raw``. That is the same spelling the converters use in + their systemic records, and the reason the content blocks are inside the + contract rather than quietly outside it. + """ + paths: set[str] = set() + for name, info in model.model_fields.items(): + path = f"{prefix}{name}" + if path in skip: + continue + paths.add(path) + nested = _nested_models(info.annotation) + if not nested: + continue + child = f"{path}[]." if _is_list_of_models(info.annotation) else f"{path}." + for model_type in nested: + paths |= _model_paths(model_type, child, skip) + return paths + + +#: Set by the converter on every trace and every event, so an absence is +#: impossible rather than undeclared. Everything else has to be filled or +#: declared, per instance. +STRUCTURAL = frozenset( + { + "ir_version", + "extensions", + "provenance", + "provenance.source_format", + "provenance.producer", + "provenance.captured_at", + "events", + "events[].index", + "events[].kind", + "events[].provenance", + "events[].provenance.source_format", + "events[].provenance.producer", + "events[].provenance.captured_at", + # A required model field: a block with no kind cannot be constructed. + "events[].tool_call.content[].kind", + } +) + + +def _is_absent(value: Any) -> bool: + """True only for ``None`` — the IR's "the source did not carry this". + + ``{}`` and ``[]`` are **not** absences. §8.2 makes that the whole point of + the tri-state rule: ``arguments={}`` is "the source carried an empty + argument map" and an ``OPAQUE`` block whose ``raw`` is ``{}`` carried an + empty object. Treating them as missing would make the guard demand a + declaration for a value that was observed. + + The known limit this inherits — stated when `ACP → IR` first met it — is + that a converter can still dodge a declaration by writing ``{}`` where it + means ``None``. Neither this guard nor invariant 7 can see that; only + reading the produced document can. + """ + return value is None + + +def _addressable_paths() -> set[str]: + """Every path a hub loss record could legitimately name.""" + return _model_paths(CanonicalTrace, "", frozenset({"losses"})) | {"losses"} + + +def _contract_paths() -> set[str]: + """The IR fields this edge owes an answer for, as systemic paths.""" + return _model_paths(CanonicalTrace, "", frozenset({"losses"})) - STRUCTURAL + + +def _systemic(field: str) -> str: + """A concrete record's path, rewritten to the systemic form it answers.""" + import re + + return re.sub(r"\[\d+\]", "[]", field) + + +def _instances(path: str, trace: CanonicalTrace) -> list[str]: + """*path* expanded to one concrete path per instance in *trace*. + + Exempts ``tool_call`` and its descendants on an event that is not a tool + call: invariant 3 does not merely permit the absence, it **requires** it, so + a declaration there would be asserting a loss the IR forbids having. + """ + if "events[]" not in path: + return [path] + concrete: list[str] = [] + for index, event in enumerate(trace.events): + tail = path[len("events[].") :] + if tail.startswith("tool_call") and event.kind is not EventKind.TOOL_CALL: + continue + here = f"events[{index}].{tail}" + if "content[]" not in here: + concrete.append(here) + continue + blocks = event.tool_call.content if event.tool_call else [] + concrete.extend( + here.replace("content[]", f"content[{position}]", 1) + for position in range(len(blocks)) + ) + return concrete + + +def _covered(concrete: str, declared: set[str]) -> bool: + """True when a record addresses *concrete*, its systemic form, or an ancestor. + + The ancestor rule is the IR's own "address the outermost absent node": a + conversion with no usage at all declares ``usage``, not + ``usage.input_tokens``, and the declaration covers everything beneath it. + """ + for candidate in (concrete, _systemic(concrete)): + parts = candidate.split(".") + for cut in range(len(parts), 0, -1): + if ".".join(parts[:cut]) in declared: + return True + return False + + +def _unanswered(trace: CanonicalTrace) -> list[str]: + """Concrete paths that are neither filled nor declared — one per instance. + + This is the per-instance form of the contract. An earlier version asked only + whether a *field* was ever filled or ever declared anywhere in the trace, + which let a field that is filled on one event and silently empty on another + pass; ``events[i].tool_call.started_at`` was exactly that hole. + """ + declared = { + record.field + for record in (trace.losses.records if trace.losses else []) + if record.space is PathSpace.HUB + } + document = trace.model_dump(mode="json") + missing: list[str] = [] + for path in sorted(_contract_paths()): + for concrete in _instances(path, trace): + found, value = resolve_ir_path(document, concrete) + if found and not _is_absent(value): + continue + if not _covered(concrete, declared): + missing.append(concrete) + return missing + + +def test_the_contract_path_list_is_not_empty_or_trivial(): + """The guard below is only worth anything if it enumerates something. + + Derived from the models, so a field added to the IR appears here without + anyone remembering to add it — which is the point. The content-block paths + are asserted explicitly because they were outside the contract until a + review found them there. + """ + paths = _contract_paths() + assert len(paths) > 25 + assert { + "trace_id", + "session_id", + "agent.agent_version", + "outcome.stop_reason", + "usage.cost_usd", + "events[].role", + "events[].tool_call.arguments", + "events[].tool_call.started_at", + "events[].tool_call.finished_at", + "events[].tool_call.content[].text", + "events[].tool_call.content[].raw", + "events[].usage.total_tokens", + } <= paths + + +@pytest.mark.parametrize( + "trace_factory", + [ + pytest.param(only_trace, id="producer-fixture"), + pytest.param(lambda: convert(span()), id="bare-span"), + pytest.param(lambda: convert(tool_span()), id="tool-span-with-nothing"), + pytest.param( + lambda: convert( + tool_span( + status={"code": "STATUS_CODE_ERROR"}, + attributes=attrs( + (GEN_AI_TOOL_NAME, S("read")), + (GEN_AI_TOOL_CALL_ID, S("t1")), + (GEN_AI_TOOL_CALL_ARGUMENTS, {"kvlistValue": {"values": []}}), + ("gen_ai.tool.call.result", S("out")), + ), + ) + ), + id="tool-span-with-everything", + ), + pytest.param( + lambda: convert( + tool_span( + attributes=attrs( + ( + "gen_ai.tool.call.result", + {"kvlistValue": {"values": []}}, + ) + ) + ) + ), + id="opaque-result-block", + ), + pytest.param( + lambda: convert( + span(attributes=attrs(("gen_ai.usage.input_tokens", I("5")))), + span(), + ), + id="usage-on-one-span-only", + ), + pytest.param( + lambda: convert( + span(attributes=attrs(("gen_ai.usage.input_tokens", I("5")))), + span(attributes=attrs(("gen_ai.usage.output_tokens", I("6")))), + ), + id="usage-fields-split-across-spans", + ), + pytest.param( + lambda: convert({"traceId": TRACE_ID, "spanId": ROOT_SPAN_ID}), + id="span-with-nothing-at-all", + ), + ], +) +def test_every_ir_field_is_filled_or_declared_on_every_instance(trace_factory): + """The guard that makes an undeclared loss a test failure. + + **Per instance, not per field.** An earlier version asked only whether a + field was filled *somewhere* or declared *somewhere*, which let a field that + the fixture happens to fill pass forever — including on payloads where it is + empty and nothing declares it. ``events[i].tool_call.started_at`` was + exactly that hole, and the ``tool-span-with-nothing`` case is the payload + that walked through it. + + The parameters matter as much as the assertion: a single trace can only + exercise the instances it contains, so the corpus has to include the + degenerate shapes — a span with no attributes, a tool call with no result, + usage on some spans but not others. + """ + assert _unanswered(trace_factory()) == [] + + +def test_the_guard_catches_the_hole_it_was_strengthened_for(): + """Anti-tautology, aimed at the specific defect a review found. + + A tool span with no readable timestamps leaves ``tool_call.started_at`` and + ``.finished_at`` empty. Before the fix nothing declared them and the + field-level guard was satisfied by the fixture; now their absence is + declared, and removing the declarations has to fail. + """ + trace = convert(tool_span_without_timestamps()) + assert _unanswered(trace) == [] + assert trace.losses is not None + + stripped = trace.model_copy( + update={ + "losses": LossReport( + direction=trace.losses.direction, + records=[ + record + for record in trace.losses.records + if "tool_call.started_at" not in record.field + and "tool_call.finished_at" not in record.field + ], + ) + } + ) + missing = _unanswered(stripped) + assert "events[0].tool_call.started_at" in missing + assert "events[0].tool_call.finished_at" in missing + + +def test_the_guard_fails_when_a_systemic_declaration_is_removed(): + """The other half: a systemic record covers every instance, so losing it + must fail on every instance rather than on none.""" + trace = only_trace() + assert trace.losses is not None + stripped = trace.model_copy( + update={ + "losses": LossReport( + direction=trace.losses.direction, + records=[ + record + for record in trace.losses.records + if record.field != "events[].role" + ], + ) + } + ) + missing = _unanswered(stripped) + assert missing == [f"events[{index}].role" for index in range(len(trace.events))] + + +def test_a_content_block_declares_the_payload_field_it_does_not_carry(): + """F-8: the content descendants are inside the contract, not beside it. + + A ``TEXT`` block read from a string result has no ``raw`` — the attribute + value *is* the text — and an ``OPAQUE`` block has no rendered ``text``. + Neither absence is structural, so each is declared at its own concrete path, + the way `ir_from_atif` declares ``content[].raw``. + """ + text_block = convert( + tool_span(attributes=attrs(("gen_ai.tool.call.result", S("out")))) + ) + assert ( + record_for(text_block, "events[0].tool_call.content[0].raw").loss_class + is LossClass.UNSUPPORTED + ) + assert _unanswered(text_block) == [] + + opaque = convert( + tool_span( + attributes=attrs( + ("gen_ai.tool.call.result", {"kvlistValue": {"values": []}}) + ) + ) + ) + assert opaque.events[0].tool_call.content[0].kind is ContentBlockKind.OPAQUE + assert ( + record_for(opaque, "events[0].tool_call.content[0].text").loss_class + is LossClass.UNSUPPORTED + ) + assert _unanswered(opaque) == [] + + +def test_a_non_tool_event_owes_no_tool_call_declaration(): + """Invariant 3 forbids the payload, so its absence is not a loss. + + Without this exemption the guard would demand a declaration that the IR's + own validator rejects the alternative to — and every plain span in every + payload would need one. + """ + trace = convert(span(name="plain")) + assert trace.events[0].tool_call is None + assert _unanswered(trace) == [] + assert not [ + record + for record in trace.losses.records + if record.field.startswith("events[0].tool_call") + ] + + +def test_every_declared_path_names_a_field_that_exists(): + """The other direction: a typo in a path is a declaration nobody can check.""" + trace = only_trace() + assert trace.losses is not None + known = _addressable_paths() + unknown = sorted( + { + _systemic(record.field) + for record in trace.losses.records + if record.space is PathSpace.HUB + } + - known + ) + assert unknown == [], unknown + + +def test_every_concrete_loss_path_resolves_in_the_canonical_encoding(): + """A declaration a reader of the JSON cannot find is not a declaration. + + Same property the ACP and ATIF edges are held to, and the one whose + violation `exclude_none=True` produced in §8.4. + """ + trace = only_trace() + assert trace.losses is not None + canonical = trace.model_dump(mode="json") + concrete = [ + record.field + for record in trace.losses.records + if record.space is PathSpace.HUB + and record.field.startswith("events[") + and not record.field.startswith("events[]") + ] + assert concrete + unresolved = [ + field for field in concrete if not resolve_ir_path(canonical, field)[0] + ] + assert unresolved == [], unresolved + + lean = trace.model_dump(mode="json", exclude_none=True) + assert any(not resolve_ir_path(lean, field)[0] for field in concrete) + + +def test_stripping_an_arguments_declaration_makes_the_trace_invalid(): + """Invariant 7, exercised against this edge rather than against a fixture. + + The IR promises that a converter which quietly fails to carry arguments + produces an *invalid* trace. This is that promise, checked on the document + this edge actually produces. + """ + trace = convert(tool_span()) + assert validate_trace(trace) == [] + assert trace.losses is not None + stripped = trace.model_copy( + update={ + "losses": LossReport( + direction=trace.losses.direction, + records=[ + record + for record in trace.losses.records + if record.field != "events[0].tool_call.arguments" + ], + ) + } + ) + issues = validate_trace(stripped) + assert len(issues) == 1 + assert "absence must be declared" in issues[0] + + +def test_a_target_space_record_cannot_satisfy_a_hub_invariant(): + """The space is part of the address, not decoration.""" + trace = convert(tool_span()) + assert trace.losses is not None + relabelled = trace.model_copy( + update={ + "losses": LossReport( + direction=trace.losses.direction, + records=[ + record.model_copy(update={"space": PathSpace.TARGET}) + if record.field == "events[0].tool_call.arguments" + else record + for record in trace.losses.records + ], + ) + } + ) + assert validate_trace(relabelled) != [] + + +def test_the_report_does_not_grow_with_the_trace(): + """Systemic losses are declared once, so declared absence stays affordable. + + Two spans of the same shape add per-span records only; the systemic set is + identical. `ir_from_acp` measured the same property, and it is what keeps a + 50-tool-call trace from carrying 50 copies of one sentence. + """ + one = convert(tool_span()) + many = convert(*[tool_span() for _ in range(20)]) + systemic = lambda trace: { # noqa: E731 - local, and reads better inline + record.field + for record in trace.losses.records + if record.field.startswith("events[]") or "[" not in record.field + } + assert systemic(one) == systemic(many) + per_span = len(many.losses.records) - len(systemic(many)) + assert per_span == 20 * (len(one.losses.records) - len(systemic(one))) + + +# --------------------------------------------------------------------------- +# Isolation +# --------------------------------------------------------------------------- + + +def test_this_edge_imports_nothing_but_the_hub(): + """The family rule, checked at the new member rather than only globally. + + ``tests/trajectories/test_trace_ir.py`` asserts that nothing outside the IR + family imports it. This is the other half: the new edge does not reach into + a run path either, so the proposal stays deletable. + """ + source = Path(ir_from_otel_module.__file__).read_text(encoding="utf-8") + imported: set[str] = set() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add(node.module or "") + benchflow_imports = {name for name in imported if name.startswith("benchflow")} + assert benchflow_imports == { + "benchflow.trajectories.ir", + "benchflow.trajectories._otlp_anyvalue", + }, sorted(benchflow_imports) + + +def test_the_anyvalue_decoder_knows_nothing_about_the_hub(): + """The split is only worth making if it is a real boundary. + + `_otlp_anyvalue` turns protobuf JSON wrappers into Python values. If it ever + imports the IR or the loss model, it has stopped being a decoder and the + module has become a second place where mapping decisions live. + """ + source = Path(anyvalue_module.__file__).read_text(encoding="utf-8") + imported: set[str] = set() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add(node.module or "") + assert [name for name in imported if name.startswith("benchflow")] == [] + assert [name for name in imported if name.startswith("opentelemetry")] == [] + + +def test_the_edge_takes_no_opentelemetry_dependency(): + """The versions are pinned so the mapping is checkable, not so it can import. + + Adding ``opentelemetry`` as a runtime dependency would change ``uv.lock``, + which nine CI jobs verify with ``uv sync --locked``. The whole point of + reading dictionaries is that this slice costs the lock file nothing. + """ + source = Path(ir_from_otel_module.__file__).read_text(encoding="utf-8") + for node in ast.walk(ast.parse(source)): + if isinstance(node, (ast.Import, ast.ImportFrom)): + names = ( + [alias.name for alias in node.names] + if isinstance(node, ast.Import) + else [node.module or ""] + ) + assert not any(name.startswith("opentelemetry") for name in names), names + + +def test_no_input_dictionary_is_mutated(): + """The reader is a reader: the caller's payload comes back unchanged.""" + document = payload() + before = json.dumps(document, sort_keys=True) + otlp_json_to_ir(document) + assert json.dumps(document, sort_keys=True) == before diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py index 651a98661..34000219c 100644 --- a/tests/trajectories/test_trace_ir.py +++ b/tests/trajectories/test_trace_ir.py @@ -549,6 +549,8 @@ def test_the_oracle_record_has_its_own_kind_and_role(): "ir_from_acp", "ir_to_atif", "ir_from_atif", + "ir_from_otel", + "_otlp_anyvalue", "ir_round_trip", ) """The unwired modules of the proposal: the IR and its converters. From d4a105f61d0995f6b48ab50a3d25333e53615689 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Tue, 18 Aug 2026 20:12:54 +0200 Subject: [PATCH 14/23] docs(trace-interop): record the ingest-only OTel scope as our own decision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds §8.12. No code changes: the edge it describes is the one already committed in 796e436d, and `IR -> OTel` is not implemented here or anywhere. For this implementation OpenTelemetry is an ingest boundary. `OTLP/JSON -> IR` exists; `IR -> OTel` is deferred. What the section is careful about is *whose* decision that is: it is ours, it is current, and it is reversible. No maintainer has said the emitter is out of scope, and none has said it is required — open question 5 is untouched and stays on the list in §8.11. The reasoning is five things checkable in the tree rather than a reading of anyone's intent: no outbound OTel consumer exists, no exporter or backend is integrated, the OTel support that did exist was a receiver, the consumer named by the task's own issue body reads ACP, and there is no contract to build an emitter against. That last one is the load-bearing reason — without a consumer, the span tree for a rollout, the id encoding and the commitment to an `_incubating` vocabulary would all be decided by this implementation alone, and the hub is the one place a wrong answer propagates to every format. Three things the section refuses to claim, because each would be an overreach in a different direction: that `IR -> OTel` will never be needed; that the `<->` in the task title has been resolved (it has not, and this decision is not evidence about what it meant); and that the task has been narrowed — if the answer comes back "both", the work is an additional edge, not a redesign. The reversal condition is a consumer or a contract, and one constraint holds until then: the IR must not grow a span-tree field, an id-minting policy or an OTel-shaped slot in anticipation. The hub is a superset of what BenchFlow can observe, and a field added for a converter that does not exist has no evidence behind it. The emitter, when written, takes the IR as it finds it and declares what it cannot express — the contract every outbound edge already has. --- docs/trace-interop.md | 87 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/docs/trace-interop.md b/docs/trace-interop.md index 54debc4c1..dfdabbb34 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -467,11 +467,11 @@ them. in `normalize_acp_usage`? 5. Is OpenTelemetry wanted as an inbound receiver, an outbound emitter, or both? The only implementation ever written was inbound, and it was removed - deliberately. **Partly acted on, not answered:** §8.3 implements the inbound - half (`OTel → IR`) because it is useful whichever way the question is - resolved — it lets BenchFlow *read* traces from instrumented agents without - committing to emitting any. The emitter is not written, and §8.11 lists the - decisions it would depend on. + deliberately. **Still unanswered by anyone who owns the answer.** §8.3 + implements the inbound half (`OTel → IR`); §8.12 records *our* decision to + scope this implementation to ingest and to defer the emitter, and is careful + to say that the decision is ours rather than a reading of this question. The + emitter is not written, and §8.11 lists the decisions it would depend on. **Still open.** §8 takes a *provisional* position on question 1 — a canonical hub — and implements it in isolation so it can be reviewed as code. That is a @@ -960,6 +960,10 @@ look like, which ids to mint, which encoding to write them in, and whether emitting `gen_ai.*` attributes commits the project to an `_incubating` vocabulary. Those are in §8.11. +**This is a deferral we chose, not one we were given.** §8.12 records it as a +scope decision with its reasoning and its reversal conditions, so a reader can +tell it apart from the maintainer decision that is still outstanding. + ### 8.4 A worked example **FACT.** The document below is produced by the models in `ir.py` and compared @@ -1641,7 +1645,8 @@ it. the package a real dependency. 8. **Is an `IR → OTel` emitter wanted at all?** Open question 5, still open for the outbound half. It needs answers to 1, 2 and 4 before it can be written - without inventing a span tree for BenchFlow rollouts. + without inventing a span tree for BenchFlow rollouts. §8.12 defers it in the + meantime; deferring is not answering, and the question stays on this list. #### Human verification @@ -1709,3 +1714,73 @@ narrowed by the fact that a person signed the steps off. - **Nothing is wired.** `ir_from_otel.py` and `_otlp_anyvalue.py` join the closed family of §8.7; no run path imports either, and no artifact changes because they exist. + +### 8.12 Scope decision: OpenTelemetry is ingest-only in this implementation + +> **DECISION — ours, current, and reversible.** This section records a choice +> made by the author of this work. **It is not a maintainer ruling, and nothing +> here should be read as one.** No maintainer has stated that `IR → OTel` is out +> of scope, and none has stated that it is required. + +**The decision.** For this implementation, OpenTelemetry is an **ingest +boundary**. The edge that exists is `OTLP/JSON → Canonical Trace IR` (§8.3). +`Canonical Trace IR → OTel` is **deferred**, not rejected. + +#### Why defer rather than build + +Five observations, each checkable in the tree today, and none of them a +statement about what the maintainers want: + +1. **No outbound OTel consumer exists in the repository.** Nothing reads spans + BenchFlow would produce. +2. **No OTel exporter or backend is integrated.** There is no OTLP endpoint, no + collector configuration, and no observability backend referenced anywhere in + `src/` or `docs/`. +3. **The OTel support that did exist was inbound.** The removed `OTelCollector` + (§4.2) was an OTLP/HTTP *receiver*; it contained no emitter. +4. **The consumer named by the task's own issue body consumes ACP.** The Viewer + work reads `trajectory/acp_trajectory.jsonl` and explicitly declines to + become another trace schema. +5. **There is no concrete contract to build the emitter against.** Without a + consumer, every question §8.11 lists — the span tree for a rollout, which ids + to mint, which encoding to write them in, whether to commit to an + `_incubating` vocabulary — would be answered by this implementation alone. + Those answers would be **speculative policy baked into the hub**, and the hub + is the one place where a wrong answer propagates to every other format. + +Building an edge against no consumer is how a format acquires conventions nobody +chose. Deferring costs nothing that cannot be recovered; guessing does. + +#### What this decision explicitly does not claim + +- **It does not claim `IR → OTel` will never be needed.** It claims there is + nothing to build it against *today*. +- **It does not reinterpret the `<->` in the task title.** That symbol is + ambiguous — it can mean bidirectional conversion on every edge, or simply "among + these three formats" — and this decision is **not** evidence that ingest-only + is what it meant. The ambiguity is unchanged and is still a maintainer's to + resolve (§6, question 5). +- **It does not narrow the task.** If the answer comes back "both", the work is + an additional edge, not a redesign. + +#### Reversal conditions, and what must not happen first + +The deferral ends when **either** exists: + +- a **consumer** that would read BenchFlow-emitted spans — a collector, a + backend, a hosted viewer, a downstream tool; or +- a **contract** that fixes the open questions independently of this + implementation — a maintainer decision, a vendored specification, or a + convention this project agrees to follow. + +Until then, **the IR must not be changed in anticipation of the emitter.** No +span-tree field, no id-minting policy, no OTel-shaped slot may be added to +`ir.py` on the argument that an emitter would want it. The hub is a superset of +what BenchFlow can *observe* (§8.2), and a field added for a converter that does +not exist is a field with no evidence behind it. When the emitter is written it +takes the IR as it finds it and declares whatever it cannot express — which is +the same contract every other outbound edge is held to (§8.3, `IR → ATIF`). + +Adding the edge later is cheap by construction: the outbound direction is one +module and one loss report, exactly as `ir_to_atif.py` is, and the family test in +§8.7 is what keeps that addition from quietly wiring anything. From ab2e0010182d0a25ab6770603772b0a91f570f21 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Tue, 18 Aug 2026 20:27:47 +0200 Subject: [PATCH 15/23] docs(trace-interop): state the OTel direction as one answer, not two halves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two inconsistencies §8.7 was left with after Slice F. "Not implemented, deliberately: `IR -> OTel`" was true but said nothing about whose deliberation, which is the whole distinction §8.12 exists to draw. The status now reads as one state — the OTel direction is implemented inbound and deferred outbound, by a decision of ours, reversibly — instead of as an implemented edge sitting next to an unexplained absence. The isolation paragraph still claimed "each converter imports one benchflow module — the IR", which stopped being true when `_otlp_anyvalue` was split out of the OTel edge. It now says what each converter actually imports, and keeps the property that was the point: every converter reads or writes its format as data rather than through the module that already handles it. Documentation only. No code, no test, and no `IR -> OTel`. --- docs/trace-interop.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/trace-interop.md b/docs/trace-interop.md index dfdabbb34..4e6e050ad 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -1210,8 +1210,12 @@ here. invariants, the validation suite, this section, four converters — `ACP → IR`, `IR → ATIF`, `ATIF → IR`, `OTLP/JSON → IR` — each with its loss report, and the round-trip measurement over the ATIF pair (§8.10). -- **Not implemented, deliberately:** `IR → OTel`, any wiring into a run path, - any on-disk artifact, any capture-layer enrichment. +- **Deferred by a decision of ours, reversibly:** `IR → OTel` — see §8.12 for + the reasoning and the conditions that end the deferral. The OTel direction is + *implemented inbound and deferred outbound*, which is one state and not two + half-answers. +- **Not implemented:** any wiring into a run path, any on-disk artifact, any + capture-layer enrichment. - **Unchanged:** every existing format, exporter, artifact and code path. `export_atif.py` in particular is untouched and remains the only writer of `trainer/atif.json`. @@ -1221,10 +1225,14 @@ The isolation property is unchanged in substance and restated at a new boundary: `ir_from_otel.py`, `_otlp_anyvalue.py` and `ir_round_trip.py` form a closed family that may import each other, and `test_only_the_ir_family_imports_the_ir` asserts that nothing else in -`src/benchflow` imports any of them. Each converter imports one benchflow module -— the IR — and reads or writes its own format as data; in particular -`ir_to_atif` does not import `export_atif`, and pins the shared schema version by -test instead. +`src/benchflow` imports any of them. + +Every converter reads or writes its own format **as data**, never through the +module that already handles it: `ir_to_atif` does not import `export_atif` and +pins the shared schema version by test instead, and `ir_from_otel` imports no +`opentelemetry` package at all. Each imports the IR, and `ir_from_otel` also +imports `_otlp_anyvalue` — the decoder split out of it, which knows nothing +about the hub and is the only family member that imports none of the others. ### 8.8 What the first converter showed From 67ccd12f837c2b15c1f8fd6e632dc7e775c63ca4 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Tue, 18 Aug 2026 22:11:25 +0200 Subject: [PATCH 16/23] feat(trajectories): write ACP capture events from the canonical IR, unwired MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fifth edge of the hub, and the first one whose target refuses partial output. Nothing imports it, nothing writes its result to disk, `_capture.py` is untouched and still the only producer of the format, and `uv.lock` is intact. The target is the **ACP-session capture event format** — the three record shapes Slice A's schema pins — and deliberately not the `acp_trajectory.jsonl` artifact. That file also holds oracle records, `hosted_env` records and session-factory `steps`, and §2.1 already recorded that no artifact-level contract exists in this repository. An edge cannot target a format nobody has defined; this one targets the part that is defined and says so. Every record shape has `additionalProperties: false` and six required fields on a tool call, so this is the first outbound edge that can be *unable* to write. It fails closed: `ir_to_acp_capture_events` either represents every event or raises `AcpCaptureNotRepresentable` — a `ValueError` carrying the blocking `LossRecord`s, following `PrimeSftTrajectoryJsonlError` and the `ValueError` `ir_to_atif` already raises. A partial event list is indistinguishable from a complete one once it leaves the function: the target has no envelope, no count and no marker for "some events are missing". `acp_capture_blockers` answers the same question without an exception, through the same pass, so asking and converting cannot disagree. Nothing is invented. `status` is never synthesized — ACP's vocabulary is closed, every member asserts a lifecycle state, and a fabricated one would reach the viewer and be shown to a person as an observation. `ORACLE` and `UNKNOWN` have no record shape in the contract, so they are refused rather than dropped, and the schema is not widened to admit them. A content block with no source block is refused: ACP stores wire blocks verbatim, two shapes are known to be consumed, and choosing one would be inventing structure. `title` is the single exception, written as `""` only because the contract documents that as its representation of an absent title, and declared SYNTHESIZED because a reader cannot tell it from an observed empty one. `tool_call_id` carries identical documentation and is deliberately not treated the same way: an empty title is a missing label, an empty id would turn an absent identity into an observable one. **`name_semantics` gates the `kind` slot**, and a review is what put it there. An ACP `kind` is a *category* — `ToolKind` calls itself "Category tag for tool calls" — while ATIF's `function_name` and OTel's `gen_ai.tool.name` name particular tools. An earlier version wrote the name through regardless, so an ATIF document with a usable status exported `kind="read_file"`: a tool name in a category slot, with only a DROPPED record to show for it. That is not a normalization a reader can undo, it is a reinterpretation. Only `name_semantics == "acp_kind"` is writable now, and the string is never inspected — a `function_name` of `read` collides with a real ToolKind member and is refused anyway. Representability is decided by the data and never by the lineage. `Provenance` is not read, and a test asserts the module never references it: an OTel-derived trace carrying every required value would export, and an ACP-derived trace missing one does not. `ACP → IR → ACP` reproduces its input exactly — structural equality and key-order preservation — on the two captured rollouts of §5.2 and on the suite's fixture with its drop-one subsets. Three corpora, not a universality claim. 77 tests, every emitted record validated against the published schema. Human-verified end to end, G1-G10, against `e2e-g/PROCEDURE.md`; §8.3 records the limits that sign-off does not extend to — no artifact-level support, no exportable oracle or unknown events, no general ATIF/OTel tool results, no `OTel → Viewer`, and no wiring. --- docs/trace-interop.md | 203 ++++- src/benchflow/trajectories/ir_to_acp.py | 770 +++++++++++++++++ tests/trajectories/test_ir_to_acp.py | 1002 +++++++++++++++++++++++ tests/trajectories/test_trace_ir.py | 1 + 4 files changed, 1960 insertions(+), 16 deletions(-) create mode 100644 src/benchflow/trajectories/ir_to_acp.py create mode 100644 tests/trajectories/test_ir_to_acp.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md index 4e6e050ad..95645f7ca 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -171,10 +171,10 @@ content blocks permissive for this reason. ### 2.4 Records in the file that the ACP-session emitter did not produce -**FACT.** Two other sources can put records into `acp_trajectory.jsonl`. Neither -goes through `_events_to_trajectory`, and the schema models neither. Note that -the artifact *path* is the same in every mode, so the same filename can hold -different record shapes depending on how the rollout ran. +**FACT.** **Three** other sources can put records into `acp_trajectory.jsonl`. +None goes through `_events_to_trajectory`, and the schema models none of them. +Note that the artifact *path* is the same in every mode, so the same filename +can hold different record shapes depending on how the rollout ran. **Session-factory Sessions.** `_snapshot_session_trajectory` duck-types on the ACP streaming attributes; a `benchflow.agents.protocol.Session` has none, so the @@ -192,8 +192,40 @@ final write depends on the trajectory list the caller passes to mutually exclusive today, and **no production path emits a mixed ACP + oracle artifact**. +**`hosted_env`, which writes the file directly.** `_reconstruct_trajectory` +(`hosted_env.py:655`) rebuilds a trajectory from a vf-eval `results.jsonl` and +`hosted_env.py:552` writes it to the artifact path **without going through +`TrajectoryWriter`**. Its own docstrings call the output "ACP-shaped" and +"ACP-style session events", and the resemblance is real but partial: the records +reuse the `user_message` / `agent_message` type strings while carrying the text +under **`content`** rather than `text`, adding `ts` and `example_index`, and +introducing a **fourth type, `reward`**, with `value` and `source` +(`hosted_env.py:720-767`). Every one of those records fails the §2.1 schema — +on the required field, on `additionalProperties`, and for `reward` on the type +enum. This is provenance-tagged: `trajectory_source="hosted_env"`. + +**A fourth record family is recognized by consumers, and nothing in this +repository writes it.** `viewer.py:573` branches on +`{session_meta, response_item, event_msg, turn_context}`, and +`publish/traj_report.py:243-246` reads the same set. No site under `src/` emits +those types. They are an **externally produced** shape: `cli/traj.py:549` +identifies `session_meta` as a **Codex** payload, alongside Claude's `cwd` +events, and that CLI uploads sessions recorded by those harnesses. So this is a +foreign trace format the consumers tolerate, not a BenchFlow producer — worth +separating, because "a consumer knows how to read it" and "something here writes +it" are different facts and only the first is shown. + So the file is not a superset of the ACP-session emitter's output; it holds one of several possible shapes, and no single site defines its full vocabulary. + +**There is, however, a partial typed contract nobody has connected to this +schema**: `TrajectorySource = Literal["acp", "scraped", "partial_acp", +"hosted_env"]` (`models.py:17`) records *which lineage produced the file* and +reaches `result.json`. It has four members and covers neither the oracle records +nor the family above — in oracle mode `"oracle"` is the **agent name**, not the +trajectory source. So the artifact's provenance is typed while its *content* is +not, which is the gap §2.1 describes from the other side. + Whether `oracle` belongs in the ACP trajectory contract or is a separate downstream concern is an open question (see §6), and this document does not settle it. @@ -623,9 +655,11 @@ Three design choices carry most of the weight: **`ACP → IR`** ([`ir_from_acp.py`](../src/benchflow/trajectories/ir_from_acp.py)), **`IR → ATIF`** ([`ir_to_atif.py`](../src/benchflow/trajectories/ir_to_atif.py)), -**`ATIF → IR`** ([`ir_from_atif.py`](../src/benchflow/trajectories/ir_from_atif.py)) -and **`OTLP/JSON → IR`** -([`ir_from_otel.py`](../src/benchflow/trajectories/ir_from_otel.py)) **are +**`ATIF → IR`** ([`ir_from_atif.py`](../src/benchflow/trajectories/ir_from_atif.py)), +**`OTLP/JSON → IR`** +([`ir_from_otel.py`](../src/benchflow/trajectories/ir_from_otel.py)) and +**`IR → ACP capture events`** +([`ir_to_acp.py`](../src/benchflow/trajectories/ir_to_acp.py)) **are implemented**, all unwired and still provisional. `IR → OTel` is not, and is deliberately not sketched — §8.11 lists the decisions it depends on. @@ -689,10 +723,12 @@ trace length. unwired: `export_atif.py` is untouched and still the only writer of `trainer/atif.json`. -**FACT — the hub reproduces the direct exporter.** For any conformant capture -input, `ir_to_atif(acp_events_to_ir(events), prompts=P)` produces the same -document as `trajectory_to_atif_record(events=events, prompts=P)`, with one -deliberate exception (oracle, below). Asserted by +**FACT — the hub reproduces the direct exporter on everything measured.** +`ir_to_atif(acp_events_to_ir(events), prompts=P)` produces the same document as +`trajectory_to_atif_record(events=events, prompts=P)`, with one deliberate +exception (oracle, below). **Measured, not proved for all inputs**: the evidence +is the corpus named next, and no argument here establishes the property for +capture inputs outside it. Asserted by `tests/trajectories/test_ir_to_atif.py` over events driven through the production capture path and nine further shapes, and confirmed against the two real rollouts of §5.2: for both, the document produced through the hub is @@ -951,6 +987,138 @@ the whole run, which a batch does not promise. When no span carries them at all, the same fields become `UNSUPPORTED` instead. Two different reasons for the same empty field, kept apart. +#### IR → ACP capture events + +**Implemented** ([`ir_to_acp.py`](../src/benchflow/trajectories/ir_to_acp.py)), +unwired, in memory only. + +**The target is the ACP-session capture event format, not the +`acp_trajectory.jsonl` artifact.** §2.1 and §2.4 are the reason: the file holds +records from four sources — the ACP-session emitter, `_run_oracle`, +`hosted_env`, and whatever a session-factory `Session` puts in `steps` — plus a +fifth family the viewer reads and nothing here writes. **No artifact-level +contract exists**, so there is nothing for an edge to target at that level. The +capture format *is* defined, by the §2.1 schema, and that is what this edge +writes. Whether the artifact should acquire a contract is untouched by this +slice. + +| IR | ACP capture | Class | Loss recorded | +|---|---|---|---| +| `USER_MESSAGE` / `AGENT_MESSAGE` / `AGENT_REASONING` + text | `text_event` with the matching `type` | preserved | — including an observed `""` | +| `TOOL_CALL` with every required value | `tool_call_event` | preserved | — | +| `TIMEOUT` with `outcome == "wall_clock_timeout"` and its three extension keys | `agent_timeout_event` | preserved | — recovered by name from `extensions` | +| `tool_call.title` absent | `""` | **synthesized** | `events[i].tool_call.title` — the contract's own representation of an absent title | +| `content[].raw` | the block, verbatim | preserved | `…content[i].kind` — the IR's text/opaque classification is not part of the wire shape | +| several `reasoning_segments` | one `agent_thought` | dropped | `events[i].reasoning_segments` — §5 loss #10, in the writing direction | +| `arguments`, `name_semantics`, per-event and per-call timestamps, `role`, `source_type`, `usage`, `extensions` | *(no slot)* | dropped | one record each, when the trace carries them | +| `trace_id`, `session_id`, `started_at`, `finished_at`, `agent.*`, `usage`, `outcome.*`, trace `extensions` | *(no envelope at all)* | dropped | one record each | + +**Five things make an event unrepresentable, and each refuses rather than +invents.** A tool call with no ACP `status`, no `tool_call_id` or no `kind`; a +content block with no source block to write; a text event with no text; a +timeout missing or mistyping any of its four required values; and an `ORACLE` or +`UNKNOWN` event, which have no record shape in the contract at all. + +**`status` is the one that matters.** The ACP vocabulary is closed and every +member asserts a lifecycle state, so there is no neutral value and the IR's +`ToolStatus.UNKNOWN` has no counterpart. A synthesized status would reach the +viewer and be displayed to a person as an observation. It is never written. + +**Fail closed.** A trace with one unrepresentable event yields **no** events — +`AcpCaptureNotRepresentable`, a `ValueError` carrying the blocking +`LossRecord`s. The target has no envelope, no count and no marker for "some +events are missing", so a partial list is indistinguishable from a complete one +the moment it leaves the converter. + +**Representability is decided by the data, never by the provenance.** An +OTel-derived trace that carries every required value exports; an ACP-derived +trace missing one does not. `Provenance` is not read, and a test asserts the +module never references it. Today ACP-derived traces usually export and +OTel-derived ones usually do not — that is an observation about what those edges +currently carry, not a rule. + +**`name_semantics` is part of that data, and it gates the `kind` slot.** ACP's +`kind` is a *category* — `ToolKind` calls itself "Category tag for tool calls", +`_canonical_tool_kind` defaults an absent one to `other`, and the production +values (`execute`, `edit`, `fetch`, `think`) are categories. ATIF's +`function_name` and OTel's `gen_ai.tool.name` name *particular tools*. Writing +one of those into `kind` would not be a normalization a reader could undo; it +would assert a category nobody observed. So only `name_semantics == "acp_kind"` +may be written, and **the string is never inspected** — a `function_name` of +`read` coincides with a real `ToolKind` member and is refused anyway. + +**Why `content[].raw` is required, and what it costs.** The schema leaves +`content_block` permissive (`type: object`), so this is **not** a constraint the +contract imposes — it is a conservative choice not to invent wire structure. ACP +stores `session/update.content` exactly as the wire delivered it, and two shapes +are known to be consumed: the nested `{"type": "content", "content": {...}}` and +the flat `{"text": …}`. A block that carries rendered text but no source block +gives no way to choose between them. + +The cost is concrete and worth stating plainly: **`ATIF → IR` and `OTLP → IR` +both produce content blocks with `text` and no `raw`, so a tool result that did +not come from ACP is normally not representable today.** Relaxing this would +require defining a canonical ACP content-block construction policy — which shape +to write, and on whose authority. **No such policy is defined here**, and adding +one is a decision rather than an implementation detail. + +**What the round trip has actually been measured on.** `ACP → IR → ACP` +reproduces its input with **structural equality** on three corpora: the two +captured rollouts of §5.2 — H1, five records, and H2, four records ending in a +real wall-clock timeout — and the suite's conformant fixture of five records +together with its five drop-one subsets. **Key-order preservation** and **schema +validity** are checked separately, the second over six further shapes. Byte +equality through `redact_acp_trajectory_jsonl` also holds, but adds nothing +beyond key order: the serializer redacts both sides, so redaction cancels. + +**No universality is claimed or demonstrated.** There is no property test, and +three corpora are three corpora: nothing here establishes the behaviour for +conformant inputs outside them. What *is* structural is the alternative — the +edge either reproduces its input or refuses, so an unmeasured input cannot +silently produce a degraded record. It produces none. + +#### Human verification of the ACP edge + +**HUMAN E2E VERIFIED, 2026-08-18, G1–G10 all pass**, against +`e2e-g/PROCEDURE.md`. Every step was carried out and judged by a person rather +than by the suite. + +| | what was verified | +|---|---| +| G1 | H1 and H2 are real BenchFlow rollouts — `result.json` carries `trajectory_source: "acp"`, the typed lineage of `models.py:17`, and each record's keys are the emitter's literal writes | +| G2 | `ACP → IR → ACP` read event by event: structural equality **and** key-order preservation on H1 (5 records) and H2 (4 records) | +| G3 | every loss path resolves in the canonical encoding; **no unexpected synthesis**; `name_semantics: "acp_kind"` recorded `NORMALIZED`, not `DROPPED` | +| G4 | the laundering path is closed — a `function_name` is refused even when its value spells `read`, `write`, `bash` or another real `ToolKind` member; the OTel fixture shows 8 blockers in four independent kinds | +| G5 | fail-closed atomicity: the blocker is on the last event and **no partial prefix** is returned | +| G6 | `title=None` → `""` with `SYNTHESIZED`; an observed empty title synthesizes nothing; `tool_call_id=None` is refused | +| G7 | representability and output are independent of provenance, and an ACP-derived trace with an `UNKNOWN` status is still refused | +| G8 | 12 emitted records validated, 0 errors, and all four negative controls rejected | +| G9 | 27/27 mutations caught, no `MISSED`, no `SKIP`, exit 0; the files restored, proven by a green `tests/trajectories` | +| G10 | no importer outside the IR family, no wiring, no file I/O; `uv.lock`, `ir.py`, the schema and the viewer unchanged | + +**The verification found two defects, both in the procedure, neither in the +converter.** G4 described the OTel fixture's blockers as three independent kinds +when there are four, and the inspector's G7 header said "two traces" while +printing four. Both are corrected in the scaffolding, which lives outside the +repository and is not part of this change. + +**What the sign-off does not extend to.** Each of these is unchanged by it: + +- **Not** that every conformant ACP document round-trips. The measured corpora + are H1, H2 and the suite fixture with its subsets. +- **Not** that the `acp_trajectory.jsonl` **artifact** is supported. The target + is the capture *event format* of §2.1; that file also holds oracle records, + `hosted_env` records and session-factory `steps`, and §2.4 records that no + artifact-level contract exists. +- **Not** that `ORACLE` or `UNKNOWN` events are exportable — they have no record + shape in the contract and are refused. +- **Not** that ATIF or OTel tool results are generally exportable. Their content + blocks carry `text` and no `raw`, and their names are not ACP kinds. +- **Not** that `OTel → Viewer` works. The OTel fixture is four independent kinds + of refusal away from exporting, and nothing here changes that. +- **Not** that anything is wired. There is no runtime path, and this edge writes + no file — it returns records. + #### IR → OpenTelemetry **Not implemented, and deliberately not sketched further.** The inbound edge @@ -1207,9 +1375,12 @@ here. ### 8.7 Status - **Implemented:** the IR types, the loss model with its path spaces, the - invariants, the validation suite, this section, four converters — `ACP → IR`, - `IR → ATIF`, `ATIF → IR`, `OTLP/JSON → IR` — each with its loss report, and - the round-trip measurement over the ATIF pair (§8.10). + invariants, the validation suite, this section, five converters — `ACP → IR`, + `IR → ATIF`, `ATIF → IR`, `OTLP/JSON → IR`, `IR → ACP capture events` — each + with its loss report; the round-trip measurement over the ATIF pair (§8.10), + and the ACP pair's round-trip evidence in §8.3 and its test suite. There is no + §8.10-style measurement for the ACP pair: `ACP → IR → ACP` either reproduces + its input exactly or refuses, so there is no partial preservation to quantify. - **Deferred by a decision of ours, reversibly:** `IR → OTel` — see §8.12 for the reasoning and the conditions that end the deferral. The OTel direction is *implemented inbound and deferred outbound*, which is one state and not two @@ -1222,8 +1393,8 @@ here. The isolation property is unchanged in substance and restated at a new boundary: `ir.py`, `ir_from_acp.py`, `ir_to_atif.py`, `ir_from_atif.py`, -`ir_from_otel.py`, `_otlp_anyvalue.py` and `ir_round_trip.py` form a closed -family that may import each other, and +`ir_from_otel.py`, `ir_to_acp.py`, `_otlp_anyvalue.py` and `ir_round_trip.py` +form a closed family that may import each other, and `test_only_the_ir_family_imports_the_ir` asserts that nothing else in `src/benchflow` imports any of them. diff --git a/src/benchflow/trajectories/ir_to_acp.py b/src/benchflow/trajectories/ir_to_acp.py new file mode 100644 index 000000000..998389cbd --- /dev/null +++ b/src/benchflow/trajectories/ir_to_acp.py @@ -0,0 +1,770 @@ +"""Canonical Trace IR → ACP-session capture events (Slice G). + +> **PROVISIONAL.** Companion to :mod:`benchflow.trajectories.ir`, itself an +> unapproved proposal (`docs/trace-interop.md` §8). Nothing imports this module, +> nothing writes its output to disk, and no capture path, exporter or artifact +> changes because it exists. + +## What this edge targets, and what it does not + +**The target is the ACP-session capture event format** — the record vocabulary +pinned by Slice A's JSON Schema and produced by +`benchflow.trajectories._capture._events_to_trajectory`. Three record shapes, +each with `additionalProperties: false`. + +**The target is *not* the `acp_trajectory.jsonl` artifact.** That file holds +records from several producers — the ACP-session emitter, `_run_oracle`, +`hosted_env._row_to_acp_events`, and whatever a session-factory `Session` puts +in `steps` — and §2.1 records that **no artifact-level contract exists in this +repository**. An edge cannot target a format nobody has defined, so this one +targets the part that *is* defined. Output is a list of event dictionaries in +memory; writing them anywhere is not this module's business. + +## Representability is a property of the data, never of the provenance + +A trace is exportable when its events carry what the ACP contract requires. It +is **not** exportable because it came from ACP, and not unexportable because it +came from OTel or ATIF. :attr:`~benchflow.trajectories.ir.Provenance` is never +read here, and a test asserts that: a hand-built trace carrying every required +value exports whatever its ``source_format`` says, and an ACP-derived trace +missing one does not. + +The practical consequence today is that ACP-derived traces usually export and +OTel-derived ones usually do not — but that is an observation about what those +edges currently carry, not a rule this module applies. + +## Fail closed + +**A conversion either represents every event or represents none of it.** +:func:`ir_to_acp_capture_events` raises :class:`AcpCaptureNotRepresentable` +rather than returning the events it managed to build, because a partial event +list is indistinguishable from a complete one once it leaves this function — the +target has no envelope, no count and no marker for "some events are missing". +Returning one would be handing a caller a trajectory that silently lost a tool +call and looks like a successful conversion. + +The exception is a ``ValueError`` subclass, following +`PrimeSftTrajectoryJsonlError` and the ``ValueError`` `ir_to_atif` already +raises, and it carries the blockers as +:class:`~benchflow.trajectories.ir.LossRecord` values — the loss model this +family already uses to address a field by path. A caller that wants to *ask* +rather than to *handle* calls :func:`acp_capture_blockers` and gets the same +records with no exception involved. + +## What is never invented + +- **`status` is never synthesized.** The ACP vocabulary is + ``pending`` / ``in_progress`` / ``completed`` / ``failed`` / ``cancelled``, + and every member asserts something about a tool call's lifecycle. There is no + neutral value, the IR's :attr:`~benchflow.trajectories.ir.ToolStatus.UNKNOWN` + has no counterpart, and a fabricated status reaching the viewer would be + displayed to a person as an observation. A tool call whose status the IR does + not know is **not representable**, full stop. +- **`oracle` and `unknown` events are outside the codomain.** The schema models + three record shapes and neither of these is one of them. They are not dropped + to make a conversion succeed, the schema is not widened to admit them, and no + schema-invalid record is emitted: a trace containing one is not representable. +- **No trace-level value is smuggled in.** The capture format is a flat event + stream with no envelope, so `trace_id`, `session_id`, the agent block, usage + and outcome have nowhere to go. They are declared dropped, not attached to an + event that never carried them. + +## The `kind` slot takes a category, not a tool name + +ACP's `kind` is a **category tag** — `benchflow.acp.types.ToolKind` says so in +its own docstring, `_canonical_tool_kind` defaults an absent one to `"other"`, +and the values seen in production (`execute`, `edit`, `fetch`, `think`) are +categories. ATIF's `function_name` and OTel's `gen_ai.tool.name` are names of +*particular tools*. + +So a tool call is writable into that slot **only when +:attr:`~benchflow.trajectories.ir.ToolCall.name_semantics` is `"acp_kind"`**. +Any other label is a refusal, not a normalization: a normalization is a value a +reader can undo with convention knowledge, and there is nothing to undo here — +the record would simply assert a category nobody observed. + +The string itself is never inspected. A `function_name` of `"read"` coincides +with a real `ToolKind` member and is still refused, because matching the +vocabulary by accident is not being drawn from it, and a rule that looked at the +value would admit exactly the cases most likely to be wrong. + +## The one place an empty string is written for an absence + +`title` — and only because the contract says so. Slice A's schema documents the +field as *"Human-readable label. Empty string when the agent supplied none."*, +so ``""`` is the contract's own representation of an absent title rather than +this converter's invention. It is still declared +:attr:`~benchflow.trajectories.ir.LossClass.SYNTHESIZED`, because a reader of +the output cannot tell that empty string from one the agent really supplied. + +`tool_call_id` carries the *same* documentation — *"May be the empty string when +the agent omitted it"* — and is deliberately **not** treated the same way here. +Extending the argument to a second field is a decision worth taking explicitly +rather than by analogy; until then a tool call with no id is not representable. + +## What is recovered from ``extensions`` + +Only the three `agent_timeout` fields, and only because +:func:`~benchflow.trajectories.ir_from_acp.acp_events_to_ir` puts them there by +name: ``timeout_sec``, ``pending_tool_call_ids`` and +``terminal_trajectory_complete`` are keys that edge preserved explicitly, so +reading them back is deterministic rather than a search. Each is type-checked +against what the schema requires; a missing or wrongly-typed one makes the event +not representable rather than defaulted. +""" + +from __future__ import annotations + +from typing import Any + +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlock, + EventKind, + LossClass, + LossRecord, + LossReport, + PathSpace, + ToolCall, + ToolStatus, + TraceEvent, +) + +LOSS_DIRECTION = "ir->acp" + +ACP_TEXT_TYPE: dict[EventKind, str] = { + EventKind.USER_MESSAGE: "user_message", + EventKind.AGENT_MESSAGE: "agent_message", + EventKind.AGENT_REASONING: "agent_thought", +} +"""The three IR kinds Slice A's ``text_event.type`` enum admits.""" + +ACP_TOOL_STATUSES: frozenset[str] = frozenset( + {"pending", "in_progress", "completed", "failed", "cancelled"} +) +"""``tool_call_event.status`` — a CLOSED enum in the schema. + +:attr:`~benchflow.trajectories.ir.ToolStatus.UNKNOWN` is deliberately absent: the +IR has a value for "the source carried a status this converter could not map", +and ACP has none. +""" + +ACP_KIND_SEMANTICS = "acp_kind" +"""The only :attr:`~benchflow.trajectories.ir.ToolCall.name_semantics` value +that may be written into ACP's ``kind`` slot. + +``ir_from_acp`` writes this label when it reads a capture record's ``kind``. +Any other label — ``function_name``, ``gen_ai.tool.name`` — says the name is +not a category, and this edge refuses rather than reinterpreting it. +""" + +ACP_TIMEOUT_REASON = "wall_clock_timeout" +"""``agent_timeout_event.reason`` — a single-member enum. ``record_agent_timeout`` +is its only producer and hardcodes this value.""" + +TIMEOUT_EXTENSION_KEYS: tuple[str, ...] = ( + "timeout_sec", + "pending_tool_call_ids", + "terminal_trajectory_complete", +) +"""The keys `ir_from_acp` preserves verbatim on a timeout event's extensions.""" + +# Key order matches ``_events_to_trajectory`` exactly. It costs nothing and it +# is what lets the round-trip anchor be checked on the serialized bytes as well +# as on the structures; see the module's test suite. +_TEXT_KEYS = ("type", "text") +_TOOL_CALL_KEYS = ("type", "tool_call_id", "kind", "title", "status", "content") +_TIMEOUT_KEYS = ("type", "reason", *TIMEOUT_EXTENSION_KEYS) + + +class AcpCaptureNotRepresentable(ValueError): + """A trace cannot become ACP capture events without inventing a value. + + Carries the blockers as :class:`~benchflow.trajectories.ir.LossRecord` + values so the failure names the field that caused it by path, in the same + vocabulary every other edge of this family uses, instead of a bare message a + caller would have to parse. + """ + + def __init__(self, blockers: list[LossRecord]) -> None: + self.blockers: tuple[LossRecord, ...] = tuple(blockers) + first = blockers[0] if blockers else None + summary = ( + f"{len(blockers)} value(s) cannot be represented as ACP capture " + f"events; first: {first.field} — {first.detail}" + if first is not None + else "trace is not representable as ACP capture events" + ) + super().__init__(summary) + + +def acp_capture_blockers(trace: CanonicalTrace) -> list[LossRecord]: + """What stops *trace* from being exportable, without raising. + + An empty list means :func:`ir_to_acp_capture_events` will succeed. Each + record addresses the IR field responsible, in + :attr:`~benchflow.trajectories.ir.PathSpace.HUB`, so a caller can report the + obstruction without catching anything. + """ + return _convert(trace)[1] + + +def ir_to_acp_capture_events( + trace: CanonicalTrace, +) -> tuple[list[dict[str, Any]], LossReport]: + """Build the ACP capture events for *trace*, or refuse. + + Returns the events **and** the report of what the target could not carry. + The report is returned rather than attached: one trace may be converted to + several targets and none of those conversions describes how the trace came + to exist, so :attr:`~benchflow.trajectories.ir.CanonicalTrace.losses` is left + alone and *trace* is not modified. + + Raises :class:`AcpCaptureNotRepresentable` when any event needs a value the + IR does not have. Nothing partial is returned — see the module docstring. + """ + events, blockers, losses = _convert(trace) + if blockers: + raise AcpCaptureNotRepresentable(blockers) + return events, losses + + +def _convert( + trace: CanonicalTrace, +) -> tuple[list[dict[str, Any]], list[LossRecord], LossReport]: + """One pass: the events, what blocked, and what the target could not carry. + + Both public entry points go through here, so asking whether a trace is + representable and converting it cannot disagree. + """ + losses = LossReport(direction=LOSS_DIRECTION) + blockers: list[LossRecord] = [] + events: list[dict[str, Any]] = [] + + for event in trace.events: + record, event_blockers = _event_to_acp(event, losses) + blockers.extend(event_blockers) + if record is not None: + events.append(record) + + _declare_trace_level(trace, losses) + return events, blockers, losses + + +def _blocker(field: str, detail: str) -> LossRecord: + """A record that makes a conversion impossible rather than lossy. + + ``UNSUPPORTED`` is the honest class: the value is not in the trace, so no + change to this converter could produce it. It is not ``DROPPED`` — nothing + was discarded here — and it is emphatically not ``SYNTHESIZED``, which is + the class this edge exists to avoid needing. + """ + return LossRecord( + field=field, + space=PathSpace.HUB, + loss_class=LossClass.UNSUPPORTED, + detail=detail, + ) + + +def _event_to_acp( + event: TraceEvent, losses: LossReport +) -> tuple[dict[str, Any] | None, list[LossRecord]]: + """One IR event as one ACP capture record, or the reasons it cannot be.""" + where = f"events[{event.index}]" + + if event.kind in ACP_TEXT_TYPE: + return _text_event(event, where, losses) + if event.kind is EventKind.TOOL_CALL: + return _tool_call_event(event, where, losses) + if event.kind is EventKind.TIMEOUT: + return _timeout_event(event, where, losses) + + # ORACLE and UNKNOWN. The schema models three record shapes and neither of + # these is one of them; §2.4 documents the oracle record as something the + # capture emitter never produces. Widening the schema, emitting an invalid + # record, or dropping the event would each be a different way of pretending + # the codomain is bigger than it is. + return None, [ + _blocker( + where, + f"an event of kind {event.kind.value!r} has no ACP capture record " + "shape; the Slice A contract defines text, tool_call and " + "agent_timeout events only", + ) + ] + + +def _text_event( + event: TraceEvent, where: str, losses: LossReport +) -> tuple[dict[str, Any] | None, list[LossRecord]]: + """A user / agent / thought record. + + The schema requires ``text`` and documents ``""`` as a value the capture + path really records — an unconditionally captured empty prompt — rather than + as the representation of an absent one. So an empty string passes through as + the observation it is, and a ``None`` is a blocker rather than an empty + string: writing one would turn "the source carried no text" into "the source + carried empty text", which is exactly the tri-state collapse §8.2 forbids. + """ + is_reasoning = event.kind is EventKind.AGENT_REASONING + source_field = "reasoning" if is_reasoning else "text" + value = event.reasoning if is_reasoning else event.text + + if value is None: + return None, [ + _blocker( + f"{where}.{source_field}", + f"a {ACP_TEXT_TYPE[event.kind]} record requires text and the " + f"event carries none; the contract documents the empty string as " + "an observed value, not as a way to write an absence", + ) + ] + + _declare_event_level(event, where, losses) + if is_reasoning and event.text is not None: + losses.add( + f"{where}.text", + LossClass.DROPPED, + "an agent_thought record carries only its reasoning text; the ACP " + "capture format has no second text slot on the same event", + ) + if ( + is_reasoning + and event.reasoning_segments is not None + and len(event.reasoning_segments) > 1 + ): + losses.add( + f"{where}.reasoning_segments", + LossClass.DROPPED, + f"{len(event.reasoning_segments)} thought segments become one " + "agent_thought record; the boundary between them is not " + "representable, which is §5 loss #10 in the writing direction", + ) + return dict(zip(_TEXT_KEYS, (ACP_TEXT_TYPE[event.kind], value), strict=True)), [] + + +def _tool_call_event( + event: TraceEvent, where: str, losses: LossReport +) -> tuple[dict[str, Any] | None, list[LossRecord]]: + """A tool-call record, or every reason it cannot be built. + + All blockers are collected rather than short-circuiting on the first: a + caller fixing a producer wants the whole list, and reporting one field at a + time would make an unrepresentable trace look like a sequence of small + problems. + """ + call = event.tool_call + if call is None: # pragma: no cover - invariant 3 forbids it + return None, [ + _blocker(f"{where}.tool_call", "tool_call event with no tool_call payload") + ] + + blockers: list[LossRecord] = [] + + status = _acp_status(call, where, blockers) + call_id = _acp_call_id(call, where, blockers) + kind = _acp_kind(call, where, blockers) + content = _acp_content(call, where, blockers) + + if blockers: + return None, blockers + + _declare_event_level(event, where, losses) + title = _acp_title(call, where, losses) + _declare_tool_call_losses(event, call, where, losses) + + return ( + dict( + zip( + _TOOL_CALL_KEYS, + ("tool_call", call_id, kind, title, status, content), + strict=True, + ) + ), + [], + ) + + +def _acp_status(call: ToolCall, where: str, blockers: list[LossRecord]) -> str | None: + """The one field this edge will never invent.""" + if call.status is not None and call.status.value in ACP_TOOL_STATUSES: + return call.status.value + observed = ( + "no status" + if call.status is None + else f"the status {call.status.value!r}, which ACP has no member for" + ) + blockers.append( + _blocker( + f"{where}.tool_call.status", + f"the tool call carries {observed}; the ACP vocabulary is " + f"{sorted(ACP_TOOL_STATUSES)} and every member asserts something " + "about the call's lifecycle, so none of them can stand in for an " + "unknown one", + ) + ) + if call.status is ToolStatus.UNKNOWN: + # Worth its own sentence: this is not a gap in the trace, it is the IR + # saying something ACP cannot say. + blockers[-1] = _blocker( + f"{where}.tool_call.status", + "the IR records this status as UNKNOWN — the source carried one this " + "family could not map — and ACP's closed enum has no member with " + "that meaning; writing any of the five would assert a lifecycle " + "state nobody observed", + ) + return None + + +def _acp_call_id(call: ToolCall, where: str, blockers: list[LossRecord]) -> str | None: + """``tool_call_id``, which is required and is not defaulted here. + + The schema documents ``""`` as what the capture path records when the agent + omitted an id, which is the same argument that lets ``title`` be written + empty. It is not applied here on purpose — see the module docstring. + """ + if call.call_id is not None: + return call.call_id + blockers.append( + _blocker( + f"{where}.tool_call.call_id", + "the tool call carries no id; the contract documents the empty " + "string for an id the agent omitted, but this edge does not write " + "one for an id the trace never had", + ) + ) + return None + + +def _acp_kind(call: ToolCall, where: str, blockers: list[LossRecord]) -> str | None: + """``kind`` — required, and only writable from a name that *is* an ACP kind. + + An ACP ``kind`` is a **category**, not a tool name. + ``benchflow.acp.types.ToolKind`` calls itself "Category tag for tool calls, + used for metrics and trajectory display", `_canonical_tool_kind` defaults an + absent one to ``"other"``, and the values seen in production — ``execute``, + ``edit``, ``fetch``, ``think`` — are categories too. + + ATIF's ``function_name`` and OTel's ``gen_ai.tool.name`` are names of + *particular tools*. Writing one into this slot would not be a normalization, + which is a value a reader can undo with convention knowledge; it would be a + **reinterpretation** — the record would assert a category that was never + observed, and nothing downstream could tell. The IR carries + :attr:`~benchflow.trajectories.ir.ToolCall.name_semantics` precisely so this + edge does not have to guess, and refusing is what makes that field + load-bearing rather than decorative. + + So the representability rule reads the semantics, not the string: + + - no ``name`` — nothing to write, and the contract documents no meaning for + an empty ``kind`` the way it does for an empty ``title``; + - no ``name_semantics`` — the trace does not say what kind of name it holds, + and an unlabelled name is not evidence of a category; + - ``name_semantics`` other than ``"acp_kind"`` — the trace says explicitly + that this is *not* an ACP kind. + + **The string is never inspected.** A ``function_name`` of ``"read"`` + coincides with a real ``ToolKind`` member and is still refused: matching the + vocabulary by accident is not the same as being drawn from it, and a rule + that looked at the value would silently admit exactly the cases most likely + to be wrong. Provenance is not consulted either — ``name_semantics`` is + trace data, and a hand-built trace that labels its names ``acp_kind`` + exports whatever its ``source_format`` says. + """ + # Only this function's own findings decide its return value; the list is + # shared with the other field readers and may already hold theirs. + before = len(blockers) + if call.name is None: + blockers.append( + _blocker( + f"{where}.tool_call.name", + "the tool call carries no name and ACP requires a kind; unlike " + "title, the contract documents no meaning for an empty one", + ) + ) + if call.name_semantics is None: + blockers.append( + _blocker( + f"{where}.tool_call.name_semantics", + "the trace does not say what kind of name this is, and ACP's " + "kind is a category rather than a tool name; an unlabelled name " + "is not evidence that it was drawn from that vocabulary", + ) + ) + elif call.name_semantics != ACP_KIND_SEMANTICS: + blockers.append( + _blocker( + f"{where}.tool_call.name_semantics", + f"the name is a {call.name_semantics!r}, and ACP's kind is a " + "category tag; writing it into that slot would assert a category " + "nobody observed, which is a reinterpretation rather than a " + "normalization a reader could undo", + ) + ) + return None if len(blockers) > before else call.name + + +def _acp_title(call: ToolCall, where: str, losses: LossReport) -> str: + """``title``, defaulting to ``""`` **because the contract says so**. + + Slice A documents the field as "Empty string when the agent supplied none", + so the empty string is the contract's representation of an absent title + rather than this converter's invention. It is declared ``SYNTHESIZED`` + anyway: a reader of the output cannot distinguish it from an empty title the + agent really supplied, and that indistinguishability is the loss. + """ + if call.title is not None: + return call.title + losses.add( + f"{where}.tool_call.title", + LossClass.SYNTHESIZED, + "the trace carries no title and ACP requires one; the empty string is " + "the contract's own representation of an absent title, and is written " + "here as such — a reader cannot tell it from an observed empty title", + ) + return "" + + +def _acp_content( + call: ToolCall, where: str, blockers: list[LossRecord] +) -> list[dict[str, Any]] | None: + """The captured output blocks, verbatim. + + ACP stores ``session/update.content`` as the wire delivered it and the + schema keeps ``content_block`` permissive for exactly that reason, so the + only faithful thing to write is the block the IR preserved in + :attr:`~benchflow.trajectories.ir.ContentBlock.raw`. A block with no ``raw`` + carries text but no wire shape, and choosing one — the nested ACP form or + the flat form, both of which `content_blocks_to_text` accepts — would be + this converter inventing structure the source never had. + + An empty list is not a blocker: the schema documents it as what a call with + no output records. + """ + blocks: list[dict[str, Any]] = [] + for position, block in enumerate(call.content): + raw = _raw_block(block) + if raw is None: + blockers.append( + _blocker( + f"{where}.tool_call.content[{position}].raw", + "the content block carries rendered text but not the source " + "block it came from; ACP stores wire blocks verbatim and " + "there is no single shape to wrap the text in", + ) + ) + continue + blocks.append(raw) + return None if blockers else blocks + + +def _raw_block(block: ContentBlock) -> dict[str, Any] | None: + return block.raw + + +def _declare_event_level(event: TraceEvent, where: str, losses: LossReport) -> None: + """IR event fields no ACP record shape has a slot for. + + Declared only when the event actually carries the value — the outbound rule + Slice D adopted: an edge declares what it loses given the trace in hand, + not what it would lose given a fuller one. + """ + for field in ("started_at", "finished_at", "usage", "role", "source_type"): + if getattr(event, field) is not None: + losses.add( + f"{where}.{field}", + LossClass.DROPPED, + f"ACP capture records have no {field} slot", + ) + if event.extensions and event.kind is not EventKind.TIMEOUT: + losses.add( + f"{where}.extensions", + LossClass.DROPPED, + "the record shapes forbid additional properties, so carried source " + "fields have nowhere to go", + ) + + +def _declare_tool_call_losses( + event: TraceEvent, call: ToolCall, where: str, losses: LossReport +) -> None: + """What a tool-call record cannot carry, given this call.""" + if call.arguments is not None: + losses.add( + f"{where}.tool_call.arguments", + LossClass.DROPPED, + "the ACP capture record has no arguments field; ACPSession." + "handle_update never read rawInput, so the format never grew one", + "§5 loss #1", + ) + if call.name_semantics is not None: + losses.add( + f"{where}.tool_call.name_semantics", + LossClass.NORMALIZED, + f"{call.name_semantics!r} is not written anywhere in the record; it " + "is recoverable only from the convention that an ACP kind is an ACP " + "kind, which is what reading the file as ACP already assumes", + ) + for field in ("started_at", "finished_at"): + if getattr(call, field) is not None: + losses.add( + f"{where}.tool_call.{field}", + LossClass.DROPPED, + "ToolCallRecord tracks both in memory and the capture format " + "serializes neither", + "§5 loss #3", + ) + for position, block in enumerate(call.content): + if block.raw is not None and block.text is not None: + # The block is written verbatim, so nothing is lost — but the IR's + # classification of it is, and a consumer re-reading the record has + # to re-derive text from the wire shape. + losses.add( + f"{where}.tool_call.content[{position}].kind", + LossClass.NORMALIZED, + "the block is written verbatim; the IR's text/opaque " + "classification is not part of the wire shape and is re-derived " + "by whoever reads it back", + ) + if event.text is not None: + losses.add( + f"{where}.text", + LossClass.DROPPED, + "a tool_call record has no text slot; a step's message rides on the " + "event in some source formats and cannot here", + ) + + +def _timeout_event( + event: TraceEvent, where: str, losses: LossReport +) -> tuple[dict[str, Any] | None, list[LossRecord]]: + """An ``agent_timeout`` record, rebuilt from the fields the inbound edge kept. + + Four required values, and every one of them is checked rather than + defaulted. ``reason`` is a single-member enum, so an outcome that is not + exactly ``wall_clock_timeout`` cannot be written as one; the other three are + read from ``extensions`` by the names + :func:`~benchflow.trajectories.ir_from_acp.acp_events_to_ir` preserved them + under, which makes the recovery deterministic rather than a search. + """ + blockers: list[LossRecord] = [] + + if event.outcome != ACP_TIMEOUT_REASON: + blockers.append( + _blocker( + f"{where}.outcome", + f"agent_timeout.reason is a single-member enum " + f"({ACP_TIMEOUT_REASON!r}) and the event's outcome is " + f"{event.outcome!r}; there is no other reason the record can state", + ) + ) + + values: dict[str, Any] = {} + checks = { + "timeout_sec": ( + lambda v: not isinstance(v, bool) and isinstance(v, (int, float)), + "a number", + ), + "pending_tool_call_ids": (lambda v: isinstance(v, list), "an array"), + "terminal_trajectory_complete": (lambda v: isinstance(v, bool), "a boolean"), + } + for key, (ok, expected) in checks.items(): + if key not in event.extensions: + blockers.append( + _blocker( + f"{where}.extensions.{key}", + f"agent_timeout requires {key} and the event carries none; " + "it is preserved by name when the trace came through the ACP " + "edge, and inventing a budget that was never observed is not " + "a substitute", + ) + ) + continue + value = event.extensions[key] + if not ok(value): + blockers.append( + _blocker( + f"{where}.extensions.{key}", + f"{key} is {type(value).__name__} and the contract requires " + f"{expected}", + ) + ) + continue + values[key] = value + + if blockers: + return None, blockers + + _declare_event_level(event, where, losses) + unused = set(event.extensions) - set(TIMEOUT_EXTENSION_KEYS) + if unused: + losses.add( + f"{where}.extensions", + LossClass.DROPPED, + "extension keys with no agent_timeout field: " + ", ".join(sorted(unused)), + ) + return ( + dict( + zip( + _TIMEOUT_KEYS, + ( + "agent_timeout", + ACP_TIMEOUT_REASON, + *(values[key] for key in TIMEOUT_EXTENSION_KEYS), + ), + strict=True, + ) + ), + [], + ) + + +def _declare_trace_level(trace: CanonicalTrace, losses: LossReport) -> None: + """Everything the capture format has no envelope for. + + The target is a flat stream of event records. There is no header, no + trailer and no per-file object, so this is not a list of fields ACP happens + to lack — it is the whole of the trace level, and no future ACP record shape + could take it without the artifact growing a concept it does not have. + + Declared only for values this trace actually carries, per the outbound rule. + """ + for field in ("trace_id", "session_id", "started_at", "finished_at"): + if getattr(trace, field) is not None: + losses.add( + field, + LossClass.DROPPED, + "the ACP capture format is a flat event stream with no envelope " + "to carry a trace-level value", + ) + for field, value in trace.agent.model_dump().items(): + if value is not None: + losses.add( + f"agent.{field}", + LossClass.DROPPED, + "no capture record identifies the agent; the artifact's readers " + "take that from result.json", + ) + if trace.usage is not None: + losses.add( + "usage", + LossClass.DROPPED, + "token accounting is routed to result.json and has never been part " + "of the capture event stream", + "§5 losses #6, #8", + ) + for field, value in trace.outcome.model_dump().items(): + if value is not None: + losses.add( + f"outcome.{field}", + LossClass.DROPPED, + "the capture stream has no run-outcome record; a timeout is " + "expressible only as its own event", + ) + if trace.extensions: + losses.add( + "extensions", + LossClass.DROPPED, + "trace-level extensions have no envelope to ride in", + ) diff --git a/tests/trajectories/test_ir_to_acp.py b/tests/trajectories/test_ir_to_acp.py new file mode 100644 index 000000000..e59096140 --- /dev/null +++ b/tests/trajectories/test_ir_to_acp.py @@ -0,0 +1,1002 @@ +"""Conversion suite for ``canonical Trace IR → ACP capture events`` (Slice G). + +The target is the **ACP-session capture event format** pinned by Slice A's JSON +Schema, not the `acp_trajectory.jsonl` artifact — §2.1 records that no +artifact-level contract exists, and several producers write records into that +file that the schema deliberately does not model. + +Two properties carry most of the weight here: + +- **the round-trip anchor** — a conformant ACP event list converted to the IR + and back is the same event list, which is what makes "this edge writes the + format" a measurement rather than an assertion; +- **fail closed** — a trace with one unrepresentable event yields no events at + all, because a partial list is indistinguishable from a complete one once it + leaves the function. + +Everything else is the negative space around those: the values this edge refuses +to invent, and the proof that refusing is what actually happens. + +Nothing here writes to disk and nothing imports a run path. `_capture.py` is +imported read-only, as the producer whose records this edge has to reproduce. +""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path +from typing import Any + +import jsonschema +import pytest + +from benchflow.trajectories import ir_to_acp as ir_to_acp_module +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlock, + ContentBlockKind, + EventKind, + LossClass, + ModelInfo, + PathSpace, + Provenance, + Role, + ToolCall, + ToolStatus, + TraceEvent, + TraceOutcome, + TraceUsage, + validate_trace, +) +from benchflow.trajectories.ir_from_acp import acp_events_to_ir +from benchflow.trajectories.ir_to_acp import ( + ACP_TIMEOUT_REASON, + ACP_TOOL_STATUSES, + LOSS_DIRECTION, + AcpCaptureNotRepresentable, + acp_capture_blockers, + ir_to_acp_capture_events, +) +from benchflow.trajectories.types import redact_acp_trajectory_jsonl +from tests.trajectories.test_trace_ir import resolve_ir_path + +SCHEMA_PATH = ( + Path(__file__).resolve().parents[2] + / "src/benchflow/trajectories/schemas/acp-capture-event-v1.schema.json" +) +SCHEMA = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) +VALIDATOR = jsonschema.Draft202012Validator(SCHEMA) + + +def assert_schema_valid(events: list[dict[str, Any]]) -> None: + """Every emitted record must validate against the Slice A contract. + + This is the check that makes "the target is the capture format" mean + something: the edge is held to the published schema rather than to whatever + it happens to produce. + """ + for position, record in enumerate(events): + errors = sorted(VALIDATOR.iter_errors(record), key=lambda e: e.json_path) + assert errors == [], (position, record, [e.message for e in errors]) + + +CONFORMANT: list[dict[str, Any]] = [ + {"type": "user_message", "text": "solve the task"}, + {"type": "agent_thought", "text": "reading the repo"}, + { + "type": "tool_call", + "tool_call_id": "tc-1", + "kind": "execute", + "title": "ls -la", + "status": "completed", + "content": [{"type": "content", "content": {"type": "text", "text": "out"}}], + }, + {"type": "agent_message", "text": ""}, + { + "type": "agent_timeout", + "reason": "wall_clock_timeout", + "timeout_sec": 300, + "pending_tool_call_ids": ["tc-9"], + "terminal_trajectory_complete": True, + }, +] +"""One record of every shape the emitter produces, including the two empty +values the contract documents: an empty `agent_message` and a text-empty title +is exercised separately.""" + + +def tool_event(index: int = 0, **overrides: Any) -> TraceEvent: + """A representable tool-call event, before overrides.""" + call_fields: dict[str, Any] = { + "call_id": "tc-1", + "name": "execute", + # An ACP kind is a category, and only a name labelled as one may be + # written into that slot — so a representable tool call must say so. + "name_semantics": "acp_kind", + "title": "ls", + "status": ToolStatus.COMPLETED, + "arguments": {}, + "content": [ + ContentBlock(kind=ContentBlockKind.TEXT, text="o", raw={"text": "o"}) + ], + } + for key in list(overrides): + if key in call_fields: + call_fields[key] = overrides.pop(key) + event: dict[str, Any] = { + "index": index, + "kind": EventKind.TOOL_CALL, + "tool_call": ToolCall(**call_fields), + "provenance": Provenance(source_format="hand-built"), + } + event.update(overrides) + return TraceEvent(**event) + + +def timeout_event(index: int = 0, **overrides: Any) -> TraceEvent: + extensions = overrides.pop( + "extensions", + { + "timeout_sec": 300, + "pending_tool_call_ids": [], + "terminal_trajectory_complete": True, + }, + ) + event: dict[str, Any] = { + "index": index, + "kind": EventKind.TIMEOUT, + "outcome": ACP_TIMEOUT_REASON, + "extensions": extensions, + "provenance": Provenance(source_format="hand-built"), + } + event.update(overrides) + return TraceEvent(**event) + + +def trace_of(*events: TraceEvent, **overrides: Any) -> CanonicalTrace: + payload: dict[str, Any] = { + "events": list(events), + "provenance": Provenance(source_format="hand-built"), + } + payload.update(overrides) + return CanonicalTrace(**payload) + + +def blocker_fields(trace: CanonicalTrace) -> list[str]: + return [record.field for record in acp_capture_blockers(trace)] + + +# --------------------------------------------------------------------------- +# The anchor: ACP → IR → ACP +# --------------------------------------------------------------------------- + + +def test_a_conformant_event_list_survives_the_round_trip(): + """The property the whole slice rests on. + + If this fails, the edge is not writing the capture format — it is writing + something that resembles it. + + **Scope of the evidence.** This is measured on `CONFORMANT` — five records, + one of each shape the emitter produces — and, in the drop-one test below, on + its five subsets. It is **not** a demonstration that every conformant + capture document round-trips: there is no property test and no corpus taken + from captured rollouts. What holds for unmeasured inputs is the weaker but + structural guarantee that the edge either reproduces its input or refuses, + so an unmeasured shape cannot quietly produce a degraded record. + """ + trace = acp_events_to_ir(CONFORMANT) + assert validate_trace(trace) == [] + events, report = ir_to_acp_capture_events(trace) + + assert events == CONFORMANT + assert report.direction == LOSS_DIRECTION + assert_schema_valid(events) + + +def test_the_round_trip_preserves_key_order_too(): + """Structural equality does not pin key order; this does. + + It matters because the artifact is JSONL: two records with the same keys in + a different order are the same object and different bytes, and the capture + file is read by tools that diff it. + """ + trace = acp_events_to_ir(CONFORMANT) + events, _ = ir_to_acp_capture_events(trace) + for original, produced in zip(CONFORMANT, events, strict=True): + assert list(original.keys()) == list(produced.keys()) + + +def test_the_round_trip_is_byte_identical_through_the_project_serializer(): + """True, and worth stating precisely rather than loudly. + + `redact_acp_trajectory_jsonl` is what every write path in the repository + uses, and the two serializations are identical. But this is **not an + independent property of the conversion**: the serializer applies redaction + to both sides, so redaction cancels, and what byte equality adds over + structural equality is exactly the key order the previous test pins. The + honest claim for this edge is structural equality plus key order; the byte + result is their consequence. + """ + trace = acp_events_to_ir(CONFORMANT) + events, _ = ir_to_acp_capture_events(trace) + assert redact_acp_trajectory_jsonl(events) == redact_acp_trajectory_jsonl( + CONFORMANT + ) + + +def test_the_serializer_redacts_which_is_why_the_byte_claim_is_qualified(): + """The reason the test above is not the headline. + + A secret-shaped value is rewritten on the way out, by the serializer and not + by this edge. A byte claim stated over that function would be partly a claim + about redaction. + """ + events = [{"type": "agent_message", "text": "AKIAIOSFODNN7EXAMPLE"}] + assert "AKIAIOSFODNN7EXAMPLE" not in redact_acp_trajectory_jsonl(events) + + +@pytest.mark.parametrize("dropped", range(len(CONFORMANT))) +def test_every_record_shape_round_trips_on_its_own(dropped): + """The anchor holds per shape, not only for the full list. + + Guards against a mapping that is right in aggregate because two errors + cancel — dropping one record and mis-emitting another would still produce a + list of the same length. + """ + subset = [e for i, e in enumerate(CONFORMANT) if i != dropped] + events, _ = ir_to_acp_capture_events(acp_events_to_ir(subset)) + assert events == subset + assert_schema_valid(events) + + +# --------------------------------------------------------------------------- +# Representability is about the data, not the provenance +# --------------------------------------------------------------------------- + + +def test_provenance_is_never_read_to_decide_representability(): + """Two traces, identical data, different declared origin. + + The rule is that the ACP contract is satisfied by values, not by lineage. A + converter that consulted `provenance` would be deciding a data question with + a metadata answer, and would make an OTel trace that grew the missing fields + permanently unexportable. + """ + outputs = [] + for source in ("acp-capture-v1", "otel", "atif", "invented"): + event = tool_event(provenance=Provenance(source_format=source)) + trace = trace_of(event, provenance=Provenance(source_format=source)) + events, _ = ir_to_acp_capture_events(trace) + outputs.append(events) + assert all(out == outputs[0] for out in outputs) + assert_schema_valid(outputs[0]) + + +def test_the_module_never_references_provenance(): + """Static half of the rule above, so it cannot be reintroduced quietly.""" + source = Path(ir_to_acp_module.__file__).read_text(encoding="utf-8") + tree = ast.parse(source) + reads = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Attribute) + and node.attr in {"provenance", "source_format"} + ] + assert reads == [], [ast.unparse(node) for node in reads] + + +def test_an_acp_derived_trace_can_still_be_unrepresentable(): + """The other direction of the same rule. + + Origin does not buy a pass: an ACP-derived trace whose status this family + could not map is refused exactly like any other. + """ + trace = acp_events_to_ir( + [ + { + "type": "tool_call", + "tool_call_id": "tc-1", + "kind": "execute", + "title": "ls", + "status": "something_new", + "content": [], + } + ] + ) + assert trace.events[0].tool_call.status is ToolStatus.UNKNOWN + assert blocker_fields(trace) == ["events[0].tool_call.status"] + + +# --------------------------------------------------------------------------- +# The value this edge will never invent +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("status", [None, ToolStatus.UNKNOWN]) +def test_a_tool_call_with_no_acp_status_is_refused(status): + """No member of the ACP enum can stand in for an unknown lifecycle state.""" + trace = trace_of(tool_event(status=status)) + blockers = acp_capture_blockers(trace) + assert [b.field for b in blockers] == ["events[0].tool_call.status"] + assert blockers[0].loss_class is LossClass.UNSUPPORTED + assert blockers[0].space is PathSpace.HUB + + with pytest.raises(AcpCaptureNotRepresentable) as excinfo: + ir_to_acp_capture_events(trace) + assert excinfo.value.blockers == tuple(blockers) + + +def test_no_acp_status_string_appears_anywhere_when_the_status_is_unknown(): + """The strongest form: not "it raised", but "it never wrote one". + + A converter could raise *and* have built a record with a fabricated status + on the way; this asserts the five vocabulary members are absent from + everything the call produces, including the exception. + """ + trace = trace_of(tool_event(status=None)) + with pytest.raises(AcpCaptureNotRepresentable) as excinfo: + ir_to_acp_capture_events(trace) + rendered = str(excinfo.value) + repr(excinfo.value.blockers) + for member in ACP_TOOL_STATUSES: + assert f"'{member}'" not in rendered.replace( + str(sorted(ACP_TOOL_STATUSES)), "" + ), member + + +@pytest.mark.parametrize( + ("field", "path"), + [("call_id", "events[0].tool_call.call_id"), ("name", "events[0].tool_call.name")], +) +def test_a_tool_call_missing_a_required_identifier_is_refused(field, path): + """`kind` and `tool_call_id` are required and are not defaulted. + + The contract documents an empty `tool_call_id` for an id the *agent* + omitted; this edge does not extend that to an id the *trace* never had. That + restraint is deliberate and is a decision worth reviewing rather than an + oversight — see the module docstring. + """ + trace = trace_of(tool_event(**{field: None})) + assert blocker_fields(trace) == [path] + + +# --------------------------------------------------------------------------- +# The kind slot: a category, not a tool name +# --------------------------------------------------------------------------- + + +def test_an_acp_kind_passes_through_unchanged(): + """The only semantics ACP's `kind` slot accepts, and it is preserved.""" + trace = acp_events_to_ir([CONFORMANT[2]]) + assert trace.events[0].tool_call.name_semantics == "acp_kind" + events, _ = ir_to_acp_capture_events(trace) + assert events[0]["kind"] == "execute" + assert_schema_valid(events) + + +@pytest.mark.parametrize( + "semantics", + ["function_name", "gen_ai.tool.name", "span_name", "something_new"], +) +def test_a_name_that_is_not_an_acp_kind_is_refused(semantics): + """ACP's `kind` is a category tag (`ToolKind`: "Category tag for tool calls"). + + ATIF's `function_name` and OTel's `gen_ai.tool.name` name *particular tools*. + Writing one into this slot is not a normalization a reader could undo — it + asserts a category nobody observed. `name_semantics` exists so this edge does + not have to guess, and refusing is what makes the field load-bearing. + """ + trace = trace_of(tool_event(name="read_file", name_semantics=semantics)) + blockers = acp_capture_blockers(trace) + assert [b.field for b in blockers] == ["events[0].tool_call.name_semantics"] + assert semantics in blockers[0].detail + + +def test_an_unlabelled_name_is_refused_and_says_why(): + """An unlabelled name is not evidence that it came from the vocabulary. + + The detail is asserted, not just the path: "the trace does not say what kind + of name this is" and "the name is a `function_name`" are different findings, + and a reader fixing a producer needs to know which one they have. Found by + mutation — collapsing the two branches left the path identical and only the + message wrong, which the earlier version of this test could not see. + """ + trace = trace_of(tool_event(name="execute", name_semantics=None)) + blockers = acp_capture_blockers(trace) + assert [b.field for b in blockers] == ["events[0].tool_call.name_semantics"] + assert "does not say what kind of name" in blockers[0].detail + + +def test_a_foreign_name_that_looks_like_an_acp_kind_is_still_refused(): + """The insidious case, and the reason the string is never inspected. + + `read` is a real `ToolKind` member. A `function_name` that happens to spell + it is still a function name: matching the vocabulary by accident is not the + same as being drawn from it, and a rule that looked at the value would + silently admit exactly the cases most likely to be wrong. + """ + for name in ("read", "write", "other", "bash", "search", "browser", "skill"): + trace = trace_of(tool_event(name=name, name_semantics="function_name")) + assert blocker_fields(trace) == ["events[0].tool_call.name_semantics"], name + + +def test_an_atif_derived_tool_call_cannot_launder_a_function_name(): + """The regression this rule was added for, end to end from the real edge. + + Before it, an ATIF document whose `extra.status` made the call otherwise + representable exported `kind="read_file"` — a tool name in a category slot, + with only a `DROPPED` record on `name_semantics` to show for it. + """ + from benchflow.trajectories.ir_from_atif import atif_to_ir + + trace = atif_to_ir( + { + "steps": [ + { + "source": "agent", + "tool_calls": [ + { + "tool_call_id": "call_1", + "function_name": "read_file", + "arguments": {}, + "extra": {"status": "completed", "title": "read"}, + } + ], + } + ] + } + ) + call = trace.events[0].tool_call + assert (call.name, call.name_semantics) == ("read_file", "function_name") + assert "events[0].tool_call.name_semantics" in blocker_fields(trace) + with pytest.raises(AcpCaptureNotRepresentable): + ir_to_acp_capture_events(trace) + + +def test_an_otel_derived_tool_call_is_refused_on_semantics_too(): + """Blocked on status today; this pins that it is *also* blocked on semantics, + so making the status representable would not open the laundering path.""" + trace = trace_of( + tool_event( + name="read_file", + name_semantics="gen_ai.tool.name", + status=ToolStatus.COMPLETED, + ) + ) + assert blocker_fields(trace) == ["events[0].tool_call.name_semantics"] + + +def test_the_semantics_rule_does_not_consult_provenance(): + """`name_semantics` is trace data; `source_format` is not consulted.""" + for source in ("acp-capture-v1", "otel", "atif"): + allowed = trace_of( + tool_event( + name_semantics="acp_kind", + provenance=Provenance(source_format=source), + ), + provenance=Provenance(source_format=source), + ) + refused = trace_of( + tool_event( + name_semantics="function_name", + provenance=Provenance(source_format=source), + ), + provenance=Provenance(source_format=source), + ) + assert acp_capture_blockers(allowed) == [], source + assert blocker_fields(refused) == ["events[0].tool_call.name_semantics"], source + + +def test_a_missing_name_and_a_missing_semantics_are_reported_separately(): + """Two different absences, two different paths.""" + trace = trace_of(tool_event(name=None, name_semantics=None)) + assert sorted(blocker_fields(trace)) == [ + "events[0].tool_call.name", + "events[0].tool_call.name_semantics", + ] + + +def test_a_content_block_with_no_source_block_is_refused(): + """ACP stores wire blocks verbatim; there is no single shape to invent.""" + trace = trace_of( + tool_event(content=[ContentBlock(kind=ContentBlockKind.TEXT, text="o")]) + ) + assert blocker_fields(trace) == ["events[0].tool_call.content[0].raw"] + + +def test_every_blocker_is_reported_not_just_the_first(): + """A producer being fixed wants the whole list in one pass.""" + trace = trace_of(tool_event(status=None, call_id=None, name=None)) + assert sorted(blocker_fields(trace)) == [ + "events[0].tool_call.call_id", + "events[0].tool_call.name", + "events[0].tool_call.status", + ] + + +# --------------------------------------------------------------------------- +# The one documented empty string +# --------------------------------------------------------------------------- + + +def test_a_missing_title_becomes_the_empty_string_and_says_so(): + """Allowed because the contract defines it, declared because a reader cannot + tell it from an observed empty title.""" + trace = trace_of(tool_event(title=None)) + events, report = ir_to_acp_capture_events(trace) + assert events[0]["title"] == "" + assert_schema_valid(events) + + records = report.for_field("events[0].tool_call.title") + assert len(records) == 1 + assert records[0].loss_class is LossClass.SYNTHESIZED + assert "contract" in records[0].detail + + +def test_an_observed_empty_title_declares_nothing(): + """The complement — otherwise the record above would be unconditional and + would say nothing about which titles were real.""" + trace = trace_of(tool_event(title="")) + events, report = ir_to_acp_capture_events(trace) + assert events[0]["title"] == "" + assert report.for_field("events[0].tool_call.title") == [] + + +def test_a_missing_text_is_refused_rather_than_emptied(): + """`text` is not `title`. + + The schema documents the empty string as a value the capture path really + records — an unconditionally captured empty prompt — not as a way to write + an absence. Emitting one for `None` would collapse the tri-state §8.2 is + built on. + """ + event = TraceEvent( + index=0, + kind=EventKind.USER_MESSAGE, + text=None, + provenance=Provenance(source_format="hand-built"), + ) + assert blocker_fields(trace_of(event)) == ["events[0].text"] + + +def test_carried_source_fields_never_reach_the_record(): + """`additionalProperties: false` on every shape, so extensions stay out. + + An IR event can carry arbitrary source keys the inbound edge preserved. + Merging them into the record would be the easiest way to lose nothing — and + would emit a schema-invalid record, which is a worse outcome than declaring + the loss. Found by mutation: nothing asserted this until a mutation that + merged `extensions` into the record left the suite green. + """ + event = TraceEvent( + index=0, + kind=EventKind.USER_MESSAGE, + text="go", + extensions={"ts": "2026-08-18T00:00:00", "example_index": 0}, + provenance=Provenance(source_format="hand-built"), + ) + events, report = ir_to_acp_capture_events(trace_of(event)) + + assert events == [{"type": "user_message", "text": "go"}] + assert_schema_valid(events) + record = report.for_field("events[0].extensions") + assert len(record) == 1 + assert record[0].loss_class is LossClass.DROPPED + + +def test_a_tool_call_records_extensions_are_dropped_not_merged(): + """The same property on the shape with the most required fields.""" + trace = trace_of(tool_event(extensions={"raw_input": {"cmd": "ls"}})) + events, report = ir_to_acp_capture_events(trace) + assert set(events[0]) == { + "type", + "tool_call_id", + "kind", + "title", + "status", + "content", + } + assert_schema_valid(events) + assert report.for_field("events[0].extensions") + + +def test_a_timeout_keeps_its_three_keys_and_drops_the_rest(): + """The one shape that reads `extensions`, so the boundary is worth pinning: + the three contract keys are consumed, anything else is declared.""" + trace = trace_of( + timeout_event( + extensions={ + "timeout_sec": 300, + "pending_tool_call_ids": [], + "terminal_trajectory_complete": True, + "stray": "value", + } + ) + ) + events, report = ir_to_acp_capture_events(trace) + assert "stray" not in events[0] + assert_schema_valid(events) + records = report.for_field("events[0].extensions") + assert len(records) == 1 + assert "stray" in records[0].detail + + +def test_an_observed_empty_text_is_written_as_observed(): + trace = trace_of( + TraceEvent( + index=0, + kind=EventKind.AGENT_MESSAGE, + text="", + provenance=Provenance(source_format="hand-built"), + ) + ) + events, _ = ir_to_acp_capture_events(trace) + assert events == [{"type": "agent_message", "text": ""}] + assert_schema_valid(events) + + +# --------------------------------------------------------------------------- +# Outside the codomain +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", [EventKind.ORACLE, EventKind.UNKNOWN]) +def test_an_event_outside_the_codomain_is_refused(kind): + """Not dropped, not emitted invalid, and not a reason to widen the schema. + + §2.4 documents the oracle record as something the capture emitter never + produces; `unknown` is unmodelled by definition. The schema has three record + shapes and this edge writes only those three. + """ + event = TraceEvent( + index=0, + kind=kind, + source_type="oracle" if kind is EventKind.ORACLE else "mystery", + extensions={"command": "solve.sh", "return_code": 0}, + provenance=Provenance(source_format="hand-built"), + ) + blockers = acp_capture_blockers(trace_of(event)) + assert [b.field for b in blockers] == ["events[0]"] + assert kind.value in blockers[0].detail + + +def test_an_oracle_trace_produces_no_records_at_all(): + """The failure is total, so nothing can leak into a caller's file.""" + trace = acp_events_to_ir( + [{"type": "oracle", "command": "solve.sh", "return_code": 0, "stdout": "ok"}] + ) + assert trace.events[0].kind is EventKind.ORACLE + with pytest.raises(AcpCaptureNotRepresentable): + ir_to_acp_capture_events(trace) + + +# --------------------------------------------------------------------------- +# Timeout: four required values, none defaulted +# --------------------------------------------------------------------------- + + +def test_a_timeout_rebuilds_from_the_keys_the_inbound_edge_preserved(): + trace = acp_events_to_ir([CONFORMANT[4]]) + events, _ = ir_to_acp_capture_events(trace) + assert events == [CONFORMANT[4]] + assert_schema_valid(events) + + +@pytest.mark.parametrize( + ("extensions", "expected"), + [ + ( + {}, + [ + "events[0].extensions.timeout_sec", + "events[0].extensions.pending_tool_call_ids", + "events[0].extensions.terminal_trajectory_complete", + ], + ), + ( + {"timeout_sec": 300, "pending_tool_call_ids": []}, + ["events[0].extensions.terminal_trajectory_complete"], + ), + ( + { + "timeout_sec": "300", + "pending_tool_call_ids": [], + "terminal_trajectory_complete": True, + }, + ["events[0].extensions.timeout_sec"], + ), + ( + { + "timeout_sec": True, + "pending_tool_call_ids": [], + "terminal_trajectory_complete": True, + }, + ["events[0].extensions.timeout_sec"], + ), + ( + { + "timeout_sec": 300, + "pending_tool_call_ids": "none", + "terminal_trajectory_complete": True, + }, + ["events[0].extensions.pending_tool_call_ids"], + ), + ( + { + "timeout_sec": 300, + "pending_tool_call_ids": [], + "terminal_trajectory_complete": 1, + }, + ["events[0].extensions.terminal_trajectory_complete"], + ), + ], +) +def test_a_timeout_missing_or_mistyped_fields_is_refused(extensions, expected): + """No budget is invented, and no type is coerced. + + `timeout_sec: True` is its own case: `bool` is an `int` subclass in Python + and a timeout budget of `True` is a malformed record, not the number 1. + """ + trace = trace_of(timeout_event(extensions=extensions)) + assert blocker_fields(trace) == expected + + +def test_a_timeout_with_another_reason_is_refused(): + """`reason` is a single-member enum; there is no other reason to state.""" + trace = trace_of(timeout_event(outcome="idle_timeout")) + assert blocker_fields(trace) == ["events[0].outcome"] + + +# --------------------------------------------------------------------------- +# Fail closed +# --------------------------------------------------------------------------- + + +def test_one_unrepresentable_event_refuses_the_whole_trace(): + """No partial success. + + The target has no envelope, no count and no marker for "some events are + missing", so a partial list is indistinguishable from a complete one the + moment it leaves this function. + """ + trace = trace_of( + TraceEvent( + index=0, + kind=EventKind.USER_MESSAGE, + text="go", + provenance=Provenance(source_format="hand-built"), + ), + tool_event(index=1, status=None), + TraceEvent( + index=2, + kind=EventKind.AGENT_MESSAGE, + text="done", + provenance=Provenance(source_format="hand-built"), + ), + ) + assert validate_trace(trace) == [] + with pytest.raises(AcpCaptureNotRepresentable) as excinfo: + ir_to_acp_capture_events(trace) + assert [b.field for b in excinfo.value.blockers] == ["events[1].tool_call.status"] + + +def test_the_inspection_helper_and_the_converter_never_disagree(): + """Both entry points share one pass, so "is it representable" and "convert + it" cannot answer differently.""" + for trace in ( + acp_events_to_ir(CONFORMANT), + trace_of(tool_event(status=None)), + trace_of(timeout_event(extensions={})), + ): + blockers = acp_capture_blockers(trace) + if blockers: + with pytest.raises(AcpCaptureNotRepresentable): + ir_to_acp_capture_events(trace) + else: + ir_to_acp_capture_events(trace) + + +def test_the_error_is_the_repositorys_existing_idiom(): + """A `ValueError` subclass defined in the exporter module, like + `PrimeSftTrajectoryJsonlError` and the `ValueError` `ir_to_atif` raises.""" + assert issubclass(AcpCaptureNotRepresentable, ValueError) + + +# --------------------------------------------------------------------------- +# The loss report +# --------------------------------------------------------------------------- + + +def test_nothing_trace_level_is_smuggled_into_an_event(): + """The capture format is a flat stream with no envelope.""" + trace = trace_of( + tool_event(), + trace_id="t-1", + session_id="s-1", + agent=ModelInfo(agent_name="a", model="m"), + usage=TraceUsage(input_tokens=5), + outcome=TraceOutcome(status=None, stop_reason="end_turn"), + extensions={"x": 1}, + ) + events, report = ir_to_acp_capture_events(trace) + serialized = json.dumps(events) + for leaked in ("t-1", "s-1", "end_turn"): + assert leaked not in serialized + declared = {r.field for r in report.records} + assert { + "trace_id", + "session_id", + "agent.agent_name", + "agent.model", + "usage", + "outcome.stop_reason", + "extensions", + } <= declared + + +def test_the_outbound_edge_declares_only_what_this_trace_loses(): + """A trace carrying nothing extra declares nothing extra — the rule Slice D + adopted, so a report stays a statement about the conversion in hand.""" + _, report = ir_to_acp_capture_events(trace_of(tool_event(arguments=None))) + assert report.for_field("trace_id") == [] + assert report.for_field("usage") == [] + assert report.for_field("events[0].tool_call.arguments") == [] + + +def test_every_declared_loss_path_resolves_in_the_canonical_encoding(): + """A declaration a reader of the trace cannot find is not a declaration.""" + trace = acp_events_to_ir(CONFORMANT) + _, report = ir_to_acp_capture_events(trace) + document = trace.model_dump(mode="json") + unresolved = [ + record.field + for record in report.records + if record.space is PathSpace.HUB + and "[]" not in record.field + and not resolve_ir_path(document, record.field)[0] + ] + assert unresolved == [], unresolved + + +def test_no_required_acp_field_is_filled_by_an_undeclared_default(): + """Every required field is either carried or declared. + + `title` is the only one written for an absence, and it has a record. If any + other required field ever starts being defaulted, this fails. + """ + trace = trace_of(tool_event(title=None)) + events, report = ir_to_acp_capture_events(trace) + declared = {r.field for r in report.records} + record = events[0] + for field, ir_path in ( + ("tool_call_id", "events[0].tool_call.call_id"), + ("kind", "events[0].tool_call.name"), + ("title", "events[0].tool_call.title"), + ("status", "events[0].tool_call.status"), + ): + carried = getattr( + trace.events[0].tool_call, + { + "tool_call_id": "call_id", + "kind": "name", + "title": "title", + "status": "status", + }[field], + ) + assert record[field] is not None + assert carried is not None or ir_path in declared, field + + +def test_the_report_grows_only_in_its_per_event_half(): + """Declared absence has to stay affordable, as on every other edge. + + The trace-level half is declared once per conversion and must not scale; + the per-event half scales exactly linearly. Asserting the total would be + asserting their sum, which hides which half moved. + """ + + def split(report): + per_event = [r for r in report.records if r.field.startswith("events[")] + return len(report.records) - len(per_event), len(per_event) + + one_trace, one_event = split( + ir_to_acp_capture_events(acp_events_to_ir(CONFORMANT))[1] + ) + ten_trace, ten_event = split( + ir_to_acp_capture_events(acp_events_to_ir(CONFORMANT * 10))[1] + ) + assert one_trace == ten_trace, "the trace-level half must not scale" + assert ten_event == 10 * one_event, "the per-event half must scale linearly" + + +def test_no_record_is_ever_written_for_an_event_that_blocked(): + """Fail-closed at the record level, not only at the function level.""" + trace = trace_of(tool_event(index=0), tool_event(index=1, status=None)) + with pytest.raises(AcpCaptureNotRepresentable): + ir_to_acp_capture_events(trace) + assert len(acp_capture_blockers(trace)) == 1 + + +# --------------------------------------------------------------------------- +# Schema conformance of everything this edge emits +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "trace_factory", + [ + pytest.param(lambda: acp_events_to_ir(CONFORMANT), id="round-trip"), + pytest.param(lambda: trace_of(tool_event(title=None)), id="synthesized-title"), + pytest.param( + lambda: trace_of(tool_event(title="", call_id="")), id="observed-empties" + ), + pytest.param(lambda: trace_of(tool_event(content=[])), id="no-content"), + pytest.param(lambda: trace_of(timeout_event()), id="timeout"), + pytest.param( + lambda: trace_of( + TraceEvent( + index=0, + kind=EventKind.AGENT_REASONING, + reasoning="t", + reasoning_segments=["t"], + role=Role.AGENT, + provenance=Provenance(source_format="hand-built"), + ) + ), + id="reasoning", + ), + ], +) +def test_every_successful_output_validates_against_the_slice_a_schema(trace_factory): + events, _ = ir_to_acp_capture_events(trace_factory()) + assert events + assert_schema_valid(events) + + +def test_the_schema_used_here_is_the_published_one(): + """Guards against the suite validating a copy that drifted.""" + assert SCHEMA["$id"].endswith("acp-capture-event-v1.schema.json") + assert [ref["$ref"].split("/")[-1] for ref in SCHEMA["oneOf"]] == [ + "text_event", + "tool_call_event", + "agent_timeout_event", + ] + + +def test_the_status_vocabulary_matches_the_schema(): + """The constant this edge refuses against is the schema's own enum.""" + assert ( + set(SCHEMA["$defs"]["tool_call_event"]["properties"]["status"]["enum"]) + == ACP_TOOL_STATUSES + ) + assert ( + ACP_TIMEOUT_REASON + in SCHEMA["$defs"]["agent_timeout_event"]["properties"]["reason"]["enum"] + ) + + +# --------------------------------------------------------------------------- +# Isolation +# --------------------------------------------------------------------------- + + +def test_this_edge_imports_nothing_but_the_hub(): + source = Path(ir_to_acp_module.__file__).read_text(encoding="utf-8") + imported: set[str] = set() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add(node.module or "") + assert {name for name in imported if name.startswith("benchflow")} == { + "benchflow.trajectories.ir" + } + + +def test_the_edge_writes_nothing_to_disk(): + """It returns records; persisting them is a caller's decision and a wiring + question this slice does not open.""" + source = Path(ir_to_acp_module.__file__).read_text(encoding="utf-8") + for forbidden in ("open(", "write_text", "Path(", "TrajectoryWriter"): + assert forbidden not in source, forbidden diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py index 34000219c..dc334ee92 100644 --- a/tests/trajectories/test_trace_ir.py +++ b/tests/trajectories/test_trace_ir.py @@ -550,6 +550,7 @@ def test_the_oracle_record_has_its_own_kind_and_role(): "ir_to_atif", "ir_from_atif", "ir_from_otel", + "ir_to_acp", "_otlp_anyvalue", "ir_round_trip", ) From 476611f429d71b291123149e128bcd4f432ab447 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Tue, 18 Aug 2026 23:51:41 +0200 Subject: [PATCH 17/23] trajectories: join the measured round trip to the declared loss contract The round-trip harness measures what changed (D) and reads only the traces. Each converter separately declares what it could not carry, or had to invent (L). Nothing joined the two, so a divergence no edge had declared looked exactly like one that was. ir_conformance.py is that join, kept out of both halves so the measurement stays independent of the declarations it is supposed to be checked against. Two things the naive D subset-of L misses, and what the gate does instead: - The sides address different path spaces. ir_to_atif writes schema_version and step_id into the ATIF document and can only declare them in TARGET space: a hub path would address a node the input trace does not contain, which ir.py's resolution invariant correctly rejects. ir_from_atif then reads them back into the hub, so the round trip observes values the input never had, without either edge having lied. TARGET_TO_HUB relates the two, and a test checks every entry against real values rather than asserting it. It is a bridge and not an exemption: remove the SYNTHESIZED record and the gate fails anyway. - Events are fused away, so some divergence is structural rather than a mishandled field. structure_explained requires every value that went missing to be one a vanished event was holding - per value, not per path - so an edit to an event that survived the trip is still caught. ir_to_atif now declares schema_version and steps[].step_id as SYNTHESIZED instead of leaving them undeclared. SYNTHESIZED and not NORMALIZED, because neither has a source value being reshaped: the dialect string is not an observation about the run, and step positions are not the IR's event index renumbered. The rules: an undeclared fabrication is always a violation, with no allowlist for structural metadata; a loss at a path the target can hold is a violation; a transformed path must account for both the values that left and the values that arrived, since TRANSFORMED conflates the two. Mutation testing found a real gap - the suite would have accepted a NORMALIZED record where the rule asks for a SYNTHESIZED one - and a test now covers it. Still unwired. ir_conformance joins the IR family in the isolation test, nothing on a run path imports it, and no artifact changes. ir.py, ir_round_trip.py, the viewer and every runtime module are untouched. tests/trajectories: 699 passed. Mutation harness: 11/11 applied and caught. Human verification H1-H8 recorded in docs/trace-interop.md section 8.13. --- docs/trace-interop.md | 133 +++++- src/benchflow/trajectories/ir_conformance.py | 348 ++++++++++++++ src/benchflow/trajectories/ir_to_atif.py | 45 ++ tests/trajectories/test_ir_conformance.py | 448 +++++++++++++++++++ tests/trajectories/test_ir_to_atif.py | 6 + tests/trajectories/test_trace_ir.py | 1 + 6 files changed, 978 insertions(+), 3 deletions(-) create mode 100644 src/benchflow/trajectories/ir_conformance.py create mode 100644 tests/trajectories/test_ir_conformance.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md index 95645f7ca..614130630 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -1378,7 +1378,8 @@ here. invariants, the validation suite, this section, five converters — `ACP → IR`, `IR → ATIF`, `ATIF → IR`, `OTLP/JSON → IR`, `IR → ACP capture events` — each with its loss report; the round-trip measurement over the ATIF pair (§8.10), - and the ACP pair's round-trip evidence in §8.3 and its test suite. There is no + and the ACP pair's round-trip evidence in §8.3 and its test suite; and the + loss-bounded conformance gate that joins the two halves (§8.13). There is no §8.10-style measurement for the ACP pair: `ACP → IR → ACP` either reproduces its input exactly or refuses, so there is no partial preservation to quantify. - **Deferred by a decision of ours, reversibly:** `IR → OTel` — see §8.12 for @@ -1393,8 +1394,8 @@ here. The isolation property is unchanged in substance and restated at a new boundary: `ir.py`, `ir_from_acp.py`, `ir_to_atif.py`, `ir_from_atif.py`, -`ir_from_otel.py`, `ir_to_acp.py`, `_otlp_anyvalue.py` and `ir_round_trip.py` -form a closed family that may import each other, and +`ir_from_otel.py`, `ir_to_acp.py`, `_otlp_anyvalue.py`, `ir_round_trip.py` and +`ir_conformance.py` form a closed family that may import each other, and `test_only_the_ir_family_imports_the_ir` asserts that nothing else in `src/benchflow` imports any of them. @@ -1963,3 +1964,129 @@ the same contract every other outbound edge is held to (§8.3, `IR → ATIF`). Adding the edge later is cheap by construction: the outbound direction is one module and one loss report, exactly as `ir_to_atif.py` is, and the family test in §8.7 is what keeps that addition from quietly wiring anything. + +### 8.13 Loss-bounded conformance: is the observed divergence inside the declared contract? + +§8.10 measures **D** — what actually changed across a round trip, read off the +two traces and never off the reports. Each converter separately produces **L** — +what it declared it could not carry, or had to invent. The two are produced +independently on purpose, and +[`test_the_harness_reads_traces_and_not_reports`](../tests/trajectories/test_ir_round_trip.py) +exists to keep the measurement from confirming the converters against +themselves. + +[`ir_conformance.py`](../src/benchflow/trajectories/ir_conformance.py) is the +third thing: the **join**. It answers one question — *is every observed +divergence accounted for by something the contract declared?* — and it is +deliberately not folded into either half, because a measurement that consulted +the declarations would stop being a measurement. + +#### The naive form does not hold, for two reasons worth stating + +The obvious gate is `D(T, R(T)) ⊆ L(T)`. Implemented literally it fails on a +correct round trip, and both failures are real properties of the model rather +than bugs to paper over. + +**The two sides address different spaces.** D speaks in canonical *hub* paths of +the IR. L speaks in three (`PathSpace`), and an outbound edge legitimately +declares a value in `TARGET` space using its target's vocabulary. `ir_to_atif` +writes `schema_version` into the ATIF document and declares it *there*, because +at that moment the value has no IR antecedent: a hub path would address a node +the trace being converted does not contain, and `ir.py`'s invariant that every +hub record resolves in the trace would — correctly — reject it. Then +`ir_from_atif` reads the value faithfully back into `extensions.schema_version`, +and the round trip observes a value the input never had. **Neither edge lied. +The fabrication is a property of the composition**, and the join is the only +place that can see it. + +`TARGET_TO_HUB` is the bridge, three entries, each a fact about what +`ir_from_atif` does rather than a convention: + +| declared at (target) | observed at (hub) | why | +|---|---|---| +| `schema_version` | `extensions.schema_version` | the inbound edge stores the document's dialect under the trace's extensions | +| `steps[].step_id` | `events[].extensions.step_id` | ...and each step's id under its event's | +| `steps[].message` | `events[].text` | an ATIF step message becomes the event's text | + +**This is a bridge, not an exemption.** Remove the `SYNTHESIZED` record and the +gate fails; the map only lets a declaration made honestly in one space answer +for the path it occupies in the other. `test_every_bridge_entry_is_real` runs a +trip and checks each correspondence against actual values, so an entry cannot +rot into a lie while still reading as coverage. + +**Not every divergence is a field-level fact.** When an event is fused away, the +values it held stop appearing at *every* path it populated. Those divergences +are caused by a change in the event sequence, not by a converter mishandling a +field. `structure_explained` is how the join says so **without** exempting the +paths involved: the values that went missing must be exactly values held by +events of a kind that lost instances. It is per value, not per path — a kind +vanishing does not license *any* change at a path those events touched. An edit +to a surviving event leaves a missing value nobody was holding, and fails. + +#### The rules + +1. **Undeclared fabrication is always a violation.** Values coming back with + none going in requires a `SYNTHESIZED` record at that path, directly or + through the bridge. There is no allowlist, and **structural metadata is not + exempt**: `schema_version` and `steps[].step_id` are now declared by + `ir_to_atif` like anything else, and clear the gate the ordinary way. +2. **A representable loss is a violation.** Values in, none out, at a path the + target *does* have a slot for, is a gap in our own edge. +3. **A transformed path must account for both of its sides.** Values that + disappeared: declared, or structure-explained. Values that appeared: + declared. `TRANSFORMED` conflates the two, and a rule checking only the first + would let an arbitrary insertion through — a mutation confirms it does not. + +#### Why `schema_version` and `step_id` are `SYNTHESIZED` and not `NORMALIZED` + +`NORMALIZED` says a source value was reshaped. Neither of these has a source +value: the ATIF dialect string is not an observation about the run, and the +step positions are not the IR's event index renumbered — that index is a +position in a different sequence, over events some of which never become steps. +Calling them normalizations would assert a correspondence that does not exist, +which is the same laundering §8.3 refuses for `name_semantics`. + +#### Human verification of the conformance gate + +**HUMAN E2E VERIFIED, 2026-08-18, H1–H8 all pass**, against +`e2e-h/PROCEDURE.md`. Every step was carried out and judged by a person rather +than by the suite. + +| | what was verified | +|---|---| +| H1 | the working tree is exactly the six files under review, and the family/isolation tests pass — nothing outside the IR family imports any of it | +| H2 | `test_ir_conformance.py` green with **no skips**, so the two captured rollouts really ran rather than silently vanishing with the evidence tree | +| H3 | `D`, `L`, `D ∩ L` and `D \ L` read by hand on both rollouts: the bridge reduces `D \ L(hub)` to `D \ L`, every residual is structure-explained, and the gate reports no violation | +| H4 | each `TARGET_TO_HUB` entry corresponds to a real inbound/outbound behaviour, not to a convenience; the bridge tests pass | +| H5 | the gate can be made to fail: removing the `schema_version` record, removing `steps[].step_id`, or relabelling `SYNTHESIZED` as `NORMALIZED` each make the trip non-conformant | +| H6 | the structural rule is not a blanket exemption — an edit to an event that *survived* the trip is caught even though other events vanished | +| H7 | mutation harness: baseline green, every mutation applied and every one caught, no `ANCHOR NOT FOUND`, no `MISSED`, the file restored and `git status` unchanged | +| H8 | full suite on `HEAD` and on the working tree compared directly: the same pre-existing failures on both, empty diff, tree restored | + +**The verification found one defect, in the procedure, not in the gate.** H8's +full-suite command could stop at an interactive GitHub credential prompt +part-way through a long run, which reads as a hang rather than as a failure. +`GIT_TERMINAL_PROMPT=0` is now on every H8 command so a credential problem +fails visibly instead of blocking the step. + +#### What a green gate does not establish + +- **Not that the contract is *right*.** An edge that declares `SYNTHESIZED` on + a field and then invents it passes here. The gate checks that declaration and + observation **agree**; it cannot check that the declaration is true. It is a + consistency check between two independently produced artefacts, which is + exactly as much as a join can be. +- **Not a universal property.** It was run on finite corpora — the two captured + rollouts of §5.2 and Slice E's fixture. A clean result is evidence about + those traces, and no corpus was built to make it come out that way. +- **Nothing about OpenTelemetry.** There is no OTel round trip here because + there is no `IR → OTel` edge to close one with (§8.12). The OTel direction is + ingest-only, and a gate cannot measure a loop that does not exist. +- **Little about ACP.** The `Representability` column travels from §8.10's ATIF + capability table, so rule 2 is meaningful for the ATIF pair and for nothing + else. The ACP loop is carried only as a **regression guard**, with that rule + filtered out where it does not apply, and the test says so. +- **Nothing about a run.** No wiring was added. The gate reads two traces and + two loss reports that a test constructed; it is not on any run path, writes + no artifact, and `ir.py`, `ir_round_trip.py`, the viewer and every runtime + module are untouched by it. diff --git a/src/benchflow/trajectories/ir_conformance.py b/src/benchflow/trajectories/ir_conformance.py new file mode 100644 index 000000000..00596c08b --- /dev/null +++ b/src/benchflow/trajectories/ir_conformance.py @@ -0,0 +1,348 @@ +"""Loss-bounded conformance — is the observed divergence inside the declared contract? + +> **PROVISIONAL.** Part of the unwired IR family (`docs/trace-interop.md` §8.7). +> Nothing imports it from a run path and it changes no artifact. + +`ir_round_trip` measures **D**: what actually differs between a trace and its +round trip, read off the two traces and never off the reports. The converters +produce **L**: what each edge declared it could not carry, or had to invent. +Neither knows about the other, and that separation is load-bearing — +`test_the_harness_reads_traces_and_not_reports` exists to keep the measurement +from confirming the converters against themselves. + +This module is the **third** thing: the join. It answers one question — *is every +observed divergence accounted for by something the contract declared?* — and it +is deliberately not folded into either half. + +## Why the join is not a set comparison + +The obvious form, ``D(T, R(T)) ⊆ L(T)``, does not typecheck against the model +as it stands, for two reasons that are worth stating rather than papering over. + +**The two sides address different spaces.** D speaks in canonical *hub* paths of +the IR (``events[].text``, indices collapsed). L speaks in three spaces +(:class:`~benchflow.trajectories.ir.PathSpace`), and an outbound edge +legitimately declares a value in ``TARGET`` space using its *target's* +vocabulary. `ir_to_atif` writes ``schema_version`` into the ATIF document and +declares it there, because at that moment the value has no IR antecedent — a hub +path would address a node the trace being converted does not contain. Then +`ir_from_atif` reads it faithfully back into ``extensions.schema_version``, and +the round trip sees a value the input never had. **Neither edge lied; the +fabrication is a property of the composition.** :data:`TARGET_TO_HUB` is the +bridge, and every entry in it is verified by a test rather than asserted. + +**Not every divergence is a field-level fact.** When an event is fused away, the +values it held stop appearing — at every path it populated. Those divergences +are caused by a change in the event sequence, not by any converter mishandling a +field. :func:`structure_explained` is how the join says so **without** a blanket +exemption for the paths involved: the values that went missing must be exactly +values held by events of a kind that lost instances. An arbitrary edit to a +surviving event produces a missing value no vanished event held, and fails. + +## The rules + +1. **Undeclared fabrication is always a violation.** A path with values coming + back and none going in must have a ``SYNTHESIZED`` declaration at that path — + directly, or through the bridge. There is no allowlist, and structural + metadata is not exempt: ``schema_version`` and ``step_id`` are declared like + anything else, with a detail that says what they are. +2. **A representable loss is a violation.** Values that went in, none that came + back, at a path the target *does* have a slot for, is a gap in our own edge. +3. **A transformed path must account for both of its sides.** Values that + disappeared: declared, or structure-explained. Values that appeared: + declared. ``TRANSFORMED`` conflates the two, and a rule that checked only the + first would let an arbitrary insertion through. + +## What a clean run does not establish + +That the contract is *right*. An edge that declares ``SYNTHESIZED`` on a field +and then invents it passes here — the gate checks that the declaration and the +observation agree, not that the declaration is true. It is a consistency check +between two independently produced artefacts, which is exactly as much as a +join can be. +""" + +from __future__ import annotations + +import re +from collections import Counter +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from benchflow.trajectories.ir import CanonicalTrace, LossClass, LossReport, PathSpace +from benchflow.trajectories.ir_round_trip import ( + Representability, + RoundTripOutcome, + RoundTripReport, + _values_by_path, +) + +TARGET_TO_HUB: dict[str, str] = { + "schema_version": "extensions.schema_version", + "steps[].step_id": "events[].extensions.step_id", + "steps[].message": "events[].text", +} +"""Where a value declared in ``TARGET`` space lands in the hub on the way back. + +Three entries, all for the `ATIF` pair, and each one is a fact about +`ir_from_atif` rather than a convention: it stores the document's +``schema_version`` under the trace's ``extensions``, each step's ``step_id`` +under its event's, and a step ``message`` becomes the event's ``text``. + +**This is a bridge, not an exemption.** Without a ``SYNTHESIZED`` record at the +target path the gate still fails; the map only lets the join find a declaration +that was made honestly in the other space. ``test_every_bridge_entry_is_real`` +runs a trip and checks each correspondence against the values, so an entry +cannot rot into a lie. +""" + + +class Rule(BaseModel): + """Marker for the rule a violation broke, so failures are greppable.""" + + model_config = ConfigDict(extra="forbid") + + name: str + text: str + + +UNDECLARED_FABRICATION = Rule( + name="undeclared-fabrication", + text="values came back that never went in, and no SYNTHESIZED record " + "declares them at that path", +) +REPRESENTABLE_LOSS = Rule( + name="representable-loss", + text="values went in and none came back, at a path the target has a slot " + "for — a gap in our own edge rather than a cost of the format", +) +UNEXPLAINED_DISAPPEARANCE = Rule( + name="unexplained-disappearance", + text="values stopped appearing at a path that declares nothing, and they " + "were not held by the events the round trip fused away", +) +UNDECLARED_INSERTION = Rule( + name="undeclared-insertion", + text="values appeared at a transformed path with no record declaring them", +) + + +class Violation(BaseModel): + """One observed divergence the declared contract does not account for.""" + + model_config = ConfigDict(extra="forbid") + + path: str + outcome: RoundTripOutcome + rule: str + detail: str + declared: list[str] = Field(default_factory=list) + """The loss classes found at this path, if any — empty is the usual cause.""" + + def __str__(self) -> str: # pragma: no cover - diagnostics + return f"[{self.rule}] {self.path} ({self.outcome.value}): {self.detail}" + + +def canonical(field: str) -> str: + """A loss record's path in the shape :mod:`ir_round_trip` compares by. + + ``events[3].tool_call.arguments`` and the systemic ``events[].tool_call. + arguments`` are the same path once indices collapse, which is the same + normalization ``_canonical_leaves`` performs on the trace side. + """ + return re.sub(r"\[\d+\]", "[]", field) + + +def declared_classes(path: str, *reports: LossReport | None) -> set[LossClass]: + """Every loss class declared at *path*, across the edges of one round trip. + + Hub records match directly. Target records match through + :data:`TARGET_TO_HUB`, which is what lets a synthesis declared honestly in + the target's vocabulary answer for the hub path it later occupies. + """ + found: set[LossClass] = set() + for report in reports: + if report is None: + continue + for record in report.records: + here = canonical(record.field) + if record.space is PathSpace.HUB: + matches = here == path + elif record.space is PathSpace.TARGET: + matches = TARGET_TO_HUB.get(here) == path + else: + # SOURCE records describe the format a trace came *from*; they + # say nothing about a hub path and must not answer for one. + matches = False + if matches: + found.add(record.loss_class) + return found + + +def vanished_kinds(report: RoundTripReport) -> dict[str, int]: + """Event kinds that lost instances across the trip, with how many. + + This is the only structural signal :class:`RoundTripReport` carries, and it + is enough: alignment between individual events is not recoverable after a + conversion that fuses and drops them, and guessing at it would be the + invention this family refuses. + """ + before, after = Counter(report.kinds_before), Counter(report.kinds_after) + return { + kind: count - after.get(kind, 0) + for kind, count in before.items() + if count > after.get(kind, 0) + } + + +def _values_held_by(trace: CanonicalTrace, kinds: set[str], path: str) -> Counter: + """What the events of *kinds* contribute at *path*, as a multiset. + + Each event is measured on its own so its values are attributed to it rather + than to the trace, which is what makes the structural rule specific to the + events that actually vanished. + """ + held: Counter = Counter() + for event in trace.events: + if event.kind.value not in kinds: + continue + one = trace.model_copy(update={"events": [event]}) + for value in _values_by_path(one).get(path, []): + held[repr(value)] += 1 + return held + + +def structure_explained( + before: CanonicalTrace, + after: CanonicalTrace, + report: RoundTripReport, + path: str, +) -> bool: + """True when everything that stopped appearing at *path* left with an event. + + The rule is deliberately narrow. It does not say "some events vanished, so + changes at this path are fine" — that would excuse an arbitrary edit + whenever any merge happened. It says: **every value missing from this path + is a value an event of a vanished kind was holding.** A surviving event + whose text was rewritten leaves a missing value nobody held, and fails. + """ + kinds = set(vanished_kinds(report)) + if not kinds: + return False + missing = Counter(map(repr, _values_by_path(before).get(path, []))) - Counter( + map(repr, _values_by_path(after).get(path, [])) + ) + if not missing: + return True + return not (missing - _values_held_by(before, kinds, path)) + + +def _appeared(before: CanonicalTrace, after: CanonicalTrace, path: str) -> Counter: + return Counter(map(repr, _values_by_path(after).get(path, []))) - Counter( + map(repr, _values_by_path(before).get(path, [])) + ) + + +def conformance_violations( + before: CanonicalTrace, + after: CanonicalTrace, + report: RoundTripReport, + *reports: LossReport | None, +) -> list[Violation]: + """Every observed divergence the declared contract does not account for. + + An empty list is the gate passing. *reports* are the loss reports of the + edges the trip went through, in any order — typically the outbound one the + converter returned and the inbound one carried on ``after``. + + Rule 2 reads :attr:`FieldComparison.representability`, which + `ir_round_trip` fills from its ATIF capability table. That column is + meaningful for the ATIF pair and not for any other; a loop with no ``LOST`` + comparison is unaffected either way. + """ + violations: list[Violation] = [] + + for comparison in report.comparisons: + path = comparison.path + declared = declared_classes(path, *reports) + names = sorted(loss.value for loss in declared) + + if comparison.outcome is RoundTripOutcome.FABRICATED: + if LossClass.SYNTHESIZED not in declared: + violations.append( + Violation( + path=path, + outcome=comparison.outcome, + rule=UNDECLARED_FABRICATION.name, + detail=( + f"{comparison.after_count} value(s) came back and none " + f"went in; no SYNTHESIZED record addresses this path, " + f"directly or through the target bridge" + ), + declared=names, + ) + ) + continue + + if comparison.outcome is RoundTripOutcome.LOST: + if comparison.representability is Representability.REPRESENTABLE: + violations.append( + Violation( + path=path, + outcome=comparison.outcome, + rule=REPRESENTABLE_LOSS.name, + detail=( + f"{comparison.before_count} value(s) went in and none " + "came back, at a path the target can hold" + ), + declared=names, + ) + ) + continue + + if comparison.outcome is not RoundTripOutcome.TRANSFORMED: + continue + + # Both sides of a transformation have to be accounted for separately: + # values leaving and values arriving are different events with different + # explanations, and one rule covering both would hide the second. + if not declared and not structure_explained(before, after, report, path): + violations.append( + Violation( + path=path, + outcome=comparison.outcome, + rule=UNEXPLAINED_DISAPPEARANCE.name, + detail=( + "values stopped appearing here, nothing declares the path, " + "and they were not held by the events that vanished " + f"(kinds: {sorted(vanished_kinds(report)) or 'none'})" + ), + declared=names, + ) + ) + continue + + if _appeared(before, after, path) and not declared: + violations.append( + Violation( + path=path, + outcome=comparison.outcome, + rule=UNDECLARED_INSERTION.name, + detail=( + "values appeared at this path that were not in the input, " + "and no record declares them" + ), + declared=names, + ) + ) + + return violations + + +def conformance_summary(violations: list[Violation]) -> dict[str, int]: + """Violation counts per rule — for reading a failure at a glance.""" + summary: dict[str, Any] = {} + for violation in violations: + summary[violation.rule] = summary.get(violation.rule, 0) + 1 + return summary diff --git a/src/benchflow/trajectories/ir_to_atif.py b/src/benchflow/trajectories/ir_to_atif.py index 00c29817c..7d05876b3 100644 --- a/src/benchflow/trajectories/ir_to_atif.py +++ b/src/benchflow/trajectories/ir_to_atif.py @@ -407,6 +407,7 @@ def flush_thoughts() -> None: ) _declare_systemic_losses(trace, losses) + _declare_structural_metadata(steps, losses) record: dict[str, Any] = {"schema_version": ATIF_SCHEMA_VERSION} if trace.session_id: @@ -417,6 +418,50 @@ def flush_thoughts() -> None: return record, losses +def _declare_structural_metadata( + steps: list[dict[str, Any]], losses: LossReport +) -> None: + """Declare the two values this edge writes that describe the *document*. + + Neither is a statement about the run, and both are deterministic — one is a + constant, the other a position. That is precisely why they were easy to miss: + a value that is obviously not an observation still *becomes* one once a + reader takes it back into the hub. `ATIF → IR` puts `schema_version` into + the trace's ``extensions`` and each `step_id` into its event's, faithfully, + because they really are in the document it is reading. So the round trip + ends with two values the input never had, and only a declaration here + distinguishes them from an observation. + + ``SYNTHESIZED`` rather than ``NORMALIZED``: there is no source value being + reshaped. Nothing in the IR corresponds to either of them. + + The space is ``TARGET`` and the paths are the ATIF document's, because that + is where this edge writes them and neither has an IR antecedent. A hub path + would be a lie twice over — it would address a node the trace being + converted does not contain, and the resolvability guard would be right to + reject it. + """ + losses.add( + "schema_version", + LossClass.SYNTHESIZED, + f"structural metadata of the ATIF document: the literal " + f"{ATIF_SCHEMA_VERSION!r} identifying the dialect, written on every " + "record. It is not an observation about the run, and the IR carries no " + "field for a target format's own version", + space=PathSpace.TARGET, + ) + if steps: + losses.add( + "steps[].step_id", + LossClass.SYNTHESIZED, + f"structural metadata the exporter introduces: {len(steps)} step " + "positions, dense from 1 over the steps actually emitted. No id was " + "observed in the source — the IR's event index is a position in a " + "different sequence, and this is not that index renumbered", + space=PathSpace.TARGET, + ) + + def _declare_systemic_losses(trace: CanonicalTrace, losses: LossReport) -> None: """Losses that hold for the conversion rather than for one event. diff --git a/tests/trajectories/test_ir_conformance.py b/tests/trajectories/test_ir_conformance.py new file mode 100644 index 000000000..a9ed74b56 --- /dev/null +++ b/tests/trajectories/test_ir_conformance.py @@ -0,0 +1,448 @@ +"""The conformance gate: every observed divergence is accounted for, and the +gate still fails when it should. + +Half of this file is the gate passing on real rollouts. The other half is the +part that gives the first half its meaning — each rule is shown *failing* on an +input built to break exactly it. A gate that cannot be made to fail proves +nothing about the runs where it passes. + +No corpus is built here. The rollouts are the ones already captured for the +`ATIF` human verification, and the rich fixture is Slice E's. +""" + +from __future__ import annotations + +import json +import pathlib +from typing import Any + +import pytest + +from benchflow.trajectories.ir import LossClass, LossReport, PathSpace, Role +from benchflow.trajectories.ir_conformance import ( + REPRESENTABLE_LOSS, + TARGET_TO_HUB, + UNDECLARED_FABRICATION, + UNDECLARED_INSERTION, + UNEXPLAINED_DISAPPEARANCE, + canonical, + conformance_summary, + conformance_violations, + declared_classes, + structure_explained, + vanished_kinds, +) +from benchflow.trajectories.ir_from_acp import acp_events_to_ir +from benchflow.trajectories.ir_round_trip import ( + Representability, + RoundTripOutcome, + _values_by_path, + compare_traces, + round_trip_through_atif, +) +from benchflow.trajectories.ir_to_acp import ir_to_acp_capture_events +from tests.trajectories.test_atif_preservation import _rich_events + +EVIDENCE = pathlib.Path(__file__).resolve().parents[2].parent / "e2e-a2" / "evidence" + + +def _rollout(name: str) -> list[dict[str, Any]] | None: + """A captured `ACP` rollout, or ``None`` when the evidence tree is absent. + + The captures live outside the clone, next to the human-verification + scaffolding. The suite must stay green without them, so every test that + wants one skips rather than fails — and :func:`_rich_events` covers the + same rules from inside the repo, so a skip never leaves a rule untested. + """ + path = EVIDENCE / name / "acp_trajectory.jsonl" + if not path.is_file(): + return None + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _gate(events: list[dict[str, Any]]): + """Run the `ATIF` loop and join the measurement to the declarations.""" + trip = round_trip_through_atif(events, session_id="s", agent_name="a") + return trip, conformance_violations( + trip.before, trip.after, trip.report, trip.outbound, trip.inbound + ) + + +def _captured(name: str): + events = _rollout(name) + if events is None: + pytest.skip(f"captured rollout {name!r} is not in this tree") + return _gate(events) + + +# -------------------------------------------------------------------------- +# the gate passing +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name", ["h1", "h2"]) +def test_a_captured_rollout_stays_inside_its_declared_contract(name): + """The whole point: on a real run, D is covered by L. + + Nothing here asserts *how many* divergences there are. The count is a + property of the rollout and of every converter it passes through, and + pinning it would turn an honest improvement into a test failure. + """ + trip, violations = _captured(name) + assert violations == [], "\n".join(str(v) for v in violations) + # ...and the run has to be non-trivial, or the assertion above is vacuous. + assert any( + c.outcome is not RoundTripOutcome.PRESERVED for c in trip.report.comparisons + ) + + +def test_the_rich_fixture_stays_inside_its_declared_contract(): + """Same gate, on the in-repo fixture — so it holds with no evidence tree.""" + _, violations = _gate(_rich_events()) + assert violations == [], "\n".join(str(v) for v in violations) + + +def test_the_acp_loop_is_a_regression_guard(): + """`IR → ACP → IR′` declares what it changes too. + + Slice G's loop is included as a guard, not as a second contract: the + representability column travels from the ATIF table and is not about ACP, + so only the rules that do not consult it are meaningful here. + """ + before = acp_events_to_ir(_rich_events()) + events, outbound = ir_to_acp_capture_events(before) + after = acp_events_to_ir(events) + report = compare_traces(before, after) + violations = [ + v + for v in conformance_violations(before, after, report, outbound, after.losses) + if v.rule != REPRESENTABLE_LOSS.name + ] + assert violations == [], "\n".join(str(v) for v in violations) + + +def test_every_bridge_entry_is_real(): + """Each `TARGET_TO_HUB` entry is checked against values, not asserted. + + A bridge that drifted out of step with `ir_from_atif` would silently + forgive fabrications at a path nobody declares, which is the one failure + mode the map could introduce. So: the target value the document carries has + to actually turn up at the hub path it claims to reach. + """ + trip, _ = _gate(_rich_events()) + hub = _values_by_path(trip.after) + + assert trip.document["schema_version"] in hub["extensions.schema_version"] + step_ids = {step["step_id"] for step in trip.document["steps"]} + assert step_ids & set(hub["events[].extensions.step_id"]) + if any(step.get("message") == "" for step in trip.document["steps"]): + assert "" in hub["events[].text"] + + assert set(TARGET_TO_HUB) == { + "schema_version", + "steps[].step_id", + "steps[].message", + }, "a new bridge entry needs a check above before it can be trusted" + + +def test_the_two_structural_values_are_declared_and_not_exempted(): + """`schema_version` and `step_id` pass the gate the ordinary way. + + They are fabrications — the round trip really does produce values the input + never had — and they clear rule 1 only because `ir_to_atif` declares them + ``SYNTHESIZED``. There is no branch in the gate that knows their names. + """ + trip, _ = _gate(_rich_events()) + fabricated = { + c.path + for c in trip.report.comparisons + if c.outcome is RoundTripOutcome.FABRICATED + } + assert {"extensions.schema_version", "events[].extensions.step_id"} <= fabricated + + for path in ("extensions.schema_version", "events[].extensions.step_id"): + assert LossClass.SYNTHESIZED in declared_classes( + path, trip.outbound, trip.inbound + ) + + +# -------------------------------------------------------------------------- +# the gate failing — one input per rule +# -------------------------------------------------------------------------- + + +def _without(report: LossReport, field: str) -> LossReport: + """The same report with every record at *field* removed.""" + return report.model_copy( + update={"records": [r for r in report.records if r.field != field]} + ) + + +def test_removing_a_synthesized_declaration_fails_the_gate(): + """Rule 1, on the exact path the user asked not to exempt. + + Drop `ir_to_atif`'s ``schema_version`` record and the fabrication it + covered has nothing left to stand on — which is what makes the passing case + above evidence of a declaration rather than of an allowlist. + """ + trip, clean = _gate(_rich_events()) + assert clean == [] + + violations = conformance_violations( + trip.before, + trip.after, + trip.report, + _without(trip.outbound, "schema_version"), + trip.inbound, + ) + assert [v.path for v in violations] == ["extensions.schema_version"] + assert violations[0].rule == UNDECLARED_FABRICATION.name + + +def test_removing_the_step_id_declaration_fails_the_gate(): + """Rule 1 again, for the second structural value.""" + trip, _ = _gate(_rich_events()) + violations = conformance_violations( + trip.before, + trip.after, + trip.report, + _without(trip.outbound, "steps[].step_id"), + trip.inbound, + ) + assert [v.path for v in violations] == ["events[].extensions.step_id"] + assert violations[0].rule == UNDECLARED_FABRICATION.name + + +def test_a_declaration_of_the_wrong_class_does_not_excuse_a_fabrication(): + """Rule 1 asks for ``SYNTHESIZED`` specifically, not for any record at all. + + ``NORMALIZED`` says a source value was reshaped. A value that was never in + the input has no source value to reshape, so a record of that class at the + path is not an account of the fabrication — it is a different claim that + happens to share an address. + + Found by mutating the rule into "is anything declared here", which the + suite did not notice until this test existed. + """ + trip, _ = _gate(_rich_events()) + mislabelled = LossReport(direction=trip.outbound.direction) + for record in trip.outbound.records: + mislabelled.add( + record.field, + LossClass.NORMALIZED + if record.field == "schema_version" + else record.loss_class, + record.detail, + space=record.space, + ) + + assert declared_classes("extensions.schema_version", mislabelled) == { + LossClass.NORMALIZED + }, "the path must still be declared, or this tests an empty lookup" + + violations = conformance_violations( + trip.before, trip.after, trip.report, mislabelled, trip.inbound + ) + assert ("extensions.schema_version", UNDECLARED_FABRICATION.name) in { + (v.path, v.rule) for v in violations + } + + +def test_a_fabrication_at_a_path_nobody_declares_fails_the_gate(): + """Rule 1, for a value no edge has ever heard of. + + An undeclared field appears in the reconstructed trace, and the gate says + so without needing to know what it means. + """ + trip, _ = _gate(_rich_events()) + after = trip.after.model_copy( + update={"extensions": {**trip.after.extensions, "invented": "no-one-said-so"}} + ) + report = compare_traces(trip.before, after) + violations = conformance_violations( + trip.before, after, report, trip.outbound, trip.inbound + ) + assert [v.path for v in violations if v.rule == UNDECLARED_FABRICATION.name] == [ + "extensions.invented" + ] + + +def test_editing_a_surviving_event_fails_the_gate(): + """Rule 3, and the reason ``structure_explained`` is narrow. + + Some events genuinely vanish in this loop, so a rule of the form "changes + are fine when a merge happened" would wave this through. The value edited + here belongs to an event that *survived*, so no vanished event was holding + it, and the gate catches it. + """ + trip, clean = _gate(_rich_events()) + assert clean == [] + + survivors = list(trip.after.events) + edited = survivors[0].model_copy(update={"role": Role.ORACLE}) + after = trip.after.model_copy(update={"events": [edited, *survivors[1:]]}) + + report = compare_traces(trip.before, after) + violations = conformance_violations( + trip.before, after, report, trip.outbound, trip.inbound + ) + assert UNEXPLAINED_DISAPPEARANCE.name in {v.rule for v in violations} + assert "events[].role" in {v.path for v in violations} + + +def test_losing_a_representable_field_fails_the_gate(): + """Rule 2: a gap in our own edge, not a cost of the format.""" + trip, _ = _gate(_rich_events()) + stripped = [e.model_copy(update={"text": None}) for e in trip.after.events] + after = trip.after.model_copy(update={"events": stripped}) + + report = compare_traces(trip.before, after) + assert any( + c.path == "events[].text" + and c.outcome is RoundTripOutcome.LOST + and c.representability is Representability.REPRESENTABLE + for c in report.comparisons + ) + violations = conformance_violations( + trip.before, after, report, trip.outbound, trip.inbound + ) + assert ("events[].text", REPRESENTABLE_LOSS.name) in { + (v.path, v.rule) for v in violations + } + + +def test_an_undeclared_insertion_at_a_transformed_path_fails_the_gate(): + """Rule 3's second half — the side a one-sided rule would miss. + + ``TRANSFORMED`` covers both values leaving and values arriving. Here the + loss side is structure-explained, and a value is *added* on top; the gate + has to object to the addition on its own. + """ + trip, _ = _gate(_rich_events()) + events = list(trip.after.events) + kinds = set(vanished_kinds(trip.report)) + assert kinds, "this fixture must lose at least one event kind" + + # `role` is carried by structure alone, so an insertion is the only fault. + extra = events[0].model_copy(update={"index": 999, "role": Role.ORACLE}) + after = trip.after.model_copy(update={"events": [*events, extra]}) + report = compare_traces(trip.before, after) + violations = conformance_violations( + trip.before, after, report, trip.outbound, trip.inbound + ) + assert ("events[].role", UNDECLARED_INSERTION.name) in { + (v.path, v.rule) for v in violations + } + + +def test_the_gate_is_not_satisfied_by_the_measurement_alone(): + """With no declarations at all, a real run is full of violations. + + This is the negative control for the whole join: the observation half does + not justify itself, and every clean result above is the *pair* agreeing. + """ + trip, _ = _gate(_rich_events()) + violations = conformance_violations(trip.before, trip.after, trip.report) + assert len(violations) >= 3 + assert UNDECLARED_FABRICATION.name in conformance_summary(violations) + + +# -------------------------------------------------------------------------- +# the pieces +# -------------------------------------------------------------------------- + + +def test_structure_explained_needs_a_vanished_kind(): + """No event lost, no structural excuse — even for a real transformation.""" + trip, _ = _gate(_rich_events()) + report = trip.report.model_copy( + update={"kinds_before": dict(trip.report.kinds_after)} + ) + assert not structure_explained(trip.before, trip.after, report, "events[].kind") + + +def test_structure_explained_does_not_cover_a_value_nobody_held(): + """The rule is per value, not per path. + + A kind vanishing does not license *any* change at a path the vanished + events touched — only the disappearance of the values they were holding. + """ + trip, _ = _gate(_rich_events()) + assert structure_explained(trip.before, trip.after, trip.report, "events[].kind") + + survivors = list(trip.after.events) + edited = survivors[0].model_copy(update={"role": Role.ORACLE}) + after = trip.after.model_copy(update={"events": [edited, *survivors[1:]]}) + assert not structure_explained(trip.before, after, trip.report, "events[].role") + + +def test_canonical_collapses_indices_the_way_the_measurement_does(): + assert canonical("events[3].tool_call.arguments") == ( + "events[].tool_call.arguments" + ) + assert canonical("events[].text") == "events[].text" + assert canonical("extensions.schema_version") == "extensions.schema_version" + + +def test_a_target_record_only_answers_for_the_path_it_bridges_to(): + """The bridge is a map, not a wildcard over the target space.""" + report = LossReport(direction="ir->atif") + report.add( + "steps[].step_id", LossClass.SYNTHESIZED, "structural", space=PathSpace.TARGET + ) + assert declared_classes("events[].extensions.step_id", report) == { + LossClass.SYNTHESIZED + } + assert declared_classes("events[].index", report) == set() + assert declared_classes("steps[].step_id", report) == set() + + +def test_a_hub_record_answers_only_at_its_own_path(): + report = LossReport(direction="ir->atif") + report.add("events[].text", LossClass.NORMALIZED, "reshaped", space=PathSpace.HUB) + assert declared_classes("events[].text", report) == {LossClass.NORMALIZED} + assert declared_classes("steps[].message", report) == set() + + +def test_vanished_kinds_reports_only_net_losses(): + trip, _ = _gate(_rich_events()) + vanished = vanished_kinds(trip.report) + assert vanished + for kind, count in vanished.items(): + assert count == trip.report.kinds_before[kind] - trip.report.kinds_after.get( + kind, 0 + ) + assert count > 0 + + +def test_the_gate_does_not_mutate_what_it_reads(): + """It is a join over two artefacts, and it leaves both as it found them.""" + trip, _ = _gate(_rich_events()) + snapshot = ( + trip.before.model_dump_json(), + trip.after.model_dump_json(), + trip.report.model_dump_json(), + trip.outbound.model_dump_json(), + ) + conformance_violations( + trip.before, trip.after, trip.report, trip.outbound, trip.inbound + ) + assert snapshot == ( + trip.before.model_dump_json(), + trip.after.model_dump_json(), + trip.report.model_dump_json(), + trip.outbound.model_dump_json(), + ) + + +def test_a_missing_report_is_tolerated_not_treated_as_a_declaration(): + """``None`` in the reports is absence of evidence, and it must not pass.""" + trip, _ = _gate(_rich_events()) + violations = conformance_violations( + trip.before, trip.after, trip.report, None, None + ) + assert violations, "no declarations must not read as everything declared" diff --git a/tests/trajectories/test_ir_to_atif.py b/tests/trajectories/test_ir_to_atif.py index f001ee76e..a4341a352 100644 --- a/tests/trajectories/test_ir_to_atif.py +++ b/tests/trajectories/test_ir_to_atif.py @@ -400,6 +400,12 @@ def test_target_only_values_are_declared_in_the_target_space(): "steps[1]", "steps[].message", "final_metrics.total_steps", + # Structural metadata of the document. Deterministic and obviously not + # observations, which is why they were missed — but `ATIF → IR` reads + # both back into the hub, so the round trip ends with two values the + # input never had, and only these records tell them apart from data. + "schema_version", + "steps[].step_id", } assert all( r.loss_class is LossClass.SYNTHESIZED for r in report.by_space(PathSpace.TARGET) diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py index dc334ee92..05988b3d3 100644 --- a/tests/trajectories/test_trace_ir.py +++ b/tests/trajectories/test_trace_ir.py @@ -553,6 +553,7 @@ def test_the_oracle_record_has_its_own_kind_and_role(): "ir_to_acp", "_otlp_anyvalue", "ir_round_trip", + "ir_conformance", ) """The unwired modules of the proposal: the IR and its converters. From 360ec6885052824d80da3e75eb28d78ea136de34 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Wed, 19 Aug 2026 00:54:57 +0200 Subject: [PATCH 18/23] trajectories: project a canonical trace onto viewer trace steps ir_to_view.py converts a trace into the step list a viewer page renders. It produces steps and deliberately not a whole payload: of a payload's five fields, only the steps are a function of the trace. rollout_name is a directory name, verifier is four sidecar files, and meta comes from result.json and timing.json - the IR has no slot for task_name, skill_mode, reward, partial_trajectory or trajectory_source at all. Returning a payload here would mean synthesizing run metadata to fill a shape rather than converting a trace, so the trace-level fields the IR does carry are declared UNSUPPORTED: outside this edge's codomain, not lost by it. Assembly belongs to the wiring slice, which has the directory. The wire shape is read from the viewer package proposed in #1034, at the commit recorded in VIEW_SCHEMA_ORIGIN. Nothing is imported, vendored or fetched from that branch - it is unmerged, and the family rule is the one ir_to_atif already follows for ATIF: read the target format as data, pin what matters by test, never reach into the module that handles it. That pin protects this edge from drifting; it cannot notice #1034 changing. tool.name_semantics is a seventh key on an object #1034 defines with six, and it is here because the alternative is inference. #1034 derives a tool's display category with tool_hue(kind, title), which scans both strings for needles: an ATIF function_name of 'execute' becomes the execute category, and an OTel gen_ai.tool.name of 'read_file' becomes 'read' because the word appears in it. This edge emits a hue only when the semantics say acp_kind and the value is already a member of the display vocabulary - membership, never inference - and otherwise the neutral 'other', which the renderer maps to the secondary tokens and which therefore asserts nothing. On the corpora that means 'execute' is spelled identically in the ACP and ATIF rows and gets a category in one only. Unlike the ACP edge this one never raises. A viewer is a display: an event it cannot type must still reach the page. Every event becomes exactly one step, UNKNOWN and ORACLE included, and what the shape cannot hold goes to the loss report. ORACLE has no member in the step vocabulary, so it lands on 'unknown' with its identity in the type slot, and both untyped kinds carry a serialization of the canonical IR event - not a raw source record, since the IR does not hold one, and the loss record says so. Sentinels are declared per slot and only where the target has no null to write. An absent title and an observed empty one both render as "", so the report is the only place that difference survives; a block the IR holds with no text contributes no string rather than having its raw form rendered as JSON. Still unwired. ir_to_view joins the IR family in the isolation test, nothing on a run path imports it, no page is rendered and no artifact changes. ir.py, ir_round_trip.py and the existing viewer.py are untouched. tests/trajectories: 752 passed (699 + 53). Mutation harness: 13/13 applied and caught. Human verification V1-V10 recorded in docs/trace-interop.md section 8.14, along with what it does not cover. --- docs/trace-interop.md | 181 ++++- src/benchflow/trajectories/ir_to_view.py | 544 +++++++++++++++ tests/trajectories/test_ir_to_view.py | 819 +++++++++++++++++++++++ tests/trajectories/test_trace_ir.py | 1 + 4 files changed, 1538 insertions(+), 7 deletions(-) create mode 100644 src/benchflow/trajectories/ir_to_view.py create mode 100644 tests/trajectories/test_ir_to_view.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md index 614130630..5f9c08419 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -1375,11 +1375,12 @@ here. ### 8.7 Status - **Implemented:** the IR types, the loss model with its path spaces, the - invariants, the validation suite, this section, five converters — `ACP → IR`, - `IR → ATIF`, `ATIF → IR`, `OTLP/JSON → IR`, `IR → ACP capture events` — each - with its loss report; the round-trip measurement over the ATIF pair (§8.10), - and the ACP pair's round-trip evidence in §8.3 and its test suite; and the - loss-bounded conformance gate that joins the two halves (§8.13). There is no + invariants, the validation suite, this section, six converters — `ACP → IR`, + `IR → ATIF`, `ATIF → IR`, `OTLP/JSON → IR`, `IR → ACP capture events`, + `IR → viewer trace steps` (§8.14) — each with its loss report; the round-trip + measurement over the ATIF pair (§8.10), and the ACP pair's round-trip evidence + in §8.3 and its test suite; and the loss-bounded conformance gate that joins + the two halves (§8.13). There is no §8.10-style measurement for the ACP pair: `ACP → IR → ACP` either reproduces its input exactly or refuses, so there is no partial preservation to quantify. - **Deferred by a decision of ours, reversibly:** `IR → OTel` — see §8.12 for @@ -1394,8 +1395,9 @@ here. The isolation property is unchanged in substance and restated at a new boundary: `ir.py`, `ir_from_acp.py`, `ir_to_atif.py`, `ir_from_atif.py`, -`ir_from_otel.py`, `ir_to_acp.py`, `_otlp_anyvalue.py`, `ir_round_trip.py` and -`ir_conformance.py` form a closed family that may import each other, and +`ir_from_otel.py`, `ir_to_acp.py`, `_otlp_anyvalue.py`, `ir_round_trip.py`, +`ir_conformance.py` and `ir_to_view.py` form a closed family that may import +each other, and `test_only_the_ir_family_imports_the_ir` asserts that nothing else in `src/benchflow` imports any of them. @@ -2090,3 +2092,168 @@ fails visibly instead of blocking the step. two loss reports that a test constructed; it is not on any run path, writes no artifact, and `ir.py`, `ir_round_trip.py`, the viewer and every runtime module are untouched by it. + +### 8.14 `Canonical Trace IR → viewer trace steps` + +[`ir_to_view.py`](../src/benchflow/trajectories/ir_to_view.py) converts a trace +into the step list a BenchFlow viewer page renders. It is unwired: no run path +imports it, no page is built from it, and no artifact changes. + +#### It produces steps, and deliberately not a payload + +A viewer payload has five fields and only one of them is a function of the +trace. `rollout_name` is a directory name; `verifier` is four sidecar files; +`meta` is `result.json` and `timing.json`. The IR has no slot for `task_name`, +`skill_mode`, `reward`, `partial_trajectory` or `trajectory_source` **at all**, +so an edge that returned a whole payload would be **synthesizing run metadata +to fill a shape** rather than converting a trace. The trace-level fields the IR +*does* carry — `session_id`, `agent`, `usage`, `outcome`, the trace's own +start and finish — are declared `UNSUPPORTED`, which says they are outside this +edge's codomain rather than lost by it: the viewer shows them, from artifacts +this edge never sees. Assembly belongs to the wiring slice, which has the +directory. + +#### Where the shape comes from, and how provisional that is + +The wire shape is read from the viewer package proposed in +`benchflow-ai/benchflow#1034`, at the commit recorded in `VIEW_SCHEMA_ORIGIN`. +**Nothing is imported, vendored or fetched from that branch** — it is unmerged, +and the family rule is the one §8.3 already follows for ATIF: read the target +format as data, pin what matters by test, never reach into the module that +handles it. `test_the_vocabularies_are_frozen` freezes our copy. + +Be clear about what that buys: it protects **our** contract from drifting. It +cannot notice #1034 changing. That correspondence was checked by a person once, +against the recorded commit, and re-checking it is a human step. + +#### The one addition + +`tool.name_semantics` is **ours**. #1034's `ToolCall` has six fields; this is a +seventh, and it carries :attr:`ToolCall.name_semantics` through unchanged. +Without it the viewer boundary cannot tell an ACP *category* from a *function +name* from a *span name*, and that is exactly the gap #1034 fills by inference: + +```python +_HUE_INFER = (("read", ("read", "cat", "view", "ls", "list")), ...) +def tool_hue(kind: str, title: str) -> ToolHue: + if kind in _KNOWN_HUES: return kind + hay = f"{kind} {title}".lower() + for hue, needles in _HUE_INFER: + if any(needle in hay for needle in needles): return hue +``` + +On the three corpora that reads: an ACP `read` is a read (correct); an ATIF +`function_name` of `execute` becomes the `execute` category (a name read as a +category); an OTel `gen_ai.tool.name` of `read_file` becomes `read` (a category +inferred from a substring). This edge does none of it. A hue is emitted only +when the semantics say `acp_kind` **and** the value already **is** a member of +the display vocabulary — membership, never inference — and otherwise the +neutral `other`, which the renderer maps to the secondary/border tokens and +which therefore asserts nothing. Emitting an extra key is additive, but it is a +divergence from #1034's contract and not an agreed extension to it. + +#### Declare, don't refuse + +Unlike §8.3's ACP edge, this one never raises. A viewer is a display: an event +it cannot type must still reach the page. **Every event produces exactly one +step** — pinned by test on both captured rollouts and on hand-built traces — +and what the shape cannot hold goes to the loss report. + +| IR event kind | step kind | how identity survives | +|---|---|---| +| `USER_MESSAGE` | `prompt` | observed `text` | +| `AGENT_MESSAGE` | `message` | observed `text` | +| `AGENT_REASONING` | `thought` | `reasoning`, not `text` | +| `TOOL_CALL` | `tool` | the tool object, `name_semantics` included | +| `TIMEOUT` | `timeout` | stays **typed**; `reason` from the event's terminal signal, the rest from `extensions` | +| `ORACLE` | `unknown` | `type: "oracle"` — the vocabulary has no oracle member | +| `UNKNOWN` | `unknown` | `type` = the source's own type string | + +`source_type` is carried for **every** kind, not only the untyped ones: a +normalized kind whose source string was discarded is a lossy rename. + +For the two kinds with no typed slot the step's `text` is a serialization of +the **canonical IR event**. #1034's own unknown branch serializes the raw ACP +dict it read off disk; this edge has no such document, so what it renders +carries IR field names, and the loss record says so. A page showing it must +label it the same way — it is not the producer's payload. + +#### Absent is not observed-empty + +Six viewer slots have no null to write. Each substitution is declared on its +own, and **only** when the source value was absent: an observed `""` title and +a missing one both render as `""`, and the report is the only place that +difference survives. Blocks are the same principle: a block the IR holds with +no text contributes no string, declared and omitted, because rendering its +`raw` as JSON would put tool output on the page that no tool ever produced. An +observed empty string is kept — #1034 filters falsy strings out, and that is a +small silent loss this edge does not reproduce. + +#### Measured on the three corpora + +**FACT — machine measurement, not a human verification.** + +| corpus | events → steps | tools (`kind`, `name_semantics`, `hue`) | loss records | +|---|---|---|---| +| ACP H1 | 5 → 5 | `execute`/`acp_kind`/**`execute`**, `read`/`acp_kind`/**`read`** | 17 | +| ACP H2 | 4 → 4 | `think`/`acp_kind`/**`think`** | 17 | +| ATIF (§8.10 fixture) | 3 → 3 | `execute`/`function_name`/**`other`** | 18 | +| OTel (§8.11 fixture) | 5 → 5 | `read_file`, `write_file` / `gen_ai.tool.name` / **`other`** | 23 | + +The third and fourth rows are the point. `execute` is spelled identically in +rows one and three and gets a category in one and not the other, because the +difference is in the semantics and not in the string. + +OTel is also the only corpus with timestamps — all five steps carry `t` and +`dur` — and the only one whose untyped events carry the whole span in +`extensions`, `parentSpanId` included. It is also the only one with real tool +arguments, which the viewer shape has no slot for and which are declared +`DROPPED`: an observed value reaching no field. + +Counts here are snapshots of these fixtures, not thresholds. + +#### Human verification of the viewer edge + +**HUMAN E2E VERIFIED, 2026-08-19, V1–V10 all pass**, against +`e2e-view/PROCEDURE.md`. Every step was carried out and judged by a person +reading the values the edge produced, not by re-running the suite. + +| | what was verified | +|---|---| +| V1 | the working tree is exactly the four files under review; `ir.py`, `ir_round_trip.py`, the existing `viewer.py` and every runtime module untouched; the family/import test green | +| V2 | steps are 1-based and dense, no `label` is ever written, `type` is preserved on the **known** kinds too, absent optionals are omitted rather than nulled, and the tool object carries seven keys including `name_semantics` | +| V3 | ACP H1 5→5, ACP H2 4→4, ATIF 3→3, OTel 5→5 — one step per event on every corpus, no silent drop | +| V4 | a real `acp_kind` can produce a semantic hue; `function_name` and `gen_ai.tool.name` stay `other`; no substring and no title inference, checked with titles deliberately stuffed with category words | +| V5 | `ORACLE` and `UNKNOWN` both reach the page; the diagnostic text is declared a serialization of the canonical IR event and not a raw source record; OTel parentage survives inside `extensions` | +| V6 | absent and observed-empty are distinguishable **only** through the loss report, and are; H2's real timeout is carried, not synthesized; an opaque block with no text is `DROPPED` rather than invented | +| V7 | `role`, `content[].kind`/`raw`, per-event usage, `reasoning_segments` and `arguments` are declared `DROPPED`; trace-level fields are `UNSUPPORTED` as outside this edge's codomain; no run metadata appears on any step | +| V8 | 13 valid mutations applied and caught, no `ANCHOR NOT FOUND`, no `MISSED`, no collection error, restore green | +| V9 | 752 collected = 699 baseline + 53 new; a node-id comparison against `HEAD` shows no pre-existing test missing; ruff, format and `ty` green | +| V10 | the same four files at the end; `e2e-view/` never wrote into the clone | + +**The verification found three defects, all in the mutation harness, none in +the converter.** Two mutations were no-ops — one kept the declaration it was +meant to remove, the other was overwritten by a later dict entry. The third +rewrote a `losses.add(` call into a tuple display, which is a `SyntaxError` +once the call's `space=` keyword is inside it: the module stopped importing and +pytest reported a **collection error** rather than a failure, so the harness +counted as "caught" a run that only proved a broken file will not import. All +three were rewritten outside the repo, and the procedure now names `error` +instead of `failed` as a second way a mutation can fail to mean what it claims. + +#### What this verification does not cover + +- **No wiring, and no page.** Nothing imports this edge from a run path, no + renderer was run, no HTML was produced, and no artifact on disk changes. That + the steps are correct is not evidence that they *render* correctly. +- **The contract pin protects our side only.** `VIEW_STEP_KINDS`, + `VIEW_TOOL_HUES` and the emitted key sets are frozen by test, which stops + *this* edge from drifting. It cannot notice `benchflow-ai/benchflow#1034` + changing — that branch is unmerged and moving, and re-checking the + correspondence to `VIEW_SCHEMA_ORIGIN` is a human step. +- **`name_semantics` is ours.** It is a seventh key on an object #1034 defines + with six: additive, and a divergence from that contract rather than a + maintainer-approved extension to it. +- **`ORACLE` stays `unknown`.** It keeps its identity in `type`, but it will + remain an untyped step until the viewer contract gains a member for it — + which is a change to propose upstream, not one to assume. diff --git a/src/benchflow/trajectories/ir_to_view.py b/src/benchflow/trajectories/ir_to_view.py new file mode 100644 index 000000000..44c22af27 --- /dev/null +++ b/src/benchflow/trajectories/ir_to_view.py @@ -0,0 +1,544 @@ +"""Canonical Trace IR → BenchFlow viewer trace steps — outbound, provisional. + +> **PROVISIONAL.** Part of the unwired IR family (`docs/trace-interop.md` §8.7). +> Nothing imports it from a run path, no artifact changes, and no page is +> rendered from it. + +This edge produces **only the step list** — the part of a viewer page that is +actually a function of the trace. It deliberately does not build a +``ViewerPayload``: that document's other four fields (``rollout_name``, +``meta``, ``verifier``, and the schema version) come from ``result.json``, +``timing.json``, the ``verifier/`` sidecars and the rollout directory's name. +The IR carries none of them and has no slot for ``task_name``, ``skill_mode``, +``reward``, ``partial_trajectory`` or ``trajectory_source`` at all, so +assembling a payload here would mean **synthesizing run metadata to fill a +shape** rather than converting a trace. Assembly belongs to the wiring slice, +which has the directory. + +## What the shape is, and how provisional that is + +The wire shape is taken from the viewer package proposed in +`benchflow-ai/benchflow#1034`, read at :data:`VIEW_SCHEMA_ORIGIN`. **No code is +imported from it and none is vendored** — that branch is unmerged, and the +family rule is the one `ir_to_atif` already follows for ATIF: read the target +format as data, pin what matters by test, never reach into the module that +handles it. The constants below are our copy of that vocabulary, and +`tests/trajectories/test_ir_to_view.py` freezes them. That protects **our** +contract from drifting; it cannot notice #1034 changing. + +## The one addition, and why it is here + +``tool.name_semantics`` is **ours** — #1034's ``ToolCall`` has six fields and +this is a seventh. It carries :attr:`ToolCall.name_semantics` through +unchanged, because without it the viewer boundary loses the only thing that +distinguishes an ACP *category* (``execute``, ``read``) from a *function name* +(``read_file``) from a *span name*. #1034 resolves that by +``tool_hue(kind, title)``, which infers a category from substrings of the two +strings — so an OTel ``gen_ai.tool.name`` of ``read_file`` acquires the ``read`` +category because the word "read" appears in it. This edge does not do that, and +does not need to: the hue it emits is neutral unless a real category was +observed. Emitting one extra key is additive — the renderer reads named fields +— but it is a divergence from #1034's contract, not an agreed extension to it. + +## Loss regime: declare, don't refuse + +Unlike `ir_to_acp`, this edge never raises. A viewer is a display: an event it +cannot type must still reach the page. Every event produces exactly one step, +`UNKNOWN` and `ORACLE` included, and what the shape cannot hold is written to +the :class:`LossReport` instead of being dropped. Sentinels are declared +**per slot**, and only where the target has no null to write: an absent title +and an observed empty title both render as ``""`` and must not read as the same +observation, so only the absent one produces a record. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from typing import Any + +from benchflow.trajectories.ir import ( + CanonicalTrace, + EventKind, + LossClass, + LossReport, + PathSpace, + ToolCall, + TraceEvent, +) + +VIEW_SCHEMA_ORIGIN = "benchflow-ai/benchflow#1034@79695125" +"""The viewer design this shape was read from, at the exact commit audited. + +Recorded so the provenance of every constant below is checkable by a person. +It is not a dependency: nothing here fetches, imports or vendors that branch. +""" + +LOSS_DIRECTION = "ir->view" + +VIEW_STEP_KINDS: tuple[str, ...] = ( + "prompt", + "message", + "thought", + "tool", + "timeout", + "unknown", +) +"""The renderer's step vocabulary. Note there is no ``oracle`` member.""" + +VIEW_TOOL_HUES: tuple[str, ...] = ( + "read", + "edit", + "execute", + "fetch", + "search", + "think", + "skill", + "other", +) +"""Display hues. The renderer whitelists exactly these and maps each to a CSS +custom-property set; ``other`` resolves to the neutral secondary/border tokens, +which is what makes it usable as "no category was observed" rather than as a +claim about the tool.""" + +NEUTRAL_HUE = "other" +"""The member of :data:`VIEW_TOOL_HUES` that asserts nothing.""" + +ACP_KIND_SEMANTICS = "acp_kind" +"""The one :attr:`ToolCall.name_semantics` value that *is* a category. + +Defined here rather than imported from `ir_to_acp` so this edge does not depend +on the ACP edge to know what an ACP kind is; a test pins the two equal.""" + +STEP_KIND: dict[EventKind, str] = { + EventKind.USER_MESSAGE: "prompt", + EventKind.AGENT_MESSAGE: "message", + EventKind.AGENT_REASONING: "thought", + EventKind.TOOL_CALL: "tool", + EventKind.TIMEOUT: "timeout", + EventKind.ORACLE: "unknown", + EventKind.UNKNOWN: "unknown", +} +"""Every :class:`EventKind`, mapped. Total by test, so a new kind cannot reach +the viewer by vanishing from it.""" + +TRACE_LEVEL_PATHS: tuple[str, ...] = ( + "trace_id", + "session_id", + "agent", + "usage", + "outcome", + "started_at", + "finished_at", +) +"""Trace-level fields this edge deliberately does not read. + +They are not losses of the viewer shape — `meta` holds most of them — and they +are not losses of this edge either, because a step list is not where they go. +Declaring them ``UNSUPPORTED`` says exactly that, at paths a reader can resolve +in the trace, instead of leaving the omission unexplained.""" + +DIAGNOSTIC_KINDS = frozenset({EventKind.ORACLE, EventKind.UNKNOWN}) +"""Kinds with no typed slot, rendered as a serialization of the canonical +event. `ORACLE` is here because #1034's ``StepKind`` has no member for it.""" + +_TIMEOUT_SEC_KEY = "timeout_sec" +_TIMEOUT_PENDING_KEY = "pending_tool_call_ids" +_TIMEOUT_COMPLETE_KEY = "terminal_trajectory_complete" + + +def _epoch(value: datetime) -> float: + return value.timestamp() + + +def _diagnostic_text(event: TraceEvent) -> str: + """The canonical event, serialized — **not** a source record. + + The distinction is the point. #1034's own unknown branch serializes the raw + ACP dict it read off disk; this edge has no such record, only the IR event + the inbound converter built from one. Presenting that as a raw payload + would assert a source document the IR does not possess, so the loss record + names it for what it is and the wiring slice is expected to label it in the + page the same way. + """ + return json.dumps( + event.model_dump(mode="json"), ensure_ascii=False, sort_keys=True, indent=2 + ) + + +def _hue(call: ToolCall, where: str, losses: LossReport) -> str: + """The display hue, by direct membership or not at all. + + Two ways to reach a real hue, and both require the source to have said so: + the semantics must be :data:`ACP_KIND_SEMANTICS` — an ACP ``kind`` *is* a + category — and the value must already **be** a member of the display + vocabulary. There is no third path. No substring is examined, no title is + consulted, and :attr:`Provenance.source_format` is never read: where a + trace came from is not evidence about what a field means. + """ + if call.name_semantics == ACP_KIND_SEMANTICS and call.name in VIEW_TOOL_HUES: + return call.name + + if call.name_semantics is None: + reason = "the IR carries no semantics for this tool name" + elif call.name_semantics != ACP_KIND_SEMANTICS: + reason = ( + f"{call.name_semantics!r} is a name, not a category — inferring one " + "from the string would be the reinterpretation this family refuses" + ) + else: + reason = ( + f"the observed category {call.name!r} is outside the viewer's " + "display vocabulary" + ) + losses.add( + f"{where}.tool_call.name_semantics", + LossClass.SYNTHESIZED, + f"the viewer step requires a hue and {reason}; the neutral " + f"{NEUTRAL_HUE!r} was written, which asserts no category", + space=PathSpace.HUB, + ) + return NEUTRAL_HUE + + +def _content_texts(call: ToolCall, where: str, losses: LossReport) -> list[str]: + """The text actually observed, and nothing standing in for what was not. + + A block the IR holds as ``OPAQUE`` — or a ``TEXT`` block whose text is + ``None`` — has no string to contribute. Rendering its ``raw`` as JSON would + manufacture a tool observation that the capture never made, so the block is + declared and omitted. An observed empty string is kept: it is a value. + """ + texts: list[str] = [] + for position, block in enumerate(call.content): + if block.text is None: + losses.add( + f"{where}.tool_call.content[{position}]", + LossClass.DROPPED, + f"a {block.kind.value!r} block with no text; the viewer holds " + "tool output as strings and this edge does not invent one from " + "the block's raw form", + space=PathSpace.HUB, + ) + continue + texts.append(block.text) + return texts + + +def _tool_payload(call: ToolCall, where: str, losses: LossReport) -> dict[str, Any]: + """The tool object, with every non-nullable slot accounted for.""" + if call.call_id is None: + losses.add( + f"{where}.tool_call.call_id", + LossClass.SYNTHESIZED, + "the viewer requires an id string and the source carried no id " + 'field at all, so "" was written; it is not an observed empty id', + space=PathSpace.HUB, + ) + if call.name is None: + losses.add( + f"{where}.tool_call.name", + LossClass.SYNTHESIZED, + 'the viewer requires a name string and the IR carries none, so "" ' + "was written; the renderer's own fallback label is its business, " + "not an observation this edge should make", + space=PathSpace.HUB, + ) + if call.title is None: + losses.add( + f"{where}.tool_call.title", + LossClass.SYNTHESIZED, + "the viewer requires a title string and the source carried none, " + 'so "" was written; an observed empty title declares nothing here', + space=PathSpace.HUB, + ) + if call.status is None: + losses.add( + f"{where}.tool_call.status", + LossClass.SYNTHESIZED, + "the viewer requires a status string and the IR carries none, so " + '"" was written; the renderer shows "?" for it, and that is a ' + "display choice rather than a status the run had", + space=PathSpace.HUB, + ) + if call.arguments is not None: + losses.add( + f"{where}.tool_call.arguments", + LossClass.DROPPED, + "the viewer's tool object has no slot for arguments; the values " + "are observed and reach no field", + space=PathSpace.HUB, + ) + + return { + "id": call.call_id or "", + "kind": call.name or "", + "title": call.title or "", + "status": call.status.value if call.status is not None else "", + "content": _content_texts(call, where, losses), + "hue": _hue(call, where, losses), + "name_semantics": call.name_semantics, + } + + +def _timeout_payload( + event: TraceEvent, where: str, losses: LossReport +) -> dict[str, Any]: + """The typed timeout object — this kind does not fall through to unknown. + + Two of the four slots accept ``None`` and so represent their own absence; + the other two do not, and each sentinel is declared on its own. + """ + extensions = event.extensions + if event.outcome is None: + losses.add( + f"{where}.outcome", + LossClass.SYNTHESIZED, + "the viewer requires a timeout reason string and the event carries " + 'no terminal signal, so "" was written', + space=PathSpace.HUB, + ) + pending_raw = extensions.get(_TIMEOUT_PENDING_KEY) + if pending_raw is None: + losses.add( + f"{where}.extensions", + LossClass.SYNTHESIZED, + f"the viewer requires a list of pending tool-call ids and the event " + f"carries no {_TIMEOUT_PENDING_KEY!r}, so [] was written; it is not " + "an observation that none were pending", + space=PathSpace.HUB, + ) + pending: list[str] = [] + else: + pending = [str(item) for item in pending_raw] + + return { + "reason": event.outcome or "", + "timeout_sec": extensions.get(_TIMEOUT_SEC_KEY), + "pending": pending, + "complete": extensions.get(_TIMEOUT_COMPLETE_KEY), + } + + +def _timestamps(event: TraceEvent, step: dict[str, Any], is_tool: bool) -> bool: + """Attach ``t``/``dur`` when observed. Returns whether anything was written. + + A tool step prefers the call's own window when it has one — that is the + narrower observation — and falls back to the event's. ``dur`` needs both + ends and a non-negative interval; a finish before a start is not a duration + and is left out rather than clamped. + """ + started, finished = event.started_at, event.finished_at + if is_tool and event.tool_call is not None and event.tool_call.started_at: + started = event.tool_call.started_at + finished = event.tool_call.finished_at or finished + if started is None: + return False + step["t"] = _epoch(started) + if finished is not None and finished >= started: + step["dur"] = _epoch(finished) - _epoch(started) + return True + + +def _declare_codomain(losses: LossReport) -> None: + """What this edge is not for. + + These are not step-level losses and must not be counted as any: the trace + carries them, and the document that would hold them is assembled elsewhere + from artifacts this edge never sees. Calling them ``DROPPED`` would say the + viewer cannot represent them, which is false — it says nothing about them + here because here is the wrong place. + """ + for path in TRACE_LEVEL_PATHS: + losses.add( + path, + LossClass.UNSUPPORTED, + "run metadata, not a step. The viewer holds this in `meta`, which " + "the wiring slice builds from result.json and timing.json — it is " + "outside this edge's codomain rather than something the edge lost", + space=PathSpace.HUB, + ) + losses.add( + "steps[].label", + LossClass.UNSUPPORTED, + "prompt labels number the prompts a run was given, which live in " + "prompts.json; a trace does not know its own prompt ordinals, so this " + "edge never writes the key", + space=PathSpace.TARGET, + ) + + +def _declare_systemic(losses: LossReport, trace: CanonicalTrace) -> None: + """Properties of the mapping itself, declared once rather than per event.""" + losses.add( + "steps[].i", + LossClass.SYNTHESIZED, + f"viewer step positions, dense from 1 over the {len(trace.events)} " + "steps emitted. The IR's event index is a position in a different " + "sequence and this is not that index renumbered", + space=PathSpace.TARGET, + ) + losses.add( + "events[].kind", + LossClass.NORMALIZED, + "the IR's event vocabulary is projected onto the viewer's six step " + "kinds; oracle has no member and shares 'unknown' with unrecognized " + "records, which is why the source type is carried alongside", + space=PathSpace.HUB, + ) + + if any(event.role is not None for event in trace.events): + losses.add( + "events[].role", + LossClass.DROPPED, + "the viewer step has no slot for who an event is attributable to; " + "the renderer labels a step from its kind alone", + space=PathSpace.HUB, + ) + if any( + event.tool_call is not None and event.tool_call.content + for event in trace.events + ): + losses.add( + "events[].tool_call.content[].kind", + LossClass.DROPPED, + "tool output is a list of strings in the viewer; whether a block " + "was text or opaque does not survive", + space=PathSpace.HUB, + ) + losses.add( + "events[].tool_call.content[].raw", + LossClass.DROPPED, + "the source block is kept verbatim in the IR and has no viewer slot", + space=PathSpace.HUB, + ) + losses.add( + "steps[].tool.content", + LossClass.NORMALIZED, + "content blocks become plain strings, in order, keeping only text " + "that was observed", + space=PathSpace.TARGET, + ) + if any(event.usage is not None for event in trace.events): + losses.add( + "events[].usage", + LossClass.DROPPED, + "the viewer aggregates usage at the run level in `meta`; a step has " + "no usage slot, so a per-event observation reaches no field", + space=PathSpace.HUB, + ) + if any(event.reasoning_segments for event in trace.events): + losses.add( + "events[].reasoning_segments", + LossClass.DROPPED, + "thought boundaries are not expanded into separate steps by this " + "slice: there is no contract yet for doing so without also emitting " + "the joined `reasoning`, which would show the same text twice", + space=PathSpace.HUB, + ) + if any( + event.tool_call is not None and event.tool_call.status for event in trace.events + ): + losses.add( + "steps[].tool.status", + LossClass.NORMALIZED, + "the IR's status enum becomes the viewer's status string", + space=PathSpace.TARGET, + ) + if any( + event.tool_call is not None + and event.tool_call.name_semantics == ACP_KIND_SEMANTICS + and event.tool_call.name in VIEW_TOOL_HUES + for event in trace.events + ): + losses.add( + "steps[].tool.hue", + LossClass.NORMALIZED, + "an observed ACP category that is already a member of the display " + "vocabulary is carried across directly; membership is tested, never " + "inferred from the string", + space=PathSpace.TARGET, + ) + if any(event.kind in DIAGNOSTIC_KINDS for event in trace.events): + losses.add( + "steps[].text", + LossClass.SYNTHESIZED, + "events with no typed viewer slot are rendered as a serialization " + "of the **canonical IR event**. It is not a raw source record — the " + "IR does not hold one — and a page showing it must say so", + space=PathSpace.TARGET, + ) + + +def ir_to_view_steps(trace: CanonicalTrace) -> tuple[list[dict[str, Any]], LossReport]: + """Project a canonical trace onto the viewer's step list. + + Returns the steps and a report of everything the shape could not carry. + **Every event becomes exactly one step** — there is no branch that skips a + record, and a test pins the counts equal — so an event this edge cannot + type is visible on the page as a diagnostic rather than absent from it. + + The input is not modified and its own inbound report is not touched: a + trace may be converted to many targets and none of them describes it. + """ + losses = LossReport(direction=LOSS_DIRECTION) + _declare_codomain(losses) + _declare_systemic(losses, trace) + + steps: list[dict[str, Any]] = [] + for event in trace.events: + where = f"events[{event.index}]" + kind = STEP_KIND[event.kind] + step: dict[str, Any] = {"i": len(steps) + 1, "kind": kind} + + if event.source_type is not None: + step["type"] = event.source_type + elif event.kind is EventKind.ORACLE: + # The IR observed oracle-ness in `kind`; without a source string the + # step would land in the same undifferentiated 'unknown' as a record + # nobody recognized. Writing the kind here reshapes an observation, + # it does not invent one. + step["type"] = EventKind.ORACLE.value + losses.add( + "steps[].type", + LossClass.NORMALIZED, + "the event kind was written into the type slot for an oracle " + "record that carried no source type string of its own", + space=PathSpace.TARGET, + ) + + if event.kind is EventKind.AGENT_REASONING: + if event.reasoning is not None: + step["text"] = event.reasoning + elif event.kind in DIAGNOSTIC_KINDS: + step["text"] = _diagnostic_text(event) + elif event.text is not None: + step["text"] = event.text + + if event.kind is EventKind.TOOL_CALL: + if event.tool_call is None: + losses.add( + f"{where}.tool_call", + LossClass.DROPPED, + "a tool-call event with no tool call; the step keeps its " + "kind and carries no tool object", + space=PathSpace.HUB, + ) + else: + step["tool"] = _tool_payload(event.tool_call, where, losses) + elif event.kind is EventKind.TIMEOUT: + step["timeout"] = _timeout_payload(event, where, losses) + + _timestamps(event, step, is_tool=event.kind is EventKind.TOOL_CALL) + steps.append(step) + + if any("t" in step for step in steps): + losses.add( + "steps[].t", + LossClass.NORMALIZED, + "observed timestamps become epoch seconds; a tool step prefers the " + "call's own window over the event's when it has one", + space=PathSpace.TARGET, + ) + + return steps, losses diff --git a/tests/trajectories/test_ir_to_view.py b/tests/trajectories/test_ir_to_view.py new file mode 100644 index 000000000..c6ec98f43 --- /dev/null +++ b/tests/trajectories/test_ir_to_view.py @@ -0,0 +1,819 @@ +"""`IR → viewer trace steps`: the shape, the vocabulary, and the refusals. + +The interesting half of this file is the second one. A converter that maps +fields is easy to test by mapping them back; what this edge actually promises +is *negative* — that it will not infer a tool category from a string, will not +manufacture tool output, will not let an event vanish, and will not confuse an +absent value with an observed empty one. Each of those has a test that fails +when the promise is removed. +""" + +from __future__ import annotations + +import json +import pathlib +from datetime import UTC, datetime +from typing import Any + +import pytest + +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlock, + ContentBlockKind, + EventKind, + LossClass, + PathSpace, + Provenance, + Role, + ToolCall, + ToolStatus, + TraceEvent, + TraceUsage, + validate_trace, +) +from benchflow.trajectories.ir_from_acp import acp_events_to_ir +from benchflow.trajectories.ir_from_atif import atif_to_ir +from benchflow.trajectories.ir_from_otel import otlp_json_to_ir +from benchflow.trajectories.ir_to_acp import ACP_KIND_SEMANTICS as ACP_EDGE_SEMANTICS +from benchflow.trajectories.ir_to_atif import ir_to_atif +from benchflow.trajectories.ir_to_view import ( + ACP_KIND_SEMANTICS, + DIAGNOSTIC_KINDS, + LOSS_DIRECTION, + NEUTRAL_HUE, + STEP_KIND, + TRACE_LEVEL_PATHS, + VIEW_SCHEMA_ORIGIN, + VIEW_STEP_KINDS, + VIEW_TOOL_HUES, + ir_to_view_steps, +) +from tests.trajectories.test_atif_preservation import _rich_events +from tests.trajectories.test_ir_from_otel import PRODUCER_PAYLOAD_JSON +from tests.trajectories.test_trace_ir import resolve_ir_path + +EVIDENCE = pathlib.Path(__file__).resolve().parents[2].parent / "e2e-a2" / "evidence" + +_PROV = Provenance(source_format="hand-built") + + +def _trace(*events: TraceEvent) -> CanonicalTrace: + return CanonicalTrace(provenance=_PROV, events=list(events)) + + +def _event(index: int = 0, **kwargs: Any) -> TraceEvent: + kwargs.setdefault("kind", EventKind.AGENT_MESSAGE) + kwargs.setdefault("provenance", _PROV) + return TraceEvent(index=index, **kwargs) + + +def _tool_event(index: int = 0, **call: Any) -> TraceEvent: + call.setdefault("name", "execute") + call.setdefault("name_semantics", ACP_KIND_SEMANTICS) + return _event(index, kind=EventKind.TOOL_CALL, tool_call=ToolCall(**call)) + + +def _records_at(report, field: str) -> list: + return [r for r in report.records if r.field == field] + + +def _rollout(name: str) -> list[dict[str, Any]]: + path = EVIDENCE / name / "acp_trajectory.jsonl" + if not path.is_file(): + pytest.skip(f"captured rollout {name!r} is not in this tree") + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +# --------------------------------------------------------------------------- +# The frozen contract +# --------------------------------------------------------------------------- + + +def test_the_vocabularies_are_frozen(): + """Changing either is changing the wire contract, not an implementation + detail — and the origin comment is what makes the values checkable.""" + assert VIEW_STEP_KINDS == ( + "prompt", + "message", + "thought", + "tool", + "timeout", + "unknown", + ) + assert VIEW_TOOL_HUES == ( + "read", + "edit", + "execute", + "fetch", + "search", + "think", + "skill", + "other", + ) + assert NEUTRAL_HUE in VIEW_TOOL_HUES + assert VIEW_SCHEMA_ORIGIN == "benchflow-ai/benchflow#1034@79695125" + + +def test_every_event_kind_maps_to_a_step_kind(): + """Totality is the anti-drop guarantee at the type level. + + A new `EventKind` with no entry raises here rather than reaching the viewer + by being quietly skipped — which is exactly how both existing normalizers + lose unrecognized records today. + """ + assert set(STEP_KIND) == set(EventKind) + assert set(STEP_KIND.values()) <= set(VIEW_STEP_KINDS) + + +def test_the_acp_category_constant_matches_the_acp_edge(): + """Two edges, one meaning of "this name is a category". + + Pinned rather than imported so the viewer edge does not depend on the ACP + edge, and so a rename on either side is a failing test instead of a silent + divergence in what counts as a real category. + """ + assert ACP_KIND_SEMANTICS == ACP_EDGE_SEMANTICS + + +def test_the_emitted_keys_are_frozen_per_kind(): + """The wire shape, pinned key by key.""" + trace = _trace( + _event(0, kind=EventKind.USER_MESSAGE, source_type="user_message", text="hi"), + _event(1, kind=EventKind.AGENT_MESSAGE, source_type="agent_message", text="yo"), + _event( + 2, + kind=EventKind.AGENT_REASONING, + source_type="agent_thought", + reasoning="hm", + ), + _event( + 3, + kind=EventKind.TOOL_CALL, + source_type="tool_call", + tool_call=ToolCall( + call_id="c1", + name="execute", + name_semantics=ACP_KIND_SEMANTICS, + title="ls", + status=ToolStatus.COMPLETED, + ), + ), + _event( + 4, + kind=EventKind.TIMEOUT, + source_type="agent_timeout", + outcome="wall_clock_timeout", + extensions={ + "timeout_sec": 30.0, + "pending_tool_call_ids": ["c1"], + "terminal_trajectory_complete": False, + }, + ), + _event(5, kind=EventKind.UNKNOWN, source_type="future_record"), + ) + steps, _ = ir_to_view_steps(trace) + + assert [set(step) for step in steps] == [ + {"i", "kind", "type", "text"}, + {"i", "kind", "type", "text"}, + {"i", "kind", "type", "text"}, + {"i", "kind", "type", "tool"}, + {"i", "kind", "type", "timeout"}, + {"i", "kind", "type", "text"}, + ] + assert set(steps[3]["tool"]) == { + "id", + "kind", + "title", + "status", + "content", + "hue", + "name_semantics", + } + assert set(steps[4]["timeout"]) == { + "reason", + "timeout_sec", + "pending", + "complete", + } + + +def test_optional_keys_are_omitted_when_absent_not_nulled(): + """#1034's renderer reads key presence, so a null is not an omission. + + ``label`` is never written at all: prompt ordinals come from prompts.json, + which is the wiring slice's input and not a property of a trace. + """ + steps, report = ir_to_view_steps(_trace(_event(0, kind=EventKind.AGENT_MESSAGE))) + assert set(steps[0]) == {"i", "kind"} + assert "label" not in steps[0] + assert _records_at(report, "steps[].label")[0].loss_class is LossClass.UNSUPPORTED + + +def test_step_numbering_is_dense_from_one_and_declared(): + trace = _trace(*(_event(i) for i in range(4))) + steps, report = ir_to_view_steps(trace) + assert [step["i"] for step in steps] == [1, 2, 3, 4] + record = _records_at(report, "steps[].i")[0] + assert record.loss_class is LossClass.SYNTHESIZED + assert record.space is PathSpace.TARGET + + +# --------------------------------------------------------------------------- +# Mapping +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("kind", "expected"), + [ + (EventKind.USER_MESSAGE, "prompt"), + (EventKind.AGENT_MESSAGE, "message"), + (EventKind.AGENT_REASONING, "thought"), + (EventKind.TOOL_CALL, "tool"), + (EventKind.TIMEOUT, "timeout"), + (EventKind.ORACLE, "unknown"), + (EventKind.UNKNOWN, "unknown"), + ], +) +def test_each_event_kind_reaches_its_step_kind(kind, expected): + event = ( + _tool_event(0) + if kind is EventKind.TOOL_CALL + else _event(0, kind=kind, text="t" if kind is EventKind.USER_MESSAGE else None) + ) + steps, _ = ir_to_view_steps(_trace(event)) + assert steps[0]["kind"] == expected + + +def test_reasoning_is_read_from_reasoning_not_text(): + """The IR keeps thoughts in their own field; the viewer has one text slot.""" + trace = _trace( + _event(0, kind=EventKind.AGENT_REASONING, reasoning="deliberating", text=None) + ) + steps, _ = ir_to_view_steps(trace) + assert steps[0] == {"i": 1, "kind": "thought", "text": "deliberating"} + + +def test_source_type_survives_on_known_kinds_too(): + """Not only on `unknown` — #1034 keeps it for unknown steps alone, and a + normalized kind with its source string discarded is a lossy rename.""" + trace = _trace( + _event( + 0, + kind=EventKind.AGENT_REASONING, + source_type="agent_thought", + reasoning="x", + ), + _event(1, kind=EventKind.USER_MESSAGE, source_type="user", text="y"), + ) + steps, _ = ir_to_view_steps(trace) + assert [step["type"] for step in steps] == ["agent_thought", "user"] + + +def test_no_event_is_ever_dropped(): + """One step per event, whatever the event is. + + The corpora include kinds with no typed slot; if any branch could skip a + record this count would drift, and the page would be missing history with + nothing saying so. + """ + for events in (_rollout("h1"), _rollout("h2")): + trace = acp_events_to_ir(events) + steps, _ = ir_to_view_steps(trace) + assert len(steps) == len(trace.events) + + weird = _trace( + _event(0, kind=EventKind.UNKNOWN, source_type=None), + _event(1, kind=EventKind.ORACLE), + _event(2, kind=EventKind.TOOL_CALL, tool_call=None), + ) + steps, _ = ir_to_view_steps(weird) + assert len(steps) == 3 + + +# --------------------------------------------------------------------------- +# ORACLE and UNKNOWN +# --------------------------------------------------------------------------- + + +def test_an_oracle_event_keeps_its_identity_without_a_typed_slot(): + """`StepKind` has no oracle member, so the type slot carries the fact.""" + trace = acp_events_to_ir( + [ + { + "type": "oracle", + "command": "python solve.py", + "return_code": 0, + "stdout": "ok\n", + } + ] + ) + steps, _ = ir_to_view_steps(trace) + assert steps[0]["kind"] == "unknown" + assert steps[0]["type"] == "oracle" + + body = json.loads(steps[0]["text"]) + assert body["extensions"] == { + "command": "python solve.py", + "return_code": 0, + "stdout": "ok\n", + } + assert body["kind"] == "oracle" + + +def test_an_oracle_with_no_source_string_still_says_it_is_an_oracle(): + """Reshaping the observed kind, not inventing a source record.""" + steps, report = ir_to_view_steps( + _trace(_event(0, kind=EventKind.ORACLE, source_type=None)) + ) + assert steps[0]["type"] == "oracle" + assert _records_at(report, "steps[].type")[0].loss_class is LossClass.NORMALIZED + + +def test_the_diagnostic_text_is_the_canonical_event_not_a_source_record(): + """The distinction the loss record is required to make. + + #1034's unknown branch serializes the raw ACP dict it read off disk. This + edge has no such document — only the IR event built from one — so what it + renders carries IR field names, and the report says so rather than letting + a page present it as the producer's own payload. + """ + trace = acp_events_to_ir([{"type": "mystery", "a": 1}]) + steps, report = ir_to_view_steps(trace) + + body = json.loads(steps[0]["text"]) + assert body["provenance"]["source_format"] == "acp-trajectory-unknown" + assert body["extensions"] == {"type": "mystery", "a": 1} + assert set(body) >= {"index", "kind", "source_type", "provenance", "extensions"} + + record = _records_at(report, "steps[].text")[0] + assert record.loss_class is LossClass.SYNTHESIZED + assert "canonical IR event" in record.detail + assert "not a raw source record" in record.detail + + +def test_an_unknown_without_a_source_type_omits_the_key(): + steps, _ = ir_to_view_steps( + _trace(_event(0, kind=EventKind.UNKNOWN, source_type=None)) + ) + assert "type" not in steps[0] + assert steps[0]["kind"] == "unknown" + + +# --------------------------------------------------------------------------- +# Timeout stays typed +# --------------------------------------------------------------------------- + + +def test_a_timeout_is_typed_and_not_a_diagnostic_blob(): + trace = acp_events_to_ir( + [ + { + "type": "agent_timeout", + "reason": "wall_clock_timeout", + "timeout_sec": 30.0, + "pending_tool_call_ids": ["c1"], + "terminal_trajectory_complete": False, + } + ] + ) + steps, report = ir_to_view_steps(trace) + assert steps[0]["kind"] == "timeout" + assert "text" not in steps[0] + assert steps[0]["timeout"] == { + "reason": "wall_clock_timeout", + "timeout_sec": 30.0, + "pending": ["c1"], + "complete": False, + } + assert not [r for r in report.records if r.field.startswith("steps[].text")] + + +def test_a_timeout_missing_everything_declares_each_sentinel(): + """Two of the four slots take null and represent their own absence; the + other two do not, and each substitute is declared on its own.""" + trace = _trace(_event(0, kind=EventKind.TIMEOUT, outcome=None, extensions={})) + steps, report = ir_to_view_steps(trace) + + assert steps[0]["timeout"] == { + "reason": "", + "timeout_sec": None, + "pending": [], + "complete": None, + } + assert _records_at(report, "events[0].outcome")[0].loss_class is ( + LossClass.SYNTHESIZED + ) + pending = _records_at(report, "events[0].extensions")[0] + assert pending.loss_class is LossClass.SYNTHESIZED + assert "not an observation that none were pending" in pending.detail + + +# --------------------------------------------------------------------------- +# Hue: membership, never inference +# --------------------------------------------------------------------------- + + +def test_a_real_acp_category_becomes_its_hue(): + steps, report = ir_to_view_steps( + _trace(_tool_event(0, name="read", name_semantics=ACP_KIND_SEMANTICS)) + ) + assert steps[0]["tool"]["hue"] == "read" + assert _records_at(report, "steps[].tool.hue")[0].loss_class is ( + LossClass.NORMALIZED + ) + + +@pytest.mark.parametrize( + ("name", "semantics"), + [ + ("read", "function_name"), + ("read_file", "gen_ai.tool.name"), + ("read", "span_name"), + ("read", None), + ], +) +def test_a_name_never_becomes_a_category(name, semantics): + """The laundering this edge exists to refuse. + + ``function_name="read"`` spells a real category exactly, and + ``gen_ai.tool.name="read_file"`` contains one; #1034's ``tool_hue`` returns + ``read`` for both. Neither is an observation that the tool *is* a read. + """ + steps, report = ir_to_view_steps( + _trace(_tool_event(0, name=name, name_semantics=semantics)) + ) + assert steps[0]["tool"]["hue"] == NEUTRAL_HUE + assert steps[0]["tool"]["kind"] == name + assert steps[0]["tool"]["name_semantics"] == semantics + + record = _records_at(report, "events[0].tool_call.name_semantics")[0] + assert record.loss_class is LossClass.SYNTHESIZED + + +def test_the_title_is_never_consulted_for_a_hue(): + """`tool_hue` reads ``kind + " " + title``; this edge reads neither string + for meaning. A title full of category words changes nothing.""" + steps, _ = ir_to_view_steps( + _trace( + _tool_event( + 0, + name="mystery_tool", + name_semantics="function_name", + title="bash grep read write search fetch", + ) + ) + ) + assert steps[0]["tool"]["hue"] == NEUTRAL_HUE + + +def test_a_category_outside_the_display_vocabulary_is_neutral(): + """`delete` and `move` are real ACP kinds with no hue; membership is tested + directly rather than approximated to the nearest colour.""" + steps, report = ir_to_view_steps( + _trace(_tool_event(0, name="delete", name_semantics=ACP_KIND_SEMANTICS)) + ) + assert steps[0]["tool"]["hue"] == NEUTRAL_HUE + assert ( + "outside the viewer's display vocabulary" + in _records_at(report, "events[0].tool_call.name_semantics")[0].detail + ) + + +def test_provenance_is_not_evidence_about_semantics(): + """Where a trace came from does not decide what its fields mean. + + An ACP-provenanced trace whose call is labelled a function name is still a + function name; the edge reads ``name_semantics`` and nothing else. + """ + event = _tool_event(0, name="read", name_semantics="function_name") + event = event.model_copy( + update={"provenance": Provenance(source_format="acp-capture-v1")} + ) + trace = CanonicalTrace( + provenance=Provenance(source_format="acp-capture-v1"), events=[event] + ) + steps, _ = ir_to_view_steps(trace) + assert steps[0]["tool"]["hue"] == NEUTRAL_HUE + + +def test_name_semantics_survives_the_boundary(): + """Our additive seventh key. Without it the viewer cannot tell the three + corpora apart, which is what makes substring inference look reasonable.""" + for semantics in (ACP_KIND_SEMANTICS, "function_name", "gen_ai.tool.name", None): + steps, _ = ir_to_view_steps( + _trace(_tool_event(0, name="x", name_semantics=semantics)) + ) + assert steps[0]["tool"]["name_semantics"] == semantics + + +# --------------------------------------------------------------------------- +# Absent is not observed-empty +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("field", "wire", "path"), + [ + ("call_id", "id", "events[0].tool_call.call_id"), + ("name", "kind", "events[0].tool_call.name"), + ("title", "title", "events[0].tool_call.title"), + ], +) +def test_an_absent_string_is_declared_and_an_observed_empty_one_is_not( + field, wire, path +): + """Both render as ``""``; only one of them is a substitution. + + The report is the only place the difference survives, so a shared branch + that treated ``None`` and ``""`` alike would erase a real distinction the + IR went to some trouble to keep. + """ + absent, _ = ir_to_view_steps(_trace(_tool_event(0, **{field: None}))) + observed, observed_report = ir_to_view_steps(_trace(_tool_event(0, **{field: ""}))) + + assert absent[0]["tool"][wire] == "" + assert observed[0]["tool"][wire] == "" + assert _records_at( + ir_to_view_steps(_trace(_tool_event(0, **{field: None})))[1], path + ) + assert not _records_at(observed_report, path) + + +def test_an_absent_status_is_declared(): + steps, report = ir_to_view_steps(_trace(_tool_event(0, status=None))) + assert steps[0]["tool"]["status"] == "" + record = _records_at(report, "events[0].tool_call.status")[0] + assert record.loss_class is LossClass.SYNTHESIZED + + +def test_a_status_enum_becomes_its_string(): + steps, report = ir_to_view_steps( + _trace(_tool_event(0, status=ToolStatus.IN_PROGRESS)) + ) + assert steps[0]["tool"]["status"] == "in_progress" + assert _records_at(report, "steps[].tool.status")[0].loss_class is ( + LossClass.NORMALIZED + ) + + +# --------------------------------------------------------------------------- +# Content +# --------------------------------------------------------------------------- + + +def test_a_block_with_no_text_is_declared_and_not_invented(): + """The viewer holds tool output as strings. A block that has none + contributes none — serializing its ``raw`` would put a string on the page + that no tool ever emitted.""" + call = ToolCall( + name="execute", + name_semantics=ACP_KIND_SEMANTICS, + content=[ + ContentBlock(kind=ContentBlockKind.TEXT, text="stdout"), + ContentBlock( + kind=ContentBlockKind.OPAQUE, raw={"type": "image", "data": "…"} + ), + ], + ) + steps, report = ir_to_view_steps( + _trace(_event(0, kind=EventKind.TOOL_CALL, tool_call=call)) + ) + assert steps[0]["tool"]["content"] == ["stdout"] + record = _records_at(report, "events[0].tool_call.content[1]")[0] + assert record.loss_class is LossClass.DROPPED + assert "does not invent one" in record.detail + + +def test_an_observed_empty_block_is_kept(): + """#1034 filters falsy strings out; an empty observation is still one.""" + call = ToolCall( + name="execute", + name_semantics=ACP_KIND_SEMANTICS, + content=[ContentBlock(kind=ContentBlockKind.TEXT, text="")], + ) + steps, _ = ir_to_view_steps( + _trace(_event(0, kind=EventKind.TOOL_CALL, tool_call=call)) + ) + assert steps[0]["tool"]["content"] == [""] + + +def test_arguments_are_declared_dropped_when_observed(): + """The OTel corpus is the only one that has real arguments, and the shape + has nowhere to put them.""" + steps, report = ir_to_view_steps( + _trace(_tool_event(0, arguments={"path": "/repo/README.md"})) + ) + assert "arguments" not in steps[0]["tool"] + assert _records_at(report, "events[0].tool_call.arguments")[0].loss_class is ( + LossClass.DROPPED + ) + + +# --------------------------------------------------------------------------- +# Timestamps +# --------------------------------------------------------------------------- + + +def test_timestamps_become_epoch_seconds_when_observed(): + start = datetime(2025, 8, 18, 6, 53, 20, tzinfo=UTC) + end = datetime(2025, 8, 18, 6, 53, 24, tzinfo=UTC) + trace = _trace(_event(0, started_at=start, finished_at=end)) + steps, report = ir_to_view_steps(trace) + assert steps[0]["t"] == start.timestamp() + assert steps[0]["dur"] == 4.0 + assert _records_at(report, "steps[].t")[0].loss_class is LossClass.NORMALIZED + + +def test_a_finish_before_a_start_is_not_a_duration(): + start = datetime(2025, 8, 18, 6, 53, 24, tzinfo=UTC) + end = datetime(2025, 8, 18, 6, 53, 20, tzinfo=UTC) + steps, _ = ir_to_view_steps(_trace(_event(0, started_at=start, finished_at=end))) + assert "dur" not in steps[0] + + +def test_a_tool_step_prefers_the_calls_own_window(): + outer = datetime(2025, 8, 18, 6, 0, 0, tzinfo=UTC) + inner = datetime(2025, 8, 18, 6, 0, 5, tzinfo=UTC) + event = _event( + 0, + kind=EventKind.TOOL_CALL, + started_at=outer, + tool_call=ToolCall( + name="execute", + name_semantics=ACP_KIND_SEMANTICS, + started_at=inner, + finished_at=inner, + ), + ) + steps, _ = ir_to_view_steps(_trace(event)) + assert steps[0]["t"] == inner.timestamp() + + +def test_no_timestamps_means_no_keys_and_no_declaration(): + steps, report = ir_to_view_steps(_trace(_event(0))) + assert "t" not in steps[0] and "dur" not in steps[0] + assert not _records_at(report, "steps[].t") + + +# --------------------------------------------------------------------------- +# Run metadata is not step metadata +# --------------------------------------------------------------------------- + + +def test_trace_level_fields_are_unsupported_here_not_dropped(): + """They are not losses of this edge. Saying `DROPPED` would claim the + viewer cannot show a model name, which is false — `meta` shows it, built + from artifacts a trace does not contain.""" + trace = CanonicalTrace( + provenance=_PROV, + session_id="s1", + usage=TraceUsage(input_tokens=10), + events=[_event(0, text="x")], + ) + steps, report = ir_to_view_steps(trace) + + flat = json.dumps(steps) + assert "s1" not in flat + for path in TRACE_LEVEL_PATHS: + record = _records_at(report, path)[0] + assert record.loss_class is LossClass.UNSUPPORTED + assert record.space is PathSpace.HUB + + +def test_per_event_usage_is_a_real_loss_and_says_so(): + trace = _trace(_event(0, usage=TraceUsage(input_tokens=1204), text="x")) + _, report = ir_to_view_steps(trace) + assert _records_at(report, "events[].usage")[0].loss_class is LossClass.DROPPED + + +def test_role_and_reasoning_segments_are_declared_when_present(): + trace = _trace( + _event(0, role=Role.AGENT, text="x"), + _event( + 1, + kind=EventKind.AGENT_REASONING, + reasoning="a\n\nb", + reasoning_segments=["a", "b"], + ), + ) + steps, report = ir_to_view_steps(trace) + assert steps[1]["text"] == "a\n\nb" + assert len(steps) == 2, "segments must not silently become extra steps" + assert _records_at(report, "events[].role")[0].loss_class is LossClass.DROPPED + segments = _records_at(report, "events[].reasoning_segments")[0] + assert segments.loss_class is LossClass.DROPPED + assert "same text twice" in segments.detail + + +# --------------------------------------------------------------------------- +# Report hygiene +# --------------------------------------------------------------------------- + + +def test_every_hub_record_resolves_in_the_trace(): + for events in (_rollout("h1"), _rollout("h2")): + trace = acp_events_to_ir(events) + _, report = ir_to_view_steps(trace) + canonical = trace.model_dump(mode="json") + for record in report.records: + if record.space is not PathSpace.HUB: + continue + if record.field.startswith("events[]"): + continue + assert resolve_ir_path(canonical, record.field)[0], record.field + + +def test_target_records_are_not_readable_as_ir_paths(): + trace = acp_events_to_ir(_rich_events()) + _, report = ir_to_view_steps(trace) + canonical = trace.model_dump(mode="json") + for record in report.records: + if record.space is PathSpace.TARGET: + assert not resolve_ir_path(canonical, record.field)[0], record.field + + +def test_the_conversion_leaves_the_input_alone(): + trace = acp_events_to_ir(_rich_events()) + before = trace.model_dump_json() + inbound = len(trace.losses.records) if trace.losses else 0 + + _, report = ir_to_view_steps(trace) + + assert trace.model_dump_json() == before + assert (len(trace.losses.records) if trace.losses else 0) == inbound + assert report is not trace.losses + assert report.direction == LOSS_DIRECTION + assert validate_trace(trace) == [] + + +# --------------------------------------------------------------------------- +# The three corpora +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name", ["h1", "h2"]) +def test_a_captured_acp_rollout_converts(name): + trace = acp_events_to_ir(_rollout(name)) + steps, report = ir_to_view_steps(trace) + + assert len(steps) == len(trace.events) + assert {step["kind"] for step in steps} <= set(VIEW_STEP_KINDS) + for step in steps: + if step["kind"] != "tool": + continue + # Every captured ACP call is a category, and every category these + # rollouts use is in the display vocabulary. + assert step["tool"]["name_semantics"] == ACP_KIND_SEMANTICS + assert report.records + + +def test_the_atif_fixture_converts_without_gaining_a_category(): + """ATIF puts an ACP kind in a `function_name` slot (§8.3). The value here + is literally ``execute`` — a hue member — and it still must not become one. + """ + document, _ = ir_to_atif(acp_events_to_ir(_rich_events())) + trace = atif_to_ir(document) + steps, _ = ir_to_view_steps(trace) + + tools = [step["tool"] for step in steps if step["kind"] == "tool"] + assert tools, "the fixture must contain a tool call" + for tool in tools: + assert tool["name_semantics"] == "function_name" + assert tool["hue"] == NEUTRAL_HUE + assert len(steps) == len(trace.events) + + +def test_the_otel_fixture_converts_and_keeps_what_it_has(): + traces, _ = otlp_json_to_ir(json.loads(PRODUCER_PAYLOAD_JSON)) + assert traces + for trace in traces: + steps, _ = ir_to_view_steps(trace) + assert len(steps) == len(trace.events) + + for step, event in zip(steps, trace.events, strict=True): + if event.source_type is not None: + assert step["type"] == event.source_type + + for step in steps: + if step["kind"] == "tool": + assert step["tool"]["name_semantics"] == "gen_ai.tool.name" + assert step["tool"]["hue"] == NEUTRAL_HUE + + # OTel is the only corpus with timestamps, and the only one whose + # unknown events carry the whole span — parentage included. + assert any("t" in step for step in steps) + unknown = [ + step + for step, event in zip(steps, trace.events, strict=True) + if event.kind in DIAGNOSTIC_KINDS + ] + for step in unknown: + body = json.loads(step["text"]) + assert body["extensions"]["otel"]["span"]["name"] == step["type"] diff --git a/tests/trajectories/test_trace_ir.py b/tests/trajectories/test_trace_ir.py index 05988b3d3..857dc8e47 100644 --- a/tests/trajectories/test_trace_ir.py +++ b/tests/trajectories/test_trace_ir.py @@ -554,6 +554,7 @@ def test_the_oracle_record_has_its_own_kind_and_role(): "_otlp_anyvalue", "ir_round_trip", "ir_conformance", + "ir_to_view", ) """The unwired modules of the proposal: the IR and its converters. From 4cffda28bc77214efd7d85db36b1450e514ed259 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Wed, 19 Aug 2026 06:50:08 +0200 Subject: [PATCH 19/23] viewer: factor the ACP page's cards out of the renderer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure extraction, no behaviour change: `_page`, `_prompt_block`, `_message_block`, `_thought_block` and `_result_block` come out of `_render_acp_events` so a second producer of blocks can emit the same page without copying its markup. The block builders take text the caller has already escaped and truncated. That is what keeps the extraction pure: the two prompt sites do those two things in opposite orders (`prompts.json` slices then escapes, an inline `user_message` escapes then slices), and a helper that picked one order would have changed the other site's output. The asymmetry is pinned by a test and left for its own commit. `tests/trajectories/test_viewer_primitives.py` freezes each card's markup and the assembled body of a whole ACP page, plus three properties of the current renderer that are facts rather than bugs: an `agent_timeout` and an unrecognized record reach no card, and a tool card carries kind/title/status but not the tool's output. Verified byte-identical on six pages rendered from the two captured rollouts of Slice A2 and from raw ACP and Codex session files — same SHA-256 before and after, ruff format included. --- src/benchflow/trajectories/viewer.py | 101 +++++++---- tests/trajectories/test_viewer_primitives.py | 180 +++++++++++++++++++ 2 files changed, 245 insertions(+), 36 deletions(-) create mode 100644 tests/trajectories/test_viewer_primitives.py diff --git a/src/benchflow/trajectories/viewer.py b/src/benchflow/trajectories/viewer.py index 733e103cb..5c70d81b5 100644 --- a/src/benchflow/trajectories/viewer.py +++ b/src/benchflow/trajectories/viewer.py @@ -482,6 +482,65 @@ def _load_result_json(rollout_dir: Path) -> dict: return parsed if isinstance(parsed, dict) else {} +# ── Page primitives ─────────────────────────────────────────────────── +# +# The markup below was inline inside _render_acp_events. It is factored out +# unchanged so a second producer of blocks can emit the same page without +# copying the markup — and so that changing a card means changing one place. +# +# Each block builder takes text that the caller has already escaped and +# truncated. That is deliberate: the two prompt sites differ in the order they +# do those two things (one slices then escapes, the other escapes then slices), +# and a helper that picked one would change the other's output. Escaping stays +# the caller's responsibility, at the site that knows what the value is. + + +def _page(title: str, blocks: list[str]) -> str: + """The shared page shell: one stylesheet, a wordmark header, the blocks.""" + return f""" +benchflow — {html.escape(title)} + +
{_WORDMARK_HTML}

{html.escape(title)}

+{"".join(blocks)} +""" + + +def _prompt_block(label: str, escaped_text: str) -> str: + """A prompt card. *label* and *escaped_text* are emitted as given.""" + return ( + f'
' + f'
{label}
' + f'
{escaped_text}
' + f"
" + ) + + +def _message_block(escaped_text: str) -> str: + """An agent message card.""" + return f'
{escaped_text}
' + + +def _thought_block(escaped_text: str) -> str: + """An agent reasoning card.""" + return f'
{escaped_text}
' + + +def _result_block(result_data: dict) -> str: + """The run summary card, read from result.json rather than from a trace.""" + agent = html.escape(result_data.get("agent_name", "?")) + rewards = result_data.get("rewards", {}) + n_tools = result_data.get("n_tool_calls", 0) + n_prompts = result_data.get("n_prompts", 0) + return ( + f'
' + f'
RESULT
' + f'
Agent: {agent} | Rewards: {rewards} | ' + f"Tool calls: {n_tools} | Prompts: {n_prompts}
" + f"
" + ) + + def _render_acp_events( title: str, events: list[dict], @@ -495,12 +554,7 @@ def _render_acp_events( has_inline_prompts = any(e.get("type") == "user_message" for e in events) if not has_inline_prompts: for i, prompt in enumerate(prompts or []): - blocks.append( - f'
' - f'
PROMPT {i + 1}
' - f'
{html.escape(prompt[:500])}
' - f"
" - ) + blocks.append(_prompt_block(f"PROMPT {i + 1}", html.escape(prompt[:500]))) # Show events prompt_counter = 0 @@ -509,12 +563,7 @@ def _render_acp_events( if etype == "user_message": prompt_counter += 1 text = html.escape(event.get("text", "")) - blocks.append( - f'
' - f'
PROMPT {prompt_counter}
' - f'
{text[:500]}
' - f"
" - ) + blocks.append(_prompt_block(f"PROMPT {prompt_counter}", text[:500])) elif etype == "tool_call": kind = html.escape(event.get("kind", "")) event_title = html.escape(event.get("title", "")) @@ -532,36 +581,16 @@ def _render_acp_events( ) elif etype == "agent_message": text = html.escape(event.get("text", "")) - blocks.append( - f'
{text[:500]}
' - ) + blocks.append(_message_block(text[:500])) elif etype == "agent_thought": text = html.escape(event.get("text", "")) - blocks.append( - f'
{text[:500]}
' - ) + blocks.append(_thought_block(text[:500])) # Result summary if result_data: - agent = html.escape(result_data.get("agent_name", "?")) - rewards = result_data.get("rewards", {}) - n_tools = result_data.get("n_tool_calls", 0) - n_prompts = result_data.get("n_prompts", 0) - blocks.append( - f'
' - f'
RESULT
' - f'
Agent: {agent} | Rewards: {rewards} | ' - f"Tool calls: {n_tools} | Prompts: {n_prompts}
" - f"
" - ) + blocks.append(_result_block(result_data)) - return f""" -benchflow — {html.escape(title)} - -
{_WORDMARK_HTML}

{html.escape(title)}

-{"".join(blocks)} -""" + return _page(title, blocks) def _join_with_divider(blocks: list[str]) -> str: diff --git a/tests/trajectories/test_viewer_primitives.py b/tests/trajectories/test_viewer_primitives.py new file mode 100644 index 000000000..74f2f5d61 --- /dev/null +++ b/tests/trajectories/test_viewer_primitives.py @@ -0,0 +1,180 @@ +"""The viewer's page primitives, frozen at the markup they were extracted with. + +Guards the pure refactor in PR #984 (Slice I): ``_page``, ``_prompt_block``, +``_message_block``, ``_thought_block`` and ``_result_block`` were lifted out of +``_render_acp_events`` so a second producer of blocks can emit the same page. +The refactor claimed byte-identical output; these tests are what keeps that +claim true, by pinning each card's markup and the assembled body of a whole +ACP page. + +They also pin one thing the refactor deliberately did **not** repair: the two +prompt sites truncate and escape in opposite orders. Fixing that is a change to +the legacy renderer's output and belongs to its own commit, not to an +extraction. +""" + +from __future__ import annotations + +import json + +from benchflow.trajectories.viewer import ( + _message_block, + _page, + _prompt_block, + _render_acp_events, + _result_block, + _thought_block, +) + +ACP_EVENTS = [ + {"type": "user_message", "text": "count the lines"}, + {"type": "agent_thought", "text": "reading the file"}, + { + "type": "tool_call", + "tool_call_id": "call_1", + "kind": "execute", + "title": "wc -l /etc/hostname", + "status": "completed", + "content": [{"type": "content", "content": {"type": "text", "text": "1"}}], + }, + {"type": "agent_message", "text": "the answer is 1"}, +] + +RESULT_DATA = { + "agent_name": "gemini-cli", + "rewards": {"reward": 1.0}, + "n_tool_calls": 1, + "n_prompts": 1, +} + +# The exact body the current renderer emits for ACP_EVENTS + RESULT_DATA — +# everything between the header and . +FROZEN_BODY = ( + '
' + '
PROMPT 1
' + '
count the lines
' + "
" + '
reading the file
' + '
' + '
execute wc -l /etc/hostname
' + '
completed
' + "
" + '
the answer is 1
' + '
' + '
RESULT
' + "
Agent: gemini-cli | Rewards: {'reward': 1.0} | " + "Tool calls: 1 | Prompts: 1
" + "
" +) + + +def _body(page: str) -> str: + """The block region of a rendered page.""" + return page.split("\n", 1)[1].rsplit("\n", 1)[0] + + +def test_the_acp_page_body_is_frozen(): + page = _render_acp_events("t", ACP_EVENTS, RESULT_DATA, None) + assert _body(page) == FROZEN_BODY + + +def test_each_card_is_the_markup_it_was_extracted_with(): + assert _prompt_block("PROMPT 1", "hi") == ( + '
' + '
PROMPT 1
' + '
hi
' + "
" + ) + assert ( + _message_block("hi") + == '
hi
' + ) + assert _thought_block("hi") == ( + '
hi
' + ) + assert _result_block(RESULT_DATA) == ( + '
' + '
RESULT
' + "
Agent: gemini-cli | Rewards: {'reward': 1.0} | " + "Tool calls: 1 | Prompts: 1
" + "
" + ) + + +def test_the_page_shell_carries_the_stylesheet_and_the_escaped_title(): + page = _page("a & b", ['
']) + assert page.startswith("") + assert "benchflow — a & b" in page + assert "

a & b

" in page + assert ".acc-bash" in page, "the one stylesheet ships with every page" + assert page.rstrip().endswith("") + assert '
' in page + + +def test_the_block_builders_emit_what_they_are_given(): + """Escaping is the caller's job, at the site that knows what the value is. + + Pinned because it is what lets the two prompt sites keep their different + truncation orders, and what lets a second producer truncate before + escaping without changing anything here. + """ + raw = "" + assert raw in _message_block(raw) + assert raw in _prompt_block("L", raw) + + +def test_the_two_prompt_sites_still_truncate_in_opposite_orders(): + """Documented, not repaired, by the extraction. + + ``prompts.json`` entries are sliced then escaped; inline ``user_message`` + text is escaped then sliced, so a long prompt can lose its last entity to + the 500-char cut. Changing either is a behaviour change and needs its own + commit. + """ + long_amp = "a" * 498 + "&&&" + + from_prompts_json = _render_acp_events("t", [], None, [long_amp]) + assert "&" in from_prompts_json + + inline = _render_acp_events( + "t", [{"type": "user_message", "text": long_amp}], None, None + ) + body = _body(inline) + assert "&am" in body or "&a" in body or "&" in body, body[-80:] + + +def test_unknown_event_types_and_timeouts_are_absent_from_the_legacy_page(): + """The current renderer's four branches, stated as a fact rather than a bug. + + Slice I's adapter exists because of this line: an ``agent_timeout`` and an + unrecognized record reach no card here. Pinned so the day someone adds a + branch, the adapter's reason for existing is revisited too. + """ + events = [ + {"type": "agent_timeout", "reason": "wall_clock_timeout", "timeout_sec": 90.0}, + {"type": "reward", "value": 1.0}, + ] + body = _body(_render_acp_events("t", events, None, None)) + assert body == "" + assert "timeout" not in _render_acp_events("t", events, None, None).lower() + + +def test_tool_output_text_is_absent_from_the_legacy_tool_card(): + """Also a fact, not a bug: the tool card carries kind, title and status.""" + marker = "MARKER-TOOL-OUTPUT" + events = [ + { + "type": "tool_call", + "tool_call_id": "c1", + "kind": "execute", + "title": "echo", + "status": "completed", + "content": [ + {"type": "content", "content": {"type": "text", "text": marker}} + ], + } + ] + assert marker not in _render_acp_events("t", events, None, None) + assert json.dumps(events[0]["content"]) not in _render_acp_events( + "t", events, None, None + ) From 050203417a344e8fe86f6f0dafa1298bb881bd91 Mon Sep 17 00:00:00 2001 From: Galius5136 Date: Wed, 19 Aug 2026 06:57:53 +0200 Subject: [PATCH 20/23] trajectories: render canonical trace steps through the current viewer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ir_to_view_html` closes the chain `ir_to_view` opened: a viewer step list becomes one HTML page, built from the card builders `viewer.py` already renders its ACP page with. The dependency runs one way — this module imports the viewer, the viewer imports no part of the IR family — so what crosses the boundary is a plain step document and the renderer stays a renderer. It is written against the viewer on main. #1034 stays a design reference for the step vocabulary and nothing else: no code from that branch is imported, vendored or fetched, and this edge works whether or not it lands. It does not rebuild steps into ACP capture events. The IR holds records ACP has no type for, a status it cannot spell, and tool names whose semantics are the point; forging capture events would launder all three back into the assumption the hub exists to remove. Measured against `_render_acp_events` on the two captured rollouts: H2's four events render three cards there and the word "timeout" appears nowhere, an unrecognized record reaches no card, and a tool card carries kind/title/status but not the tool's output. Those are that renderer's four branches, not bugs; this edge has six step kinds to place, places them, and declares the difference. `test_viewer_primitives.py` pins the legacy behaviour as a fact so the day a branch is added, this edge is revisited with it. Classification is a table over the eight display hues, never a substring: `viewer._tool_accent_class` is not imported and an AST test asserts the name never appears here. The same `execute` gets `acc-bash` as an ACP kind and stays neutral as an ATIF `function_name` in the same run; `read_file` from OTel stays neutral too. `think` resolves to the neutral accent because the stylesheet has none for it, and that one is declared DROPPED rather than hidden. `name_semantics` rides in the metrics line and in `data-name-semantics`; a step with no typed slot is labelled `Canonical IR representation` so the page never passes a reconstructed event off as a source record; cuts carry an explicit marker and a NORMALIZED record. `render_trace` returns both reports without merging them, and this edge never addresses the hub — it has the IR on neither side. Unwired: nothing under src/benchflow calls it. `python -m benchflow.trajectories.ir_to_view_html [out.html]` is how a person looks at a page. Docs in §8.15, family list and status in §8.7. --- docs/trace-interop.md | 146 ++++- src/benchflow/trajectories/ir_to_view_html.py | 547 ++++++++++++++++++ tests/trajectories/test_ir_to_view_html.py | 475 +++++++++++++++ tests/trajectories/test_trace_ir.py | 1 + 4 files changed, 1166 insertions(+), 3 deletions(-) create mode 100644 src/benchflow/trajectories/ir_to_view_html.py create mode 100644 tests/trajectories/test_ir_to_view_html.py diff --git a/docs/trace-interop.md b/docs/trace-interop.md index 5f9c08419..cdbddbdff 100644 --- a/docs/trace-interop.md +++ b/docs/trace-interop.md @@ -1377,7 +1377,9 @@ here. - **Implemented:** the IR types, the loss model with its path spaces, the invariants, the validation suite, this section, six converters — `ACP → IR`, `IR → ATIF`, `ATIF → IR`, `OTLP/JSON → IR`, `IR → ACP capture events`, - `IR → viewer trace steps` (§8.14) — each with its loss report; the round-trip + `IR → viewer trace steps` (§8.14) — each with its loss report; the page edge + that renders those steps through the current viewer's own cards (§8.15); the + round-trip measurement over the ATIF pair (§8.10), and the ACP pair's round-trip evidence in §8.3 and its test suite; and the loss-bounded conformance gate that joins the two halves (§8.13). There is no @@ -1396,11 +1398,16 @@ here. The isolation property is unchanged in substance and restated at a new boundary: `ir.py`, `ir_from_acp.py`, `ir_to_atif.py`, `ir_from_atif.py`, `ir_from_otel.py`, `ir_to_acp.py`, `_otlp_anyvalue.py`, `ir_round_trip.py`, -`ir_conformance.py` and `ir_to_view.py` form a closed family that may import -each other, and +`ir_conformance.py`, `ir_to_view.py` and `ir_to_view_html.py` form a closed +family that may import each other, and `test_only_the_ir_family_imports_the_ir` asserts that nothing else in `src/benchflow` imports any of them. +`ir_to_view_html` is the one member that imports a runtime module — it renders +through `viewer`'s card builders rather than copying their markup — and the +dependency runs one way only: no part of `viewer` imports any part of the +family. + Every converter reads or writes its own format **as data**, never through the module that already handles it: `ir_to_atif` does not import `export_atif` and pins the shared schema version by test instead, and `ir_from_otel` imports no @@ -2257,3 +2264,136 @@ instead of `failed` as a second way a mutation can fail to mean what it claims. - **`ORACLE` stays `unknown`.** It keeps its identity in `type`, but it will remain an untyped step until the viewer contract gains a member for it — which is a change to propose upstream, not one to assume. + +### 8.15 `viewer trace steps → a page the current viewer renders` + +[`ir_to_view_html.py`](../src/benchflow/trajectories/ir_to_view_html.py) closes +the chain §8.14 opened. It takes the step list and produces one HTML page, +using the card builders `benchflow/trajectories/viewer.py` already renders its +ACP page with: + +``` + → ir_from_* → CanonicalTrace → ir_to_view_steps → ir_to_view_html → page +``` + +It is written against the viewer **on `main`**, not against the reviewer-grade +package proposed in `benchflow-ai/benchflow#1034`. That branch is a design +reference for the step vocabulary (§8.14) and nothing more: no code is +imported, vendored or fetched from it, and this edge composes with the current +renderer whether or not that PR ever lands. + +#### The direction of the dependency + +One way, and it is the point. `ir_to_view_html` imports `viewer`; `viewer` +imports no part of the IR family. What crosses the boundary is a step list — a +plain document of six kinds and named keys — so the renderer stays a renderer +and the hub stays format-neutral. `test_only_the_ir_family_imports_the_ir` +still passes unchanged: this module joins the closed family rather than opening +it. + +#### What it refuses to do + +It does not rebuild steps into ACP capture events and hand them to +`_render_acp_events`. That would be a lie in the middle of the chain: the IR +holds records ACP has no type for (`oracle`, and anything `unknown`), a status +ACP cannot spell, and a tool name whose *semantics* are the whole reason the +hub exists. Forging capture events would launder all three straight back into +the assumption the conversion was built to stop. + +#### What the current page does not show, measured + +Against `viewer._render_acp_events` on the two captured rollouts of §8.3: + +| | legacy ACP page | this edge | +|---|---|---| +| H1 (5 events) | 5 cards | 5 cards | +| H2 (4 events, ends in a real timeout) | **3 cards**, and the word "timeout" appears nowhere in the page | 4 cards, one of them a typed timeout | +| H2 + one unrecognized record | **3 cards** | 5 cards, the last one a labelled diagnostic | +| tool output | not on the page at all | shown when the block carried text | +| ATIF-only rollout directory | `

No trajectory files found

` (32 bytes) | 5 cards | + +None of that is a bug being fixed. `_render_acp_events` has four branches — +`user_message`, `tool_call`, `agent_message`, `agent_thought` — and an +`agent_timeout` or an unknown record simply reaches none of them. This edge has +six step kinds to place, so it places them, and declares the difference instead +of presenting it as a repair. Three tests in +`tests/trajectories/test_viewer_primitives.py` pin the legacy behaviour as a +fact, so the day a branch is added, this edge's reason for existing is +revisited with it. + +#### Classification: a table, not a substring + +`viewer._tool_accent_class` scans the tool kind for needles and, failing that, +the human title. Executed on `main`: `read_file → acc-read`, +`bash -lc 'ls -la' → acc-bash`. It is not imported here, and an AST test +asserts the name never appears in this module. + +Instead the hue that `ir_to_view` already decided — by membership, only when +the source said the string *is* a category — is mapped through `HUE_ACCENT`, +a table over the eight display hues. Two of them resolve to the neutral accent +for opposite reasons, and only one of the two is a loss: + +- `other` — no category was observed. Nothing is lost; neutral is the claim. +- `think` — a real ACP kind the current stylesheet has no accent for. Declared + `DROPPED` at `steps[i].tool.hue`, with the note that the category is still + legible because it is the card's own label. Adding an accent for it is a + viewer decision, not one to take inside a converter. + +The same run, converted two ways, is the whole slice on one screen: H1's +`execute` arrives as an ACP `kind` and gets `acc-bash`; the identical string in +that run's `trainer/atif.json` arrives as a `function_name` and stays neutral. +OTel's `read_file` and `write_file` stay neutral for the same reason. + +#### `name_semantics` on the page + +The page has no payload, so preserving `name_semantics` means rendering it. It +appears twice per tool card: in the metrics line (`completed · +name_semantics: acp_kind`), so a reader sees what kind of name they are looking +at, and as `data-name-semantics`, so a browser-level check can assert it +without parsing prose. `data-hue`, `data-tool-id` and `data-source-type` ride +along the same way. + +#### Diagnostics say which document they are showing + +A step with no typed slot is rendered under the label **`Canonical IR +representation`**, followed by its source type and the serialized event. The +label is not decoration: §8.14 emits the body of those steps as a +serialization of the *canonical IR event*, because the IR holds no source +record — and the legacy path's unknown branch, when it grows one, will be +showing the raw capture entry instead. A page that displayed ours under the +source's own type would claim a document that does not exist. + +#### Cuts are announced + +The legacy card cuts message text at 500 characters silently, and escapes +before cutting — so a long prompt can lose an entity to the knife. This edge +keeps the same 500 for message-shaped text (2000 for tool output, 4000 for a +diagnostic body), cuts **before** escaping, appends +`… [truncated, N more characters]` to the page, and records the cut as +`NORMALIZED`. The legacy asymmetry is pinned by a test and left alone: changing +it is a behaviour change to that renderer and belongs to its own commit. + +#### Two reports, not one + +`render_trace` returns the page and **both** reports — `ir->view` from §8.14 and +`view->html` from here — without merging them. They address different +documents, and a merged report would have to name a space neither edge has. +This is also the one edge in the family with the IR on *neither* side, so +`PathSpace.HUB` is never used: `SOURCE` addresses the step list it was given, +`TARGET` the page it produced, and no record of this edge can be joined to a +hub path by mistake. + +#### Status + +- **Unwired in this section.** Nothing under `src/benchflow` calls it. The way + to look at a page is + `python -m benchflow.trajectories.ir_to_view_html [out.html]`, + which is an entry point rather than a run path. +- **The viewer refactor that precedes it is pure.** `_page`, `_prompt_block`, + `_message_block`, `_thought_block` and `_result_block` were lifted out of + `_render_acp_events` unchanged; six pages rendered from the captured + rollouts and from raw ACP and Codex session files have the same SHA-256 + before and after. +- **Not claimed:** no human end-to-end verification of the rendered pages, and + no browser-level check of any kind. The pages have been produced and read as + text; that is not the same as looking at them. diff --git a/src/benchflow/trajectories/ir_to_view_html.py b/src/benchflow/trajectories/ir_to_view_html.py new file mode 100644 index 000000000..00c85793c --- /dev/null +++ b/src/benchflow/trajectories/ir_to_view_html.py @@ -0,0 +1,547 @@ +"""Viewer trace steps → a page the current viewer's primitives render. + +> **PROVISIONAL.** Part of the IR family (`docs/trace-interop.md` §8.7). It is +> the family's only module that imports a runtime module, and nothing in a run +> path imports *it* — see §8.15. + +`ir_to_view.ir_to_view_steps` projects a canonical trace onto the viewer's step +vocabulary. This module turns that step list into one HTML page, using the card +builders `benchflow.trajectories.viewer` already emits its ACP page with. The +chain is:: + + -> ir_from_* -> CanonicalTrace -> ir_to_view_steps -> here -> page + +and the direction of the dependency is one-way: this module imports the viewer, +the viewer never imports the IR. A step list is a plain document — six kinds, +named keys — so the renderer stays a renderer. + +## What it is not + +It does not rebuild the steps into ACP capture events and hand them to +`_render_acp_events`. That would be a lie in the middle of the chain: the IR +holds records ACP has no type for (`oracle`, anything `unknown`), a status ACP +cannot spell, and a tool name whose *semantics* are the whole point. Forging +capture events would launder all three back into the shape the conversion +exists to stop assuming. + +## What the current page gains, and why those are additions rather than fixes + +Measured against `viewer._render_acp_events` on the two captured rollouts: + +- an `agent_timeout` reaches **no card** there — H2's four events render three + blocks and the word "timeout" appears nowhere on the page; +- an unrecognized record reaches no card either; +- a tool card carries kind, title and status, and **not** the tool's output. + +None of that is broken: those are the four branches that renderer has. This +edge has six step kinds to place, so it places them, and declares the +difference rather than presenting it as a repair. + +## The classification rule + +The hue arrives already decided by `ir_to_view`, which emits a category only +when the source said the string *is* one. Here it is mapped to an accent class +by **membership in a table**, never by inspecting the string. +`viewer._tool_accent_class` — which scans the kind and then the title for +needles, so a `function_name` of `read_file` becomes the read accent — is +deliberately not imported, and a test pins that it is not. +""" + +from __future__ import annotations + +import html +import json +from typing import Any, NamedTuple + +from benchflow.trajectories import viewer +from benchflow.trajectories.ir import CanonicalTrace, LossClass, LossReport, PathSpace +from benchflow.trajectories.ir_to_view import ( + NEUTRAL_HUE, + VIEW_TOOL_HUES, + ir_to_view_steps, +) + +LOSS_DIRECTION = "view->html" +"""The one edge in the family with the IR on **neither** side. + +:class:`~benchflow.trajectories.ir.PathSpace` is defined relative to the hub, so +its two non-hub members are read here as: ``SOURCE`` addresses the step list +this module was given, ``TARGET`` addresses the page it produced. ``HUB`` is +never used, so no record of this edge can be joined to a hub path by mistake. +""" + +HUE_ACCENT: dict[str, str] = { + "execute": "acc-bash", + "read": "acc-read", + "edit": "acc-edit", + "fetch": "acc-web", + "search": "acc-web", + "skill": "acc-agent", + "think": "acc-other", + "other": "acc-other", +} +"""Display hue → the accent class the current stylesheet defines. + +A table, checked by membership. Two hues map to ``acc-other`` for opposite +reasons: ``other`` because no category was observed, and ``think`` because the +stylesheet has no accent for one — the second is a real loss and is declared as +such. Adding an accent for ``think`` is a viewer decision, not ours to take +inside a converter. +""" + +NEUTRAL_ACCENT = "acc-other" + +DIAGNOSTIC_LABEL = "Canonical IR representation" +"""What the card for an untypeable event says it is showing. + +`ir_to_view` renders those events as a serialization of the **canonical IR +event**; the IR holds no source record to show instead. A page that displayed +it under the source's own type would be claiming a document it does not have. +""" + +TEXT_PREVIEW = 500 +"""Message/thought/prompt cut, the same one the legacy ACP card uses.""" + +TOOL_OUTPUT_PREVIEW = 2000 +DIAGNOSTIC_PREVIEW = 4000 + + +def _truncate(text: str, limit: int, where: str, losses: LossReport) -> str: + """Cut *text* to *limit* with a visible marker and a declared record. + + The cut happens **before** escaping — a marker is appended to the plain + string, then the whole thing is escaped once — so it can never bisect an + entity. Silence is the thing being avoided here: a page that shortens a + tool's output without saying so is a page that lies about the run. + """ + if len(text) <= limit: + return text + dropped = len(text) - limit + losses.add( + where, + LossClass.NORMALIZED, + f"shown to {limit} characters of {len(text)}; the page carries an " + f"explicit marker for the {dropped} it does not show", + space=PathSpace.SOURCE, + ) + return f"{text[:limit]}\n… [truncated, {dropped} more characters]" + + +def _esc(text: str, limit: int, where: str, losses: LossReport) -> str: + return html.escape(_truncate(text, limit, where, losses)) + + +def _accent(hue: Any, where: str, losses: LossReport) -> str: + """The accent class for an already-decided hue. No string is inspected.""" + if hue not in VIEW_TOOL_HUES: + losses.add( + where, + LossClass.DROPPED, + f"{hue!r} is outside the display vocabulary this edge was written " + f"against; the neutral accent was used, which asserts no category", + space=PathSpace.SOURCE, + ) + return NEUTRAL_ACCENT + accent = HUE_ACCENT[hue] + if hue != NEUTRAL_HUE and accent == NEUTRAL_ACCENT: + losses.add( + where, + LossClass.DROPPED, + f"the observed category {hue!r} has no accent in the viewer's " + f"stylesheet, so the strip stays neutral; the category itself is " + f"still legible — it is the card's own label", + space=PathSpace.SOURCE, + ) + return accent + + +def _data_attrs(pairs: dict[str, Any]) -> str: + """``data-*`` attributes for the values a card has no visible slot for.""" + out = [] + for name, value in pairs.items(): + if value is None: + continue + out.append(f' data-{name}="{html.escape(str(value), quote=True)}"') + return "".join(out) + + +def _tool_card(step: dict[str, Any], where: str, losses: LossReport) -> str: + """A tool call: the legacy card's three fields, plus what it has no slot for. + + ``name_semantics`` rides in the metrics line *and* in a ``data-`` attribute: + the first so a reader sees whether ``execute`` was a category or a function + name, the second so a browser check can assert it without parsing prose. + """ + tool = step["tool"] + accent = _accent(tool.get("hue"), f"{where}.tool.hue", losses) + semantics = tool.get("name_semantics") + + kind = html.escape(str(tool.get("kind", ""))) + title = html.escape(str(tool.get("title", ""))) + status = html.escape(str(tool.get("status", ""))) + meta = ( + f"{status or '—'} · name_semantics: {html.escape(str(semantics or 'unknown'))}" + ) + + if tool.get("id"): + losses.add( + f"{where}.tool.id", + LossClass.DROPPED, + "the tool call id is carried in a data- attribute; the card has no " + "visible slot for it", + space=PathSpace.SOURCE, + ) + + body = "" + content = tool.get("content") or [] + if content: + joined = "\n\n".join(str(item) for item in content) + body = ( + f'
' + f"{_esc(joined, TOOL_OUTPUT_PREVIEW, f'{where}.tool.content', losses)}" + f"
" + ) + + attrs = _data_attrs( + { + "name-semantics": semantics, + "hue": tool.get("hue"), + "tool-id": tool.get("id") or None, + "source-type": step.get("type"), + } + ) + return ( + f'
' + f'
{kind} {title}
' + f'
{meta}
' + f"{body}" + f"
" + ) + + +def _timeout_card(step: dict[str, Any], where: str, losses: LossReport) -> str: + """A typed timeout — the kind the current renderer has no branch for.""" + info = step["timeout"] + reason = html.escape(str(info.get("reason") or "")) + pending = info.get("pending") or [] + details = [f"timeout_sec: {html.escape(str(info.get('timeout_sec')))}"] + details.append(f"pending tool calls: {html.escape(str(len(pending)))}") + if pending: + details.append(html.escape(", ".join(str(p) for p in pending))) + details.append( + f"terminal trajectory complete: {html.escape(str(info.get('complete')))}" + ) + attrs = _data_attrs({"step-kind": "timeout", "source-type": step.get("type")}) + return ( + f'
' + f'
agent timeout {reason}
' + f'
{" · ".join(details)}
' + f"
" + ) + + +def _diagnostic_card(step: dict[str, Any], where: str, losses: LossReport) -> str: + """An event with no typed slot, labelled for what its body actually is.""" + source_type = html.escape(str(step.get("type") or "no source type")) + text = str(step.get("text") or "") + body = "" + if text: + body = ( + f'
' + f"{_esc(text, DIAGNOSTIC_PREVIEW, f'{where}.text', losses)}" + f"
" + ) + attrs = _data_attrs( + { + "step-kind": "unknown", + "source-type": step.get("type"), + "diagnostic": "canonical-ir", + } + ) + return ( + f'
' + f'
{html.escape(DIAGNOSTIC_LABEL)} ' + f"{source_type}
" + f"{body}" + f"
" + ) + + +def _provenance_card(step_count: int, source_format: str | None) -> str: + """One line saying which renderer produced the page below it.""" + source = html.escape(source_format or "canonical trace") + return ( + f'
' + f'
Rendered from the canonical Trace IR ' + f"({source}) — {step_count} steps, one per canonical event.
" + f"
" + ) + + +def view_steps_to_html( + title: str, + steps: list[dict[str, Any]], + result_data: dict[str, Any] | None = None, + *, + prompts: list[str] | None = None, + source_format: str | None = None, +) -> tuple[str, LossReport]: + """Render a viewer step list as one page, with what the page cannot hold. + + *prompts* follows the legacy renderer's own rule — the run's ``prompts.json`` + is shown only when the step list carries no prompt of its own, because + showing both prints the same text twice. Prompt ordinals are a property of + the run, not of the trace, which is why they are assigned here. + + *result_data* is ``result.json`` and reaches the page through the viewer's + own summary card, unchanged. + """ + losses = LossReport(direction=LOSS_DIRECTION) + blocks: list[str] = [_provenance_card(len(steps), source_format)] + + prompt_counter = 0 + if not any(step.get("kind") == "prompt" for step in steps): + for position, prompt in enumerate(prompts or []): + prompt_counter += 1 + blocks.append( + viewer._prompt_block( + f"PROMPT {prompt_counter}", + _esc(str(prompt), TEXT_PREVIEW, f"prompts[{position}]", losses), + ) + ) + + for index, step in enumerate(steps): + where = f"steps[{index}]" + kind = step.get("kind") + + if kind == "prompt": + prompt_counter += 1 + blocks.append( + viewer._prompt_block( + f"PROMPT {prompt_counter}", + _esc( + str(step.get("text") or ""), + TEXT_PREVIEW, + f"{where}.text", + losses, + ), + ) + ) + elif kind == "message": + blocks.append( + viewer._message_block( + _esc( + str(step.get("text") or ""), + TEXT_PREVIEW, + f"{where}.text", + losses, + ) + ) + ) + elif kind == "thought": + blocks.append( + viewer._thought_block( + _esc( + str(step.get("text") or ""), + TEXT_PREVIEW, + f"{where}.text", + losses, + ) + ) + ) + elif kind == "tool" and isinstance(step.get("tool"), dict): + blocks.append(_tool_card(step, where, losses)) + elif kind == "timeout" and isinstance(step.get("timeout"), dict): + blocks.append(_timeout_card(step, where, losses)) + else: + # Everything else — 'unknown', and a typed kind whose payload is + # missing — becomes a labelled diagnostic. No step is skipped. + blocks.append(_diagnostic_card(step, where, losses)) + if kind not in (None, "unknown"): + losses.add( + where, + LossClass.NORMALIZED, + f"a {kind!r} step carrying no {kind!r} payload was rendered " + "as a diagnostic rather than dropped", + space=PathSpace.SOURCE, + ) + + if kind not in ("tool", "timeout", "unknown") and step.get("type") is not None: + losses.add( + f"{where}.type", + LossClass.DROPPED, + "the source record's own type reaches no field on a prompt, " + "message or thought card; those cards are the viewer's and are " + "emitted unchanged", + space=PathSpace.SOURCE, + ) + + _declare_page_level(steps, losses) + + if result_data: + blocks.append(viewer._result_block(result_data)) + + return viewer._page(title, blocks), losses + + +def _declare_page_level(steps: list[dict[str, Any]], losses: LossReport) -> None: + """Properties of the page, declared once instead of per step.""" + if any("t" in step for step in steps): + losses.add( + "steps[].t", + LossClass.DROPPED, + "observed timestamps reach no field: this page has no timeline", + space=PathSpace.SOURCE, + ) + if any("dur" in step for step in steps): + losses.add( + "steps[].dur", + LossClass.DROPPED, + "observed tool durations reach no field on this page", + space=PathSpace.SOURCE, + ) + if steps: + losses.add( + "steps[].i", + LossClass.DROPPED, + "step positions are not printed; the page keeps them as document " + "order and nothing renumbers", + space=PathSpace.SOURCE, + ) + losses.add( + "provenance", + LossClass.SYNTHESIZED, + "the page opens with a line naming the renderer that produced it; " + "no step carries that text", + space=PathSpace.TARGET, + ) + + +class RenderedTrace(NamedTuple): + """A page plus the two reports that describe how it was reached. + + The reports are **not** merged. They address different documents — one the + canonical trace, the other the step list — and a single report claiming + both would have to name a space neither edge has. + """ + + html: str + steps_losses: LossReport + page_losses: LossReport + + +def render_trace( + title: str, + trace: CanonicalTrace, + result_data: dict[str, Any] | None = None, + *, + prompts: list[str] | None = None, +) -> RenderedTrace: + """Render a canonical trace: `ir_to_view_steps`, then this page. + + The trace is not modified and its own inbound report is left alone. + """ + steps, steps_losses = ir_to_view_steps(trace) + page, page_losses = view_steps_to_html( + title, + steps, + result_data, + prompts=prompts, + source_format=trace.provenance.source_format, + ) + return RenderedTrace(page, steps_losses, page_losses) + + +# --------------------------------------------------------------------------- +# A way to look at one, without wiring anything into a run path +# --------------------------------------------------------------------------- + + +def _load(path: Any) -> Any: + from pathlib import Path + + return json.loads(Path(path).read_text(encoding="utf-8", errors="replace")) + + +def _main(argv: list[str]) -> int: + """``python -m benchflow.trajectories.ir_to_view_html [out.html]``. + + *path* is a rollout directory (its ACP capture, else its ``trainer/atif.json``), + an ACP ``.jsonl`` session file, an ATIF document, or an OTLP/JSON export. + Nothing in BenchFlow calls this; it exists so a person can look at the page. + """ + from pathlib import Path + + from benchflow.trajectories.export_atif import ROLLOUT_ATIF_RELPATH + from benchflow.trajectories.ir_from_acp import acp_events_to_ir + from benchflow.trajectories.ir_from_atif import atif_to_ir + from benchflow.trajectories.ir_from_otel import otlp_json_to_ir + + if not argv: + print( + "usage: python -m benchflow.trajectories.ir_to_view_html " + " [out.html]" + ) + return 2 + + path = Path(argv[0]) + out = Path(argv[1]) if len(argv) > 1 else path.with_suffix(".ir.html") + result_data: dict[str, Any] = {} + prompts: list[str] | None = None + traces: list[CanonicalTrace] = [] + + if path.is_dir(): + result_path = path / "result.json" + if result_path.exists(): + loaded = _load(result_path) + result_data = loaded if isinstance(loaded, dict) else {} + prompts_path = path / "prompts.json" + if prompts_path.exists(): + loaded = _load(prompts_path) + prompts = loaded if isinstance(loaded, list) else None + acp = path / "trajectory" / "acp_trajectory.jsonl" + atif = path / ROLLOUT_ATIF_RELPATH + if acp.exists(): + traces = [ + acp_events_to_ir(viewer._parse_jsonl(acp.read_text(encoding="utf-8"))) + ] + elif atif.exists(): + traces = [atif_to_ir(_load(atif))] + else: + print(f"no ACP capture and no {ROLLOUT_ATIF_RELPATH} in {path}") + return 1 + elif path.suffix == ".jsonl": + traces = [ + acp_events_to_ir(viewer._parse_jsonl(path.read_text(encoding="utf-8"))) + ] + else: + document = _load(path) + if isinstance(document, dict) and "resourceSpans" in document: + traces, _ = otlp_json_to_ir(document) + else: + traces = [atif_to_ir(document)] + + if not traces: + print(f"no trace could be read from {path}") + return 1 + + written = [] + for index, trace in enumerate(traces): + target = out if index == 0 else out.with_name(f"{out.stem}-{index}{out.suffix}") + rendered = render_trace(path.name, trace, result_data, prompts=prompts) + target.write_text(rendered.html, encoding="utf-8", newline="") + written.append(target) + print( + f"{target} ({len(trace.events)} events, " + f"{len(rendered.steps_losses.records)} ir->view records, " + f"{len(rendered.page_losses.records)} view->html records)" + ) + if len(written) > 1: + print(f"{len(written)} pages: the payload carried {len(written)} trace ids") + return 0 + + +if __name__ == "__main__": # pragma: no cover - manual entry point + import sys + + raise SystemExit(_main(sys.argv[1:])) diff --git a/tests/trajectories/test_ir_to_view_html.py b/tests/trajectories/test_ir_to_view_html.py new file mode 100644 index 000000000..00fb828e8 --- /dev/null +++ b/tests/trajectories/test_ir_to_view_html.py @@ -0,0 +1,475 @@ +"""`viewer trace steps → page`: what reaches the page, and what refuses to. + +The positive half is easy to state — six step kinds, six cards, nothing +skipped. The half worth having is negative: a tool name never acquires a +category from its spelling, a diagnostic body never passes for a source +record, a cut is never silent, and the legacy renderer keeps emitting exactly +what it emitted before (pinned next door in +``test_viewer_primitives.py``). +""" + +from __future__ import annotations + +import ast +import json +import pathlib + +import pytest + +from benchflow.trajectories import viewer +from benchflow.trajectories.ir import ( + CanonicalTrace, + ContentBlock, + ContentBlockKind, + EventKind, + LossClass, + PathSpace, + Provenance, + ToolCall, + ToolStatus, + TraceEvent, + validate_trace, +) +from benchflow.trajectories.ir_from_acp import acp_events_to_ir +from benchflow.trajectories.ir_from_atif import atif_to_ir +from benchflow.trajectories.ir_from_otel import otlp_json_to_ir +from benchflow.trajectories.ir_to_atif import ir_to_atif +from benchflow.trajectories.ir_to_view import ( + ACP_KIND_SEMANTICS, + VIEW_TOOL_HUES, + ir_to_view_steps, +) +from benchflow.trajectories.ir_to_view_html import ( + DIAGNOSTIC_LABEL, + HUE_ACCENT, + LOSS_DIRECTION, + NEUTRAL_ACCENT, + TOOL_OUTPUT_PREVIEW, + render_trace, + view_steps_to_html, +) +from tests.trajectories.test_atif_preservation import _rich_events +from tests.trajectories.test_ir_from_otel import PRODUCER_PAYLOAD_JSON + +EVIDENCE = pathlib.Path(__file__).resolve().parents[2].parent / "e2e-a2" / "evidence" + +_PROV = Provenance(source_format="hand-built") + +H1_EVENTS = [ + {"type": "user_message", "text": "count the lines in /etc/hostname"}, + {"type": "agent_thought", "text": "[cwd /app] reading the file"}, + { + "type": "tool_call", + "tool_call_id": "call_1317590", + "kind": "execute", + "title": "wc -l < /etc/hostname", + "status": "completed", + "content": [ + {"type": "content", "content": {"type": "text", "text": "TOOL-OUTPUT-1"}} + ], + }, + { + "type": "tool_call", + "tool_call_id": "call_832088", + "kind": "read", + "title": "answer.txt", + "status": "completed", + "content": [{"type": "content", "content": {"type": "text", "text": "1\n"}}], + }, + {"type": "agent_message", "text": "answer.txt holds 1"}, +] + +H2_EVENTS = [ + {"type": "user_message", "text": "run both commands"}, + { + "type": "tool_call", + "tool_call_id": "call_1131058", + "kind": "think", + "title": "Update topic", + "status": "completed", + "content": [], + }, + {"type": "agent_thought", "text": "[cwd /app] sleeping"}, + { + "type": "agent_timeout", + "reason": "wall_clock_timeout", + "timeout_sec": 90.0, + "pending_tool_call_ids": ["call_1131058"], + "terminal_trajectory_complete": True, + }, +] + + +def _trace(*events: TraceEvent) -> CanonicalTrace: + return CanonicalTrace(provenance=_PROV, events=list(events)) + + +def _tool_event(index: int = 0, **call) -> TraceEvent: + call.setdefault("title", "") + call.setdefault("status", ToolStatus.COMPLETED) + return TraceEvent( + index=index, + kind=EventKind.TOOL_CALL, + provenance=_PROV, + tool_call=ToolCall(**call), + ) + + +def _page(trace: CanonicalTrace, **kwargs) -> str: + return render_trace("t", trace, **kwargs).html + + +def _accents(page: str) -> list[str]: + import re + + return re.findall(r'
list[str]: + import re + + return re.findall(r'data-name-semantics="([^"]*)"', page) + + +def _captured(name: str) -> list[dict]: + path = EVIDENCE / name / "acp_trajectory.jsonl" + if not path.is_file(): + pytest.skip(f"captured rollout {name!r} is not in this tree") + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +# --------------------------------------------------------------------------- +# The classification rule +# --------------------------------------------------------------------------- + + +def test_the_accent_table_is_total_over_the_display_vocabulary(): + assert set(HUE_ACCENT) == set(VIEW_TOOL_HUES) + + +def test_every_accent_it_can_emit_is_defined_in_the_stylesheet(): + for accent in set(HUE_ACCENT.values()) | {NEUTRAL_ACCENT}: + assert f".{accent}" in viewer._VIEWER_CSS, accent + + +def test_the_module_never_reaches_for_the_substring_classifier(): + """The one import that would undo the slice, asserted absent by AST.""" + from benchflow.trajectories import ir_to_view_html + + source = pathlib.Path(ir_to_view_html.__file__).read_text(encoding="utf-8") + tree = ast.parse(source) + names = { + node.attr for node in ast.walk(tree) if isinstance(node, ast.Attribute) + } | {node.id for node in ast.walk(tree) if isinstance(node, ast.Name)} + assert "_tool_accent_class" not in names + assert "_TOOL_ACCENTS" not in names + + +def test_an_acp_category_gets_its_accent_and_a_function_name_does_not(): + """The same string, twice, from the same run — the slice in one test.""" + as_category = _trace(_tool_event(name="execute", name_semantics=ACP_KIND_SEMANTICS)) + as_function = _trace(_tool_event(name="execute", name_semantics="function_name")) + + assert _accents(_page(as_category)) == ["acc-bash"] + assert _accents(_page(as_function)) == [NEUTRAL_ACCENT] + assert "execute" in _page(as_function), "the name is still shown, only uncoloured" + + +def test_a_span_name_never_acquires_a_category_from_its_spelling(): + trace = _trace(_tool_event(name="read_file", name_semantics="gen_ai.tool.name")) + assert _accents(_page(trace)) == [NEUTRAL_ACCENT] + # The classifier the legacy card uses would have said otherwise. + assert viewer._tool_accent_class("read_file") == "acc-read" + + +def test_the_title_is_never_consulted_either(): + trace = _trace( + _tool_event( + name="mystery", name_semantics="function_name", title="bash -lc 'ls -la'" + ) + ) + assert _accents(_page(trace)) == [NEUTRAL_ACCENT] + assert viewer._tool_accent_class("bash -lc 'ls -la'") == "acc-bash" + + +def test_an_observed_category_with_no_accent_is_declared_not_hidden(): + """`think` is a real ACP kind and the stylesheet has no strip for it.""" + trace = _trace(_tool_event(name="think", name_semantics=ACP_KIND_SEMANTICS)) + rendered = render_trace("t", trace) + assert _accents(rendered.html) == [NEUTRAL_ACCENT] + assert "think" in rendered.html + declared = [ + r + for r in rendered.page_losses.records + if r.field.endswith(".tool.hue") and r.loss_class is LossClass.DROPPED + ] + assert declared, rendered.page_losses.records + + +def test_name_semantics_is_observable_on_every_tool_card(): + trace = acp_events_to_ir(H1_EVENTS) + page = _page(trace) + assert _semantics(page) == [ACP_KIND_SEMANTICS, ACP_KIND_SEMANTICS] + assert page.count("name_semantics: acp_kind") == 2 + + +# --------------------------------------------------------------------------- +# The four corpora +# --------------------------------------------------------------------------- + + +def test_acp_h1_renders_one_card_per_event(): + trace = acp_events_to_ir(H1_EVENTS) + page = _page(trace) + assert page.count('
{DIAGNOSTIC_LABEL}") == 2 + assert "session_meta" in page + assert "oracle" in page + assert page.count('data-diagnostic="canonical-ir"') == 2 + + +def test_an_unrecognized_acp_record_reaches_the_page_the_legacy_card_drops(): + events = [*H2_EVENTS, {"type": "reward", "value": 1.0}] + page = _page(acp_events_to_ir(events)) + legacy = viewer._render_acp_events("t", events, None, None) + + assert "reward" not in legacy + assert "reward" in page + assert DIAGNOSTIC_LABEL in page + + +def test_the_diagnostic_body_is_the_canonical_event_and_says_so(): + event = TraceEvent( + index=0, kind=EventKind.UNKNOWN, provenance=_PROV, source_type="session_meta" + ) + page = _page(_trace(event)) + + assert DIAGNOSTIC_LABEL in page + # The body is the canonical event, not a source record: its keys are the + # IR's, and a reader is told which document they are looking at. + assert ""kind": "unknown"" in page + assert ""provenance"" in page + + +def test_tool_output_reaches_the_page_when_it_was_observed(): + trace = acp_events_to_ir(H1_EVENTS) + page = _page(trace) + assert "TOOL-OUTPUT-1" in page + assert "TOOL-OUTPUT-1" not in viewer._render_acp_events("t", H1_EVENTS, None, None) + + +def test_a_block_with_no_text_contributes_nothing_rather_than_a_placeholder(): + trace = _trace( + _tool_event( + name="execute", + name_semantics=ACP_KIND_SEMANTICS, + content=[ContentBlock(kind=ContentBlockKind.OPAQUE, raw={"blob": "x"})], + ) + ) + page = _page(trace) + assert "blob" not in page + assert '
' not in page + + +# --------------------------------------------------------------------------- +# Cuts, escaping, prompts +# --------------------------------------------------------------------------- + + +def test_a_cut_is_announced_on_the_page_and_in_the_report(): + long_output = "x" * (TOOL_OUTPUT_PREVIEW + 25) + trace = _trace( + _tool_event( + name="execute", + name_semantics=ACP_KIND_SEMANTICS, + content=[ContentBlock(kind=ContentBlockKind.TEXT, text=long_output)], + ) + ) + rendered = render_trace("t", trace) + assert "[truncated, 25 more characters]" in rendered.html + cuts = [ + r for r in rendered.page_losses.records if r.loss_class is LossClass.NORMALIZED + ] + assert any("truncated" in r.detail or "does not show" in r.detail for r in cuts) + + +def test_trajectory_text_is_escaped_everywhere_it_lands(): + payload = "" + trace = _trace( + TraceEvent( + index=0, kind=EventKind.AGENT_MESSAGE, provenance=_PROV, text=payload + ), + _tool_event( + 1, + name="execute", + name_semantics=ACP_KIND_SEMANTICS, + title=payload, + content=[ContentBlock(kind=ContentBlockKind.TEXT, text=payload)], + ), + TraceEvent( + index=2, kind=EventKind.UNKNOWN, provenance=_PROV, source_type=payload + ), + ) + page = _page(trace) + assert "" + "y" * 600 + page, losses = view_steps_to_html( + "t", [{"i": 1, "kind": "message", "text": "b", "reasoning": payload}] + ) + assert "