From 7f4491a2c1927cf11babb81fcf83e621c0be486a Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Sun, 13 Sep 2026 23:22:43 +0300 Subject: [PATCH] fix(llm): retry a sub-call without response_format when the endpoint refuses structured outputs Memory sub-calls (query rewriter, link generator, vote runner, distill) send OpenAI Structured Outputs. Endpoints without the feature refuse the request: OpenRouter pinned to Z.AI's endpoint for z-ai/glm-5.3-flash answers 404 "No endpoints found" because its "Filter by Parameters" routing step drops that endpoint, and vendor APIs without json_schema answer 400/422 naming the field. The sub-call then failed on every run (the rewriter failed 18 of 18 times in a live session), and because every cloud OpenAiHttpError classifies as transport, each refusal also advanced the provider fallback chain. OpenAiProvider.complete now recognises that refusal narrowly and sends the same body once more without response_format. The prompts still ask for their text formats and every parser reads them when the reply is not JSON, so only decode enforcement is lost. The retry happens inside the provider, so a handled refusal never reaches runWithFallback. The provider and model pair is remembered for the process once the stripped send is accepted, later sub-calls skip the field up front, and the first time logs one warn line. Not treated as refusals: 401/402/403/429/5xx, timeouts, network failures, size rejections, and the "messages must contain the word json" 400, which is a prompt problem on an endpoint that does support the feature. Streamed requests never carry response_format and are unchanged. --- AGENTS.md | 14 +- src/llm/provider/openai/openai-provider.ts | 36 ++-- .../openai/structured-output-fallback.test.ts | 192 ++++++++++++++++++ .../openai/structured-output-fallback.ts | 118 +++++++++++ .../structured-output-refusal.fixture.ts | 32 +++ .../openai/structured-output-refusal.test.ts | 119 +++++++++++ .../openai/structured-output-refusal.ts | 76 +++++++ ...lm-fallback-seam-structured-output.test.ts | 151 ++++++++++++++ 8 files changed, 725 insertions(+), 13 deletions(-) create mode 100644 src/llm/provider/openai/structured-output-fallback.test.ts create mode 100644 src/llm/provider/openai/structured-output-fallback.ts create mode 100644 src/llm/provider/openai/structured-output-refusal.fixture.ts create mode 100644 src/llm/provider/openai/structured-output-refusal.test.ts create mode 100644 src/llm/provider/openai/structured-output-refusal.ts create mode 100644 src/runtime/llm-fallback-seam-structured-output.test.ts diff --git a/AGENTS.md b/AGENTS.md index 21fa60c0..1e8881fa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -658,6 +658,8 @@ There is currently no dedicated workspace-memory, retrieval, embeddings, or reso A three-channel cross-session memory subsystem lives in [src/memory/](src/memory/) and exposes itself to the agent via six tools in [src/tools/memory/](src/tools/memory/). The full description is in [MEMORY.md](MEMORY.md); this section is the engineering summary. The v2 roadmap (paths B+C+E+P: reactive graph, periodic consolidation, vote curation, procedure templates) lives in [MEMORY_FABRIC_V2.md](MEMORY_FABRIC_V2.md) and rolls out in strict-gated phases. Plan-level deviation from doc §9 invariant 2: v2 pays the stable-prefix KV-cache invalidation **twice** (once when `### lessons` lands in phase 5, once when `### procedures` lands in phase 7b) instead of the doc's intended single combined release — the strict-gates rollout requires evaluation windows between the two prefix-touching phases. +Cloud sub-runners (query rewriter, link generator, vote runner, distill) ask for OpenAI Structured Outputs but never depend on them: an endpoint that refuses `response_format` gets the same request once more without it and is remembered for the rest of the run, and every sub-runner parser reads the prompt's text format whenever the reply is not JSON — see §"Structured-output refusal (sub-calls)". + ### Memory-v2 phase 1B — hybrid FTS5 + embedding recall (opt-in) Lives in [src/memory/embeddings/](src/memory/embeddings/) and is **off in config until the operator enables it from the TUI Models tab** (download + start embedding model). When turned on, it adds a second `llama-server` process dedicated to `/embedding` requests and blends BM25 hits with cosine similarity over a `memory_embeddings` table (schema v5). @@ -2262,7 +2264,7 @@ Deliberately out of scope: an opt-in whole-disk / drive index (the issue sketche ## LLM reliability policy -Three narrow retry layers sit between the agent loop and the model server, plus one single-shot request repair (§"Credit-limit repair (HTTP 402)"). All are deliberately bounded and never replay already-executed tool calls, and none of them ever replays output a caller has already seen: +Three narrow retry layers sit between the agent loop and the model server, plus two single-shot request repairs (§"Credit-limit repair (HTTP 402)" and §"Structured-output refusal (sub-calls)"). All are deliberately bounded and never replay already-executed tool calls, and none of them ever replays output a caller has already seen: 1. **Parser retry (step-executor).** If the first `parseToolCall` on a completion throws, the executor calls the unary `llmComplete` exactly once more with the same prompt/slot and re-parses. A `parse_retry` event is emitted for observability. If the second attempt also fails, the original error (with a raw-output preview) is thrown. The streaming path always falls back to unary for the retry so partial SSE deltas are not double-emitted. 2. **Transport retry (LlamaServerClient).** `complete()` and the initial pre-body fetch of `completeStream()` are wrapped in a bounded retry governed by `llama.completionRetries` (default 3) and `llama.completionRetryBackoffMs` (default 150ms, exponential with ±20% jitter). Retries fire **only** for network errors (`LlamaServerError.status === null`) and HTTP 5xx. Grammar/validation 4xx and abort signals short-circuit immediately. Once the SSE body starts streaming, no further retries happen — the conversation state on the server is considered indeterminate. @@ -2287,6 +2289,16 @@ Not a retry layer — one targeted repair of one specific, self-describing refus Pinned by [parse-credit-limit.test.ts](src/llm/provider/openai/parse-credit-limit.test.ts), [plan-credit-limit-retry.test.ts](src/llm/provider/openai/plan-credit-limit-retry.test.ts) and the `credit-limit (402) recovery` block in [openai-http.test.ts](src/llm/provider/openai/openai-http.test.ts). +### Structured-output refusal (sub-calls) + +Also a repair, not a retry layer. The memory sub-runners (query rewriter, link generator, vote runner, distill) send OpenAI Structured Outputs (`response_format: { type: "json_schema" }`), and some endpoints have none: OpenRouter pinned to one vendor's endpoint answers `404 No endpoints found` with a routing funnel whose `Filter by Parameters` step dropped that endpoint, and vendor APIs without `json_schema` answer 400/422 naming the field. Every cloud `OpenAiHttpError` classifies `transport`, so that refusal used to fail the sub-call on every run **and** advance the fallback chain each time — onto a local link that may not even be running. + +- **Detection is narrow and fails closed.** [structured-output-refusal.ts](src/llm/provider/openai/structured-output-refusal.ts) accepts only a 404 `No endpoints found` with parameter-filter evidence (the `requested parameters` sentence, a structured-output field name, or a `Filter by Parameters` step whose count dropped), and a 400/422 naming `response_format`, `json_schema`, `json_object` or `structured output(s)`. Excluded: size rejections, and the `'messages' must contain the word 'json'` 400 — a prompt problem on an endpoint that does support the feature, which stripping would paper over with a downgrade for the rest of the run. Never 401/402/403/429/5xx, our own timeout, or a network failure. +- **One send without the field, below the chain.** [structured-output-fallback.ts](src/llm/provider/openai/structured-output-fallback.ts) runs inside `OpenAiProvider.complete` and re-sends the same body minus `response_format`, so a handled refusal never reaches `runWithFallback`, never advances it and never trips its breaker. The prompts still ask for their text formats (``, `LINK`, `UPVOTE`, `LESSON`) and each parser reads them whenever the reply is not JSON, so the answer survives and only decode enforcement is lost. It applies only when dropping `responseFormat` changes the wire: a request with `tools` never sends it, and a `response_format` set through `extraBody` is the operator's and stays. Streamed requests never carry `responseFormat` and are untouched. +- **Remembered once confirmed.** The (provider id, model) pair is recorded for the life of the process only when the stripped send is accepted — a retry that fails too propagates its own error and records nothing — and later sub-calls to that pair skip `response_format` up front, with no failed round trip. The record lives outside the provider instance, so it survives a provider rebuild on config save. The first record logs one `warn`: `llm: "" does not support structured outputs (response_format) for ; memory sub-calls fall back to prompt-only output for the rest of this run.` + +Pinned by [structured-output-refusal.test.ts](src/llm/provider/openai/structured-output-refusal.test.ts), [structured-output-fallback.test.ts](src/llm/provider/openai/structured-output-fallback.test.ts) and [llm-fallback-seam-structured-output.test.ts](src/runtime/llm-fallback-seam-structured-output.test.ts). + ### Failure taxonomy Every terminal failure the agent loop surfaces is normalised into a canonical `LlmFailureCategory` before `step_error` / `loop_failed` fire. The classes live in [src/llm/reliability/](src/llm/reliability/) and carry specialised fields for postmortem use. diff --git a/src/llm/provider/openai/openai-provider.ts b/src/llm/provider/openai/openai-provider.ts index 613e16ff..31485e35 100644 --- a/src/llm/provider/openai/openai-provider.ts +++ b/src/llm/provider/openai/openai-provider.ts @@ -42,6 +42,7 @@ import { adaptQwenTaggedToolResponse, } from "./qwen-tagged-tool-response-adapter.js"; import type { CreditLimitLogger } from "./plan-credit-limit-retry.js"; +import { sendWithStructuredOutputFallback } from "./structured-output-fallback.js"; export interface OpenAiProviderOptions { id: string; @@ -148,19 +149,30 @@ export class OpenAiProvider implements LlmProvider { } async complete(request: CompletionRequest): Promise { - const body = buildOpenAiChatBody( - request, - this.defaultChatModel, - false, - this.extraBody, - this.maxOutputTokens, - this.strictTools, - ); - const json = await openAiPostJson( - this.http, - `${this.apiPathPrefix}/chat/completions`, - body, + // Unary only: sub-calls carry `response_format`, streamed turns never do. + const json = await sendWithStructuredOutputFallback( + { + providerId: this.id, + model: this.defaultChatModel, + logger: this.http.logger, + }, request, + (req) => + buildOpenAiChatBody( + req, + this.defaultChatModel, + false, + this.extraBody, + this.maxOutputTokens, + this.strictTools, + ), + (body) => + openAiPostJson( + this.http, + `${this.apiPathPrefix}/chat/completions`, + body, + request, + ), ); const adapted = this.taggedToolCompatibility === "qwen" diff --git a/src/llm/provider/openai/structured-output-fallback.test.ts b/src/llm/provider/openai/structured-output-fallback.test.ts new file mode 100644 index 00000000..443b1f56 --- /dev/null +++ b/src/llm/provider/openai/structured-output-fallback.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { + CompletionRequest, + ResponseFormatJsonSchema, +} from "../completion-types.js"; +import { OpenAiHttpError } from "./openai-http.js"; +import { OpenAiProvider } from "./openai-provider.js"; +import { OPENROUTER_PARAMETER_REFUSAL_BODY } from "./structured-output-refusal.fixture.js"; + +const responseFormat: ResponseFormatJsonSchema = { + name: "query_rewriter", + schema: { + type: "object", + properties: { rewritten_query: { type: "string" } }, + required: ["rewritten_query"], + additionalProperties: false, + }, +}; + +const ENVELOPE = "deploy the kastel app"; + +type Reply = () => Response; + +const ok = + (content: string): Reply => + () => + new Response( + JSON.stringify({ + model: "z-ai/glm-5.3-flash", + choices: [{ message: { role: "assistant", content }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + +const status = + (code: number, body: string): Reply => + () => + new Response(body, { status: code }); + +const refusal = status(404, OPENROUTER_PARAMETER_REFUSAL_BODY); + +/** + * Scripted fetch that records every request body. A request beyond the + * script answers 418 — non-retryable, so an unexpected send fails the + * test loudly instead of being absorbed by the retry budget. + */ +function scriptedFetch(replies: Reply[]) { + const bodies: Array> = []; + const impl = vi.fn(async (_url: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body)) as Record); + const next = replies.shift(); + return next ? next() : new Response("unexpected extra request", { status: 418 }); + }); + return { fetchImpl: impl as unknown as typeof fetch, bodies, replies }; +} + +let seq = 0; +function makeProvider( + fetchImpl: typeof fetch, + opts: { id?: string; model?: string } = {}, +) { + const warn = vi.fn(); + const provider = new OpenAiProvider({ + id: opts.id ?? `openrouter-${++seq}`, + baseUrl: "https://openrouter.example", + apiKey: "k", + defaultChatModel: opts.model ?? "z-ai/glm-5.3-flash", + fetchImpl, + logger: { warn }, + }); + return { provider, warn }; +} + +const subcall: CompletionRequest = { prompt: "rewrite", maxTokens: 256, responseFormat }; + +describe("OpenAiProvider.complete — structured-output refusal fallback", () => { + it("retries once without response_format and returns that answer", async () => { + const net = scriptedFetch([refusal, ok(ENVELOPE)]); + const { provider, warn } = makeProvider(net.fetchImpl); + + const result = await provider.complete(subcall); + + expect(result.content).toBe(ENVELOPE); + expect(net.bodies).toHaveLength(2); + expect(net.bodies[0]).toHaveProperty("response_format.type", "json_schema"); + expect(net.bodies[1]).not.toHaveProperty("response_format"); + // Only the field is dropped: the retry is otherwise the same request. + const { response_format: _sent, ...firstWithout } = net.bodies[0]!; + expect(net.bodies[1]).toEqual(firstWithout); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]![0]).toContain(provider.id); + expect(warn.mock.calls[0]![0]).toContain("does not support structured outputs"); + }); + + it("remembers the refusal: the next sub-call skips response_format with no failed round trip", async () => { + const net = scriptedFetch([refusal, ok(ENVELOPE), ok(ENVELOPE)]); + const { provider, warn } = makeProvider(net.fetchImpl); + + await provider.complete(subcall); + await provider.complete(subcall); + + expect(net.bodies).toHaveLength(3); + expect(net.bodies[2]).not.toHaveProperty("response_format"); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it("keys the memory by provider id and model, and outlives the provider instance", async () => { + const id = `openrouter-${++seq}`; + const first = scriptedFetch([refusal, ok(ENVELOPE)]); + await makeProvider(first.fetchImpl, { id }).provider.complete(subcall); + + const rebuilt = scriptedFetch([ok(ENVELOPE)]); + await makeProvider(rebuilt.fetchImpl, { id }).provider.complete(subcall); + expect(rebuilt.bodies[0]).not.toHaveProperty("response_format"); + + const otherProvider = scriptedFetch([ok(ENVELOPE)]); + await makeProvider(otherProvider.fetchImpl).provider.complete(subcall); + expect(otherProvider.bodies[0]).toHaveProperty("response_format"); + + const otherModel = scriptedFetch([ok(ENVELOPE)]); + await makeProvider(otherModel.fetchImpl, { + id, + model: "openai/gpt-5.4-mini", + }).provider.complete(subcall); + expect(otherModel.bodies[0]).toHaveProperty("response_format"); + }); + + it.each([ + ["invalid key", "API key not valid. Please pass a valid API key."], + ["context length", "This model's maximum context length is 32768 tokens."], + [ + "json word", + "'messages' must contain the word 'json' in some form, to use 'response_format' of type 'json_object'.", + ], + ])("propagates a non-refusal 400 (%s) unchanged, without a retry", async (_label, message) => { + const body = JSON.stringify({ error: { message } }); + const net = scriptedFetch([status(400, body), ok(ENVELOPE)]); + const { provider, warn } = makeProvider(net.fetchImpl); + + const err = await provider.complete(subcall).catch((e: unknown) => e); + + expect(err).toBeInstanceOf(OpenAiHttpError); + expect((err as OpenAiHttpError).status).toBe(400); + expect((err as OpenAiHttpError).message).toBe(`openai provider 400: ${body}`); + expect(net.bodies).toHaveLength(1); + expect(warn).not.toHaveBeenCalled(); + }); + + it("has no retry path for a request without responseFormat", async () => { + const net = scriptedFetch([refusal, ok(ENVELOPE)]); + const { provider } = makeProvider(net.fetchImpl); + + await expect(provider.complete({ prompt: "turn" })).rejects.toMatchObject({ status: 404 }); + expect(net.bodies).toHaveLength(1); + }); + + it("has no retry path when tools kept response_format off the wire", async () => { + const net = scriptedFetch([refusal, ok(ENVELOPE)]); + const { provider } = makeProvider(net.fetchImpl); + const tools: NonNullable = [ + { type: "function", function: { name: "emit", parameters: { type: "object" } } }, + ]; + + await expect(provider.complete({ ...subcall, tools })).rejects.toMatchObject({ status: 404 }); + expect(net.bodies).toHaveLength(1); + expect(net.bodies[0]).not.toHaveProperty("response_format"); + }); + + it("propagates a failed retry's own error and remembers nothing", async () => { + const other = JSON.stringify({ error: { message: "Provider returned error" } }); + const net = scriptedFetch([refusal, status(400, other), ok(ENVELOPE)]); + const { provider, warn } = makeProvider(net.fetchImpl); + + await expect(provider.complete(subcall)).rejects.toMatchObject({ status: 400 }); + await provider.complete(subcall); + + expect(net.bodies).toHaveLength(3); + expect(net.bodies[2]).toHaveProperty("response_format"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("leaves streaming alone: a refusal on completeStream is not retried", async () => { + const net = scriptedFetch([refusal, ok(ENVELOPE)]); + const { provider } = makeProvider(net.fetchImpl); + + const stream = provider.completeStream(subcall); + await expect(stream.next()).rejects.toMatchObject({ status: 404 }); + expect(net.bodies).toHaveLength(1); + }); +}); diff --git a/src/llm/provider/openai/structured-output-fallback.ts b/src/llm/provider/openai/structured-output-fallback.ts new file mode 100644 index 00000000..4de60740 --- /dev/null +++ b/src/llm/provider/openai/structured-output-fallback.ts @@ -0,0 +1,118 @@ +import type { CompletionRequest } from "../completion-types.js"; +import { isStructuredOutputRefusal } from "./structured-output-refusal.js"; + +/** Minimal logging surface this fallback needs (satisfied by `StructuredLogger`). */ +export interface StructuredOutputLogger { + warn(message: string, context?: Record): void; +} + +/** + * The (provider id, model) pairs whose endpoint refused structured + * outputs, for the lifetime of the process. + * + * Kept out of the provider instance on purpose: a provider is rebuilt on + * hot-swap and on every config write, and forgetting the refusal there + * would put the failed round trip back on every sub-call after each + * save. Not persisted either — a vendor that ships `json_schema` support + * is picked up on the next start, which is the cheapest re-probe there is. + */ +export class StructuredOutputRefusals { + private readonly pairs = new Set(); + + has(providerId: string, model: string): boolean { + return this.pairs.has(pairKey(providerId, model)); + } + + /** Record a pair; `true` only the first time, which is when to log. */ + record(providerId: string, model: string): boolean { + const key = pairKey(providerId, model); + if (this.pairs.has(key)) return false; + this.pairs.add(key); + return true; + } +} + +/** The process-wide record every `OpenAiProvider` consults. */ +export const structuredOutputRefusals = new StructuredOutputRefusals(); + +function pairKey(providerId: string, model: string): string { + return JSON.stringify([providerId, model]); +} + +export interface StructuredOutputFallbackContext { + providerId: string; + model: string; + logger?: StructuredOutputLogger | undefined; + /** Defaults to the process-wide record; injected by tests. */ + refusals?: StructuredOutputRefusals; +} + +/** + * Send a unary completion, and when its endpoint refuses the + * `response_format` the request carried, send it once more without it. + * + * Why this is safe: `response_format` is only set by the memory + * sub-runners (query rewriter, link generator, vote runner, distill), and + * every one of their prompts still asks for its text format — the + * `` envelope, `LINK` / `UPVOTE` / `LESSON` lines — which + * their parsers read whenever the reply is not JSON. Losing the schema + * costs decode enforcement, not the answer. + * + * Why it lives here, below the fallback chain: every cloud + * `OpenAiHttpError` classifies as `transport`, so an unhandled refusal + * advanced `runWithFallback` to the next link — for a sub-call whose + * request was the only thing wrong — and did so again on every sub-call. + * Handled inside the provider's `complete`, the chain never sees it. + * + * Bounded: exactly one extra send, not wrapped again. The refusal is + * remembered only once that send is accepted, which is what proves the + * field was the problem; a retry that fails too propagates its own error + * and leaves the provider untouched. Afterwards the pair skips + * `response_format` up front, with no failed round trip. + * + * Only a request where dropping `responseFormat` changes the wire takes + * this path: a request that also carries `tools` never sends + * `response_format` (`buildOpenAiChatBody`), and one set through + * `extraBody` is the operator's and is never removed. + */ +export async function sendWithStructuredOutputFallback( + ctx: StructuredOutputFallbackContext, + request: CompletionRequest, + buildBody: (request: CompletionRequest) => Record, + send: (body: Record) => Promise, +): Promise { + const body = buildBody(request); + if (!request.responseFormat) return send(body); + const promptOnly: CompletionRequest = { ...request }; + delete promptOnly.responseFormat; + const promptOnlyBody = buildBody(promptOnly); + if (promptOnlyBody.response_format === body.response_format) { + return send(body); + } + const refusals = ctx.refusals ?? structuredOutputRefusals; + if (refusals.has(ctx.providerId, ctx.model)) return send(promptOnlyBody); + try { + return await send(body); + } catch (err) { + if (request.signal?.aborted || !isStructuredOutputRefusal(err)) throw err; + const result = await send(promptOnlyBody); + if (refusals.record(ctx.providerId, ctx.model)) { + ctx.logger?.warn(structuredOutputFallbackMessage(ctx), { + provider: ctx.providerId, + model: ctx.model, + status: err.status, + }); + } + return result; + } +} + +export function structuredOutputFallbackMessage( + ctx: Pick, +): string { + return ( + `llm: "${ctx.providerId}" does not support structured outputs ` + + `(response_format) for ${ctx.model}; memory sub-calls fall back to ` + + `prompt-only output for the rest of this run.` + ); +} diff --git a/src/llm/provider/openai/structured-output-refusal.fixture.ts b/src/llm/provider/openai/structured-output-refusal.fixture.ts new file mode 100644 index 00000000..c6b0e370 --- /dev/null +++ b/src/llm/provider/openai/structured-output-refusal.fixture.ts @@ -0,0 +1,32 @@ +/** + * Test fixture: OpenRouter's 404 when routing filters leave no endpoint. + * `counts` are the funnel's `endpoint_count` per step, in the order + * OpenRouter sends them. + */ +export function openRouterRoutingFunnelBody( + counts: [number, number, number, number], +): string { + return JSON.stringify({ + error: { + message: "No endpoints found for z-ai/glm-5.3-flash.", + code: 404, + metadata: { + routing_funnel: [ + { step: "Initial Endpoints", endpoint_count: counts[0] }, + { step: "Filter by Parameters", endpoint_count: counts[1] }, + { step: "Apply Status Sorting", endpoint_count: counts[2] }, + { step: "Filter by Fallback", endpoint_count: counts[3] }, + ], + }, + }, + }); +} + +/** + * The body OpenRouter sent live (2026-09-13) for every rewriter, link and + * vote sub-call with `provider.order: ["z-ai"]`: the parameter step drops + * Z.AI's endpoint (27 → 20), and the pinned order then leaves nothing. + */ +export const OPENROUTER_PARAMETER_REFUSAL_BODY = openRouterRoutingFunnelBody([ + 27, 20, 20, 0, +]); diff --git a/src/llm/provider/openai/structured-output-refusal.test.ts b/src/llm/provider/openai/structured-output-refusal.test.ts new file mode 100644 index 00000000..602f34d9 --- /dev/null +++ b/src/llm/provider/openai/structured-output-refusal.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "vitest"; + +import { OpenAiHttpError } from "./openai-http.js"; +import { isStructuredOutputRefusal } from "./structured-output-refusal.js"; +import { + OPENROUTER_PARAMETER_REFUSAL_BODY, + openRouterRoutingFunnelBody as routingFunnel, +} from "./structured-output-refusal.fixture.js"; + +/** Build the error exactly as `httpErrorFromResponse` does: a 300-char preview. */ +function httpError(status: number, body: string): OpenAiHttpError { + return new OpenAiHttpError( + `openai provider ${status}: ${body.slice(0, 300)}`, + status, + "https://openrouter.ai/api/v1/chat/completions", + false, + null, + "openrouter", + ); +} + +const errorBody = (message: string) => JSON.stringify({ error: { message } }); + +describe("isStructuredOutputRefusal", () => { + it("reads OpenRouter's live 404 routing refusal through the 300-char preview", () => { + expect(OPENROUTER_PARAMETER_REFUSAL_BODY.length).toBeGreaterThan(300); + expect( + isStructuredOutputRefusal( + httpError(404, OPENROUTER_PARAMETER_REFUSAL_BODY), + ), + ).toBe(true); + }); + + it("reads the require_parameters 404 sentence as a refusal", () => { + const body = errorBody( + "No endpoints found that can handle the requested parameters. To learn more about provider routing, visit: https://openrouter.ai/docs/provider-routing", + ); + expect(isStructuredOutputRefusal(httpError(404, body))).toBe(true); + }); + + it("does not blame parameters when the parameter step removed no endpoint", () => { + expect( + isStructuredOutputRefusal(httpError(404, routingFunnel([27, 27, 27, 0]))), + ).toBe(false); + }); + + it.each([ + [ + "data-policy funnel", + errorBody( + "No endpoints found matching your data policy (Free model publication).", + ), + ], + [ + "unknown model", + errorBody("The model `z-ai/nope` does not exist or you do not have access."), + ], + ["bare 404", "Not Found"], + ])("does not read a 404 %s as a refusal", (_label, body) => { + expect(isStructuredOutputRefusal(httpError(404, body))).toBe(false); + }); + + it.each([ + [ + 400, + errorBody( + "Invalid parameter: 'response_format' of type 'json_schema' is not supported with this model.", + ), + ], + [422, JSON.stringify({ detail: "Structured outputs are not supported." })], + [400, errorBody("json_object response format is unavailable for this model")], + [400, errorBody("Unsupported field: json_schema")], + ])("reads a %i naming the feature as a refusal", (status, body) => { + expect(isStructuredOutputRefusal(httpError(status, body))).toBe(true); + }); + + it("leaves the 'must contain the word json' 400 to the prompt, not the wire", () => { + const body = errorBody( + "<400> InternalError.Algo.InvalidParameter: 'messages' must contain the word 'json' in some form, to use 'response_format' of type 'json_object'.", + ); + expect(isStructuredOutputRefusal(httpError(400, body))).toBe(false); + }); + + it.each([ + [ + "context length that also names response_format", + "Prompt plus response_format schema exceed the maximum context length of 32768 tokens.", + ], + [ + "context length", + "This model's maximum context length is 32768 tokens. Please reduce the length of the messages.", + ], + ["invalid key", "API key not valid. Please pass a valid API key."], + ])("does not read a 400 %s as a refusal", (_label, message) => { + expect(isStructuredOutputRefusal(httpError(400, errorBody(message)))).toBe( + false, + ); + }); + + it.each([401, 402, 403, 429, 500, 503])( + "never reads a %i as a refusal, whatever the body says", + (status) => { + const body = errorBody("response_format json_schema is not supported"); + expect(isStructuredOutputRefusal(httpError(status, body))).toBe(false); + }, + ); + + it("never reads a network failure, our own timeout, or an untyped error as a refusal", () => { + const url = "https://openrouter.ai/api/v1/chat/completions"; + const wording = "response_format json_schema is not supported"; + expect( + isStructuredOutputRefusal(new OpenAiHttpError(wording, null, url)), + ).toBe(false); + expect( + isStructuredOutputRefusal(new OpenAiHttpError(wording, 400, url, true)), + ).toBe(false); + expect(isStructuredOutputRefusal(new Error(`400 ${wording}`))).toBe(false); + }); +}); diff --git a/src/llm/provider/openai/structured-output-refusal.ts b/src/llm/provider/openai/structured-output-refusal.ts new file mode 100644 index 00000000..c0124503 --- /dev/null +++ b/src/llm/provider/openai/structured-output-refusal.ts @@ -0,0 +1,76 @@ +import { isRequestSizeRejection } from "../../reliability/request-size-rejection.js"; +import { OpenAiHttpError } from "./openai-http.js"; + +/** + * Did an endpoint refuse a request because it cannot serve OpenAI + * Structured Outputs (`response_format: { type: "json_schema" }`)? + * + * Only asked about a request whose body actually carried + * `response_format` — the memory sub-runners' calls (see + * `sendWithStructuredOutputFallback`). `true` means "the same request + * without `response_format` is worth one send", nothing more: the caller + * retries once and remembers the refusal only when that retry is + * accepted, so a misread body costs one round trip, never a permanent + * downgrade of a provider that does support the feature. + * + * Deliberately narrow, and it fails closed — an unrecognised body is + * `false` and the provider's error propagates untouched: + * + * - **404** only as OpenRouter's routing refusal: `No endpoints found` + * plus evidence that parameter filtering emptied the funnel — the + * `requested parameters` sentence (sent under + * `provider.require_parameters`), a structured-output field name, or a + * `Filter by Parameters` funnel step. When that step's count and the + * one before it are both readable and equal, parameters removed + * nothing and the funnel was emptied elsewhere (data policy, a pinned + * provider order), so it is not a refusal. A bare 404 is a wrong model + * id or base URL. + * - **400 / 422** whose body names the feature: `response_format`, + * `json_schema`, `json_object`, or `structured output(s)`. Excluded: + * size rejections (`isRequestSizeRejection` — the request is too big, + * and dropping a field does not change that), and the OpenAI/DashScope + * `'messages' must contain the word 'json'` 400. That one is a prompt + * problem on an endpoint that *does* support structured outputs: + * stripping would get the call through, but it would also downgrade + * the provider for the rest of the process over a missing word. + * - Never 401/402/403/429/5xx, our own timeout, or a network failure + * (`status === null`). Those say nothing about the request's shape, + * and the retry budget and the fallback chain already own them. + * + * The body reaches us as the first 300 characters of the error message + * (`httpErrorFromResponse`); OpenRouter lists the parameter step second + * in its funnel, well inside that preview. + */ +export function isStructuredOutputRefusal( + err: unknown, +): err is OpenAiHttpError { + if (!(err instanceof OpenAiHttpError) || err.timedOut) return false; + if (err.status === 404) return isRoutingRefusal(err.message); + if (err.status !== 400 && err.status !== 422) return false; + if (JSON_WORD_REQUIRED.test(err.message)) return false; + if (isRequestSizeRejection(err)) return false; + return FEATURE_WORDING.test(err.message); +} + +const FEATURE_WORDING = + /response_format|json_schema|json_object|structured[\s_-]*outputs?/i; +const JSON_WORD_REQUIRED = /must contain the word\W+json/i; +const NO_ENDPOINTS = /no endpoints found/i; +const REQUESTED_PARAMETERS = /requested parameters/i; +const PARAMETER_STEP = /filter by parameters/i; +/** The step before `Filter by Parameters`, then that step: two counts. */ +const PARAMETER_STEP_COUNTS = + /"endpoint_count"\s*:\s*(\d+)\s*\}\s*,\s*\{\s*"step"\s*:\s*"Filter by Parameters"\s*,\s*"endpoint_count"\s*:\s*(\d+)/i; + +function isRoutingRefusal(text: string): boolean { + if (!NO_ENDPOINTS.test(text)) return false; + if (REQUESTED_PARAMETERS.test(text) || FEATURE_WORDING.test(text)) { + return true; + } + if (!PARAMETER_STEP.test(text)) return false; + const counts = PARAMETER_STEP_COUNTS.exec(text); + // Unreadable counts (a reshaped funnel, a cut preview): the step's + // presence is the evidence, and the confirming retry is the backstop. + if (!counts) return true; + return Number(counts[2]) < Number(counts[1]); +} diff --git a/src/runtime/llm-fallback-seam-structured-output.test.ts b/src/runtime/llm-fallback-seam-structured-output.test.ts new file mode 100644 index 00000000..e1f6ac31 --- /dev/null +++ b/src/runtime/llm-fallback-seam-structured-output.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it, vi } from "vitest"; + +import { ProviderFallbackChain } from "../llm/fallback/index.js"; +import { DEFAULT_FALLBACK_TIMING } from "../llm/fallback/fallback-config.js"; +import type { + CompletionRequest, + ResponseFormatJsonSchema, +} from "../llm/provider/completion-types.js"; +import type { LlmProvider } from "../llm/provider/llm-provider.js"; +import { + fakeAnswer, + fakeProvider, +} from "../llm/provider/fake-provider.fixture.js"; +import { OpenAiProvider } from "../llm/provider/openai/openai-provider.js"; +import { OPENROUTER_PARAMETER_REFUSAL_BODY } from "../llm/provider/openai/structured-output-refusal.fixture.js"; +import { + createFallbackCompleter, + createFallbackStreamer, + type FallbackSeamDeps, +} from "./llm-fallback-seam.js"; + +/** + * The fallback chain is the reason the refusal is handled inside the + * provider: every cloud `OpenAiHttpError` classifies `transport`, so a + * refusal that escaped `complete` advanced the chain to the next link — + * for a sub-call whose only defect was a field the endpoint lacks. + */ + +const responseFormat: ResponseFormatJsonSchema = { + name: "memory_votes", + schema: { type: "object", properties: {}, additionalProperties: false }, +}; + +const params = { + prompt: "vote", + grammar: 'root ::= "ok"', + slotId: -1, + sessionId: "s1", + tools: [], +} as const; + +function seamDeps(providers: Map) { + const chain = new ProviderFallbackChain({ + resolve: () => ({ chain: ["cloud", "local"], timing: DEFAULT_FALLBACK_TIMING }), + }); + const advanceFrom = vi.spyOn(chain, "advanceFrom"); + const deps: FallbackSeamDeps = { + fallbackChain: chain, + resolveSlice: (providerId) => { + const provider = providers.get(providerId)!; + return { provider, transport: provider.capabilities.toolTransport }; + }, + recordUnaryUsage: () => {}, + recordStreamUsage: () => {}, + }; + return { deps, advanceFrom }; +} + +/** A real OpenAI-compatible cloud link: 404 refusal first, then a line-grammar answer. */ +function refusingCloud(id: string) { + const bodies: Array> = []; + const replies = [ + () => new Response(OPENROUTER_PARAMETER_REFUSAL_BODY, { status: 404 }), + () => + new Response( + JSON.stringify({ + choices: [ + { message: { role: "assistant", content: "UPVOTE memory:12" }, finish_reason: "stop" }, + ], + }), + { status: 200 }, + ), + ]; + const fetchImpl = vi.fn(async (_url: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body)) as Record); + return replies.shift()?.() ?? new Response("unexpected", { status: 418 }); + }); + const provider = new OpenAiProvider({ + id, + baseUrl: "https://openrouter.example", + apiKey: "k", + defaultChatModel: "z-ai/glm-5.3-flash", + fetchImpl: fetchImpl as unknown as typeof fetch, + logger: { warn: () => {} }, + }); + return { provider, bodies }; +} + +describe("fallback seam — structured-output refusal", () => { + it("serves a refused sub-call from the same cloud link without advancing the chain", async () => { + // A unique provider id: the refusal memory is process-wide. + const cloud = refusingCloud("cloud"); + const localServe = vi.fn(async () => fakeAnswer("local")); + const { deps, advanceFrom } = seamDeps( + new Map([ + ["cloud", cloud.provider], + ["local", fakeProvider("local", "grammar", localServe)], + ]), + ); + + const result = await createFallbackCompleter(deps)({ ...params, responseFormat }); + + expect(result.content).toBe("UPVOTE memory:12"); + expect(result.servedTransport).toBe("native_tools"); + expect(cloud.bodies).toHaveLength(2); + expect(cloud.bodies[1]).not.toHaveProperty("response_format"); + expect(advanceFrom).not.toHaveBeenCalled(); + expect(localServe).not.toHaveBeenCalled(); + }); + + it("control: the same 404 on a request without responseFormat still advances the chain", async () => { + // Proves the test above is not vacuous — this refusal body is one the + // chain falls over on when nothing handles it. + const { provider } = refusingCloud("cloud-control"); + const localServe = vi.fn(async () => fakeAnswer("local")); + const { deps, advanceFrom } = seamDeps( + new Map([ + ["cloud", provider], + ["local", fakeProvider("local", "grammar", localServe)], + ]), + ); + + const result = await createFallbackCompleter(deps)(params); + + expect(advanceFrom).toHaveBeenCalledTimes(1); + expect(result.modelId).toBe("local-model"); + }); + + it("never hands responseFormat to a streamed request, while the unary seam does", async () => { + const seen: CompletionRequest[] = []; + const serve = async (request: CompletionRequest) => { + seen.push(request); + return fakeAnswer("cloud"); + }; + const { deps } = seamDeps( + new Map([ + ["cloud", fakeProvider("cloud", "native_tools", serve)], + ["local", fakeProvider("local", "grammar", serve)], + ]), + ); + + const stream = createFallbackStreamer(deps)({ ...params, responseFormat }); + let next = await stream.next(); + while (!next.done) next = await stream.next(); + await createFallbackCompleter(deps)({ ...params, responseFormat }); + + expect(seen).toHaveLength(2); + expect(seen[0]).not.toHaveProperty("responseFormat"); + expect(seen[1]).toHaveProperty("responseFormat", responseFormat); + }); +});