From 67e5a9b0e6028e4eff31725d03518160a646cd8e Mon Sep 17 00:00:00 2001 From: Yang Date: Sun, 16 Aug 2026 18:46:24 +0800 Subject: [PATCH 01/10] feat(agents): bind reasoning effort and fast mode to each provider chain entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both controls are stored per chain BINDING, beside the model, not once per Agent. A chain mixes Providers and models for fallback and the legal effort levels follow the model, so a single Agent-wide value is necessarily invalid for at least one entry: codex advertises `ultra`, Claude never does, and Haiku 4.5 accepts no effort at all. Nothing is hard-coded. The manifest declares only WHETHER a CLI accepts each control; the legal values are discovered per model alongside the model ids, reusing the existing per-credential probe. An absent level list means "not discovered" and an empty one means "discovered, none exist" — collapsing the two would make an unprobeable credential look like a model with no levels. Execution validates neither, deliberately: by then only the model string survives and it can still change mid-Run through fallback. So `diagnose` reports a control bound to a Provider whose CLI has no such setting (provider_reasoning_effort_unsupported / provider_fast_mode_unsupported, severity warn), and a level the MODEL rejects is reported by the CLI itself with the accepted set named in its error. Engine plumbing: - claude-code passes `--effort ` and `--settings {"fastMode":true}`. minVersion rises to 2.1.47, bisected against npm: 2.1.45 rejects `--effort` and 2.1.46 was never published. - codex passes `-c model_reasoning_effort=…` and `-c service_tier="priority"`, re-passed on resume because `-c` is accepted there. Fast mode is only ever REQUESTED — the plan, the model and the endpoint each hold a veto and the CLI settles it at run time. The result log therefore records what was SERVED, with the server's own answer beating the client's intent: claude reports `usage.speed`, so a claimed `on` served at standard records as `denied`. codex reports nothing about the tier at all, which is why an absent state is never read as `off`. The eligibility probe fails OPEN. Every failure path resolves to undefined rather than `available: false`, because refusing a feature on the strength of a failed probe is worse than letting the run report the outcome itself. clearProviderBinding clears both alongside the model — without it a codex `ultra` leaks onto a Claude fallback that has no such level. Evaluation freezes both in configSnapshot, so two tasks differing only in reasoning depth are no longer indistinguishable. --- apps/api/src/db/schema.pg.ts | 2 + apps/api/src/db/schema.sqlite.ts | 2 + .../cli-invocation-surface.snapshot.json | 6 +- .../__tests__/list-available-models.test.ts | 195 +++++++++ .../engine/__tests__/provider-catalog.test.ts | 27 ++ .../engine/__tests__/reasoning-args.test.ts | 407 ++++++++++++++++++ apps/api/src/engine/claude-code.ts | 195 ++++++++- apps/api/src/engine/codex-agent.ts | 110 ++++- apps/api/src/engine/model-capabilities.ts | 65 +++ apps/api/src/engine/provider-catalog.ts | 25 ++ apps/api/src/engine/types.ts | 30 +- .../agent-execution-diagnose.test.ts | 71 +++ .../src/lib/__tests__/cli-installer.test.ts | 18 +- .../lib/__tests__/evaluation-snapshot.test.ts | 107 ++++- .../provider-binding-reasoning.test.ts | 90 ++++ apps/api/src/lib/agent-execution-diagnose.ts | 25 ++ apps/api/src/lib/agent-helpers.ts | 17 + apps/api/src/lib/evaluation-snapshot.ts | 37 +- .../routes/__tests__/evaluation-tasks.test.ts | 2 + apps/web/src/components/stream-log-item.tsx | 36 ++ apps/web/src/content/manual/en/04-agents.md | 22 + .../src/content/manual/en/05-evaluation.md | 5 +- apps/web/src/content/manual/zh/04-agents.md | 22 + .../src/content/manual/zh/05-evaluation.md | 5 +- apps/web/src/hooks/use-agents.ts | 2 + apps/web/src/locales/en.json | 27 +- apps/web/src/locales/zh.json | 27 +- .../__tests__/config-tab-mcp-warning.test.tsx | 2 + .../__tests__/provider-capabilities.test.ts | 156 +++++++ .../__tests__/provider-chain.test.ts | 31 ++ .../web/src/pages/agent-detail/config-tab.tsx | 132 +++++- .../agent-detail/provider-capabilities.ts | 86 ++++ .../src/pages/agent-detail/provider-chain.ts | 5 + apps/web/src/pages/agent-detail/types.ts | 15 +- .../src/pages/agent-detail/use-agent-form.ts | 6 + docs/PRODUCT.md | 2 +- docs/agent/api-permissions.md | 3 +- docs/agent/core-concepts-notes.md | 5 +- docs/core-concepts.md | 7 +- e2e/tests/agents/provider-chain.spec.ts | 74 ++++ .../shared/src/__tests__/agent-schema.test.ts | 49 +++ .../src/__tests__/probe-models-schema.test.ts | 83 ++++ .../src/__tests__/provider-manifest.test.ts | 27 ++ packages/shared/src/schemas/agent.ts | 25 ++ packages/shared/src/schemas/probe-models.ts | 48 ++- packages/shared/src/schemas/provider.ts | 25 +- 46 files changed, 2322 insertions(+), 36 deletions(-) create mode 100644 apps/api/src/engine/__tests__/reasoning-args.test.ts create mode 100644 apps/api/src/engine/model-capabilities.ts create mode 100644 apps/api/src/lib/__tests__/provider-binding-reasoning.test.ts create mode 100644 packages/shared/src/__tests__/probe-models-schema.test.ts diff --git a/apps/api/src/db/schema.pg.ts b/apps/api/src/db/schema.pg.ts index 52edcdc6..a8cd086b 100644 --- a/apps/api/src/db/schema.pg.ts +++ b/apps/api/src/db/schema.pg.ts @@ -1245,6 +1245,8 @@ export const evaluationTasks = pgTable( providerId: string | null providerName: string | null model: string | null + reasoningEffort: string | null + fastMode: boolean | null systemPrompt: string capturedAt: string }>() diff --git a/apps/api/src/db/schema.sqlite.ts b/apps/api/src/db/schema.sqlite.ts index a7945872..24fb3198 100644 --- a/apps/api/src/db/schema.sqlite.ts +++ b/apps/api/src/db/schema.sqlite.ts @@ -1232,6 +1232,8 @@ export const evaluationTasks = sqliteTable( providerId: string | null providerName: string | null model: string | null + reasoningEffort: string | null + fastMode: boolean | null systemPrompt: string capturedAt: string }>() diff --git a/apps/api/src/engine/__tests__/cli-invocation-surface.snapshot.json b/apps/api/src/engine/__tests__/cli-invocation-surface.snapshot.json index 32351e10..06f2da0a 100644 --- a/apps/api/src/engine/__tests__/cli-invocation-surface.snapshot.json +++ b/apps/api/src/engine/__tests__/cli-invocation-surface.snapshot.json @@ -29,16 +29,18 @@ }, "claude-code": { "sourceFile": "apps/api/src/engine/claude-code.ts", - "minVersion": null, + "minVersion": "2.1.47", "surface": [ "--allowedTools=mcp__*", "--append-system-prompt", "--dangerously-skip-permissions", + "--effort", "--json", "--model", "--output-format", "--permission-mode=plan", "--resume", + "--settings", "--verbose", "-p", "auth", @@ -67,7 +69,9 @@ "--sandbox=workspace-write", "--skip-git-repo-check", "-c", + "-c=model_reasoning_effort=${}", "-c=openai_base_url=${}", + "-c=service_tier=${}", "debug", "exec", "login", diff --git a/apps/api/src/engine/__tests__/list-available-models.test.ts b/apps/api/src/engine/__tests__/list-available-models.test.ts index 64644599..23529201 100644 --- a/apps/api/src/engine/__tests__/list-available-models.test.ts +++ b/apps/api/src/engine/__tests__/list-available-models.test.ts @@ -867,3 +867,198 @@ describe('ClaudeCodeEngine.listAvailableModels', () => { expect(result.models).toEqual(['claude-opus-4-7', 'claude-sonnet-4-6']) }) }) + +// ============================================================ +// Reasoning-effort discovery +// +// The level set is a property of the MODEL, not of the Provider: codex +// advertises `ultra`, Claude never does; Claude Opus 4.5 has neither `xhigh` +// nor `max`; Haiku 4.5 accepts no effort at all. These tests pin that the +// levels travel with the model id rather than being inferred from the kind. +// ============================================================ +describe('reasoning effort discovery', () => { + const claude = new ClaudeCodeEngine({ + path: 'claude', + apiKey: '', + baseUrl: '', + timeoutMinutes: 5, + force: false, + approveMcps: true, + defaultWorkDir: '/tmp', + }) + const codex = new CodexAgentEngine({ + path: 'codex', + apiKey: '', + timeoutMinutes: 5, + force: false, + approveMcps: true, + defaultWorkDir: '/tmp', + }) + + let fetchSpy: ReturnType + + beforeEach(() => { + mockDnsLookup.mockReset() + mockDnsLookup.mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) + fetchSpy = vi.spyOn(globalThis, 'fetch') + }) + + afterEach(() => { + fetchSpy.mockRestore() + }) + + function mockModelsResponse(body: unknown) { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ) + } + + it('claude-code: carries the levels each model reports, and they differ per model', async () => { + mockModelsResponse({ + data: [ + { + id: 'claude-opus-4-8', + capabilities: { + effort: { + supported: true, + low: { supported: true }, + medium: { supported: true }, + high: { supported: true }, + xhigh: { supported: true }, + max: { supported: true }, + }, + }, + }, + { + id: 'claude-opus-4-5-20251101', + capabilities: { + effort: { + supported: true, + low: { supported: true }, + medium: { supported: true }, + high: { supported: true }, + xhigh: { supported: false }, + max: { supported: false }, + }, + }, + }, + ], + }) + + const result = await claude.listAvailableModels({ + authMode: 'oauth', + oauthToken: 'sk-ant-oat01-abc', + }) + + expect(result.models).toEqual(['claude-opus-4-8', 'claude-opus-4-5-20251101']) + expect(result.modelCapabilities?.['claude-opus-4-8']?.reasoningEfforts).toEqual([ + { value: 'low' }, + { value: 'medium' }, + { value: 'high' }, + { value: 'xhigh' }, + { value: 'max' }, + ]) + expect( + result.modelCapabilities?.['claude-opus-4-5-20251101']?.reasoningEfforts?.map( + (option) => option.value, + ), + ).toEqual(['low', 'medium', 'high']) + }) + + it('claude-code: reports an empty level list for a model that supports no effort', async () => { + mockModelsResponse({ + data: [{ id: 'claude-haiku-4-5-20251001', capabilities: { effort: { supported: false } } }], + }) + + const result = await claude.listAvailableModels({ + authMode: 'oauth', + oauthToken: 'sk-ant-oat01-abc', + }) + + // Empty, not absent: discovery answered the question, and the answer was "none". + expect(result.modelCapabilities?.['claude-haiku-4-5-20251001']?.reasoningEfforts).toEqual([]) + }) + + it('claude-code: leaves capabilities unknown when a proxy returns bare model ids', async () => { + mockModelsResponse({ data: [{ id: 'deepseek-v4-flash' }, { id: 'internal-model' }] }) + + const result = await claude.listAvailableModels({ + authMode: 'apiKey', + baseUrl: 'https://llm-proxy.example.com', + apiKey: 'sk-xxx', + }) + + expect(result.models).toEqual(['deepseek-v4-flash', 'internal-model']) + // Unknown must not be reported as "no levels" — the UI treats the two differently. + expect(result.modelCapabilities).toBeUndefined() + }) + + it('codex: carries levels, their descriptions and the model default', async () => { + const child = new MockChildProcess() + mockSpawn.mockReturnValue(child) + const catalog = JSON.stringify({ + models: [ + { + slug: 'gpt-5.6-sol', + visibility: 'list', + default_reasoning_level: 'low', + supported_reasoning_levels: [ + { effort: 'low', description: 'Fast responses with lighter reasoning' }, + { effort: 'ultra', description: 'Maximum reasoning with automatic task delegation' }, + ], + }, + ], + }) + + const promise = codex.listAvailableModels({ authMode: 'apiKey' }) + settle(child, catalog, 0) + const result = await promise + + expect(result.models).toEqual(['gpt-5.6-sol']) + expect(result.modelCapabilities?.['gpt-5.6-sol']).toEqual({ + reasoningEfforts: [ + { value: 'low', description: 'Fast responses with lighter reasoning' }, + { value: 'ultra', description: 'Maximum reasoning with automatic task delegation' }, + ], + defaultReasoningEffort: 'low', + }) + }) + + it('codex: omits an entry for a model that reports no reasoning metadata', async () => { + const child = new MockChildProcess() + mockSpawn.mockReturnValue(child) + const catalog = JSON.stringify({ + models: [{ slug: 'gpt-5.6-sol', visibility: 'list' }], + }) + + const promise = codex.listAvailableModels({ authMode: 'apiKey' }) + settle(child, catalog, 0) + const result = await promise + + expect(result.models).toEqual(['gpt-5.6-sol']) + expect(result.modelCapabilities).toBeUndefined() + }) + + it('codex: drops a level token that is not a plain lowercase word', async () => { + const child = new MockChildProcess() + mockSpawn.mockReturnValue(child) + const catalog = JSON.stringify({ + models: [ + { + slug: 'gpt-5.6-sol', + visibility: 'list', + supported_reasoning_levels: [{ effort: 'high' }, { effort: '--not-a-level' }], + }, + ], + }) + + const promise = codex.listAvailableModels({ authMode: 'apiKey' }) + settle(child, catalog, 0) + const result = await promise + + expect(result.modelCapabilities?.['gpt-5.6-sol']?.reasoningEfforts).toEqual([{ value: 'high' }]) + }) +}) diff --git a/apps/api/src/engine/__tests__/provider-catalog.test.ts b/apps/api/src/engine/__tests__/provider-catalog.test.ts index d27e5c99..5a0f880f 100644 --- a/apps/api/src/engine/__tests__/provider-catalog.test.ts +++ b/apps/api/src/engine/__tests__/provider-catalog.test.ts @@ -187,3 +187,30 @@ describe('evaluateProviderVersion', () => { }) }) }) + +describe('reasoning capability declaration', () => { + it('declares reasoning effort only for the CLIs that accept one', () => { + const declaring = Object.entries(BUILTIN_PROVIDER_MANIFESTS) + .filter(([, manifest]) => manifest.capabilities.reasoningEffort) + .map(([kind]) => kind) + + expect(declaring).toEqual(['claude-code', 'codex']) + }) + + it('declares fast mode only for the CLIs that accept one', () => { + const declaring = Object.entries(BUILTIN_PROVIDER_MANIFESTS) + .filter(([, manifest]) => manifest.capabilities.fastMode) + .map(([kind]) => kind) + + expect(declaring).toEqual(['claude-code', 'codex']) + }) + + it('keeps the two dimensions separate from the boolean execution options', () => { + // executionOptions drives the Agent-wide advanced switches; effort and fast + // mode belong to a single provider chain entry and must not leak into it. + for (const manifest of Object.values(BUILTIN_PROVIDER_MANIFESTS)) { + expect(manifest.capabilities.executionOptions).not.toContain('reasoningEffort') + expect(manifest.capabilities.executionOptions).not.toContain('fastMode') + } + }) +}) diff --git a/apps/api/src/engine/__tests__/reasoning-args.test.ts b/apps/api/src/engine/__tests__/reasoning-args.test.ts new file mode 100644 index 00000000..40c3737d --- /dev/null +++ b/apps/api/src/engine/__tests__/reasoning-args.test.ts @@ -0,0 +1,407 @@ +/** + * Reasoning effort and fast mode reach the CLI. + * + * Both settings are resolved from the provider binding that is actually running + * (see provider-binding-reasoning.test.ts) and end up on the command line here. + * The rule these tests pin is that an unset control passes NOTHING: the CLI's + * own default is the fallback, never a value a2wave invented. + */ +import { EventEmitter } from 'node:events' +import { PassThrough } from 'node:stream' +import { afterEach, describe, expect, it, vi } from 'vitest' + +const mockSpawn = vi.hoisted(() => vi.fn()) + +vi.mock('node:child_process', () => ({ + execFile: vi.fn(), + spawn: mockSpawn, +})) + +import { ClaudeCodeEngine } from '../claude-code.js' +import { CodexAgentEngine } from '../codex-agent.js' + +class MockChildProcess extends EventEmitter { + stdout = new PassThrough() + stderr = new PassThrough() + stdin = null + pid = 4242 + kill = vi.fn() +} + +type StreamRequest = Record + +function getExecuteStream(engine: ClaudeCodeEngine | CodexAgentEngine) { + return ( + engine as unknown as { + executeStreamWithModel: (request: StreamRequest, model: string) => Promise + } + ).executeStreamWithModel.bind(engine) +} + +function lastSpawnArgs(): string[] { + const call = mockSpawn.mock.calls.at(-1) + if (!call) throw new Error('spawn was not called') + return call[1] as string[] +} + +function finishClaudeRun(child: MockChildProcess) { + child.stdout.write( + `${JSON.stringify({ type: 'result', subtype: 'success', result: 'ok', duration_ms: 1 })}\n`, + ) + child.emit('close', 0) +} + +function finishCodexRun(child: MockChildProcess) { + child.stdout.write(`${JSON.stringify({ type: 'thread.started', thread_id: 't1' })}\n`) + child.stdout.write(`${JSON.stringify({ type: 'turn.completed' })}\n`) + child.emit('close', 0) +} + +afterEach(() => vi.clearAllMocks()) + +describe('claude-code reasoning arguments', () => { + const engineConfig = { + path: 'claude', + apiKey: '', + baseUrl: '', + timeoutMinutes: 5, + force: true, + approveMcps: true, + defaultWorkDir: '/tmp', + } + + async function runWith(agentConfig: Record, chatId?: string) { + const child = new MockChildProcess() + mockSpawn.mockReturnValue(child) + const engine = new ClaudeCodeEngine(engineConfig) + const promise = getExecuteStream(engine)( + { taskId: 't', workDir: '/tmp', prompt: 'hi', agentConfig, ...(chatId ? { chatId } : {}) }, + 'claude-opus-4-8', + ) + finishClaudeRun(child) + await promise + return lastSpawnArgs() + } + + it('passes the configured level as --effort', async () => { + const args = await runWith({ reasoningEffort: 'xhigh' }) + + expect(args).toEqual(expect.arrayContaining(['--effort', 'xhigh'])) + }) + + it('turns fast mode on through --settings, the only headless entry point', async () => { + const args = await runWith({ fastMode: true }) + + const index = args.indexOf('--settings') + expect(index).toBeGreaterThan(-1) + expect(JSON.parse(args[index + 1] as string)).toEqual({ fastMode: true }) + }) + + it('passes neither flag when neither is configured', async () => { + const args = await runWith({}) + + expect(args).not.toContain('--effort') + expect(args).not.toContain('--settings') + }) + + it('passes no --settings when fast mode is explicitly off', async () => { + // Off must mean "say nothing", not "say false": the CLI reads its own + // settings files too, and a2wave has no business overriding a user default + // it was never asked to touch. + const args = await runWith({ fastMode: false }) + + expect(args).not.toContain('--settings') + }) + + it('keeps both on a resumed session', async () => { + const args = await runWith({ reasoningEffort: 'high', fastMode: true }, 'sess_1') + + expect(args).toEqual(expect.arrayContaining(['--resume', 'sess_1'])) + expect(args).toEqual(expect.arrayContaining(['--effort', 'high'])) + expect(args).toContain('--settings') + }) +}) + +describe('codex reasoning arguments', () => { + const baseConfig = { + path: 'codex', + apiKey: '', + timeoutMinutes: 5, + force: false, + approveMcps: true, + defaultWorkDir: '/tmp', + } + + async function runWith(agentConfig: Record, chatId?: string) { + const child = new MockChildProcess() + mockSpawn.mockReturnValue(child) + const engine = new CodexAgentEngine(baseConfig) + const promise = getExecuteStream(engine)( + { taskId: 't', workDir: '/tmp', prompt: 'hi', agentConfig, ...(chatId ? { chatId } : {}) }, + 'gpt-5.6-sol', + ) + finishCodexRun(child) + await promise + return lastSpawnArgs() + } + + it('passes the configured level as a config override', async () => { + const args = await runWith({ reasoningEffort: 'ultra' }) + + expect(args).toEqual(expect.arrayContaining(['-c', 'model_reasoning_effort="ultra"'])) + }) + + it('requests the faster service tier for fast mode', async () => { + const args = await runWith({ fastMode: true }) + + expect(args).toEqual(expect.arrayContaining(['-c', 'service_tier="priority"'])) + }) + + it('passes neither override when neither is configured', async () => { + const args = await runWith({}) + + expect(args.join(' ')).not.toContain('model_reasoning_effort') + expect(args.join(' ')).not.toContain('service_tier') + }) + + it('keeps both on a resumed session', async () => { + // `-c` is accepted on `codex exec resume`, unlike --sandbox, so the settings + // must be re-passed: a resumed turn would otherwise silently drop back to + // the CLI defaults halfway through a conversation. + const args = await runWith({ reasoningEffort: 'high', fastMode: true }, 'thread_1') + + expect(args.slice(0, 3)).toEqual(['exec', 'resume', 'thread_1']) + expect(args).toEqual(expect.arrayContaining(['-c', 'model_reasoning_effort="high"'])) + expect(args).toEqual(expect.arrayContaining(['-c', 'service_tier="priority"'])) + }) +}) + +/** + * Fast mode is requested, never guaranteed: it also needs first-party Anthropic + * auth, a model that supports it and an eligible plan. The CLI settles that at + * run time and reports the outcome on its result line, so a2wave records what + * actually happened instead of leaving the switch as the only evidence. + */ +describe('claude-code fast mode outcome', () => { + const engineConfig = { + path: 'claude', + apiKey: '', + baseUrl: '', + timeoutMinutes: 5, + force: true, + approveMcps: true, + defaultWorkDir: '/tmp', + } + + async function runReporting(resultLine: Record) { + const child = new MockChildProcess() + mockSpawn.mockReturnValue(child) + const entries: Array> = [] + const engine = new ClaudeCodeEngine(engineConfig) + const promise = getExecuteStream(engine)( + { + taskId: 't', + workDir: '/tmp', + prompt: 'hi', + agentConfig: { fastMode: true }, + onLogEntry: (entry: Record) => entries.push(entry), + }, + 'claude-opus-4-8', + ) + child.stdout.write(`${JSON.stringify(resultLine)}\n`) + child.emit('close', 0) + await promise + return entries.find((entry) => entry.type === 'result') + } + + it('records that the run really got the faster path', async () => { + const result = await runReporting({ + type: 'result', + subtype: 'success', + result: 'ok', + duration_ms: 1, + fast_mode_state: 'on', + }) + + expect(result?.fastModeState).toBe('on') + }) + + it('records that it did not, which the switch alone cannot tell you', async () => { + const result = await runReporting({ + type: 'result', + subtype: 'success', + result: 'ok', + duration_ms: 1, + fast_mode_state: 'off', + }) + + expect(result?.fastModeState).toBe('off') + }) + + it('passes through a cooldown verdict verbatim rather than folding it into off', async () => { + const result = await runReporting({ + type: 'result', + subtype: 'success', + result: 'ok', + duration_ms: 1, + fast_mode_state: 'cooldown', + }) + + expect(result?.fastModeState).toBe('cooldown') + }) + + it('omits the field when the CLI reports nothing, rather than inventing "off"', async () => { + // Older CLIs and every non-Claude engine report no such state. Defaulting to + // "off" would claim a verdict nobody issued. + const result = await runReporting({ + type: 'result', + subtype: 'success', + result: 'ok', + duration_ms: 1, + }) + + expect(result).toBeDefined() + expect(result?.fastModeState).toBeUndefined() + }) + + it('ignores a value that is not one of the states the CLI defines', async () => { + const result = await runReporting({ + type: 'result', + subtype: 'success', + result: 'ok', + duration_ms: 1, + fast_mode_state: { nested: true }, + }) + + expect(result?.fastModeState).toBeUndefined() + }) +}) + +/** + * The log records what the SERVER did, not what the client asked for. + * + * `fast_mode_state` is the CLI's own intent — it flips to `on` as soon as the + * request is allowed to leave. `usage.speed` is what Anthropic actually served. + * The two disagree on exactly the case that matters: an account without usage + * credits gets `fast_mode_state: on` and `speed: standard`, no error. Recording + * the intent would put a green "Fast" marker on a run that never ran fast. + */ +describe('claude-code fast mode verdict prefers the served speed', () => { + const engineConfig = { + path: 'claude', + apiKey: '', + baseUrl: '', + timeoutMinutes: 5, + force: true, + approveMcps: true, + defaultWorkDir: '/tmp', + } + + async function stateOf(resultLine: Record) { + const child = new MockChildProcess() + mockSpawn.mockReturnValue(child) + const entries: Array> = [] + const engine = new ClaudeCodeEngine(engineConfig) + const promise = getExecuteStream(engine)( + { + taskId: 't', + workDir: '/tmp', + prompt: 'hi', + agentConfig: { fastMode: true }, + onLogEntry: (entry: Record) => entries.push(entry), + }, + 'claude-opus-4-8', + ) + child.stdout.write(`${JSON.stringify(resultLine)}\n`) + child.emit('close', 0) + await promise + return entries.find((entry) => entry.type === 'result')?.fastModeState + } + + const base = { type: 'result', subtype: 'success', result: 'ok', duration_ms: 1 } + + it('reports on only when the server actually served the faster path', async () => { + expect(await stateOf({ ...base, fast_mode_state: 'on', usage: { speed: 'fast' } })).toBe('on') + }) + + it('reports "denied" when the client asked but the server served standard', async () => { + // The real shape of an account with usage credits disabled: allowed out, + // billed and served as standard, no error anywhere. Distinct from + // "requested", which means nobody ever answered. + expect(await stateOf({ ...base, fast_mode_state: 'on', usage: { speed: 'standard' } })).toBe( + 'denied', + ) + }) + + it('reports off when nothing was asked for and nothing was served', async () => { + expect(await stateOf({ ...base, fast_mode_state: 'off', usage: { speed: 'standard' } })).toBe( + 'off', + ) + }) + + it('keeps a cooldown verdict, which the served speed cannot express', async () => { + expect( + await stateOf({ ...base, fast_mode_state: 'cooldown', usage: { speed: 'standard' } }), + ).toBe('cooldown') + }) + + it('falls back to the client verdict when the server reports no speed', async () => { + expect(await stateOf({ ...base, fast_mode_state: 'on' })).toBe('on') + }) + + it('stays absent when neither source says anything', async () => { + expect(await stateOf(base)).toBeUndefined() + }) +}) + +/** + * codex's `--json` stream reports only a thread id and token usage — no model, + * no duration, and nothing at all about the service tier. The first two are + * facts a2wave already holds and simply failed to write down; the third has no + * source and is therefore left unstated rather than guessed. + */ +describe('codex run log completeness', () => { + const baseConfig = { + path: 'codex', + apiKey: '', + timeoutMinutes: 5, + force: false, + approveMcps: true, + defaultWorkDir: '/tmp', + } + + async function entriesOf(agentConfig: Record) { + const child = new MockChildProcess() + mockSpawn.mockReturnValue(child) + const entries: Array> = [] + const engine = new CodexAgentEngine(baseConfig) + const promise = getExecuteStream(engine)( + { + taskId: 't', + workDir: '/tmp', + prompt: 'hi', + agentConfig, + onLogEntry: (entry: Record) => entries.push(entry), + }, + 'gpt-5.6-sol', + ) + finishCodexRun(child) + await promise + return entries + } + + it('records the model it ran, which the engine never reports', async () => { + const init = (await entriesOf({})).find( + (entry) => entry.type === 'system' && entry.subtype === 'init', + ) + + expect(init?.model).toBe('gpt-5.6-sol') + }) + + it('records a duration measured by the platform', async () => { + const result = (await entriesOf({})).find((entry) => entry.type === 'result') + + expect(typeof result?.durationMs).toBe('number') + expect(result?.durationMs).toBeGreaterThanOrEqual(0) + }) +}) diff --git a/apps/api/src/engine/claude-code.ts b/apps/api/src/engine/claude-code.ts index 4468deb2..4cb5794e 100644 --- a/apps/api/src/engine/claude-code.ts +++ b/apps/api/src/engine/claude-code.ts @@ -2,6 +2,7 @@ import { execFileSync } from 'node:child_process' import { existsSync, readFileSync } from 'node:fs' import { homedir } from 'node:os' import { join } from 'node:path' +import type { FastModeAvailability, ModelCapabilities, ReasoningEffortOption } from '@a2wave/shared' import { Agent as UndiciAgent } from 'undici' import { unsetEnv } from '../lib/env-utils.js' import { logger } from '../lib/logger.js' @@ -16,6 +17,7 @@ import { BaseCliAgentEngine, type CliEngineBaseConfig, stripPromptArg } from './ import { toDisplayExecParams } from './exec-params.js' import { createHeartbeatTracker } from './heartbeat.js' import { runStatusProbe, truncateForRaw } from './login-status-helper.js' +import { finalizeModelCapabilities, isReasoningEffortValue } from './model-capabilities.js' import { buildSafeAgentProcessEnv, omitRuntimeEnvKeys, @@ -54,6 +56,134 @@ function clearClaudeCredentialEnv(env: NodeJS.ProcessEnv): void { for (const key of CLAUDE_CREDENTIAL_ENV_KEYS) unsetEnv(env, key) } +/** + * Read the reasoning-effort levels one model advertises on `GET /v1/models`. + * + * Anthropic reports them as `capabilities.effort`, a `supported` flag sitting + * alongside one nested object per level. Levels are collected by walking that + * object rather than by consulting a list of level names: the names are exactly + * what must not be hard-coded here, and they already differ between models + * (Opus 4.5 has no `xhigh`, Haiku 4.5 has no effort at all). Skipping the + * `supported` flag needs no special case either — it is a boolean, not an + * object, so it fails the per-level shape check on its own. + * + * Returns `undefined` when the model reports no effort capability at all, which + * is what a proxy standing in for the vendor endpoint does. That is "unknown" + * and is deliberately distinct from the empty array returned for a model that + * says effort is unsupported. + */ +function readEffortCapability(capabilities: unknown): ReasoningEffortOption[] | undefined { + if (!capabilities || typeof capabilities !== 'object') return undefined + const effort = (capabilities as { effort?: unknown }).effort + if (!effort || typeof effort !== 'object') return undefined + if ((effort as { supported?: unknown }).supported === false) return [] + + const options: ReasoningEffortOption[] = [] + for (const [level, detail] of Object.entries(effort as Record)) { + if (!detail || typeof detail !== 'object') continue + if ((detail as { supported?: unknown }).supported !== true) continue + if (!isReasoningEffortValue(level)) continue + options.push({ value: level }) + } + return options +} + +const CLAUDE_FAST_MODE_URL = `${CLAUDE_OAUTH_BASE_URL}/api/claude_code_penguin_mode` + +/** + * Ask Anthropic whether these credentials may use fast mode. + * + * The switch alone cannot answer this: fast mode is premium usage, and an + * account can be refused for several distinct reasons the operator would + * otherwise only discover by reading a finished run. The CLI gates on the same + * endpoint, which is why a2wave can pre-empt the outcome instead of guessing + * from the model name. + * + * **Advisory only, by construction.** It is an internal CLI endpoint rather than + * a published contract, so every failure path — non-200, malformed body, network + * error, timeout — resolves to `undefined` ("not answered") and never to + * `available: false`. A vanished endpoint therefore degrades to today's + * behaviour (offer the switch, report the outcome afterwards) instead of locking + * a working feature out. + * + * Skipped entirely when the binding points at a proxy: the question belongs to + * Anthropic, and a proxy's answer — or its 404 — would say nothing about it. + */ +async function probeFastModeAvailability( + token: string, + useBearer: boolean, +): Promise { + try { + const { addresses } = await resolvePublicUrl(CLAUDE_FAST_MODE_URL, undefined, { + allowPrivateDnsAnswers: true, + }) + const dispatcher = new UndiciAgent({ connect: { lookup: createPinnedLookup(addresses) } }) + try { + const res = await safeFetch(CLAUDE_FAST_MODE_URL, { + method: 'GET', + headers: { + ...(useBearer + ? { Authorization: `Bearer ${token}`, 'anthropic-beta': 'oauth-2025-04-20' } + : { 'x-api-key': token }), + 'anthropic-version': '2023-06-01', + }, + signal: AbortSignal.timeout(8_000), + maxRedirects: 0, + dispatcher, + } as Parameters[1]) + if (!res.ok) return undefined + const body = (await res.json()) as { enabled?: unknown; disabled_reason?: unknown } + if (typeof body.enabled !== 'boolean') return undefined + return { + available: body.enabled, + ...(body.enabled || typeof body.disabled_reason !== 'string' + ? {} + : { reason: body.disabled_reason.slice(0, 64) }), + } + } finally { + await dispatcher.close().catch(() => {}) + } + } catch (err) { + logger.debug( + { err: err instanceof Error ? err.message : String(err) }, + '[claude-code] fast mode availability probe skipped', + ) + return undefined + } +} + +/** + * What actually happened to fast mode on this run. + * + * Two sources disagree, and only one of them is evidence: + * - `usage.speed` is the speed Anthropic **served** — the fact. + * - `fast_mode_state` is the CLI's own **intent**; it reads `on` as soon as the + * request is allowed to leave the client. + * + * They part company on the case that matters most. An account without usage + * credits gets `fast_mode_state: on` and `speed: standard`, with no error + * anywhere — recording the intent would mark that run "Fast" when it ran, and + * billed, at normal speed. So the served speed wins whenever it is reported, and + * `denied` — asked for, confirmed served at standard — gets its own verdict + * rather than being flattened into `on` or `off`: it is the one state the + * operator can act on (enable usage credits), and the switch alone cannot show it. + * + * The third verdict, `requested`, belongs to engines that never answer at all + * (see codex): the request went out and nothing contradicted it, which is + * strictly more than "off" and strictly less than "served". + * + * `cooldown` survives because the served speed cannot express it. + */ +export function resolveFastModeState( + servedSpeed: unknown, + claimedState: unknown, +): string | undefined { + const claimed = typeof claimedState === 'string' ? claimedState : undefined + if (typeof servedSpeed !== 'string') return claimed + if (servedSpeed === 'fast') return 'on' + return claimed === 'on' ? 'denied' : (claimed ?? 'off') +} + /** * High-frequency counter-style system events in the Claude Code CLI stream * have no diagnostic value on their own — e.g. `thinking_tokens` fires once per @@ -389,13 +519,32 @@ export class ClaudeCodeEngine extends BaseCliAgentEngine { } } - const json = (await res.json()) as { data?: Array<{ id?: string }> } - const models = (json.data ?? []) - .map((m) => m.id) - .filter((id): id is string => typeof id === 'string' && id.length > 0) + const json = (await res.json()) as { data?: Array<{ id?: string; capabilities?: unknown }> } + const entries = (json.data ?? []).filter( + (m): m is { id: string; capabilities?: unknown } => + typeof m.id === 'string' && m.id.length > 0, + ) + const models = entries.map((m) => m.id) + + const capabilitiesByModel = new Map() + for (const entry of entries) { + const efforts = readEffortCapability(entry.capabilities) + if (efforts) capabilitiesByModel.set(entry.id, { reasoningEfforts: efforts }) + } + const modelCapabilities = finalizeModelCapabilities(capabilitiesByModel) + + // Only meaningful against Anthropic's own endpoint — a proxy neither owns + // the entitlement nor can speak for it. + const fastMode = usesFixedClaudeOauthEndpoint + ? await probeFastModeAvailability(key, true) + : undefined logger.info({ url, count: models.length }, '[claude-code] listAvailableModels success') - return { models } + return { + models, + ...(modelCapabilities ? { modelCapabilities } : {}), + ...(fastMode ? { fastMode } : {}), + } } catch (err) { const message = err instanceof Error ? err.message : String(err) logger.warn({ url, err: message }, '[claude-code] listAvailableModels failed') @@ -443,6 +592,9 @@ export class ClaudeCodeEngine extends BaseCliAgentEngine { readOnly: agentConfig?.readOnly !== undefined ? Boolean(agentConfig.readOnly) : undefined, force: agentConfig?.force !== undefined ? Boolean(agentConfig.force) : undefined, approveMcps: approveMcpsOverride, + reasoningEffort: + typeof agentConfig?.reasoningEffort === 'string' ? agentConfig.reasoningEffort : undefined, + fastMode: agentConfig?.fastMode === true, }) const execEnv = this.buildEnv( agentEnv, @@ -463,6 +615,10 @@ export class ClaudeCodeEngine extends BaseCliAgentEngine { cwd: resolvedWorkDir, authMode, authHeaderStyle: authMode === 'apiKey' ? authHeaderStyle : undefined, + ...(typeof agentConfig?.reasoningEffort === 'string' + ? { reasoningEffort: agentConfig.reasoningEffort } + : {}), + ...(agentConfig?.fastMode === true ? { fastMode: true } : {}), baseUrl: resolvedBaseUrl, timeout: streamTimeoutMs, runtimeHome: request.runtimeContext?.home.dir, @@ -651,11 +807,20 @@ export class ClaudeCodeEngine extends BaseCliAgentEngine { if (resultIsError) { resultErrorText = resultText || 'Claude Code returned an error result' } + // Recorded rather than inferred, and taken from the server's answer + // rather than the client's request — see resolveFastModeState. A run + // where neither source says anything keeps the field absent, because + // defaulting to "off" would state a verdict nobody issued. + const fastModeState = resolveFastModeState( + (data.usage as { speed?: unknown } | undefined)?.speed, + data.fast_mode_state, + ) onLogEntry?.({ type: 'result', subtype: resultIsError ? 'error' : 'success', durationMs: typeof data.duration_ms === 'number' ? data.duration_ms : undefined, ...(lastUsage ? { usage: lastUsage } : {}), + ...(fastModeState ? { fastModeState } : {}), ts: Date.now(), }) break @@ -751,7 +916,13 @@ This rule only applies to explicit model/version questions. For general "who are model: string, outputFormat: 'json' | 'stream-json', chatId?: string, - extras?: { readOnly?: boolean; force?: boolean; approveMcps?: boolean }, + extras?: { + readOnly?: boolean + force?: boolean + approveMcps?: boolean + reasoningEffort?: string + fastMode?: boolean + }, ): string[] { const args = ['-p', prompt, '--output-format', outputFormat] // Claude CLI 要求: --print(-p) + --output-format stream-json 必须同时带 --verbose @@ -769,6 +940,18 @@ This rule only applies to explicit model/version questions. For general "who are } if (chatId) args.push('--resume', chatId) if (model) args.push('--model', model) + // Unset means "say nothing": the CLI's own default is the fallback, never a + // level a2wave picked. Which levels are legal belongs to the model and is + // discovered per credential, so nothing is validated here — an unsupported + // level is rejected by the CLI with the accepted set named in the error, + // which is a better answer than any table this process could keep. + if (extras?.reasoningEffort) args.push('--effort', extras.reasoningEffort) + // Fast mode has no flag of its own; the CLI reads it from settings. Passing + // it inline keeps it scoped to this run instead of writing into the user's + // settings file, and carries no credential, so it needs no masking. Whether + // the run actually gets the faster path depends on the model, the plan and + // the endpoint — the CLI reports the outcome as `fast_mode_state`. + if (extras?.fastMode) args.push('--settings', JSON.stringify({ fastMode: true })) // 模型有非空值时注入身份覆盖 prompt（修 CLI 内置过时模型清单导致 agent 答错版本） if (model?.trim()) { args.push('--append-system-prompt', ClaudeCodeEngine.buildIdentityPrompt(model)) diff --git a/apps/api/src/engine/codex-agent.ts b/apps/api/src/engine/codex-agent.ts index 3da2eaae..5807dc35 100644 --- a/apps/api/src/engine/codex-agent.ts +++ b/apps/api/src/engine/codex-agent.ts @@ -16,6 +16,7 @@ */ import { createHash } from 'node:crypto' +import type { ModelCapabilities } from '@a2wave/shared' import { unsetEnv } from '../lib/env-utils.js' import { logger } from '../lib/logger.js' import { BaseCliAgentEngine, type CliEngineBaseConfig } from './cli-engine-base.js' @@ -28,6 +29,11 @@ import { toDisplayExecParams } from './exec-params.js' import { createHeartbeatTracker } from './heartbeat.js' import { runStatusProbe, truncateForRaw } from './login-status-helper.js' import type { ResolvedMcpServer } from './mcp-sync.js' +import { + finalizeModelCapabilities, + isReasoningEffortValue, + toReasoningEffortOptions, +} from './model-capabilities.js' import { buildSafeAgentProcessEnv, omitRuntimeEnvKeys, @@ -47,6 +53,48 @@ import type { } from './types.js' import { mapCodexUsage } from './usage.js' +/** + * One entry of `codex debug models`. Only the fields a2wave reads are typed; + * the CLI reports considerably more (pricing hints, base instructions, speed + * tiers) that the platform has no business persisting. + */ +interface CodexCatalogEntry { + slug?: string + visibility?: string + default_reasoning_level?: unknown + supported_reasoning_levels?: unknown +} + +/** + * Read the reasoning levels one catalog entry advertises. + * + * codex reports them per model, with its own wording for each level, and the + * set genuinely varies — `ultra` exists for some models and not others. The + * descriptions are carried through so the picker can explain the levels in the + * CLI's own words rather than in wording a2wave invented. + * + * A model that reports neither field yields an empty object, which + * `finalizeModelCapabilities` then drops: nothing was discovered, so nothing is + * claimed. + */ +function readReasoningCapability(entry: CodexCatalogEntry): ModelCapabilities { + const capabilities: ModelCapabilities = {} + + if (Array.isArray(entry.supported_reasoning_levels)) { + capabilities.reasoningEfforts = toReasoningEffortOptions( + entry.supported_reasoning_levels.map((level) => ({ + value: (level as { effort?: unknown })?.effort, + description: (level as { description?: unknown })?.description, + })), + ) + } + if (isReasoningEffortValue(entry.default_reasoning_level)) { + capabilities.defaultReasoningEffort = entry.default_reasoning_level + } + + return capabilities +} + const ENGINE_TYPE = 'codex' const CODEX_MCP_TOOL_TIMEOUT_SEC = 660 const A2WAVE_AGENT_ROUTER_MCP_NAME = 'a2wave-agent-router' @@ -361,7 +409,7 @@ export class CodexAgentEngine extends BaseCliAgentEngine { } } - let parsed: { models?: Array<{ slug?: string; visibility?: string }> } + let parsed: { models?: CodexCatalogEntry[] } try { parsed = JSON.parse(result.stdout) } catch (err) { @@ -378,10 +426,11 @@ export class CodexAgentEngine extends BaseCliAgentEngine { } } - const models = (parsed.models ?? []) - .filter((m) => m.visibility === 'list') - .map((m) => m.slug) - .filter((slug): slug is string => typeof slug === 'string' && slug.length > 0) + const visible = (parsed.models ?? []).filter( + (m): m is CodexCatalogEntry & { slug: string } => + m.visibility === 'list' && typeof m.slug === 'string' && m.slug.length > 0, + ) + const models = visible.map((m) => m.slug) if (models.length === 0) { return { @@ -391,11 +440,17 @@ export class CodexAgentEngine extends BaseCliAgentEngine { } } + const capabilitiesByModel = new Map() + for (const entry of visible) { + capabilitiesByModel.set(entry.slug, readReasoningCapability(entry)) + } + const modelCapabilities = finalizeModelCapabilities(capabilitiesByModel) + logger.info( { count: models.length, sample: models.slice(0, 3) }, '[codex] listAvailableModels success', ) - return { models } + return modelCapabilities ? { models, modelCapabilities } : { models } } protected async executeStreamWithModel( @@ -447,12 +502,17 @@ export class CodexAgentEngine extends BaseCliAgentEngine { ) } + // codex reports no duration of its own, so the platform times the stream. + const streamStartedAt = Date.now() const promptTransport = buildCodexPromptTransport(prompt) const args = this.buildArgs(promptTransport.promptArg, model, inputChatId, { readOnly: agentConfig?.readOnly !== undefined ? Boolean(agentConfig.readOnly) : undefined, force: agentConfig?.force !== undefined ? Boolean(agentConfig.force) : undefined, mcpConfigOverride: mcpInjection?.configOverride, openaiBaseUrl: authMode === 'apiKey' ? perAgentBaseUrl : undefined, + reasoningEffort: + typeof agentConfig?.reasoningEffort === 'string' ? agentConfig.reasoningEffort : undefined, + fastMode: agentConfig?.fastMode === true, }) const execEnv = this.buildEnv(agentEnv, mcpInjection?.env, runtimeEnv, perAgentApiKey, authMode) const resolvedApiKey = perAgentApiKey || this.config.apiKey @@ -464,6 +524,14 @@ export class CodexAgentEngine extends BaseCliAgentEngine { args: filteredArgs, cwd: resolvedWorkDir, authMode, + // Reported as named fields rather than read off the argv above: every + // `-c` value is blanket-redacted in the log, and neither of these is a + // secret worth losing to that rule — knowing which level a slow run used + // is exactly what the exec params are for. + ...(typeof agentConfig?.reasoningEffort === 'string' + ? { reasoningEffort: agentConfig.reasoningEffort } + : {}), + ...(agentConfig?.fastMode === true ? { fastMode: true } : {}), proxyConfigured: authMode === 'apiKey' && Boolean(perAgentBaseUrl), mcpCount: agentConfig?.resolvedMcpServers?.length ?? 0, mcpNames: (agentConfig?.resolvedMcpServers as ResolvedMcpServer[] | undefined)?.map( @@ -520,9 +588,13 @@ export class CodexAgentEngine extends BaseCliAgentEngine { case 'session': sessionId = ev.chatId logger.info({ taskId, sessionId }, '[codex] Thread started') + // codex's JSON stream never names the model, so the run log would + // otherwise show which engine ran but not what it ran — the one + // detail that changes with every provider-chain fallback. onLogEntry?.({ type: 'system', subtype: 'init', + ...(model ? { model } : {}), ts: Date.now(), }) break @@ -535,10 +607,20 @@ export class CodexAgentEngine extends BaseCliAgentEngine { // chat turns use separate exec processes, including exec resume. lastUsage = mapCodexUsage(ev.usage) ?? lastUsage logger.info({ taskId, usage: ev.usage }, '[codex] Turn completed') + // Measured here rather than read from the stream: codex reports no + // duration, and the platform has been holding the start time all along. + // codex's stream never reports which service tier was served, but it + // does reject a tier the model does not advertise (warning, then + // dropping it). So a run that got this far with fast mode on has + // been requested AND accepted by the CLI — more than intent, less + // than confirmation. `requested` states exactly that; claiming `on` + // would assert a backend answer nobody gave. onLogEntry?.({ type: 'result', subtype: 'success', + durationMs: Date.now() - streamStartedAt, ...(lastUsage ? { usage: lastUsage } : {}), + ...(agentConfig?.fastMode === true ? { fastModeState: 'requested' } : {}), ts: Date.now(), }) break @@ -550,6 +632,7 @@ export class CodexAgentEngine extends BaseCliAgentEngine { onLogEntry?.({ type: 'result', subtype: 'error', + durationMs: Date.now() - streamStartedAt, ts: Date.now(), }) break @@ -681,6 +764,8 @@ export class CodexAgentEngine extends BaseCliAgentEngine { force?: boolean mcpConfigOverride?: string openaiBaseUrl?: string + reasoningEffort?: string + fastMode?: boolean }, ): string[] { const isResume = !!chatId @@ -696,6 +781,19 @@ export class CodexAgentEngine extends BaseCliAgentEngine { if (extras?.mcpConfigOverride) { args.push('-c', extras.mcpConfigOverride) } + // Both are re-passed on resume: unlike --sandbox, `-c` is accepted by + // `codex exec resume`, and a resumed turn that dropped them would silently + // continue the conversation on the CLI defaults. Unset passes nothing. + if (extras?.reasoningEffort) { + args.push('-c', `model_reasoning_effort=${tomlString(extras.reasoningEffort)}`) + } + // Fast mode is the `priority` service tier. codex validates the tier against + // what the selected model advertises and omits it with a warning when the + // model has no such tier, so an unsupported combination degrades to normal + // speed rather than failing the run. + if (extras?.fastMode) { + args.push('-c', `service_tier=${tomlString('priority')}`) + } // sandbox / bypass flags: resume inherits the original session policy, so // they are not re-passed diff --git a/apps/api/src/engine/model-capabilities.ts b/apps/api/src/engine/model-capabilities.ts new file mode 100644 index 00000000..67486eb4 --- /dev/null +++ b/apps/api/src/engine/model-capabilities.ts @@ -0,0 +1,65 @@ +/** + * Shared helpers for the per-model metadata that model discovery returns + * alongside the model ids. + * + * a2wave deliberately keeps no model catalog: which reasoning-effort levels a + * model accepts is asked of the CLI (or the vendor endpoint) per credential and + * never written down, so it cannot drift from what the account can really run. + * These helpers exist so every engine reports that answer in one shape, and so + * one distinction is preserved everywhere: **absent means "not discovered", + * empty means "discovered, and there are none"**. Collapsing the two would make + * a proxy that returns bare model ids look identical to a model that genuinely + * takes no effort setting, and the UI has to tell them apart. + */ +import type { ModelCapabilities, ReasoningEffortOption } from '@a2wave/shared' +import { reasoningEffortValueSchema } from '@a2wave/shared' + +/** + * Whether a token is shaped like a level a CLI could be handed as an argument. + * + * Discovery output is third-party text, and the value ends up in an argv entry, + * so anything that is not a plain lowercase word is dropped rather than + * forwarded. This validates the shape only — the legal SET is whatever the + * source reported, which is the whole point of discovering it. + */ +export function isReasoningEffortValue(value: unknown): value is string { + return reasoningEffortValueSchema.safeParse(value).success +} + +/** Build the option list for one model, dropping tokens that fail the shape check. */ +export function toReasoningEffortOptions( + levels: Array<{ value: unknown; description?: unknown }>, +): ReasoningEffortOption[] { + const options: ReasoningEffortOption[] = [] + for (const level of levels) { + if (!isReasoningEffortValue(level.value)) continue + options.push( + typeof level.description === 'string' && level.description + ? { value: level.value, description: level.description.slice(0, 200) } + : { value: level.value }, + ) + } + return options +} + +/** A model entry only earns a place in the response if it actually reports something. */ +function saysSomething(capabilities: ModelCapabilities): boolean { + return ( + capabilities.reasoningEfforts !== undefined || capabilities.defaultReasoningEffort !== undefined + ) +} + +/** + * Collapse the per-model entries into the response field, or `undefined` when + * discovery learned nothing about any model — the "unknown" signal the UI needs + * in order to disable the control and say why instead of offering an empty list. + */ +export function finalizeModelCapabilities( + entries: Map, +): Record | undefined { + const result: Record = {} + for (const [modelId, capabilities] of entries) { + if (saysSomething(capabilities)) result[modelId] = capabilities + } + return Object.keys(result).length > 0 ? result : undefined +} diff --git a/apps/api/src/engine/provider-catalog.ts b/apps/api/src/engine/provider-catalog.ts index b61523a8..edd5219e 100644 --- a/apps/api/src/engine/provider-catalog.ts +++ b/apps/api/src/engine/provider-catalog.ts @@ -218,6 +218,8 @@ export const BUILTIN_PROVIDER_MANIFESTS = { credentialFields: { apiKey: [required('apiKey')] }, mcpDelivery: { mode: 'workspace-file', defaultPath: '.cursor/mcp.json' }, executionOptions: ['readOnly', 'force'], + reasoningEffort: false, + fastMode: false, sessionResume: true, sandbox: 'cli-controlled', localSessionLoginCommand: 'cursor-agent login', @@ -233,6 +235,13 @@ export const BUILTIN_PROVIDER_MANIFESTS = { }, mcpDelivery: { mode: 'workspace-file', defaultPath: '.mcp.json' }, executionOptions: ['readOnly', 'force', 'approveMcps'], + // `--effort` takes a level, and which levels are legal is a property of the + // model, so the values are probed rather than declared. Fast mode is the + // opposite: a plain switch (`--settings {"fastMode":true}`) with nothing to + // discover — whether a run really gets the faster path depends on the model, + // the plan and the endpoint, and the run reports that itself. + reasoningEffort: true, + fastMode: true, sessionResume: true, // Native OS-level sandbox: macOS Seatbelt / Linux bubblewrap; the platform can // force it on non-bypassably via managed settings. @@ -247,6 +256,12 @@ export const BUILTIN_PROVIDER_MANIFESTS = { credentialFields: { apiKey: [required('apiKey'), optional('baseUrl')] }, mcpDelivery: { mode: 'runtime-injection' }, executionOptions: ['readOnly', 'force', 'cleanResult'], + // `-c model_reasoning_effort=`; the levels come from `codex debug + // models`, which reports a different set per model. Fast mode is the + // `priority` service tier — a switch, but one codex silently omits (with a + // warning) when the selected model does not advertise it. + reasoningEffort: true, + fastMode: true, sessionResume: true, sandbox: 'native', localSessionLoginCommand: 'codex login', @@ -259,6 +274,8 @@ export const BUILTIN_PROVIDER_MANIFESTS = { credentialFields: {}, mcpDelivery: { mode: 'runtime-injection' }, executionOptions: [], + reasoningEffort: false, + fastMode: false, sessionResume: true, // No OS-level sandbox — only an in-process tool-approval gate, no filesystem // or network isolation. @@ -272,6 +289,8 @@ export const BUILTIN_PROVIDER_MANIFESTS = { credentialFields: { apiKey: [required('apiKey')] }, mcpDelivery: { mode: 'workspace-file', defaultPath: '.mcp.json' }, executionOptions: ['readOnly', 'force', 'approveMcps'], + reasoningEffort: false, + fastMode: false, sessionResume: true, sandbox: 'cli-controlled', localSessionLoginCommand: 'qodercli login', @@ -284,6 +303,8 @@ export const BUILTIN_PROVIDER_MANIFESTS = { credentialFields: { apiKey: [required('apiKey'), optional('baseUrl')] }, mcpDelivery: { mode: 'workspace-file', defaultPath: '.trae/mcp.json' }, executionOptions: ['readOnly', 'force', 'approveMcps'], + reasoningEffort: false, + fastMode: false, sessionResume: true, sandbox: 'cli-controlled', localSessionLoginCommand: 'traecli', @@ -301,6 +322,8 @@ export const BUILTIN_PROVIDER_MANIFESTS = { // `-p` rejects --yolo/--auto/--plan and always runs under the `auto` // permission policy, so readOnly/force/approveMcps have no valid flag. executionOptions: [], + reasoningEffort: false, + fastMode: false, sessionResume: true, // No OS-level sandbox — only an in-process approval gate, which `-p` // resolves automatically. @@ -320,6 +343,8 @@ export const BUILTIN_PROVIDER_MANIFESTS = { // one, but silently installing code is outside the Provider contract. mcpDelivery: { mode: 'none' }, executionOptions: ['readOnly'], + reasoningEffort: false, + fastMode: false, sessionResume: true, sandbox: 'unsupported', localSessionLoginCommand: 'pi', diff --git a/apps/api/src/engine/types.ts b/apps/api/src/engine/types.ts index 105dc862..de7e85b4 100644 --- a/apps/api/src/engine/types.ts +++ b/apps/api/src/engine/types.ts @@ -115,7 +115,21 @@ export type StreamLogEntry = ts: number } | { type: 'tool_heartbeat'; callId: string; toolName: string; elapsedMs: number; ts: number } - | { type: 'result'; subtype: string; durationMs?: number; usage?: TokenUsage; ts: number } + | { + type: 'result' + subtype: string + durationMs?: number + usage?: TokenUsage + /** + * Whether the run actually got the faster path, as reported by the CLI — + * `on` / `off` / `cooldown`. Fast mode is only ever *requested*: the model, + * the account plan and the endpoint all have a veto, and the CLI settles it + * at run time. Absent when the engine reports nothing, which is not the + * same as `off`. + */ + fastModeState?: string + ts: number + } | { type: 'error'; message: string; ts: number } | { type: 'retry'; attempt: number; nextAttemptIn: number; ts: number } | { type: 'exec_params'; engine: string; params: Record; ts: number } @@ -207,6 +221,20 @@ export interface ListModelsOptions { export interface ModelListResult { /** Available model ids retrieved (populated on success, empty array on failure) */ models: string[] + /** + * Per-model metadata, keyed by model id — today the reasoning-effort levels + * that model accepts and its default. Absent when the source reported none + * (a proxy standing in for the vendor endpoint usually returns bare ids); + * an entry with an EMPTY level list is the different, positive answer "this + * model takes no effort setting". + */ + modelCapabilities?: Record + /** + * Whether these credentials may use fast mode. Absent when the engine cannot + * ask, or the answer did not arrive — never inferred, and never a reason to + * block the control. + */ + fastMode?: import('@a2wave/shared').FastModeAvailability /** Failure signal — a non-empty value in either field means failure */ error?: string /** Failure code: 'unsupported_mode' / 'http_error' / 'no_account_models' / 'cli_failed' / 'timeout' / 'spawn_failed' / 'parse_failed' */ diff --git a/apps/api/src/lib/__tests__/agent-execution-diagnose.test.ts b/apps/api/src/lib/__tests__/agent-execution-diagnose.test.ts index a5554724..5f830b4a 100644 --- a/apps/api/src/lib/__tests__/agent-execution-diagnose.test.ts +++ b/apps/api/src/lib/__tests__/agent-execution-diagnose.test.ts @@ -398,3 +398,74 @@ describe('collectAgentExecutionChecks', () => { expect(checks.some((c) => c.id === 'provider_auth_mode_unsupported')).toBe(true) }) }) + +describe('reasoning controls bound to a Provider that cannot use them', () => { + /** + * The web form clears both controls when the Provider of a chain entry + * changes, but an imported Agent or a direct API write can still leave a level + * attached to a CLI that has no such flag. It is silently dropped at run time, + * so diagnose is the only place the operator would ever learn about it. + */ + it('warns when a reasoning level is configured on a Provider without the setting', async () => { + mockBuildAgentConfig.mockReturnValue({ + engineType: 'cursor', + model: 'composer-1', + reasoningEffort: 'xhigh', + }) + mockProviderGet.mockReturnValue({ id: 'prv_1', kind: 'cursor', name: 'Cursor CLI' }) + + const checks = await collectAgentExecutionChecks( + row({ id: 'a1', providerId: 'prv_1', type: 'cursor', providerApiKey: 'k' }), + ) + + const check = checks.find((c) => c.id === 'provider_reasoning_effort_unsupported') + expect(check?.severity).toBe('warn') + expect(check?.message).toContain('xhigh') + }) + + it('warns when fast mode is on for a Provider that has no fast mode', async () => { + mockBuildAgentConfig.mockReturnValue({ + engineType: 'cursor', + model: 'composer-1', + fastMode: true, + }) + mockProviderGet.mockReturnValue({ id: 'prv_1', kind: 'cursor', name: 'Cursor CLI' }) + + const checks = await collectAgentExecutionChecks( + row({ id: 'a1', providerId: 'prv_1', type: 'cursor', providerApiKey: 'k' }), + ) + + expect( + checks.some((c) => c.id === 'provider_fast_mode_unsupported' && c.severity === 'warn'), + ).toBe(true) + }) + + it('stays quiet for a Provider that does support both', async () => { + mockBuildAgentConfig.mockReturnValue({ + engineType: 'claude-code', + model: 'claude-opus-4-8', + reasoningEffort: 'xhigh', + fastMode: true, + }) + mockProviderGet.mockReturnValue({ id: 'prv_1', kind: 'claude-code', name: 'Claude Code' }) + + const checks = await collectAgentExecutionChecks( + row({ id: 'a1', providerId: 'prv_1', type: 'cursor', providerApiKey: 'k' }), + ) + + expect(checks.some((c) => c.id === 'provider_reasoning_effort_unsupported')).toBe(false) + expect(checks.some((c) => c.id === 'provider_fast_mode_unsupported')).toBe(false) + }) + + it('stays quiet when neither control is configured', async () => { + mockBuildAgentConfig.mockReturnValue({ engineType: 'cursor', model: 'composer-1' }) + mockProviderGet.mockReturnValue({ id: 'prv_1', kind: 'cursor', name: 'Cursor CLI' }) + + const checks = await collectAgentExecutionChecks( + row({ id: 'a1', providerId: 'prv_1', type: 'cursor', providerApiKey: 'k' }), + ) + + expect(checks.some((c) => c.id === 'provider_reasoning_effort_unsupported')).toBe(false) + expect(checks.some((c) => c.id === 'provider_fast_mode_unsupported')).toBe(false) + }) +}) diff --git a/apps/api/src/lib/__tests__/cli-installer.test.ts b/apps/api/src/lib/__tests__/cli-installer.test.ts index 4fa6c519..77d11f55 100644 --- a/apps/api/src/lib/__tests__/cli-installer.test.ts +++ b/apps/api/src/lib/__tests__/cli-installer.test.ts @@ -280,7 +280,7 @@ describe('install state', () => { * stale. */ describe('minimum version floor', () => { - // qoder's preset floor is 1.0.0; claude-code declares none, and codegraph is + // qoder's preset floor is 1.0.0; codex declares none, and codegraph is // a non-Provider tool that has no preset at all. const FLOOR_LOCK = { providers: [ @@ -298,6 +298,20 @@ describe('install state', () => { allowScripts: true, }, }, + { + kind: 'codex', + version: '0.144.5', + binary: 'codex', + versionArgs: ['--version'], + expectedVersionOutput: '0.144.5', + install: { + type: 'npm' as const, + package: '@openai/codex', + tarball: 'https://registry.npmjs.org/@openai/codex/-/codex-0.144.5.tgz', + integrity: 'sha512-test', + allowScripts: false, + }, + }, ...LOCK.providers, ], tools: LOCK.tools, @@ -359,7 +373,7 @@ describe('install state', () => { }) it('reports no floor for a Provider that declares none', async () => { - const state = await stateOf('claude-code', '2.0.1') + const state = await stateOf('codex', '0.144.5') expect(state.minVersion).toBeNull() expect(state.meetsMinimum).toBeNull() diff --git a/apps/api/src/lib/__tests__/evaluation-snapshot.test.ts b/apps/api/src/lib/__tests__/evaluation-snapshot.test.ts index 70b12d72..94c51cdf 100644 --- a/apps/api/src/lib/__tests__/evaluation-snapshot.test.ts +++ b/apps/api/src/lib/__tests__/evaluation-snapshot.test.ts @@ -97,7 +97,7 @@ describe('buildEvaluationSnapshot', () => { expect(snapshot).not.toHaveProperty('providerChain') }) - it('exposes exactly the five allowlisted keys and nothing else', async () => { + it('exposes exactly the allowlisted keys and nothing else', async () => { const snapshot = await snapshotOf(createTestAgent(), { providerId: 'prv_1', providerName: 'Claude Code', @@ -110,9 +110,11 @@ describe('buildEvaluationSnapshot', () => { expect(Object.keys(snapshot).sort()).toEqual([ 'capturedAt', + 'fastMode', 'model', 'providerId', 'providerName', + 'reasoningEffort', 'systemPrompt', ]) expect(JSON.stringify(snapshot)).not.toContain('leak-me') @@ -158,9 +160,11 @@ describe('buildStoredEvaluationSnapshot', () => { expect(JSON.stringify(stored)).not.toContain('sk-super-secret') expect(Object.keys(stored).sort()).toEqual([ 'capturedAt', + 'fastMode', 'model', 'providerId', 'providerName', + 'reasoningEffort', 'systemPrompt', ]) }) @@ -241,3 +245,104 @@ describe('applyEvaluationSnapshot', () => { expect(config.model).toBe('old-model') }) }) + +describe('reasoning controls in the snapshot', () => { + beforeEach(() => { + buildAgentConfigMock.mockReset() + }) + + it('captures the effort and fast mode the run will actually use', async () => { + const snapshot = await snapshotOf(createTestAgent(), { + providerId: 'prv_1', + providerName: 'Claude Code', + model: 'claude-opus-4-8', + systemPrompt: '', + reasoningEffort: 'xhigh', + fastMode: true, + }) + + expect(snapshot.reasoningEffort).toBe('xhigh') + expect(snapshot.fastMode).toBe(true) + }) + + it('records "not configured" rather than inventing a level', async () => { + const snapshot = await snapshotOf(createTestAgent(), { + providerId: 'prv_1', + providerName: 'Claude Code', + model: 'claude-opus-4-8', + systemPrompt: '', + }) + + expect(snapshot.reasoningEffort).toBeNull() + expect(snapshot.fastMode).toBeNull() + }) + + it('restores the captured effort over an Agent edited after the task was created', async () => { + // Effort changes what a run costs and how it answers. Replaying a set at a + // different level and filing the results under the same task would publish a + // comparison whose variables silently moved. + const agent = createTestAgent() + const live = { + providerId: 'prv_1', + providerName: 'Claude Code', + model: 'claude-opus-4-8', + reasoningEffort: 'low', + providerChain: [ + { + providerId: 'prv_1', + providerName: 'Claude Code', + engineType: 'claude-code', + model: 'claude-opus-4-8', + reasoningEffort: 'low', + }, + ], + } + + const config = applyEvaluationSnapshot( + live as never, + { + providerId: 'prv_1', + model: 'claude-opus-4-8', + systemPrompt: '', + reasoningEffort: 'xhigh', + fastMode: true, + } as never, + agent as never, + ) + + expect(config.reasoningEffort).toBe('xhigh') + expect(config.fastMode).toBe(true) + // executeWithRetry re-reads providerChain and reapplies its first entry, so + // pinning only the top level would be undone before the first turn runs. + const chain = config.providerChain as Array> + expect(chain[0]?.reasoningEffort).toBe('xhigh') + expect(chain[0]?.fastMode).toBe(true) + }) + + it('leaves a task created before these fields existed on the live configuration', async () => { + const agent = createTestAgent() + const live = { + providerId: 'prv_1', + providerName: 'Claude Code', + model: 'claude-opus-4-8', + reasoningEffort: 'high', + providerChain: [ + { + providerId: 'prv_1', + providerName: 'Claude Code', + engineType: 'claude-code', + model: 'claude-opus-4-8', + reasoningEffort: 'high', + }, + ], + } + + const config = applyEvaluationSnapshot( + live as never, + { providerId: 'prv_1', model: 'claude-opus-4-8', systemPrompt: '' } as never, + agent as never, + ) + + expect(config.reasoningEffort).toBe('high') + }) +}) diff --git a/apps/api/src/lib/__tests__/provider-binding-reasoning.test.ts b/apps/api/src/lib/__tests__/provider-binding-reasoning.test.ts new file mode 100644 index 00000000..bea65498 --- /dev/null +++ b/apps/api/src/lib/__tests__/provider-binding-reasoning.test.ts @@ -0,0 +1,90 @@ +/** + * Reasoning effort and fast mode travel with the provider binding. + * + * `applyProviderBinding` is the single switch point for both provider fallback + * (execute-with-retry) and evaluation snapshot restore, so whatever it fails to + * carry — or fails to clear — is what a run silently executes with. These tests + * pin both directions. + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('../../db/client.js', () => ({ db: {} })) +vi.mock('../../env.js', () => ({ env: {} })) + +import { + type AgentConfig, + type ResolvedProviderBinding, + applyProviderBinding, + clearProviderBinding, +} from '../agent-helpers.js' + +function binding(overrides: Partial = {}): ResolvedProviderBinding { + return { + id: 'pc_1', + providerId: 'prv_claude', + providerName: 'Claude Code', + providerKind: 'claude-code', + engineType: 'claude-code', + authMode: 'apiKey', + mcpDelivery: { mode: 'workspace-file', defaultPath: '.mcp.json' }, + ...overrides, + } +} + +describe('provider binding reasoning controls', () => { + it('carries both controls onto the config the engine reads', () => { + const config: AgentConfig = {} + + applyProviderBinding( + config, + binding({ model: 'claude-opus-4-8', reasoningEffort: 'xhigh', fastMode: true }), + ) + + expect(config.model).toBe('claude-opus-4-8') + expect(config.reasoningEffort).toBe('xhigh') + expect(config.fastMode).toBe(true) + }) + + it('leaves both unset when the binding configures neither', () => { + const config: AgentConfig = {} + + applyProviderBinding(config, binding({ model: 'claude-opus-4-8' })) + + expect(config.reasoningEffort).toBeUndefined() + expect(config.fastMode).toBeUndefined() + }) + + it('does not carry the previous entry’s effort across a provider fallback', () => { + const config: AgentConfig = {} + + applyProviderBinding( + config, + binding({ model: 'gpt-5.6-sol', providerKind: 'codex', reasoningEffort: 'ultra' }), + ) + // `ultra` exists for codex and for no Claude model. Leaking it onto the next + // binding would hand the Claude CLI a level it rejects, turning a graceful + // fallback into a hard failure on the entry that was supposed to rescue the run. + applyProviderBinding(config, binding({ model: 'claude-opus-4-8' })) + + expect(config.reasoningEffort).toBeUndefined() + expect(config.model).toBe('claude-opus-4-8') + }) + + it('does not leave fast mode on after falling back to a binding that never asked for it', () => { + const config: AgentConfig = {} + + applyProviderBinding(config, binding({ model: 'claude-opus-4-8', fastMode: true })) + applyProviderBinding(config, binding({ providerKind: 'cursor', engineType: 'cursor' })) + + expect(config.fastMode).toBeUndefined() + }) + + it('clears both when the binding is cleared', () => { + const config: AgentConfig = { reasoningEffort: 'high', fastMode: true } + + clearProviderBinding(config) + + expect(config.reasoningEffort).toBeUndefined() + expect(config.fastMode).toBeUndefined() + }) +}) diff --git a/apps/api/src/lib/agent-execution-diagnose.ts b/apps/api/src/lib/agent-execution-diagnose.ts index 64ea1917..d3de73c8 100644 --- a/apps/api/src/lib/agent-execution-diagnose.ts +++ b/apps/api/src/lib/agent-execution-diagnose.ts @@ -146,6 +146,31 @@ export async function collectAgentExecutionChecks(agent: AgentRow): Promise + | (Pick & + Partial>) | null | undefined, agent?: AgentRow, @@ -105,10 +120,24 @@ export function applyEvaluationSnapshot( // snapshot provider's endpoint would disclose it to the wrong host. applyProviderBinding(config, pinnedChain[0]) - // After applyProviderBinding, which sets model from the binding. + // After applyProviderBinding, which sets these from the binding. + // + // A snapshot value of null falls back to the live binding, exactly as the + // model does: null cannot distinguish "captured while unset" from "captured + // before this field existed", and inheriting the live value is the reading + // that keeps pre-existing tasks running unchanged. const model = snapshot.model ?? pinnedChain[0].model + const reasoningEffort = snapshot.reasoningEffort ?? pinnedChain[0].reasoningEffort + const fastMode = snapshot.fastMode ?? pinnedChain[0].fastMode if (model) config.model = model - config.providerChain = pinnedChain.map((item) => ({ ...item, model: model ?? item.model })) + if (reasoningEffort) config.reasoningEffort = reasoningEffort + if (fastMode !== undefined) config.fastMode = fastMode + config.providerChain = pinnedChain.map((item) => ({ + ...item, + model: model ?? item.model, + reasoningEffort: reasoningEffort ?? item.reasoningEffort, + fastMode: fastMode ?? item.fastMode, + })) } else if (snapshot.providerId && snapshot.providerId !== liveConfig.providerId) { // The snapshot provider is no longer bound to this Agent — it was unbound // or disabled while the task sat in the queue. @@ -148,6 +177,8 @@ export async function buildEvaluationSnapshot(agent: AgentRow): Promise { providerId: 'prv_1', providerName: 'Claude Code', model: 'claude-opus-4-8', + reasoningEffort: null, + fastMode: null, systemPrompt: 'You are helpful.', capturedAt: '2026-07-20T00:00:00.000Z', } diff --git a/apps/web/src/components/stream-log-item.tsx b/apps/web/src/components/stream-log-item.tsx index ed8621a9..3c7d6150 100644 --- a/apps/web/src/components/stream-log-item.tsx +++ b/apps/web/src/components/stream-log-item.tsx @@ -265,6 +265,19 @@ export function StreamLogItem({ entry, baseTs }: { entry: StreamLogEntry; baseTs {formatTokens(entry.usage.outputTokens)} )} + {/* Only rendered when the engine reported a verdict. Fast mode is + requested, never guaranteed — the model, the plan and the endpoint + each get a veto — so this is the only place the operator learns + whether the switch actually did anything. */} + {entry.fastModeState && ( + + {t(`runLog.fastMode.${entry.fastModeState}`, { + defaultValue: t('runLog.fastMode.unknown', { state: entry.fastModeState }), + })} + + )} ) @@ -354,12 +367,35 @@ export function StreamLogsTimeline({ const durationLabel = durationMs != null ? `${(durationMs / 1000).toFixed(1)}s` : undefined const model = logs.find((e) => e.type === 'system' && e.subtype === 'init' && 'model' in e) const modelLabel = model && 'model' in model ? (model as { model?: string }).model : undefined + // Both markers sit next to the model they belong to, and both are bare + // tokens: the summary is a one-line scan, not a report. The level is rendered + // verbatim — it is the CLI's own vocabulary, so a label in front of it adds + // width without adding meaning. Fast mode is marked only when it actually + // applied: "off" is the norm and would be noise on every run, and the full + // verdict (off, cooldown) stays on the result row inside. + const execParams = logs.find((e) => e.type === 'exec_params') + const effortLevel = + execParams && 'params' in execParams && typeof execParams.params.reasoningEffort === 'string' + ? execParams.params.reasoningEffort + : undefined + // Marked unless there is positive evidence it did not happen. Engines differ + // in how much they admit — Claude reports the served speed, codex reports + // nothing about the tier at all — and surfacing that gap as a different + // marker made an ordinary codex run look broken. Only `denied`, where the + // engine explicitly said it served standard, drops the marker; the result row + // inside carries the exact wording either way. + const fastState = + resultEntry && 'fastModeState' in resultEntry ? resultEntry.fastModeState : undefined + const fastMarker = + fastState === 'on' || fastState === 'requested' ? t('runLog.fastSummary') : null const parts = [ toolCalls > 0 ? t('runLog.toolCallsSummary', { count: toolCalls }) : null, messages > 0 ? t('runLog.messagesSummary', { count: messages }) : null, durationLabel, modelLabel, + effortLevel, + fastMarker, ].filter(Boolean) return ( diff --git a/apps/web/src/content/manual/en/04-agents.md b/apps/web/src/content/manual/en/04-agents.md index 39b546a4..3d85e704 100644 --- a/apps/web/src/content/manual/en/04-agents.md +++ b/apps/web/src/content/manual/en/04-agents.md @@ -17,6 +17,7 @@ On the Agent detail page you can configure: - **Name / Icon / Description**: identifying information. - **System Prompt**: the Agent's core persona and rules, supporting Mustache variables (e.g. `{{message}}`). - **Provider and model**: choose the execution engine and specific model; supports a **Provider chain** (primary + fallback). +- **Reasoning effort and fast mode**: supported by some engines (Claude Code, Codex) and configured **beside each chain entry's model** — see "Reasoning Effort and Fast Mode" below. - **Credential mode (authMode)**: - `apiKey`: injects an API Key (e.g. `ANTHROPIC_API_KEY`). - `oauth`: injects an OAuth Token (`CLAUDE_CODE_OAUTH_TOKEN`, Claude Code only). @@ -30,6 +31,27 @@ On the Agent detail page you can configure: - **Evaluation**: verify config changes and compare models, see [Evaluation](/wiki/evaluation). - **Publish channels and triggers**: see [Trigger Methods](/wiki/triggers). +## Reasoning Effort and Fast Mode + +Some execution engines — currently **Claude Code** and **Codex** — let you tune how deeply the model thinks and how fast it answers. Both controls sit **beside the model of each Provider chain entry**, not once per Agent: a chain can mix engines and models, and each model offers different levels. + +### Reasoning effort + +A higher level means deeper reasoning, at more time and cost. **The available levels come from the selected model**, not from a list the platform maintains: a2wave fetches them together with the model list, so what you see is what these credentials can really use with that model. + +- **Leave it empty** to use the CLI's own default. +- Switching models refreshes the levels. A level the new model also offers is kept as is; one it does not offer falls back to **that model's own default** (or to empty, meaning the CLI default, when discovery reports no default). +- Some models (lightweight ones, typically) accept no level at all, and the field says so. + +> [!NOTE] +> Behind a self-hosted proxy the model endpoint usually returns model names only, with no level list. The field is then disabled and says no level information was discovered. Leaving it empty is fine — the run uses the CLI default — and a level configured earlier is not lost. + +### Fast mode + +Runs at a higher output speed, usually at premium pricing. It is a plain switch with no levels. + +**Turning it on does not by itself guarantee it applies**: that also depends on the model, the account plan and how the engine is reached (a third-party proxy typically cannot enable it). When a condition is unmet the run simply proceeds at normal speed without failing. To see whether a given run actually used it, check the execution parameters in its [run record](/wiki/runs). + ## Creating an Agent 1. Go to the "Agents" page and click **New Agent**. diff --git a/apps/web/src/content/manual/en/05-evaluation.md b/apps/web/src/content/manual/en/05-evaluation.md index dcc139db..a007d873 100644 --- a/apps/web/src/content/manual/en/05-evaluation.md +++ b/apps/web/src/content/manual/en/05-evaluation.md @@ -51,7 +51,7 @@ A task can be **cancelled** while it runs: a queued task is cancelled immediatel > If the service restarts mid-evaluation, the interrupted task is marked **Failed** with the reason "Interrupted by a server restart" rather than being left stuck on "Running". Just start it again — finished tasks and recorded verdicts are unaffected. > [!NOTE] -> A task freezes the **provider, model and prompt** in use when it was created and runs against that snapshot, so editing the Agent while a task is queued cannot change what it measures. If the snapshotted provider is removed from the Agent before the task starts, the task **fails** and names the missing provider — rather than quietly running on a different one and filing the results under the original. +> A task freezes the **provider, model, reasoning effort / fast mode and prompt** in use when it was created and runs against that snapshot, so editing the Agent while a task is queued cannot change what it measures. If the snapshotted provider is removed from the Agent before the task starts, the task **fails** and names the missing provider — rather than quietly running on a different one and filing the results under the original. > [!NOTE] > Evaluation uses the Agent's **currently saved** config — publishing is not required. If you just edited the config, save it before running. Evaluation runs do not appear in [Run History](/wiki/runs) and are excluded from statistics and leaderboards. @@ -70,6 +70,7 @@ This is where evaluation pays off. Every task freezes a **config snapshot** reco - **Provider** (execution engine) - **Model** +- **Reasoning effort and fast mode** (where the Provider supports them) - **System prompt** Expand "Config snapshot" in the task detail to view it. The task list is ordered newest first, so comparing pass rates across tasks shows which configuration performs best. @@ -86,7 +87,7 @@ When a task's config differs from the previous one, the list flags **model chang > These flags are what let you read a score change correctly. Above, "prompt v2" dropped because the prompt changed, not because the model regressed — without the flag it's easy to blame the wrong thing. > [!NOTE] -> The snapshot records only provider, model and prompt. **No credentials are ever stored** (API keys, OAuth tokens and the like never reach the snapshot). Mounted Skills, MCP servers and knowledge bases are outside its scope. +> The snapshot records only provider, model, reasoning effort / fast mode and prompt. **No credentials are ever stored** (API keys, OAuth tokens and the like never reach the snapshot). Mounted Skills, MCP servers and knowledge bases are outside its scope. ## Permissions diff --git a/apps/web/src/content/manual/zh/04-agents.md b/apps/web/src/content/manual/zh/04-agents.md index 23fbf8a0..b1b83a56 100644 --- a/apps/web/src/content/manual/zh/04-agents.md +++ b/apps/web/src/content/manual/zh/04-agents.md @@ -17,6 +17,7 @@ Agent 是 a2wave 的核心编排单元。一个 Agent = **系统提示词 + 执 - **名称 / 图标 / 描述**：标识信息。 - **系统提示词（System Prompt）**：Agent 的核心人设与规则，支持 Mustache 变量（如 `{{message}}`）。 - **Provider 与模型**：选择执行引擎与具体模型；支持 **Provider 链**（主用 + 回退），最多 5 个，详见下方「Provider 链与失败切换」。 +- **推理档位与快速模式**：部分引擎（Claude Code、Codex）支持，配置在**每一档 Provider 的模型旁边**，详见下方「推理档位与快速模式」。 - **凭证模式（authMode）**： - `apiKey`：注入 API Key（如 `ANTHROPIC_API_KEY`）。 - `oauth`：注入 OAuth Token（`CLAUDE_CODE_OAUTH_TOKEN`，仅 Claude Code）。 @@ -32,6 +33,27 @@ Agent 是 a2wave 的核心编排单元。一个 Agent = **系统提示词 + 执 - **评测**：改完配置后验证效果、对比不同模型，见 [评测](/wiki/evaluation)。 - **发布渠道与触发**：见 [触发方式](/wiki/triggers)。 +## 推理档位与快速模式 + +部分执行引擎（目前是 **Claude Code** 和 **Codex**）允许调节「思考深度」和「输出速度」。这两个开关**跟着模型走**，配置在 Provider 链里每一档的模型下方，而不是整个 Agent 一份——因为链上可以混用不同引擎和模型，各自能选的档位不一样。 + +### 推理档位 + +档位越高，模型思考越深入，耗时和费用也越高。**可选档位由所选模型决定**，不是平台写死的：a2wave 在拉取模型列表时一并把该模型支持的档位取回来，所以你看到的选项就是这套凭证下这个模型真正能用的。 + +- **留空**表示不指定，直接用 CLI 自己的默认档位。 +- 换模型后档位选项会跟着变。原来选的档位新模型也有就保持不变；新模型没有这一档时，自动回落到**该模型的默认档位**（拉取不到默认值时留空，运行时即 CLI 默认）。 +- 有些模型（例如轻量级模型）本身就不支持推理档位，此时会提示「该模型不支持推理档位」。 + +> [!NOTE] +> 如果你用的是自建代理地址，代理通常只返回模型名、不返回档位清单，此时选择框会置灰并提示「未拉取到档位信息」。留空即可，运行时会使用 CLI 的默认档位；已经配好的档位不会因此丢失。 + +### 快速模式 + +以更高的输出速度运行，通常按更高的价格计费。它是一个纯开关，没有档位可选。 + +**是否真正生效不由这个开关单独决定**：还取决于所用模型、账号套餐以及接入方式（例如经过第三方代理时通常无法启用）。开关打开后如果条件不满足，本次执行会以正常速度运行，不会报错。想确认某次执行到底有没有用上，看该次[运行记录](/wiki/runs)里的执行参数。 + ## Provider 链与失败切换 一个 Agent 可以配置**最多 5 个** Provider，按列表顺序组成一条链：第一个是主用，其余是回退。执行失败时按下面的规则决定「原地重试」还是「换下一个」： diff --git a/apps/web/src/content/manual/zh/05-evaluation.md b/apps/web/src/content/manual/zh/05-evaluation.md index 2555c3d2..d319d1bb 100644 --- a/apps/web/src/content/manual/zh/05-evaluation.md +++ b/apps/web/src/content/manual/zh/05-evaluation.md @@ -51,7 +51,7 @@ > 如果服务在评测执行途中重启，正在跑的任务会被标记为 **失败**（原因写明「被服务重启中断」），不会一直卡在「运行中」。重新发起即可，已经跑完的任务和评审结论不受影响。 > [!NOTE] -> 任务创建时会冻结当时的 **Provider + 模型 + 提示词**，执行时按这份快照跑，因此排队期间修改 Agent 不会影响已提交的任务。但如果快照里的 Provider 在任务开始前被移出该 Agent，任务会 **失败** 并说明是哪个 Provider 不可用——而不是换一个 Provider 跑完、却把结果记在原来那个头上。 +> 任务创建时会冻结当时的 **Provider + 模型 + 推理档位 / 快速模式 + 提示词**，执行时按这份快照跑，因此排队期间修改 Agent 不会影响已提交的任务。但如果快照里的 Provider 在任务开始前被移出该 Agent，任务会 **失败** 并说明是哪个 Provider 不可用——而不是换一个 Provider 跑完、却把结果记在原来那个头上。 > [!NOTE] > 评测使用的是 Agent **当前已保存**的配置，不需要先发布。如果你刚改完配置还没保存，请先保存再发起评测。评测执行不会出现在[运行记录](/wiki/runs)里，也不会计入统计和排行榜。 @@ -70,6 +70,7 @@ - **Provider**（执行引擎） - **模型** +- **推理档位与快速模式**（若该 Provider 支持） - **系统提示词** 在任务详情页展开「配置快照」即可查看。任务列表按时间倒序排列，直接对比不同任务的通过率，就能看出哪套配置表现更好。 @@ -86,7 +87,7 @@ > 有这个标记才能正确解读分数波动。上表中「提示词 v2」通过率下降，原因是提示词改了，而不是模型退化——没有标记的话很容易归错因。 > [!NOTE] -> 配置快照只记录 Provider、模型和提示词，**不含任何密钥**（API Key、OAuth Token 等一律不落库）。挂载的 Skill、MCP、知识库不在快照范围内。 +> 配置快照只记录 Provider、模型、推理档位 / 快速模式和提示词，**不含任何密钥**（API Key、OAuth Token 等一律不落库）。挂载的 Skill、MCP、知识库不在快照范围内。 ## 权限 diff --git a/apps/web/src/hooks/use-agents.ts b/apps/web/src/hooks/use-agents.ts index fc412e55..a48b3d64 100644 --- a/apps/web/src/hooks/use-agents.ts +++ b/apps/web/src/hooks/use-agents.ts @@ -500,6 +500,8 @@ export type StreamLogEntry = type: 'result' subtype: string durationMs?: number + /** CLI's own verdict on fast mode: `on` / `off` / `cooldown`; absent when unreported. */ + fastModeState?: string usage?: { inputTokens?: number outputTokens?: number diff --git a/apps/web/src/locales/en.json b/apps/web/src/locales/en.json index 1d4663c1..a6497ff1 100644 --- a/apps/web/src/locales/en.json +++ b/apps/web/src/locales/en.json @@ -114,6 +114,7 @@ "input": "Input", "output": "Output", "execLog": "Execution log", + "fastSummary": "Fast", "execParams": "Exec params", "result": "Result", "notFound": "Log not found", @@ -158,7 +159,15 @@ "filterProblems": "Errors / retries", "pageInfo": "Page {{page}} / {{total}}", "prevPage": "Prev", - "nextPage": "Next" + "nextPage": "Next", + "fastMode": { + "on": "fast mode served", + "requested": "fast mode requested; this engine does not report the served speed", + "denied": "fast mode requested, served at standard speed", + "off": "fast mode off", + "cooldown": "fast mode cooling down", + "unknown": "fast mode: {{state}}" + } }, "streaming": { "thinking": "Thinking…", @@ -1264,6 +1273,22 @@ "askModeDesc": "Run in read-only mode (--mode ask), MCP unavailable", "forceMode": "Force Mode", "forceModeDesc": "Skip workspace trust prompt (--force)", + "reasoningEffort": "Reasoning effort", + "reasoningEffortDesc": "Levels come from the selected model. Higher means deeper reasoning at more time and cost. Leave empty to use the CLI's own default.", + "reasoningEffortPlaceholder": "Use the CLI default", + "reasoningEffortDefaultOption": "{{level}} (model default)", + "reasoningEffortNone": "This model takes no reasoning level", + "reasoningEffortNoneDesc": "Discovery reported no levels for this model. Pick a different model to see options.", + "reasoningEffortUnknown": "No level information discovered", + "reasoningEffortUnknownDesc": "These credentials return no level list for this model — a self-hosted proxy usually reports model ids only. Leave it empty; the run uses the CLI default.", + "fastMode": "Fast mode", + "fastModeDesc": "Runs at a higher output speed, usually at premium pricing. Whether a run really gets it depends on the model, the plan and the endpoint — the run record shows the actual state.", + "fastModeBlocked": { + "extra_usage_disabled": "This account has usage credits disabled, so fast mode cannot run. Enable it with /usage-credits in the Claude client.", + "free": "Fast mode requires a paid subscription.", + "preference": "Fast mode is disabled by your organization.", + "unknown": "Fast mode is unavailable for these credentials." + }, "cleanResult": "Keep Final Answer Only", "cleanResultDesc": "Codex CLI only. Save and send only the final assistant response.", "maxConcurrency": "Max Concurrency", diff --git a/apps/web/src/locales/zh.json b/apps/web/src/locales/zh.json index ac5fc88c..eecbe4c4 100644 --- a/apps/web/src/locales/zh.json +++ b/apps/web/src/locales/zh.json @@ -114,6 +114,7 @@ "input": "输入", "output": "输出", "execLog": "执行日志", + "fastSummary": "Fast", "execParams": "执行参数", "result": "运行结果", "notFound": "日志未找到", @@ -158,7 +159,15 @@ "filterProblems": "错误/重试", "pageInfo": "{{page}} / {{total}} 页", "prevPage": "上一页", - "nextPage": "下一页" + "nextPage": "下一页", + "fastMode": { + "on": "快速模式已生效", + "requested": "已请求快速模式，该引擎不回报实际执行速度", + "denied": "已请求快速模式，服务端按普通速度执行", + "off": "快速模式未生效", + "cooldown": "快速模式冷却中", + "unknown": "快速模式：{{state}}" + } }, "streaming": { "thinking": "思考中…", @@ -1264,6 +1273,22 @@ "askModeDesc": "以只读模式运行（--mode ask），无法使用 MCP", "forceMode": "强制模式", "forceModeDesc": "跳过工作区信任确认（--force）", + "reasoningEffort": "推理档位", + "reasoningEffortDesc": "档位由所选模型下发，越高越深入思考、耗时和费用也越高。留空则用 CLI 自己的默认值。", + "reasoningEffortPlaceholder": "使用 CLI 默认档位", + "reasoningEffortDefaultOption": "{{level}}（模型默认）", + "reasoningEffortNone": "该模型不支持推理档位", + "reasoningEffortNoneDesc": "拉取结果显示这个模型没有可选档位，换一个模型即可看到选项。", + "reasoningEffortUnknown": "未拉取到档位信息", + "reasoningEffortUnknownDesc": "当前凭证拉取不到该模型的档位清单（自建代理通常只返回模型名）。留空即可，运行时会用 CLI 默认档位。", + "fastMode": "快速模式", + "fastModeDesc": "以更高的输出速度运行，通常按更高价格计费。是否真正生效取决于模型、账号套餐和接入方式，以运行记录中的实际状态为准。", + "fastModeBlocked": { + "extra_usage_disabled": "该账号未开启额外用量，无法使用快速模式。在 Claude 客户端执行 /usage-credits 开启后即可。", + "free": "快速模式需要付费订阅。", + "preference": "所在组织已禁用快速模式。", + "unknown": "当前凭证不可用快速模式。" + }, "cleanResult": "仅保留最终回答", "cleanResultDesc": "仅 Codex CLI 生效。只保存并发送最后一段 assistant 回复。", "maxConcurrency": "最大并发数", diff --git a/apps/web/src/pages/agent-detail/__tests__/config-tab-mcp-warning.test.tsx b/apps/web/src/pages/agent-detail/__tests__/config-tab-mcp-warning.test.tsx index ac9e2598..c395c231 100644 --- a/apps/web/src/pages/agent-detail/__tests__/config-tab-mcp-warning.test.tsx +++ b/apps/web/src/pages/agent-detail/__tests__/config-tab-mcp-warning.test.tsx @@ -37,6 +37,8 @@ const capabilities: ProviderCapabilities = { // The Provider under test cannot deliver MCP — this is what the warning is about. mcpDelivery: { mode: 'none' }, executionOptions: [], + reasoningEffort: false, + fastMode: false, sessionResume: false, sandbox: 'native', } diff --git a/apps/web/src/pages/agent-detail/__tests__/provider-capabilities.test.ts b/apps/web/src/pages/agent-detail/__tests__/provider-capabilities.test.ts index b0049bbe..f4ab2d3d 100644 --- a/apps/web/src/pages/agent-detail/__tests__/provider-capabilities.test.ts +++ b/apps/web/src/pages/agent-detail/__tests__/provider-capabilities.test.ts @@ -10,6 +10,8 @@ import { modelProbePolicy, normalizeAuthMode, providersWithoutMcpDelivery, + reasoningEffortAfterModelChange, + reasoningEffortSelectState, resolveModelProbeErrorTranslation, visibleCredentialFieldsFor, } from '../provider-capabilities' @@ -27,6 +29,8 @@ const capabilities: ProviderCapabilities = { }, mcpDelivery: { mode: 'runtime-injection' }, executionOptions: ['readOnly'], + reasoningEffort: false, + fastMode: false, sessionResume: true, sandbox: 'native', localSessionLoginCommand: 'custom login', @@ -514,3 +518,155 @@ describe('Provider capability UI helpers', () => { }) }) }) + +describe('reasoningEffortSelectState', () => { + const supported = { reasoningEffort: true } as unknown as ProviderCapabilities + const unsupported = { reasoningEffort: false } as unknown as ProviderCapabilities + + it('renders nothing for a Provider whose CLI has no such setting', () => { + expect(reasoningEffortSelectState(unsupported, undefined, 'gpt-5.6-sol')).toEqual({ + kind: 'unsupported', + }) + }) + + it('renders nothing when the Provider is not resolved yet', () => { + expect(reasoningEffortSelectState(undefined, undefined, '')).toEqual({ kind: 'unsupported' }) + }) + + it('offers the levels discovery reported for the selected model', () => { + const state = reasoningEffortSelectState( + supported, + { + 'claude-opus-4-8': { + reasoningEfforts: [{ value: 'low' }, { value: 'xhigh' }], + defaultReasoningEffort: 'xhigh', + }, + }, + 'claude-opus-4-8', + ) + + expect(state).toEqual({ + kind: 'options', + options: [{ value: 'low' }, { value: 'xhigh' }], + defaultValue: 'xhigh', + }) + }) + + it('follows the model select — a second model gets its own levels', () => { + const capabilities = { + 'claude-opus-4-8': { reasoningEfforts: [{ value: 'xhigh' }] }, + 'claude-opus-4-5-20251101': { reasoningEfforts: [{ value: 'high' }] }, + } + + expect(reasoningEffortSelectState(supported, capabilities, 'claude-opus-4-5-20251101')).toEqual( + { + kind: 'options', + options: [{ value: 'high' }], + }, + ) + }) + + it('separates "this model takes no level" from "nothing was discovered"', () => { + // A proxy that returns bare model ids reports nothing; Haiku reports an + // empty list. Both would render an empty dropdown, but only one of them is + // the user's fault to fix, so they must not share a message. + expect( + reasoningEffortSelectState( + supported, + { 'claude-haiku-4-5': { reasoningEfforts: [] } }, + 'claude-haiku-4-5', + ), + ).toEqual({ kind: 'none' }) + + expect(reasoningEffortSelectState(supported, undefined, 'deepseek-v4-flash')).toEqual({ + kind: 'unknown', + }) + }) + + it('reports unknown for a model missing from an otherwise populated probe result', () => { + expect( + reasoningEffortSelectState( + supported, + { 'claude-opus-4-8': { reasoningEfforts: [] } }, + 'other-model', + ), + ).toEqual({ kind: 'unknown' }) + }) + + it('reports unknown before a model has been chosen', () => { + expect( + reasoningEffortSelectState(supported, { 'claude-opus-4-8': { reasoningEfforts: [] } }, ''), + ).toEqual({ kind: 'unknown' }) + }) +}) + +describe('reasoningEffortAfterModelChange', () => { + const supported = { reasoningEffort: true } as unknown as ProviderCapabilities + const capabilities = { + 'claude-opus-4-8': { reasoningEfforts: [{ value: 'high' }, { value: 'xhigh' }] }, + // No default: Anthropic's model endpoint lists levels without naming one. + 'claude-sonnet-4-6': { reasoningEfforts: [{ value: 'high' }] }, + 'claude-haiku-4-5': { reasoningEfforts: [] }, + // codex reports a per-model default alongside the levels. + 'gpt-5.6-sol': { + reasoningEfforts: [{ value: 'low' }, { value: 'medium' }, { value: 'ultra' }], + defaultReasoningEffort: 'low', + }, + } + + it('keeps a level the new model still accepts', () => { + expect( + reasoningEffortAfterModelChange(supported, capabilities, 'claude-sonnet-4-6', 'high'), + ).toBe('high') + }) + + it('falls back to the new model’s default when the level does not carry over', () => { + // `ultra` exists on this codex model but not on every one; when a switch + // lands on a model that reports a default, the field states that default + // rather than going blank. + expect(reasoningEffortAfterModelChange(supported, capabilities, 'gpt-5.6-sol', 'xhigh')).toBe( + 'low', + ) + }) + + it('clears when the new model rejects the level and names no default', () => { + // Opus 4.8 has xhigh, Sonnet 4.6 does not, and Anthropic reports no default + // to fall back to. Empty passes no flag, so the CLI's own default applies — + // the same outcome, just not spelled out in the field. + expect( + reasoningEffortAfterModelChange(supported, capabilities, 'claude-sonnet-4-6', 'xhigh'), + ).toBeUndefined() + }) + + it('carries the level over unchanged when the new model still offers it', () => { + expect(reasoningEffortAfterModelChange(supported, capabilities, 'gpt-5.6-sol', 'medium')).toBe( + 'medium', + ) + }) + + it('drops any level when the new model accepts none, default or not', () => { + expect( + reasoningEffortAfterModelChange(supported, capabilities, 'claude-haiku-4-5', 'high'), + ).toBeUndefined() + }) + + it('keeps the level when the new model’s levels were never discovered', () => { + // Unknown is not evidence of invalidity — behind a proxy nothing is ever + // discovered, and clearing here would silently drop a working setting. + expect( + reasoningEffortAfterModelChange(supported, capabilities, 'deepseek-v4-flash', 'high'), + ).toBe('high') + }) + + it('keeps the level when nothing was probed at all', () => { + expect(reasoningEffortAfterModelChange(supported, undefined, 'claude-opus-4-8', 'high')).toBe( + 'high', + ) + }) + + it('stays undefined when no level was configured', () => { + expect( + reasoningEffortAfterModelChange(supported, capabilities, 'claude-haiku-4-5', undefined), + ).toBeUndefined() + }) +}) diff --git a/apps/web/src/pages/agent-detail/__tests__/provider-chain.test.ts b/apps/web/src/pages/agent-detail/__tests__/provider-chain.test.ts index dc3174b9..ba23847e 100644 --- a/apps/web/src/pages/agent-detail/__tests__/provider-chain.test.ts +++ b/apps/web/src/pages/agent-detail/__tests__/provider-chain.test.ts @@ -215,3 +215,34 @@ describe('provider-chain submission helpers', () => { }) }) }) + +describe('reasoning controls survive serialization', () => { + it('keeps each entry’s own effort and fast mode', () => { + const chain = serializeProviderChainEntries([ + chainEntry({ id: 'chain_codex', model: 'gpt-5.6-sol', reasoningEffort: 'ultra' }), + chainEntry({ + id: 'chain_claude', + model: 'claude-opus-4-8', + reasoningEffort: 'xhigh', + fastMode: true, + }), + ]) + + expect(chain.map((entry) => entry.reasoningEffort)).toEqual(['ultra', 'xhigh']) + expect(chain.map((entry) => entry.fastMode)).toEqual([undefined, true]) + }) + + it('omits an unset effort rather than serializing an empty string', () => { + // An empty string would fail the schema's token check and reject the save; + // "not configured" has to reach the API as an absent field. + const [entry] = serializeProviderChainEntries([chainEntry({ reasoningEffort: '' })]) + + expect(entry.reasoningEffort).toBeUndefined() + }) + + it('omits fast mode when it is off', () => { + const [entry] = serializeProviderChainEntries([chainEntry({ fastMode: false })]) + + expect(entry.fastMode).toBeUndefined() + }) +}) diff --git a/apps/web/src/pages/agent-detail/config-tab.tsx b/apps/web/src/pages/agent-detail/config-tab.tsx index 3ba2d680..03bd06cf 100644 --- a/apps/web/src/pages/agent-detail/config-tab.tsx +++ b/apps/web/src/pages/agent-detail/config-tab.tsx @@ -46,6 +46,8 @@ import { modelProbePolicy, normalizeAuthMode, providersWithoutMcpDelivery, + reasoningEffortAfterModelChange, + reasoningEffortSelectState, resolveModelProbeErrorTranslation, visibleCredentialFieldsFor, } from './provider-capabilities' @@ -431,6 +433,12 @@ export function ConfigTab({ ...entry, ...patch, dynamicModels: undefined, + // The levels belong to the previous credential's models. Keeping + // a chosen level across the switch would offer, and save, a + // token the new Provider may not accept. + modelCapabilities: undefined, + fastModeAvailability: undefined, + reasoningEffort: undefined, probeError: undefined, } : { ...entry, ...patch } @@ -483,6 +491,8 @@ export function ConfigTab({ probeError: result.error, probeErrorCode: result.code, dynamicModels: undefined, + modelCapabilities: undefined, + fastModeAvailability: undefined, }) } else { updateProviderEntry(entry.id, { @@ -490,6 +500,8 @@ export function ConfigTab({ probeError: undefined, probeErrorCode: undefined, dynamicModels: result.models, + modelCapabilities: result.modelCapabilities, + fastModeAvailability: result.fastMode, }) } } catch (e) { @@ -498,6 +510,8 @@ export function ConfigTab({ probeError: e instanceof Error ? e.message : String(e), probeErrorCode: undefined, dynamicModels: undefined, + modelCapabilities: undefined, + fastModeAvailability: undefined, }) } }, @@ -825,6 +839,18 @@ export function ConfigTab({ } return base })() + // The level set follows the MODEL, not the Provider, so it is read + // from this entry's probe result for whichever model is selected. + const effortState = reasoningEffortSelectState( + capabilities, + entry.modelCapabilities, + entry.model, + ) + const showFastModeOption = Boolean(capabilities?.fastMode) + // Blocked only on a definite "no" from the vendor for these + // credentials. Absent availability means the question was never + // answered, and that is not evidence. + const fastModeBlocked = entry.fastModeAvailability?.available === false const needProbeManual = probePolicy === 'manualButton' const needProbeAuto = probePolicy === 'autoOnMount' // The auto-probe effect deliberately does not retry after a failure, @@ -1319,7 +1345,19 @@ export function ConfigTab({ data-testid={`provider-chain-model-select-${index}`} onChange={(val) => { const next = Array.isArray(val) ? (val[val.length - 1] ?? '') : val - updateProviderEntry(entry.id, { model: next }) + // The options follow the model on their own; the + // selected level has to be re-checked against the + // new model, or a level it rejects stays selected + // and gets saved. + updateProviderEntry(entry.id, { + model: next, + reasoningEffort: reasoningEffortAfterModelChange( + capabilities, + entry.modelCapabilities, + next, + entry.reasoningEffort, + ), + }) if (index === 0) setValue('model', next, { shouldDirty: true }) }} filterOption={selectFilterOption} @@ -1331,6 +1369,98 @@ export function ConfigTab({ // 让浮层走 Antd 全局 z-index，escape 局部层叠上下文。 getPopupContainer={() => document.body} /> + {/* Reasoning effort sits beside the model it belongs + to: the legal levels come from the model, so a + chain that mixes Providers cannot share one value. + Four states, because an empty dropdown means two + very different things — see + `reasoningEffortSelectState`. */} + {effortState.kind !== 'unsupported' && ( +
+ +