Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 118 additions & 0 deletions apps/desktop/src/hooks/use-franklin-chat.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
import { useState } from "react";
import { act, fireEvent, render, renderHook, screen } from "@testing-library/react";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { ModelSelect } from "../components/ModelSelect";
import type { AgentSendPayload, ModelInfo, ServerMsg } from "../lib/wire";

const transport = vi.hoisted(() => ({
models: [] as ModelInfo[],
stream: vi.fn((_kind: string, _payload: AgentSendPayload, _handle: (message: ServerMsg) => void) => () => {}),
emit: vi.fn(),
}));

vi.mock("./use-models", () => ({
useModels: () => ({ models: transport.models, isLoading: false }),
}));
vi.mock("../lib/ws", () => ({ agent: { stream: transport.stream, emit: transport.emit } }));

const { CHAT_MODELS, DEFAULT_CHAT_MODEL, useFranklinChat } = await import("./use-franklin-chat");
type ChatMessage = import("./use-franklin-chat").ChatMessage;
const setMessages = vi.fn();
const ensureConvId = () => "auto-test-conversation";
const renderChat = () => renderHook(() => useFranklinChat([], setMessages, ensureConvId));
const liveModel: ModelInfo = { id: "google/gemini-2.5-flash", label: "Gemini 2.5 Flash", free: false, group: "Google" };

function finishTurn(index: number, text?: string) {
const handle = transport.stream.mock.calls[index][2];
act(() => {
if (text) handle({ id: "turn", kind: "agent.text", payload: { text } });
handle({ id: "turn", kind: "agent.done", payload: {} });
});
}

function Composer() {
const [messages, updateMessages] = useState<ChatMessage[]>([]);
const chat = useFranklinChat(messages, updateMessages, ensureConvId);
return <>
<ModelSelect models={chat.models} model={chat.model} onChange={chat.setModel} disabled={chat.isBusy} />
<button onClick={() => chat.send("Explain this market")}>Send test message</button>
<output>{messages.map(message => message.content).join("\n")}</output>
</>;
}

beforeEach(() => {
transport.models = [];
transport.stream.mockClear();
transport.emit.mockClear();
setMessages.mockClear();
});

describe("Desktop Auto model selection", () => {
it("offers Auto before the catalog is available without changing the user's default", () => {
const { result } = renderChat();
expect(CHAT_MODELS.some(model => model.id === "blockrun/auto")).toBe(true);
expect(result.current.models[0]).toMatchObject({ id: "blockrun/auto", label: "Auto" });
expect(result.current.models[0].free).not.toBe(true);
expect(result.current.model).toBe(DEFAULT_CHAT_MODEL);
});

it("keeps Auto available when the live catalog replaces the fallback list", () => {
const { result, rerender } = renderChat();
act(() => result.current.setModel("blockrun/auto"));
transport.models = [liveModel];
rerender();
expect(result.current.selectedModel?.label).toBe("Auto");
expect(result.current.models.map(model => model.id)).toEqual(["blockrun/auto", liveModel.id]);
});

it("does not duplicate or falsely label Auto as free if a backend also returns it", () => {
transport.models = [liveModel, { id: "blockrun/auto", label: "Old Auto", group: "Legacy", free: true }];
const { result } = renderChat();
const auto = result.current.models.filter(model => model.id === "blockrun/auto");
expect(auto).toHaveLength(1);
expect(auto[0].label).toBe("Auto");
expect(auto[0].free).not.toBe(true);
});

it("selects Auto through the real dropdown and preserves it after a routed reply", () => {
transport.models = [{ id: DEFAULT_CHAT_MODEL, label: "Claude Opus 4.8", free: false, group: "Anthropic" }, liveModel];
render(<Composer />);
fireEvent.click(screen.getByRole("button", { name: "Claude Opus 4.8" }));
fireEvent.click(screen.getByRole("option", { name: /^Auto$/ }));
fireEvent.click(screen.getByRole("button", { name: "Send test message" }));
expect(transport.stream.mock.calls[0][1]).toMatchObject({ model: "blockrun/auto", mode: "chat" });
finishTurn(0, "Auto → deepseek/deepseek-v4-pro\n\nA routed answer.");
expect(screen.getByRole("status").textContent).toContain("Auto → deepseek/deepseek-v4-pro");
expect(screen.getByRole("button", { name: /^Auto$/ })).toBeTruthy();
fireEvent.click(screen.getByRole("button", { name: "Send test message" }));
expect(transport.stream.mock.calls[1][1].model).toBe("blockrun/auto");
});

it("sends Auto for regeneration and still allows an explicit manual model", () => {
const { result } = renderChat();
act(() => result.current.setModel("blockrun/auto"));
act(() => result.current.send("First prompt"));
finishTurn(0);
act(() => result.current.regenerate());
expect(transport.stream.mock.calls[1][1].model).toBe("blockrun/auto");
finishTurn(1);
act(() => result.current.setModel(liveModel.id));
act(() => result.current.send("Manual prompt"));
expect(transport.stream.mock.calls[2][1].model).toBe(liveModel.id);
});

it("keeps Auto out of media pickers and preserves the selected media model", () => {
const { result } = renderChat();
act(() => result.current.setModel("blockrun/auto"));
act(() => result.current.setMode("image"));
expect(result.current.models.some(model => model.id === "blockrun/auto")).toBe(false);
act(() => result.current.setModel("google/nano-banana-pro"));
act(() => result.current.send("A golden F"));
const payload = transport.stream.mock.calls[0][1];
expect(payload.model).toBeUndefined();
expect(payload.text).toContain("google/nano-banana-pro");
finishTurn(0);
act(() => result.current.setMode("chat"));
expect(result.current.model).toBe("blockrun/auto");
});
});
12 changes: 11 additions & 1 deletion apps/desktop/src/hooks/use-franklin-chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,14 @@ export interface ChatModel {
contextWindow?: number;
}

// Routing profiles are resolved by the CLI, so the gateway catalog does not
// include them. Keep Auto available before and after that catalog loads.
const AUTO_CHAT_MODEL: ChatModel = { id: "blockrun/auto", label: "Auto", group: "Automatic" };

// Curated fallback lineup (mirrors Franklin's /model picker) used until the CLI
// returns its live catalog over models.list.
export const CHAT_MODELS: ChatModel[] = [
AUTO_CHAT_MODEL,
{ id: "nvidia/nemotron-nano-9b-v2", label: "Nemotron Nano 9B v2", free: true, group: "Free" },
{ id: "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", label: "Nemotron 3 Nano Omni", free: true, group: "Free" },
{ id: "nvidia/mistral-nemotron", label: "Mistral Nemotron", free: true, group: "Free" },
Expand Down Expand Up @@ -179,7 +184,12 @@ export function useFranklinChat(

// Prefer the CLI's live catalog for chat; fall back to the curated list.
const chatModels: ChatModel[] = liveModels.length
? liveModels.map((m) => ({ id: m.id, label: m.label, free: m.free, group: m.group, contextWindow: m.contextWindow }))
? [
AUTO_CHAT_MODEL,
...liveModels
.filter((m) => m.id !== AUTO_CHAT_MODEL.id)
.map((m) => ({ id: m.id, label: m.label, free: m.free, group: m.group, contextWindow: m.contextWindow })),
]
: CHAT_MODELS;

const models = mode === "image" ? IMAGE_MODELS : mode === "video" ? VIDEO_MODELS : mode === "music" ? MUSIC_MODELS : chatModels;
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"desktop:package:win": "npm run dist:win --workspace @blockrun/franklin-desktop",
"dev": "tsc --watch",
"start": "node dist/index.js",
"test": "npm run build && node --test --test-concurrency=4 --test-reporter=spec test/local.mjs test/api-key.local.mjs test/solana-migration.local.mjs test/serve-security.local.mjs test/skills.local.mjs test/skills-management.local.mjs test/repair.mjs test/model-resolver.mjs test/exa.local.mjs test/audit-batch.local.mjs test/polymarket.local.mjs test/market.local.mjs test/hooks.local.mjs test/scheduler.local.mjs test/trade-plan.local.mjs test/goal.local.mjs test/memory.local.mjs test/agent-host.local.mjs test/reservation.local.mjs test/model-catalog.local.mjs test/arc.local.mjs test/subprocess-env.local.mjs test/withdraw.local.mjs test/orders.local.mjs test/payment-hold.local.mjs test/experience.local.mjs",
"test": "npm run build && node --test --test-concurrency=4 --test-reporter=spec test/local.mjs test/api-key.local.mjs test/solana-migration.local.mjs test/serve-security.local.mjs test/desktop-auto-routing.local.mjs test/skills.local.mjs test/skills-management.local.mjs test/repair.mjs test/model-resolver.mjs test/exa.local.mjs test/audit-batch.local.mjs test/polymarket.local.mjs test/market.local.mjs test/hooks.local.mjs test/scheduler.local.mjs test/trade-plan.local.mjs test/goal.local.mjs test/memory.local.mjs test/agent-host.local.mjs test/reservation.local.mjs test/model-catalog.local.mjs test/arc.local.mjs test/subprocess-env.local.mjs test/withdraw.local.mjs test/orders.local.mjs test/payment-hold.local.mjs test/experience.local.mjs",
"test:e2e": "npm run build && node --test --test-reporter=spec test/e2e.mjs",
"e2e:polymarket:readonly": "npm run build && node scripts/polymarket-e2e-readonly.mjs",
"test:free-models": "npm run build && node --test --test-reporter=spec test/free-model-matrix.mjs",
Expand Down
160 changes: 160 additions & 0 deletions test/desktop-auto-routing.local.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
/** Desktop routing integration: real serve socket + agent loop, fake loopback
* gateway only. A child process gets a disposable HOME/USERPROFILE and no
* inherited credentials. No live wallet, account, model or service is used. */
import assert from 'node:assert/strict';
import { fork } from 'node:child_process';
import { once } from 'node:events';
import fs from 'node:fs';
import http from 'node:http';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import test from 'node:test';
import WebSocket from 'ws';

const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const SELF = fileURLToPath(import.meta.url);

if (process.argv[2] === '--fixture') {
const gateway = new URL(process.env.FRANKLIN_TEST_GATEWAY);
assert.equal(gateway.hostname, '127.0.0.1');
const nativeFetch = globalThis.fetch;
globalThis.fetch = async (input, init) => {
const url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url);
if (url.origin === gateway.origin) return nativeFetch(input, init);
// Catalog refreshes and optional background services must never escape
// this fixture. Return promptly so a network outage cannot hide a failure.
return Response.json({ error: 'External network disabled by test fixture' }, { status: 503 });
};
const { API_URLS } = await import('../dist/config.js');
API_URLS.base = gateway.origin;
const { startServer } = await import('../dist/serve/server.js');
await startServer({ port: 0, workDir: process.env.FRANKLIN_TEST_WORKDIR, debug: false });
} else {
test('Desktop serve routes Auto to concrete models and preserves explicit model changes', { timeout: 45_000 }, async (t) => {
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'franklin-desktop-auto-'));
const home = path.join(temp, 'home');
const workDir = path.join(temp, 'workspace');
fs.mkdirSync(home);
fs.mkdirSync(workDir);
const requests = [];
const gateway = http.createServer(async (req, res) => {
if (req.method !== 'POST' || req.url !== '/v1/messages') {
res.writeHead(503, { 'content-type': 'application/json' });
res.end(JSON.stringify({ error: 'Only the fake messages endpoint is available' }));
return;
}
let raw = '';
for await (const chunk of req) raw += chunk;
const payload = JSON.parse(raw);
requests.push(payload);
res.writeHead(200, { 'content-type': 'text/event-stream' });
const event = (kind, data) => res.write(`event: ${kind}\ndata: ${JSON.stringify(data)}\n\n`);
event('message_start', { message: { usage: { input_tokens: 5, output_tokens: 0 } } });
event('content_block_start', { index: 0, content_block: { type: 'text', text: '' } });
event('content_block_delta', { index: 0, delta: { type: 'text_delta', text: `Offline answer from ${payload.model}` } });
event('content_block_stop', { index: 0 });
event('message_delta', { delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 5 } });
event('message_stop', {});
res.end('data: [DONE]\n\n');
});
await new Promise(resolve => gateway.listen(0, '127.0.0.1', resolve));
t.after(async () => {
gateway.closeAllConnections();
await new Promise(resolve => gateway.close(resolve));
fs.rmSync(temp, { recursive: true, force: true });
});

const child = fork(SELF, ['--fixture'], {
cwd: ROOT,
execArgv: [],
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
env: {
PATH: process.env.PATH ?? '',
...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}),
HOME: home,
USERPROFILE: home,
APPDATA: path.join(home, 'AppData', 'Roaming'),
LOCALAPPDATA: path.join(home, 'AppData', 'Local'),
RUNCODE_CHAIN: 'base',
FRANKLIN_TEST_GATEWAY: `http://127.0.0.1:${gateway.address().port}`,
FRANKLIN_TEST_WORKDIR: workDir,
FRANKLIN_DESKTOP_WORKSPACE_BOUNDARY: '1',
FRANKLIN_SERVE_DISCOVERY: 'off',
FRANKLIN_CLOUD_SYNC: 'off',
FRANKLIN_NO_AUDIT: '1',
FRANKLIN_NO_PERSIST: '1',
FRANKLIN_NO_EXPERIENCE: '1',
},
});
let diagnostics = '';
child.stdout.on('data', chunk => { diagnostics = (diagnostics + chunk).slice(-4000); });
child.stderr.on('data', chunk => { diagnostics = (diagnostics + chunk).slice(-4000); });
t.after(async () => {
if (child.exitCode === null) {
const exited = once(child, 'exit');
child.kill();
await exited;
}
});
const port = await new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error(`serve readiness timeout: ${diagnostics}`)), 10_000);
child.once('error', reject);
child.once('exit', code => { clearTimeout(timer); reject(new Error(`serve exited (${code}): ${diagnostics}`)); });
child.on('message', message => {
if (message?.type === 'franklin:server-ready') {
clearTimeout(timer);
resolve(message.port);
}
});
});
const socket = new WebSocket(`ws://127.0.0.1:${port}/agent`);
await once(socket, 'open');
t.after(() => socket.terminate());
let sequence = 0;
async function turn(model, text) {
const id = `auto-routing-${++sequence}`;
const events = [];
await new Promise((resolve, reject) => {
const timer = setTimeout(() => finish(new Error(`turn timeout: ${diagnostics}`)), 10_000);
const onMessage = raw => {
const message = JSON.parse(raw.toString());
if (message.id !== id) return;
events.push(message);
if (message.kind === 'agent.error' || message.kind === 'error') finish(new Error(message.payload?.message ?? 'Agent error'));
else if (message.kind === 'agent.done') finish();
};
function finish(error) {
clearTimeout(timer);
socket.off('message', onMessage);
if (error) reject(error); else resolve();
}
socket.on('message', onMessage);
socket.send(JSON.stringify({ id, kind: 'agent.send', payload: { sessionId: null, convId: 'desktop-auto', model, text } }));
});
return events.filter(event => event.kind === 'agent.text').map(event => event.payload.text).join('');
}
const mainRequests = () => requests.filter(request => request.tools?.some(tool => tool.name === 'Glob'));

const manualFirst = await turn('anthropic/claude-haiku-4.5', 'Reply briefly: manual-first');
assert.doesNotMatch(manualFirst, /Auto →/);
assert.equal(mainRequests().at(-1)?.model, 'anthropic/claude-haiku-4.5');

const firstAuto = await turn('blockrun/auto', 'Reply briefly to this greeting: auto-first');
const firstResolved = firstAuto.match(/Auto → ([^*\s]+)/)?.[1];
assert.ok(firstResolved?.includes('/'), 'Auto announces a concrete provider/model');
assert.notEqual(firstResolved, 'blockrun/auto');
assert.equal(mainRequests().at(-1)?.model, firstResolved, 'the gateway gets the model announced to Desktop');

const nextAuto = await turn('blockrun/auto', 'Reply briefly to this greeting: auto-second');
const nextResolved = nextAuto.match(/Auto → ([^*\s]+)/)?.[1];
assert.ok(nextResolved, 'the next turn keeps automatic routing enabled');
assert.equal(mainRequests().at(-1)?.model, nextResolved);

const manualLast = await turn('deepseek/deepseek-v4-pro', 'Reply briefly: manual-again');
assert.doesNotMatch(manualLast, /Auto →/);
assert.equal(mainRequests().at(-1)?.model, 'deepseek/deepseek-v4-pro');
assert.ok(requests.length >= 4);
assert.ok(requests.every(request => request.model && !request.model.startsWith('blockrun/')), 'no virtual routing profile reaches the gateway');
});
}
Loading