Skip to content

fix(llm): allow embedding-only openai-compatible providers without defaultChatModel #410

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Bartok9 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(llm): allow embedding-only openai-compatible providers without defaultChatModel #410

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions src/llm/provider/openai/openai-provider.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,3 +262,54 @@ describe("OpenAiProvider strictTools wiring", () => {
});
});
});

describe("OpenAiProvider without defaultChatModel", () => {
it("throws on complete() when defaultChatModel is not configured", async () => {
const p = new OpenAiProvider({
id: "test",
baseUrl: "https://example.invalid",
apiKey: "test-key",
fetchImpl: fakeFetch({
role: "assistant",
content: "ok",
}) as unknown as typeof fetch,
});
await expect(p.complete({ prompt: "hi" })).rejects.toThrow(
/has no defaultChatModel configured/,
);
});

it("throws on completeStream() when defaultChatModel is not configured", async () => {
const p = new OpenAiProvider({
id: "test",
baseUrl: "https://example.invalid",
apiKey: "test-key",
fetchImpl: fakeStreamFetch("ok") as unknown as typeof fetch,
});
const stream = p.completeStream({ prompt: "hi" });
await expect(stream.next()).rejects.toThrow(
/has no defaultChatModel configured/,
);
});

it("throws on describeImage() when defaultChatModel is not configured", async () => {
const p = new OpenAiProvider({
id: "test",
baseUrl: "https://example.invalid",
apiKey: "test-key",
supportsVision: true,
fetchImpl: fakeFetch({
role: "assistant",
content: "ok",
}) as unknown as typeof fetch,
});
await expect(
p.describeImage({
prompt: "describe",
images: [
{ id: 0, bytes: new Uint8Array(), mimeType: "image/png" },
],
}),
).rejects.toThrow(/has no defaultChatModel configured/);
});
});
38 changes: 28 additions & 10 deletions src/llm/provider/openai/openai-provider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ export interface OpenAiProviderOptions {
id: string;
baseUrl: string;
apiKey: string;
defaultChatModel: string;
defaultChatModel?: string;
headers?: Record<string, string>;
/**
* Header that carries the API key when the service does not accept
Expand Down Expand Up @@ -104,7 +104,7 @@ export class OpenAiProvider implements LlmProvider {
readonly capabilities: ProviderCapabilities;

private readonly http: OpenAiHttpDeps;
private readonly defaultChatModel: string;
private readonly defaultChatModel: string | undefined;
private readonly apiPathPrefix: string;
private readonly taggedToolCompatibility: "qwen" | undefined;
private readonly extraBody: Record<string, unknown> | undefined;
Expand Down Expand Up @@ -157,18 +157,24 @@ export class OpenAiProvider implements LlmProvider {
}

async complete(request: CompletionRequest): Promise<CompletionResult> {
const model = this.defaultChatModel;
if (!model) {
throw new Error(
`openai-compatible provider "${this.name}" has no defaultChatModel configured`,
);
}
// Unary only: sub-calls carry `response_format`, streamed turns never do.
const json = await sendWithStructuredOutputFallback(
{
providerId: this.id,
model: this.defaultChatModel,
model,
logger: this.http.logger,
},
request,
(req) =>
buildOpenAiChatBody(
req,
this.defaultChatModel,
model,
false,
this.extraBody,
this.maxOutputTokens,
Expand All @@ -187,15 +193,21 @@ export class OpenAiProvider implements LlmProvider {
this.taggedToolCompatibility === "qwen"
? adaptQwenTaggedToolResponse(json, request)
: json;
return normaliseOpenAiChatResponse(adapted, this.defaultChatModel);
return normaliseOpenAiChatResponse(adapted, model);
}

async *completeStream(
request: CompletionRequest,
): AsyncGenerator<StreamChunk, CompletionResult, void> {
const model = this.defaultChatModel;
if (!model) {
throw new Error(
`openai-compatible provider "${this.name}" has no defaultChatModel configured`,
);
}
const body = buildOpenAiChatBody(
request,
this.defaultChatModel,
model,
true,
this.extraBody,
this.maxOutputTokens,
Expand Down Expand Up @@ -334,7 +346,7 @@ export class OpenAiProvider implements LlmProvider {
}
const final = completionFromStreamFinal(
streamFinal,
this.defaultChatModel,
model,
accumulated,
accumulatedReasoning,
);
Expand Down Expand Up @@ -396,9 +408,15 @@ export class OpenAiProvider implements LlmProvider {
if (!this.capabilities.vision) {
throw new VisionUnsupportedError(this.name);
}
const model = this.defaultChatModel;
if (!model) {
throw new Error(
`openai-compatible provider "${this.name}" has no defaultChatModel configured`,
);
}
return describeImageViaOpenAi(
this.http,
this.defaultChatModel,
model,
request,
this.apiPathPrefix,
this.providerPreferences,
Expand All @@ -421,7 +439,7 @@ function normalizeApiPathPrefix(prefix: string): string {

function completionFromStreamFinal(
streamFinal: StreamFinalResult | void,
defaultChatModel: string,
defaultChatModel: string | undefined,
accumulated: string,
accumulatedReasoning: string,
): CompletionResult {
Expand All @@ -444,7 +462,7 @@ function completionFromStreamFinal(
},
cacheHitTokens: 0,
slotId: -1,
modelId: streamFinal?.modelId ?? defaultChatModel,
modelId: streamFinal?.modelId ?? defaultChatModel ?? null,
usage,
toolCalls: streamFinal?.toolCalls,
finishReason,
Expand Down
74 changes: 74 additions & 0 deletions src/llm/provider/registry/provider-registry.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,4 +87,78 @@ describe("ProviderRegistry", () => {
}),
).rejects.toThrow(/unknown llm provider kind/);
});

it("allows embedding-only openai-compatible provider without defaultChatModel", async () => {
registerBuiltInProviderKinds();
const fakeConfig = {
...getConfig(),
llm: {
activeTextProvider: "chat-provider",
activeEmbeddingProvider: "embed-only",
toolTransport: "auto" as const,
providers: [
{
id: "chat-provider",
kind: "openai-compatible",
baseUrl: "https://example.invalid",
defaultChatModel: "gpt-4",
},
{
id: "embed-only",
kind: "openai-compatible",
baseUrl: "https://example.invalid",
defaultEmbeddingModel: "nomic-embed-text",
userModels: [
{
id: "nomic-embed-text",
kind: "embedding" as const,
dim: 768,
},
],
},
],
},
} as AtomicAgentConfig;
const registry = await ProviderRegistry.fromConfig(fakeConfig, {
config: fakeConfig,
llamaClient: {} as never,
getProfile: () => ({}) as never,
logger: { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} },
});
expect(registry.listIds()).toContain("embed-only");
});

it("rejects non-embedding openai-compatible provider without defaultChatModel", async () => {
registerBuiltInProviderKinds();
const fakeConfig = {
...getConfig(),
llm: {
activeTextProvider: "chat-provider",
activeEmbeddingProvider: "embed-only",
toolTransport: "auto" as const,
providers: [
{
id: "chat-provider",
kind: "openai-compatible",
baseUrl: "https://example.invalid",
defaultChatModel: "gpt-4",
},
{
id: "bad-provider",
kind: "openai-compatible",
baseUrl: "https://example.invalid",
userModels: [{ id: "gpt-4", kind: "chat" as const }],
},
],
},
} as AtomicAgentConfig;
await expect(
ProviderRegistry.fromConfig(fakeConfig, {
config: fakeConfig,
llamaClient: {} as never,
getProfile: () => ({}) as never,
logger: { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} },
}),
).rejects.toThrow(/requires defaultChatModel/);
});
});
26 changes: 22 additions & 4 deletions src/llm/provider/registry/register-built-in-providers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,18 @@ export function registerBuiltInProviderKinds(): void {

registerProviderKind("openai-compatible", (ctx) => {
const entry = ctx.entry;
if (!entry.baseUrl || !entry.defaultChatModel) {
const isEmbeddingOnly =
entry.userModels !== undefined &&
entry.userModels.length > 0 &&
entry.userModels.every((m) => m.kind === "embedding");
if (!entry.baseUrl) {
throw new Error(
`openai-compatible provider "${entry.id}" requires baseUrl and defaultChatModel`,
`openai-compatible provider "${entry.id}" requires baseUrl`,
);
}
if (!isEmbeddingOnly && !entry.defaultChatModel) {
throw new Error(
`openai-compatible provider "${entry.id}" requires defaultChatModel`,
);
}
return new OpenAiProvider({
Expand All @@ -73,9 +82,18 @@ export function registerBuiltInProviderKinds(): void {

registerProviderKind("qwen-openai-compatible", (ctx) => {
const entry = ctx.entry;
if (!entry.baseUrl || !entry.defaultChatModel) {
const isEmbeddingOnly =
entry.userModels !== undefined &&
entry.userModels.length > 0 &&
entry.userModels.every((m) => m.kind === "embedding");
if (!entry.baseUrl) {
throw new Error(
`qwen-openai-compatible provider "${entry.id}" requires baseUrl`,
);
}
if (!isEmbeddingOnly && !entry.defaultChatModel) {
throw new Error(
`qwen-openai-compatible provider "${entry.id}" requires baseUrl and defaultChatModel`,
`qwen-openai-compatible provider "${entry.id}" requires defaultChatModel`,
);
}
return new OpenAiProvider({
Expand Down
Loading