Skip to content

Commit 261a88e

Browse files
committed
Verify the FTS language on the serving path before the first query
Migration-time verification only protects hosts that run the migration runner at boot; one that migrated once and later changed FTS_LANGUAGE would silently degrade recall with nothing rechecking. The knowledge plane now awaits the same read-only catalog check, memoized, before its first search or capture — the fail-fast validate step (Hibernate validate, Rails check_all_pending!) adapted to a synchronous mount. Success is remembered forever; a failure clears the memo so a transient connection error at first use does not poison the plane until restart. The verifier is exported for hosts that prefer running it from a real readiness probe.
1 parent 19158fb commit 261a88e

6 files changed

Lines changed: 154 additions & 14 deletions

File tree

‎IMPLEMENTATION.md‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -131,10 +131,13 @@ An ordered slice of a version's text, keyed by `(version_id, ordinal)`
131131
that powers the lexical search channel — this is the only place FTS is
132132
computed; no separate FTS table exists. Its language comes from
133133
`FTS_LANGUAGE` at migration time; the query side binds the same configured
134-
language as a `regconfig` parameter, and `runKnowledgeMigrations` verifies
135-
the column's actual language against the configuration (read back from the
136-
catalog) so a mismatch fails at startup instead of silently degrading
137-
recall. Chunks are **never** reused across
134+
language as a `regconfig` parameter. The invariant is verified twice, both
135+
read-only against the catalog: `runKnowledgeMigrations` checks after
136+
applying (the deploy step), and the knowledge plane runs the same check
137+
once, memoized, before its first query (the serving path) — so a mismatch
138+
or unmigrated schema fails loudly on first use regardless of who ran the
139+
migrations. Hosts with a readiness probe can call the exported
140+
`verifyFtsLanguage` there instead. Chunks are **never** reused across
138141
versions — every new version gets a fresh full insert of its own chunks.
139142

140143
### `knowledge_entity` / `knowledge_edge`

‎src/core/fts-language.test.ts‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { describe, expect, it } from "bun:test";
22
import { readFile } from "node:fs/promises";
33
import { join } from "node:path";
44
import {
5+
createFtsVerification,
56
DEFAULT_FTS_LANGUAGE,
67
FTS_LANGUAGE_TOKEN,
78
parseFtsLanguage,
@@ -91,3 +92,108 @@ describe("verifyFtsLanguage", () => {
9192
).rejects.toThrow("schema not migrated");
9293
});
9394
});
95+
96+
describe("createFtsVerification", () => {
97+
const ENGLISH_EXPR = "to_tsvector('english'::regconfig, text)";
98+
99+
it("verifies once and memoizes success", async () => {
100+
let calls = 0;
101+
const ensure = createFtsVerification(
102+
{
103+
query: (sqlText: string) => {
104+
calls += 1;
105+
return Promise.resolve(
106+
sqlText.includes("pg_ts_config") ? [{ ok: 1 }] : [{ expr: ENGLISH_EXPR }],
107+
);
108+
},
109+
},
110+
"english",
111+
);
112+
await ensure();
113+
await ensure();
114+
// one pg_ts_config lookup + one catalog read, never repeated
115+
expect(calls).toBe(2);
116+
});
117+
118+
it("clears the memo on failure so a transient error does not poison the plane", async () => {
119+
let attempt = 0;
120+
const ensure = createFtsVerification(
121+
{
122+
query: (sqlText: string) => {
123+
if (attempt === 0 && sqlText.includes("pg_ts_config")) {
124+
attempt += 1;
125+
return Promise.reject(new Error("connection refused"));
126+
}
127+
return Promise.resolve(
128+
sqlText.includes("pg_ts_config") ? [{ ok: 1 }] : [{ expr: ENGLISH_EXPR }],
129+
);
130+
},
131+
},
132+
"english",
133+
);
134+
await expect(ensure()).rejects.toThrow("connection refused");
135+
await ensure();
136+
});
137+
138+
it("keeps throwing on a real mismatch", async () => {
139+
const ensure = createFtsVerification(
140+
{
141+
query: (sqlText: string) =>
142+
Promise.resolve(
143+
sqlText.includes("pg_ts_config") ? [{ ok: 1 }] : [{ expr: ENGLISH_EXPR }],
144+
),
145+
},
146+
"german",
147+
);
148+
await expect(ensure()).rejects.toThrow('built with "english"');
149+
await expect(ensure()).rejects.toThrow('built with "english"');
150+
});
151+
});
152+
153+
describe("createFtsVerification concurrency", () => {
154+
const ENGLISH_EXPR = "to_tsvector('english'::regconfig, text)";
155+
156+
it("concurrent first calls share one in-flight verification", async () => {
157+
let calls = 0;
158+
let release!: () => void;
159+
const gate = new Promise<void>((r) => (release = r));
160+
const ensure = createFtsVerification(
161+
{
162+
query: async (sqlText: string) => {
163+
calls += 1;
164+
await gate;
165+
return sqlText.includes("pg_ts_config") ? [{ ok: 1 }] : [{ expr: ENGLISH_EXPR }];
166+
},
167+
},
168+
"english",
169+
);
170+
const a = ensure();
171+
const b = ensure();
172+
release();
173+
await Promise.all([a, b]);
174+
expect(calls).toBe(2);
175+
});
176+
177+
it("concurrent callers all see a shared failure, then a retry succeeds", async () => {
178+
let attempt = 0;
179+
const ensure = createFtsVerification(
180+
{
181+
query: (sqlText: string) => {
182+
if (attempt === 0 && sqlText.includes("pg_ts_config")) {
183+
attempt += 1;
184+
return Promise.reject(new Error("connection refused"));
185+
}
186+
return Promise.resolve(
187+
sqlText.includes("pg_ts_config") ? [{ ok: 1 }] : [{ expr: ENGLISH_EXPR }],
188+
);
189+
},
190+
},
191+
"english",
192+
);
193+
const a = ensure();
194+
const b = ensure();
195+
await expect(a).rejects.toThrow("connection refused");
196+
await expect(b).rejects.toThrow("connection refused");
197+
await ensure();
198+
});
199+
});

‎src/core/fts-language.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,25 @@ export function parseFtsLanguage(raw: string | undefined): string {
3030
return raw;
3131
}
3232

33-
interface FtsVerifySqlClient {
33+
/**
34+
* Memoized serving-path wrapper around verifyFtsLanguage: the first call
35+
* verifies, success is remembered forever, and a failure clears the memo so
36+
* a transient DB error at first use does not poison the plane until
37+
* restart (a real mismatch keeps failing on every retry anyway).
38+
*/
39+
export function createFtsVerification(
40+
client: FtsVerifySqlClient,
41+
ftsLanguage: string,
42+
): () => Promise<void> {
43+
let verified: Promise<void> | undefined;
44+
return () =>
45+
(verified ??= verifyFtsLanguage(client, ftsLanguage).catch((err) => {
46+
verified = undefined;
47+
throw err;
48+
}));
49+
}
50+
51+
export interface FtsVerifySqlClient {
3452
query: (sql: string, params: readonly unknown[]) => Promise<Array<Record<string, unknown>>>;
3553
}
3654

‎src/index.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ export type { KnowledgePlane } from "./knowledge.ts";
2727
export { CaptureLog, type CaptureEvent } from "./capture-log.ts";
2828
// Migrations
2929
export { runKnowledgeMigrations } from "./migrations.ts";
30+
export {
31+
type FtsVerifySqlClient,
32+
parseFtsLanguage,
33+
verifyFtsLanguage,
34+
} from "./core/fts-language.ts";
3035
// Granular mount (compose your own)
3136
export { mountKnowledgeRoutes, type GrantConfig } from "./routes/mount.ts";
3237

‎src/knowledge.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import type { EngineConfig } from "./config.ts";
66
import { log } from "./log.ts";
77
import { createDb, type Db, type RawSql } from "./db/client.ts";
8+
import { createFtsVerification } from "./core/fts-language.ts";
9+
import { createEmbedRegistrySqlClient } from "./core/embed-sql.ts";
810
import type { VisibilitySpec } from "./core/schemas/document.ts";
911
import { captureDocument } from "./services/capture.ts";
1012
import {
@@ -57,9 +59,21 @@ export function createKnowledgePlane(config: KnowledgeConfig): KnowledgePlane {
5759
const { db, sql }: { db: Db; sql: RawSql } = createDb(engineConfig);
5860
const deps = { db, sql, config: engineConfig };
5961

62+
// Serving-path schema validation, industry-standard fail-fast shape
63+
// (Hibernate validate / Rails check_all_pending!): the mount is
64+
// synchronous, so "before accepting traffic" becomes a memoized check
65+
// awaited by the first query. Read-only; migration stays a deploy step.
66+
// Hosts with a real readiness probe can call verifyFtsLanguage there
67+
// instead — this memo then resolves against an already-verified schema.
68+
const ensureVerified = createFtsVerification(
69+
createEmbedRegistrySqlClient(sql),
70+
engineConfig.ftsLanguage,
71+
);
72+
6073
return {
6174
async search(params) {
6275
try {
76+
await ensureVerified();
6377
const result = await hybridSearch(deps, {
6478
tenantId: params.tenantId,
6579
principalId: params.principalId,
@@ -114,6 +128,7 @@ export function createKnowledgePlane(config: KnowledgeConfig): KnowledgePlane {
114128
},
115129

116130
async capture(params) {
131+
await ensureVerified();
117132
const adapter = params.adapter ?? "mcp";
118133
const externalRef =
119134
params.externalRef ??

‎src/migrations.ts‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
parseFtsLanguage,
1212
verifyFtsLanguage,
1313
} from "./core/fts-language.ts";
14+
import { createEmbedRegistrySqlClient } from "./core/embed-sql.ts";
1415

1516
const MIGRATIONS_DIR = join(import.meta.dir, "..", "migrations");
1617

@@ -58,15 +59,7 @@ export async function runKnowledgeMigrations(
5859
// generated column was actually built with; a previously-migrated
5960
// database under a different language must fail loudly here, not
6061
// degrade recall silently at query time.
61-
await verifyFtsLanguage(
62-
{
63-
query: async (text, params) =>
64-
(await sql.unsafe(text, [...params] as never[])) as unknown as Array<
65-
Record<string, unknown>
66-
>,
67-
},
68-
ftsLanguage,
69-
);
62+
await verifyFtsLanguage(createEmbedRegistrySqlClient(sql), ftsLanguage);
7063
} finally {
7164
await sql.end();
7265
}

0 commit comments

Comments
 (0)