From ea4d59e592a45460c37a9bd04426c71657e7be58 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 05:36:07 -0600 Subject: [PATCH 01/12] fix(hnsw): auto-scale efConstruction with graph size so edge quality holds at 1M+ nodes At a constant efConstruction (100), edge-selection quality erodes as the corpus grows until true neighbours become unreachable at any search ef: recall@10 at 1M nodes plateaued at 0.94-0.97 across ef 512-1536, while rebuilding at efConstruction 200 restored 0.985/0.997 and made queries faster at the same ef (21% fewer nodes visited). Scale it with sqrt(nodes/250K) from the same memoized node count the search-side auto-scale resolves, capped at 512; an explicitly configured efConstruction stays authoritative. Closes #2180 Co-Authored-By: Claude Fable 5 --- DESIGN.md | 23 +++++ .../HierarchicalNavigableSmallWorld.ts | 26 +++++- unitTests/resources/vectorIndex.test.js | 91 +++++++++++++++++++ 3 files changed, 138 insertions(+), 2 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 8b8a19dcb..5322103f9 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -791,6 +791,29 @@ graphs being identical, though equal metrics do not prove it. It is the expected the upper layers are sparse enough that a greedy walk reaches the same entry point, which is why standard HNSW descends this way. +## `efConstruction` auto-scales with the graph, for the same reason search `ef` does + +The connection-building pass selects each node's stored edges from a candidate list of +`efConstruction` entries. Held at a constant (100) while the corpus grows, edge quality erodes in a +way no search-side setting can compensate: at 1M nodes (768-dim, int8, calibrated hard corpus) +recall@10 fell to 0.935 and sweeping the search `ef` from 512 to 1536 only reached 0.957 raw / 0.967 +set at 4.7x the latency — the missing neighbours were not deep in the candidate list, they were +unreachable. Rebuilding the identical corpus (same seed, same level assignments) with +`efConstruction` 200 restored 0.985/0.997 and made queries _faster_ at the same `ef` (3,110 nodes +visited vs 3,948 — better-selected edges route more directly). Quantization contributed ~1.5 points +(float32 rebuild: 0.952); construction quality was the dominant term. Full sweep in #2180. + +So when the schema does not configure `efConstruction`, it scales as `base * sqrt(nodes / +AUTO_EFC_REF)` from the same memoized node count the search-side auto-scale already resolves, +capped at `AUTO_EFC_MAX`. Scaling starts at 250K nodes: efC 100 held recall through 500K +(0.978), so smaller graphs — the common case — build exactly as before. The sqrt shape mirrors the +search-side scale; the cost is build time (1.77x at 1M for efC 200), paid only by tables that +actually grow large, and partly returned as cheaper queries. An explicit `efConstruction` stays +authoritative: it is a per-index decision about build cost, and it also seeds the search `ef`, so +overriding it would surprise twice. Nodes indexed before the graph crossed a scale threshold keep +their original edges — the scale applies to inserts from that point on, and a reindex rebuilds +uniformly at the final size's efC. + ## An approximate index returns at most `ef` rows, so `limit` has to reach it Layer 0 keeps at most `ef` candidates, and ef resolves from the auto-scale, not from the query. A diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index f028ff07d..757951545 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -73,6 +73,21 @@ const ROUTING_EF = 1; // AUTO_EF_MAX so the worst case stays the same order as the index's own auto-scaled ceiling; a // caller who genuinely wants more sets an explicit `ef` and owns the cost. const LIMIT_EF_MAX = 4 * AUTO_EF_MAX; +// Auto-scaled construction ef, used only when an index does not explicitly configure efConstruction. +// The connection-building pass in index() selects each node's stored edges from a candidate list of +// this size; held constant while the corpus grows, edge quality erodes until true neighbours become +// unreachable at ANY search ef — at 1M nodes (768-dim, int8) recall@10 plateaued at 0.94–0.97 from +// ef 512 to 1536, while efConstruction 200 restored 0.985/0.997 AND made queries faster at the same +// ef (21% fewer nodes visited; better-selected edges route more directly). See #2180 for the sweep. +// Scaling starts at AUTO_EFC_REF nodes so smaller graphs build exactly as before; sqrt keeps growth +// gentle; the cap bounds per-insert cost (build time was 1.77x at 1M for efC 200). An explicitly +// configured efConstruction is a per-index decision about build cost and stays authoritative. +const AUTO_EFC_REF = 250_000; +const AUTO_EFC_MAX = 512; +function autoScaleEfConstruction(base: number, nodeCount: number): number { + const scaled = Math.round(base * Math.sqrt(Math.max(1, nodeCount / AUTO_EFC_REF))); + return Math.min(AUTO_EFC_MAX, Math.max(base, scaled)); +} // How long a resolved graph size is reused before it is looked up again (see approximateNodeCount). // ef moves with the square root of the count and is capped, so a slightly stale size is immaterial; // this only has to be short enough that a table growing from empty picks up a larger ef promptly. @@ -198,6 +213,7 @@ export class HierarchicalNavigableSmallWorld { distance: (a: number[], b: number[]) => number; int8 = true; // store vectors as int8-quantized bins by default; opt out with `quantization: "none"` efSearchConfigured = false; // whether the schema set an explicit search ef; if not, search ef auto-scales with N + efConstructionConfigured = false; // whether the schema set an explicit efConstruction; if not, it auto-scales with N // Caches the Int8Array-converted clone of a frozen (decoded-from-disk) int8 node, keyed by the // frozen node the object store hands back. WeakMap so entries are collected when the store evicts // the frozen node — without it, every cache hit on a frozen node would re-slice and re-clone. @@ -214,6 +230,7 @@ export class HierarchicalNavigableSmallWorld { this.int8 = options?.quantization !== 'none'; // Respect an explicitly-configured search ef (or efConstruction, which seeds it); otherwise auto-scale. this.efSearchConfigured = options?.efConstructionSearch !== undefined || options?.efConstruction !== undefined; + this.efConstructionConfigured = options?.efConstruction !== undefined; this.distance = options?.distance === 'euclidean' ? euclideanDistance @@ -375,9 +392,14 @@ export class HierarchicalNavigableSmallWorld { connections[i] = []; } - // Connect the new element to neighbors at its level and below + // Connect the new element to neighbors at its level and below. The candidate-list size + // auto-scales with the graph unless the schema pinned it; approximateNodeCount is memoized, + // so this stays O(1) per insert. + const efConstruction = this.efConstructionConfigured + ? this.efConstruction + : autoScaleEfConstruction(this.efConstruction, this.approximateNodeCount()); for (let l = Math.min(level, currentLevel); l >= 0; l--) { - let neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, l, options); + let neighbors = this.searchLayer(vector, entryPointId, entryPoint, efConstruction, l, options); neighbors = neighbors.slice(0, this.M << 1) as SearchResults; if (neighbors.length === 0 && l === 0) { diff --git a/unitTests/resources/vectorIndex.test.js b/unitTests/resources/vectorIndex.test.js index ff029b714..deedce13b 100644 --- a/unitTests/resources/vectorIndex.test.js +++ b/unitTests/resources/vectorIndex.test.js @@ -596,6 +596,97 @@ describe('HNSW graph-size resolution (drives the ef auto-scale)', () => { }); }); +describe('HNSW construction ef auto-scale (#2180)', () => { + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; + let T; + let Pinned; + + before(() => { + setupTestDBPath(); + setMainIsWorker(true); + T = table({ + table: 'HNSWEfcAutoScaleTest', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'vector', indexed: { type: 'HNSW', distance: 'cosine' }, type: 'Array' }, + ], + }); + Pinned = table({ + table: 'HNSWEfcPinnedTest', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'vector', indexed: { type: 'HNSW', distance: 'cosine', efConstruction: 64 }, type: 'Array' }, + ], + }); + }); + + after(() => { + T.dropTable(); + Pinned.dropTable(); + }); + + // The efs the connection pass actually searched with during one put. The routing descent runs at + // ROUTING_EF (1), so every ef > 1 seen during index() is the resolved efConstruction. + async function connectionEfsDuringPut(Table, id) { + const customIndex = Table.indices.vector.customIndex; + const original = Object.getPrototypeOf(customIndex).searchLayer; + const efs = new Set(); + customIndex.searchLayer = function (queryVector, entryPointId, entryPoint, ef, level, ...rest) { + if (ef > 1) efs.add(ef); + return original.call(this, queryVector, entryPointId, entryPoint, ef, level, ...rest); + }; + try { + const a = (id / 100) * Math.PI * 2; + await Table.put(id, { vector: [Math.cos(a), Math.sin(a), (id % 5) / 5] }); + } finally { + delete customIndex.searchLayer; + } + return efs; + } + + // Force the memoized node count so the scale point is exercised without building a 1M-node graph. + // The memo TTL (10s) comfortably covers one put. + function mockNodeCount(Table, count) { + const customIndex = Table.indices.vector.customIndex; + customIndex.nodeCount = count; + customIndex.nodeCountAt = Date.now(); + } + + it('builds small graphs at the base efConstruction, unchanged', async () => { + for (let i = 0; i < 20; i++) { + const a = (i / 20) * Math.PI * 2; + await T.put(i, { vector: [Math.cos(a), Math.sin(a), (i % 5) / 5] }); + } + const efs = await connectionEfsDuringPut(T, 20); + assert.deepStrictEqual([...efs], [100], `expected the base efConstruction below the scale point, got ${[...efs]}`); + }); + + it('scales the connection candidate list once the graph passes the reference size', async () => { + mockNodeCount(T, 1_000_000); + // base 100 * sqrt(1M / 250K) = 200 — the point measured in #2180 (recall 0.935 -> 0.985) + const efs = await connectionEfsDuringPut(T, 21); + assert.deepStrictEqual([...efs], [200], `expected the auto-scaled efConstruction at 1M nodes, got ${[...efs]}`); + }); + + it('caps the auto-scale at AUTO_EFC_MAX', async () => { + mockNodeCount(T, 100_000_000); + const efs = await connectionEfsDuringPut(T, 22); + assert.deepStrictEqual([...efs], [512], `expected the capped efConstruction, got ${[...efs]}`); + }); + + it('leaves an explicitly configured efConstruction authoritative at any graph size', async () => { + for (let i = 0; i < 5; i++) { + const a = (i / 5) * Math.PI * 2; + await Pinned.put(i, { vector: [Math.cos(a), Math.sin(a), (i % 5) / 5] }); + } + mockNodeCount(Pinned, 1_000_000); + const efs = await connectionEfsDuringPut(Pinned, 5); + assert.deepStrictEqual([...efs], [64], `expected the schema-configured efConstruction, got ${[...efs]}`); + }); +}); + describe('HNSW greedy routing above layer 0 (ROUTING_EF)', () => { if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; let T; From 3831f4f6b0ff6a01478d088accb3b19df509a6f6 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 05:36:07 -0600 Subject: [PATCH 02/12] =?UTF-8?q?feat(bench):=20--stream=20mode=20for=20hn?= =?UTF-8?q?sw-scale.js=20=E2=80=94=20corpus=20sizes=20past=20what=20a=20fl?= =?UTF-8?q?oat=20pool=20can=20hold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-pass generation from a dedicated RNG stream (independent of the Math.random the index draws levels from, so both passes replay identical rows): pass 1 collects evenly-spaced query source rows, pass 2 feeds each row to the index and folds it into the brute-force ground truth as it streams by. Peak memory is the graph plus one row, vs a 15GB pool at 5M x 768 dims. Co-Authored-By: Claude Fable 5 --- benchmarks/hnsw-scale.js | 218 +++++++++++++++++++++++++++++++++------ 1 file changed, 188 insertions(+), 30 deletions(-) diff --git a/benchmarks/hnsw-scale.js b/benchmarks/hnsw-scale.js index 366d9781d..592b01be9 100644 --- a/benchmarks/hnsw-scale.js +++ b/benchmarks/hnsw-scale.js @@ -11,6 +11,8 @@ * node benchmarks/hnsw-scale.js [--n=5000,10000,25000] [--dims=768] [--queries=50] * [--clusters=N] [--ef=auto] [--quantization=int8|none] * [--upper-ef=N] (override ef used above layer 0) + * [--stream] (two-pass generation; no float pool held — for N + * where the pool alone would not fit in memory) * [--json=path] */ @@ -55,6 +57,8 @@ const UPPER_EF = // median pairwise distance is ~0.37, not ~0.95), so any ef/recall conclusion has to be confirmed on // real vectors before it is trusted. const CORPUS = argv.corpus ? String(argv.corpus) : undefined; +const STREAM = argv.stream !== undefined; +if (STREAM && CORPUS) throw new Error('--stream generates synthetically; it cannot be combined with --corpus'); const INTRA_COS = Number(argv['intra-cos'] ?? 0.75); const NOISE = argv.noise !== undefined ? Number(argv.noise) : Math.sqrt((1 / (INTRA_COS * INTRA_COS) - 1) / DIMS); @@ -210,6 +214,148 @@ function rowToArray(pool, i, dims) { return out; } +/** gauss() over a caller-owned RNG, so a stream's draws are independent of Math.random. */ +function gaussStream(rand) { + let spare = null; + return () => { + if (spare !== null) { + const s = spare; + spare = null; + return s; + } + let u, v, s; + do { + u = rand() * 2 - 1; + v = rand() * 2 - 1; + s = u * u + v * v; + } while (s === 0 || s >= 1); + const mul = Math.sqrt((-2 * Math.log(s)) / s); + spare = v * mul; + return u * mul; + }; +} + +/** + * Row-at-a-time corpus generator for --stream: same Gaussian mixture as buildCorpus, but driven by + * its own RNG stream so re-instantiating it replays the identical rows regardless of how many draws + * the index makes from Math.random for level assignment in between. Returns a reused buffer — the + * caller copies what it keeps. + */ +function corpusStream(n, dims, nClusters, seed) { + const rand = mulberry32(seed); + const g = gaussStream(rand); + const centroids = new Float32Array(nClusters * dims); + for (let c = 0; c < nClusters; c++) { + let mag = 0; + for (let d = 0; d < dims; d++) { + const x = g(); + centroids[c * dims + d] = x; + mag += x * x; + } + mag = Math.sqrt(mag) || 1; + for (let d = 0; d < dims; d++) centroids[c * dims + d] /= mag; + } + const row = new Float32Array(dims); + let i = 0; + return () => { + if (i >= n) return null; + const c = (rand() * nClusters) | 0; + let mag = 0; + for (let d = 0; d < dims; d++) { + const x = centroids[c * dims + d] + g() * NOISE; + row[d] = x; + mag += x * x; + } + mag = Math.sqrt(mag) || 1; + for (let d = 0; d < dims; d++) row[d] /= mag; + i++; + return row; + }; +} + +/** + * Two-pass streaming build. Pass 1 replays the corpus to collect the query source rows (evenly + * spaced, like loadCorpus's holdout) and perturbs them into queries on yet another RNG stream. + * Pass 2 replays it again, feeding each row to the index and folding it into the brute-force + * ground truth and the corpus diagnostic as it streams by. Peak memory is the graph plus one row; + * GT folding time is measured per row and excluded from buildMs. + */ +function streamBuild(hnsw, n, dims, nClusters) { + const srcIdx = new Set(); + for (let q = 0; q < N_QUERIES; q++) srcIdx.add(Math.floor((q * n) / N_QUERIES)); + const sources = []; + { + const next = corpusStream(n, dims, nClusters, SEED + 1); + for (let i = 0; i < n; i++) { + const row = next(); + if (srcIdx.has(i)) sources.push(row.slice()); + } + } + const qg = gaussStream(mulberry32(SEED + 2)); + const queries = sources.map((src) => { + const v = Array.from(src); + let mag = 0; + for (let d = 0; d < dims; d++) { + v[d] += qg() * NOISE * 0.5; + mag += v[d] * v[d]; + } + mag = Math.sqrt(mag) || 1; + for (let d = 0; d < dims; d++) v[d] /= mag; + return v; + }); + + const gtTop = queries.map(() => []); // per query: ascending [{dist, i}], length <= TOP_K + const probes = Math.min(20, queries.length); + const sepNearBest = new Float64Array(probes).fill(Infinity); + let sepRandSum = 0; + const next = corpusStream(n, dims, nClusters, SEED + 1); + const buildStart = performance.now(); + let gtMsAcc = 0; + let lastLog = buildStart; + for (let i = 0; i < n; i++) { + const row = next(); + hnsw.index('r' + i, rowToArray(row, 0, dims), undefined, {}); + const t0 = performance.now(); + for (let q = 0; q < queries.length; q++) { + const query = queries[q]; + let dot = 0; + for (let d = 0; d < dims; d++) dot += row[d] * query[d]; + const dist = 1 - dot; + const top = gtTop[q]; + if (top.length < TOP_K || dist < top[top.length - 1].dist) { + let at = top.length; + while (at > 0 && top[at - 1].dist > dist) at--; + top.splice(at, 0, { dist, i }); + if (top.length > TOP_K) top.pop(); + } + if (i < 3000 && q < probes) { + if (dist < sepNearBest[q]) sepNearBest[q] = dist; + if (i < 200) sepRandSum += dist; + } + } + gtMsAcc += performance.now() - t0; + if ((i + 1) % 250_000 === 0) { + const now = performance.now(); + console.log( + ` … ${i + 1}/${n} indexed, ${((now - buildStart) / 60000).toFixed(1)}min elapsed, ` + + `${(250_000 / ((now - lastLog) / 1000)).toFixed(0)} rows/s this chunk` + ); + lastLog = now; + } + } + const buildMs = performance.now() - buildStart - gtMsAcc; + let sepNear = 0; + for (let p = 0; p < probes; p++) sepNear += sepNearBest[p]; + return { + buildMs, + queries, + groundTruth: gtTop.map((top) => new Set(top.map((t) => t.i))), + gtMs: gtMsAcc, + sepNear: sepNear / probes, + sepRand: sepRandSum / (probes * 200), + }; +} + /** Ground truth over unit-normalized rows: cosine distance = 1 - dot. */ function bruteForceTopK(pool, n, dims, query, k) { const dist = new Float64Array(n); @@ -302,7 +448,11 @@ for (const N of SIZES) { global.gc?.(); Math.random = mulberry32(SEED); // identical corpus + level assignments for every configuration _spare = null; - const { pool, queryPool } = CORPUS ? loadCorpus(CORPUS, N, DIMS, N_QUERIES) : buildCorpus(N, DIMS, nClusters); + const { pool, queryPool } = STREAM + ? {} + : CORPUS + ? loadCorpus(CORPUS, N, DIMS, N_QUERIES) + : buildCorpus(N, DIMS, nClusters); const store = new MemoryStore(); const options = { distance: 'cosine', quantization: QUANTIZATION }; @@ -313,38 +463,46 @@ for (const N of SIZES) { const hnsw = new HierarchicalNavigableSmallWorld(store, options); instrument(hnsw); - const buildStart = performance.now(); - for (let i = 0; i < N; i++) hnsw.index('r' + i, rowToArray(pool, i, DIMS), undefined, {}); - const buildMs = performance.now() - buildStart; + let buildMs; + let queries = []; + let groundTruth; + let gtMs = 0; + let sepNear = 0; + let sepRand = 0; + if (STREAM) { + ({ buildMs, queries, groundTruth, gtMs, sepNear, sepRand } = streamBuild(hnsw, N, DIMS, nClusters)); + } else { + const buildStart = performance.now(); + for (let i = 0; i < N; i++) hnsw.index('r' + i, rowToArray(pool, i, DIMS), undefined, {}); + buildMs = performance.now() - buildStart; + } const shape = graphShape(store); - // queries drawn from the same distribution (perturbed corpus rows) - const queries = []; - for (let q = 0; q < N_QUERIES; q++) { - if (queryPool) { - // real corpus: a held-out embedding, never indexed - queries.push(rowToArray(queryPool, q, DIMS)); - continue; - } - // synthetic: perturb an indexed row so rank 1 is not trivially the query itself - const src = (Math.random() * N) | 0; - const v = rowToArray(pool, src, DIMS); - let mag = 0; - for (let d = 0; d < DIMS; d++) { - v[d] += gauss() * NOISE * 0.5; - mag += v[d] * v[d]; + // queries drawn from the same distribution (perturbed corpus rows); --stream built its own above + if (!STREAM) + for (let q = 0; q < N_QUERIES; q++) { + if (queryPool) { + // real corpus: a held-out embedding, never indexed + queries.push(rowToArray(queryPool, q, DIMS)); + continue; + } + // synthetic: perturb an indexed row so rank 1 is not trivially the query itself + const src = (Math.random() * N) | 0; + const v = rowToArray(pool, src, DIMS); + let mag = 0; + for (let d = 0; d < DIMS; d++) { + v[d] += gauss() * NOISE * 0.5; + mag += v[d] * v[d]; + } + mag = Math.sqrt(mag) || 1; + for (let d = 0; d < DIMS; d++) v[d] /= mag; + queries.push(v); } - mag = Math.sqrt(mag) || 1; - for (let d = 0; d < DIMS; d++) v[d] /= mag; - queries.push(v); - } // corpus diagnostic: how separated is the true nearest neighbour from a random row? // (if these are close, the corpus is effectively uniform and no ANN can help) - let sepNear = 0; - let sepRand = 0; - { + if (!STREAM) { const probes = 20; for (let p = 0; p < probes; p++) { const q = queries[p % queries.length]; @@ -361,11 +519,11 @@ for (const N of SIZES) { } sepNear /= probes; sepRand /= probes * 200; - } - const gtStart = performance.now(); - const groundTruth = queries.map((q) => bruteForceTopK(pool, N, DIMS, q, TOP_K)); - const gtMs = performance.now() - gtStart; + const gtStart = performance.now(); + groundTruth = queries.map((q) => bruteForceTopK(pool, N, DIMS, q, TOP_K)); + gtMs = performance.now() - gtStart; + } for (const efSpec of EF_SWEEP) { const queryEf = efSpec === 'auto' ? undefined : Number(efSpec); From c487c674b00aaf68f309b88016fbb28402495e72 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 05:48:48 -0600 Subject: [PATCH 03/12] =?UTF-8?q?fix(hnsw):=20review=20fixes=20=E2=80=94?= =?UTF-8?q?=20read=20the=20id=20counter=20directly=20on=20the=20write=20pa?= =?UTF-8?q?th,=20own=20the=20count=20caveats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the pre-push cross-model review: resolve the build-side node count with resolveNodeCount() (one atomic load — the counter is always initialized on the write path) instead of the memoized approximateNodeCount(), removing TTL lag during bulk loads; document that the count is a lifetime high-water mark (churn-table build-cost caveat, bounded at the cap), that a post-restart reindex repeats the ramp, and that pinning efConstruction also pins search ef; update the stale schema.graphql description; fix the sepRand diagnostic below 200 rows; trim comments that restated DESIGN.md. Co-Authored-By: Claude Fable 5 --- DESIGN.md | 31 +++++++++++++------ benchmarks/hnsw-scale.js | 4 +-- .../HierarchicalNavigableSmallWorld.ts | 21 +++++-------- schema.graphql | 3 +- unitTests/resources/vectorIndex.test.js | 24 +++++++------- 5 files changed, 47 insertions(+), 36 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 5322103f9..9708d5036 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -804,15 +804,28 @@ visited vs 3,948 — better-selected edges route more directly). Quantization co (float32 rebuild: 0.952); construction quality was the dominant term. Full sweep in #2180. So when the schema does not configure `efConstruction`, it scales as `base * sqrt(nodes / -AUTO_EFC_REF)` from the same memoized node count the search-side auto-scale already resolves, -capped at `AUTO_EFC_MAX`. Scaling starts at 250K nodes: efC 100 held recall through 500K -(0.978), so smaller graphs — the common case — build exactly as before. The sqrt shape mirrors the -search-side scale; the cost is build time (1.77x at 1M for efC 200), paid only by tables that -actually grow large, and partly returned as cheaper queries. An explicit `efConstruction` stays -authoritative: it is a per-index decision about build cost, and it also seeds the search `ef`, so -overriding it would surprise twice. Nodes indexed before the graph crossed a scale threshold keep -their original edges — the scale applies to inserts from that point on, and a reindex rebuilds -uniformly at the final size's efC. +AUTO_EFC_REF)`, capped at `AUTO_EFC_MAX`, read on each insert directly from the id counter +(`resolveNodeCount` — always initialized on the write path, so this is one atomic load; the +search-side memo exists to keep the _fallback_ seek off the query path, which the write path never +takes). Scaling starts at 250K nodes: efC 100 held recall through 500K (0.978), so smaller graphs — +the common case — build exactly as before. The sqrt shape mirrors the search-side scale; the cost +is build time (1.77x at 1M for efC 200), paid only by tables that actually grow large, and partly +returned as cheaper queries. An explicit `efConstruction` stays authoritative: it is a per-index +decision about build cost — though note it also seeds the search `ef`, so pinning it to cut build +cost also pins query-time `ef`; there is currently no "pinned build, auto search" combination. + +Two caveats are accepted deliberately, both inherited from the count being a lifetime high-water +mark of allocated node ids rather than a live count. First, churn: a table that deletes heavily +(TTL eviction, delete-and-reinsert ingest) reads high forever, so its build-side efC can sit at the +cap while the live graph is small — bounded at `AUTO_EFC_MAX` (5.12x base build cost), it wastes +build CPU but never hurts recall. The search side accepted the same over-count as "slightly +generous ef" on an opt-in read path; the write path inherits it as a known cost until a live count +exists (tracked follow-up). Second, ramp history: nodes indexed before the graph crossed a scale +threshold keep their original edges — the scale applies to inserts from that point on. A reindex in +a live process rebuilds roughly uniformly (the id counter keeps its high-water mark), but a reindex +after a restart re-seeds the counter from the largest id in the rebuilding store and therefore +repeats the ramp — its first 250K nodes rebuild at the base efC. Both converge to the same steady +state as the graph grows past the knee. ## An approximate index returns at most `ef` rows, so `limit` has to reach it diff --git a/benchmarks/hnsw-scale.js b/benchmarks/hnsw-scale.js index 592b01be9..c02c4bed1 100644 --- a/benchmarks/hnsw-scale.js +++ b/benchmarks/hnsw-scale.js @@ -352,7 +352,7 @@ function streamBuild(hnsw, n, dims, nClusters) { groundTruth: gtTop.map((top) => new Set(top.map((t) => t.i))), gtMs: gtMsAcc, sepNear: sepNear / probes, - sepRand: sepRandSum / (probes * 200), + sepRand: sepRandSum / (probes * Math.min(200, n)), }; } @@ -518,7 +518,7 @@ for (const N of SIZES) { sepNear += best; } sepNear /= probes; - sepRand /= probes * 200; + sepRand /= probes * Math.min(200, N); const gtStart = performance.now(); groundTruth = queries.map((q) => bruteForceTopK(pool, N, DIMS, q, TOP_K)); diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index 757951545..679d50455 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -74,14 +74,9 @@ const ROUTING_EF = 1; // caller who genuinely wants more sets an explicit `ef` and owns the cost. const LIMIT_EF_MAX = 4 * AUTO_EF_MAX; // Auto-scaled construction ef, used only when an index does not explicitly configure efConstruction. -// The connection-building pass in index() selects each node's stored edges from a candidate list of -// this size; held constant while the corpus grows, edge quality erodes until true neighbours become -// unreachable at ANY search ef — at 1M nodes (768-dim, int8) recall@10 plateaued at 0.94–0.97 from -// ef 512 to 1536, while efConstruction 200 restored 0.985/0.997 AND made queries faster at the same -// ef (21% fewer nodes visited; better-selected edges route more directly). See #2180 for the sweep. -// Scaling starts at AUTO_EFC_REF nodes so smaller graphs build exactly as before; sqrt keeps growth -// gentle; the cap bounds per-insert cost (build time was 1.77x at 1M for efC 200). An explicitly -// configured efConstruction is a per-index decision about build cost and stays authoritative. +// At a constant efConstruction, edge quality erodes as the graph grows until true neighbours become +// unreachable at ANY search ef; the cap bounds per-insert cost. Measurements and policy in +// DESIGN.md ("efConstruction auto-scales with the graph") and #2180. const AUTO_EFC_REF = 250_000; const AUTO_EFC_MAX = 512; function autoScaleEfConstruction(base: number, nodeCount: number): number { @@ -228,7 +223,7 @@ export class HierarchicalNavigableSmallWorld { this.indexStore.encoder.useFloat32 = FLOAT32_OPTIONS.ALWAYS; } this.int8 = options?.quantization !== 'none'; - // Respect an explicitly-configured search ef (or efConstruction, which seeds it); otherwise auto-scale. + // Respect an explicitly-configured ef (efConstruction seeds the search ef too); otherwise auto-scale both. this.efSearchConfigured = options?.efConstructionSearch !== undefined || options?.efConstruction !== undefined; this.efConstructionConfigured = options?.efConstruction !== undefined; this.distance = @@ -392,12 +387,12 @@ export class HierarchicalNavigableSmallWorld { connections[i] = []; } - // Connect the new element to neighbors at its level and below. The candidate-list size - // auto-scales with the graph unless the schema pinned it; approximateNodeCount is memoized, - // so this stays O(1) per insert. + // The id counter is always initialized on the write path (the node id was just allocated + // from it), so resolveNodeCount is a plain atomic read here — no memo, no TTL lag. It is a + // lifetime high-water mark, not a live count; see DESIGN.md for the churn-table caveat. const efConstruction = this.efConstructionConfigured ? this.efConstruction - : autoScaleEfConstruction(this.efConstruction, this.approximateNodeCount()); + : autoScaleEfConstruction(this.efConstruction, this.resolveNodeCount()); for (let l = Math.min(level, currentLevel); l >= 0; l--) { let neighbors = this.searchLayer(vector, entryPointId, entryPoint, efConstruction, l, options); neighbors = neighbors.slice(0, this.M << 1) as SearchResults; diff --git a/schema.graphql b/schema.graphql index 3820724af..0d10d1ef5 100644 --- a/schema.graphql +++ b/schema.graphql @@ -188,7 +188,8 @@ directive @indexed( distance: String """ Construction effort/recall parameter (HNSW). Higher values improve recall at - the cost of build time and memory. + the cost of build time and memory. When omitted, it auto-scales with graph + size; setting it pins both the build-side value and the search-side default. """ efConstruction: Int """ diff --git a/unitTests/resources/vectorIndex.test.js b/unitTests/resources/vectorIndex.test.js index deedce13b..5506079cf 100644 --- a/unitTests/resources/vectorIndex.test.js +++ b/unitTests/resources/vectorIndex.test.js @@ -646,12 +646,17 @@ describe('HNSW construction ef auto-scale (#2180)', () => { return efs; } - // Force the memoized node count so the scale point is exercised without building a 1M-node graph. - // The memo TTL (10s) comfortably covers one put. - function mockNodeCount(Table, count) { + // Force the resolved node count (the write path reads it directly, not through the memo) so the + // scale point is exercised without building a 1M-node graph. Instance-level shadow; the count + // source itself is covered by the graph-size describe above. + async function withNodeCount(Table, count, run) { const customIndex = Table.indices.vector.customIndex; - customIndex.nodeCount = count; - customIndex.nodeCountAt = Date.now(); + customIndex.resolveNodeCount = () => count; + try { + return await run(); + } finally { + delete customIndex.resolveNodeCount; + } } it('builds small graphs at the base efConstruction, unchanged', async () => { @@ -664,15 +669,13 @@ describe('HNSW construction ef auto-scale (#2180)', () => { }); it('scales the connection candidate list once the graph passes the reference size', async () => { - mockNodeCount(T, 1_000_000); // base 100 * sqrt(1M / 250K) = 200 — the point measured in #2180 (recall 0.935 -> 0.985) - const efs = await connectionEfsDuringPut(T, 21); + const efs = await withNodeCount(T, 1_000_000, () => connectionEfsDuringPut(T, 21)); assert.deepStrictEqual([...efs], [200], `expected the auto-scaled efConstruction at 1M nodes, got ${[...efs]}`); }); it('caps the auto-scale at AUTO_EFC_MAX', async () => { - mockNodeCount(T, 100_000_000); - const efs = await connectionEfsDuringPut(T, 22); + const efs = await withNodeCount(T, 100_000_000, () => connectionEfsDuringPut(T, 22)); assert.deepStrictEqual([...efs], [512], `expected the capped efConstruction, got ${[...efs]}`); }); @@ -681,8 +684,7 @@ describe('HNSW construction ef auto-scale (#2180)', () => { const a = (i / 5) * Math.PI * 2; await Pinned.put(i, { vector: [Math.cos(a), Math.sin(a), (i % 5) / 5] }); } - mockNodeCount(Pinned, 1_000_000); - const efs = await connectionEfsDuringPut(Pinned, 5); + const efs = await withNodeCount(Pinned, 1_000_000, () => connectionEfsDuringPut(Pinned, 5)); assert.deepStrictEqual([...efs], [64], `expected the schema-configured efConstruction, got ${[...efs]}`); }); }); From adb7f089aa731bebbac2de12c89fc916575ba532 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 05:55:12 -0600 Subject: [PATCH 04/12] fix(hnsw): ensure the shared id counter on the connection pass, not only at id allocation An update-only worker after a restart never allocates a node id, so the counter stayed uninitialized and the direct count resolution fell back to a reverse seek on every update. Extract the create-or-attach into ensureIdIncrementer() and call it from the connection pass: one seek on the first write after open, atomic loads after. Co-Authored-By: Claude Fable 5 --- .../HierarchicalNavigableSmallWorld.ts | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index 679d50455..f7868a774 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -270,24 +270,8 @@ export class HierarchicalNavigableSmallWorld { // that won't collide with the node ids, so we can't have a collision with internal if (!nodeId) { if (!vector) return; // didn't exist before, doesn't exist now, nothing to do - if (!this.idIncrementer) { - let largestNodeId = 0; - for (const key of this.indexStore.getKeys({ - reverse: true, - limit: 1, - start: Infinity, - end: 0, - transaction: options.transaction, - })) { - if (typeof key === 'number') largestNodeId = key; - } - - this.idIncrementer = new BigInt64Array([BigInt(largestNodeId) + 1n]); - this.idIncrementer = new BigInt64Array( - this.indexStore.getUserSharedBuffer('next-id', this.idIncrementer.buffer) - ); - } - nodeId = Number(Atomics.add(this.idIncrementer, 0, 1n)); + this.ensureIdIncrementer(options); + nodeId = Number(Atomics.add(this.idIncrementer!, 0, 1n)); this.indexStore.put(safeKey, nodeId, options); } const updatedNodes = new Map(); @@ -387,12 +371,15 @@ export class HierarchicalNavigableSmallWorld { connections[i] = []; } - // The id counter is always initialized on the write path (the node id was just allocated - // from it), so resolveNodeCount is a plain atomic read here — no memo, no TTL lag. It is a + // The counter is ensured here (not only at id allocation) because an update-only worker + // after a restart never allocates an id — without it, every update would pay the reverse + // seek. Ensured, resolveNodeCount is a plain atomic read — no memo, no TTL lag. It is a // lifetime high-water mark, not a live count; see DESIGN.md for the churn-table caveat. - const efConstruction = this.efConstructionConfigured - ? this.efConstruction - : autoScaleEfConstruction(this.efConstruction, this.resolveNodeCount()); + let efConstruction = this.efConstruction; + if (!this.efConstructionConfigured) { + this.ensureIdIncrementer(options); + efConstruction = autoScaleEfConstruction(this.efConstruction, this.resolveNodeCount()); + } for (let l = Math.min(level, currentLevel); l >= 0; l--) { let neighbors = this.searchLayer(vector, entryPointId, entryPoint, efConstruction, l, options); neighbors = neighbors.slice(0, this.M << 1) as SearchResults; @@ -742,6 +729,27 @@ export class HierarchicalNavigableSmallWorld { return this.nodeCount; } + /** + * Create-or-attach the shared id counter, seeded from a one-time reverse seek to the largest node + * id. getUserSharedBuffer returns the existing shared buffer when another worker created it + * first, so the seed only matters for whoever wins the race. + */ + private ensureIdIncrementer(options?: any): void { + if (this.idIncrementer) return; + let largestNodeId = 0; + for (const key of this.indexStore.getKeys({ + reverse: true, + limit: 1, + start: Infinity, + end: 0, + transaction: options?.transaction, + })) { + if (typeof key === 'number') largestNodeId = key; + } + this.idIncrementer = new BigInt64Array([BigInt(largestNodeId) + 1n]); + this.idIncrementer = new BigInt64Array(this.indexStore.getUserSharedBuffer('next-id', this.idIncrementer.buffer)); + } + /** O(1) node count — the shared id counter, else a single reverse seek to the largest node id. */ private resolveNodeCount(): number { if (this.idIncrementer) return Number(Atomics.load(this.idIncrementer, 0)); From 436f3e5cc613e855acaca8529c29159117ebdd1d Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 05:56:55 -0600 Subject: [PATCH 05/12] =?UTF-8?q?docs(schema):=20correct=20the=20efConstru?= =?UTF-8?q?ctionSearch=20description=20=E2=80=94=20it=20is=20the=20query-t?= =?UTF-8?q?ime=20ef?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- schema.graphql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/schema.graphql b/schema.graphql index 0d10d1ef5..bb79477d4 100644 --- a/schema.graphql +++ b/schema.graphql @@ -206,8 +206,9 @@ directive @indexed( """ mL: Int """ - Search-time effort/recall parameter used during construction (implementation- - specific). Larger values typically yield better accuracy. + Search-time effort/recall parameter (HNSW): the candidate-list size used when + querying. Larger values typically yield better accuracy at the cost of query + latency. When omitted, it auto-scales with graph size. """ efConstructionSearch: Int ) on FIELD_DEFINITION From 342c1abe8686a4aee3b62323e5d24ee3e0a5c4bf Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 05:07:36 -0600 Subject: [PATCH 06/12] feat(hnsw): resume the search-ef auto-scale past its 512 plateau on large graphs AUTO_EF_MAX was calibrated when layers above 0 were searched at the full ef, which made large efs cost seconds; after the greedy-descent fix the same headroom costs tens of milliseconds, and the measured decay at a pinned 512 on well-built graphs (set-recall 0.997 -> 0.955 -> 0.935 across 1M/2M/5M) is recall left on the table. Past 1M nodes the scale resumes from the plateau (512 * sqrt(nodes/1M)) up to AUTO_EF_CEILING (2048, binding ~16M); the 5M point resolves 1,145, bracketed by the measured ef-1024 sweep there (0.985 set-recall). Raise AUTO_EFC_MAX to 1024 on the same curve (validated to 447 at 5M), and re-derive LIMIT_EF_MAX from the new ceiling. Latency-first apps pin efConstructionSearch or a per-query ef; graphs past tens of millions should shard. Co-Authored-By: Claude Fable 5 --- DESIGN.md | 16 +++++++- .../HierarchicalNavigableSmallWorld.ts | 37 ++++++++++++++----- unitTests/resources/vectorIndex.test.js | 28 ++++++++++++-- 3 files changed, 67 insertions(+), 14 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 9708d5036..bb367cade 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -791,7 +791,7 @@ graphs being identical, though equal metrics do not prove it. It is the expected the upper layers are sparse enough that a greedy walk reaches the same entry point, which is why standard HNSW descends this way. -## `efConstruction` auto-scales with the graph, for the same reason search `ef` does +## `efConstruction` and the search-`ef` ceiling both auto-scale with the graph The connection-building pass selects each node's stored edges from a candidate list of `efConstruction` entries. Held at a constant (100) while the corpus grows, edge quality erodes in a @@ -814,6 +814,20 @@ returned as cheaper queries. An explicit `efConstruction` stays authoritative: i decision about build cost — though note it also seeds the search `ef`, so pinning it to cut build cost also pins query-time `ef`; there is currently no "pinned build, auto search" combination. +The search side scales past its old plateau for the same reason. `AUTO_EF_MAX` (512, pinned from +~13K nodes) was calibrated when layers above 0 were searched at the full `ef`, which made large efs +cost seconds; after the greedy-descent fix the same headroom costs tens of milliseconds (ef 1024 at +5M nodes: ~45ms p50), and holding the pin leaves measured recall on the table — set-recall at a +pinned 512 on well-built graphs decays 0.997 → 0.955 → 0.935 across 1M/2M/5M. So past +`AUTO_EF_LARGE_REF` (1M nodes, where 512 was last measured sufficient) the scale resumes from the +plateau — `512 * sqrt(nodes / 1M)` — up to `AUTO_EF_CEILING` (2048, binding at ~16M). The 5M point +resolves 1,145, bracketed by the measured ef-1024 sweep there (0.985 set). The default's query +latency therefore grows as sqrt(N) on large tables; that is the recall-first trade chosen here, and +apps preferring latency pin `efConstructionSearch` or a per-query `ef`. Both ceilings are finite on +purpose: total build work grows as N^1.5 under sqrt scaling, and past roughly tens of millions of +nodes per graph, sharded medium graphs beat one huge graph on build and query cost alike — scaling +the constants further is the wrong tool there. + Two caveats are accepted deliberately, both inherited from the count being a lifetime high-water mark of allocated node ids rather than a live count. First, churn: a table that deletes heavily (TTL eviction, delete-and-reinsert ingest) reads high forever, so its build-side efC can sit at the diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index f7868a774..10ea2f13a 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -43,11 +43,16 @@ function dequantizeInt8(q: Int8Array, scale: number): number[] { // Auto-scaled search ef, used only when an index does not explicitly configure efConstructionSearch // and a query does not pass its own ef. A fixed ef makes recall decay as the graph grows (it explores -// a shrinking fraction of the graph), so ef grows with sqrt(node count), capped to bound search cost. -// Constants from a recall/latency-vs-N sweep (768-dim cosine, int8): ef≈400 holds ~0.8 recall@10 from -// 5K–30K, and the recall/latency tradeoff is steep (ef 800 at 30K ≈ 0.92 recall but ~2s p50), so the -// cap deliberately favors latency — apps wanting higher recall set efConstructionSearch or a per-query -// ef. Tune as graph build quality / larger-N data improves. +// a shrinking fraction of the graph), so ef grows with sqrt(node count) in two regimes, with a +// ceiling to bound search cost. The first regime's constants come from the original recall/latency +// sweep (5K-30K, 768-dim cosine, int8) and plateau at AUTO_EF_MAX from ~13K nodes; that plateau was +// calibrated when layers above 0 were searched at the full ef, which made large efs cost seconds. +// After the greedy-descent fix (#2125) ef 1024 at 5M nodes costs ~45ms, and the measured decay at a +// pinned 512 (set-recall 0.997 -> 0.955 -> 0.935 across 1M/2M/5M on well-built graphs, #2181) is +// recall left on the table, so past AUTO_EF_LARGE_REF nodes the scale resumes from that plateau and +// runs to AUTO_EF_CEILING. Validated against the same sweeps: the second regime resolves 1,145 at +// 5M, and the measured ef-1024 point there holds 0.985. Apps preferring latency pin +// efConstructionSearch or a per-query ef; graphs past ~tens of millions of nodes should shard. const AUTO_EF_BASE = 100; // The index store holds a graph node plus a primary-key mapping per record, so a key count is twice // the node count. Sizes here are in nodes; this converts back for the one consumer still calibrated @@ -58,7 +63,15 @@ const INDEX_KEYS_PER_NODE = 2; // a live count, so the resolved ef can differ from that formula by one at a rounding boundary. const AUTO_EF_REF = 500; const AUTO_EF_MAX = 512; +// Nodes at which the second regime starts: 512 was measured sufficient through 1M (set-recall +// 0.997) and short from 2M up, so the resumed curve is anchored to pass through (1M, 512). +const AUTO_EF_LARGE_REF = 1_000_000; +const AUTO_EF_CEILING = 2048; function autoScaleEf(nodeCount: number): number { + if (nodeCount > AUTO_EF_LARGE_REF) { + const scaled = Math.round(AUTO_EF_MAX * Math.sqrt(nodeCount / AUTO_EF_LARGE_REF)); + return Math.min(AUTO_EF_CEILING, scaled); + } const scaled = Math.round(AUTO_EF_BASE * Math.sqrt(Math.max(1, nodeCount / AUTO_EF_REF))); return Math.min(AUTO_EF_MAX, Math.max(AUTO_EF_BASE, scaled)); } @@ -69,16 +82,20 @@ function autoScaleEf(nodeCount: number): number { const ROUTING_EF = 1; // Ceiling on the ef a query's own `offset + limit` can ask for. `limit` is unprivileged and set on // every request, so this bounds what a caller can make one thread do synchronously: layer 0 holds -// `ef` candidates in a sorted array with an O(len) insert. Kept within a small multiple of -// AUTO_EF_MAX so the worst case stays the same order as the index's own auto-scaled ceiling; a -// caller who genuinely wants more sets an explicit `ef` and owns the cost. -const LIMIT_EF_MAX = 4 * AUTO_EF_MAX; +// `ef` candidates in a sorted array with an O(len) insert. Kept within a small multiple of the +// index's own auto-scaled ceiling so the worst case stays the same order; a caller who genuinely +// wants more sets an explicit `ef` and owns the cost. +const LIMIT_EF_MAX = 2 * AUTO_EF_CEILING; // Auto-scaled construction ef, used only when an index does not explicitly configure efConstruction. // At a constant efConstruction, edge quality erodes as the graph grows until true neighbours become // unreachable at ANY search ef; the cap bounds per-insert cost. Measurements and policy in // DESIGN.md ("efConstruction auto-scales with the graph") and #2180. const AUTO_EFC_REF = 250_000; -const AUTO_EFC_MAX = 512; +// Validated to 447 (the 5M point, where the resulting graph held 0.985 set-recall at ef 1024); the +// headroom to 1024 is the same sqrt curve extrapolated, binding at ~26M nodes. Build cost grows +// with the curve (N^1.5 total under sqrt scaling), which is why the cap stays finite: past ~tens of +// millions of nodes, sharded medium graphs beat one huge graph on both build and query cost. +const AUTO_EFC_MAX = 1024; function autoScaleEfConstruction(base: number, nodeCount: number): number { const scaled = Math.round(base * Math.sqrt(Math.max(1, nodeCount / AUTO_EFC_REF))); return Math.min(AUTO_EFC_MAX, Math.max(base, scaled)); diff --git a/unitTests/resources/vectorIndex.test.js b/unitTests/resources/vectorIndex.test.js index 5506079cf..e536dd82c 100644 --- a/unitTests/resources/vectorIndex.test.js +++ b/unitTests/resources/vectorIndex.test.js @@ -675,9 +675,31 @@ describe('HNSW construction ef auto-scale (#2180)', () => { }); it('caps the auto-scale at AUTO_EFC_MAX', async () => { + // 100 * sqrt(100M / 250K) = 2000, capped at 1024 const efs = await withNodeCount(T, 100_000_000, () => connectionEfsDuringPut(T, 22)); - assert.deepStrictEqual([...efs], [512], `expected the capped efConstruction, got ${[...efs]}`); - }); + assert.deepStrictEqual([...efs], [1024], `expected the capped efConstruction, got ${[...efs]}`); + }); + + // The search-side scale resumes past AUTO_EF_LARGE_REF (the 512 plateau was measured sufficient + // through 1M nodes and short from 2M up — set-recall 0.997 -> 0.955 -> 0.935 across 1M/2M/5M). + // The second regime is anchored to pass through (1M, 512); the 5M point resolves 1,145, bracketed + // by the measured ef-1024 sweep there (0.985 set-recall). + for (const [nodes, expected] of [ + [1_000_000, 512], + [2_000_000, 724], + [5_000_000, 1145], + [100_000_000, 2048], + ]) { + it(`resolves search ef ${expected} at ${nodes.toLocaleString('en-US')} nodes`, async () => { + const efs = await withNodeCount(T, nodes, async () => { + const customIndex = T.indices.vector.customIndex; + customIndex.nodeCountAt = 0; // expire the memo so the search re-resolves through the shadow + const layer0Ef = await captureLayer0Ef(T, { limit: 10 }); + return layer0Ef; + }); + assert.strictEqual(efs, expected, `expected the auto-scaled search ef at ${nodes} nodes, got ${efs}`); + }); + } it('leaves an explicitly configured efConstruction authoritative at any graph size', async () => { for (let i = 0; i < 5; i++) { @@ -850,7 +872,7 @@ describe('HNSW limit above the resolved search ef', () => { ]) { it(`bounds the limit-derived ef at LIMIT_EF_MAX under ${label}`, async () => { const layer0Ef = await captureLayer0Ef(T, query); - assert.strictEqual(layer0Ef, 4 * 512, `layer-0 ef should be LIMIT_EF_MAX (${4 * 512}), got ${layer0Ef}`); + assert.strictEqual(layer0Ef, 2 * 2048, `layer-0 ef should be LIMIT_EF_MAX (${2 * 2048}), got ${layer0Ef}`); }); } From 276e58d64204e69af5f4428d1a507854c7d71640 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 05:22:52 -0600 Subject: [PATCH 07/12] =?UTF-8?q?fix(hnsw):=20round-4=20review=20fixes=20?= =?UTF-8?q?=E2=80=94=20decouple=20the=20filtered=20budget=20from=20the=20r?= =?UTF-8?q?aised=20ef=20ceiling,=20guard=20the=20write-path=20count?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The filtered-traversal budget (maxVisits, #1241) no longer follows the second-regime search ef: each budgeted visit is a synchronous record load + predicate evaluation, so an auto-scaled ef contributes at most AUTO_EF_MAX to the budget (explicit per-query or schema ef still raises it — the caller owns that cost). Guard the connection pass's counter ensure/resolve so a write can never fail on an ef heuristic with a safe fallback. Debug-log the resolved efC once per value so replica-divergent build quality is diagnosable. Correct the stale 5.12x churn-cost figure in DESIGN.md (AUTO_EFC_MAX is now 1024: 10.24x ef, ~6-7x build time), fix a stale DESIGN.md heading pointer, and add a mechanism test proving the update-only path ensures the shared counter with one seek rather than seeking per write. Co-Authored-By: Claude Fable 5 --- DESIGN.md | 12 +++++-- .../HierarchicalNavigableSmallWorld.ts | 36 ++++++++++++++----- unitTests/resources/vectorIndex.test.js | 26 ++++++++++++++ 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index bb367cade..cd8b5fb4d 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -823,7 +823,12 @@ pinned 512 on well-built graphs decays 0.997 → 0.955 → 0.935 across 1M/2M/5M plateau — `512 * sqrt(nodes / 1M)` — up to `AUTO_EF_CEILING` (2048, binding at ~16M). The 5M point resolves 1,145, bracketed by the measured ef-1024 sweep there (0.985 set). The default's query latency therefore grows as sqrt(N) on large tables; that is the recall-first trade chosen here, and -apps preferring latency pin `efConstructionSearch` or a per-query `ef`. Both ceilings are finite on +apps preferring latency pin `efConstructionSearch` or a per-query `ef`. The filtered-traversal +budget (`maxVisits`, #1241) deliberately does not follow the second regime: each budgeted visit is +a synchronous record load plus predicate evaluation, so an auto-scaled ef's budget contribution +stays capped at `AUTO_EF_MAX` — the recall decision and the filtered-scan bound are separate +decisions, and an explicit ef (per-query or schema) still raises the budget for callers who own +the cost. Both ceilings are finite on purpose: total build work grows as N^1.5 under sqrt scaling, and past roughly tens of millions of nodes per graph, sharded medium graphs beat one huge graph on build and query cost alike — scaling the constants further is the wrong tool there. @@ -831,8 +836,9 @@ the constants further is the wrong tool there. Two caveats are accepted deliberately, both inherited from the count being a lifetime high-water mark of allocated node ids rather than a live count. First, churn: a table that deletes heavily (TTL eviction, delete-and-reinsert ingest) reads high forever, so its build-side efC can sit at the -cap while the live graph is small — bounded at `AUTO_EFC_MAX` (5.12x base build cost), it wastes -build CPU but never hurts recall. The search side accepted the same over-count as "slightly +cap while the live graph is small — bounded at `AUTO_EFC_MAX` (10.24x the base ef; by the measured +1.77x-wall-time-per-2x-ef relation, roughly 6–7x build time), it wastes build CPU but never hurts +recall. The search side accepted the same over-count as "slightly generous ef" on an opt-in read path; the write path inherits it as a known cost until a live count exists (tracked follow-up). Second, ramp history: nodes indexed before the graph crossed a scale threshold keep their original edges — the scale applies to inserts from that point on. A reindex in diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index 10ea2f13a..2504f2036 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -89,7 +89,7 @@ const LIMIT_EF_MAX = 2 * AUTO_EF_CEILING; // Auto-scaled construction ef, used only when an index does not explicitly configure efConstruction. // At a constant efConstruction, edge quality erodes as the graph grows until true neighbours become // unreachable at ANY search ef; the cap bounds per-insert cost. Measurements and policy in -// DESIGN.md ("efConstruction auto-scales with the graph") and #2180. +// DESIGN.md ("efConstruction and the search-ef ceiling both auto-scale with the graph") and #2180. const AUTO_EFC_REF = 250_000; // Validated to 447 (the 5M point, where the resulting graph held 0.985 set-recall at ef 1024); the // headroom to 1024 is the same sqrt curve extrapolated, binding at ~26M nodes. Build cost grows @@ -226,6 +226,7 @@ export class HierarchicalNavigableSmallWorld { int8 = true; // store vectors as int8-quantized bins by default; opt out with `quantization: "none"` efSearchConfigured = false; // whether the schema set an explicit search ef; if not, search ef auto-scales with N efConstructionConfigured = false; // whether the schema set an explicit efConstruction; if not, it auto-scales with N + private lastLoggedEfConstruction = 0; // Caches the Int8Array-converted clone of a frozen (decoded-from-disk) int8 node, keyed by the // frozen node the object store hands back. WeakMap so entries are collected when the store evicts // the frozen node — without it, every cache hit on a frozen node would re-slice and re-clone. @@ -388,14 +389,24 @@ export class HierarchicalNavigableSmallWorld { connections[i] = []; } - // The counter is ensured here (not only at id allocation) because an update-only worker - // after a restart never allocates an id — without it, every update would pay the reverse - // seek. Ensured, resolveNodeCount is a plain atomic read — no memo, no TTL lag. It is a - // lifetime high-water mark, not a live count; see DESIGN.md for the churn-table caveat. + // Ensured here (not only at id allocation) so an update-only worker reads the count with an + // atomic load instead of a per-write seek. The count is a lifetime high-water mark, not a + // live count; see DESIGN.md for the churn-table caveat. Guarded because the count is an ef + // heuristic with a safe fallback — a write must never fail on it. let efConstruction = this.efConstruction; if (!this.efConstructionConfigured) { - this.ensureIdIncrementer(options); - efConstruction = autoScaleEfConstruction(this.efConstruction, this.resolveNodeCount()); + try { + this.ensureIdIncrementer(options); + efConstruction = autoScaleEfConstruction(this.efConstruction, this.resolveNodeCount()); + } catch (error) { + logger.debug?.('could not resolve the node count for the construction ef', error); + } + if (efConstruction > this.efConstruction && this.lastLoggedEfConstruction !== efConstruction) { + // once per resolved value per process: makes replica-divergent build quality and the + // build-cost ramp diagnosable (the resolved value is otherwise surfaced nowhere) + this.lastLoggedEfConstruction = efConstruction; + logger.debug?.(`HNSW construction ef auto-scaled to ${efConstruction}`); + } } for (let l = Math.min(level, currentLevel); l >= 0; l--) { let neighbors = this.searchLayer(vector, entryPointId, entryPoint, efConstruction, l, options); @@ -1094,10 +1105,17 @@ export class HierarchicalNavigableSmallWorld { // filter, so bound layer-0 work at ef * filterExpansion nodes. Only built when a filter is active. // Deliberately `resolvedEf`, not the limit-widened ef: this budget is what stops a selective // filter crawling the whole graph, and multiplying it by a caller's limit would turn a filtered - // vector query into a record-loading scan. + // vector query into a record-loading scan. When the ef came from the auto-scale (neither a + // per-query ef nor a schema-configured one), its budget contribution is additionally capped at + // AUTO_EF_MAX: every budgeted visit is a synchronous record load + predicate evaluation, so the + // second-regime search ef (up to AUTO_EF_CEILING) must not silently quadruple the filtered + // worst case — the recall decision and the filtered-scan budget are separate decisions. Callers + // wanting a deeper filtered search set an explicit ef or filterExpansion, and own the cost. const filterState: FilterState | undefined = filter ? { - maxVisits: resolvedEf * (filterExpansion && filterExpansion > 0 ? filterExpansion : this.filterExpansion), + maxVisits: + (explicitEf || this.efSearchConfigured ? resolvedEf : Math.min(resolvedEf, AUTO_EF_MAX)) * + (filterExpansion && filterExpansion > 0 ? filterExpansion : this.filterExpansion), nodesVisited: 0, filterEvaluations: 0, } diff --git a/unitTests/resources/vectorIndex.test.js b/unitTests/resources/vectorIndex.test.js index e536dd82c..c09165511 100644 --- a/unitTests/resources/vectorIndex.test.js +++ b/unitTests/resources/vectorIndex.test.js @@ -701,6 +701,32 @@ describe('HNSW construction ef auto-scale (#2180)', () => { }); } + // The refactor's stated mechanism, not just the formula: an update-only worker (counter absent, + // as after a restart with no new inserts) must ensure the shared counter with ONE seek, then read + // it atomically — never a reverse seek per write, and never a failed write if the seek throws. + it('an update-only path ensures the shared counter once instead of seeking per write', async () => { + const customIndex = T.indices.vector.customIndex; + const saved = customIndex.idIncrementer; + customIndex.idIncrementer = undefined; + const store = customIndex.indexStore; + const originalGetKeys = store.getKeys; + let seeks = 0; + store.getKeys = function (opts, ...rest) { + if (opts?.reverse) seeks++; + return originalGetKeys.call(this, opts, ...rest); + }; + try { + // ids 0 and 1 already exist in T, so both puts take the update path and never allocate an id + await T.put(0, { vector: [1, 0, 0.2] }); + await T.put(1, { vector: [0.9, 0.1, 0.2] }); + assert(customIndex.idIncrementer, 'the update path must ensure the shared counter'); + assert.strictEqual(seeks, 1, `the counter seed should be the only reverse seek, got ${seeks}`); + } finally { + store.getKeys = originalGetKeys; + if (!customIndex.idIncrementer) customIndex.idIncrementer = saved; + } + }); + it('leaves an explicitly configured efConstruction authoritative at any graph size', async () => { for (let i = 0; i < 5; i++) { const a = (i / 5) * Math.PI * 2; From c6784e53b088abdcb999285d0a8b32727120c5fd Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 05:26:38 -0600 Subject: [PATCH 08/12] fix(hnsw): never install the id counter until the shared-buffer attach succeeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assigning the private seed array to this.idIncrementer before attaching meant a thrown attach (previously fatal to the write, made silently reachable by the round-4 guard) left this process allocating ids from a counter no other worker can see — cross-worker id collisions and graph corruption. Install only the attached shared view; on failure the counter stays unset and the next write retries. Test simulates a failing attach: the write succeeds on the base-efC fallback, no counter is installed, and the next write recovers. Co-Authored-By: Claude Fable 5 --- .../HierarchicalNavigableSmallWorld.ts | 7 +++-- unitTests/resources/vectorIndex.test.js | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index 2504f2036..c8121bd34 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -774,8 +774,11 @@ export class HierarchicalNavigableSmallWorld { })) { if (typeof key === 'number') largestNodeId = key; } - this.idIncrementer = new BigInt64Array([BigInt(largestNodeId) + 1n]); - this.idIncrementer = new BigInt64Array(this.indexStore.getUserSharedBuffer('next-id', this.idIncrementer.buffer)); + // Never install the counter until the shared attach succeeds: assigning the private seed + // array first would, on an attach failure, leave THIS process allocating ids nobody else can + // see — cross-worker id collisions. Left unset, the next write simply retries the ensure. + const seed = new BigInt64Array([BigInt(largestNodeId) + 1n]); + this.idIncrementer = new BigInt64Array(this.indexStore.getUserSharedBuffer('next-id', seed.buffer)); } /** O(1) node count — the shared id counter, else a single reverse seek to the largest node id. */ diff --git a/unitTests/resources/vectorIndex.test.js b/unitTests/resources/vectorIndex.test.js index c09165511..815016d63 100644 --- a/unitTests/resources/vectorIndex.test.js +++ b/unitTests/resources/vectorIndex.test.js @@ -727,6 +727,32 @@ describe('HNSW construction ef auto-scale (#2180)', () => { } }); + // A failed shared-buffer attach must not install a private counter (ids allocated from it would + // collide with other workers'). The write still succeeds (base efC fallback) and the next write + // retries the ensure against the healthy store. + it('does not install a private counter when the shared attach fails, and retries next write', async () => { + const customIndex = T.indices.vector.customIndex; + const saved = customIndex.idIncrementer; + customIndex.idIncrementer = undefined; + const store = customIndex.indexStore; + const originalGetUserSharedBuffer = store.getUserSharedBuffer; + store.getUserSharedBuffer = () => { + throw new Error('simulated shared-buffer attach failure'); + }; + try { + await T.put(0, { vector: [0.8, 0.2, 0.1] }); // update path; efC falls back, write succeeds + assert.strictEqual(customIndex.idIncrementer, undefined, 'a failed attach must not install a counter'); + } finally { + store.getUserSharedBuffer = originalGetUserSharedBuffer; + } + try { + await T.put(1, { vector: [0.7, 0.3, 0.1] }); + assert(customIndex.idIncrementer, 'the ensure must retry once the store is healthy'); + } finally { + if (!customIndex.idIncrementer) customIndex.idIncrementer = saved; + } + }); + it('leaves an explicitly configured efConstruction authoritative at any graph size', async () => { for (let i = 0; i < 5; i++) { const a = (i / 5) * Math.PI * 2; From afe769fd7f35e26215f4b250b8e72f4b2a73b81b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 22:12:02 -0600 Subject: [PATCH 09/12] Address HNSW autoscale review feedback Add second-regime filtered-budget coverage, preserve construction scaling when a shared-counter attach fails, expose resolved efConstruction in the benchmark, and document the operator-facing cost and configuration tradeoffs. Co-Authored-By: GPT-5 Codex --- DESIGN.md | 53 +++++++++++------- benchmarks/hnsw-scale.js | 24 +++++++- .../HierarchicalNavigableSmallWorld.ts | 39 ++++++++----- schema.graphql | 7 ++- unitTests/resources/vectorIndex.test.js | 55 ++++++++++++++++--- 5 files changed, 134 insertions(+), 44 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index cd8b5fb4d..91fd42a49 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -803,16 +803,21 @@ unreachable. Rebuilding the identical corpus (same seed, same level assignments) visited vs 3,948 — better-selected edges route more directly). Quantization contributed ~1.5 points (float32 rebuild: 0.952); construction quality was the dominant term. Full sweep in #2180. -So when the schema does not configure `efConstruction`, it scales as `base * sqrt(nodes / -AUTO_EFC_REF)`, capped at `AUTO_EFC_MAX`, read on each insert directly from the id counter -(`resolveNodeCount` — always initialized on the write path, so this is one atomic load; the -search-side memo exists to keep the _fallback_ seek off the query path, which the write path never -takes). Scaling starts at 250K nodes: efC 100 held recall through 500K (0.978), so smaller graphs — -the common case — build exactly as before. The sqrt shape mirrors the search-side scale; the cost -is build time (1.77x at 1M for efC 200), paid only by tables that actually grow large, and partly -returned as cheaper queries. An explicit `efConstruction` stays authoritative: it is a per-index -decision about build cost — though note it also seeds the search `ef`, so pinning it to cut build -cost also pins query-time `ef`; there is currently no "pinned build, auto search" combination. +So when the schema does not configure `efConstruction`, it scales as `AUTO_EF_BASE * sqrt(nodes / +AUTO_EFC_REF)`, capped at `AUTO_EFC_MAX`. The healthy write path reads the count directly from the +shared id counter: one atomic load with no memo lag during bulk ingest. If an update-only worker +cannot attach that counter, it warns once, falls back to the memoized reverse seek, and retries the +attach after the memo TTL; a new insert still requires the shared counter rather than risking ids +from a private counter. Scaling starts at 250K nodes: efC 100 held recall through 500K (0.978), so +smaller graphs — the common case — build exactly as before. The sqrt shape mirrors the search-side +scale; the cost is build time (1.77x at 1M for efC 200), paid only by tables that actually grow +large, and partly returned as cheaper queries. + +An explicit `efConstruction` stays authoritative and is structural, so changing it triggers a full +index rebuild. It also seeds the search `ef`: setting `efConstruction: 100` alone cuts query effort +to 100. Retaining the former large-graph search default while opting out of build scaling requires +an explicit `efConstructionSearch` as well (512 after the former auto-scale reached its plateau). +There is currently no "pinned build, auto search" combination. The search side scales past its old plateau for the same reason. `AUTO_EF_MAX` (512, pinned from ~13K nodes) was calibrated when layers above 0 were searched at the full `ef`, which made large efs @@ -836,16 +841,24 @@ the constants further is the wrong tool there. Two caveats are accepted deliberately, both inherited from the count being a lifetime high-water mark of allocated node ids rather than a live count. First, churn: a table that deletes heavily (TTL eviction, delete-and-reinsert ingest) reads high forever, so its build-side efC can sit at the -cap while the live graph is small — bounded at `AUTO_EFC_MAX` (10.24x the base ef; by the measured -1.77x-wall-time-per-2x-ef relation, roughly 6–7x build time), it wastes build CPU but never hurts -recall. The search side accepted the same over-count as "slightly -generous ef" on an opt-in read path; the write path inherits it as a known cost until a live count -exists (tracked follow-up). Second, ramp history: nodes indexed before the graph crossed a scale -threshold keep their original edges — the scale applies to inserts from that point on. A reindex in -a live process rebuilds roughly uniformly (the id counter keeps its high-water mark), but a reindex -after a restart re-seeds the counter from the largest id in the rebuilding store and therefore -repeats the ramp — its first 250K nodes rebuild at the base efC. Both converge to the same steady -state as the graph grows past the knee. +cap while the live graph is small. The 6–7x build-time extrapolation applies to a comparably large +graph; it is not a bound for a small rolling window. When efC exceeds the live graph size, the +candidate list cannot fill and an insert can traverse a large fraction of the graph before storing +only `M << 1` edges. This wastes throughput without improving recall. The search side accepted the +same over-count as "slightly generous ef" on an opt-in read path; the write path inherits it as a +known cost until a live count exists (tracked follow-up). Second, ramp history: nodes indexed before +the graph crossed a scale threshold keep their original edges — the scale applies to inserts from +that point on. A reindex in a live process rebuilds roughly uniformly (the id counter keeps its +high-water mark), but a reindex after a restart re-seeds the counter from the largest id in the +rebuilding store and therefore repeats the ramp — its first 250K nodes rebuild at the base efC. +Later inserts add reverse edges to older nodes, but a default-ramp 1M build has not been compared +directly with the uniform-200 A/B. The larger default-ramp runs reached 0.988 set-recall at 2M and +0.985 at 5M when searched at ef 1024, which shows that the measured neighbours remained reachable +at those sizes without proving uniform convergence. + +Deletes have a separate tail-latency cost: connectivity repair can synchronously reinsert an orphan +and up to 256 nodes from a severed island. Those reinserts use the current auto-scaled efC, so the +per-insert build multiplier can land hundreds of times within one delete. ## An approximate index returns at most `ef` rows, so `limit` has to reach it diff --git a/benchmarks/hnsw-scale.js b/benchmarks/hnsw-scale.js index c02c4bed1..5a76d8d77 100644 --- a/benchmarks/hnsw-scale.js +++ b/benchmarks/hnsw-scale.js @@ -10,10 +10,14 @@ * Run (after npm run build): * node benchmarks/hnsw-scale.js [--n=5000,10000,25000] [--dims=768] [--queries=50] * [--clusters=N] [--ef=auto] [--quantization=int8|none] + * [--ef-construction=N] [--ef-sweep=auto,N,...] + * [--intra-cos=X] [--seed=N] * [--upper-ef=N] (override ef used above layer 0) * [--stream] (two-pass generation; no float pool held — for N * where the pool alone would not fit in memory) * [--json=path] + * `--ef-construction` also seeds the default search ef. Use the same explicit `--ef-sweep` + * across construction A/B runs to compare both graphs at a common search ef. */ const { performance } = require('node:perf_hooks'); @@ -377,6 +381,7 @@ function pct(sorted, p) { let inSearch = false; let currentEf = 0; +let currentEfConstruction = 0; const layerStats = { visitsByLevel: new Map(), timeByLevel: new Map(), callsByLevel: new Map() }; function resetLayerStats() { layerStats.visitsByLevel.clear(); @@ -405,6 +410,7 @@ function instrument(hnsw) { // query — the same value, and read from the index rather than recomputed here, so this cannot // drift when the auto-scale formula changes. if (inSearch && level === 0) currentEf = ef; + else if (!inSearch && level === 0) currentEfConstruction = ef; if (inSearch && UPPER_EF !== undefined && level > 0 && currentEf) ef = UPPER_EF === 'match' ? currentEf : UPPER_EF; else if (inSearch && UPPER_EF !== undefined && UPPER_EF !== 'match' && level > 0) ef = UPPER_EF; else if (!inSearch && BUILD_UPPER_EF !== undefined && level > 0) ef = BUILD_UPPER_EF; @@ -438,6 +444,7 @@ function graphShape(store) { const rows = []; console.log( `\nHNSW scaling sweep — dims=${DIMS}, queries=${N_QUERIES}, k=${TOP_K}, quantization=${QUANTIZATION}, ef=${EF_OPT}` + + `, efC=${EF_CONSTRUCTION ?? 'auto'}, efSweep=${EF_SWEEP.join(',')}` + (UPPER_EF !== undefined ? `, upper-ef=${UPPER_EF}` : '') + (BUILD_UPPER_EF !== undefined ? `, build-upper-ef=${BUILD_UPPER_EF}` : '') + `, intraCos=${INTRA_COS}, sigma=${NOISE.toFixed(4)}\n` @@ -462,6 +469,7 @@ for (const N of SIZES) { if (EF_CONSTRUCTION !== undefined) options.efConstruction = EF_CONSTRUCTION; const hnsw = new HierarchicalNavigableSmallWorld(store, options); instrument(hnsw); + currentEfConstruction = 0; let buildMs; let queries = []; @@ -478,6 +486,7 @@ for (const N of SIZES) { } const shape = graphShape(store); + const effectiveEfConstruction = currentEfConstruction || hnsw.efConstruction; // queries drawn from the same distribution (perturbed corpus rows); --stream built its own above if (!STREAM) @@ -563,6 +572,7 @@ for (const N of SIZES) { clusters: nClusters, buildMs: Math.round(buildMs), msPerInsert: +(buildMs / N).toFixed(2), + efConstruction: effectiveEfConstruction, efSpec: String(efSpec), ef: effectiveEf, p50: +pct(latencies, 50).toFixed(2), @@ -586,7 +596,7 @@ for (const N of SIZES) { rows.push(row); console.log( `N=${String(N).padStart(7)} build ${String(row.buildMs).padStart(7)}ms (${row.msPerInsert} ms/ins) ` + - `ef=${String(row.ef).padStart(4)}${efSpec === 'auto' ? '*' : ' '} p50 ${String(row.p50).padStart(8)}ms p95 ${String(row.p95).padStart(8)}ms ` + + `efC=${String(row.efConstruction).padStart(4)} ef=${String(row.ef).padStart(4)}${efSpec === 'auto' ? '*' : ' '} p50 ${String(row.p50).padStart(8)}ms p95 ${String(row.p95).padStart(8)}ms ` + `µs/vec ${String(row.usPerVector).padStart(6)} visited ${String(row.visited).padStart(7)} (${String(row.visitedPctOfGraph).padStart(5)}% of graph; L0 ${row.l0Visited}, upper ${row.upperVisited}, upper ${row.upperTimePct}% of time) ` + `d(nn)/d(rand) ${row.dNear}/${row.dRand} recall@${TOP_K} raw ${row.recall} set ${row.recallSet} returned ${row.returned} deg0 ${row.avgDegree0} levels ${row.levels}` ); @@ -610,7 +620,17 @@ if (argv.json) { fs.writeFileSync( argv.json, JSON.stringify( - { dims: DIMS, queries: N_QUERIES, k: TOP_K, quantization: QUANTIZATION, ef: EF_OPT, upperEf: UPPER_EF, rows }, + { + dims: DIMS, + queries: N_QUERIES, + k: TOP_K, + quantization: QUANTIZATION, + ef: EF_OPT, + efConstruction: EF_CONSTRUCTION ?? 'auto', + efSweep: EF_SWEEP, + upperEf: UPPER_EF, + rows, + }, null, 2 ) diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index c8121bd34..ac42f1c90 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -96,9 +96,9 @@ const AUTO_EFC_REF = 250_000; // with the curve (N^1.5 total under sqrt scaling), which is why the cap stays finite: past ~tens of // millions of nodes, sharded medium graphs beat one huge graph on both build and query cost. const AUTO_EFC_MAX = 1024; -function autoScaleEfConstruction(base: number, nodeCount: number): number { - const scaled = Math.round(base * Math.sqrt(Math.max(1, nodeCount / AUTO_EFC_REF))); - return Math.min(AUTO_EFC_MAX, Math.max(base, scaled)); +function autoScaleEfConstruction(nodeCount: number): number { + const scaled = Math.round(AUTO_EF_BASE * Math.sqrt(Math.max(1, nodeCount / AUTO_EFC_REF))); + return Math.min(AUTO_EFC_MAX, Math.max(AUTO_EF_BASE, scaled)); } // How long a resolved graph size is reused before it is looked up again (see approximateNodeCount). // ef moves with the square root of the count and is capped, so a slightly stale size is immaterial; @@ -227,6 +227,8 @@ export class HierarchicalNavigableSmallWorld { efSearchConfigured = false; // whether the schema set an explicit search ef; if not, search ef auto-scales with N efConstructionConfigured = false; // whether the schema set an explicit efConstruction; if not, it auto-scales with N private lastLoggedEfConstruction = 0; + private idIncrementerRetryAt = 0; + private idIncrementerFailureLogged = false; // Caches the Int8Array-converted clone of a frozen (decoded-from-disk) int8 node, keyed by the // frozen node the object store hands back. WeakMap so entries are collected when the store evicts // the frozen node — without it, every cache hit on a frozen node would re-slice and re-clone. @@ -389,18 +391,11 @@ export class HierarchicalNavigableSmallWorld { connections[i] = []; } - // Ensured here (not only at id allocation) so an update-only worker reads the count with an - // atomic load instead of a per-write seek. The count is a lifetime high-water mark, not a - // live count; see DESIGN.md for the churn-table caveat. Guarded because the count is an ef - // heuristic with a safe fallback — a write must never fail on it. + // An update-only worker may not have attached the id counter yet. The healthy path is one + // atomic load; a failed attach falls back to the memoized seek and retries after its TTL. let efConstruction = this.efConstruction; if (!this.efConstructionConfigured) { - try { - this.ensureIdIncrementer(options); - efConstruction = autoScaleEfConstruction(this.efConstruction, this.resolveNodeCount()); - } catch (error) { - logger.debug?.('could not resolve the node count for the construction ef', error); - } + efConstruction = autoScaleEfConstruction(this.resolveConstructionNodeCount(options)); if (efConstruction > this.efConstruction && this.lastLoggedEfConstruction !== efConstruction) { // once per resolved value per process: makes replica-divergent build quality and the // build-cost ramp diagnosable (the resolved value is otherwise surfaced nowhere) @@ -757,6 +752,24 @@ export class HierarchicalNavigableSmallWorld { return this.nodeCount; } + private resolveConstructionNodeCount(options?: any): number { + const now = Date.now(); + if (this.idIncrementer || now >= this.idIncrementerRetryAt) { + try { + this.ensureIdIncrementer(options); + this.idIncrementerRetryAt = 0; + return this.resolveNodeCount(); + } catch (error) { + this.idIncrementerRetryAt = now + NODE_COUNT_TTL; + if (!this.idIncrementerFailureLogged) { + this.idIncrementerFailureLogged = true; + logger.warn?.('could not attach the shared HNSW id counter; using a memoized node count', error); + } + } + } + return this.approximateNodeCount(); + } + /** * Create-or-attach the shared id counter, seeded from a one-time reverse seek to the largest node * id. getUserSharedBuffer returns the existing shared buffer when another worker created it diff --git a/schema.graphql b/schema.graphql index bb79477d4..887958f58 100644 --- a/schema.graphql +++ b/schema.graphql @@ -189,7 +189,8 @@ directive @indexed( """ Construction effort/recall parameter (HNSW). Higher values improve recall at the cost of build time and memory. When omitted, it auto-scales with graph - size; setting it pins both the build-side value and the search-side default. + size. Setting it pins both the build-side value and the search-side default; + changing it rebuilds the index. """ efConstruction: Int """ @@ -208,7 +209,9 @@ directive @indexed( """ Search-time effort/recall parameter (HNSW): the candidate-list size used when querying. Larger values typically yield better accuracy at the cost of query - latency. When omitted, it auto-scales with graph size. + latency. When both this and efConstruction are omitted, it auto-scales with + graph size, continuing past one million nodes up to 2048. Pin it when query + latency matters more than recall on large tables. """ efConstructionSearch: Int ) on FIELD_DEFINITION diff --git a/unitTests/resources/vectorIndex.test.js b/unitTests/resources/vectorIndex.test.js index 815016d63..4ff8afe20 100644 --- a/unitTests/resources/vectorIndex.test.js +++ b/unitTests/resources/vectorIndex.test.js @@ -701,6 +701,32 @@ describe('HNSW construction ef auto-scale (#2180)', () => { }); } + it('caps the filtered visit budget when the auto-scaled search ef exceeds AUTO_EF_MAX', async () => { + const customIndex = T.indices.vector.customIndex; + const originalSearchLayer = Object.getPrototypeOf(customIndex).searchLayer; + let layer0Ef = 0; + const budgets = []; + customIndex.searchLayer = function (v, epId, ep, ef, level, options, distanceFn, filter, filterState) { + if (level === 0) layer0Ef = ef; + if (filterState) budgets.push(filterState.maxVisits); + return originalSearchLayer.call(this, v, epId, ep, ef, level, options, distanceFn, filter, filterState); + }; + try { + await withNodeCount(T, 5_000_000, () => { + customIndex.nodeCountAt = 0; + customIndex.search({ target: [1, 0, 0], comparator: 'sort' }, { transaction: undefined }, () => true); + }); + } finally { + delete customIndex.searchLayer; + } + assert.strictEqual(layer0Ef, 1145, 'the test must reach the second search-ef regime'); + assert.deepStrictEqual( + [...new Set(budgets)], + [512 * customIndex.filterExpansion], + 'the automatic filtered budget must stay capped at AUTO_EF_MAX' + ); + }); + // The refactor's stated mechanism, not just the formula: an update-only worker (counter absent, // as after a restart with no new inserts) must ensure the shared counter with ONE seek, then read // it atomically — never a reverse seek per write, and never a failed write if the seek throws. @@ -727,29 +753,44 @@ describe('HNSW construction ef auto-scale (#2180)', () => { } }); - // A failed shared-buffer attach must not install a private counter (ids allocated from it would - // collide with other workers'). The write still succeeds (base efC fallback) and the next write - // retries the ensure against the healthy store. - it('does not install a private counter when the shared attach fails, and retries next write', async () => { + it('falls back to a memoized count and backs off after a shared-counter attach failure', async () => { const customIndex = T.indices.vector.customIndex; const saved = customIndex.idIncrementer; + const savedNodeCount = customIndex.nodeCount; + const savedNodeCountAt = customIndex.nodeCountAt; + const savedRetryAt = customIndex.idIncrementerRetryAt; + const savedFailureLogged = customIndex.idIncrementerFailureLogged; customIndex.idIncrementer = undefined; + customIndex.nodeCountAt = 0; const store = customIndex.indexStore; const originalGetUserSharedBuffer = store.getUserSharedBuffer; + let attachAttempts = 0; store.getUserSharedBuffer = () => { + attachAttempts++; throw new Error('simulated shared-buffer attach failure'); }; + customIndex.resolveNodeCount = () => 1_000_000; try { - await T.put(0, { vector: [0.8, 0.2, 0.1] }); // update path; efC falls back, write succeeds + const firstEfs = await connectionEfsDuringPut(T, 0); + const secondEfs = await connectionEfsDuringPut(T, 1); assert.strictEqual(customIndex.idIncrementer, undefined, 'a failed attach must not install a counter'); + assert.deepStrictEqual([...firstEfs], [200], 'the fallback count must preserve the construction scale'); + assert.deepStrictEqual([...secondEfs], [200], 'the memoized fallback must preserve the scale'); + assert.strictEqual(attachAttempts, 1, 'the attach must not be retried on every update'); } finally { store.getUserSharedBuffer = originalGetUserSharedBuffer; + delete customIndex.resolveNodeCount; + customIndex.idIncrementerRetryAt = 0; } try { await T.put(1, { vector: [0.7, 0.3, 0.1] }); - assert(customIndex.idIncrementer, 'the ensure must retry once the store is healthy'); + assert(customIndex.idIncrementer, 'the attach must be retryable once the backoff expires'); } finally { - if (!customIndex.idIncrementer) customIndex.idIncrementer = saved; + customIndex.idIncrementer = saved; + customIndex.nodeCount = savedNodeCount; + customIndex.nodeCountAt = savedNodeCountAt; + customIndex.idIncrementerRetryAt = savedRetryAt; + customIndex.idIncrementerFailureLogged = savedFailureLogged; } }); From 8e3b2cb3e538a12a907d98e2fe148f51f32e56fe Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 22:39:52 -0600 Subject: [PATCH 10/12] Harden HNSW autoscale fallback Preserve the construction scale when shared counter attachment fails, bound retries, pass the active write transaction through fallback seeks, and document the remaining large-graph cost caveats. Co-Authored-By: GPT-5 Codex --- DESIGN.md | 15 ++- .../HierarchicalNavigableSmallWorld.ts | 54 ++++---- unitTests/resources/vectorIndex.test.js | 122 +++++++++++++++--- 3 files changed, 144 insertions(+), 47 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 91fd42a49..345e4ad85 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -884,10 +884,11 @@ puts a store lookup back on every query whose `limit` exceeds the table — the whole change removed, reintroduced in miniature. An `ef` above the node count is free anyway: the traversal is bounded by the nodes it can reach, so it ends at the graph, not at `ef`. -The filter budget deliberately does not follow a limit-derived `ef`. `maxVisits = ef * filterExpansion` -(#1241) is what stops a selective filter crawling the graph and loading a record per visit, so it is -computed from the `ef` the index resolved for itself. Multiplying it by a caller's `limit` would turn -a filtered vector query into a record-loading scan wearing an index's clothes. +The filter budget deliberately does not follow a limit-derived `ef`. It is computed from the `ef` +the index resolved for itself, with an automatically scaled `ef` capped at `AUTO_EF_MAX` before it is +multiplied by `filterExpansion`; explicit schema or per-query `ef` values remain authoritative. +Multiplying the budget by a caller's `limit` would turn a filtered vector query into a record-loading +scan wearing an index's clothes. Paging a vector search is best-effort, not a stable partition. Each page re-runs the approximate search at a different `ef` (`offset 0, limit 250` resolves 250; `offset 250, limit 200` resolves 450), @@ -897,8 +898,10 @@ empty" defect; it does not make offsets a cursor. Callers who need stability sho large enough for the whole result set, or pin an explicit `ef`. One consumer is still calibrated in index-store keys rather than nodes: `estimateCountAsSort`, the -planner's cost estimate for a vector sort. It is scaled by `INDEX_KEYS_PER_NODE` so the unit switch -does not silently shift which condition the planner chooses to lead with. +planner's cost estimate for a vector sort. It is scaled by `INDEX_KEYS_PER_NODE` so the count-source +unit switch does not shift the estimate on its own. The ef term remains the configured search value, +not the runtime auto-scaled value, so the planner increasingly underestimates vector traversal cost +as an automatically scaled graph grows. ## Env-config empty objects mean three different things (`config/harperConfigEnvVars.ts`) diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index ac42f1c90..d4700aa61 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -209,22 +209,17 @@ export class HierarchicalNavigableSmallWorld { // a value of 1 is extremely aggressive. optimizeRouting = 0.5; nodesVisitedCount = 0; - // Visit-budget multiplier for predicate-aware traversal (#1241). When a filter is selective enough - // that layer-0 results never fill `ef`, the "closest candidate worse than worst result" stop rule - // never triggers, so maxVisits = ef * filterExpansion caps how many nodes an under-filled filtered - // search visits before returning what it has (approximate, like all ANN). It does NOT bound a filter - // that fills `ef` — that terminates naturally. Default 24 (not the 8 the design sketch assumed): - // this HNSW visits a large fraction of the graph per query, so filling `ef` at selectivity `s` needs - // ~ef/s visits; a multiplier of 24 fills down to ~4% selectivity before the budget bites, keeping - // recall at or above post-filtering across the range the query planner routes to traversal. Raising - // it is nearly free for condition-derived filters (they fill and self-terminate); it mainly trades - // latency for recall on selective *function* predicates. Per-query override via the search options. + // Visit-budget multiplier for predicate-aware traversal (#1241). Under-filled filtered searches + // stop after the resolved budget ef * filterExpansion visits; automatic search ef contributes at + // most AUTO_EF_MAX, while explicit schema/query ef remains authoritative. A filter that fills its + // candidate list terminates naturally before this bound. Raising the multiplier mainly trades + // latency for recall on selective function predicates. Per-query override via the search options. filterExpansion = 24; idIncrementer: BigInt64Array | undefined; distance: (a: number[], b: number[]) => number; int8 = true; // store vectors as int8-quantized bins by default; opt out with `quantization: "none"` - efSearchConfigured = false; // whether the schema set an explicit search ef; if not, search ef auto-scales with N + efSearchConfigured = false; // whether the schema pins search ef directly or through efConstruction efConstructionConfigured = false; // whether the schema set an explicit efConstruction; if not, it auto-scales with N private lastLoggedEfConstruction = 0; private idIncrementerRetryAt = 0; @@ -558,8 +553,7 @@ export class HierarchicalNavigableSmallWorld { } } this.indexStore.remove(nodeId, options); - // Remove the safeKey→nodeId mapping so the key count used by autoScaleEf stays accurate - // and a re-insert of this primary key gets a fresh node rather than the deleted node's id. + // A re-insert of this primary key must get a fresh node rather than the deleted node's id. this.indexStore.remove(safeKey, options); } const needsReindexing = new Map(); @@ -744,20 +738,22 @@ export class HierarchicalNavigableSmallWorld { * measurements behind both. The memo is the only gate on how often the size is resolved; nothing on * the query path may bypass it, or the O(1) lookup becomes per-query work again. */ - private approximateNodeCount(): number { + private approximateNodeCount(options?: any): number { const now = Date.now(); if (this.nodeCountAt > 0 && now - this.nodeCountAt < NODE_COUNT_TTL) return this.nodeCount; - this.nodeCount = this.resolveNodeCount(); + this.nodeCount = this.resolveNodeCount(options); this.nodeCountAt = now; return this.nodeCount; } private resolveConstructionNodeCount(options?: any): number { + if (this.idIncrementer) return this.resolveNodeCount(); const now = Date.now(); - if (this.idIncrementer || now >= this.idIncrementerRetryAt) { + if (now >= this.idIncrementerRetryAt) { try { this.ensureIdIncrementer(options); this.idIncrementerRetryAt = 0; + this.idIncrementerFailureLogged = false; return this.resolveNodeCount(); } catch (error) { this.idIncrementerRetryAt = now + NODE_COUNT_TTL; @@ -767,7 +763,7 @@ export class HierarchicalNavigableSmallWorld { } } } - return this.approximateNodeCount(); + return this.approximateNodeCount(options); } /** @@ -791,14 +787,28 @@ export class HierarchicalNavigableSmallWorld { // array first would, on an attach failure, leave THIS process allocating ids nobody else can // see — cross-worker id collisions. Left unset, the next write simply retries the ensure. const seed = new BigInt64Array([BigInt(largestNodeId) + 1n]); - this.idIncrementer = new BigInt64Array(this.indexStore.getUserSharedBuffer('next-id', seed.buffer)); + const sharedBuffer = this.indexStore.getUserSharedBuffer('next-id', seed.buffer); + if ( + !sharedBuffer || + sharedBuffer.byteLength < BigInt64Array.BYTES_PER_ELEMENT || + sharedBuffer.byteLength % BigInt64Array.BYTES_PER_ELEMENT !== 0 + ) { + throw new Error('Shared HNSW id counter buffer is unusable'); + } + this.idIncrementer = new BigInt64Array(sharedBuffer); } /** O(1) node count — the shared id counter, else a single reverse seek to the largest node id. */ - private resolveNodeCount(): number { + private resolveNodeCount(options?: any): number { if (this.idIncrementer) return Number(Atomics.load(this.idIncrementer, 0)); try { - for (const key of this.indexStore.getKeys({ reverse: true, limit: 1, start: Infinity, end: 0 })) { + for (const key of this.indexStore.getKeys({ + reverse: true, + limit: 1, + start: Infinity, + end: 0, + transaction: options?.transaction, + })) { if (typeof key === 'number') return key + 1; } } catch (error) { @@ -1095,8 +1105,8 @@ export class HierarchicalNavigableSmallWorld { if (!Array.isArray(target)) throw new ClientError('The target vector must be an array'); const options = context.transaction; // should have a nested RocksDB transaction - // Resolve search ef: per-query ef wins; else an explicitly-configured efConstructionSearch; - // else auto-scale with the graph size so recall holds as the table grows. + // Resolve search ef: per-query ef wins; else use the schema-pinned value (from either ef option); + // otherwise auto-scale with the graph size so recall holds as the table grows. let effectiveEf = this.efConstructionSearch; const explicitEf = ef !== undefined && ef > 0; if (explicitEf) effectiveEf = ef; diff --git a/unitTests/resources/vectorIndex.test.js b/unitTests/resources/vectorIndex.test.js index 4ff8afe20..bef1e52a7 100644 --- a/unitTests/resources/vectorIndex.test.js +++ b/unitTests/resources/vectorIndex.test.js @@ -600,6 +600,7 @@ describe('HNSW construction ef auto-scale (#2180)', () => { if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; let T; let Pinned; + let SearchPinned; before(() => { setupTestDBPath(); @@ -620,11 +621,24 @@ describe('HNSW construction ef auto-scale (#2180)', () => { { name: 'vector', indexed: { type: 'HNSW', distance: 'cosine', efConstruction: 64 }, type: 'Array' }, ], }); + SearchPinned = table({ + table: 'HNSWSearchEfPinnedTest', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { + name: 'vector', + indexed: { type: 'HNSW', distance: 'cosine', efConstructionSearch: 1500 }, + type: 'Array', + }, + ], + }); }); after(() => { T.dropTable(); Pinned.dropTable(); + SearchPinned.dropTable(); }); // The efs the connection pass actually searched with during one put. The routing descent runs at @@ -659,6 +673,31 @@ describe('HNSW construction ef auto-scale (#2180)', () => { } } + async function captureFilteredSearch(Table, count, searchCondition = {}) { + const customIndex = Table.indices.vector.customIndex; + const originalSearchLayer = Object.getPrototypeOf(customIndex).searchLayer; + let layer0Ef = 0; + const budgets = []; + customIndex.searchLayer = function (v, epId, ep, ef, level, options, distanceFn, filter, filterState) { + if (level === 0) layer0Ef = ef; + if (filterState) budgets.push(filterState.maxVisits); + return originalSearchLayer.call(this, v, epId, ep, ef, level, options, distanceFn, filter, filterState); + }; + try { + await withNodeCount(Table, count, () => { + customIndex.nodeCountAt = 0; + customIndex.search( + { target: [1, 0, 0], comparator: 'sort', ...searchCondition }, + { transaction: undefined }, + () => true + ); + }); + } finally { + delete customIndex.searchLayer; + } + return { layer0Ef, budgets: [...new Set(budgets)] }; + } + it('builds small graphs at the base efConstruction, unchanged', async () => { for (let i = 0; i < 20; i++) { const a = (i / 20) * Math.PI * 2; @@ -703,30 +742,30 @@ describe('HNSW construction ef auto-scale (#2180)', () => { it('caps the filtered visit budget when the auto-scaled search ef exceeds AUTO_EF_MAX', async () => { const customIndex = T.indices.vector.customIndex; - const originalSearchLayer = Object.getPrototypeOf(customIndex).searchLayer; - let layer0Ef = 0; - const budgets = []; - customIndex.searchLayer = function (v, epId, ep, ef, level, options, distanceFn, filter, filterState) { - if (level === 0) layer0Ef = ef; - if (filterState) budgets.push(filterState.maxVisits); - return originalSearchLayer.call(this, v, epId, ep, ef, level, options, distanceFn, filter, filterState); - }; - try { - await withNodeCount(T, 5_000_000, () => { - customIndex.nodeCountAt = 0; - customIndex.search({ target: [1, 0, 0], comparator: 'sort' }, { transaction: undefined }, () => true); - }); - } finally { - delete customIndex.searchLayer; - } + const { layer0Ef, budgets } = await captureFilteredSearch(T, 5_000_000); assert.strictEqual(layer0Ef, 1145, 'the test must reach the second search-ef regime'); assert.deepStrictEqual( - [...new Set(budgets)], + budgets, [512 * customIndex.filterExpansion], 'the automatic filtered budget must stay capped at AUTO_EF_MAX' ); }); + it('lets an explicit query ef raise the filtered visit budget', async () => { + const customIndex = T.indices.vector.customIndex; + const { layer0Ef, budgets } = await captureFilteredSearch(T, 5_000_000, { ef: 3000 }); + assert.strictEqual(layer0Ef, 3000); + assert.deepStrictEqual(budgets, [3000 * customIndex.filterExpansion]); + }); + + it('lets an explicit schema ef raise the filtered visit budget', async () => { + for (let i = 0; i < 3; i++) await SearchPinned.put(i, { vector: [1, i / 10, 0] }); + const customIndex = SearchPinned.indices.vector.customIndex; + const { layer0Ef, budgets } = await captureFilteredSearch(SearchPinned, 5_000_000); + assert.strictEqual(layer0Ef, 1500); + assert.deepStrictEqual(budgets, [1500 * customIndex.filterExpansion]); + }); + // The refactor's stated mechanism, not just the formula: an update-only worker (counter absent, // as after a restart with no new inserts) must ensure the shared counter with ONE seek, then read // it atomically — never a reverse seek per write, and never a failed write if the seek throws. @@ -764,12 +803,21 @@ describe('HNSW construction ef auto-scale (#2180)', () => { customIndex.nodeCountAt = 0; const store = customIndex.indexStore; const originalGetUserSharedBuffer = store.getUserSharedBuffer; + const originalGetKeys = store.getKeys; let attachAttempts = 0; + const seekTransactions = []; + store.getKeys = function* (options) { + if (options?.reverse) { + seekTransactions.push(options.transaction); + yield 999_999; + return; + } + yield* originalGetKeys.call(this, options); + }; store.getUserSharedBuffer = () => { attachAttempts++; throw new Error('simulated shared-buffer attach failure'); }; - customIndex.resolveNodeCount = () => 1_000_000; try { const firstEfs = await connectionEfsDuringPut(T, 0); const secondEfs = await connectionEfsDuringPut(T, 1); @@ -777,14 +825,22 @@ describe('HNSW construction ef auto-scale (#2180)', () => { assert.deepStrictEqual([...firstEfs], [200], 'the fallback count must preserve the construction scale'); assert.deepStrictEqual([...secondEfs], [200], 'the memoized fallback must preserve the scale'); assert.strictEqual(attachAttempts, 1, 'the attach must not be retried on every update'); + assert.strictEqual( + seekTransactions.length, + 2, + 'the first update should seek once to attach and once to fall back' + ); + assert(seekTransactions[0], 'the counter seed seek must use the write transaction'); + assert.strictEqual(seekTransactions[1], seekTransactions[0], 'the fallback seek must use the same transaction'); } finally { store.getUserSharedBuffer = originalGetUserSharedBuffer; - delete customIndex.resolveNodeCount; + store.getKeys = originalGetKeys; customIndex.idIncrementerRetryAt = 0; } try { await T.put(1, { vector: [0.7, 0.3, 0.1] }); assert(customIndex.idIncrementer, 'the attach must be retryable once the backoff expires'); + assert.strictEqual(customIndex.idIncrementerFailureLogged, false, 'recovery must re-arm the warning'); } finally { customIndex.idIncrementer = saved; customIndex.nodeCount = savedNodeCount; @@ -794,6 +850,34 @@ describe('HNSW construction ef auto-scale (#2180)', () => { } }); + it('does not cache an unusable shared-counter buffer', async () => { + const customIndex = T.indices.vector.customIndex; + const saved = customIndex.idIncrementer; + const savedNodeCount = customIndex.nodeCount; + const savedNodeCountAt = customIndex.nodeCountAt; + const savedRetryAt = customIndex.idIncrementerRetryAt; + const savedFailureLogged = customIndex.idIncrementerFailureLogged; + customIndex.idIncrementer = undefined; + customIndex.nodeCountAt = 0; + const store = customIndex.indexStore; + const originalGetUserSharedBuffer = store.getUserSharedBuffer; + store.getUserSharedBuffer = () => new ArrayBuffer(0); + customIndex.resolveNodeCount = () => 1_000_000; + try { + const efs = await connectionEfsDuringPut(T, 0); + assert.strictEqual(customIndex.idIncrementer, undefined); + assert.deepStrictEqual([...efs], [200]); + } finally { + store.getUserSharedBuffer = originalGetUserSharedBuffer; + delete customIndex.resolveNodeCount; + customIndex.idIncrementer = saved; + customIndex.nodeCount = savedNodeCount; + customIndex.nodeCountAt = savedNodeCountAt; + customIndex.idIncrementerRetryAt = savedRetryAt; + customIndex.idIncrementerFailureLogged = savedFailureLogged; + } + }); + it('leaves an explicitly configured efConstruction authoritative at any graph size', async () => { for (let i = 0; i < 5; i++) { const a = (i / 5) * Math.PI * 2; From 3792c84f7d7c57e0069c2f321e50966f4039b840 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 22:43:18 -0600 Subject: [PATCH 11/12] Document the filtered-search budget default Retain the selectivity rationale for filterExpansion while keeping the revised auto-scale budget description concise. Co-Authored-By: GPT-5 Codex --- resources/indexes/HierarchicalNavigableSmallWorld.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index d4700aa61..cf01aa462 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -213,7 +213,8 @@ export class HierarchicalNavigableSmallWorld { // stop after the resolved budget ef * filterExpansion visits; automatic search ef contributes at // most AUTO_EF_MAX, while explicit schema/query ef remains authoritative. A filter that fills its // candidate list terminates naturally before this bound. Raising the multiplier mainly trades - // latency for recall on selective function predicates. Per-query override via the search options. + // latency for recall on selective function predicates; 24 fills to roughly 4% selectivity before + // the budget binds. Per-query override via the search options. filterExpansion = 24; idIncrementer: BigInt64Array | undefined; From 7ac9caa2901cc820c01e0a1d2a7e3d462792b311 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 20 Aug 2026 11:25:30 -0600 Subject: [PATCH 12/12] Respect schema HNSW search ceilings Co-Authored-By: GPT-5 Codex --- DESIGN.md | 8 +-- benchmarks/hnsw-scale.js | 19 ++++-- .../HierarchicalNavigableSmallWorld.ts | 40 +++++++---- unitTests/resources/vectorIndex.test.js | 66 +++++++++++++++---- 4 files changed, 98 insertions(+), 35 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 345e4ad85..b876b23e9 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -872,10 +872,10 @@ request. Any future approximate index needs the same plumbing. Two bounds keep that from becoming a new problem. `ef` drives a synchronous traversal that holds every admitted candidate in a sorted array with an O(len) insert, so a limit-derived `ef` is capped at `LIMIT_EF_MAX`; without it, ordinary deep pagination (`offset` in the millions) would walk the -whole graph on the event loop, which is worse than the truncation being fixed. And a per-query `ef` -stays authoritative: it is an explicit cost ceiling, so it bounds the result set rather than being -raised by the limit. A schema-level `efConstructionSearch` is a default rather than a per-request -decision, so it does not block the floor. +whole graph on the event loop, which is worse than the truncation being fixed. And schema-level or +per-query `ef` values stay authoritative: each is an explicit cost ceiling, so it bounds the result +set rather than being raised by the limit. Only automatically scaled indexes widen toward +`LIMIT_EF_MAX` to satisfy a larger bounded request. `LIMIT_EF_MAX` is the _only_ bound on the widening — deliberately not also the graph size. Clamping there is tempting and costs more than it saves: the memoized size reads low while a table grows, so diff --git a/benchmarks/hnsw-scale.js b/benchmarks/hnsw-scale.js index 5a76d8d77..15c0c4fd2 100644 --- a/benchmarks/hnsw-scale.js +++ b/benchmarks/hnsw-scale.js @@ -381,7 +381,8 @@ function pct(sorted, p) { let inSearch = false; let currentEf = 0; -let currentEfConstruction = 0; +let efConstructionStart = 0; +let efConstructionEnd = 0; const layerStats = { visitsByLevel: new Map(), timeByLevel: new Map(), callsByLevel: new Map() }; function resetLayerStats() { layerStats.visitsByLevel.clear(); @@ -410,7 +411,10 @@ function instrument(hnsw) { // query — the same value, and read from the index rather than recomputed here, so this cannot // drift when the auto-scale formula changes. if (inSearch && level === 0) currentEf = ef; - else if (!inSearch && level === 0) currentEfConstruction = ef; + else if (!inSearch && level === 0) { + if (!efConstructionStart) efConstructionStart = ef; + efConstructionEnd = ef; + } if (inSearch && UPPER_EF !== undefined && level > 0 && currentEf) ef = UPPER_EF === 'match' ? currentEf : UPPER_EF; else if (inSearch && UPPER_EF !== undefined && UPPER_EF !== 'match' && level > 0) ef = UPPER_EF; else if (!inSearch && BUILD_UPPER_EF !== undefined && level > 0) ef = BUILD_UPPER_EF; @@ -469,7 +473,8 @@ for (const N of SIZES) { if (EF_CONSTRUCTION !== undefined) options.efConstruction = EF_CONSTRUCTION; const hnsw = new HierarchicalNavigableSmallWorld(store, options); instrument(hnsw); - currentEfConstruction = 0; + efConstructionStart = 0; + efConstructionEnd = 0; let buildMs; let queries = []; @@ -486,7 +491,8 @@ for (const N of SIZES) { } const shape = graphShape(store); - const effectiveEfConstruction = currentEfConstruction || hnsw.efConstruction; + const effectiveEfConstructionStart = efConstructionStart || hnsw.efConstruction; + const effectiveEfConstructionEnd = efConstructionEnd || hnsw.efConstruction; // queries drawn from the same distribution (perturbed corpus rows); --stream built its own above if (!STREAM) @@ -572,7 +578,8 @@ for (const N of SIZES) { clusters: nClusters, buildMs: Math.round(buildMs), msPerInsert: +(buildMs / N).toFixed(2), - efConstruction: effectiveEfConstruction, + efConstructionStart: effectiveEfConstructionStart, + efConstructionEnd: effectiveEfConstructionEnd, efSpec: String(efSpec), ef: effectiveEf, p50: +pct(latencies, 50).toFixed(2), @@ -596,7 +603,7 @@ for (const N of SIZES) { rows.push(row); console.log( `N=${String(N).padStart(7)} build ${String(row.buildMs).padStart(7)}ms (${row.msPerInsert} ms/ins) ` + - `efC=${String(row.efConstruction).padStart(4)} ef=${String(row.ef).padStart(4)}${efSpec === 'auto' ? '*' : ' '} p50 ${String(row.p50).padStart(8)}ms p95 ${String(row.p95).padStart(8)}ms ` + + `efC=${String(row.efConstructionStart).padStart(4)}→${String(row.efConstructionEnd).padEnd(4)} ef=${String(row.ef).padStart(4)}${efSpec === 'auto' ? '*' : ' '} p50 ${String(row.p50).padStart(8)}ms p95 ${String(row.p95).padStart(8)}ms ` + `µs/vec ${String(row.usPerVector).padStart(6)} visited ${String(row.visited).padStart(7)} (${String(row.visitedPctOfGraph).padStart(5)}% of graph; L0 ${row.l0Visited}, upper ${row.upperVisited}, upper ${row.upperTimePct}% of time) ` + `d(nn)/d(rand) ${row.dNear}/${row.dRand} recall@${TOP_K} raw ${row.recall} set ${row.recallSet} returned ${row.returned} deg0 ${row.avgDegree0} levels ${row.levels}` ); diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index cf01aa462..1a4906725 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -83,8 +83,8 @@ const ROUTING_EF = 1; // Ceiling on the ef a query's own `offset + limit` can ask for. `limit` is unprivileged and set on // every request, so this bounds what a caller can make one thread do synchronously: layer 0 holds // `ef` candidates in a sorted array with an O(len) insert. Kept within a small multiple of the -// index's own auto-scaled ceiling so the worst case stays the same order; a caller who genuinely -// wants more sets an explicit `ef` and owns the cost. +// index's own auto-scaled ceiling so the worst case stays the same order. Schema and per-query ef +// pins are authoritative cost ceilings; only an automatically scaled index widens from `limit`. const LIMIT_EF_MAX = 2 * AUTO_EF_CEILING; // Auto-scaled construction ef, used only when an index does not explicitly configure efConstruction. // At a constant efConstruction, edge quality erodes as the graph grows until true neighbours become @@ -391,7 +391,12 @@ export class HierarchicalNavigableSmallWorld { // atomic load; a failed attach falls back to the memoized seek and retries after its TTL. let efConstruction = this.efConstruction; if (!this.efConstructionConfigured) { - efConstruction = autoScaleEfConstruction(this.resolveConstructionNodeCount(options)); + // Graph size only tunes a heuristic; a write must never fail because it could not be resolved. + try { + efConstruction = autoScaleEfConstruction(this.resolveConstructionNodeCount(options)); + } catch (error) { + logger.debug?.('could not resolve the HNSW construction node count; using the base ef', error); + } if (efConstruction > this.efConstruction && this.lastLoggedEfConstruction !== efConstruction) { // once per resolved value per process: makes replica-divergent build quality and the // build-cost ramp diagnosable (the resolved value is otherwise surfaced nowhere) @@ -788,15 +793,22 @@ export class HierarchicalNavigableSmallWorld { // array first would, on an attach failure, leave THIS process allocating ids nobody else can // see — cross-worker id collisions. Left unset, the next write simply retries the ensure. const seed = new BigInt64Array([BigInt(largestNodeId) + 1n]); - const sharedBuffer = this.indexStore.getUserSharedBuffer('next-id', seed.buffer); - if ( - !sharedBuffer || - sharedBuffer.byteLength < BigInt64Array.BYTES_PER_ELEMENT || - sharedBuffer.byteLength % BigInt64Array.BYTES_PER_ELEMENT !== 0 - ) { - throw new Error('Shared HNSW id counter buffer is unusable'); + try { + const sharedBuffer = this.indexStore.getUserSharedBuffer('next-id', seed.buffer); + if ( + !sharedBuffer || + sharedBuffer.byteLength < BigInt64Array.BYTES_PER_ELEMENT || + sharedBuffer.byteLength % BigInt64Array.BYTES_PER_ELEMENT !== 0 + ) { + throw new Error('Shared HNSW id counter buffer is unusable'); + } + this.idIncrementer = new BigInt64Array(sharedBuffer); + } catch (error) { + // Reuse the transactional seed seek as the degraded count instead of seeking a second time. + this.nodeCount = largestNodeId + 1; + this.nodeCountAt = Date.now(); + throw error; } - this.idIncrementer = new BigInt64Array(sharedBuffer); } /** O(1) node count — the shared id counter, else a single reverse seek to the largest node id. */ @@ -1120,12 +1132,12 @@ export class HierarchicalNavigableSmallWorld { // to cover the request, up to LIMIT_EF_MAX — `ef` sizes a synchronous traversal that holds every // admitted candidate in a sorted array with an O(len) insert, so an unbounded one lets a plain // `limit` stall the thread. Past the ceiling the result set is still short, as it was before. - // A per-query `ef` is left authoritative: it is an explicit cost ceiling, and a caller who sets - // one has said what they are willing to spend. + // Schema and per-query `ef` pins are authoritative cost ceilings. Only an automatically scaled + // index widens toward LIMIT_EF_MAX to cover a larger bounded request. // The ceiling is the only bound: clamping to the graph size as well would need a count exact as // of this query — the memo reads low while a table grows, truncating the very limit this // honours — and an ef above the node count is free, the traversal ending at the graph, not ef. - if (minResults !== undefined && !explicitEf && minResults > effectiveEf) { + if (minResults !== undefined && !explicitEf && !this.efSearchConfigured && minResults > effectiveEf) { effectiveEf = Math.max(effectiveEf, Math.min(minResults, LIMIT_EF_MAX)); } // Predicate-aware traversal budget (#1241): matches accrue slower than visits under a selective diff --git a/unitTests/resources/vectorIndex.test.js b/unitTests/resources/vectorIndex.test.js index bef1e52a7..c87ca1843 100644 --- a/unitTests/resources/vectorIndex.test.js +++ b/unitTests/resources/vectorIndex.test.js @@ -698,6 +698,22 @@ describe('HNSW construction ef auto-scale (#2180)', () => { return { layer0Ef, budgets: [...new Set(budgets)] }; } + function captureLimitDerivedLayer0Ef(Table, minResults) { + const customIndex = Table.indices.vector.customIndex; + const originalSearchLayer = customIndex.searchLayer; + let layer0Ef = 0; + customIndex.searchLayer = function (v, epId, ep, ef, level, ...rest) { + if (level === 0) layer0Ef = ef; + return originalSearchLayer.call(this, v, epId, ep, ef, level, ...rest); + }; + try { + customIndex.search({ target: [1, 0, 0], comparator: 'sort' }, { transaction: undefined }, undefined, minResults); + } finally { + customIndex.searchLayer = originalSearchLayer; + } + return layer0Ef; + } + it('builds small graphs at the base efConstruction, unchanged', async () => { for (let i = 0; i < 20; i++) { const a = (i / 20) * Math.PI * 2; @@ -766,6 +782,17 @@ describe('HNSW construction ef auto-scale (#2180)', () => { assert.deepStrictEqual(budgets, [1500 * customIndex.filterExpansion]); }); + it('leaves an explicit schema ef authoritative over the limit-derived floor', async () => { + for (const [Table, expectedEf] of [ + [Pinned, 64], + [SearchPinned, 1500], + ]) { + for (let i = 0; i < 3; i++) await Table.put(i, { vector: [1, i / 10, 0] }); + const layer0Ef = captureLimitDerivedLayer0Ef(Table, 3000); + assert.strictEqual(layer0Ef, expectedEf, 'a schema search-ef pin must cap limit-derived widening'); + } + }); + // The refactor's stated mechanism, not just the formula: an update-only worker (counter absent, // as after a restart with no new inserts) must ensure the shared counter with ONE seek, then read // it atomically — never a reverse seek per write, and never a failed write if the seek throws. @@ -825,23 +852,17 @@ describe('HNSW construction ef auto-scale (#2180)', () => { assert.deepStrictEqual([...firstEfs], [200], 'the fallback count must preserve the construction scale'); assert.deepStrictEqual([...secondEfs], [200], 'the memoized fallback must preserve the scale'); assert.strictEqual(attachAttempts, 1, 'the attach must not be retried on every update'); - assert.strictEqual( - seekTransactions.length, - 2, - 'the first update should seek once to attach and once to fall back' - ); + assert.strictEqual(seekTransactions.length, 1, 'the attach seed seek must also supply the fallback count'); assert(seekTransactions[0], 'the counter seed seek must use the write transaction'); - assert.strictEqual(seekTransactions[1], seekTransactions[0], 'the fallback seek must use the same transaction'); - } finally { store.getUserSharedBuffer = originalGetUserSharedBuffer; store.getKeys = originalGetKeys; customIndex.idIncrementerRetryAt = 0; - } - try { await T.put(1, { vector: [0.7, 0.3, 0.1] }); assert(customIndex.idIncrementer, 'the attach must be retryable once the backoff expires'); assert.strictEqual(customIndex.idIncrementerFailureLogged, false, 'recovery must re-arm the warning'); } finally { + store.getUserSharedBuffer = originalGetUserSharedBuffer; + store.getKeys = originalGetKeys; customIndex.idIncrementer = saved; customIndex.nodeCount = savedNodeCount; customIndex.nodeCountAt = savedNodeCountAt; @@ -850,6 +871,19 @@ describe('HNSW construction ef auto-scale (#2180)', () => { } }); + it('keeps construction count resolution best-effort on the write path', async () => { + const customIndex = T.indices.vector.customIndex; + customIndex.resolveConstructionNodeCount = () => { + throw new RangeError('simulated unusable counter'); + }; + try { + const efs = await connectionEfsDuringPut(T, 2); + assert.deepStrictEqual([...efs], [100], 'count resolution failure must retain the base efConstruction'); + } finally { + delete customIndex.resolveConstructionNodeCount; + } + }); + it('does not cache an unusable shared-counter buffer', async () => { const customIndex = T.indices.vector.customIndex; const saved = customIndex.idIncrementer; @@ -861,15 +895,25 @@ describe('HNSW construction ef auto-scale (#2180)', () => { customIndex.nodeCountAt = 0; const store = customIndex.indexStore; const originalGetUserSharedBuffer = store.getUserSharedBuffer; + const originalGetKeys = store.getKeys; + let seekCount = 0; + store.getKeys = function* (options) { + if (options?.reverse) { + seekCount++; + yield 999_999; + return; + } + yield* originalGetKeys.call(this, options); + }; store.getUserSharedBuffer = () => new ArrayBuffer(0); - customIndex.resolveNodeCount = () => 1_000_000; try { const efs = await connectionEfsDuringPut(T, 0); assert.strictEqual(customIndex.idIncrementer, undefined); assert.deepStrictEqual([...efs], [200]); + assert.strictEqual(seekCount, 1, 'the unusable buffer fallback must reuse the attach seed seek'); } finally { store.getUserSharedBuffer = originalGetUserSharedBuffer; - delete customIndex.resolveNodeCount; + store.getKeys = originalGetKeys; customIndex.idIncrementer = saved; customIndex.nodeCount = savedNodeCount; customIndex.nodeCountAt = savedNodeCountAt;