diff --git a/integrationTests/apiTests/rest.test.mjs b/integrationTests/apiTests/rest.test.mjs index a5e0e99d7..829dd9325 100644 --- a/integrationTests/apiTests/rest.test.mjs +++ b/integrationTests/apiTests/rest.test.mjs @@ -45,7 +45,8 @@ type SubObject @table(audit: false) @export { } `; -const CONFIG_YAML = `rest: true +const CONFIG_YAML = `rest: + exactCount: true graphqlSchema: files: '*.graphql' graphql: true @@ -68,6 +69,26 @@ const SUBOBJECT_ROWS = [ { id: '5', relatedId: '5', any: 'any-5' }, ]; +// Second component whose REST mount uses the default (exact counting NOT opted in). +const SCHEMA_GATE_GRAPHQL = ` +type GatedWidget @table @export(rest: true, mqtt: false) { + id: ID @primaryKey + name: String @indexed +} +`; + +const CONFIG_GATE_YAML = `rest: true +graphqlSchema: + files: '*.graphql' +graphql: true +`; + +const GATE_ROWS = [ + { id: '1', name: 'w-1' }, + { id: '2', name: 'w-2' }, + { id: '3', name: 'w-3' }, +]; + const skipSuite = process.platform === 'win32'; suite('REST query syntax', { skip: skipSuite }, (ctx) => { @@ -244,4 +265,175 @@ suite('REST query syntax', { skip: skipSuite }, (ctx) => { ) .expect(200); }); + + // `Prefer: count=` (pagination total-count) — emits Content-Range/Range-Unit/Preference-Applied. + test('[rest] count=exact emits an exact Content-Range', () => { + return client + .reqRest('/Related/?sort(id)&limit(2)') + .set('Prefer', 'count=exact') + .expect('Range-Unit', 'items') + .expect('Content-Range', 'items 0-1/5') + .expect('Preference-Applied', 'count=exact') + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect((r) => + assert.ok( + (r.headers['access-control-expose-headers'] || '').includes('Content-Range'), + `expected Content-Range to be exposed for CORS, got: ${r.headers['access-control-expose-headers']}` + ) + ) + .expect(200); + }); + + test('[rest] count=exact reflects the offset window but a total independent of it', () => { + return client + .reqRest('/Related/?sort(id)&limit(1,3)') // offset 1, 2 rows + .set('Prefer', 'count=exact') + .expect('Content-Range', 'items 1-2/5') + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect(200); + }); + + test('[rest] count=exact on a filtered query counts only matches', () => { + return client + .reqRest('/Related/?name==name-2&limit(10)') + .set('Prefer', 'count=exact') + .expect('Content-Range', 'items 0-0/1') + .expect('Preference-Applied', 'count=exact') + .expect(200); + }); + + test('[rest] count=estimated emits a numeric total flagged estimated', () => { + return client + .reqRest('/Related/?sort(id)&limit(2)') + .set('Prefer', 'count=estimated') + .expect('Preference-Applied', 'count=estimated') + .expect((r) => assert.match(r.headers['content-range'], /^items 0-1\/\d+$/, r.text)) + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect(200); + }); + + test('[rest] an uncomputable total reports items .../* but still echoes the requested mode', () => { + // `name != x` estimates to Infinity, so the total is unavailable. The header must still say + // count=estimated (the mode applied), not count=none — the client asked, it just can't be given. + return client + .reqRest('/Related/?name!=name-2&limit(2)') + .set('Prefer', 'count=estimated') + .expect('Preference-Applied', 'count=estimated') + .expect((r) => assert.match(r.headers['content-range'], /^items 0-\d+\/\*$/, r.text)) + .expect(200); + }); + + test('[rest] no Prefer header means no Content-Range (opt-in only)', () => { + return client + .reqRest('/Related/?sort(id)&limit(2)') + .expect((r) => assert.equal(r.headers['content-range'], undefined, r.text)) + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect(200); + }); + + test('[rest] HEAD with count=exact returns the count header and no body', () => { + return request(client.restURL) + .head('/Related/?sort(id)&limit(2)') + .set(client.headers) + .set('Prefer', 'count=exact') + .expect('Content-Range', 'items 0-1/5') + .expect((r) => assert.ok(!r.body || Object.keys(r.body).length === 0, r.text)) + .expect(200); + }); + + test('[rest] an oversized page limit falls through to streaming with no count', () => { + // A limit past the max count-page size must not materialize a count page — the request is served + // normally (all rows) with no Content-Range, rather than buffering an unbounded page. + return client + .reqRest('/Related/?sort(id)&limit(0,20000)') + .set('Prefer', 'count=exact') + .expect((r) => assert.equal(r.headers['content-range'], undefined, r.text)) + .expect((r) => assert.equal(r.body.length, 5, r.text)) + .expect(200); + }); + + test('[rest] a deep-page offset past the scan budget falls through with no count', () => { + // limit(start,end) with a huge start is a huge offset; the count path must not iterate an unbounded + // offset before its guardrail engages, so the request falls through with no Content-Range. + return client + .reqRest('/Related/?sort(id)&limit(2000000,2000010)') + .set('Prefer', 'count=exact') + .expect((r) => assert.equal(r.headers['content-range'], undefined, r.text)) + .expect(200); + }); + + test('[rest] a collection read declares Vary: Prefer', () => { + // So a shared cache keys on Prefer and never serves count headers to a request that did not ask. + return client + .reqRest('/Related/?sort(id)&limit(2)') + .expect((r) => assert.match(r.headers['vary'] || '', /\bPrefer\b/i, r.text)) + .expect(200); + }); + + test('[rest] DELETE with a limit and Prefer: count is not misrouted to the count path', () => { + // Regression: the count preference is GET/HEAD-only. A DELETE that also carried limit()+Prefer used + // to receive a materialized array from search() and throw instead of deleting. + return client + .req() + .send({ operation: 'insert', table: 'Related', records: [{ id: 'del-me', name: 'to-delete' }] }) + .expect(200) + .then(() => + request(client.restURL) + .delete('/Related/?id==del-me&limit(10)') + .set(client.headers) + .set('Prefer', 'count=exact') + .expect((r) => assert.ok(r.status >= 200 && r.status < 300, `expected 2xx, got ${r.status}: ${r.text}`)) + ) + .then(() => client.reqRest('/Related/?id==del-me').expect((r) => assert.equal(r.body.length, 0, r.text))); + }); +}); + +// exactCount is a per-REST-mount policy, so it needs its own instance: two components exporting at +// the root path share one mount (the handler dedupes), and this mount's default config would otherwise +// bleed onto the main suite's routes (which opt in with exactCount: true). +suite('REST count default (exact not opted in)', { skip: skipSuite }, (ctx) => { + let client; + + before(async () => { + await startHarper(ctx, { config: {}, env: {} }); + client = createApiClient(ctx.harper); + + await installAppComponent(client, { + project: 'appCountGate', + files: { 'schema.graphql': SCHEMA_GATE_GRAPHQL, 'config.yaml': CONFIG_GATE_YAML }, + probePath: '/GatedWidget/', + restartTimeoutMs: 120000, + }); + + await client + .req() + .send({ operation: 'insert', table: 'GatedWidget', records: GATE_ROWS }) + .expect((r) => assert.ok(r.body.message.includes('inserted 3 of 3 records'), r.text)) + .expect(200); + }); + + after(async () => { + await teardownHarper(ctx); + }); + + // Default (no exactCount opt-in): count=exact is served as a cheap estimate instead. + test('[rest] default downgrades count=exact to estimated', () => { + return client + .reqRest('/GatedWidget/?sort(id)&limit(2)') + .set('Prefer', 'count=exact') + .expect('Preference-Applied', 'count=estimated') + .expect((r) => assert.match(r.headers['content-range'], /^items 0-1\/\d+$/, r.text)) + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect(200); + }); + + // estimated still works normally on a default mount. + test('[rest] default leaves count=estimated unchanged', () => { + return client + .reqRest('/GatedWidget/?sort(id)&limit(2)') + .set('Prefer', 'count=estimated') + .expect('Preference-Applied', 'count=estimated') + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect(200); + }); }); diff --git a/resources/RequestTarget.ts b/resources/RequestTarget.ts index ef9d3a145..22a140fdf 100644 --- a/resources/RequestTarget.ts +++ b/resources/RequestTarget.ts @@ -36,6 +36,8 @@ export class RequestTarget extends URLSearchParams { declare select?: Select; /** Return an explanation of the query order */ declare explain?: boolean; + /** Request a total count of matching records for pagination (REST `Prefer: count=exact|estimated`). */ + declare count?: 'exact' | 'estimated'; /** Force the query to be executed in the order of conditions */ declare enforceExecutionOrder?: boolean; declare lazy?: boolean; diff --git a/resources/Table.ts b/resources/Table.ts index bcf0b5971..884633fc7 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -55,6 +55,7 @@ import { searchByIndex, findAttribute, estimateCondition, + estimatedEntryCount, flattenKey, COERCIBLE_OPERATORS, executeConditions, @@ -128,6 +129,17 @@ const EVICTION_BATCH_SIZE = 100; // letting an unbounded number of open transactions (and their snapshots) accumulate. const MAX_INFLIGHT_EVICTION_BATCHES = 4; const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]); +// Guardrails for `Prefer: count=exact`: once the requested page has been collected, counting the rest +// of the match set is bounded by BOTH a row cap and a wall-clock budget, so a paginated read can't turn +// into an unbounded scan. Exceeding either reports an unknown total (Content-Range `.../*`) rather than +// truncating the page. These bound the count tail, not the page itself; a genuinely expensive query +// (large filtered full-scan, in-memory sort) should still be gated by config before broad exposure. +const MAX_EXACT_COUNT_SCAN = 1_000_000; +const MAX_EXACT_COUNT_MS = 1_000; +// Largest page a `Prefer: count=` request will materialize. A request whose limit exceeds this (or is +// not a finite, non-negative integer, e.g. `limit(Infinity)`/`limit(foo)`) falls through to the normal +// streaming path with no count, so a count request can't be coerced into buffering an unbounded page. +const MAX_COUNT_PAGE = 10_000; envMngr.initSync(); const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const LOCK_TIMEOUT = 10000; @@ -3430,6 +3442,10 @@ export function makeTable(options) { } } const select = target.select; + // Whether the caller supplied real filter conditions — read from the raw request, NOT the + // planner-augmented `conditions` (which by now may carry a synthetic `sort` pseudo-condition and + // injected full-scan condition). Used to pick the count-estimate source below. + const hasUserConditions = Array.isArray(target.conditions) && target.conditions.length > 0; if (conditions.length === 0) { conditions = [{ attribute: primaryKey, comparator: 'greater_than', value: true }]; } @@ -3513,12 +3529,91 @@ export function makeTable(options) { readTxn, transformToRecord ); + const offset = target.offset || 0; + const end = target.limit !== undefined ? offset + (target.limit as number) : undefined; + // `Prefer: count=` (REST pagination): materialize the requested page and attach a total record + // count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once, + // windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/ + // table estimate. Opt-in only — the default streaming path below is untouched. + // + // Requires a bounded page AND window. Counting is a pagination feature; both the limit and the + // offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the + // window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/ + // oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity), + // limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the + // normal streaming path with no count. This bounds the offset too: without it a huge offset would + // postpone the exact guardrail (which only engages past the page) until that offset was scanned. + const pageLimit = target.limit as number; + if ( + target.count && + Number.isInteger(pageLimit) && + pageLimit >= 0 && + pageLimit <= MAX_COUNT_PAGE && + Number.isInteger(offset) && + offset >= 0 && + offset + pageLimit <= MAX_EXACT_COUNT_SCAN + ) { + const wantExact = target.count === 'exact'; + const pageEnd = offset + pageLimit; + const countStart = performance.now(); + return (async () => { + const page: any = []; + let scanned = 0; + let exact = true; + try { + for await (const record of results) { + if (scanned >= offset && scanned < pageEnd) page.push(record); + scanned++; + // The page window [offset, pageEnd) is always collected in full first — the guardrail + // only ever abandons the running TOTAL, never truncates the page body. + if (scanned >= pageEnd) { + if (!wantExact) break; // `estimated` needs nothing past the page + // `exact` keeps counting the tail, bounded by a row cap AND a time budget so a + // large match set can't turn a bounded page fetch into an unbounded scan. + if (scanned > MAX_EXACT_COUNT_SCAN || performance.now() - countStart > MAX_EXACT_COUNT_MS) { + exact = false; + break; + } + } + } + } finally { + // We own the iteration here (no results.onDone consumer), so release the read + // transaction unconditionally — including when the drain throws — or the snapshot leaks. + txn.doneReadTxn(); + } + let total: number | null; + if (wantExact) { + total = exact ? scanned : null; + } else if (boundRowFilter || typeof target.vectorFilter === 'function') { + // An opaque row/vector filter shapes the result but isn't reflected in the index/condition + // estimate; guessing would both mislead and disclose cardinality the filter hides. + total = null; + } else if (!hasUserConditions) { + total = estimatedEntryCount(primaryStore); + } else { + // Estimate from the real conditions only — drop the planner's synthetic `sort` + // pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality. + const est = estimateCondition(TableResource)({ + conditions: conditions.filter((c: any) => c.comparator !== 'sort'), + operator: operator ? String(operator).toLowerCase() : 'and', + }); + total = isFinite(est) ? Math.round(est) : null; + } + // For an estimate, never report a total below the last row actually returned — keeps the + // Content-Range valid (start-end/total) when an estimate undershoots a non-empty page. + // Exact totals are authoritative (and an empty page past the end must not be clamped up). + if (!wantExact && total != null && page.length > 0 && total < offset + page.length) { + total = offset + page.length; + } + page.recordCount = total; + page.recordCountExact = wantExact && exact; + page.selectApplied = true; + page.getColumns = getColumns; + return page; + })() as any; + } // apply any offset/limit after all the sorting and filtering - if (target.offset || target.limit !== undefined) - results = results.slice( - target.offset, - target.limit !== undefined ? (target.offset || 0) + target.limit : undefined - ); + if (target.offset || target.limit !== undefined) results = results.slice(offset, end); results.onDone = () => { results.onDone = null; // ensure that it isn't called twice txn.doneReadTxn(); diff --git a/resources/search.ts b/resources/search.ts index c1f7dcafd..07eb4d4ab 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -1619,7 +1619,7 @@ export function flattenKey(key) { return key; } -function estimatedEntryCount(store) { +export function estimatedEntryCount(store) { const now = Date.now(); if ((store.estimatedEntryCountExpires || 0) < now) { // use getStats for LMDB because it is fast path, otherwise RocksDB can handle fast path on its own diff --git a/server/REST.ts b/server/REST.ts index 49e0c7be6..6ce143580 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -8,7 +8,7 @@ import { Resources } from '../resources/Resources.ts'; import { Resource, missingMethod, allowedMethods } from '../resources/Resource.ts'; import { IterableEventQueue } from '../resources/IterableEventQueue.ts'; import { transaction } from '../resources/transaction.ts'; -import { Headers, mergeHeaders } from '../server/serverHelpers/Headers.ts'; +import { Headers, mergeHeaders, addVaryHeader } from '../server/serverHelpers/Headers.ts'; import { generateJsonApi } from '../resources/openApi.ts'; import { getConfigPath } from '../config/configUtils.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; @@ -117,6 +117,39 @@ async function findInactiveComponent(url: string): Promise { } } +/** + * Emit RFC 7233-style pagination headers for a `Prefer: count=` request. Table.search returns the page + * with a `recordCount` (total matching records, or null when an exact scan hit its guardrail or an + * estimate was suppressed by an opaque filter). `Content-Range: items -/` lets a + * client paginate — `` is `*` when unavailable. `Preference-Applied` echoes the count mode the + * server actually applied (`exact` or `estimated`, after any per-mount downgrade), so a `.../*` total + * reads as "that mode was applied but the total is unavailable" rather than "no count was requested". + * All three headers are added to `Access-Control-Expose-Headers` so a browser can read them cross-origin + * (they aren't safelisted). + */ +function setCountHeaders(headers: Headers, offset: number, mode: string, page: any) { + const total = page.recordCount; + const len = Array.isArray(page) ? page.length : 0; + const range = len > 0 ? `${offset}-${offset + len - 1}` : '*'; + const totalStr = typeof total === 'number' ? String(total) : '*'; + headers.set('Range-Unit', 'items'); + headers.set('Content-Range', `items ${range}/${totalStr}`); + headers.set('Preference-Applied', `count=${mode}`); + // Append (don't overwrite) so a resource that already exposed its own headers keeps them. Compare + // case-insensitive comma tokens, not substrings, so an unrelated existing token (e.g. + // `X-Content-Range-Metadata`) doesn't suppress the real `Content-Range` token. + const exposed = headers.get('Access-Control-Expose-Headers'); + const existing = new Set( + (Array.isArray(exposed) ? exposed.join(',') : exposed || '') + .split(',') + .map((token) => token.trim().toLowerCase()) + .filter(Boolean) + ); + for (const name of ['Content-Range', 'Range-Unit', 'Preference-Applied']) { + if (!existing.has(name.toLowerCase())) headers.append('Access-Control-Expose-Headers', name, true); + } +} + async function http(request: Request, nextHandler, resources: Resources, httpOptions: any) { const headersObject = request.headers.asObject; const isSse = headersObject.accept === 'text/event-stream'; @@ -165,6 +198,27 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt (target as any).async = true; resource = entry.Resource; + // Pagination total-count opt-in (no default): `Prefer: count=exact|estimated`. Table.search + // reads target.count to compute the total emitted as Content-Range below. Only honored on + // GET/HEAD reads: setting it for other methods would hand their `search()` a materialized array + // instead of the AsyncIterable they iterate (e.g. a collection DELETE at Table.ts). + const prefer = headersObject['prefer']; + if (prefer && (method === 'GET' || method === 'HEAD')) { + // Exact counting scans the full matched set, so it is opt-in per mount via + // `rest: { exactCount: true }` (default off); count=exact is otherwise served as a cheap + // estimate. Estimated is always available. Accept a string `"true"` too, since not every + // config source coerces to a boolean. + const exactEnabled = (httpOptions as any).exactCount === true || (httpOptions as any).exactCount === 'true'; + for (const pref of parseHeaderValue(prefer as any)) { + const mode = (pref?.value as string | undefined)?.toLowerCase(); + if (pref?.name === 'count' && (mode === 'exact' || mode === 'estimated')) { + // A count=exact request on a mount that hasn't opted in is downgraded to estimated, + // signaled back to the client via Preference-Applied. + (target as any).count = mode === 'exact' && !exactEnabled ? 'estimated' : mode; + break; + } + } + } } if ((resource as any)?.isCaching) { const cacheControl = headersObject['cache-control']; @@ -348,9 +402,24 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt } // TODO: Handle 201 Created if (responseData !== undefined) { + if ( + (target as any)?.count && + (method === 'GET' || method === 'HEAD') && + Array.isArray(responseData) && + (responseData as any).recordCount !== undefined + ) { + // Array.isArray guards the single-record path: a record that happens to carry a + // `recordCount` attribute must not be mistaken for a count page (Table.search only ever + // returns the count as an array). + setCountHeaders(headers, (target as any).offset || 0, (target as any).count, responseData); + } responseObject.body = serialize(responseData, request, responseObject); if (method === 'HEAD') responseObject.body = undefined; // we want everything else to be the same as GET, but then omit the body } + // A collection read's count headers vary by the request's `Prefer` value; serialize() just reset + // `Vary`, so declare it here (after serialization) — otherwise a shared cache could serve count + // headers to a request that didn't ask, or a cached non-count response to one that did. + if ((method === 'GET' || method === 'HEAD') && (target as any)?.isCollection) addVaryHeader(headers, 'Prefer'); return responseObject; } catch (error) { error ??= new Error('Unknown error occurred'); diff --git a/unitTests/resources/queryCount.test.js b/unitTests/resources/queryCount.test.js new file mode 100644 index 000000000..92448bbf1 --- /dev/null +++ b/unitTests/resources/queryCount.test.js @@ -0,0 +1,163 @@ +require('../testUtils'); +const assert = require('node:assert'); +const { setupTestDBPath } = require('../testUtils'); +const { table } = require('#src/resources/databases'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + +// Covers the `Prefer: count=` pagination support in Table.search: a `count` target returns the +// requested page (offset/limit window) as an array carrying `recordCount` (total matching records) +// and `recordCountExact`, instead of the default lazy streaming iterable. +describe('Table.search count (REST pagination total-count)', () => { + let CountTable; + const TOTAL = 20; + const GROUP_A = 12; // ids 0..11 + const GROUP_B = TOTAL - GROUP_A; // ids 12..19 + + before(async function () { + setupTestDBPath(); + setMainIsWorker(true); + CountTable = table({ + table: 'QueryCountTable', + database: 'test', + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'group', indexed: true }, { name: 'name' }], + }); + let last; + for (let i = 0; i < TOTAL; i++) { + last = CountTable.put({ id: i, group: i < GROUP_A ? 'a' : 'b', name: 'n-' + i }); + } + await last; + }); + + it('exact: whole-collection page carries the exact total', async function () { + const page = await CountTable.search({ limit: 5, offset: 0, count: 'exact' }); + assert.ok(Array.isArray(page)); + assert.strictEqual(page.length, 5); + assert.strictEqual(page.recordCount, TOTAL); + assert.strictEqual(page.recordCountExact, true); + }); + + it('exact: total is independent of the offset/limit window', async function () { + const nearEnd = await CountTable.search({ limit: 5, offset: 18, count: 'exact' }); + assert.strictEqual(nearEnd.length, 2); // only ids 18,19 remain + assert.strictEqual(nearEnd.recordCount, TOTAL); + + const pastEnd = await CountTable.search({ limit: 5, offset: 25, count: 'exact' }); + assert.strictEqual(pastEnd.length, 0); + assert.strictEqual(pastEnd.recordCount, TOTAL); + assert.strictEqual(pastEnd.recordCountExact, true); + }); + + it('exact: a filtered query counts only matching records', async function () { + const paged = await CountTable.search({ + conditions: [{ attribute: 'group', value: 'a' }], + limit: 5, + count: 'exact', + }); + assert.strictEqual(paged.length, 5); + assert.strictEqual(paged.recordCount, GROUP_A); + assert.strictEqual(paged.recordCountExact, true); + + // a limit wide enough to cover the whole matched set gives page == matches, count == matches + const all = await CountTable.search({ + conditions: [{ attribute: 'group', value: 'b' }], + limit: 100, + count: 'exact', + }); + assert.strictEqual(all.length, GROUP_B); + assert.strictEqual(all.recordCount, GROUP_B); + }); + + it('count without a limit falls through to streaming (no unbounded drain)', async function () { + // A count is a pagination feature; without a limit the page would be the entire matched set, so the + // request is served by the normal streaming path with no count instead of materializing everything. + const results = CountTable.search({ conditions: [{ attribute: 'group', value: 'b' }], count: 'exact' }); + assert.ok(!Array.isArray(results), 'count without a limit must not materialize a page'); + assert.strictEqual(results.recordCount, undefined); + let n = 0; + for await (const _ of results) n++; + assert.strictEqual(n, GROUP_B); // still returns every matching row, just no count + }); + + it('a non-finite, negative, or oversized limit falls through to streaming (no count)', function () { + // The count page must be a finite, non-negative integer no larger than the max count-page size; + // anything else (limit(Infinity)/limit(foo)->NaN, a negative, or an oversized limit) must not + // materialize a count page. + for (const limit of [Infinity, NaN, -1, 20000]) { + const results = CountTable.search({ limit, count: 'exact' }); + assert.ok(!Array.isArray(results), `limit=${limit} must not materialize a count page`); + assert.strictEqual(results.recordCount, undefined); + } + }); + + it('a negative or oversized-window offset falls through to streaming (no count)', function () { + // The offset must be a finite non-negative integer, and offset+limit must be within the scan + // budget — otherwise a huge offset would postpone the exact guardrail until it had been scanned. + for (const t of [ + { offset: -5, limit: 10 }, + { offset: 2_000_000, limit: 10 }, + ]) { + const results = CountTable.search({ ...t, count: 'exact' }); + assert.ok(!Array.isArray(results), `offset=${t.offset} must not materialize a count page`); + assert.strictEqual(results.recordCount, undefined); + } + }); + + it('estimated: returns the page plus a positive estimate, flagged non-exact', async function () { + const page = await CountTable.search({ limit: 5, count: 'estimated' }); + assert.strictEqual(page.length, 5); + assert.strictEqual(typeof page.recordCount, 'number'); + assert.ok(page.recordCount > 0, `expected a positive estimate, got ${page.recordCount}`); + assert.strictEqual(page.recordCountExact, false); + + const filtered = await CountTable.search({ + conditions: [{ attribute: 'group', value: 'a' }], + limit: 3, + count: 'estimated', + }); + assert.strictEqual(filtered.length, 3); + assert.ok(filtered.recordCount > 0); + assert.strictEqual(filtered.recordCountExact, false); + }); + + it('default (no count): still returns the lazy streaming iterable, not a materialized page', async function () { + const results = CountTable.search({ limit: 5 }); + assert.ok(!Array.isArray(results), 'default search must not materialize an array'); + assert.strictEqual(results.recordCount, undefined); + let n = 0; + for await (const _ of results) n++; + assert.strictEqual(n, 5); + }); + + it('estimated: a sorted whole-collection estimate is not halved by the planner sort condition', async function () { + // Regression: the synthetic `sort` pseudo-condition used to flip hasUserConditions and feed + // estimateCondition, yielding ~entryCount/2 and impossible ranges (e.g. items 3-4/3). + const page = await CountTable.search({ sort: { attribute: 'id' }, offset: 3, limit: 3, count: 'estimated' }); + assert.strictEqual(page.length, 3); + assert.ok( + page.recordCount >= 3 + page.length, + `range must be valid: total ${page.recordCount} vs page end ${3 + page.length}` + ); + assert.ok( + page.recordCount >= TOTAL * 0.75, + `sorted estimate ${page.recordCount} should track table size ${TOTAL}, not half it` + ); + }); + + it('estimated: an opaque rowFilter yields an unknown total (null), not a misleading estimate', async function () { + const page = await CountTable.search({ rowFilter: (r) => r.group === 'a', limit: 3, count: 'estimated' }); + assert.ok(page.length <= 3); + assert.ok( + page.every((r) => r.group === 'a'), + 'page must honor the rowFilter' + ); + assert.strictEqual(page.recordCount, null); + assert.strictEqual(page.recordCountExact, false); + }); + + it('exact: honors a rowFilter in both the page and the count', async function () { + const page = await CountTable.search({ rowFilter: (r) => r.group === 'b', limit: 100, count: 'exact' }); + assert.strictEqual(page.length, GROUP_B); + assert.strictEqual(page.recordCount, GROUP_B); + assert.strictEqual(page.recordCountExact, true); + }); +}); diff --git a/unitTests/resources/queryCountBytes.test.js b/unitTests/resources/queryCountBytes.test.js new file mode 100644 index 000000000..c35292c9c --- /dev/null +++ b/unitTests/resources/queryCountBytes.test.js @@ -0,0 +1,72 @@ +require('../testUtils'); +const assert = require('node:assert'); +const { setupTestDBPath } = require('../testUtils'); +const { table } = require('#src/resources/databases'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + +// Verifies that a `Prefer: count=` page — which is materialized and returned AFTER Table.search +// releases its read transaction — does not hand back Bytes fields that alias the (now-released) read +// buffer. If decoded Bytes are zero-copy views into the read snapshot, churning reads/writes after the +// count would mutate the already-returned page. +describe('Table.search count with Bytes columns (read-buffer safety)', () => { + let BytesTable; + const N = 5; + const LEN = 64; + + before(async function () { + setupTestDBPath(); + setMainIsWorker(true); + BytesTable = table({ + table: 'BytesCountTable', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'data', type: 'Bytes' }, + ], + }); + let last; + for (let i = 0; i < N; i++) { + last = BytesTable.put({ id: i, data: new Uint8Array(LEN).fill(i + 1) }); + } + await last; + }); + + it('exact count returns intact Bytes that survive post-release buffer churn', async function () { + const page = await BytesTable.search({ limit: N, count: 'exact' }); + assert.strictEqual(page.recordCount, N); + assert.strictEqual(page.length, N); + + // correctness immediately after the count released its read txn + for (const rec of page) { + assert.ok(rec.data && rec.data.length === LEN, `record ${rec.id} missing bytes`); + assert.ok( + [...rec.data].every((b) => b === rec.id + 1), + `record ${rec.id} bytes wrong right after count: ${[...rec.data.slice(0, 4)]}` + ); + } + + // Snapshot the returned bytes, then churn writes + reads to reuse read buffers. + const snapshots = page.map((r) => [...r.data]); + for (let r = 0; r < 60; r++) { + await BytesTable.put({ id: 100 + r, data: new Uint8Array(LEN).fill(150 + (r % 100)) }); + } + for (let round = 0; round < 5; round++) { + // eslint-disable-next-line no-unused-vars + for await (const _ of BytesTable.search({ limit: 500 })) { + } + } + + // The already-returned page must be unchanged — no aliasing of the released read buffer. + page.forEach((rec, i) => { + assert.deepStrictEqual( + [...rec.data], + snapshots[i], + `record ${rec.id} bytes changed after churn — count page aliased the released read buffer` + ); + assert.ok( + [...rec.data].every((b) => b === rec.id + 1), + `record ${rec.id} bytes corrupted after churn: ${[...rec.data.slice(0, 4)]}` + ); + }); + }); +});