feat(rest): total-count pagination via Prefer: count= (Content-Range)

integrationTests/apiTests/rest.test.mjs

@@ -45,7 +45,8 @@ type SubObject @table(audit: false) @export {
 }
 `;
 
-const CONFIG_YAML = `rest: true
+const CONFIG_YAML = `rest:
+  exactCount: true
 graphqlSchema:
   files: '*.graphql'
 graphql: true
@@ -68,6 +69,26 @@ const SUBOBJECT_ROWS = [
 	{ id: '5', relatedId: '5', any: 'any-5' },
 ];
 
+// Second component whose REST mount uses the default (exact counting NOT opted in).
+const SCHEMA_GATE_GRAPHQL = `
+type GatedWidget @table @export(rest: true, mqtt: false) {
+	id: ID @primaryKey
+	name: String @indexed
+}
+`;
+
+const CONFIG_GATE_YAML = `rest: true
+graphqlSchema:
+  files: '*.graphql'
+graphql: true
+`;
+
+const GATE_ROWS = [
+	{ id: '1', name: 'w-1' },
+	{ id: '2', name: 'w-2' },
+	{ id: '3', name: 'w-3' },
+];
+
 const skipSuite = process.platform === 'win32';
 
 suite('REST query syntax', { skip: skipSuite }, (ctx) => {
@@ -244,4 +265,175 @@ suite('REST query syntax', { skip: skipSuite }, (ctx) => {
 			)
 			.expect(200);
 	});
+
+	// `Prefer: count=` (pagination total-count) — emits Content-Range/Range-Unit/Preference-Applied.
+	test('[rest] count=exact emits an exact Content-Range', () => {
+		return client
+			.reqRest('/Related/?sort(id)&limit(2)')
+			.set('Prefer', 'count=exact')
+			.expect('Range-Unit', 'items')
+			.expect('Content-Range', 'items 0-1/5')
+			.expect('Preference-Applied', 'count=exact')
+			.expect((r) => assert.equal(r.body.length, 2, r.text))
+			.expect((r) =>
+				assert.ok(
+					(r.headers['access-control-expose-headers'] || '').includes('Content-Range'),
+					`expected Content-Range to be exposed for CORS, got: ${r.headers['access-control-expose-headers']}`
+				)
+			)
+			.expect(200);
+	});
+
+	test('[rest] count=exact reflects the offset window but a total independent of it', () => {
+		return client
+			.reqRest('/Related/?sort(id)&limit(1,3)') // offset 1, 2 rows
+			.set('Prefer', 'count=exact')
+			.expect('Content-Range', 'items 1-2/5')
+			.expect((r) => assert.equal(r.body.length, 2, r.text))
+			.expect(200);
+	});
+
+	test('[rest] count=exact on a filtered query counts only matches', () => {
+		return client
+			.reqRest('/Related/?name==name-2&limit(10)')
+			.set('Prefer', 'count=exact')
+			.expect('Content-Range', 'items 0-0/1')
+			.expect('Preference-Applied', 'count=exact')
+			.expect(200);
+	});
+
+	test('[rest] count=estimated emits a numeric total flagged estimated', () => {
+		return client
+			.reqRest('/Related/?sort(id)&limit(2)')
+			.set('Prefer', 'count=estimated')
+			.expect('Preference-Applied', 'count=estimated')
+			.expect((r) => assert.match(r.headers['content-range'], /^items 0-1\/\d+$/, r.text))
+			.expect((r) => assert.equal(r.body.length, 2, r.text))
+			.expect(200);
+	});
+
+	test('[rest] an uncomputable total reports items .../* but still echoes the requested mode', () => {
+		// `name != x` estimates to Infinity, so the total is unavailable. The header must still say
+		// count=estimated (the mode applied), not count=none — the client asked, it just can't be given.
+		return client
+			.reqRest('/Related/?name!=name-2&limit(2)')
+			.set('Prefer', 'count=estimated')
+			.expect('Preference-Applied', 'count=estimated')
+			.expect((r) => assert.match(r.headers['content-range'], /^items 0-\d+\/\*$/, r.text))
+			.expect(200);
+	});
+
+	test('[rest] no Prefer header means no Content-Range (opt-in only)', () => {
+		return client
+			.reqRest('/Related/?sort(id)&limit(2)')
+			.expect((r) => assert.equal(r.headers['content-range'], undefined, r.text))
+			.expect((r) => assert.equal(r.body.length, 2, r.text))
+			.expect(200);
+	});
+
+	test('[rest] HEAD with count=exact returns the count header and no body', () => {
+		return request(client.restURL)
+			.head('/Related/?sort(id)&limit(2)')
+			.set(client.headers)
+			.set('Prefer', 'count=exact')
+			.expect('Content-Range', 'items 0-1/5')
+			.expect((r) => assert.ok(!r.body || Object.keys(r.body).length === 0, r.text))
+			.expect(200);
+	});
+
+	test('[rest] an oversized page limit falls through to streaming with no count', () => {
+		// A limit past the max count-page size must not materialize a count page — the request is served
+		// normally (all rows) with no Content-Range, rather than buffering an unbounded page.
+		return client
+			.reqRest('/Related/?sort(id)&limit(0,20000)')
+			.set('Prefer', 'count=exact')
+			.expect((r) => assert.equal(r.headers['content-range'], undefined, r.text))
+			.expect((r) => assert.equal(r.body.length, 5, r.text))
+			.expect(200);
+	});
+
+	test('[rest] a deep-page offset past the scan budget falls through with no count', () => {
+		// limit(start,end) with a huge start is a huge offset; the count path must not iterate an unbounded
+		// offset before its guardrail engages, so the request falls through with no Content-Range.
+		return client
+			.reqRest('/Related/?sort(id)&limit(2000000,2000010)')
+			.set('Prefer', 'count=exact')
+			.expect((r) => assert.equal(r.headers['content-range'], undefined, r.text))
+			.expect(200);
+	});
+
+	test('[rest] a collection read declares Vary: Prefer', () => {
+		// So a shared cache keys on Prefer and never serves count headers to a request that did not ask.
+		return client
+			.reqRest('/Related/?sort(id)&limit(2)')
+			.expect((r) => assert.match(r.headers['vary'] || '', /\bPrefer\b/i, r.text))
+			.expect(200);
+	});
+
+	test('[rest] DELETE with a limit and Prefer: count is not misrouted to the count path', () => {
+		// Regression: the count preference is GET/HEAD-only. A DELETE that also carried limit()+Prefer used
+		// to receive a materialized array from search() and throw instead of deleting.
+		return client
+			.req()
+			.send({ operation: 'insert', table: 'Related', records: [{ id: 'del-me', name: 'to-delete' }] })
+			.expect(200)
+			.then(() =>
+				request(client.restURL)
+					.delete('/Related/?id==del-me&limit(10)')
+					.set(client.headers)
+					.set('Prefer', 'count=exact')
+					.expect((r) => assert.ok(r.status >= 200 && r.status < 300, `expected 2xx, got ${r.status}: ${r.text}`))
+			)
+			.then(() => client.reqRest('/Related/?id==del-me').expect((r) => assert.equal(r.body.length, 0, r.text)));
+	});
+});
+
+// exactCount is a per-REST-mount policy, so it needs its own instance: two components exporting at
+// the root path share one mount (the handler dedupes), and this mount's default config would otherwise
+// bleed onto the main suite's routes (which opt in with exactCount: true).
+suite('REST count default (exact not opted in)', { skip: skipSuite }, (ctx) => {
+	let client;
+
+	before(async () => {
+		await startHarper(ctx, { config: {}, env: {} });
+		client = createApiClient(ctx.harper);
+
+		await installAppComponent(client, {
+			project: 'appCountGate',
+			files: { 'schema.graphql': SCHEMA_GATE_GRAPHQL, 'config.yaml': CONFIG_GATE_YAML },
+			probePath: '/GatedWidget/',
+			restartTimeoutMs: 120000,
+		});
+
+		await client
+			.req()
+			.send({ operation: 'insert', table: 'GatedWidget', records: GATE_ROWS })
+			.expect((r) => assert.ok(r.body.message.includes('inserted 3 of 3 records'), r.text))
+			.expect(200);
+	});
+
+	after(async () => {
+		await teardownHarper(ctx);
+	});
+
+	// Default (no exactCount opt-in): count=exact is served as a cheap estimate instead.
+	test('[rest] default downgrades count=exact to estimated', () => {
+		return client
+			.reqRest('/GatedWidget/?sort(id)&limit(2)')
+			.set('Prefer', 'count=exact')
+			.expect('Preference-Applied', 'count=estimated')
+			.expect((r) => assert.match(r.headers['content-range'], /^items 0-1\/\d+$/, r.text))
+			.expect((r) => assert.equal(r.body.length, 2, r.text))
+			.expect(200);
+	});
+
+	// estimated still works normally on a default mount.
+	test('[rest] default leaves count=estimated unchanged', () => {
+		return client
+			.reqRest('/GatedWidget/?sort(id)&limit(2)')
+			.set('Prefer', 'count=estimated')
+			.expect('Preference-Applied', 'count=estimated')
+			.expect((r) => assert.equal(r.body.length, 2, r.text))
+			.expect(200);
+	});
 });

resources/RequestTarget.ts

@@ -36,6 +36,8 @@ export class RequestTarget extends URLSearchParams {
 	declare select?: Select;
 	/**	 Return an explanation of the query order */
 	declare explain?: boolean;
+	/** Request a total count of matching records for pagination (REST `Prefer: count=exact|estimated`). */
+	declare count?: 'exact' | 'estimated';
 	/**	 Force the query to be executed in the order of conditions */
 	declare enforceExecutionOrder?: boolean;
 	declare lazy?: boolean;

resources/Table.ts

@@ -55,6 +55,7 @@ import {
 	searchByIndex,
 	findAttribute,
 	estimateCondition,
+	estimatedEntryCount,
 	flattenKey,
 	COERCIBLE_OPERATORS,
 	executeConditions,
@@ -128,6 +129,17 @@ const EVICTION_BATCH_SIZE = 100;
 // letting an unbounded number of open transactions (and their snapshots) accumulate.
 const MAX_INFLIGHT_EVICTION_BATCHES = 4;
 const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
+// Guardrails for `Prefer: count=exact`: once the requested page has been collected, counting the rest
+// of the match set is bounded by BOTH a row cap and a wall-clock budget, so a paginated read can't turn
+// into an unbounded scan. Exceeding either reports an unknown total (Content-Range `.../*`) rather than
+// truncating the page. These bound the count tail, not the page itself; a genuinely expensive query
+// (large filtered full-scan, in-memory sort) should still be gated by config before broad exposure.
+const MAX_EXACT_COUNT_SCAN = 1_000_000;
+const MAX_EXACT_COUNT_MS = 1_000;
+// Largest page a `Prefer: count=` request will materialize. A request whose limit exceeds this (or is
+// not a finite, non-negative integer, e.g. `limit(Infinity)`/`limit(foo)`) falls through to the normal
+// streaming path with no count, so a count request can't be coerced into buffering an unbounded page.
+const MAX_COUNT_PAGE = 10_000;
 envMngr.initSync();
 const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
 const LOCK_TIMEOUT = 10000;
@@ -3430,6 +3442,10 @@ export function makeTable(options) {
 				}
 			}
 			const select = target.select;
+			// Whether the caller supplied real filter conditions — read from the raw request, NOT the
+			// planner-augmented `conditions` (which by now may carry a synthetic `sort` pseudo-condition and
+			// injected full-scan condition). Used to pick the count-estimate source below.
+			const hasUserConditions = Array.isArray(target.conditions) && target.conditions.length > 0;
 			if (conditions.length === 0) {
 				conditions = [{ attribute: primaryKey, comparator: 'greater_than', value: true }];
 			}
@@ -3513,12 +3529,91 @@ export function makeTable(options) {
 				readTxn,
 				transformToRecord
 			);
+			const offset = target.offset || 0;
+			const end = target.limit !== undefined ? offset + (target.limit as number) : undefined;
+			// `Prefer: count=` (REST pagination): materialize the requested page and attach a total record
+			// count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once,
+			// windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/
+			// table estimate. Opt-in only — the default streaming path below is untouched.
+			//
+			// Requires a bounded page AND window. Counting is a pagination feature; both the limit and the
+			// offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the
+			// window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/
+			// oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity),
+			// limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the
+			// normal streaming path with no count. This bounds the offset too: without it a huge offset would
+			// postpone the exact guardrail (which only engages past the page) until that offset was scanned.
+			const pageLimit = target.limit as number;
+			if (
+				target.count &&
+				Number.isInteger(pageLimit) &&
+				pageLimit >= 0 &&
+				pageLimit <= MAX_COUNT_PAGE &&
+				Number.isInteger(offset) &&
+				offset >= 0 &&
+				offset + pageLimit <= MAX_EXACT_COUNT_SCAN
+			) {
+				const wantExact = target.count === 'exact';
+				const pageEnd = offset + pageLimit;
+				const countStart = performance.now();
+				return (async () => {
+					const page: any = [];
+					let scanned = 0;
+					let exact = true;
+					try {
+						for await (const record of results) {
+							if (scanned >= offset && scanned < pageEnd) page.push(record);
+							scanned++;
+							// The page window [offset, pageEnd) is always collected in full first — the guardrail
+							// only ever abandons the running TOTAL, never truncates the page body.
+							if (scanned >= pageEnd) {
+								if (!wantExact) break; // `estimated` needs nothing past the page
+								// `exact` keeps counting the tail, bounded by a row cap AND a time budget so a
+								// large match set can't turn a bounded page fetch into an unbounded scan.
+								if (scanned > MAX_EXACT_COUNT_SCAN || performance.now() - countStart > MAX_EXACT_COUNT_MS) {
+									exact = false;
+									break;
+								}
+							}
+						}
+					} finally {
+						// We own the iteration here (no results.onDone consumer), so release the read
+						// transaction unconditionally — including when the drain throws — or the snapshot leaks.
+						txn.doneReadTxn();
+					}
+					let total: number | null;
+					if (wantExact) {
+						total = exact ? scanned : null;
+					} else if (boundRowFilter || typeof target.vectorFilter === 'function') {
+						// An opaque row/vector filter shapes the result but isn't reflected in the index/condition
+						// estimate; guessing would both mislead and disclose cardinality the filter hides.
+						total = null;
+					} else if (!hasUserConditions) {
+						total = estimatedEntryCount(primaryStore);
+					} else {
+						// Estimate from the real conditions only — drop the planner's synthetic `sort`
+						// pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality.
+						const est = estimateCondition(TableResource)({
+							conditions: conditions.filter((c: any) => c.comparator !== 'sort'),
+							operator: operator ? String(operator).toLowerCase() : 'and',
+						});
+						total = isFinite(est) ? Math.round(est) : null;
+					}
+					// For an estimate, never report a total below the last row actually returned — keeps the
+					// Content-Range valid (start-end/total) when an estimate undershoots a non-empty page.
+					// Exact totals are authoritative (and an empty page past the end must not be clamped up).
+					if (!wantExact && total != null && page.length > 0 && total < offset + page.length) {
+						total = offset + page.length;
+					}
+					page.recordCount = total;
+					page.recordCountExact = wantExact && exact;
+					page.selectApplied = true;
+					page.getColumns = getColumns;
+					return page;
+				})() as any;
+			}
 			// apply any offset/limit after all the sorting and filtering
-			if (target.offset || target.limit !== undefined)
-				results = results.slice(
-					target.offset,
-					target.limit !== undefined ? (target.offset || 0) + target.limit : undefined
-				);
+			if (target.offset || target.limit !== undefined) results = results.slice(offset, end);
 			results.onDone = () => {
 				results.onDone = null; // ensure that it isn't called twice
 				txn.doneReadTxn();

resources/search.ts

@@ -1619,7 +1619,7 @@ export function flattenKey(key) {
 	return key;
 }
 
-function estimatedEntryCount(store) {
+export function estimatedEntryCount(store) {
 	const now = Date.now();
 	if ((store.estimatedEntryCountExpires || 0) < now) {
 		// use getStats for LMDB because it is fast path, otherwise RocksDB can handle fast path on its own