From 65cc10065d86239df343a6b4c17ee20d61cd9e61 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 12 Aug 2026 17:33:46 -0600 Subject: [PATCH 01/43] fix: restore Rocks index value iteration --- resources/RocksIndexStore.ts | 8 ++++++++ unitTests/resources/expiresAtAttribute.test.js | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index c223f84090..f6a44d591d 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -10,6 +10,7 @@ import { MAXIMUM_KEY } from 'ordered-binary'; declare module '@harperfast/rocksdb-js' { interface DBI { + getValues(indexedValue: any): Iterable; getValuesCount(indexedValue: any): number; } } @@ -40,6 +41,13 @@ export class RocksIndexStore extends RocksDatabase { }); } + /** Return the primary keys stored under one exact indexed value. */ + getValues(indexedValue: any): Iterable { + return super + .getRange({ start: indexedValue, end: [indexedValue, MAXIMUM_KEY] }) + .map(({ key }) => (key.length > 2 ? key.slice(1) : key[1])); + } + /** * Translate a put with indexed value and primary key to an underlying put * @param indexedValue - ignored, only used by LMDB diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 76e2726f6f..d3a0212b31 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -115,4 +115,15 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Table.primaryStore.committed; assert.strictEqual(await Table.get(1), null); }); + + it('enumerates exact Rocks index values for the expiration sweep', async function () { + const Table = makeTable('ExpiresAtIndexValues'); + const expiresAt = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.put(2, { id: 2, expiresAt }); + await Table.put(3, { id: 3, expiresAt: expiresAt + 1 }); + await Table.primaryStore.committed; + + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)].sort(), [1, 2]); + }); }); From 5dcb7ae9768fe01445e3559b00851cd44bb2c632 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 12 Aug 2026 17:55:38 -0600 Subject: [PATCH 02/43] test: strengthen Rocks expiration range coverage --- resources/RocksIndexStore.ts | 6 +++--- unitTests/resources/expiresAtAttribute.test.js | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index f6a44d591d..847bb67e12 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -10,7 +10,7 @@ import { MAXIMUM_KEY } from 'ordered-binary'; declare module '@harperfast/rocksdb-js' { interface DBI { - getValues(indexedValue: any): Iterable; + getValues(indexedValue: any): Iterable; getValuesCount(indexedValue: any): number; } } @@ -44,8 +44,8 @@ export class RocksIndexStore extends RocksDatabase { /** Return the primary keys stored under one exact indexed value. */ getValues(indexedValue: any): Iterable { return super - .getRange({ start: indexedValue, end: [indexedValue, MAXIMUM_KEY] }) - .map(({ key }) => (key.length > 2 ? key.slice(1) : key[1])); + .getKeys({ start: indexedValue, end: [indexedValue, MAXIMUM_KEY] }) + .map((key) => (key.length > 2 ? key.slice(1) : key[1])); } /** diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index d3a0212b31..1d3a4990ad 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -119,6 +119,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { it('enumerates exact Rocks index values for the expiration sweep', async function () { const Table = makeTable('ExpiresAtIndexValues'); const expiresAt = Date.now() + 60_000; + await Table.put(0, { id: 0, expiresAt: expiresAt - 1 }); await Table.put(1, { id: 1, expiresAt }); await Table.put(2, { id: 2, expiresAt }); await Table.put(3, { id: 3, expiresAt: expiresAt + 1 }); From 6ba7c9ea3fb2d423fb8dd13042f347071d834a33 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 12 Aug 2026 18:00:36 -0600 Subject: [PATCH 03/43] fix: preserve unique index keys for expiration scans --- resources/RocksIndexStore.ts | 15 ++++++++++++++- unitTests/resources/expiresAtAttribute.test.js | 7 ++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index 847bb67e12..a87552b7d6 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -6,7 +6,7 @@ import { RocksDatabase, } from '@harperfast/rocksdb-js'; import { Id } from './ResourceInterface.ts'; -import { MAXIMUM_KEY } from 'ordered-binary'; +import { compareKeys, MAXIMUM_KEY } from 'ordered-binary'; declare module '@harperfast/rocksdb-js' { interface DBI { @@ -36,6 +36,19 @@ export class RocksIndexStore extends RocksDatabase { end = [end, MAXIMUM_KEY]; } const translatedOptions = { ...options, start, end }; + if (options.values === false) { + let first = true; + let previous: any; + return super + .getRange(translatedOptions) + .map(({ key }) => key[0]) + .filter((key) => { + if (!first && compareKeys(previous, key) === 0) return false; + first = false; + previous = key; + return true; + }); + } return super.getRange(translatedOptions).map(({ key }) => { return { key: key[0], value: key.length > 2 ? key.slice(1) : key[1] }; }); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 1d3a4990ad..cca5ae7b21 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -125,6 +125,11 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Table.put(3, { id: 3, expiresAt: expiresAt + 1 }); await Table.primaryStore.committed; - assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)].sort(), [1, 2]); + const index = Table.indices.expiresAt; + assert.deepStrictEqual( + [...index.getRange({ start: true, values: false, end: expiresAt + 1, snapshot: false })], + [expiresAt - 1, expiresAt] + ); + assert.deepStrictEqual([...index.getValues(expiresAt)].sort(), [1, 2]); }); }); From 3841fa2d81d696043bf8bb9497bea073f9cdc855 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 12 Aug 2026 18:03:02 -0600 Subject: [PATCH 04/43] test: cover expiration sweep handoff --- resources/RocksIndexStore.ts | 2 -- unitTests/resources/expiresAtAttribute.test.js | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index a87552b7d6..96bddbf031 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -10,7 +10,6 @@ import { compareKeys, MAXIMUM_KEY } from 'ordered-binary'; declare module '@harperfast/rocksdb-js' { interface DBI { - getValues(indexedValue: any): Iterable; getValuesCount(indexedValue: any): number; } } @@ -54,7 +53,6 @@ export class RocksIndexStore extends RocksDatabase { }); } - /** Return the primary keys stored under one exact indexed value. */ getValues(indexedValue: any): Iterable { return super .getKeys({ start: indexedValue, end: [indexedValue, MAXIMUM_KEY] }) diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index cca5ae7b21..50e1418ff1 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -123,6 +123,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Table.put(1, { id: 1, expiresAt }); await Table.put(2, { id: 2, expiresAt }); await Table.put(3, { id: 3, expiresAt: expiresAt + 1 }); + await Table.put([4, 'part'], { id: [4, 'part'], expiresAt: expiresAt + 2 }); await Table.primaryStore.committed; const index = Table.indices.expiresAt; @@ -131,5 +132,6 @@ describe('@expiresAt attribute is authoritative over the table default', () => { [expiresAt - 1, expiresAt] ); assert.deepStrictEqual([...index.getValues(expiresAt)].sort(), [1, 2]); + assert.deepStrictEqual([...index.getValues(expiresAt + 2)], [[4, 'part']]); }); }); From 409138c42b91979a6e0dd51a0ea2ffa1bf2a0f6f Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 12 Aug 2026 18:12:30 -0600 Subject: [PATCH 05/43] fix: expose Rocks index values to expiration sweeps --- resources/RocksIndexStore.ts | 35 ++++++++----- resources/Table.ts | 38 ++++++++++++-- .../resources/expiresAtAttribute.test.js | 49 ++++++++++++++++++- 3 files changed, 102 insertions(+), 20 deletions(-) diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index 96bddbf031..d2d2877fb3 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -7,6 +7,7 @@ import { } from '@harperfast/rocksdb-js'; import { Id } from './ResourceInterface.ts'; import { compareKeys, MAXIMUM_KEY } from 'ordered-binary'; +import { ExtendedIterable } from '@harperfast/extended-iterable'; declare module '@harperfast/rocksdb-js' { interface DBI { @@ -36,26 +37,34 @@ export class RocksIndexStore extends RocksDatabase { } const translatedOptions = { ...options, start, end }; if (options.values === false) { - let first = true; - let previous: any; - return super - .getRange(translatedOptions) - .map(({ key }) => key[0]) - .filter((key) => { - if (!first && compareKeys(previous, key) === 0) return false; - first = false; - previous = key; - return true; - }); + const { limit, ...keyOptions } = translatedOptions; + const getKeys = () => super.getKeys(keyOptions); + return new ExtendedIterable({ + *[Symbol.iterator]() { + if (limit !== undefined && limit <= 0) return; + let first = true; + let previous: any; + let yielded = 0; + for (const key of getKeys()) { + const indexedValue = key[0]; + if (!first && compareKeys(previous, indexedValue) === 0) continue; + first = false; + previous = indexedValue; + yield indexedValue; + yielded++; + if (limit !== undefined && yielded >= limit) return; + } + }, + }); } return super.getRange(translatedOptions).map(({ key }) => { return { key: key[0], value: key.length > 2 ? key.slice(1) : key[1] }; }); } - getValues(indexedValue: any): Iterable { + getValues(indexedValue: any, options: Omit = {}): Iterable { return super - .getKeys({ start: indexedValue, end: [indexedValue, MAXIMUM_KEY] }) + .getKeys({ ...options, start: [indexedValue], end: [indexedValue, MAXIMUM_KEY] }) .map((key) => (key.length > 2 ? key.slice(1) : key[1])); } diff --git a/resources/Table.ts b/resources/Table.ts index 08d11c78e3..2f0ac17714 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -124,6 +124,7 @@ const RECORD_PRUNING_INTERVAL = 60000; // one minute // Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already // coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path. const EVICTION_BATCH_SIZE = 100; +const RECORD_EXPIRATION_BATCH_LIMIT = EVICTION_BATCH_SIZE * 100; // Cap on eviction-batch commits in flight at once, so commit I/O overlaps scan/staging without // letting an unbounded number of open transactions (and their snapshots) accumulate. const MAX_INFLIGHT_EVICTION_BATCHES = 4; @@ -6312,27 +6313,54 @@ export function makeTable(options) { if (runningRecordExpiration) return; runningRecordExpiration = true; try { + let processed = 0; + let pendingEvictions: Promise[] = []; const expiresAtName = expiresAtProperty.name; const index = indices[expiresAtName]; if (!index) throw new Error(`expiresAt attribute ${expiresAtProperty} must be indexed`); - for (const key of index.getRange({ + expirationKeys: for (const key of index.getRange({ start: true, values: false, end: Date.now(), snapshot: false, })) { - for (const id of index.getValues(key)) { + for (const id of index.getValues(key, { snapshot: true })) { const recordEntry = primaryStore.getEntry(id); if (!recordEntry?.value) { // cleanup the index if the record is gone - primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); + if (primaryStore.ifVersion) { + primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); + } else { + const transaction = new RocksTransaction(primaryStore.store); + const options = { transaction }; + if (!primaryStore.getEntry(id, options)?.value) { + index.remove(key, id, options); + pendingEvictions.push( + transaction.commit().catch((error) => { + try { + transaction.abort(); + } catch {} + if (error?.code !== 'ERR_BUSY') + logger.warn?.('Error cleaning dangling expiration index', id, error); + }) + ); + } else { + transaction.abort(); + } + } } else if (recordEntry.value[expiresAtName] < Date.now()) { // make sure the record hasn't changed and won't change while removing - TableResource.evict(id, recordEntry.value, recordEntry.version); + pendingEvictions.push(Promise.resolve(TableResource.evict(id, recordEntry.value, recordEntry.version))); + } + if (++processed % EVICTION_BATCH_SIZE === 0) { + await Promise.all(pendingEvictions); + pendingEvictions = []; + await rest(); } + if (processed >= RECORD_EXPIRATION_BATCH_LIMIT) break expirationKeys; } - await rest(); } + await Promise.all(pendingEvictions); } catch (error) { logger.error?.('Error in evicting old records', error); } finally { diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 50e1418ff1..716fb5a3b4 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -120,6 +120,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const Table = makeTable('ExpiresAtIndexValues'); const expiresAt = Date.now() + 60_000; await Table.put(0, { id: 0, expiresAt: expiresAt - 1 }); + await Table.put(5, { id: 5, expiresAt: expiresAt - 1 }); await Table.put(1, { id: 1, expiresAt }); await Table.put(2, { id: 2, expiresAt }); await Table.put(3, { id: 3, expiresAt: expiresAt + 1 }); @@ -127,11 +128,55 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Table.primaryStore.committed; const index = Table.indices.expiresAt; + const expirationKeys = index.getRange({ start: true, values: false, end: expiresAt + 1, snapshot: false }); + assert.deepStrictEqual([...expirationKeys], [expiresAt - 1, expiresAt]); + assert.deepStrictEqual([...expirationKeys], [expiresAt - 1, expiresAt]); + assert.deepStrictEqual([...expirationKeys.map((value) => value)], [expiresAt - 1, expiresAt]); assert.deepStrictEqual( - [...index.getRange({ start: true, values: false, end: expiresAt + 1, snapshot: false })], + [...index.getRange({ start: true, values: false, end: expiresAt + 1, limit: 2, snapshot: false })], [expiresAt - 1, expiresAt] ); - assert.deepStrictEqual([...index.getValues(expiresAt)].sort(), [1, 2]); + const sweepEntries = []; + for (const key of expirationKeys) { + for (const id of index.getValues(key)) sweepEntries.push([key, id]); + } + assert.deepStrictEqual(sweepEntries, [ + [expiresAt - 1, 0], + [expiresAt - 1, 5], + [expiresAt, 1], + [expiresAt, 2], + ]); + assert.deepStrictEqual( + [...index.getValues(expiresAt)].sort((left, right) => left - right), + [1, 2] + ); assert.deepStrictEqual([...index.getValues(expiresAt + 2)], [[4, 'part']]); }); + + it('physically evicts expired records and transactionally cleans dangling index entries', async function () { + const originalSetInterval = global.setInterval; + let runSweep; + global.setInterval = (callback, interval) => { + if (interval === 60_000) runSweep = callback; + return { unref() {} }; + }; + let Table; + try { + Table = makeTable('ExpiresAtSweep'); + } finally { + global.setInterval = originalSetInterval; + } + assert(runSweep, 'table creation should register the expiration sweep'); + + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.put(2, { id: 2, expiresAt }); + await Table.primaryStore.committed; + Table.primaryStore.removeSync(2); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1, 2]); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + }); }); From 3ae45a3973b22d1f90c69bccf1bd68f0627db965 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Thu, 13 Aug 2026 21:24:23 -0600 Subject: [PATCH 06/43] fix: harden RocksDB expiration sweeps --- DESIGN.md | 2 + resources/DESIGN.md | 12 + resources/RocksIndexStore.ts | 75 +++- resources/Table.ts | 245 ++++++++----- resources/blob.ts | 17 +- resources/databases.ts | 6 + .../resources/closeLoadedDatabases.test.js | 44 ++- .../resources/expiresAtAttribute.test.js | 336 +++++++++++++++++- 8 files changed, 627 insertions(+), 110 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 0f9eeeab06..b98382ff00 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -53,6 +53,8 @@ The mitigations live in three places: When adding a new commit-handler early-return path: reset `write.skipped = false` at the top of the handler if you don't already, then set `write.skipped = true` immediately before the `return`. Decide first whether the audit log will reference the blob (via `auditRecordToStore`) — if it does, leave `skipped` unset. `cleanupOrphans` is the periodic safety net; don't rely on it for transactional correctness. +RocksDB eviction has the inverse ordering constraint: removing a record that already references a blob must not unlink the file until the raw transaction commits. A transactional `store.remove()` is synchronous because it only stages the delete, so `removeEntry()` cannot infer durability from its return value on this path. `TableResource.evict()` therefore carries the blob-bearing record through the commit and calls `deleteBlobsInObject()` only from the commit-success handler. Shared eviction batches must continue excluding blob-bearing records because one commit covers many staged removals and can conflict as a unit. + **Source-unavailable blobs must not abort the commit.** `startPreCommitBlobsForRecord().complete()` awaits each blob's `saving` promise; a rejection there propagates up and aborts the record's apply (the replication subscription loop catches and logs it as `error in subscription handler`). For a blob the replication source can no longer provide — evicted/expired at the origin, the receiver having flagged the rejection `sourceBlobUnavailable` (harper-pro#403) — that abort permanently wedged a replication copy stream on an expiration cache table whose TTL-evicted blobs are gone everywhere: every orphaned record's apply re-threw, the copy never advanced, and backpressure pinned at ~100%. `complete()` therefore tolerates a `sourceBlobUnavailable` rejection (`isSourceBlobUnavailable`): the record commits with a diverged blob reference, left for proactive backfill (harper-pro#388). Local/transient save faults stay unmarked and still reject, so the write aborts and a reconnect retries it — no silent loss. This is the apply/commit-side complement to the replication receiver's resume-cursor advance (harper-pro#403/#405), which handles the durability-watermark side of the same missing blob. ## Over-time transactions are aborted, not force-committed (`DatabaseTransaction`/`LMDBTransaction`) diff --git a/resources/DESIGN.md b/resources/DESIGN.md index 2706bfe3b9..e13ec7343d 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -104,6 +104,18 @@ One giant `makeTable()` factory that returns a `TableResource extends Resource` --- +## Record expiration sweeps + +RocksDB `@expiresAt` sweeps walk one bounded composite-index range at a time with a fixed cutoff and an +owned `(expiresAt, primaryKey)` cursor. A sweep never holds an iterator snapshot across an `await`, and it +continues until every index entry at or before that cutoff has been considered. This keeps memory and native +iterator lifetime bounded without leaving a permanent backlog when more than one chunk expires together. + +Dangling-index cleanup reads and writes the primary key in the same RocksDB transaction as the index removal. +The primary write is a conflict guard against concurrent resurrection: absent keys receive a transactional +remove, while retained audit tombstones are rewritten byte-for-byte so their version and retention semantics do +not change. An `ERR_BUSY` abort leaves the newly written record and index authoritative for the next sweep. + ## Path routing & parameterised routes `Resources.ts` is the registry that maps URL paths to `Resource` classes. Resources are registered (`jsResource.ts`) from a component's exports: diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index d2d2877fb3..ff0b9d359e 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -9,6 +9,12 @@ import { Id } from './ResourceInterface.ts'; import { compareKeys, MAXIMUM_KEY } from 'ordered-binary'; import { ExtendedIterable } from '@harperfast/extended-iterable'; +function cloneKey(key: any): any { + if (Array.isArray(key)) return key.map(cloneKey); + if (key instanceof Uint8Array) return key.slice(); + return key; +} + declare module '@harperfast/rocksdb-js' { interface DBI { getValuesCount(indexedValue: any): number; @@ -35,22 +41,27 @@ export class RocksIndexStore extends RocksDatabase { if ((reverse ? !inclusiveEnd : inclusiveEnd) && end !== undefined) { end = [end, MAXIMUM_KEY]; } - const translatedOptions = { ...options, start, end }; + const translatedOptions: StoreIteratorOptions & { offset?: number } = { ...options, start, end }; if (options.values === false) { - const { limit, ...keyOptions } = translatedOptions; + const { limit, offset = 0, ...keyOptions } = translatedOptions; const getKeys = () => super.getKeys(keyOptions); return new ExtendedIterable({ *[Symbol.iterator]() { if (limit !== undefined && limit <= 0) return; let first = true; let previous: any; + let skipped = 0; let yielded = 0; for (const key of getKeys()) { const indexedValue = key[0]; if (!first && compareKeys(previous, indexedValue) === 0) continue; first = false; - previous = indexedValue; - yield indexedValue; + previous = cloneKey(indexedValue); + if (skipped < Math.max(0, offset)) { + skipped++; + continue; + } + yield cloneKey(indexedValue); yielded++; if (limit !== undefined && yielded >= limit) return; } @@ -62,10 +73,60 @@ export class RocksIndexStore extends RocksDatabase { }); } + getCompositeRange(options: { after?: any[]; end: any; limit: number }): Iterable<{ + key: any; + value: Id; + cursor: any[]; + }> { + const keyOptions: StoreIteratorOptions & { snapshot?: boolean } = { + start: options.after ?? true, + end: [options.end, MAXIMUM_KEY], + limit: options.limit, + snapshot: false, + ...(options.after === undefined ? {} : { exclusiveStart: true }), + }; + return super.getKeys(keyOptions).map((key) => { + const cursor = cloneKey(key) as any[]; + return { + key: cursor[0], + value: cursor.length > 2 ? cursor.slice(1) : cursor[1], + cursor, + }; + }); + } + getValues(indexedValue: any, options: Omit = {}): Iterable { - return super - .getKeys({ ...options, start: [indexedValue], end: [indexedValue, MAXIMUM_KEY] }) - .map((key) => (key.length > 2 ? key.slice(1) : key[1])); + const { + limit, + offset = 0, + ...keyOptions + } = options as Omit & { + offset?: number; + }; + const bounds = keyOptions.reverse + ? { start: [indexedValue, MAXIMUM_KEY], end: [indexedValue], inclusiveEnd: true } + : { start: [indexedValue], end: [indexedValue, MAXIMUM_KEY] }; + const primaryKey = (key: any[]) => (key.length > 2 ? key.slice(1).map(cloneKey) : cloneKey(key[1])); + if (Math.max(0, offset) === 0 && (limit === undefined || limit > 0)) { + return super.getKeys({ ...keyOptions, ...bounds, limit }).map(primaryKey); + } + const getKeys = () => super.getKeys({ ...keyOptions, ...bounds }); + return new ExtendedIterable({ + *[Symbol.iterator]() { + if (limit !== undefined && limit <= 0) return; + let skipped = 0; + let yielded = 0; + for (const key of getKeys()) { + if (skipped < Math.max(0, offset)) { + skipped++; + continue; + } + yield primaryKey(key); + yielded++; + if (limit !== undefined && yielded >= limit) return; + } + }, + }); } /** diff --git a/resources/Table.ts b/resources/Table.ts index 2f0ac17714..90f16368dc 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -11,7 +11,7 @@ import { SYSTEM_SCHEMA_NAME, MAX_SET_TIMEOUT_MS, } from '../utility/hdbTerms.ts'; -import { type Database } from 'lmdb'; +import { asBinary, type Database } from 'lmdb'; import { Script } from 'node:vm'; import { randomUUID } from 'node:crypto'; import { getIndexedValues } from '../utility/lmdb/commonUtility.ts'; @@ -74,7 +74,13 @@ import { recordAction, recordActionBinary } from './analytics/write.ts'; import { rebuildUpdateBefore } from './crdt.ts'; import { appendHeader } from '../server/serverHelpers/Headers.ts'; import fs from 'node:fs'; -import { Blob, deleteBlobsInObject, findBlobsInObject, startPreCommitBlobsForRecord } from './blob.ts'; +import { + Blob, + deleteBlobsInObject, + findBlobsInObject, + prepareBlobDeletion, + startPreCommitBlobsForRecord, +} from './blob.ts'; import { onStorageReclamation, getStorageSpaceStats } from '../server/storageReclamation.ts'; import { RequestTarget } from './RequestTarget.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; @@ -124,7 +130,6 @@ const RECORD_PRUNING_INTERVAL = 60000; // one minute // Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already // coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path. const EVICTION_BATCH_SIZE = 100; -const RECORD_EXPIRATION_BATCH_LIMIT = EVICTION_BATCH_SIZE * 100; // Cap on eviction-batch commits in flight at once, so commit I/O overlaps scan/staging without // letting an unbounded number of open transactions (and their snapshots) accumulate. const MAX_INFLIGHT_EVICTION_BATCHES = 4; @@ -421,6 +426,9 @@ export function makeTable(options) { let cleanupPriority = 0; let lastCleanupInterval: number; let cleanupTimer: NodeJS.Timeout; + let recordExpirationInterval: NodeJS.Timeout; + let recordExpirationCancelled = false; + let missingExpirationIndexReported = false; // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup let expirationScanScheduled = false; // set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table @@ -2030,11 +2038,19 @@ export function makeTable(options) { */ static evict(id, existingRecord, existingVersion) { let entry; + let deleteBlobsAfterCommit: (() => void) | undefined; + let currentEntry; const lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction() }); let transaction = lmdbTransaction.getReadTxn(); let options = { transaction }; let committed = false; try { + // A raw Rocks read transaction can reuse an older snapshot. Reject an already-refreshed + // record before consulting that snapshot; the transaction read below guards later races. + if (!primaryStore.ifVersion) { + currentEntry = primaryStore.getEntry(id); + if (!currentEntry || currentEntry.version !== existingVersion) return; + } if (hasSourceGet || audit) { if (!existingRecord) return; entry = primaryStore.getEntry(id, options); @@ -2059,8 +2075,15 @@ export function makeTable(options) { const removal = removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion); lmdbCompletion = Promise.all([indexCleanup, removal]); } else { - updateIndices(id, existingRecord, null, options); - removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options); + const removalEntry = entry ?? currentEntry ?? primaryStore.getEntry(id); + if (!removalEntry || removalEntry.version !== existingVersion) return; + updateIndices(id, removalEntry.value, null, options); + if (removalEntry?.value && removalEntry.metadataFlags & HAS_BLOBS) { + primaryStore.remove(removalEntry.key, options); + deleteBlobsAfterCommit = prepareBlobDeletion(removalEntry.value); + } else { + removeEntry(primaryStore, removalEntry, options); + } } committed = true; // Eviction is best-effort cleanup, run fire-and-forget from the record-expiration sweep and the @@ -2083,15 +2106,24 @@ export function makeTable(options) { // as DatabaseTransaction.commit() would abort it (no tracked writes). The raw commit bypasses // DatabaseTransaction's ERR_BUSY retry, so a concurrent-write conflict rejects here — swallow it // (abandon the eviction) and log anything unexpected, rather than letting it crash the process. - return (transaction as any).commit().catch((error) => { - // The commit failed, so the read-snapshot/transaction handle is still open — release it, as the - // batched-eviction path does on its own commit failures. committed===true skips the finally abort. - try { - (transaction as any).abort(); - } catch {} - if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); - else logger.warn?.('Error evicting record', id, error); - }); + return (transaction as any).commit().then( + () => { + try { + deleteBlobsAfterCommit?.(); + } catch (error) { + logger.warn?.('Error deleting blobs for evicted record', id, error); + } + }, + (error) => { + // The commit failed, so the read-snapshot/transaction handle is still open — release it, as the + // batched-eviction path does on its own commit failures. committed===true skips the finally abort. + try { + (transaction as any).abort(); + } catch {} + if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); + else logger.warn?.('Error evicting record', id, error); + } + ); } finally { if (!committed) { // Skip path or thrown error: abort instead of committing so we don't apply @@ -5194,6 +5226,8 @@ export function makeTable(options) { } static cleanup() { deleteCallbackHandle?.remove(); + recordExpirationCancelled = true; + if (recordExpirationInterval) clearInterval(recordExpirationInterval); } static _readTxnForContext(context) { return txnForContext(context).getReadTxn(); @@ -6070,15 +6104,15 @@ export function makeTable(options) { return true; } // RocksDB-only: coalesces eviction/tombstone removals into shared transactions so the cleanup - // scan pays one commit per batch instead of one per record. Descriptors hold only the decoded - // primary key and the version seen during the scan (both stable primitives — the scanned record - // value lives in a reused iterator buffer, so it is re-read fresh at commit time). Each record is - // version-guarded inside the commit transaction, and RocksDB's optimistic conflict detection + // scan pays one commit per batch instead of one per record. Descriptors hold stable, owned keys. + // Each record is guarded inside the commit transaction, and RocksDB's optimistic conflict detection // catches anything modified between staging and commit: on conflict (ERR_BUSY) we re-stage once // into a fresh transaction (dropping the now-changed record) and otherwise skip the batch, leaving // those records for the next cleanup cycle. - function createEvictionBatcher() { - type EvictItem = { type: 'evict' | 'tombstone'; key: any; version: number }; + function createEvictionBatcher(isCancelled: () => boolean = () => false) { + type EvictItem = + | { type: 'evict' | 'tombstone'; key: any; version: number } + | { type: 'dangling-index'; key: any; indexedValue: any; index: any }; let pending: EvictItem[] = []; const inFlight = new Set>(); @@ -6088,12 +6122,28 @@ export function makeTable(options) { const options = { transaction }; let staged = 0; for (const item of items) { + if (item.type === 'dangling-index') { + const entry = primaryStore.getEntry(item.key, options); + if (entry?.value != null) continue; + if (entry) { + const encodedTombstone = primaryStore.getBinarySync(item.key, options); + if (encodedTombstone === undefined) continue; + // Deliberate no-op write: make resurrection conflict with dangling-index cleanup. + primaryStore.putSync(item.key, asBinary(encodedTombstone.slice()), options); + } else { + primaryStore.removeSync(item.key, options); + } + item.index.remove(item.indexedValue, item.key, options); + staged++; + continue; + } const entry = primaryStore.getEntry(item.key, options); if (!entry || entry.version !== item.version) continue; // gone or changed since the scan; leave for next cycle if (item.type === 'tombstone') { if (entry.value != null) continue; // resurrected since the scan } else { if (entry.value == null) continue; // already removed + if (entry.metadataFlags & HAS_BLOBS) continue; // per-record evict() owns blob/commit ordering if (hasSourceGet && primaryStore.hasLock(item.key, entry.version)) continue; // resolution in progress updateIndices(item.key, entry.value, null, options); } @@ -6105,6 +6155,7 @@ export function makeTable(options) { async function commitItems(items: EvictItem[]) { for (let attempt = 0; attempt < 2; attempt++) { + if (isCancelled() || primaryStore.rootStore.status !== 'open') return; // Create the transaction inside the try: if the store is closing mid-scan, the constructor // can throw, and this promise is not always awaited (in-flight under the cap), so an // uncaught throw here would surface as an unhandled rejection. @@ -6152,8 +6203,8 @@ export function makeTable(options) { } return { - add(type: 'evict' | 'tombstone', key: any, version: number): Promise | void { - pending.push({ type, key, version }); + add(item: EvictItem): Promise | void { + pending.push(item); if (pending.length >= EVICTION_BATCH_SIZE) { const items = pending; pending = []; @@ -6161,6 +6212,7 @@ export function makeTable(options) { } }, async drain(): Promise { + if (isCancelled() || primaryStore.rootStore.status !== 'open') pending = []; if (pending.length > 0) { const items = pending; pending = []; @@ -6266,7 +6318,7 @@ export function makeTable(options) { // Blob-bearing records delete their blob files as a non-transactional side effect, so // they stay on the per-record evict() path that preserves the existing blob/commit ordering. if (batcher && !(action === 'evict' && metadataFlags & HAS_BLOBS)) { - await batcher.add(action, key, version); + await batcher.add({ type: action, key, version }); } else { const resolution = action === 'tombstone' @@ -6304,70 +6356,99 @@ export function makeTable(options) { }); } function runRecordExpirationEviction() { - // Periodically evict expired records, searching for records who expiresAt timestamp is before now - if (getWorkerIndex() === 0) { - // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting - setInterval(async () => { - // go through each database and table and then search for expired entries - // find any entries that are set to expire before now - if (runningRecordExpiration) return; - runningRecordExpiration = true; - try { - let processed = 0; - let pendingEvictions: Promise[] = []; - const expiresAtName = expiresAtProperty.name; - const index = indices[expiresAtName]; - if (!index) throw new Error(`expiresAt attribute ${expiresAtProperty} must be indexed`); - expirationKeys: for (const key of index.getRange({ - start: true, - values: false, - end: Date.now(), - snapshot: false, - })) { - for (const id of index.getValues(key, { snapshot: true })) { - const recordEntry = primaryStore.getEntry(id); - if (!recordEntry?.value) { - // cleanup the index if the record is gone - if (primaryStore.ifVersion) { - primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); - } else { - const transaction = new RocksTransaction(primaryStore.store); - const options = { transaction }; - if (!primaryStore.getEntry(id, options)?.value) { - index.remove(key, id, options); - pendingEvictions.push( - transaction.commit().catch((error) => { - try { - transaction.abort(); - } catch {} - if (error?.code !== 'ERR_BUSY') - logger.warn?.('Error cleaning dangling expiration index', id, error); - }) - ); - } else { - transaction.abort(); - } - } - } else if (recordEntry.value[expiresAtName] < Date.now()) { - // make sure the record hasn't changed and won't change while removing - pendingEvictions.push(Promise.resolve(TableResource.evict(id, recordEntry.value, recordEntry.version))); - } - if (++processed % EVICTION_BATCH_SIZE === 0) { - await Promise.all(pendingEvictions); - pendingEvictions = []; - await rest(); + if (getWorkerIndex() !== 0) return; + const expiresAtName = expiresAtProperty.name; + + async function sweepRocks(index: any, cutoff: number) { + const batcher = createEvictionBatcher(() => recordExpirationCancelled); + let after: any[] | undefined; + try { + while (!recordExpirationCancelled && primaryStore.rootStore.status === 'open') { + const previousAfter = after; + const entries = [...index.getCompositeRange({ after, end: cutoff, limit: EVICTION_BATCH_SIZE })]; + if (entries.length === 0) break; + let completedChunk = true; + for (const entry of entries) { + if (recordExpirationCancelled || primaryStore.rootStore.status !== 'open') { + completedChunk = false; + break; + } + after = entry.cursor; + const recordEntry = primaryStore.getEntry(entry.value); + let backpressure: Promise | void; + if (recordEntry?.value == null) { + backpressure = batcher.add({ + type: 'dangling-index', + key: entry.value, + indexedValue: entry.key, + index, + }); + } else if (recordEntry.value[expiresAtName] < cutoff) { + if (recordEntry.metadataFlags & HAS_BLOBS) { + await TableResource.evict(entry.value, recordEntry.value, recordEntry.version); + } else { + backpressure = batcher.add({ + type: 'evict', + key: entry.value, + version: recordEntry.version, + }); } - if (processed >= RECORD_EXPIRATION_BATCH_LIMIT) break expirationKeys; } + if (backpressure) await backpressure; } - await Promise.all(pendingEvictions); - } catch (error) { - logger.error?.('Error in evicting old records', error); - } finally { - runningRecordExpiration = false; + if (!completedChunk) break; + if (previousAfter && compareKeys(after, previousAfter) <= 0) { + logger.warn?.(`Record expiration cursor did not advance for ${tableName}`); + break; + } + await rest(); + if (entries.length < EVICTION_BATCH_SIZE) break; } - }, RECORD_PRUNING_INTERVAL).unref(); + } finally { + await batcher.drain(); + } } + + async function sweepLmdb(index: any, cutoff: number) { + for (const key of index.getRange({ start: true, values: false, end: cutoff, snapshot: false })) { + for (const id of index.getValues(key)) { + const recordEntry = primaryStore.getEntry(id); + if (recordEntry?.value == null) { + primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); + } else if (recordEntry.value[expiresAtName] < cutoff) { + TableResource.evict(id, recordEntry.value, recordEntry.version); + } + } + await rest(); + } + } + + async function sweep() { + if (runningRecordExpiration || recordExpirationCancelled) return; + runningRecordExpiration = true; + try { + if (primaryStore.rootStore.status !== 'open') return; + const index = indices[expiresAtName]; + if (!index) { + if (!missingExpirationIndexReported) { + missingExpirationIndexReported = true; + logger.warn?.(`expiresAt attribute ${expiresAtName} must be indexed for ${tableName}`); + } + return; + } + missingExpirationIndexReported = false; + const cutoff = Date.now(); + if (isRocksDB) await sweepRocks(index, cutoff); + else await sweepLmdb(index, cutoff); + } catch (error) { + logger.error?.('Error in evicting old records', error); + } finally { + runningRecordExpiration = false; + } + } + + recordExpirationInterval = setInterval(sweep, RECORD_PRUNING_INTERVAL); + recordExpirationInterval.unref(); } function residencyFromFunction(shardOrResidencyList: ResidencyDefinition): string[] | void { if (shardOrResidencyList == undefined) return; diff --git a/resources/blob.ts b/resources/blob.ts index 094bb47fe9..50a4a0dcc2 100644 --- a/resources/blob.ts +++ b/resources/blob.ts @@ -851,11 +851,11 @@ let deletionDelay = 500; * @param blob */ export function deleteBlob(blob: Blob): void { - // do we even need to check for completion here? const filePath = getFilePathForBlob(blob as any); - if (!filePath) { - return; - } + if (filePath) scheduleBlobFileDeletion(filePath); +} + +function scheduleBlobFileDeletion(filePath: string): void { setTimeout(() => { // TODO: we need to determine when any read transaction are done with the file, and then delete it, this is a hack to just give it some time for that unlink(filePath, (error) => { @@ -863,6 +863,15 @@ export function deleteBlob(blob: Blob): void { }); }, deletionDelay); } + +export function prepareBlobDeletion(object: any): () => void { + const filePaths: string[] = []; + findBlobsInObject(object, (blob) => { + const filePath = getFilePathForBlob(blob as any); + if (filePath) filePaths.push(filePath); + }); + return () => filePaths.forEach(scheduleBlobFileDeletion); +} export function setDeletionDelay(delay: number) { deletionDelay = delay; } diff --git a/resources/databases.ts b/resources/databases.ts index 55cd4fba2b..c002353cea 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1232,6 +1232,7 @@ export async function dropDatabase(databaseName) { try { for (const tableName in dbTables) { const table = dbTables[tableName]; + table.cleanup?.(); rootStore = table.primaryStore.rootStore; if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); lmdbDatabaseEnvs.delete(rootStore.path); @@ -1303,6 +1304,11 @@ export function closeDatabase(databaseName: string): boolean { for (const tableName in dbTables) { const table: any = dbTables[tableName]; if (!table?.primaryStore) continue; + try { + table.cleanup?.(); + } catch (error) { + logger.warn(`Error cleaning up table ${databaseName}.${tableName} while closing database:`, error); + } if (table.primaryStore.rootStore) rootStores.add(table.primaryStore.rootStore); for (const indexName in table.indices || {}) { closeStore(table.indices[indexName], `index ${tableName}.${indexName}`); diff --git a/unitTests/resources/closeLoadedDatabases.test.js b/unitTests/resources/closeLoadedDatabases.test.js index 176395b7c2..b43c07b1ca 100644 --- a/unitTests/resources/closeLoadedDatabases.test.js +++ b/unitTests/resources/closeLoadedDatabases.test.js @@ -9,8 +9,16 @@ require('../testUtils'); const assert = require('node:assert'); const { setupTestDBPath } = require('../testUtils'); -const { table, database, getDatabases, closeDatabase, closeLoadedDatabases } = require('#src/resources/databases'); +const { + table, + database, + getDatabases, + closeDatabase, + closeLoadedDatabases, + dropDatabase, +} = require('#src/resources/databases'); const { registryStatus, RocksDatabase } = require('@harperfast/rocksdb-js'); +const sinon = require('sinon'); describe('RocksDB handle release', function () { before(function () { @@ -69,4 +77,38 @@ describe('RocksDB handle release', function () { assert.strictEqual(refCountFor(dbPath), 0, 'tableless database should be released'); }); + + it('closeDatabase runs table cleanup before closing its stores', function () { + const Table = table({ + table: 'expiring', + database: 'closerelease4', + attributes: [ + { attribute: 'id', isPrimaryKey: true }, + { attribute: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + const cleanup = sinon.spy(Table, 'cleanup'); + + closeDatabase('closerelease4'); + + assert.strictEqual(cleanup.callCount, 1, 'table cleanup should run before its database closes'); + cleanup.restore(); + }); + + it('dropDatabase runs table cleanup before destroying its stores', async function () { + const Table = table({ + table: 'expiring', + database: 'droprelease5', + attributes: [ + { attribute: 'id', isPrimaryKey: true }, + { attribute: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + const cleanup = sinon.spy(Table, 'cleanup'); + + await dropDatabase('droprelease5'); + + assert.strictEqual(cleanup.callCount, 1, 'table cleanup should run before its database is destroyed'); + cleanup.restore(); + }); }); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 716fb5a3b4..73c33ca5ff 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -3,6 +3,13 @@ const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); const { table } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); +const { Transaction: RocksTransaction } = require('@harperfast/rocksdb-js'); +const { asBinary } = require('lmdb'); +const sinon = require('sinon'); +const { createBlob, getFilePathForBlob, setDeletionDelay } = require('#src/resources/blob'); +const { existsSync } = require('node:fs'); +const { setTimeout: delay } = require('node:timers/promises'); +const { waitFor } = require('../waitFor.js'); // A schema @expiresAt attribute must be authoritative over the table-level expiration default, in both // directions. Previously the field only armed a separate index-pruning sweep (which can only remove @@ -17,10 +24,11 @@ describe('@expiresAt attribute is authoritative over the table default', () => { setMainIsWorker(true); }); - const makeTable = (name, expirationSeconds) => + const makeTable = (name, expirationSeconds, options = {}) => table({ table: name, database: 'test', + ...options, ...(expirationSeconds == null ? {} : { expiration: expirationSeconds }), attributes: [ { name: 'id', isPrimaryKey: true }, @@ -33,6 +41,29 @@ describe('@expiresAt attribute is authoritative over the table default', () => { return Table.primaryStore.getEntry(id)?.expiresAt; }; + const captureExpirationSweep = (createTable) => { + let runSweep; + let sweepIntervalCount = 0; + const realSetInterval = global.setInterval; + const intervalStub = sinon.stub(global, 'setInterval').callsFake((callback, interval, ...args) => { + if (interval === 60_000) { + runSweep = callback; + sweepIntervalCount++; + return { unref() {} }; + } + return realSetInterval(callback, interval, ...args); + }); + let Table; + try { + Table = createTable(); + } finally { + intervalStub.restore(); + } + assert(runSweep, 'table creation should register the expiration sweep'); + assert.strictEqual(sweepIntervalCount, 1); + return { Table, runSweep }; + }; + it('extends: a far-future field overrides a short table default', async function () { const Table = makeTable('ExpiresAtExtend', 3); const fieldExpiresAt = Date.now() + 3_600_000; // 1h, far past the 3s table default @@ -133,8 +164,12 @@ describe('@expiresAt attribute is authoritative over the table default', () => { assert.deepStrictEqual([...expirationKeys], [expiresAt - 1, expiresAt]); assert.deepStrictEqual([...expirationKeys.map((value) => value)], [expiresAt - 1, expiresAt]); assert.deepStrictEqual( - [...index.getRange({ start: true, values: false, end: expiresAt + 1, limit: 2, snapshot: false })], - [expiresAt - 1, expiresAt] + [...index.getRange({ start: true, values: false, end: expiresAt + 2, limit: 1, snapshot: false })], + [expiresAt - 1] + ); + assert.deepStrictEqual( + [...index.getRange({ start: true, values: false, end: expiresAt + 2, offset: 1, limit: 2, snapshot: false })], + [expiresAt, expiresAt + 1] ); const sweepEntries = []; for (const key of expirationKeys) { @@ -151,22 +186,57 @@ describe('@expiresAt attribute is authoritative over the table default', () => { [1, 2] ); assert.deepStrictEqual([...index.getValues(expiresAt + 2)], [[4, 'part']]); + + const firstChunk = [...index.getCompositeRange({ end: expiresAt + 2, limit: 2 })]; + const cursor = firstChunk.at(-1).cursor; + const savedCursor = structuredClone(cursor); + assert.deepStrictEqual( + [...index.getCompositeRange({ end: expiresAt + 2, limit: 20 })].map((entry) => entry.value), + [0, 5, 1, 2, 3, [4, 'part']] + ); + assert.deepStrictEqual(cursor, savedCursor); + assert.deepStrictEqual( + [...index.getCompositeRange({ after: cursor, end: expiresAt + 2, limit: 20 })].map((entry) => entry.value), + [1, 2, 3, [4, 'part']] + ); }); - it('physically evicts expired records and transactionally cleans dangling index entries', async function () { - const originalSetInterval = global.setInterval; - let runSweep; - global.setInterval = (callback, interval) => { - if (interval === 60_000) runSweep = callback; - return { unref() {} }; - }; - let Table; - try { - Table = makeTable('ExpiresAtSweep'); - } finally { - global.setInterval = originalSetInterval; + it('keeps exact Rocks index matching distinct for similar values', async function () { + const Table = table({ + table: 'SimilarIndexValues', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'label', indexed: true }, + ], + }); + await Table.put(1, { id: 1, label: 'hello' }); + await Table.put(4, { id: 4, label: 'hello' }); + await Table.put(2, { id: 2, label: 'hello world' }); + await Table.put(3, { id: 3, label: 'hell' }); + await Table.put([5, 'part'], { id: [5, 'part'], label: 'hello' }); + await Table.primaryStore.committed; + + const index = Table.indices.label; + assert.deepStrictEqual([...index.getValues('hello')], [1, 4, [5, 'part']]); + assert.deepStrictEqual([...index.getValues('hello', { offset: 1, limit: 1 })], [4]); + assert.deepStrictEqual([...index.getValues('hello', { reverse: true })], [[5, 'part'], 4, 1]); + assert.deepStrictEqual([...index.getValues('hello', { reverse: true, offset: 1, limit: 1 })], [4]); + assert.deepStrictEqual([...index.getValues('hello world')], [2]); + assert.deepStrictEqual([...index.getValues('hell')], [3]); + + const searchResults = []; + for await (const record of Table.search({ + allowFullScan: false, + conditions: [{ attribute: 'label', value: 'hello' }], + })) { + searchResults.push(record.id); } - assert(runSweep, 'table creation should register the expiration sweep'); + assert.deepStrictEqual(searchResults, [1, 4, [5, 'part']]); + }); + + it('physically evicts expired records and transactionally cleans dangling index entries', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSweep')); const expiresAt = Date.now() - 1_000; await Table.put(1, { id: 1, expiresAt }); @@ -178,5 +248,239 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await runSweep(); assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + Table.cleanup(); + }); + + it('continues an expiration sweep beyond one RocksDB chunk', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSweepContinuation')); + const expiresAt = Date.now() - 1_000; + await Promise.all(Array.from({ length: 505 }, (_, id) => Table.put(id, { id, expiresAt }))); + await Table.primaryStore.committed; + + await runSweep(); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + for (let id = 0; id < 505; id++) assert.strictEqual(Table.primaryStore.getEntry(id)?.value, undefined); + Table.cleanup(); + }); + + it('keeps a blob file when an expired-record eviction conflicts', async function () { + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'ExpiresAtBlobConflict', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + const blob = createBlob(Buffer.alloc(20_000, 7)); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: blob }); + await Table.primaryStore.committed; + const filePath = getFilePathForBlob(blob); + assert(filePath); + assert(existsSync(filePath)); + + setDeletionDelay(0); + const commitStub = sinon + .stub(RocksTransaction.prototype, 'commit') + .rejects(Object.assign(new Error('injected optimistic conflict'), { code: 'ERR_BUSY' })); + let commitRestored = false; + try { + await runSweep(); + await delay(25); + assert(Table.primaryStore.getEntry(1)?.value, 'the failed eviction must leave the record intact'); + assert(existsSync(filePath), 'a failed eviction must leave the referenced blob file intact'); + commitStub.restore(); + commitRestored = true; + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + await waitFor(() => !existsSync(filePath), { + message: 'a committed eviction must remove its blob file', + }); + } finally { + if (!commitRestored) commitStub.restore(); + setDeletionDelay(500); + Table.cleanup(); + } + }); + + it('preserves a blob record refreshed before its eviction transaction starts', async function () { + const Table = table({ + table: 'ExpiresAtBlobRefreshRace', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }); + const oldBlob = createBlob(Buffer.alloc(20_000, 5)); + const freshBlob = createBlob(Buffer.alloc(20_000, 6)); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: oldBlob }); + await Table.primaryStore.committed; + const expiredEntry = Table.primaryStore.getEntry(1); + + const freshExpiresAt = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt: freshExpiresAt, payload: freshBlob }); + await Table.primaryStore.committed; + assert.notStrictEqual(Table.primaryStore.getEntry(1).version, expiredEntry.version); + await Table.evict(1, expiredEntry.value, expiredEntry.version); + + const refreshed = Table.primaryStore.getEntry(1)?.value; + assert.strictEqual(refreshed.expiresAt, freshExpiresAt); + assert.strictEqual(refreshed.payload.id, freshBlob.id); + assert(existsSync(getFilePathForBlob(freshBlob)), 'the refreshed record must retain its blob file'); + Table.cleanup(); + }); + + it('stops a running expiration sweep when the table is cleaned up', async function () { + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'ExpiresAtSweepCancellation', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + const expiresAt = Date.now() - 1_000; + await Table.put('a', { id: 'a', expiresAt, payload: createBlob(Buffer.alloc(20_000, 3)) }); + await Table.put('b', { id: 'b', expiresAt, payload: createBlob(Buffer.alloc(20_000, 4)) }); + await Table.primaryStore.committed; + const [, secondEntry] = [...Table.indices.expiresAt.getCompositeRange({ end: Date.now(), limit: 2 })]; + assert(secondEntry, 'the sweep should have a second entry to skip'); + + const originalEvict = Table.evict; + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { + evictionStarted = true; + await blockedEviction; + return originalEvict.apply(this, args); + }); + try { + const sweep = runSweep(); + await waitFor(() => evictionStarted, { message: 'the first eviction should start' }); + Table.cleanup(); + releaseEviction(); + await sweep; + } finally { + releaseEviction(); + evictStub.restore(); + } + + assert(Table.primaryStore.getEntry(secondEntry.value)?.value, 'cleanup must stop the sweep before its next entry'); + }); + + it('preserves retained tombstones while removing their dangling expiration index entries', async function () { + const { Table, runSweep } = captureExpirationSweep(() => + makeTable('ExpiresAtTombstoneSweep', undefined, { audit: true }) + ); + + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.delete(1); + await Table.primaryStore.committed; + const tombstone = Table.primaryStore.getEntry(1); + assert(tombstone); + assert.strictEqual(tombstone.value, null); + const tombstoneMetadata = { + version: tombstone.version, + expiresAt: tombstone.expiresAt, + metadataFlags: tombstone.metadataFlags, + residencyId: tombstone.residencyId, + }; + Table.indices.expiresAt.put(expiresAt, 1); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + + await runSweep(); + const retained = Table.primaryStore.getEntry(1); + assert(retained); + assert.strictEqual(retained.value, null); + assert.deepStrictEqual( + { + version: retained.version, + expiresAt: retained.expiresAt, + metadataFlags: retained.metadataFlags, + residencyId: retained.residencyId, + }, + tombstoneMetadata + ); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + Table.cleanup(); + }); + + it('conflicts dangling cleanup with a concurrent primary resurrection', async function () { + const Table = makeTable('ExpiresAtDanglingConflict'); + const expiresAt = Date.now() + 60_000; + Table.indices.expiresAt.put(expiresAt, 1); + + const transaction = new RocksTransaction(Table.primaryStore.store); + const options = { transaction }; + assert.strictEqual(Table.primaryStore.getEntry(1, options), undefined); + Table.primaryStore.removeSync(1, options); + Table.indices.expiresAt.remove(expiresAt, 1, options); + await Table.put(1, { id: 1, expiresAt }); + await assert.rejects(transaction.commit(), (error) => error?.code === 'ERR_BUSY'); + + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.id, 1); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.expiresAt, expiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + Table.cleanup(); + }); + + it('keeps a resurrection and its index when the real sweep cleanup conflicts', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSweepConflict')); + const expiresAt = Date.now() - 1_000; + Table.indices.expiresAt.put(expiresAt, 1); + + const originalCommit = RocksTransaction.prototype.commit; + let injected = false; + const commitStub = sinon.stub(RocksTransaction.prototype, 'commit').callsFake(async function (...args) { + if (!injected) { + injected = true; + await Table.put(1, { id: 1, expiresAt, name: 'resurrected' }); + } + return originalCommit.apply(this, args); + }); + try { + await runSweep(); + } finally { + commitStub.restore(); + } + + assert(injected); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.name, 'resurrected'); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + Table.cleanup(); + }); + + it('conflicts retained-tombstone cleanup with a concurrent primary resurrection', async function () { + const Table = makeTable('ExpiresAtTombstoneConflict', undefined, { audit: true }); + const expiresAt = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.delete(1); + await Table.primaryStore.committed; + Table.indices.expiresAt.put(expiresAt, 1); + + const transaction = new RocksTransaction(Table.primaryStore.store); + const options = { transaction }; + const tombstone = Table.primaryStore.getEntry(1, options); + assert.strictEqual(tombstone.value, null); + const encodedTombstone = Table.primaryStore.getBinarySync(1, options); + Table.primaryStore.putSync(1, asBinary(encodedTombstone), options); + Table.indices.expiresAt.remove(expiresAt, 1, options); + await Table.put(1, { id: 1, expiresAt, name: 'resurrected' }); + await assert.rejects(transaction.commit(), (error) => error?.code === 'ERR_BUSY'); + + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.name, 'resurrected'); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + Table.cleanup(); }); }); From 3ba9a192fee37333622d8ab5e0407aca3cf3fef2 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Thu, 13 Aug 2026 22:35:31 -0600 Subject: [PATCH 07/43] fix: preserve sweeps when database drop is blocked --- resources/databases.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/databases.ts b/resources/databases.ts index c002353cea..f8f2ca661f 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1232,7 +1232,6 @@ export async function dropDatabase(databaseName) { try { for (const tableName in dbTables) { const table = dbTables[tableName]; - table.cleanup?.(); rootStore = table.primaryStore.rootStore; if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); lmdbDatabaseEnvs.delete(rootStore.path); @@ -1240,6 +1239,7 @@ export async function dropDatabase(databaseName) { } for (const tableName in dbTables) { + dbTables[tableName].cleanup?.(); databaseEventsEmitter.emit('dropTable', tableName, databaseName); } From f982e8ffe2f3d89fa2f151909db4a092984ae740 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Thu, 13 Aug 2026 22:50:01 -0600 Subject: [PATCH 08/43] fix: bound expiration sweep event-loop work --- resources/Table.ts | 11 +++++++++-- resources/databases.ts | 1 + unitTests/resources/expiresAtAttribute.test.js | 15 ++++++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 90f16368dc..b13088362b 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -1406,6 +1406,8 @@ export function makeTable(options) { // invisible, and the tombstone guarantees the drop completes on the // next startup (or on a same-name create). delete databases[databaseName][tableName]; + TableResource.cleanup(); + while (runningRecordExpiration) await rest(); // The above stops new source-fill writes from starting, but a write from a get() // that already returned to its caller may still be in flight. Dropping the column // families out from under that write is a genuine invariant violation, not just a @@ -2046,9 +2048,9 @@ export function makeTable(options) { let committed = false; try { // A raw Rocks read transaction can reuse an older snapshot. Reject an already-refreshed - // record before consulting that snapshot; the transaction read below guards later races. + // record before consulting it; commit-time conflict detection guards later races. if (!primaryStore.ifVersion) { - currentEntry = primaryStore.getEntry(id); + currentEntry = primaryStore.getEntry(id, { lazy: true }); if (!currentEntry || currentEntry.version !== existingVersion) return; } if (hasSourceGet || audit) { @@ -6368,6 +6370,7 @@ export function makeTable(options) { const entries = [...index.getCompositeRange({ after, end: cutoff, limit: EVICTION_BATCH_SIZE })]; if (entries.length === 0) break; let completedChunk = true; + let entriesSinceYield = 0; for (const entry of entries) { if (recordExpirationCancelled || primaryStore.rootStore.status !== 'open') { completedChunk = false; @@ -6395,6 +6398,10 @@ export function makeTable(options) { } } if (backpressure) await backpressure; + if (++entriesSinceYield >= 10) { + entriesSinceYield = 0; + await rest(); + } } if (!completedChunk) break; if (previousAfter && compareKeys(after, previousAfter) <= 0) { diff --git a/resources/databases.ts b/resources/databases.ts index f8f2ca661f..5c0256d6c5 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -967,6 +967,7 @@ function initStores( table.updatedAttributes(); } } else { + if (recreateForEngineChange) table.cleanup?.(); table = setTable( tables, tableName, diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 73c33ca5ff..ae9eb3c6c9 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -241,9 +241,11 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const expiresAt = Date.now() - 1_000; await Table.put(1, { id: 1, expiresAt }); await Table.put(2, { id: 2, expiresAt }); + await Table.put([3, 'part'], { id: [3, 'part'], expiresAt }); await Table.primaryStore.committed; Table.primaryStore.removeSync(2); - assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1, 2]); + Table.primaryStore.removeSync([3, 'part']); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1, 2, [3, 'part']]); await runSweep(); assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); @@ -378,6 +380,17 @@ describe('@expiresAt attribute is authoritative over the table default', () => { assert(Table.primaryStore.getEntry(secondEntry.value)?.value, 'cleanup must stop the sweep before its next entry'); }); + it('cancels the expiration sweep before dropping a table', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtTableDrop')); + const cleanup = sinon.spy(Table, 'cleanup'); + + await Table.dropTable(); + await runSweep(); + + assert.strictEqual(cleanup.callCount, 1); + cleanup.restore(); + }); + it('preserves retained tombstones while removing their dangling expiration index entries', async function () { const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtTombstoneSweep', undefined, { audit: true }) From 7455c808df2e5361da314d6c94085eaa909572fb Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Thu, 13 Aug 2026 22:52:47 -0600 Subject: [PATCH 09/43] fix: drain expiration sweeps before table drop --- resources/Table.ts | 10 ++++- resources/databases.ts | 6 ++- .../resources/expiresAtAttribute.test.js | 44 +++++++++++++++---- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index b13088362b..a5437f285a 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -1407,7 +1407,15 @@ export function makeTable(options) { // next startup (or on a same-name create). delete databases[databaseName][tableName]; TableResource.cleanup(); - while (runningRecordExpiration) await rest(); + const expirationDrainDeadline = Date.now() + LOCK_TIMEOUT; + while (runningRecordExpiration) { + if (Date.now() >= expirationDrainDeadline) { + throw new Error( + `dropTable() timed out waiting for the expiration sweep on ${tableName}; refusing to drop its column families while cleanup is active.` + ); + } + await rest(); + } // The above stops new source-fill writes from starting, but a write from a get() // that already returned to its caller may still be in flight. Dropping the column // families out from under that write is a genuine invariant violation, not just a diff --git a/resources/databases.ts b/resources/databases.ts index 5c0256d6c5..6c1ec17a8c 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1240,7 +1240,11 @@ export async function dropDatabase(databaseName) { } for (const tableName in dbTables) { - dbTables[tableName].cleanup?.(); + try { + dbTables[tableName].cleanup?.(); + } catch (error) { + logger.warn(`Error cleaning up table ${databaseName}.${tableName} while dropping database:`, error); + } databaseEventsEmitter.emit('dropTable', tableName, databaseName); } diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index ae9eb3c6c9..b748536d21 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -380,15 +380,43 @@ describe('@expiresAt attribute is authoritative over the table default', () => { assert(Table.primaryStore.getEntry(secondEntry.value)?.value, 'cleanup must stop the sweep before its next entry'); }); - it('cancels the expiration sweep before dropping a table', async function () { - const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtTableDrop')); - const cleanup = sinon.spy(Table, 'cleanup'); - - await Table.dropTable(); - await runSweep(); + it('waits for an active expiration sweep before dropping a table', async function () { + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'ExpiresAtTableDrop', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: createBlob(Buffer.alloc(20_000, 8)) }); + await Table.primaryStore.committed; - assert.strictEqual(cleanup.callCount, 1); - cleanup.restore(); + const originalEvict = Table.evict; + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { + evictionStarted = true; + await blockedEviction; + return originalEvict.apply(this, args); + }); + try { + const sweep = runSweep(); + await waitFor(() => evictionStarted, { message: 'the eviction should start' }); + let dropResolved = false; + const drop = Table.dropTable().then(() => (dropResolved = true)); + await delay(10); + assert.strictEqual(dropResolved, false, 'drop must wait for the active sweep'); + releaseEviction(); + await Promise.all([sweep, drop]); + } finally { + releaseEviction(); + evictStub.restore(); + } }); it('preserves retained tombstones while removing their dangling expiration index entries', async function () { From 7efafa5b5d2a5164281442b77aa6a20fd7cb2549 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Thu, 13 Aug 2026 22:56:28 -0600 Subject: [PATCH 10/43] fix: await expiration cleanup before database close --- resources/RocksIndexStore.ts | 2 +- resources/Table.ts | 8 +++- resources/databases.ts | 21 +++++++-- server/itc/serverHandlers.js | 2 +- server/jobs/jobProcess.ts | 2 +- .../resources/closeLoadedDatabases.test.js | 10 ++--- .../resources/expiresAtAttribute.test.js | 45 ++++++++++++++++++- .../resources/rocksCompressionOpen.test.js | 8 ++-- 8 files changed, 79 insertions(+), 19 deletions(-) diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index ff0b9d359e..4da0b120d8 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -161,7 +161,7 @@ export class RocksIndexStore extends RocksDatabase { */ DBI.prototype.getValuesCount = function getValuesCount(indexedValue: any) { if (this instanceof RocksIndexStore) { - return this.store.getCount(this._context, { start: indexedValue, end: [indexedValue, MAXIMUM_KEY] }); + return this.store.getCount(this._context, { start: [indexedValue], end: [indexedValue, MAXIMUM_KEY] }); } throw new Error('getValuesCount is only supported if dupSort=true'); }; diff --git a/resources/Table.ts b/resources/Table.ts index a5437f285a..769537d7c0 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -449,6 +449,7 @@ export function makeTable(options) { // it, so such a table takes the guarded serialization path rather than the raw fast path. let hasSurfacedComputed = false; let runningRecordExpiration: boolean; + let recordExpirationCompletion: Promise = Promise.resolve(); const isRocksDB = primaryStore instanceof RocksDatabase; type BigInt64ArrayAndMaxSafeId = BigInt64Array & { maxSafeId: number }; let idIncrementer: BigInt64ArrayAndMaxSafeId; @@ -5238,6 +5239,7 @@ export function makeTable(options) { deleteCallbackHandle?.remove(); recordExpirationCancelled = true; if (recordExpirationInterval) clearInterval(recordExpirationInterval); + return recordExpirationCompletion; } static _readTxnForContext(context) { return txnForContext(context).getReadTxn(); @@ -6439,7 +6441,8 @@ export function makeTable(options) { } async function sweep() { - if (runningRecordExpiration || recordExpirationCancelled) return; + if (runningRecordExpiration) return recordExpirationCompletion; + if (recordExpirationCancelled) return; runningRecordExpiration = true; try { if (primaryStore.rootStore.status !== 'open') return; @@ -6462,7 +6465,8 @@ export function makeTable(options) { } } - recordExpirationInterval = setInterval(sweep, RECORD_PRUNING_INTERVAL); + const runSweep = () => (recordExpirationCompletion = sweep()); + recordExpirationInterval = setInterval(runSweep, RECORD_PRUNING_INTERVAL); recordExpirationInterval.unref(); } function residencyFromFunction(shardOrResidencyList: ResidencyDefinition): string[] | void { diff --git a/resources/databases.ts b/resources/databases.ts index 6c1ec17a8c..b580e04c84 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1295,10 +1295,11 @@ export async function dropDatabase(databaseName) { * `resetDatabases()`/`getDatabases()` rescan reloads it (or skips it while a restore is in * progress, per the restore marker checks in the scan). */ -export function closeDatabase(databaseName: string): boolean { +export async function closeDatabase(databaseName: string): Promise { const dbTables = databases[databaseName]; if (!dbTables) return false; const rootStores = new Set(); + const cleanupCompletions: Promise[] = []; const closeStore = (store: any, description: string) => { try { store?.close?.(); @@ -1310,11 +1311,23 @@ export function closeDatabase(databaseName: string): boolean { const table: any = dbTables[tableName]; if (!table?.primaryStore) continue; try { - table.cleanup?.(); + const completion = table.cleanup?.(); + if (completion?.then) { + cleanupCompletions.push( + completion.catch((error) => + logger.warn(`Error awaiting cleanup for table ${databaseName}.${tableName}:`, error) + ) + ); + } } catch (error) { logger.warn(`Error cleaning up table ${databaseName}.${tableName} while closing database:`, error); } if (table.primaryStore.rootStore) rootStores.add(table.primaryStore.rootStore); + } + await Promise.all(cleanupCompletions); + for (const tableName in dbTables) { + const table: any = dbTables[tableName]; + if (!table?.primaryStore) continue; for (const indexName in table.indices || {}) { closeStore(table.indices[indexName], `index ${tableName}.${indexName}`); } @@ -1356,7 +1369,7 @@ export function closeDatabase(databaseName: string): boolean { * (skipped by the loop), is never restored online, and the exiting worker may still touch the job * table during teardown. Best-effort: closing failures are swallowed inside `closeDatabase`. */ -export function closeLoadedDatabases(): void { +export async function closeLoadedDatabases(): Promise { // snapshot the names first: closeDatabase() deletes from `databases` as it goes for (const databaseName of Object.keys(databases)) { const dbTables = databases[databaseName]; @@ -1373,7 +1386,7 @@ export function closeLoadedDatabases(): void { if (!isRocks && (definedDatabases?.get(databaseName) as any)?.rootStore instanceof RocksDatabase) { isRocks = true; } - if (isRocks) closeDatabase(databaseName); + if (isRocks) await closeDatabase(databaseName); } } // HNSW_NO_AUTOVERSION kill-switch: when set, a NEW index initializes as legacy rather than diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.js index 9b48ef3266..a6802e6b51 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.js @@ -50,7 +50,7 @@ async function schemaHandler(event) { // rewrite the database directory. The rescan below (resetDatabases) skips reloading it while // the restoring marker is present, and reloads it on the completion signal (marker gone). if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP && event.message.schema) { - closeDatabase(event.message.schema); + await closeDatabase(event.message.schema); } await cleanLmdbMap(event.message); await syncSchemaMetadata(event.message); diff --git a/server/jobs/jobProcess.ts b/server/jobs/jobProcess.ts index 9688a3c32b..c02127507c 100644 --- a/server/jobs/jobProcess.ts +++ b/server/jobs/jobProcess.ts @@ -91,7 +91,7 @@ const JOB_ID = JOB_NAME.substring(4); // effort — never let cleanup mask the job result. try { const { closeLoadedDatabases } = await import('../../resources/databases.ts'); - closeLoadedDatabases(); + await closeLoadedDatabases(); } catch (closeErr) { harperLogger.warn('Error releasing database handles on job worker exit:', closeErr); } diff --git a/unitTests/resources/closeLoadedDatabases.test.js b/unitTests/resources/closeLoadedDatabases.test.js index b43c07b1ca..a2b193b3ea 100644 --- a/unitTests/resources/closeLoadedDatabases.test.js +++ b/unitTests/resources/closeLoadedDatabases.test.js @@ -46,7 +46,7 @@ describe('RocksDB handle release', function () { const dbPath = rootStore.path; assert.ok(refCountFor(dbPath) > 0, 'database should be open before close'); - closeDatabase('closerelease1'); + await closeDatabase('closerelease1'); assert.strictEqual(refCountFor(dbPath), 0, 'no native handles should remain after closeDatabase'); }); @@ -58,7 +58,7 @@ describe('RocksDB handle release', function () { if (!(a instanceof RocksDatabase)) return this.skip(); assert.ok(refCountFor(a.path) > 0 && refCountFor(b.path) > 0, 'both databases should be open'); - closeLoadedDatabases(); + await closeLoadedDatabases(); assert.strictEqual(refCountFor(a.path), 0, 'database a should be released'); assert.strictEqual(refCountFor(b.path), 0, 'database b should be released'); @@ -73,12 +73,12 @@ describe('RocksDB handle release', function () { const dbPath = rootStore.path; assert.ok(refCountFor(dbPath) > 0, 'tableless database should be open'); - closeLoadedDatabases(); + await closeLoadedDatabases(); assert.strictEqual(refCountFor(dbPath), 0, 'tableless database should be released'); }); - it('closeDatabase runs table cleanup before closing its stores', function () { + it('closeDatabase runs table cleanup before closing its stores', async function () { const Table = table({ table: 'expiring', database: 'closerelease4', @@ -89,7 +89,7 @@ describe('RocksDB handle release', function () { }); const cleanup = sinon.spy(Table, 'cleanup'); - closeDatabase('closerelease4'); + await closeDatabase('closerelease4'); assert.strictEqual(cleanup.callCount, 1, 'table cleanup should run before its database closes'); cleanup.restore(); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index b748536d21..ba1f8053e9 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -1,7 +1,7 @@ require('../testUtils'); const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); -const { table } = require('#src/resources/databases'); +const { table, closeDatabase } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { Transaction: RocksTransaction } = require('@harperfast/rocksdb-js'); const { asBinary } = require('lmdb'); @@ -224,6 +224,9 @@ describe('@expiresAt attribute is authoritative over the table default', () => { assert.deepStrictEqual([...index.getValues('hello', { reverse: true, offset: 1, limit: 1 })], [4]); assert.deepStrictEqual([...index.getValues('hello world')], [2]); assert.deepStrictEqual([...index.getValues('hell')], [3]); + assert.strictEqual(index.getValuesCount('hello'), 3); + assert.strictEqual(index.getValuesCount('hello world'), 1); + assert.strictEqual(index.getValuesCount('hell'), 1); const searchResults = []; for await (const record of Table.search({ @@ -419,6 +422,46 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } }); + it('waits for an active expiration sweep before closing a database', async function () { + const database = 'ExpiresAtDatabaseClose'; + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'expiring', + database, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: createBlob(Buffer.alloc(20_000, 9)) }); + await Table.primaryStore.committed; + + const originalEvict = Table.evict; + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { + evictionStarted = true; + await blockedEviction; + return originalEvict.apply(this, args); + }); + try { + const sweep = runSweep(); + await waitFor(() => evictionStarted, { message: 'the eviction should start' }); + let closeResolved = false; + const close = closeDatabase(database).then(() => (closeResolved = true)); + await delay(10); + assert.strictEqual(closeResolved, false, 'close must wait for the active sweep'); + releaseEviction(); + await Promise.all([sweep, close]); + } finally { + releaseEviction(); + evictStub.restore(); + } + }); + it('preserves retained tombstones while removing their dangling expiration index entries', async function () { const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtTombstoneSweep', undefined, { audit: true }) diff --git a/unitTests/resources/rocksCompressionOpen.test.js b/unitTests/resources/rocksCompressionOpen.test.js index 958221f727..faabd0d31d 100644 --- a/unitTests/resources/rocksCompressionOpen.test.js +++ b/unitTests/resources/rocksCompressionOpen.test.js @@ -37,8 +37,8 @@ describe('storage.rocks.compression reaches a real RocksDB open', function () { mkdirSync(dir, { recursive: true }); }); - afterEach(function () { - closeDatabase(databaseName); + afterEach(async function () { + await closeDatabase(databaseName); rmSync(dir, { recursive: true, force: true }); }); @@ -90,13 +90,13 @@ describe('storage.rocks.compression reaches a real RocksDB open', function () { resetRocksCompression(); const initial = database({ database: databaseName }); assert.strictEqual(initial.compression.algorithm, 'none'); - closeDatabase(databaseName); + await closeDatabase(databaseName); setProperty(CONFIG_PARAMS.STORAGE_ROCKS_COMPRESSION, 'zstd'); resetRocksCompression(); const upgraded = database({ database: databaseName }); assert.strictEqual(upgraded.compression.algorithm, 'zstd'); - closeDatabase(databaseName); + await closeDatabase(databaseName); const reopenedSibling = RocksDatabase.open(dir, { name: 'sibling' }); try { From 1112e7d9d85cf6bd3a6d81e96c7ca0ad030c3b45 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Thu, 13 Aug 2026 23:05:43 -0600 Subject: [PATCH 11/43] fix: serialize expiration teardown --- resources/Table.ts | 12 ++--- resources/databases.ts | 51 ++++++++++++++++--- .../resources/closeLoadedDatabases.test.js | 16 +++++- .../resources/expiresAtAttribute.test.js | 42 ++++++++++++++- 4 files changed, 105 insertions(+), 16 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 769537d7c0..74dd7934cf 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -1411,9 +1411,9 @@ export function makeTable(options) { const expirationDrainDeadline = Date.now() + LOCK_TIMEOUT; while (runningRecordExpiration) { if (Date.now() >= expirationDrainDeadline) { - throw new Error( - `dropTable() timed out waiting for the expiration sweep on ${tableName}; refusing to drop its column families while cleanup is active.` - ); + const message = `dropTable() timed out waiting for the expiration sweep on ${tableName}; the table is unloaded and its drop will complete on restart or same-name create.`; + logger.warn?.(message); + throw new Error(message); } await rest(); } @@ -2088,10 +2088,10 @@ export function makeTable(options) { } else { const removalEntry = entry ?? currentEntry ?? primaryStore.getEntry(id); if (!removalEntry || removalEntry.version !== existingVersion) return; - updateIndices(id, removalEntry.value, null, options); - if (removalEntry?.value && removalEntry.metadataFlags & HAS_BLOBS) { + updateIndices(id, existingRecord, null, options); + if (existingRecord && removalEntry.metadataFlags & HAS_BLOBS) { primaryStore.remove(removalEntry.key, options); - deleteBlobsAfterCommit = prepareBlobDeletion(removalEntry.value); + deleteBlobsAfterCommit = prepareBlobDeletion(existingRecord); } else { removeEntry(primaryStore, removalEntry, options); } diff --git a/resources/databases.ts b/resources/databases.ts index b580e04c84..0fdfdd5301 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -96,6 +96,7 @@ const logger = forComponent('storage'); const DEFAULT_DATABASE_NAME = 'data'; const DEFINED_TABLES = Symbol('defined-tables'); +const DATABASE_CLOSE_TIMEOUT = 10_000; const DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages initSync(); /** @@ -1239,12 +1240,24 @@ export async function dropDatabase(databaseName) { rocksdbDatabaseEnvs.delete(rootStore.path); } + const cleanupCompletions: Promise[] = []; for (const tableName in dbTables) { try { - dbTables[tableName].cleanup?.(); + const completion = dbTables[tableName].cleanup?.(); + if (completion?.then) { + cleanupCompletions.push( + completion.catch((error) => + logger.warn(`Error awaiting cleanup for table ${databaseName}.${tableName}:`, error) + ) + ); + } } catch (error) { logger.warn(`Error cleaning up table ${databaseName}.${tableName} while dropping database:`, error); } + } + await Promise.all(cleanupCompletions); + + for (const tableName in dbTables) { databaseEventsEmitter.emit('dropTable', tableName, databaseName); } @@ -1295,9 +1308,21 @@ export async function dropDatabase(databaseName) { * `resetDatabases()`/`getDatabases()` rescan reloads it (or skips it while a restore is in * progress, per the restore marker checks in the scan). */ -export async function closeDatabase(databaseName: string): Promise { +const closingDatabases = new Map>(); + +export function closeDatabase(databaseName: string): Promise { + const activeClose = closingDatabases.get(databaseName); + if (activeClose) return activeClose; + const completion = closeDatabaseOnce(databaseName).finally(() => closingDatabases.delete(databaseName)); + closingDatabases.set(databaseName, completion); + return completion; +} + +async function closeDatabaseOnce(databaseName: string): Promise { const dbTables = databases[databaseName]; if (!dbTables) return false; + const tableEntries: [string, any][] = []; + for (const tableName in dbTables) tableEntries.push([tableName, dbTables[tableName]]); const rootStores = new Set(); const cleanupCompletions: Promise[] = []; const closeStore = (store: any, description: string) => { @@ -1307,8 +1332,7 @@ export async function closeDatabase(databaseName: string): Promise { logger.warn(`Error closing ${description} while closing database ${databaseName}:`, error); } }; - for (const tableName in dbTables) { - const table: any = dbTables[tableName]; + for (const [tableName, table] of tableEntries) { if (!table?.primaryStore) continue; try { const completion = table.cleanup?.(); @@ -1324,9 +1348,22 @@ export async function closeDatabase(databaseName: string): Promise { } if (table.primaryStore.rootStore) rootStores.add(table.primaryStore.rootStore); } - await Promise.all(cleanupCompletions); - for (const tableName in dbTables) { - const table: any = dbTables[tableName]; + let timeout: NodeJS.Timeout | undefined; + const timedOut = Symbol('timedOut'); + const cleanupResult = await Promise.race([ + Promise.all(cleanupCompletions), + new Promise((resolve) => { + timeout = setTimeout(() => resolve(timedOut), DATABASE_CLOSE_TIMEOUT); + timeout.unref(); + }), + ]); + if (timeout) clearTimeout(timeout); + if (cleanupResult === timedOut) { + logger.warn( + `Timed out after ${DATABASE_CLOSE_TIMEOUT}ms waiting for expiration cleanup while closing database ${databaseName}; closing its stores to avoid leaking native handles.` + ); + } + for (const [tableName, table] of tableEntries) { if (!table?.primaryStore) continue; for (const indexName in table.indices || {}) { closeStore(table.indices[indexName], `index ${tableName}.${indexName}`); diff --git a/unitTests/resources/closeLoadedDatabases.test.js b/unitTests/resources/closeLoadedDatabases.test.js index a2b193b3ea..8ab213f2b5 100644 --- a/unitTests/resources/closeLoadedDatabases.test.js +++ b/unitTests/resources/closeLoadedDatabases.test.js @@ -95,10 +95,22 @@ describe('RocksDB handle release', function () { cleanup.restore(); }); + it('coalesces concurrent closes of the same database', async function () { + const rootStore = openRocksDb('closerelease5'); + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + + const first = closeDatabase('closerelease5'); + const second = closeDatabase('closerelease5'); + assert.strictEqual(second, first, 'concurrent callers should share one close operation'); + await first; + + assert.strictEqual(refCountFor(rootStore.path), 0); + }); + it('dropDatabase runs table cleanup before destroying its stores', async function () { const Table = table({ table: 'expiring', - database: 'droprelease5', + database: 'droprelease6', attributes: [ { attribute: 'id', isPrimaryKey: true }, { attribute: 'expiresAt', expiresAt: true, indexed: true }, @@ -106,7 +118,7 @@ describe('RocksDB handle release', function () { }); const cleanup = sinon.spy(Table, 'cleanup'); - await dropDatabase('droprelease5'); + await dropDatabase('droprelease6'); assert.strictEqual(cleanup.callCount, 1, 'table cleanup should run before its database is destroyed'); cleanup.restore(); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index ba1f8053e9..7a12b90506 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -1,7 +1,7 @@ require('../testUtils'); const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); -const { table, closeDatabase } = require('#src/resources/databases'); +const { table, closeDatabase, dropDatabase } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { Transaction: RocksTransaction } = require('@harperfast/rocksdb-js'); const { asBinary } = require('lmdb'); @@ -462,6 +462,46 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } }); + it('waits for an active expiration sweep before destroying a database', async function () { + const database = 'ExpiresAtDatabaseDrop'; + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'expiring', + database, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: createBlob(Buffer.alloc(20_000, 10)) }); + await Table.primaryStore.committed; + + const originalEvict = Table.evict; + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { + evictionStarted = true; + await blockedEviction; + return originalEvict.apply(this, args); + }); + try { + const sweep = runSweep(); + await waitFor(() => evictionStarted, { message: 'the eviction should start' }); + let dropResolved = false; + const drop = dropDatabase(database).then(() => (dropResolved = true)); + await delay(10); + assert.strictEqual(dropResolved, false, 'drop must wait for the active sweep'); + releaseEviction(); + await Promise.all([sweep, drop]); + } finally { + releaseEviction(); + evictStub.restore(); + } + }); + it('preserves retained tombstones while removing their dangling expiration index entries', async function () { const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtTombstoneSweep', undefined, { audit: true }) From 030a7f59db851e84e27d69d61e156f707eaf0a0d Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Thu, 13 Aug 2026 23:08:46 -0600 Subject: [PATCH 12/43] fix: bound destructive cleanup waits --- resources/Table.ts | 3 +- resources/databases.ts | 32 ++++++++++++------- .../resources/expiresAtAttribute.test.js | 4 +++ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 74dd7934cf..e36790b040 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2089,7 +2089,8 @@ export function makeTable(options) { const removalEntry = entry ?? currentEntry ?? primaryStore.getEntry(id); if (!removalEntry || removalEntry.version !== existingVersion) return; updateIndices(id, existingRecord, null, options); - if (existingRecord && removalEntry.metadataFlags & HAS_BLOBS) { + if (removalEntry.metadataFlags & HAS_BLOBS) { + if (!existingRecord) return; primaryStore.remove(removalEntry.key, options); deleteBlobsAfterCommit = prepareBlobDeletion(existingRecord); } else { diff --git a/resources/databases.ts b/resources/databases.ts index 0fdfdd5301..b336460480 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -97,6 +97,20 @@ const logger = forComponent('storage'); const DEFAULT_DATABASE_NAME = 'data'; const DEFINED_TABLES = Symbol('defined-tables'); const DATABASE_CLOSE_TIMEOUT = 10_000; + +async function waitForTableCleanup(completions: Promise[]): Promise { + let timeout: NodeJS.Timeout | undefined; + const timedOut = Symbol('timedOut'); + const result = await Promise.race([ + Promise.all(completions), + new Promise((resolve) => { + timeout = setTimeout(() => resolve(timedOut), DATABASE_CLOSE_TIMEOUT); + timeout.unref(); + }), + ]); + if (timeout) clearTimeout(timeout); + return result !== timedOut; +} const DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages initSync(); /** @@ -1255,7 +1269,11 @@ export async function dropDatabase(databaseName) { logger.warn(`Error cleaning up table ${databaseName}.${tableName} while dropping database:`, error); } } - await Promise.all(cleanupCompletions); + if (!(await waitForTableCleanup(cleanupCompletions))) { + throw new Error( + `Timed out after ${DATABASE_CLOSE_TIMEOUT}ms waiting for expiration cleanup; refusing to destroy database ${databaseName} while cleanup is active.` + ); + } for (const tableName in dbTables) { databaseEventsEmitter.emit('dropTable', tableName, databaseName); @@ -1348,17 +1366,7 @@ async function closeDatabaseOnce(databaseName: string): Promise { } if (table.primaryStore.rootStore) rootStores.add(table.primaryStore.rootStore); } - let timeout: NodeJS.Timeout | undefined; - const timedOut = Symbol('timedOut'); - const cleanupResult = await Promise.race([ - Promise.all(cleanupCompletions), - new Promise((resolve) => { - timeout = setTimeout(() => resolve(timedOut), DATABASE_CLOSE_TIMEOUT); - timeout.unref(); - }), - ]); - if (timeout) clearTimeout(timeout); - if (cleanupResult === timedOut) { + if (!(await waitForTableCleanup(cleanupCompletions))) { logger.warn( `Timed out after ${DATABASE_CLOSE_TIMEOUT}ms waiting for expiration cleanup while closing database ${databaseName}; closing its stores to avoid leaking native handles.` ); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 7a12b90506..b464085b36 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -286,6 +286,10 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const filePath = getFilePathForBlob(blob); assert(filePath); assert(existsSync(filePath)); + const blobEntry = Table.primaryStore.getEntry(1); + await Table.evict(1, undefined, blobEntry.version); + assert(Table.primaryStore.getEntry(1)?.value, 'an eviction without the record value must fail safe'); + assert(existsSync(filePath)); setDeletionDelay(0); const commitStub = sinon From 1273f961f8056efe0b007b73a8e4d1cf32fbf70c Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 08:37:44 -0600 Subject: [PATCH 13/43] fix(storage): make expiration cleanup teardown-safe --- resources/DESIGN.md | 10 + resources/Table.ts | 250 ++++++++++++++---- resources/databases.ts | 44 ++- resources/search.ts | 23 +- .../resources/closeLoadedDatabases.test.js | 60 +++++ .../resources/expiresAtAttribute.test.js | 228 ++++++++++++++++ 6 files changed, 541 insertions(+), 74 deletions(-) diff --git a/resources/DESIGN.md b/resources/DESIGN.md index efd55c6960..17fe18f80b 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -107,6 +107,12 @@ One giant `makeTable()` factory that returns a `TableResource extends Resource` ## Record expiration sweeps +An `@expiresAt` attribute is a public timestamp field backed by the record encoder's expiration metadata. +Its index stores one canonical, non-negative epoch-millisecond number derived from that metadata, regardless of +whether the API value was a number, numeric string, ISO string, or `Date`. Writes, index rebuilds, index searches, +and scan filters apply the same normalization. `expirationIndexVersion` participates in the normal resumable +schema reindex flow so stores created before canonicalization are rebuilt without a separate open-time scan. + RocksDB `@expiresAt` sweeps walk one bounded composite-index range at a time with a fixed cutoff and an owned `(expiresAt, primaryKey)` cursor. A sweep never holds an iterator snapshot across an `await`, and it continues until every index entry at or before that cutoff has been considered. This keeps memory and native @@ -117,6 +123,10 @@ The primary write is a conflict guard against concurrent resurrection: absent ke remove, while retained audit tombstones are rewritten byte-for-byte so their version and retention semantics do not change. An `ERR_BUSY` abort leaves the newly written record and index authoritative for the next sweep. +Table cleanup is a join point for both the primary reclamation scan and the `@expiresAt` index sweep. Database +close and drop wait for that join before closing or destroying stores. A timeout fails closed: handles and +registries remain live and cleanup scheduling is resumed, so active work is never raced by teardown. + ## Path routing & parameterised routes `Resources.ts` is the registry that maps URL paths to `Resource` classes. Resources are registered (`jsResource.ts`) from a component's exports: diff --git a/resources/Table.ts b/resources/Table.ts index 72529c7600..54f9a4e4ef 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -110,6 +110,7 @@ export type Attribute = { computedFromExpression?: any; embed?: { source: string; model: string }; version?: any; + expirationIndexVersion?: number; properties?: Array; elements?: Attribute; sealed?: boolean; @@ -133,6 +134,7 @@ const EVICTION_BATCH_SIZE = 100; // Cap on eviction-batch commits in flight at once, so commit I/O overlaps scan/staging without // letting an unbounded number of open transactions (and their snapshots) accumulate. const MAX_INFLIGHT_EVICTION_BATCHES = 4; +const MAX_CLEANUP_CONCURRENCY = 50; const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]); envMngr.initSync(); const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); @@ -146,6 +148,17 @@ export function ignoreAlreadyDropped(error: any): void { if (error?.message?.includes('Column family already dropped')) return; throw error; } + +export function expirationTimestamp(value: any): number | undefined { + let timestamp = NaN; + if (typeof value === 'number' || typeof value === 'bigint') timestamp = Number(value); + else if (value instanceof Date) timestamp = value.getTime(); + else if (typeof value === 'string' && value.trim() !== '') { + const numeric = Number(value); + timestamp = Number.isFinite(numeric) ? numeric : Date.parse(value); + } + return Number.isFinite(timestamp) && timestamp >= 0 ? timestamp : undefined; +} // A frozen record we may need to copy-on-mutate before stamping it (records are immutable — decoded // records are frozen and 5.2 record caching relies on it). Only plain/record objects qualify: never // a Buffer/typed-array (spreading would corrupt the binary into a {0:.., 1:..} object) or a primitive @@ -418,6 +431,7 @@ export function makeTable(options) { if (attribute.expiresAt) expiresAtProperty = attribute; if (attribute.isPrimaryKey) primaryKeyAttribute = attribute; } + const expiresAtIndexName = expiresAtProperty?.name; let deleteCallbackHandle: { remove: () => void }; let prefetchIds = []; let prefetchCallbacks = []; @@ -427,8 +441,9 @@ export function makeTable(options) { let cleanupPriority = 0; let lastCleanupInterval: number; let cleanupTimer: NodeJS.Timeout; + let cleanupClosed = false; + let cleanupWasScheduled = false; let recordExpirationInterval: NodeJS.Timeout; - let recordExpirationCancelled = false; let missingExpirationIndexReported = false; // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup let expirationScanScheduled = false; @@ -1408,15 +1423,20 @@ export function makeTable(options) { // invisible, and the tombstone guarantees the drop completes on the // next startup (or on a same-name create). delete databases[databaseName][tableName]; - TableResource.cleanup(); - const expirationDrainDeadline = Date.now() + LOCK_TIMEOUT; - while (runningRecordExpiration) { - if (Date.now() >= expirationDrainDeadline) { - const message = `dropTable() timed out waiting for the expiration sweep on ${tableName}; the table is unloaded and its drop will complete on restart or same-name create.`; - logger.warn?.(message); - throw new Error(message); - } - await rest(); + const cleanupCompletion = TableResource.cleanup(); + let cleanupTimer: NodeJS.Timeout; + const cleanupTimedOut = Symbol('cleanupTimedOut'); + const cleanupResult = await Promise.race([ + cleanupCompletion, + new Promise((resolve) => { + cleanupTimer = setTimeout(() => resolve(cleanupTimedOut), LOCK_TIMEOUT); + }), + ]); + clearTimeout(cleanupTimer!); + if (cleanupResult === cleanupTimedOut) { + const message = `dropTable() timed out waiting for cleanup on ${tableName}; the table is unloaded and its drop will complete on restart or same-name create.`; + logger.warn?.(message); + throw new Error(message); } // The above stops new source-fill writes from starting, but a write from a get() // that already returned to its caller may still be in flight. Dropping the column @@ -2082,14 +2102,14 @@ export function makeTable(options) { // Capture both promises so a real write failure on either resolves through evict()'s catch // below rather than escaping as an unhandled rejection from the fire-and-forget callers. const indexCleanup = primaryStore.ifVersion(id, existingVersion, () => { - updateIndices(id, existingRecord, null); + updateIndices(id, existingRecord, null, undefined, undefined, entry?.expiresAt); }); const removal = removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion); lmdbCompletion = Promise.all([indexCleanup, removal]); } else { const removalEntry = entry ?? currentEntry ?? primaryStore.getEntry(id); if (!removalEntry || removalEntry.version !== existingVersion) return; - updateIndices(id, existingRecord, null, options); + updateIndices(id, existingRecord, null, options, undefined, removalEntry.expiresAt); if (removalEntry.metadataFlags & HAS_BLOBS) { if (!existingRecord) return; primaryStore.remove(removalEntry.key, options); @@ -2891,26 +2911,12 @@ export function makeTable(options) { // Read from recordToStore so the metadata matches exactly what the pruning sweep later // reads back. Falls back to the table default when the field is unset or not a timestamp. const fieldExpiresAt = expiresAtProperty ? recordToStore?.[expiresAtProperty.name] : undefined; - // Coerce only genuine timestamp shapes: a number/bigint epoch, a Date, or a numeric/ISO - // string. Booleans, empty/whitespace strings, and null/undefined fall through to NaN so a - // nonsensical field value uses the table default rather than expiring the record at epoch 0. - let fieldExpiresAtMs = NaN; - if (typeof fieldExpiresAt === 'number' || typeof fieldExpiresAt === 'bigint') - fieldExpiresAtMs = Number(fieldExpiresAt); - else if (fieldExpiresAt instanceof Date) fieldExpiresAtMs = fieldExpiresAt.getTime(); - else if (typeof fieldExpiresAt === 'string' && fieldExpiresAt.trim() !== '') { - const numeric = Number(fieldExpiresAt); - fieldExpiresAtMs = Number.isFinite(numeric) ? numeric : Date.parse(fieldExpiresAt); - } + const fieldExpiresAtMs = expirationTimestamp(fieldExpiresAt); // Only a finite, non-negative epoch counts: negatives collide with the -1 "no expiration" // sentinel (the encoder omits HAS_EXPIRATION for <0, but the field sweep would still evict a // negative field value), so treat a negative/NaN field as unset and use the table default. expiresAt = - Number.isFinite(fieldExpiresAtMs) && fieldExpiresAtMs >= 0 - ? fieldExpiresAtMs - : expirationMs - ? expirationMs + Date.now() - : -1; + fieldExpiresAtMs !== undefined ? fieldExpiresAtMs : expirationMs ? expirationMs + Date.now() : -1; } if (!fullUpdate) { // we use our own data as the basis for the audit record, which will include information about the incremental updates, even if it was overwritten by CRDT resolution @@ -2932,7 +2938,14 @@ export function makeTable(options) { } })() ); - updateIndices(id, existingRecord, recordToStore, transaction && { transaction }); + updateIndices( + id, + existingRecord, + recordToStore, + transaction && { transaction }, + expiresAt, + priorStaged?.expiresAt ?? existingEntry?.expiresAt + ); writeCommit(true); if (expiresAt >= 0) { @@ -2987,7 +3000,7 @@ export function makeTable(options) { // transaction (an audit-only commit stored no record, so it stages nothing and the // earlier staged record, if any, remains the basis) if (storeRecord) { - write.stagedEntry = { value: recordToStore }; + write.stagedEntry = { value: recordToStore, expiresAt }; // blobs this write saved are referenced by its audit entry (if it wrote one), which // then owns their lifetime; and any record an earlier write in this transaction // stored is now replaced, so mark those writes for the superseded-blob cleanup @@ -3114,7 +3127,14 @@ export function makeTable(options) { if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) { return; } - updateIndices(id, existingRecord, null, transaction && { transaction }); + updateIndices( + id, + existingRecord, + null, + transaction && { transaction }, + undefined, + priorStaged?.expiresAt ?? existingEntry?.expiresAt + ); if (audit || trackDeletes) { updateRecord( id, @@ -5295,9 +5315,20 @@ export function makeTable(options) { } static cleanup() { deleteCallbackHandle?.remove(); - recordExpirationCancelled = true; + cleanupClosed = true; + if (cleanupTimer) clearTimeout(cleanupTimer); if (recordExpirationInterval) clearInterval(recordExpirationInterval); - return recordExpirationCompletion; + return Promise.all([lastEvictionCompletion, recordExpirationCompletion]).then(() => undefined); + } + static resumeCleanup() { + if (!cleanupClosed) return; + cleanupClosed = false; + if (cleanupWasScheduled) { + lastCleanupInterval = undefined; + scheduleCleanup(); + } + if (expiresAtProperty) runRecordExpirationEviction(); + if (audit) addDeleteRemoval(); } static _readTxnForContext(context) { return txnForContext(context).getReadTxn(); @@ -5323,7 +5354,14 @@ export function makeTable(options) { if (expirationMs) TableResource.setTTLExpiration(expirationMs / 1000); if (expiresAtProperty) runRecordExpirationEviction(); return TableResource; - function updateIndices(id: any, existingRecord: any, record: any, options?: any) { + function updateIndices( + id: any, + existingRecord: any, + record: any, + options?: any, + expiresAtIndexValue?: number, + existingExpiresAtIndexValue?: number + ) { let hasChanges; // iterate the entries from the record // for-in is about 5x as fast as for-of Object.entries, and this is extremely time sensitive since it can be @@ -5340,8 +5378,26 @@ export function makeTable(options) { // the real [value, id] entry was removed, orphaning it against the now-deleted record. A record // that is present but whose attribute is null is a different case (record is a truthy object) and // still indexes under null. See harper#1894 (F-149). - const value = record == null ? undefined : resolver ? resolver(record) : record[key]; - const existingValue = existingRecord && (resolver ? resolver(existingRecord) : existingRecord[key]); + const expirationIndex = expiresAtIndexName !== undefined && key === expiresAtIndexName; + const value = + record == null + ? undefined + : expirationIndex + ? expiresAtIndexValue !== undefined && expiresAtIndexValue >= 0 + ? expiresAtIndexValue + : undefined + : resolver + ? resolver(record) + : record[key]; + const existingValue = + existingRecord && + (expirationIndex + ? existingExpiresAtIndexValue !== undefined && existingExpiresAtIndexValue >= 0 + ? existingExpiresAtIndexValue + : undefined + : resolver + ? resolver(existingRecord) + : existingRecord[key]); if (value === existingValue && !isIndexing) { continue; } @@ -6022,7 +6078,14 @@ export function makeTable(options) { sourceWrite.skipped = true; return; } - updateIndices(id, existingRecord, updatedRecord, transaction && { transaction }); + updateIndices( + id, + existingRecord, + updatedRecord, + transaction && { transaction }, + sourceContext.expiresAt, + existingEntry?.expiresAt + ); if (updatedRecord) { if (existingEntry) { context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId); @@ -6186,7 +6249,15 @@ export function makeTable(options) { function createEvictionBatcher(isCancelled: () => boolean = () => false) { type EvictItem = | { type: 'evict' | 'tombstone'; key: any; version: number } - | { type: 'dangling-index'; key: any; indexedValue: any; index: any }; + | { type: 'dangling-index'; key: any; indexedValue: any; index: any } + | { + type: 'stale-index'; + key: any; + version: number; + indexedValue: any; + currentExpiration: number | undefined; + index: any; + }; let pending: EvictItem[] = []; const inFlight = new Set>(); @@ -6211,6 +6282,25 @@ export function makeTable(options) { staged++; continue; } + if (item.type === 'stale-index') { + const entry = primaryStore.getEntry(item.key, options); + if ( + !entry || + entry.value == null || + entry.version !== item.version || + compareKeys(entry.expiresAt, item.currentExpiration) !== 0 || + compareKeys(item.indexedValue, item.currentExpiration) === 0 + ) + continue; + const encodedRecord = primaryStore.getBinarySync(item.key, options); + if (encodedRecord === undefined) continue; + primaryStore.putSync(item.key, asBinary(encodedRecord.slice()), options); + item.index.remove(item.indexedValue, item.key, options); + if (item.currentExpiration !== undefined && item.currentExpiration >= 0) + item.index.put(item.currentExpiration, item.key, options); + staged++; + continue; + } const entry = primaryStore.getEntry(item.key, options); if (!entry || entry.version !== item.version) continue; // gone or changed since the scan; leave for next cycle if (item.type === 'tombstone') { @@ -6219,7 +6309,7 @@ export function makeTable(options) { if (entry.value == null) continue; // already removed if (entry.metadataFlags & HAS_BLOBS) continue; // per-record evict() owns blob/commit ordering if (hasSourceGet && primaryStore.hasLock(item.key, entry.version)) continue; // resolution in progress - updateIndices(item.key, entry.value, null, options); + updateIndices(item.key, entry.value, null, options, undefined, entry.expiresAt); } removeEntry(primaryStore, entry, options); staged++; @@ -6296,7 +6386,31 @@ export function makeTable(options) { }, }; } + + function createCleanupOperationTracker() { + const inFlight = new Set>(); + return { + add(operation: MaybePromise | void): Promise | void { + if (operation == null) return; + const tracked = Promise.resolve(operation) + .then( + () => undefined, + (error) => { + logger.error?.('Cleanup error', error); + } + ) + .finally(() => inFlight.delete(tracked)); + inFlight.add(tracked); + if (inFlight.size >= MAX_CLEANUP_CONCURRENCY) return Promise.race(inFlight); + }, + drain(): Promise { + return Promise.all(inFlight); + }, + }; + } + function scheduleCleanup(priority?: number): Promise | void { + if (cleanupClosed) return; let runImmediately = false; if (priority) { // run immediately if there is a big increase in priority @@ -6307,6 +6421,7 @@ export function makeTable(options) { if (cleanupInterval === lastCleanupInterval && !runImmediately) return; lastCleanupInterval = cleanupInterval; if (getWorkerIndex() === getWorkerCount() - 1) { + cleanupWasScheduled = true; // run on the last thread so we aren't overloading lower-numbered threads if (cleanupTimer) clearTimeout(cleanupTimer); if (!cleanupInterval) return; @@ -6323,11 +6438,13 @@ export function makeTable(options) { ? Date.now() : Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime(); const startNextTimer = (nextScheduled) => { + if (cleanupClosed) return; logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`); // noinspection JSVoidFunctionReturnValueUsed cleanupTimer = setTimeout( () => (lastEvictionCompletion = lastEvictionCompletion.then(async () => { + if (cleanupClosed) return; // schedule the next run for when the next cleanup interval should occur (or now if it is in the past) startNextTimer(Math.max(nextScheduled + cleanupInterval, Date.now())); const rootStore = primaryStore.rootStore; @@ -6335,9 +6452,7 @@ export function makeTable(options) { clearTimeout(cleanupTimer); return; } - const MAX_CLEANUP_CONCURRENCY = 50; - const outstandingCleanupOperations = new Array(MAX_CLEANUP_CONCURRENCY); - let cleanupIndex = 0; + const operationTracker = createCleanupOperationTracker(); const evictThreshold = Math.pow(cleanupPriority, 8) * (envMngr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_EVICTIONFACTOR) ?? 100000); @@ -6370,7 +6485,7 @@ export function makeTable(options) { // RocksDB coalesces eviction/tombstone removals into shared transactions to amortize // the per-record commit cost; LMDB keeps the per-record path (eventTurnBatching already // coalesces async writes per event turn). - const batcher = isRocksDB ? createEvictionBatcher() : undefined; + const batcher = isRocksDB ? createEvictionBatcher(() => cleanupClosed) : undefined; // iterate through all entries to find expired records and deleted records for (const entry of primaryStore.getRange({ start: false, @@ -6378,6 +6493,7 @@ export function makeTable(options) { versions: true, lazy: true, // only want to access metadata most of the time })) { + if (cleanupClosed) break; const { key, value: record, version, expiresAt, metadataFlags } = entry; // if there is no auditing cleanup and we are tracking deletion, need to do cleanup of // these deletion entries (LMDB audit cleanup has its own scheduled job for this) @@ -6399,17 +6515,15 @@ export function makeTable(options) { ? removeEntry(primaryStore, entry, version) : TableResource.evict(key, record, version); if (resolution) { - await outstandingCleanupOperations[cleanupIndex]; - outstandingCleanupOperations[cleanupIndex] = resolution.catch((error) => { - logger.error?.('Cleanup error', error); - }); - if (++cleanupIndex >= MAX_CLEANUP_CONCURRENCY) cleanupIndex = 0; + const backpressure = operationTracker.add(resolution); + if (backpressure) await backpressure; } } } await rest(); } if (batcher) await batcher.drain(); + await operationTracker.drain(); logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); } catch (error) { logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); @@ -6430,21 +6544,21 @@ export function makeTable(options) { }); } function runRecordExpirationEviction() { - if (getWorkerIndex() !== 0) return; + if (getWorkerIndex() !== 0 || cleanupClosed) return; const expiresAtName = expiresAtProperty.name; async function sweepRocks(index: any, cutoff: number) { - const batcher = createEvictionBatcher(() => recordExpirationCancelled); + const batcher = createEvictionBatcher(() => cleanupClosed); let after: any[] | undefined; try { - while (!recordExpirationCancelled && primaryStore.rootStore.status === 'open') { + while (!cleanupClosed && primaryStore.rootStore.status === 'open') { const previousAfter = after; const entries = [...index.getCompositeRange({ after, end: cutoff, limit: EVICTION_BATCH_SIZE })]; if (entries.length === 0) break; let completedChunk = true; let entriesSinceYield = 0; for (const entry of entries) { - if (recordExpirationCancelled || primaryStore.rootStore.status !== 'open') { + if (cleanupClosed || primaryStore.rootStore.status !== 'open') { completedChunk = false; break; } @@ -6458,7 +6572,7 @@ export function makeTable(options) { indexedValue: entry.key, index, }); - } else if (recordEntry.value[expiresAtName] < cutoff) { + } else if (recordEntry.expiresAt < cutoff) { if (recordEntry.metadataFlags & HAS_BLOBS) { await TableResource.evict(entry.value, recordEntry.value, recordEntry.version); } else { @@ -6468,6 +6582,15 @@ export function makeTable(options) { version: recordEntry.version, }); } + } else if (compareKeys(entry.key, recordEntry.expiresAt) !== 0) { + backpressure = batcher.add({ + type: 'stale-index', + key: entry.value, + version: recordEntry.version, + indexedValue: entry.key, + currentExpiration: recordEntry.expiresAt, + index, + }); } if (backpressure) await backpressure; if (++entriesSinceYield >= 10) { @@ -6489,22 +6612,34 @@ export function makeTable(options) { } async function sweepLmdb(index: any, cutoff: number) { + const operationTracker = createCleanupOperationTracker(); for (const key of index.getRange({ start: true, values: false, end: cutoff, snapshot: false })) { + if (cleanupClosed) break; for (const id of index.getValues(key)) { const recordEntry = primaryStore.getEntry(id); + let operation: MaybePromise | void; if (recordEntry?.value == null) { - primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); - } else if (recordEntry.value[expiresAtName] < cutoff) { - TableResource.evict(id, recordEntry.value, recordEntry.version); + operation = primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); + } else if (recordEntry.expiresAt < cutoff) { + operation = TableResource.evict(id, recordEntry.value, recordEntry.version); + } else if (compareKeys(key, recordEntry.expiresAt) !== 0) { + operation = primaryStore.ifVersion(id, recordEntry.version, () => { + index.remove(key, id); + if (recordEntry.expiresAt !== undefined && recordEntry.expiresAt >= 0) + index.put(recordEntry.expiresAt, id); + }); } + const backpressure = operationTracker.add(operation); + if (backpressure) await backpressure; } await rest(); } + await operationTracker.drain(); } async function sweep() { if (runningRecordExpiration) return recordExpirationCompletion; - if (recordExpirationCancelled) return; + if (cleanupClosed) return; runningRecordExpiration = true; try { if (primaryStore.rootStore.status !== 'open') return; @@ -6516,6 +6651,7 @@ export function makeTable(options) { } return; } + if (index.isIndexing) return; missingExpirationIndexReported = false; const cutoff = Date.now(); if (isRocksDB) await sweepRocks(index, cutoff); diff --git a/resources/databases.ts b/resources/databases.ts index b336460480..1ae844c7ce 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -9,7 +9,7 @@ import { getBaseSchemaPath, getTransactionAuditStoreBasePath, } from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; -import { makeTable, ignoreAlreadyDropped } from './Table.ts'; +import { expirationTimestamp, makeTable, ignoreAlreadyDropped } from './Table.ts'; import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.ts'; import { CONFIG_PARAMS, @@ -1250,15 +1250,13 @@ export async function dropDatabase(databaseName) { const table = dbTables[tableName]; rootStore = table.primaryStore.rootStore; if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); - lmdbDatabaseEnvs.delete(rootStore.path); - rocksdbDatabaseEnvs.delete(rootStore.path); } const cleanupCompletions: Promise[] = []; for (const tableName in dbTables) { try { const completion = dbTables[tableName].cleanup?.(); - if (completion?.then) { + if (typeof completion?.then === 'function') { cleanupCompletions.push( completion.catch((error) => logger.warn(`Error awaiting cleanup for table ${databaseName}.${tableName}:`, error) @@ -1270,10 +1268,16 @@ export async function dropDatabase(databaseName) { } } if (!(await waitForTableCleanup(cleanupCompletions))) { + for (const tableName in dbTables) dbTables[tableName].resumeCleanup?.(); throw new Error( - `Timed out after ${DATABASE_CLOSE_TIMEOUT}ms waiting for expiration cleanup; refusing to destroy database ${databaseName} while cleanup is active.` + `Timed out after ${DATABASE_CLOSE_TIMEOUT}ms waiting for cleanup; refusing to destroy database ${databaseName} while cleanup is active.` ); } + for (const tableName in dbTables) { + const tableRootStore = dbTables[tableName].primaryStore.rootStore; + lmdbDatabaseEnvs.delete(tableRootStore.path); + rocksdbDatabaseEnvs.delete(tableRootStore.path); + } for (const tableName in dbTables) { databaseEventsEmitter.emit('dropTable', tableName, databaseName); @@ -1354,7 +1358,7 @@ async function closeDatabaseOnce(databaseName: string): Promise { if (!table?.primaryStore) continue; try { const completion = table.cleanup?.(); - if (completion?.then) { + if (typeof completion?.then === 'function') { cleanupCompletions.push( completion.catch((error) => logger.warn(`Error awaiting cleanup for table ${databaseName}.${tableName}:`, error) @@ -1367,8 +1371,9 @@ async function closeDatabaseOnce(databaseName: string): Promise { if (table.primaryStore.rootStore) rootStores.add(table.primaryStore.rootStore); } if (!(await waitForTableCleanup(cleanupCompletions))) { - logger.warn( - `Timed out after ${DATABASE_CLOSE_TIMEOUT}ms waiting for expiration cleanup while closing database ${databaseName}; closing its stores to avoid leaking native handles.` + for (const [, table] of tableEntries) table.resumeCleanup?.(); + throw new Error( + `Timed out after ${DATABASE_CLOSE_TIMEOUT}ms waiting for cleanup; refusing to close database ${databaseName} while cleanup is active.` ); } for (const [tableName, table] of tableEntries) { @@ -1412,7 +1417,8 @@ async function closeDatabaseOnce(databaseName: string): Promise { * handles linger process-wide (and, e.g., block an online `restore_backup` from confirming the * database is closed). The `system` database is intentionally left open: it is non-enumerable here * (skipped by the loop), is never restored online, and the exiting worker may still touch the job - * table during teardown. Best-effort: closing failures are swallowed inside `closeDatabase`. + * table during teardown. A cleanup timeout is logged here and leaves the handles open rather than + * closing them while writes are active. */ export async function closeLoadedDatabases(): Promise { // snapshot the names first: closeDatabase() deletes from `databases` as it goes @@ -1431,7 +1437,13 @@ export async function closeLoadedDatabases(): Promise { if (!isRocks && (definedDatabases?.get(databaseName) as any)?.rootStore instanceof RocksDatabase) { isRocks = true; } - if (isRocks) await closeDatabase(databaseName); + if (isRocks) { + try { + await closeDatabase(databaseName); + } catch (error) { + logger.warn(`Could not close database ${databaseName} during worker shutdown:`, error); + } + } } } // HNSW_NO_AUTOVERSION kill-switch: when set, a NEW index initializes as legacy rather than @@ -1621,7 +1633,10 @@ export function table(tableDefinition: TableDefinition): Tabl attribute.name = attribute.attribute; attribute.indexed = true; } else attribute.attribute = attribute.name; - if (attribute.expiresAt) attribute.indexed = true; + if (attribute.expiresAt) { + attribute.indexed = true; + attribute.expirationIndexVersion = 1; + } } let hasChanges; let releaseExclusiveLock: () => void; @@ -1913,6 +1928,7 @@ export function table(tableDefinition: TableDefinition): Tabl const commonChanged = !attributeDescriptor || attributeDescriptor.type !== attribute.type || + attributeDescriptor.expirationIndexVersion !== attribute.expirationIndexVersion || attributeDescriptor.nullable !== attribute.nullable || attributeDescriptor.version !== attribute.version || attributeDescriptor.enumerable !== attribute.enumerable || @@ -2168,7 +2184,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { } let outstanding = 0; // this means that a new attribute has been introduced that needs to be indexed - for (const { key, value: record } of Table.primaryStore.getRange({ + for (const { key, value: record, expiresAt } of Table.primaryStore.getRange({ start, lazy: attributesLength < 4, versions: true, @@ -2192,7 +2208,9 @@ async function runIndexing(Table, attributes, indicesToRemove) { const index = attribute.dbi; try { const resolver = attribute.resolve; - const value = record && (resolver ? resolver(record) : record[property]); + const value = attribute.expiresAt + ? expirationTimestamp(expiresAt) + : record && (resolver ? resolver(record) : record[property]); if (index.customIndex) { index.customIndex.index(key, value); didSynchronousIndexing = true; diff --git a/resources/search.ts b/resources/search.ts index c1f7dcafdf..c8957338b6 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -2,7 +2,7 @@ import { ClientError, IndexRebuildingError, Violation } from '../utility/errors/ import { OVERFLOW_MARKER, MAX_SEARCH_KEY_LENGTH, SEARCH_TYPES } from '../utility/lmdb/terms.ts'; import { compareKeys, MAXIMUM_KEY, writeKey } from 'ordered-binary'; import { SKIP } from '@harperfast/extended-iterable'; -import { INVALIDATED, EVICTED, freezeRecord } from './Table.ts'; +import { expirationTimestamp, INVALIDATED, EVICTED, freezeRecord } from './Table.ts'; import type { DirectCondition, Id } from './ResourceInterface.ts'; import { RequestTarget } from './RequestTarget.ts'; import { lastMetadata } from './RecordEncoder.ts'; @@ -14,6 +14,11 @@ const OPEN_RANGE_ESTIMATE = 0.3; const BETWEEN_ESTIMATE = 0.1; const STARTS_WITH_ESTIMATE = 0.05; +function normalizeExpirationSearchValue(value: any): any { + if (Array.isArray(value)) return value.map(normalizeExpirationSearchValue); + return expirationTimestamp(value) ?? value; +} + function getStringPrefixUpperBound(prefix: string): Uint8Array { const maximumEncodedLength = prefix.length * 3 + 3; const encodedPrefix = new Uint8Array((maximumEncodedLength + 7) & ~3); @@ -327,6 +332,9 @@ export function searchByIndex( } const isPrimaryKey = attribute_name === Table.primaryKey || attribute_name == null; const index = isPrimaryKey ? Table.primaryStore : Table.indices[attribute_name]; + if (!isPrimaryKey && findAttribute(Table.attributes, attribute_name)?.expiresAt) { + value = normalizeExpirationSearchValue(value); + } let start; let end, inclusiveEnd, exclusiveStart, stringPrefix; if (value instanceof Date) value = value.getTime(); @@ -947,6 +955,8 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar return recordFilter; } } + const normalizeExpirationValue = !isPrimaryKey && findAttribute(Table?.attributes, attribute)?.expiresAt; + if (normalizeExpirationValue) value = normalizeExpirationSearchValue(value); if (value instanceof Date) value = value.getTime(); let baseFilter; @@ -1051,6 +1061,7 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar canUseIndex?: boolean, allowObjectMatching?: boolean ) { + const normalizeRecordExpiration = !isPrimaryKey && findAttribute(Table?.attributes, attribute)?.expiresAt; let thresholdRemainingMisses: number; canUseIndex = canUseIndex && // is it a comparator that makes sense to use index @@ -1069,7 +1080,8 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar function recordFilter(record: any) { // `record` may be null/undefined when called via a nested-path filter // where an intermediate property is missing. - const value = record == null ? undefined : record[attribute]; + let value = record == null ? undefined : record[attribute]; + if (normalizeRecordExpiration) value = normalizeExpirationSearchValue(value); let matches: boolean; if (typeof value !== 'object' || !value || allowObjectMatching) matches = filter(value); else if (Array.isArray(value)) matches = value.some(filter); @@ -1168,7 +1180,9 @@ export function estimateCondition(table) { } else { // we only attempt to estimate count on equals operator because that's really all that LMDB supports (some other key-value stores like libmdbx could be considered if we need to do estimated counts of ranges at some point) const index = table.indices[attribute_name]; - condition.estimated_count = index ? index.getValuesCount(condition[1] ?? condition.value) : Infinity; + let value = condition[1] ?? condition.value; + if (findAttribute(table.attributes, attribute_name)?.expiresAt) value = normalizeExpirationSearchValue(value); + condition.estimated_count = index ? index.getValuesCount(value) : Infinity; } } else if (searchType === 'contains' || searchType === 'ends_with' || searchType === 'ne') { const attribute_name = condition[0] ?? condition.attribute; @@ -1183,8 +1197,9 @@ export function estimateCondition(table) { if (Array.isArray(condition.value) && index) { // Sum of per-value matches (over-counts duplicates but is a fine ceiling) let estimate = 0; + const normalizeExpiration = findAttribute(table.attributes, attribute_name)?.expiresAt; for (const item of condition.value) { - estimate += index.getValuesCount(item); + estimate += index.getValuesCount(normalizeExpiration ? normalizeExpirationSearchValue(item) : item); } condition.estimated_count = estimate; } else if (Array.isArray(condition.value)) { diff --git a/unitTests/resources/closeLoadedDatabases.test.js b/unitTests/resources/closeLoadedDatabases.test.js index 8ab213f2b5..9d049092d2 100644 --- a/unitTests/resources/closeLoadedDatabases.test.js +++ b/unitTests/resources/closeLoadedDatabases.test.js @@ -107,6 +107,36 @@ describe('RocksDB handle release', function () { assert.strictEqual(refCountFor(rootStore.path), 0); }); + it('fails closed and resumes cleanup when close times out', async function () { + const Table = table({ + table: 'expiring', + database: 'closetimeout6', + attributes: [ + { attribute: 'id', isPrimaryKey: true }, + { attribute: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const cleanup = sinon.stub(Table, 'cleanup').returns(new Promise(() => {})); + const resume = sinon.spy(Table, 'resumeCleanup'); + const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + const closing = closeDatabase('closetimeout6'); + const rejected = assert.rejects(closing, /refusing to close database/); + await clock.tickAsync(10_001); + await rejected; + assert.ok(refCountFor(rootStore.path) > 0, 'timed-out close must retain native handles'); + assert.strictEqual(rootStore.status, 'open'); + assert.strictEqual(resume.callCount, 1); + } finally { + clock.restore(); + cleanup.restore(); + resume.restore(); + } + await closeDatabase('closetimeout6'); + }); + it('dropDatabase runs table cleanup before destroying its stores', async function () { const Table = table({ table: 'expiring', @@ -123,4 +153,34 @@ describe('RocksDB handle release', function () { assert.strictEqual(cleanup.callCount, 1, 'table cleanup should run before its database is destroyed'); cleanup.restore(); }); + + it('fails closed and resumes cleanup when drop times out', async function () { + const Table = table({ + table: 'expiring', + database: 'droptimeout7', + attributes: [ + { attribute: 'id', isPrimaryKey: true }, + { attribute: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const cleanup = sinon.stub(Table, 'cleanup').returns(new Promise(() => {})); + const resume = sinon.spy(Table, 'resumeCleanup'); + const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + const dropping = dropDatabase('droptimeout7'); + const rejected = assert.rejects(dropping, /refusing to destroy database/); + await clock.tickAsync(10_001); + await rejected; + assert.ok(refCountFor(rootStore.path) > 0, 'timed-out drop must retain native handles'); + assert.strictEqual(rootStore.status, 'open'); + assert.strictEqual(resume.callCount, 1); + } finally { + clock.restore(); + cleanup.restore(); + resume.restore(); + } + await dropDatabase('droptimeout7'); + }); }); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index b464085b36..eb77c66d29 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -124,6 +124,50 @@ describe('@expiresAt attribute is authoritative over the table default', () => { assert.strictEqual(await storedExpiresAt(Table, 1), fieldExpiresAt); }); + it('canonicalizes supported expiration shapes in the index and in searches', async function () { + const Table = makeTable('ExpiresAtCanonicalIndex'); + const expiresAt = Date.now() + 3_600_000; + const values = [expiresAt, String(expiresAt), new Date(expiresAt).toISOString(), new Date(expiresAt)]; + for (let id = 0; id < values.length; id++) await Table.put(id, { id, expiresAt: values[id] }); + await Table.primaryStore.committed; + + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [0, 1, 2, 3]); + for (const value of values) { + const ids = []; + for await (const record of Table.search({ + allowFullScan: false, + conditions: [{ attribute: 'expiresAt', value }], + })) { + ids.push(record.id); + } + assert.deepStrictEqual(ids, [0, 1, 2, 3]); + } + Table.cleanup(); + }); + + it('rebuilds a legacy expiration index into the canonical format', async function () { + const tableName = 'ExpiresAtCanonicalRebuild'; + const Table = makeTable(tableName); + const expiresAt = Date.now() + 3_600_000; + const isoExpiresAt = new Date(expiresAt).toISOString(); + await Table.put(1, { id: 1, expiresAt: isoExpiresAt }); + await Table.primaryStore.committed; + + const attribute = Table.attributes.find((candidate) => candidate.name === 'expiresAt'); + const descriptor = Table.dbisDB.getSync(attribute.key); + delete descriptor.expirationIndexVersion; + await Table.dbisDB.put(attribute.key, descriptor); + await Table.indices.expiresAt.clear(); + await Table.indices.expiresAt.put(isoExpiresAt, 1); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(isoExpiresAt)], [1]); + + const Reloaded = makeTable(tableName); + await Reloaded.indexingOperation; + assert.deepStrictEqual([...Reloaded.indices.expiresAt.getValues(isoExpiresAt)], []); + assert.deepStrictEqual([...Reloaded.indices.expiresAt.getValues(expiresAt)], [1]); + Table.cleanup(); + }); + it('ignores non-timestamp field values (boolean / empty string) and uses the table default', async function () { const Bool = makeTable('ExpiresAtBool', 100); const Empty = makeTable('ExpiresAtEmpty', 100); @@ -136,6 +180,19 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } }); + it('does not index or evict a field-only record without a valid expiration', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtNoExpiration')); + await Table.put(1, { id: 1 }); + await Table.put(2, { id: 2, expiresAt: -1 }); + await Table.primaryStore.committed; + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(-1)], []); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.id, 1); + assert.strictEqual(Table.primaryStore.getEntry(2)?.value.id, 2); + Table.cleanup(); + }); + // End-to-end: stamping the field into the expiry metadata makes read-hiding enforce it on a // field-only table with no table default — the record is no longer served past its field time. // This also covers the RocksDB correctness half of #1481 (the field sweep is LMDB-only, so @@ -268,6 +325,34 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); + it('physically evicts an expired ISO-string value', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtIsoSweep')); + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt: new Date(expiresAt).toISOString() }); + await Table.primaryStore.committed; + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + Table.cleanup(); + }); + + it('removes a stale expired key while preserving a refreshed record and its current key', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtStaleKey')); + const expired = Date.now() - 1_000; + const current = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt: current }); + await Table.primaryStore.committed; + Table.indices.expiresAt.put(expired, 1); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.expiresAt, current); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expired)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(current)], [1]); + Table.cleanup(); + }); + it('keeps a blob file when an expired-record eviction conflicts', async function () { const { Table, runSweep } = captureExpirationSweep(() => table({ @@ -466,6 +551,57 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } }); + it('cleanup drains the primary cleanup scan and prevents writes from rearming it', async function () { + const Table = table({ + table: 'PrimaryCleanupDrain', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'payload', type: 'Blob' }, + ], + }); + let runCleanup; + const realSetTimeout = global.setTimeout; + const timeoutStub = sinon.stub(global, 'setTimeout').callsFake((callback, timeout, ...args) => { + if (!runCleanup) { + runCleanup = callback; + return { unref() {} }; + } + return realSetTimeout(callback, timeout, ...args); + }); + Table.setTTLExpiration({ scanInterval: 100 }); + assert(runCleanup); + await Table.put(1, { id: 1, payload: createBlob(Buffer.alloc(20_000, 11)) }, { expiresAt: Date.now() - 1_000 }); + await Table.primaryStore.committed; + + const originalEvict = Table.evict; + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { + evictionStarted = true; + await blockedEviction; + return originalEvict.apply(this, args); + }); + try { + const scan = runCleanup(); + await waitFor(() => evictionStarted, { message: 'the primary cleanup scan should start eviction' }); + let cleanupResolved = false; + const cleanup = Table.cleanup().then(() => (cleanupResolved = true)); + await delay(10); + assert.strictEqual(cleanupResolved, false); + releaseEviction(); + await Promise.all([scan, cleanup]); + const timerCount = timeoutStub.callCount; + await Table.put(2, { id: 2 }, { expiresAt: Date.now() + 60_000 }); + assert.strictEqual(timeoutStub.callCount, timerCount, 'cleanup must prevent a write from rearming the scan'); + } finally { + releaseEviction(); + evictStub.restore(); + timeoutStub.restore(); + } + }); + it('waits for an active expiration sweep before destroying a database', async function () { const database = 'ExpiresAtDatabaseDrop'; const { Table, runSweep } = captureExpirationSweep(() => @@ -589,6 +725,39 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); + it('does not remove a refreshed expiration key when stale-key cleanup conflicts', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtStaleConflict')); + const expired = Date.now() - 1_000; + const current = Date.now() + 60_000; + const refreshed = current + 60_000; + await Table.put(1, { id: 1, expiresAt: current }); + await Table.primaryStore.committed; + Table.indices.expiresAt.put(expired, 1); + + const originalCommit = RocksTransaction.prototype.commit; + let injected = false; + const commitStub = sinon.stub(RocksTransaction.prototype, 'commit').callsFake(async function (...args) { + if (!injected) { + injected = true; + await Table.put(1, { id: 1, expiresAt: refreshed }); + } + return originalCommit.apply(this, args); + }); + try { + await runSweep(); + } finally { + commitStub.restore(); + } + + assert(injected); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.expiresAt, refreshed); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(refreshed)], [1]); + await runSweep(); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expired)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(refreshed)], [1]); + Table.cleanup(); + }); + it('conflicts retained-tombstone cleanup with a concurrent primary resurrection', async function () { const Table = makeTable('ExpiresAtTombstoneConflict', undefined, { audit: true }); const expiresAt = Date.now() + 60_000; @@ -612,3 +781,62 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); }); + +describe('LMDB @expiresAt cleanup draining', function () { + if (process.env.HARPER_STORAGE_ENGINE !== 'lmdb') return; + + before(function () { + setupTestDBPath(); + setMainIsWorker(true); + }); + + it('does not resolve cleanup while LMDB eviction writes are active', async function () { + let runSweep; + const realSetInterval = global.setInterval; + const intervalStub = sinon.stub(global, 'setInterval').callsFake((callback, interval, ...args) => { + if (interval === 60_000) { + runSweep = callback; + return { unref() {} }; + } + return realSetInterval(callback, interval, ...args); + }); + let Table; + try { + Table = table({ + table: 'LmdbExpirationDrain', + database: 'lmdb-expiration-drain', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + } finally { + intervalStub.restore(); + } + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); + await Table.primaryStore.committed; + + const originalEvict = Table.evict; + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { + evictionStarted = true; + await blockedEviction; + return originalEvict.apply(this, args); + }); + try { + const sweep = runSweep(); + await waitFor(() => evictionStarted, { message: 'LMDB eviction should start' }); + let cleanupResolved = false; + const cleanup = Table.cleanup().then(() => (cleanupResolved = true)); + await delay(10); + assert.strictEqual(cleanupResolved, false); + releaseEviction(); + await Promise.all([sweep, cleanup]); + } finally { + releaseEviction(); + evictStub.restore(); + } + }); +}); From 533ffe10a264d51ccd2316ab1740e4b56fc375a7 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 09:13:55 -0600 Subject: [PATCH 14/43] fix(storage): preserve expiration across upgrades --- resources/DESIGN.md | 3 + resources/Table.ts | 116 ++++++++++++------ resources/databases.ts | 4 +- resources/search.ts | 21 ++-- server/itc/serverHandlers.js | 13 +- .../resources/expiresAtAttribute.test.js | 75 +++++++++-- unitTests/server/itc/serverHandlers.test.js | 42 +++++++ 7 files changed, 218 insertions(+), 56 deletions(-) diff --git a/resources/DESIGN.md b/resources/DESIGN.md index 17fe18f80b..53ac81b566 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -112,6 +112,9 @@ Its index stores one canonical, non-negative epoch-millisecond number derived fr whether the API value was a number, numeric string, ISO string, or `Date`. Writes, index rebuilds, index searches, and scan filters apply the same normalization. `expirationIndexVersion` participates in the normal resumable schema reindex flow so stores created before canonicalization are rebuilt without a separate open-time scan. +Rows created before expiration metadata was stored fall back to their public field during that rebuild and while +sweeping; an explicit no-expiration metadata sentinel never falls back. This preserves upgrade behavior without +making every open scan the table. RocksDB `@expiresAt` sweeps walk one bounded composite-index range at a time with a fixed cutoff and an owned `(expiresAt, primaryKey)` cursor. A sweep never holds an iterator snapshot across an `await`, and it diff --git a/resources/Table.ts b/resources/Table.ts index 54f9a4e4ef..d3c92bd773 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -594,6 +594,7 @@ export function makeTable(options) { static databasePath = databasePath; static databaseName = databaseName; static attributes = attributes; + static expiresAtAttributeName = expiresAtIndexName; static description = description; static properties = properties; static hidden = hidden; @@ -2068,7 +2069,7 @@ export function makeTable(options) { /** * Evicting a record will remove it from a caching table. This is not considered a canonical data change, and it is assumed that retrieving this record from the source will still yield the same record, this is only removing the local copy of the record. */ - static evict(id, existingRecord, existingVersion) { + static evict(id, existingRecord, existingVersion, existingExpiresAtIndexValue?: number) { let entry; let deleteBlobsAfterCommit: (() => void) | undefined; let currentEntry; @@ -2101,15 +2102,30 @@ export function makeTable(options) { // lmdb: the index cleanup and the record removal are both version-guarded optimistic writes. // Capture both promises so a real write failure on either resolves through evict()'s catch // below rather than escaping as an unhandled rejection from the fire-and-forget callers. + const removalEntry = entry ?? primaryStore.getEntry(id); const indexCleanup = primaryStore.ifVersion(id, existingVersion, () => { - updateIndices(id, existingRecord, null, undefined, undefined, entry?.expiresAt); + updateIndices( + id, + existingRecord, + null, + undefined, + undefined, + existingExpiresAtIndexValue ?? removalEntry?.expiresAt + ); }); - const removal = removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion); + const removal = removeEntry(primaryStore, removalEntry, existingVersion); lmdbCompletion = Promise.all([indexCleanup, removal]); } else { const removalEntry = entry ?? currentEntry ?? primaryStore.getEntry(id); if (!removalEntry || removalEntry.version !== existingVersion) return; - updateIndices(id, existingRecord, null, options, undefined, removalEntry.expiresAt); + updateIndices( + id, + existingRecord, + null, + options, + undefined, + existingExpiresAtIndexValue ?? removalEntry.expiresAt + ); if (removalEntry.metadataFlags & HAS_BLOBS) { if (!existingRecord) return; primaryStore.remove(removalEntry.key, options); @@ -6248,13 +6264,15 @@ export function makeTable(options) { // those records for the next cleanup cycle. function createEvictionBatcher(isCancelled: () => boolean = () => false) { type EvictItem = - | { type: 'evict' | 'tombstone'; key: any; version: number } + | { type: 'evict'; key: any; version: number; indexedExpiration?: number } + | { type: 'tombstone'; key: any; version: number } | { type: 'dangling-index'; key: any; indexedValue: any; index: any } | { type: 'stale-index'; key: any; version: number; indexedValue: any; + metadataExpiration: number | undefined; currentExpiration: number | undefined; index: any; }; @@ -6288,7 +6306,7 @@ export function makeTable(options) { !entry || entry.value == null || entry.version !== item.version || - compareKeys(entry.expiresAt, item.currentExpiration) !== 0 || + compareKeys(entry.expiresAt, item.metadataExpiration) !== 0 || compareKeys(item.indexedValue, item.currentExpiration) === 0 ) continue; @@ -6309,7 +6327,7 @@ export function makeTable(options) { if (entry.value == null) continue; // already removed if (entry.metadataFlags & HAS_BLOBS) continue; // per-record evict() owns blob/commit ordering if (hasSourceGet && primaryStore.hasLock(item.key, entry.version)) continue; // resolution in progress - updateIndices(item.key, entry.value, null, options, undefined, entry.expiresAt); + updateIndices(item.key, entry.value, null, options, undefined, item.indexedExpiration ?? entry.expiresAt); } removeEntry(primaryStore, entry, options); staged++; @@ -6546,6 +6564,10 @@ export function makeTable(options) { function runRecordExpirationEviction() { if (getWorkerIndex() !== 0 || cleanupClosed) return; const expiresAtName = expiresAtProperty.name; + const indexedExpiration = (entry: Entry): number | undefined => + entry.expiresAt === undefined + ? expirationTimestamp(entry.value?.[expiresAtName]) + : expirationTimestamp(entry.expiresAt); async function sweepRocks(index: any, cutoff: number) { const batcher = createEvictionBatcher(() => cleanupClosed); @@ -6572,25 +6594,30 @@ export function makeTable(options) { indexedValue: entry.key, index, }); - } else if (recordEntry.expiresAt < cutoff) { - if (recordEntry.metadataFlags & HAS_BLOBS) { - await TableResource.evict(entry.value, recordEntry.value, recordEntry.version); - } else { + } else { + const currentExpiration = indexedExpiration(recordEntry); + if (currentExpiration !== undefined && currentExpiration < cutoff) { + if (recordEntry.metadataFlags & HAS_BLOBS) { + await TableResource.evict(entry.value, recordEntry.value, recordEntry.version, currentExpiration); + } else { + backpressure = batcher.add({ + type: 'evict', + key: entry.value, + version: recordEntry.version, + indexedExpiration: currentExpiration, + }); + } + } else if (compareKeys(entry.key, currentExpiration) !== 0) { backpressure = batcher.add({ - type: 'evict', + type: 'stale-index', key: entry.value, version: recordEntry.version, + indexedValue: entry.key, + metadataExpiration: recordEntry.expiresAt, + currentExpiration, + index, }); } - } else if (compareKeys(entry.key, recordEntry.expiresAt) !== 0) { - backpressure = batcher.add({ - type: 'stale-index', - key: entry.value, - version: recordEntry.version, - indexedValue: entry.key, - currentExpiration: recordEntry.expiresAt, - index, - }); } if (backpressure) await backpressure; if (++entriesSinceYield >= 10) { @@ -6615,22 +6642,39 @@ export function makeTable(options) { const operationTracker = createCleanupOperationTracker(); for (const key of index.getRange({ start: true, values: false, end: cutoff, snapshot: false })) { if (cleanupClosed) break; - for (const id of index.getValues(key)) { - const recordEntry = primaryStore.getEntry(id); - let operation: MaybePromise | void; - if (recordEntry?.value == null) { - operation = primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); - } else if (recordEntry.expiresAt < cutoff) { - operation = TableResource.evict(id, recordEntry.value, recordEntry.version); - } else if (compareKeys(key, recordEntry.expiresAt) !== 0) { - operation = primaryStore.ifVersion(id, recordEntry.version, () => { - index.remove(key, id); - if (recordEntry.expiresAt !== undefined && recordEntry.expiresAt >= 0) - index.put(recordEntry.expiresAt, id); - }); + let afterId: any; + for (;;) { + const ids = [ + ...index.getValues(key, { + start: afterId, + exclusiveStart: afterId !== undefined, + limit: EVICTION_BATCH_SIZE, + }), + ]; + if (ids.length === 0) break; + afterId = ids.at(-1); + for (const id of ids) { + const recordEntry = primaryStore.getEntry(id); + let operation: MaybePromise | void; + if (recordEntry?.value == null) { + operation = primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); + } else { + const currentExpiration = indexedExpiration(recordEntry); + if (currentExpiration !== undefined && currentExpiration < cutoff) { + operation = TableResource.evict(id, recordEntry.value, recordEntry.version, currentExpiration); + } else if (compareKeys(key, currentExpiration) !== 0) { + operation = primaryStore.ifVersion(id, recordEntry.version, () => { + const removals = [index.remove(key, id)]; + if (currentExpiration !== undefined) removals.push(index.put(currentExpiration, id)); + return Promise.all(removals); + }); + } + } + const backpressure = operationTracker.add(operation); + if (backpressure) await backpressure; } - const backpressure = operationTracker.add(operation); - if (backpressure) await backpressure; + await operationTracker.drain(); + if (ids.length < EVICTION_BATCH_SIZE) break; } await rest(); } diff --git a/resources/databases.ts b/resources/databases.ts index 1ae844c7ce..3f7f41a535 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -2209,7 +2209,9 @@ async function runIndexing(Table, attributes, indicesToRemove) { try { const resolver = attribute.resolve; const value = attribute.expiresAt - ? expirationTimestamp(expiresAt) + ? expiresAt === undefined + ? expirationTimestamp(record?.[property]) + : expirationTimestamp(expiresAt) : record && (resolver ? resolver(record) : record[property]); if (index.customIndex) { index.customIndex.index(key, value); diff --git a/resources/search.ts b/resources/search.ts index c8957338b6..9f90dde568 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -5,7 +5,7 @@ import { SKIP } from '@harperfast/extended-iterable'; import { expirationTimestamp, INVALIDATED, EVICTED, freezeRecord } from './Table.ts'; import type { DirectCondition, Id } from './ResourceInterface.ts'; import { RequestTarget } from './RequestTarget.ts'; -import { lastMetadata } from './RecordEncoder.ts'; +import { lastMetadata, type Entry } from './RecordEncoder.ts'; import { recordAction } from './analytics/write'; import { RocksDatabase } from '@harperfast/rocksdb-js'; @@ -332,7 +332,7 @@ export function searchByIndex( } const isPrimaryKey = attribute_name === Table.primaryKey || attribute_name == null; const index = isPrimaryKey ? Table.primaryStore : Table.indices[attribute_name]; - if (!isPrimaryKey && findAttribute(Table.attributes, attribute_name)?.expiresAt) { + if (!isPrimaryKey && Table.expiresAtAttributeName === attribute_name) { value = normalizeExpirationSearchValue(value); } let start; @@ -955,7 +955,7 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar return recordFilter; } } - const normalizeExpirationValue = !isPrimaryKey && findAttribute(Table?.attributes, attribute)?.expiresAt; + const normalizeExpirationValue = !isPrimaryKey && Table?.expiresAtAttributeName === attribute; if (normalizeExpirationValue) value = normalizeExpirationSearchValue(value); if (value instanceof Date) value = value.getTime(); @@ -1061,7 +1061,7 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar canUseIndex?: boolean, allowObjectMatching?: boolean ) { - const normalizeRecordExpiration = !isPrimaryKey && findAttribute(Table?.attributes, attribute)?.expiresAt; + const normalizeRecordExpiration = !isPrimaryKey && Table?.expiresAtAttributeName === attribute; let thresholdRemainingMisses: number; canUseIndex = canUseIndex && // is it a comparator that makes sense to use index @@ -1077,10 +1077,15 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar } let misses = 0; let filteredSoFar = 3; // what we use to calculate miss rate; we give some buffer so we don't jump to indexed retrieval too quickly - function recordFilter(record: any) { + function recordFilter(record: any, entry?: Entry) { // `record` may be null/undefined when called via a nested-path filter // where an intermediate property is missing. - let value = record == null ? undefined : record[attribute]; + let value = + normalizeRecordExpiration && entry?.expiresAt !== undefined + ? entry.expiresAt + : record == null + ? undefined + : record[attribute]; if (normalizeRecordExpiration) value = normalizeExpirationSearchValue(value); let matches: boolean; if (typeof value !== 'object' || !value || allowObjectMatching) matches = filter(value); @@ -1181,7 +1186,7 @@ export function estimateCondition(table) { // we only attempt to estimate count on equals operator because that's really all that LMDB supports (some other key-value stores like libmdbx could be considered if we need to do estimated counts of ranges at some point) const index = table.indices[attribute_name]; let value = condition[1] ?? condition.value; - if (findAttribute(table.attributes, attribute_name)?.expiresAt) value = normalizeExpirationSearchValue(value); + if (table.expiresAtAttributeName === attribute_name) value = normalizeExpirationSearchValue(value); condition.estimated_count = index ? index.getValuesCount(value) : Infinity; } } else if (searchType === 'contains' || searchType === 'ends_with' || searchType === 'ne') { @@ -1197,7 +1202,7 @@ export function estimateCondition(table) { if (Array.isArray(condition.value) && index) { // Sum of per-value matches (over-counts duplicates but is a fine ceiling) let estimate = 0; - const normalizeExpiration = findAttribute(table.attributes, attribute_name)?.expiresAt; + const normalizeExpiration = table.expiresAtAttributeName === attribute_name; for (const item of condition.value) { estimate += index.getValuesCount(normalizeExpiration ? normalizeExpirationSearchValue(item) : item); } diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.js index a6802e6b51..5b2f81d33c 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.js @@ -50,7 +50,18 @@ async function schemaHandler(event) { // rewrite the database directory. The rescan below (resetDatabases) skips reloading it while // the restoring marker is present, and reloads it on the completion signal (marker gone). if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP && event.message.schema) { - await closeDatabase(event.message.schema); + for (let attempt = 0; attempt < 2; attempt++) { + try { + await closeDatabase(event.message.schema); + break; + } catch (error) { + hdbLogger.warn( + `Could not close database ${event.message.schema} for restore${attempt === 0 ? '; retrying once' : ''}:`, + error + ); + if (attempt === 1) return; + } + } } await cleanLmdbMap(event.message); await syncSchemaMetadata(event.message); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index eb77c66d29..4ce514e611 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -168,6 +168,42 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); + it('preserves expiration for legacy rows that have a field but no stored metadata', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtLegacyMetadata')); + const expiresAt = Date.now() - 1_000; + const record = { id: 1, expiresAt: new Date(expiresAt).toISOString() }; + Table.primaryStore.putSync(1, record); + Table.indices.expiresAt.put(expiresAt, 1); + assert.strictEqual(Table.primaryStore.getEntry(1).expiresAt, undefined); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + Table.cleanup(); + }); + + it('re-arms the expiration sweep after cleanup is resumed', function () { + let sweepIntervalCount = 0; + const realSetInterval = global.setInterval; + const intervalStub = sinon.stub(global, 'setInterval').callsFake((callback, interval, ...args) => { + if (interval === 60_000) { + sweepIntervalCount++; + return { unref() {} }; + } + return realSetInterval(callback, interval, ...args); + }); + try { + const Table = makeTable('ExpiresAtResumeCleanup'); + assert.strictEqual(sweepIntervalCount, 1); + Table.cleanup(); + Table.resumeCleanup(); + assert.strictEqual(sweepIntervalCount, 2); + Table.cleanup(); + } finally { + intervalStub.restore(); + } + }); + it('ignores non-timestamp field values (boolean / empty string) and uses the table default', async function () { const Bool = makeTable('ExpiresAtBool', 100); const Empty = makeTable('ExpiresAtEmpty', 100); @@ -551,7 +587,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } }); - it('cleanup drains the primary cleanup scan and prevents writes from rearming it', async function () { + it('cleanup drains the primary cleanup scan', async function () { const Table = table({ table: 'PrimaryCleanupDrain', database: 'test', @@ -561,15 +597,15 @@ describe('@expiresAt attribute is authoritative over the table default', () => { ], }); let runCleanup; - const realSetTimeout = global.setTimeout; - const timeoutStub = sinon.stub(global, 'setTimeout').callsFake((callback, timeout, ...args) => { + const setupTimerStub = sinon.stub(global, 'setTimeout').callsFake((callback) => { if (!runCleanup) { runCleanup = callback; return { unref() {} }; } - return realSetTimeout(callback, timeout, ...args); + throw new Error('setTTLExpiration scheduled more than one cleanup timer'); }); Table.setTTLExpiration({ scanInterval: 100 }); + setupTimerStub.restore(); assert(runCleanup); await Table.put(1, { id: 1, payload: createBlob(Buffer.alloc(20_000, 11)) }, { expiresAt: Date.now() - 1_000 }); await Table.primaryStore.committed; @@ -592,13 +628,10 @@ describe('@expiresAt attribute is authoritative over the table default', () => { assert.strictEqual(cleanupResolved, false); releaseEviction(); await Promise.all([scan, cleanup]); - const timerCount = timeoutStub.callCount; - await Table.put(2, { id: 2 }, { expiresAt: Date.now() + 60_000 }); - assert.strictEqual(timeoutStub.callCount, timerCount, 'cleanup must prevent a write from rearming the scan'); } finally { releaseEviction(); evictStub.restore(); - timeoutStub.restore(); + setupTimerStub.restore(); } }); @@ -790,7 +823,7 @@ describe('LMDB @expiresAt cleanup draining', function () { setMainIsWorker(true); }); - it('does not resolve cleanup while LMDB eviction writes are active', async function () { + const captureExpirationSweep = (name) => { let runSweep; const realSetInterval = global.setInterval; const intervalStub = sinon.stub(global, 'setInterval').callsFake((callback, interval, ...args) => { @@ -803,7 +836,7 @@ describe('LMDB @expiresAt cleanup draining', function () { let Table; try { Table = table({ - table: 'LmdbExpirationDrain', + table: name, database: 'lmdb-expiration-drain', attributes: [ { name: 'id', isPrimaryKey: true }, @@ -813,6 +846,28 @@ describe('LMDB @expiresAt cleanup draining', function () { } finally { intervalStub.restore(); } + return { Table, runSweep }; + }; + + it('evicts ISO expirations and repairs stale keys', async function () { + const { Table, runSweep } = captureExpirationSweep('LmdbExpirationCorrectness'); + const expired = Date.now() - 1_000; + const current = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt: new Date(expired).toISOString() }); + await Table.put(2, { id: 2, expiresAt: current }); + await Table.primaryStore.committed; + await Table.indices.expiresAt.put(expired, 2); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expired)], []); + assert.strictEqual(Table.primaryStore.getEntry(2)?.value.expiresAt, current); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(current)], [2]); + Table.cleanup(); + }); + + it('does not resolve cleanup while LMDB eviction writes are active', async function () { + const { Table, runSweep } = captureExpirationSweep('LmdbExpirationDrain'); await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); await Table.primaryStore.committed; diff --git a/unitTests/server/itc/serverHandlers.test.js b/unitTests/server/itc/serverHandlers.test.js index 4b0c693394..7e45cf6577 100644 --- a/unitTests/server/itc/serverHandlers.test.js +++ b/unitTests/server/itc/serverHandlers.test.js @@ -133,6 +133,48 @@ describe('Test hdbChildIpcHandler module', () => { await schema_handler(test_event); expect(log_error_stub).to.have.been.called; }); + + it('retries a restore close failure and still acknowledges through the schema work', async () => { + const closeStub = sandbox.stub(); + closeStub.onFirstCall().rejects(new Error('cleanup still active')); + closeStub.onSecondCall().resolves(true); + const cleanStub = sandbox.stub().resolves(); + const syncStub = sandbox.stub().resolves(); + const restoreClose = server_itc_handlers.__set__('closeDatabase', closeStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + const restoreSync = server_itc_handlers.__set__('syncSchemaMetadata', syncStub); + try { + await schema_handler({ + type: 'schema', + message: { originator: 12345, operation: 'restore_backup', schema: 'test' }, + }); + expect(closeStub).to.have.been.calledTwice; + expect(cleanStub).to.have.been.calledOnce; + expect(syncStub).to.have.been.calledOnce; + } finally { + restoreSync(); + restoreClean(); + restoreClose(); + } + }); + + it('resolves a restore close failure without rescanning an open database', async () => { + const closeStub = sandbox.stub().rejects(new Error('cleanup still active')); + const cleanStub = sandbox.stub().resolves(); + const restoreClose = server_itc_handlers.__set__('closeDatabase', closeStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + try { + await schema_handler({ + type: 'schema', + message: { originator: 12345, operation: 'restore_backup', schema: 'test' }, + }); + expect(closeStub).to.have.been.calledTwice; + expect(cleanStub).not.to.have.been.called; + } finally { + restoreClean(); + restoreClose(); + } + }); }); describe('Test componentStatusRequestHandler function', () => { From 96cf1791a7e51e83aba6ecc28628e56702b9be01 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 09:21:12 -0600 Subject: [PATCH 15/43] test(storage): clean up expiration fixtures --- unitTests/resources/expiresAtAttribute.test.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 4ce514e611..c87a57d9c6 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -1,7 +1,7 @@ require('../testUtils'); const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); -const { table, closeDatabase, dropDatabase } = require('#src/resources/databases'); +const { table: createTable, closeDatabase, dropDatabase } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { Transaction: RocksTransaction } = require('@harperfast/rocksdb-js'); const { asBinary } = require('lmdb'); @@ -11,6 +11,19 @@ const { existsSync } = require('node:fs'); const { setTimeout: delay } = require('node:timers/promises'); const { waitFor } = require('../waitFor.js'); +const activeTables = new Set(); +const table = (options) => { + const Table = createTable(options); + activeTables.add(Table); + return Table; +}; + +afterEach(async function () { + const tables = [...activeTables]; + activeTables.clear(); + await Promise.all(tables.map((Table) => Table.cleanup())); +}); + // A schema @expiresAt attribute must be authoritative over the table-level expiration default, in both // directions. Previously the field only armed a separate index-pruning sweep (which can only remove // already-past records) and was never fed into the stored expiry metadata that governs read-hiding and From 949c82abebe8a537d575bcb49bde10bfd12b8880 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 09:35:04 -0600 Subject: [PATCH 16/43] fix(storage): align source expiration indexing --- resources/Table.ts | 6 +++ resources/search.ts | 14 ++++--- .../resources/expiresAtAttribute.test.js | 42 ++++++++++++++++++- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index d3c92bd773..6268766f71 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -6024,6 +6024,9 @@ export function makeTable(options) { // 5.2 record caching relies on it — so we must not write through the frozen object). if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord }; if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id; + if (sourceContext.expiresAt === undefined && expiresAtProperty) { + sourceContext.expiresAt = expirationTimestamp(updatedRecord[expiresAtProperty.name]); + } } resolved = true; const resolvedEntry: Entry = { @@ -6644,6 +6647,7 @@ export function makeTable(options) { if (cleanupClosed) break; let afterId: any; for (;;) { + if (cleanupClosed) break; const ids = [ ...index.getValues(key, { start: afterId, @@ -6654,6 +6658,7 @@ export function makeTable(options) { if (ids.length === 0) break; afterId = ids.at(-1); for (const id of ids) { + if (cleanupClosed) break; const recordEntry = primaryStore.getEntry(id); let operation: MaybePromise | void; if (recordEntry?.value == null) { @@ -6674,6 +6679,7 @@ export function makeTable(options) { if (backpressure) await backpressure; } await operationTracker.drain(); + await rest(); if (ids.length < EVICTION_BATCH_SIZE) break; } await rest(); diff --git a/resources/search.ts b/resources/search.ts index 9f90dde568..4e57859097 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -332,7 +332,7 @@ export function searchByIndex( } const isPrimaryKey = attribute_name === Table.primaryKey || attribute_name == null; const index = isPrimaryKey ? Table.primaryStore : Table.indices[attribute_name]; - if (!isPrimaryKey && Table.expiresAtAttributeName === attribute_name) { + if (!isPrimaryKey && Table.expiresAtAttributeName !== undefined && Table.expiresAtAttributeName === attribute_name) { value = normalizeExpirationSearchValue(value); } let start; @@ -955,7 +955,8 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar return recordFilter; } } - const normalizeExpirationValue = !isPrimaryKey && Table?.expiresAtAttributeName === attribute; + const normalizeExpirationValue = + !isPrimaryKey && Table?.expiresAtAttributeName !== undefined && Table.expiresAtAttributeName === attribute; if (normalizeExpirationValue) value = normalizeExpirationSearchValue(value); if (value instanceof Date) value = value.getTime(); @@ -1061,7 +1062,8 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar canUseIndex?: boolean, allowObjectMatching?: boolean ) { - const normalizeRecordExpiration = !isPrimaryKey && Table?.expiresAtAttributeName === attribute; + const normalizeRecordExpiration = + !isPrimaryKey && Table?.expiresAtAttributeName !== undefined && Table.expiresAtAttributeName === attribute; let thresholdRemainingMisses: number; canUseIndex = canUseIndex && // is it a comparator that makes sense to use index @@ -1186,7 +1188,8 @@ export function estimateCondition(table) { // we only attempt to estimate count on equals operator because that's really all that LMDB supports (some other key-value stores like libmdbx could be considered if we need to do estimated counts of ranges at some point) const index = table.indices[attribute_name]; let value = condition[1] ?? condition.value; - if (table.expiresAtAttributeName === attribute_name) value = normalizeExpirationSearchValue(value); + if (table.expiresAtAttributeName !== undefined && table.expiresAtAttributeName === attribute_name) + value = normalizeExpirationSearchValue(value); condition.estimated_count = index ? index.getValuesCount(value) : Infinity; } } else if (searchType === 'contains' || searchType === 'ends_with' || searchType === 'ne') { @@ -1202,7 +1205,8 @@ export function estimateCondition(table) { if (Array.isArray(condition.value) && index) { // Sum of per-value matches (over-counts duplicates but is a fine ceiling) let estimate = 0; - const normalizeExpiration = table.expiresAtAttributeName === attribute_name; + const normalizeExpiration = + table.expiresAtAttributeName !== undefined && table.expiresAtAttributeName === attribute_name; for (const item of condition.value) { estimate += index.getValuesCount(normalizeExpiration ? normalizeExpirationSearchValue(item) : item); } diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index c87a57d9c6..81415012b8 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -195,8 +195,28 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); - it('re-arms the expiration sweep after cleanup is resumed', function () { + it('indexes the effective expiration returned by a source fill', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSourceFill')); + const expiresAt = Date.now() - 1_000; + Table.sourcedFrom({ + get(id) { + return { id, expiresAt: new Date(expiresAt).toISOString() }; + }, + }); + + await Table.get(1); + await waitFor(() => Table.primaryStore.getEntry(1)?.value, { message: 'source fill should be stored' }); + await Table.primaryStore.committed; + assert.strictEqual(Table.primaryStore.getEntry(1).expiresAt, expiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + }); + + it('re-arms both expiration sweeps after cleanup is resumed', function () { let sweepIntervalCount = 0; + let cleanupTimeoutCount = 0; const realSetInterval = global.setInterval; const intervalStub = sinon.stub(global, 'setInterval').callsFake((callback, interval, ...args) => { if (interval === 60_000) { @@ -205,14 +225,21 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } return realSetInterval(callback, interval, ...args); }); + const timeoutStub = sinon.stub(global, 'setTimeout').callsFake(() => { + cleanupTimeoutCount++; + return { unref() {} }; + }); try { - const Table = makeTable('ExpiresAtResumeCleanup'); + const Table = makeTable('ExpiresAtResumeCleanup', undefined, { scanInterval: 100 }); assert.strictEqual(sweepIntervalCount, 1); + assert.strictEqual(cleanupTimeoutCount, 1); Table.cleanup(); Table.resumeCleanup(); assert.strictEqual(sweepIntervalCount, 2); + assert.strictEqual(cleanupTimeoutCount, 2); Table.cleanup(); } finally { + timeoutStub.restore(); intervalStub.restore(); } }); @@ -879,6 +906,17 @@ describe('LMDB @expiresAt cleanup draining', function () { Table.cleanup(); }); + it('paginates duplicate expiration values across bounded chunks', async function () { + const { Table, runSweep } = captureExpirationSweep('LmdbExpirationPagination'); + const expiresAt = Date.now() - 1_000; + for (let id = 0; id < 205; id++) await Table.put(id, { id, expiresAt }); + await Table.primaryStore.committed; + + await runSweep(); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + for (let id = 0; id < 205; id++) assert.strictEqual(Table.primaryStore.getEntry(id)?.value, undefined); + }); + it('does not resolve cleanup while LMDB eviction writes are active', async function () { const { Table, runSweep } = captureExpirationSweep('LmdbExpirationDrain'); await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); From 35a8f08ebdffe74802578f78ab9b0023ccc09474 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 09:38:58 -0600 Subject: [PATCH 17/43] fix(storage): preserve source expiration precedence --- resources/Table.ts | 8 +++++--- unitTests/resources/expiresAtAttribute.test.js | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 6268766f71..e2f127b629 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -5971,7 +5971,6 @@ export function makeTable(options) { recordAction(resolveDuration, 'cache-resolution', tableName, null, 'success'); if (responseHeaders) appendHeader(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true); - if (expirationMs && sourceContext.expiresAt == undefined) sourceContext.expiresAt = Date.now() + expirationMs; if (updatedRecord) { if (typeof updatedRecord !== 'object') throw new Error('Only objects can be cached and stored in tables'); if (updatedRecord.status > 0 && updatedRecord.headers) { @@ -6024,8 +6023,11 @@ export function makeTable(options) { // 5.2 record caching relies on it — so we must not write through the frozen object). if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord }; if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id; - if (sourceContext.expiresAt === undefined && expiresAtProperty) { - sourceContext.expiresAt = expirationTimestamp(updatedRecord[expiresAtProperty.name]); + if (sourceContext.expiresAt === undefined) { + if (expiresAtProperty) + sourceContext.expiresAt = expirationTimestamp(updatedRecord[expiresAtProperty.name]); + if (sourceContext.expiresAt === undefined && expirationMs) + sourceContext.expiresAt = Date.now() + expirationMs; } } resolved = true; diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 81415012b8..176d8bf87b 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -196,7 +196,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { }); it('indexes the effective expiration returned by a source fill', async function () { - const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSourceFill')); + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSourceFill', 3_600)); const expiresAt = Date.now() - 1_000; Table.sourcedFrom({ get(id) { From d311c5e7be0f9cf5cb46ee3025592d5359b7d724 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 15:39:45 -0600 Subject: [PATCH 18/43] test: replace expiration sweep sinon stubs --- resources/Table.ts | 24 +- resources/databases.ts | 13 +- .../resources/closeLoadedDatabases.test.js | 84 +++-- .../resources/expiresAtAttribute.test.js | 298 +++++------------- 4 files changed, 155 insertions(+), 264 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index e2f127b629..e020651927 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -5346,6 +5346,16 @@ export function makeTable(options) { if (expiresAtProperty) runRecordExpirationEviction(); if (audit) addDeleteRemoval(); } + static runRecordExpirationSweepForTests(testHooks?: ExpirationSweepTestHooks) { + return runRecordExpirationEviction(testHooks, false) ?? Promise.resolve(); + } + static cleanupStateForTests() { + return { + closed: cleanupClosed, + cleanupScheduled: !cleanupClosed && !!cleanupTimer, + expirationScheduled: !cleanupClosed && !!recordExpirationInterval, + }; + } static _readTxnForContext(context) { return txnForContext(context).getReadTxn(); } @@ -6267,7 +6277,11 @@ export function makeTable(options) { // catches anything modified between staging and commit: on conflict (ERR_BUSY) we re-stage once // into a fresh transaction (dropping the now-changed record) and otherwise skip the batch, leaving // those records for the next cleanup cycle. - function createEvictionBatcher(isCancelled: () => boolean = () => false) { + type ExpirationSweepTestHooks = { + beforeEvict?: () => MaybePromise; + beforeBatchCommit?: () => MaybePromise; + }; + function createEvictionBatcher(isCancelled: () => boolean = () => false, testHooks?: ExpirationSweepTestHooks) { type EvictItem = | { type: 'evict'; key: any; version: number; indexedExpiration?: number } | { type: 'tombstone'; key: any; version: number } @@ -6365,6 +6379,7 @@ export function makeTable(options) { return; } try { + await testHooks?.beforeBatchCommit?.(); await transaction.commit(); return; } catch (error: any) { @@ -6566,7 +6581,7 @@ export function makeTable(options) { primaryStore.remove(id, version); }); } - function runRecordExpirationEviction() { + function runRecordExpirationEviction(testHooks?: ExpirationSweepTestHooks, schedule = true) { if (getWorkerIndex() !== 0 || cleanupClosed) return; const expiresAtName = expiresAtProperty.name; const indexedExpiration = (entry: Entry): number | undefined => @@ -6575,7 +6590,7 @@ export function makeTable(options) { : expirationTimestamp(entry.expiresAt); async function sweepRocks(index: any, cutoff: number) { - const batcher = createEvictionBatcher(() => cleanupClosed); + const batcher = createEvictionBatcher(() => cleanupClosed, testHooks); let after: any[] | undefined; try { while (!cleanupClosed && primaryStore.rootStore.status === 'open') { @@ -6603,6 +6618,7 @@ export function makeTable(options) { const currentExpiration = indexedExpiration(recordEntry); if (currentExpiration !== undefined && currentExpiration < cutoff) { if (recordEntry.metadataFlags & HAS_BLOBS) { + await testHooks?.beforeEvict?.(); await TableResource.evict(entry.value, recordEntry.value, recordEntry.version, currentExpiration); } else { backpressure = batcher.add({ @@ -6668,6 +6684,7 @@ export function makeTable(options) { } else { const currentExpiration = indexedExpiration(recordEntry); if (currentExpiration !== undefined && currentExpiration < cutoff) { + await testHooks?.beforeEvict?.(); operation = TableResource.evict(id, recordEntry.value, recordEntry.version, currentExpiration); } else if (compareKeys(key, currentExpiration) !== 0) { operation = primaryStore.ifVersion(id, recordEntry.version, () => { @@ -6716,6 +6733,7 @@ export function makeTable(options) { } const runSweep = () => (recordExpirationCompletion = sweep()); + if (!schedule) return runSweep(); recordExpirationInterval = setInterval(runSweep, RECORD_PRUNING_INTERVAL); recordExpirationInterval.unref(); } diff --git a/resources/databases.ts b/resources/databases.ts index 3f7f41a535..8425ef009c 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -96,7 +96,12 @@ const logger = forComponent('storage'); const DEFAULT_DATABASE_NAME = 'data'; const DEFINED_TABLES = Symbol('defined-tables'); -const DATABASE_CLOSE_TIMEOUT = 10_000; +const DEFAULT_DATABASE_CLOSE_TIMEOUT = 10_000; +let databaseCloseTimeout = DEFAULT_DATABASE_CLOSE_TIMEOUT; + +export function setDatabaseCloseTimeoutForTests(timeout = DEFAULT_DATABASE_CLOSE_TIMEOUT): void { + databaseCloseTimeout = timeout; +} async function waitForTableCleanup(completions: Promise[]): Promise { let timeout: NodeJS.Timeout | undefined; @@ -104,7 +109,7 @@ async function waitForTableCleanup(completions: Promise[]): Promise((resolve) => { - timeout = setTimeout(() => resolve(timedOut), DATABASE_CLOSE_TIMEOUT); + timeout = setTimeout(() => resolve(timedOut), databaseCloseTimeout); timeout.unref(); }), ]); @@ -1270,7 +1275,7 @@ export async function dropDatabase(databaseName) { if (!(await waitForTableCleanup(cleanupCompletions))) { for (const tableName in dbTables) dbTables[tableName].resumeCleanup?.(); throw new Error( - `Timed out after ${DATABASE_CLOSE_TIMEOUT}ms waiting for cleanup; refusing to destroy database ${databaseName} while cleanup is active.` + `Timed out after ${databaseCloseTimeout}ms waiting for cleanup; refusing to destroy database ${databaseName} while cleanup is active.` ); } for (const tableName in dbTables) { @@ -1373,7 +1378,7 @@ async function closeDatabaseOnce(databaseName: string): Promise { if (!(await waitForTableCleanup(cleanupCompletions))) { for (const [, table] of tableEntries) table.resumeCleanup?.(); throw new Error( - `Timed out after ${DATABASE_CLOSE_TIMEOUT}ms waiting for cleanup; refusing to close database ${databaseName} while cleanup is active.` + `Timed out after ${databaseCloseTimeout}ms waiting for cleanup; refusing to close database ${databaseName} while cleanup is active.` ); } for (const [tableName, table] of tableEntries) { diff --git a/unitTests/resources/closeLoadedDatabases.test.js b/unitTests/resources/closeLoadedDatabases.test.js index 9d049092d2..1d78450466 100644 --- a/unitTests/resources/closeLoadedDatabases.test.js +++ b/unitTests/resources/closeLoadedDatabases.test.js @@ -16,9 +16,11 @@ const { closeDatabase, closeLoadedDatabases, dropDatabase, + setDatabaseCloseTimeoutForTests, } = require('#src/resources/databases'); const { registryStatus, RocksDatabase } = require('@harperfast/rocksdb-js'); -const sinon = require('sinon'); +const { createBlob } = require('#src/resources/blob'); +const { waitFor } = require('../waitFor.js'); describe('RocksDB handle release', function () { before(function () { @@ -87,12 +89,9 @@ describe('RocksDB handle release', function () { { attribute: 'expiresAt', expiresAt: true, indexed: true }, ], }); - const cleanup = sinon.spy(Table, 'cleanup'); - await closeDatabase('closerelease4'); - assert.strictEqual(cleanup.callCount, 1, 'table cleanup should run before its database closes'); - cleanup.restore(); + assert.strictEqual(Table.cleanupStateForTests().closed, true); }); it('coalesces concurrent closes of the same database', async function () { @@ -108,32 +107,45 @@ describe('RocksDB handle release', function () { }); it('fails closed and resumes cleanup when close times out', async function () { + this.timeout(30000); const Table = table({ table: 'expiring', database: 'closetimeout6', attributes: [ { attribute: 'id', isPrimaryKey: true }, { attribute: 'expiresAt', expiresAt: true, indexed: true }, + { attribute: 'payload', type: 'Blob' }, ], }); const rootStore = Table.primaryStore.rootStore; if (!(rootStore instanceof RocksDatabase)) return this.skip(); - const cleanup = sinon.stub(Table, 'cleanup').returns(new Promise(() => {})); - const resume = sinon.spy(Table, 'resumeCleanup'); - const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + await Table.put(1, { + id: 1, + expiresAt: Date.now() - 1_000, + payload: createBlob(Buffer.alloc(20_000, 1)), + }); + await Table.primaryStore.committed; + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + const sweep = Table.runRecordExpirationSweepForTests({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'the expiration sweep should start' }); + setDatabaseCloseTimeoutForTests(25); try { - const closing = closeDatabase('closetimeout6'); - const rejected = assert.rejects(closing, /refusing to close database/); - await clock.tickAsync(10_001); - await rejected; + await assert.rejects(closeDatabase('closetimeout6'), /refusing to close database/); assert.ok(refCountFor(rootStore.path) > 0, 'timed-out close must retain native handles'); assert.strictEqual(rootStore.status, 'open'); - assert.strictEqual(resume.callCount, 1); + assert.strictEqual(Table.cleanupStateForTests().closed, false); } finally { - clock.restore(); - cleanup.restore(); - resume.restore(); + setDatabaseCloseTimeoutForTests(); + releaseEviction(); } + await sweep; await closeDatabase('closetimeout6'); }); @@ -146,41 +158,51 @@ describe('RocksDB handle release', function () { { attribute: 'expiresAt', expiresAt: true, indexed: true }, ], }); - const cleanup = sinon.spy(Table, 'cleanup'); - await dropDatabase('droprelease6'); - assert.strictEqual(cleanup.callCount, 1, 'table cleanup should run before its database is destroyed'); - cleanup.restore(); + assert.strictEqual(Table.cleanupStateForTests().closed, true); }); it('fails closed and resumes cleanup when drop times out', async function () { + this.timeout(30000); const Table = table({ table: 'expiring', database: 'droptimeout7', attributes: [ { attribute: 'id', isPrimaryKey: true }, { attribute: 'expiresAt', expiresAt: true, indexed: true }, + { attribute: 'payload', type: 'Blob' }, ], }); const rootStore = Table.primaryStore.rootStore; if (!(rootStore instanceof RocksDatabase)) return this.skip(); - const cleanup = sinon.stub(Table, 'cleanup').returns(new Promise(() => {})); - const resume = sinon.spy(Table, 'resumeCleanup'); - const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + await Table.put(1, { + id: 1, + expiresAt: Date.now() - 1_000, + payload: createBlob(Buffer.alloc(20_000, 2)), + }); + await Table.primaryStore.committed; + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + const sweep = Table.runRecordExpirationSweepForTests({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'the expiration sweep should start' }); + setDatabaseCloseTimeoutForTests(25); try { - const dropping = dropDatabase('droptimeout7'); - const rejected = assert.rejects(dropping, /refusing to destroy database/); - await clock.tickAsync(10_001); - await rejected; + await assert.rejects(dropDatabase('droptimeout7'), /refusing to destroy database/); assert.ok(refCountFor(rootStore.path) > 0, 'timed-out drop must retain native handles'); assert.strictEqual(rootStore.status, 'open'); - assert.strictEqual(resume.callCount, 1); + assert.strictEqual(Table.cleanupStateForTests().closed, false); } finally { - clock.restore(); - cleanup.restore(); - resume.restore(); + setDatabaseCloseTimeoutForTests(); + releaseEviction(); } + await sweep; await dropDatabase('droptimeout7'); }); }); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 176d8bf87b..5f4cde021b 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -5,8 +5,7 @@ const { table: createTable, closeDatabase, dropDatabase } = require('#src/resour const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { Transaction: RocksTransaction } = require('@harperfast/rocksdb-js'); const { asBinary } = require('lmdb'); -const sinon = require('sinon'); -const { createBlob, getFilePathForBlob, setDeletionDelay } = require('#src/resources/blob'); +const { createBlob, getFilePathForBlob } = require('#src/resources/blob'); const { existsSync } = require('node:fs'); const { setTimeout: delay } = require('node:timers/promises'); const { waitFor } = require('../waitFor.js'); @@ -55,26 +54,11 @@ describe('@expiresAt attribute is authoritative over the table default', () => { }; const captureExpirationSweep = (createTable) => { - let runSweep; - let sweepIntervalCount = 0; - const realSetInterval = global.setInterval; - const intervalStub = sinon.stub(global, 'setInterval').callsFake((callback, interval, ...args) => { - if (interval === 60_000) { - runSweep = callback; - sweepIntervalCount++; - return { unref() {} }; - } - return realSetInterval(callback, interval, ...args); - }); - let Table; - try { - Table = createTable(); - } finally { - intervalStub.restore(); - } - assert(runSweep, 'table creation should register the expiration sweep'); - assert.strictEqual(sweepIntervalCount, 1); - return { Table, runSweep }; + const Table = createTable(); + return { + Table, + runSweep: (testHooks) => Table.runRecordExpirationSweepForTests(testHooks), + }; }; it('extends: a far-future field overrides a short table default', async function () { @@ -214,34 +198,25 @@ describe('@expiresAt attribute is authoritative over the table default', () => { assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); }); - it('re-arms both expiration sweeps after cleanup is resumed', function () { - let sweepIntervalCount = 0; - let cleanupTimeoutCount = 0; - const realSetInterval = global.setInterval; - const intervalStub = sinon.stub(global, 'setInterval').callsFake((callback, interval, ...args) => { - if (interval === 60_000) { - sweepIntervalCount++; - return { unref() {} }; - } - return realSetInterval(callback, interval, ...args); + it('re-arms both expiration sweeps after cleanup is resumed', async function () { + const Table = makeTable('ExpiresAtResumeCleanup', undefined, { scanInterval: 100 }); + assert.deepStrictEqual(Table.cleanupStateForTests(), { + closed: false, + cleanupScheduled: true, + expirationScheduled: true, }); - const timeoutStub = sinon.stub(global, 'setTimeout').callsFake(() => { - cleanupTimeoutCount++; - return { unref() {} }; + await Table.cleanup(); + assert.deepStrictEqual(Table.cleanupStateForTests(), { + closed: true, + cleanupScheduled: false, + expirationScheduled: false, + }); + Table.resumeCleanup(); + assert.deepStrictEqual(Table.cleanupStateForTests(), { + closed: false, + cleanupScheduled: true, + expirationScheduled: true, }); - try { - const Table = makeTable('ExpiresAtResumeCleanup', undefined, { scanInterval: 100 }); - assert.strictEqual(sweepIntervalCount, 1); - assert.strictEqual(cleanupTimeoutCount, 1); - Table.cleanup(); - Table.resumeCleanup(); - assert.strictEqual(sweepIntervalCount, 2); - assert.strictEqual(cleanupTimeoutCount, 2); - Table.cleanup(); - } finally { - timeoutStub.restore(); - intervalStub.restore(); - } }); it('ignores non-timestamp field values (boolean / empty string) and uses the table default', async function () { @@ -429,54 +404,6 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); - it('keeps a blob file when an expired-record eviction conflicts', async function () { - const { Table, runSweep } = captureExpirationSweep(() => - table({ - table: 'ExpiresAtBlobConflict', - database: 'test', - attributes: [ - { name: 'id', isPrimaryKey: true }, - { name: 'expiresAt', expiresAt: true, indexed: true }, - { name: 'payload', type: 'Blob' }, - ], - }) - ); - const blob = createBlob(Buffer.alloc(20_000, 7)); - await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: blob }); - await Table.primaryStore.committed; - const filePath = getFilePathForBlob(blob); - assert(filePath); - assert(existsSync(filePath)); - const blobEntry = Table.primaryStore.getEntry(1); - await Table.evict(1, undefined, blobEntry.version); - assert(Table.primaryStore.getEntry(1)?.value, 'an eviction without the record value must fail safe'); - assert(existsSync(filePath)); - - setDeletionDelay(0); - const commitStub = sinon - .stub(RocksTransaction.prototype, 'commit') - .rejects(Object.assign(new Error('injected optimistic conflict'), { code: 'ERR_BUSY' })); - let commitRestored = false; - try { - await runSweep(); - await delay(25); - assert(Table.primaryStore.getEntry(1)?.value, 'the failed eviction must leave the record intact'); - assert(existsSync(filePath), 'a failed eviction must leave the referenced blob file intact'); - commitStub.restore(); - commitRestored = true; - - await runSweep(); - assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); - await waitFor(() => !existsSync(filePath), { - message: 'a committed eviction must remove its blob file', - }); - } finally { - if (!commitRestored) commitStub.restore(); - setDeletionDelay(500); - Table.cleanup(); - } - }); - it('preserves a blob record refreshed before its eviction transaction starts', async function () { const Table = table({ table: 'ExpiresAtBlobRefreshRace', @@ -525,24 +452,22 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const [, secondEntry] = [...Table.indices.expiresAt.getCompositeRange({ end: Date.now(), limit: 2 })]; assert(secondEntry, 'the sweep should have a second entry to skip'); - const originalEvict = Table.evict; let releaseEviction; let evictionStarted = false; const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); - const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { - evictionStarted = true; - await blockedEviction; - return originalEvict.apply(this, args); - }); try { - const sweep = runSweep(); + const sweep = runSweep({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); await waitFor(() => evictionStarted, { message: 'the first eviction should start' }); Table.cleanup(); releaseEviction(); await sweep; } finally { releaseEviction(); - evictStub.restore(); } assert(Table.primaryStore.getEntry(secondEntry.value)?.value, 'cleanup must stop the sweep before its next entry'); @@ -563,17 +488,16 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: createBlob(Buffer.alloc(20_000, 8)) }); await Table.primaryStore.committed; - const originalEvict = Table.evict; let releaseEviction; let evictionStarted = false; const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); - const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { - evictionStarted = true; - await blockedEviction; - return originalEvict.apply(this, args); - }); try { - const sweep = runSweep(); + const sweep = runSweep({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); await waitFor(() => evictionStarted, { message: 'the eviction should start' }); let dropResolved = false; const drop = Table.dropTable().then(() => (dropResolved = true)); @@ -583,7 +507,6 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Promise.all([sweep, drop]); } finally { releaseEviction(); - evictStub.restore(); } }); @@ -603,17 +526,16 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: createBlob(Buffer.alloc(20_000, 9)) }); await Table.primaryStore.committed; - const originalEvict = Table.evict; let releaseEviction; let evictionStarted = false; const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); - const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { - evictionStarted = true; - await blockedEviction; - return originalEvict.apply(this, args); - }); try { - const sweep = runSweep(); + const sweep = runSweep({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); await waitFor(() => evictionStarted, { message: 'the eviction should start' }); let closeResolved = false; const close = closeDatabase(database).then(() => (closeResolved = true)); @@ -623,55 +545,6 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Promise.all([sweep, close]); } finally { releaseEviction(); - evictStub.restore(); - } - }); - - it('cleanup drains the primary cleanup scan', async function () { - const Table = table({ - table: 'PrimaryCleanupDrain', - database: 'test', - attributes: [ - { name: 'id', isPrimaryKey: true }, - { name: 'payload', type: 'Blob' }, - ], - }); - let runCleanup; - const setupTimerStub = sinon.stub(global, 'setTimeout').callsFake((callback) => { - if (!runCleanup) { - runCleanup = callback; - return { unref() {} }; - } - throw new Error('setTTLExpiration scheduled more than one cleanup timer'); - }); - Table.setTTLExpiration({ scanInterval: 100 }); - setupTimerStub.restore(); - assert(runCleanup); - await Table.put(1, { id: 1, payload: createBlob(Buffer.alloc(20_000, 11)) }, { expiresAt: Date.now() - 1_000 }); - await Table.primaryStore.committed; - - const originalEvict = Table.evict; - let releaseEviction; - let evictionStarted = false; - const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); - const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { - evictionStarted = true; - await blockedEviction; - return originalEvict.apply(this, args); - }); - try { - const scan = runCleanup(); - await waitFor(() => evictionStarted, { message: 'the primary cleanup scan should start eviction' }); - let cleanupResolved = false; - const cleanup = Table.cleanup().then(() => (cleanupResolved = true)); - await delay(10); - assert.strictEqual(cleanupResolved, false); - releaseEviction(); - await Promise.all([scan, cleanup]); - } finally { - releaseEviction(); - evictStub.restore(); - setupTimerStub.restore(); } }); @@ -691,17 +564,16 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: createBlob(Buffer.alloc(20_000, 10)) }); await Table.primaryStore.committed; - const originalEvict = Table.evict; let releaseEviction; let evictionStarted = false; const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); - const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { - evictionStarted = true; - await blockedEviction; - return originalEvict.apply(this, args); - }); try { - const sweep = runSweep(); + const sweep = runSweep({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); await waitFor(() => evictionStarted, { message: 'the eviction should start' }); let dropResolved = false; const drop = dropDatabase(database).then(() => (dropResolved = true)); @@ -711,7 +583,6 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Promise.all([sweep, drop]); } finally { releaseEviction(); - evictStub.restore(); } }); @@ -777,20 +648,14 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const expiresAt = Date.now() - 1_000; Table.indices.expiresAt.put(expiresAt, 1); - const originalCommit = RocksTransaction.prototype.commit; let injected = false; - const commitStub = sinon.stub(RocksTransaction.prototype, 'commit').callsFake(async function (...args) { - if (!injected) { + await runSweep({ + beforeBatchCommit: async () => { + if (injected) return; injected = true; await Table.put(1, { id: 1, expiresAt, name: 'resurrected' }); - } - return originalCommit.apply(this, args); + }, }); - try { - await runSweep(); - } finally { - commitStub.restore(); - } assert(injected); assert.strictEqual(Table.primaryStore.getEntry(1)?.value.name, 'resurrected'); @@ -807,20 +672,14 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Table.primaryStore.committed; Table.indices.expiresAt.put(expired, 1); - const originalCommit = RocksTransaction.prototype.commit; let injected = false; - const commitStub = sinon.stub(RocksTransaction.prototype, 'commit').callsFake(async function (...args) { - if (!injected) { + await runSweep({ + beforeBatchCommit: async () => { + if (injected) return; injected = true; await Table.put(1, { id: 1, expiresAt: refreshed }); - } - return originalCommit.apply(this, args); + }, }); - try { - await runSweep(); - } finally { - commitStub.restore(); - } assert(injected); assert.strictEqual(Table.primaryStore.getEntry(1)?.value.expiresAt, refreshed); @@ -864,29 +723,18 @@ describe('LMDB @expiresAt cleanup draining', function () { }); const captureExpirationSweep = (name) => { - let runSweep; - const realSetInterval = global.setInterval; - const intervalStub = sinon.stub(global, 'setInterval').callsFake((callback, interval, ...args) => { - if (interval === 60_000) { - runSweep = callback; - return { unref() {} }; - } - return realSetInterval(callback, interval, ...args); + const Table = table({ + table: name, + database: 'lmdb-expiration-drain', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + ], }); - let Table; - try { - Table = table({ - table: name, - database: 'lmdb-expiration-drain', - attributes: [ - { name: 'id', isPrimaryKey: true }, - { name: 'expiresAt', expiresAt: true, indexed: true }, - ], - }); - } finally { - intervalStub.restore(); - } - return { Table, runSweep }; + return { + Table, + runSweep: (testHooks) => Table.runRecordExpirationSweepForTests(testHooks), + }; }; it('evicts ISO expirations and repairs stale keys', async function () { @@ -922,17 +770,16 @@ describe('LMDB @expiresAt cleanup draining', function () { await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); await Table.primaryStore.committed; - const originalEvict = Table.evict; let releaseEviction; let evictionStarted = false; const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); - const evictStub = sinon.stub(Table, 'evict').callsFake(async function (...args) { - evictionStarted = true; - await blockedEviction; - return originalEvict.apply(this, args); - }); try { - const sweep = runSweep(); + const sweep = runSweep({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); await waitFor(() => evictionStarted, { message: 'LMDB eviction should start' }); let cleanupResolved = false; const cleanup = Table.cleanup().then(() => (cleanupResolved = true)); @@ -942,7 +789,6 @@ describe('LMDB @expiresAt cleanup draining', function () { await Promise.all([sweep, cleanup]); } finally { releaseEviction(); - evictStub.restore(); } }); }); From 590052a5bb9e4da9ce6f820777c4445458695165 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 15:52:05 -0600 Subject: [PATCH 19/43] test: preserve cleanup invariant coverage --- resources/Table.ts | 177 +++++++++--------- .../resources/expiresAtAttribute.test.js | 73 +++++++- 2 files changed, 164 insertions(+), 86 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index e020651927..27a84ee2c9 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -440,10 +440,10 @@ export function makeTable(options) { let cleanupInterval = 86400000; let cleanupPriority = 0; let lastCleanupInterval: number; - let cleanupTimer: NodeJS.Timeout; + let cleanupTimer: NodeJS.Timeout | undefined; let cleanupClosed = false; let cleanupWasScheduled = false; - let recordExpirationInterval: NodeJS.Timeout; + let recordExpirationInterval: NodeJS.Timeout | undefined; let missingExpirationIndexReported = false; // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup let expirationScanScheduled = false; @@ -5332,8 +5332,14 @@ export function makeTable(options) { static cleanup() { deleteCallbackHandle?.remove(); cleanupClosed = true; - if (cleanupTimer) clearTimeout(cleanupTimer); - if (recordExpirationInterval) clearInterval(recordExpirationInterval); + if (cleanupTimer) { + clearTimeout(cleanupTimer); + cleanupTimer = undefined; + } + if (recordExpirationInterval) { + clearInterval(recordExpirationInterval); + recordExpirationInterval = undefined; + } return Promise.all([lastEvictionCompletion, recordExpirationCompletion]).then(() => undefined); } static resumeCleanup() { @@ -5349,6 +5355,9 @@ export function makeTable(options) { static runRecordExpirationSweepForTests(testHooks?: ExpirationSweepTestHooks) { return runRecordExpirationEviction(testHooks, false) ?? Promise.resolve(); } + static runPrimaryCleanupScanForTests(testHooks?: ExpirationSweepTestHooks) { + return (lastEvictionCompletion = lastEvictionCompletion.then(() => runPrimaryCleanupScan(testHooks))); + } static cleanupStateForTests() { return { closed: cleanupClosed, @@ -6447,6 +6456,79 @@ export function makeTable(options) { }; } + async function runPrimaryCleanupScan(testHooks?: ExpirationSweepTestHooks): Promise { + if (cleanupClosed) return; + const rootStore = primaryStore.rootStore; + if (rootStore.status !== 'open') return; + const operationTracker = createCleanupOperationTracker(); + const evictThreshold = + Math.pow(cleanupPriority, 8) * (envMngr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_EVICTIONFACTOR) ?? 100000); + const adjustedEviction = evictionMs / Math.pow(Math.max(cleanupPriority, 1), 4); + logger.debug?.( + `Starting cleanup scan for ${tableName}, evict threshold ${evictThreshold}, adjusted eviction ${adjustedEviction}ms` + ); + function shouldEvict(expiresAt: number, version: number, metadataFlags: number, record: any) { + const evictWhen = expiresAt + adjustedEviction - Date.now(); + if (evictWhen < 0) return true; + if (cleanupPriority) { + let size = primaryStore.lastSize; + if (metadataFlags & HAS_BLOBS) { + findBlobsInObject(record, (blob) => { + if (blob.size) size += blob.size; + }); + } + logger.trace?.( + `shouldEvict adjusted ${evictWhen} ${size}, ${(evictWhen * (expiresAt - version)) / size} < ${evictThreshold}` + ); + return (evictWhen * (expiresAt - version)) / size < evictThreshold; + } + return false; + } + + try { + let count = 0; + const removeDeletedRecords = !audit || isRocksDB; + const batcher = isRocksDB ? createEvictionBatcher(() => cleanupClosed, testHooks) : undefined; + for (const entry of primaryStore.getRange({ + start: false, + snapshot: false, + versions: true, + lazy: true, + })) { + if (cleanupClosed) break; + const { key, value: record, version, expiresAt, metadataFlags } = entry; + let action: 'tombstone' | 'evict' | undefined; + if (record === null && removeDeletedRecords && version + auditRetention < Date.now()) { + action = 'tombstone'; + } else if (expiresAt != undefined && shouldEvict(expiresAt, version, metadataFlags, record)) { + action = 'evict'; + count++; + } + if (action) { + if (batcher && !(action === 'evict' && metadataFlags & HAS_BLOBS)) { + await batcher.add({ type: action, key, version }); + } else { + if (action === 'evict') await testHooks?.beforeEvict?.(); + const resolution = + action === 'tombstone' + ? removeEntry(primaryStore, entry, version) + : TableResource.evict(key, record, version); + if (resolution) { + const backpressure = operationTracker.add(resolution); + if (backpressure) await backpressure; + } + } + } + await rest(); + } + if (batcher) await batcher.drain(); + await operationTracker.drain(); + logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); + } catch (error) { + logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); + } + } + function scheduleCleanup(priority?: number): Promise | void { if (cleanupClosed) return; let runImmediately = false; @@ -6483,89 +6565,14 @@ export function makeTable(options) { () => (lastEvictionCompletion = lastEvictionCompletion.then(async () => { if (cleanupClosed) return; - // schedule the next run for when the next cleanup interval should occur (or now if it is in the past) - startNextTimer(Math.max(nextScheduled + cleanupInterval, Date.now())); - const rootStore = primaryStore.rootStore; - if (rootStore.status !== 'open') { - clearTimeout(cleanupTimer); + if (primaryStore.rootStore.status !== 'open') { + if (cleanupTimer) clearTimeout(cleanupTimer); + cleanupTimer = undefined; return; } - const operationTracker = createCleanupOperationTracker(); - const evictThreshold = - Math.pow(cleanupPriority, 8) * - (envMngr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_EVICTIONFACTOR) ?? 100000); - const adjustedEviction = evictionMs / Math.pow(Math.max(cleanupPriority, 1), 4); - logger.debug?.( - `Starting cleanup scan for ${tableName}, evict threshold ${evictThreshold}, adjusted eviction ${adjustedEviction}ms` - ); - function shouldEvict(expiresAt: number, version: number, metadataFlags: number, record: any) { - const evictWhen = expiresAt + adjustedEviction - Date.now(); - if (evictWhen < 0) return true; - else if (cleanupPriority) { - let size = primaryStore.lastSize; - if (metadataFlags & HAS_BLOBS) { - findBlobsInObject(record, (blob) => { - if (blob.size) size += blob.size; - }); - } - logger.trace?.( - `shouldEvict adjusted ${evictWhen} ${size}, ${(evictWhen * (expiresAt - version)) / size} < ${evictThreshold}` - ); - // heuristic to determine if we should perform early eviction based on priority - return (evictWhen * (expiresAt - version)) / size < evictThreshold; - } - return false; - } - - try { - let count = 0; - let removeDeletedRecords = !audit || isRocksDB; - // RocksDB coalesces eviction/tombstone removals into shared transactions to amortize - // the per-record commit cost; LMDB keeps the per-record path (eventTurnBatching already - // coalesces async writes per event turn). - const batcher = isRocksDB ? createEvictionBatcher(() => cleanupClosed) : undefined; - // iterate through all entries to find expired records and deleted records - for (const entry of primaryStore.getRange({ - start: false, - snapshot: false, // we don't want to keep read transaction snapshots open - versions: true, - lazy: true, // only want to access metadata most of the time - })) { - if (cleanupClosed) break; - const { key, value: record, version, expiresAt, metadataFlags } = entry; - // if there is no auditing cleanup and we are tracking deletion, need to do cleanup of - // these deletion entries (LMDB audit cleanup has its own scheduled job for this) - let action: 'tombstone' | 'evict' | undefined; - if (record === null && removeDeletedRecords && version + auditRetention < Date.now()) { - action = 'tombstone'; - } else if (expiresAt != undefined && shouldEvict(expiresAt, version, metadataFlags, record)) { - action = 'evict'; - count++; - } - if (action) { - // Blob-bearing records delete their blob files as a non-transactional side effect, so - // they stay on the per-record evict() path that preserves the existing blob/commit ordering. - if (batcher && !(action === 'evict' && metadataFlags & HAS_BLOBS)) { - await batcher.add({ type: action, key, version }); - } else { - const resolution = - action === 'tombstone' - ? removeEntry(primaryStore, entry, version) - : TableResource.evict(key, record, version); - if (resolution) { - const backpressure = operationTracker.add(resolution); - if (backpressure) await backpressure; - } - } - } - await rest(); - } - if (batcher) await batcher.drain(); - await operationTracker.drain(); - logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); - } catch (error) { - logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); - } + // schedule the next run for when the next cleanup interval should occur (or now if it is in the past) + startNextTimer(Math.max(nextScheduled + cleanupInterval, Date.now())); + await runPrimaryCleanupScan(); resolve(undefined); cleanupPriority = 0; // reset the priority })), diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 5f4cde021b..98247a4e8c 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -5,7 +5,7 @@ const { table: createTable, closeDatabase, dropDatabase } = require('#src/resour const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { Transaction: RocksTransaction } = require('@harperfast/rocksdb-js'); const { asBinary } = require('lmdb'); -const { createBlob, getFilePathForBlob } = require('#src/resources/blob'); +const { createBlob, getFilePathForBlob, setDeletionDelay } = require('#src/resources/blob'); const { existsSync } = require('node:fs'); const { setTimeout: delay } = require('node:timers/promises'); const { waitFor } = require('../waitFor.js'); @@ -211,6 +211,12 @@ describe('@expiresAt attribute is authoritative over the table default', () => { cleanupScheduled: false, expirationScheduled: false, }); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); + assert.deepStrictEqual(Table.cleanupStateForTests(), { + closed: true, + cleanupScheduled: false, + expirationScheduled: false, + }); Table.resumeCleanup(); assert.deepStrictEqual(Table.cleanupStateForTests(), { closed: false, @@ -404,6 +410,36 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); + it('keeps a blob when eviction lacks the record and unlinks it after a committed eviction', async function () { + const Table = table({ + table: 'ExpiresAtBlobFailSafe', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }); + const blob = createBlob(Buffer.alloc(20_000, 7)); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: blob }); + await Table.primaryStore.committed; + const filePath = getFilePathForBlob(blob); + const entry = Table.primaryStore.getEntry(1); + + await Table.evict(1, undefined, entry.version); + assert(Table.primaryStore.getEntry(1)?.value, 'eviction without the record must fail safe'); + assert(existsSync(filePath), 'fail-safe eviction must keep the referenced blob'); + + setDeletionDelay(0); + try { + await Table.evict(1, entry.value, entry.version); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + await waitFor(() => !existsSync(filePath), { message: 'committed eviction should unlink the blob' }); + } finally { + setDeletionDelay(500); + } + }); + it('preserves a blob record refreshed before its eviction transaction starts', async function () { const Table = table({ table: 'ExpiresAtBlobRefreshRace', @@ -548,6 +584,41 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } }); + it('cleanup drains the primary cleanup scan', async function () { + const Table = table({ + table: 'PrimaryCleanupDrain', + database: 'test', + expiration: 1, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'payload', type: 'Blob' }, + ], + }); + await Table.put(1, { id: 1, payload: createBlob(Buffer.alloc(20_000, 11)) }, { expiresAt: Date.now() - 1_000 }); + await Table.primaryStore.committed; + + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + try { + const scan = Table.runPrimaryCleanupScanForTests({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'the primary cleanup scan should start eviction' }); + let cleanupResolved = false; + const cleanup = Table.cleanup().then(() => (cleanupResolved = true)); + await delay(10); + assert.strictEqual(cleanupResolved, false); + releaseEviction(); + await Promise.all([scan, cleanup]); + } finally { + releaseEviction(); + } + }); + it('waits for an active expiration sweep before destroying a database', async function () { const database = 'ExpiresAtDatabaseDrop'; const { Table, runSweep } = captureExpirationSweep(() => From d997b234407665383186e68cf0089ccdba79e85f Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 15:59:32 -0600 Subject: [PATCH 20/43] test: exercise eviction conflicts through sweep seam --- resources/Table.ts | 4 +- unitTests/resources/evictionBatch.test.js | 39 ------------------- .../resources/expiresAtAttribute.test.js | 17 ++++++++ 3 files changed, 20 insertions(+), 40 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 27a84ee2c9..ae625198d2 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -6491,7 +6491,7 @@ export function makeTable(options) { const batcher = isRocksDB ? createEvictionBatcher(() => cleanupClosed, testHooks) : undefined; for (const entry of primaryStore.getRange({ start: false, - snapshot: false, + snapshot: false, // avoid holding a read snapshot for the duration of a full-table scan versions: true, lazy: true, })) { @@ -6505,6 +6505,8 @@ export function makeTable(options) { count++; } if (action) { + // Blob deletion is a non-transactional side effect, so blob records stay on evict()'s + // per-record path where unlinking waits for the record removal to commit. if (batcher && !(action === 'evict' && metadataFlags & HAS_BLOBS)) { await batcher.add({ type: action, key, version }); } else { diff --git a/unitTests/resources/evictionBatch.test.js b/unitTests/resources/evictionBatch.test.js index afeedb0523..0858b9483f 100644 --- a/unitTests/resources/evictionBatch.test.js +++ b/unitTests/resources/evictionBatch.test.js @@ -21,7 +21,6 @@ const { setMainIsWorker } = require('#js/server/threads/manageThreads'); describe('Batched eviction (RocksDB)', () => { if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; let EvictTable; - let RetryTable; const makeSource = () => class extends Resource { @@ -72,12 +71,6 @@ describe('Batched eviction (RocksDB)', () => { ], }); EvictTable.sourcedFrom(makeSource()); - RetryTable = table({ - table: 'EvictRetryTable', - database: 'test', - attributes: [{ name: 'id', isPrimaryKey: true }], - }); - RetryTable.sourcedFrom(makeSource()); }); it('physically evicts >EVICTION_BATCH_SIZE records across multiple batches and cleans indices', async function () { @@ -103,36 +96,4 @@ describe('Batched eviction (RocksDB)', () => { 'index entries must be removed with the records' ); }); - - it('recovers from an optimistic commit conflict (ERR_BUSY) and still evicts', async function () { - const { Transaction } = require('@harperfast/rocksdb-js'); - const originalCommit = Transaction.prototype.commit; - let injected = false; - let armed = false; - // Inject a single ERR_BUSY into the first commit after arming, simulating an optimistic - // write-write conflict. The batcher must abort, re-stage into a fresh transaction, and commit. - Transaction.prototype.commit = async function (...args) { - if (armed && !injected) { - injected = true; - const error = new Error('injected optimistic conflict'); - error.code = 'ERR_BUSY'; - throw error; - } - return originalCommit.apply(this, args); - }; - - const ids = [1000, 1001, 1002, 1003, 1004]; - try { - RetryTable.setTTLExpiration(HOLD); // keep records resident (and un-evicted) until the conflict is armed - for (const id of ids) await RetryTable.get(id); - assert.equal(await waitForResident(RetryTable, ids), ids.length, 'records should be resident after warming'); - - armed = true; // the next commit (an eviction-batch commit) will hit the injected conflict - const resident = await evictAndWait(RetryTable, ids); - assert(injected, 'the injected ERR_BUSY conflict should have fired on a commit'); - assert.equal(resident, 0, 'eviction should still complete after retrying the conflicting batch'); - } finally { - Transaction.prototype.commit = originalCommit; - } - }); }); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 98247a4e8c..106fb5dfb1 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -370,6 +370,23 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); + it('retries a conflicting Rocks eviction batch', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtBatchConflict')); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); + await Table.primaryStore.committed; + let commitAttempts = 0; + + await runSweep({ + beforeBatchCommit() { + commitAttempts++; + if (commitAttempts === 1) throw Object.assign(new Error('injected optimistic conflict'), { code: 'ERR_BUSY' }); + }, + }); + + assert.strictEqual(commitAttempts, 2, 'the conflicting batch should be re-staged and committed once'); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + }); + it('continues an expiration sweep beyond one RocksDB chunk', async function () { const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSweepContinuation')); const expiresAt = Date.now() - 1_000; From 93e24366bb1a57b001a11a910d17ffd79eb95968 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 21:07:09 -0600 Subject: [PATCH 21/43] Preserve source-owned cache expiration --- resources/DESIGN.md | 5 +++ resources/Table.ts | 19 ++++++---- schema.graphql | 3 +- unitTests/resources/expirationWarning.test.js | 35 +++++++++++++++++++ .../resources/expiresAtAttribute.test.js | 29 +++++++++++---- 5 files changed, 77 insertions(+), 14 deletions(-) diff --git a/resources/DESIGN.md b/resources/DESIGN.md index 53ac81b566..9f062df316 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -116,6 +116,11 @@ Rows created before expiration metadata was stored fall back to their public fie sweeping; an explicit no-expiration metadata sentinel never falls back. This preserves upgrade behavior without making every open scan the table. +The expiration index represents the effective stored expiration, not an ordinary field-value index. An explicit +`options.expiresAt` or `context.expiresAt` override can therefore differ from the serialized field while remaining +authoritative for index searches and reclamation. Source/cache fills do not infer TTL from the returned field; the +source must set `sourceContext.expiresAt`, with table expiration providing the fallback. + RocksDB `@expiresAt` sweeps walk one bounded composite-index range at a time with a fixed cutoff and an owned `(expiresAt, primaryKey)` cursor. A sweep never holds an iterator snapshot across an `await`, and it continues until every index entry at or before that cutoff has been considered. This keeps memory and native diff --git a/resources/Table.ts b/resources/Table.ts index ae625198d2..013ec6a417 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -445,6 +445,7 @@ export function makeTable(options) { let cleanupWasScheduled = false; let recordExpirationInterval: NodeJS.Timeout | undefined; let missingExpirationIndexReported = false; + let missingSourceExpirationReported = false; // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup let expirationScanScheduled = false; // set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table @@ -5413,7 +5414,7 @@ export function makeTable(options) { // the real [value, id] entry was removed, orphaning it against the now-deleted record. A record // that is present but whose attribute is null is a different case (record is a truthy object) and // still indexes under null. See harper#1894 (F-149). - const expirationIndex = expiresAtIndexName !== undefined && key === expiresAtIndexName; + const expirationIndex = key === expiresAtIndexName; const value = record == null ? undefined @@ -5990,6 +5991,7 @@ export function makeTable(options) { recordAction(resolveDuration, 'cache-resolution', tableName, null, 'success'); if (responseHeaders) appendHeader(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true); + if (expirationMs && sourceContext.expiresAt == undefined) sourceContext.expiresAt = Date.now() + expirationMs; if (updatedRecord) { if (typeof updatedRecord !== 'object') throw new Error('Only objects can be cached and stored in tables'); if (updatedRecord.status > 0 && updatedRecord.headers) { @@ -6042,11 +6044,16 @@ export function makeTable(options) { // 5.2 record caching relies on it — so we must not write through the frozen object). if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord }; if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id; - if (sourceContext.expiresAt === undefined) { - if (expiresAtProperty) - sourceContext.expiresAt = expirationTimestamp(updatedRecord[expiresAtProperty.name]); - if (sourceContext.expiresAt === undefined && expirationMs) - sourceContext.expiresAt = Date.now() + expirationMs; + if ( + sourceContext.expiresAt === undefined && + expiresAtProperty && + expirationTimestamp(updatedRecord[expiresAtProperty.name]) !== undefined && + !missingSourceExpirationReported + ) { + missingSourceExpirationReported = true; + logger.warn?.( + `Source for table "${tableName}" returned an @expiresAt field without setting context.expiresAt; the field does not set cache expiration.` + ); } } resolved = true; diff --git a/schema.graphql b/schema.graphql index 3820724af5..03eded2414 100644 --- a/schema.graphql +++ b/schema.graphql @@ -163,7 +163,8 @@ Allows enumeration over a computed field, causing it to be included in serialize directive @enumerable on FIELD_DEFINITION """ -Flags the field as containing the expiration time of the entry. +Flags the field as containing the expiration time of the entry. The index follows the effective stored expiration; +an explicit write-context expiration can override the serialized field value. """ directive @expiresAt on FIELD_DEFINITION diff --git a/unitTests/resources/expirationWarning.test.js b/unitTests/resources/expirationWarning.test.js index 8f27f049f0..fbffa09629 100644 --- a/unitTests/resources/expirationWarning.test.js +++ b/unitTests/resources/expirationWarning.test.js @@ -20,6 +20,13 @@ describe('Per-record expiresAt without scheduled cleanup (#1339)', () => { const matchesWarning = (message) => typeof message === 'string' && message.includes('per-record expiresAt'); const warningsFor = (tableName) => warnings.filter(([message]) => matchesWarning(message) && message.includes(`"${tableName}"`)); + const sourceWarningsFor = (tableName) => + warnings.filter( + ([message]) => + typeof message === 'string' && + message.includes(`table "${tableName}"`) && + message.includes('@expiresAt field without setting context.expiresAt') + ); before(function () { setupTestDBPath(); @@ -97,4 +104,32 @@ describe('Per-record expiresAt without scheduled cleanup (#1339)', () => { await new Promise((resolve) => setTimeout(resolve, 50)); assert.strictEqual(warningsFor('SourcedExpiresAtTable').length, 0); }); + + it('warns once when a source returns an @expiresAt field without setting cache expiration', async function () { + const tableName = 'SourcedFieldOnlyExpiresAtTable'; + const SourcedTable = table({ + table: tableName, + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + SourcedTable.sourcedFrom( + class extends Resource { + get() { + return { id: this.getId(), expiresAt: Date.now() + 60000 }; + } + } + ); + + await SourcedTable.get(1); + await waitFor(() => sourceWarningsFor(tableName).length === 1); + assert.strictEqual(SourcedTable.primaryStore.getEntry(1)?.expiresAt, undefined); + assert.deepStrictEqual([...SourcedTable.indices.expiresAt.getRange({ start: true })], []); + + await SourcedTable.get(2); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.strictEqual(sourceWarningsFor(tableName).length, 1); + }); }); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 106fb5dfb1..63561692db 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -87,8 +87,20 @@ describe('@expiresAt attribute is authoritative over the table default', () => { it('lets an explicit options.expiresAt override the field', async function () { const Table = makeTable('ExpiresAtOptionsOverride', 3); const optionExpiresAt = Date.now() + 60_000; - await Table.put(1, { id: 1, expiresAt: Date.now() + 3_600_000 }, { expiresAt: optionExpiresAt }); + const fieldExpiresAt = optionExpiresAt + 3_600_000; + await Table.put(1, { id: 1, expiresAt: fieldExpiresAt }, { expiresAt: optionExpiresAt }); assert.strictEqual(await storedExpiresAt(Table, 1), optionExpiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(optionExpiresAt)], [1]); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + const matchingOverride = []; + for await (const record of Table.search({ + allowFullScan: false, + conditions: [{ attribute: 'expiresAt', value: optionExpiresAt }], + })) { + matchingOverride.push(record); + } + assert.strictEqual(matchingOverride.length, 1); + assert.strictEqual(matchingOverride[0].expiresAt, fieldExpiresAt); }); it('keeps the field value across a patch that does not touch it', async function () { @@ -179,20 +191,23 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); - it('indexes the effective expiration returned by a source fill', async function () { + it('uses source context rather than the returned field for cache expiration', async function () { const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSourceFill', 3_600)); - const expiresAt = Date.now() - 1_000; + const contextExpiresAt = Date.now() - 1_000; + const fieldExpiresAt = contextExpiresAt + 3_600_000; Table.sourcedFrom({ - get(id) { - return { id, expiresAt: new Date(expiresAt).toISOString() }; + get(id, context) { + context.expiresAt = contextExpiresAt; + return { id, expiresAt: new Date(fieldExpiresAt).toISOString() }; }, }); await Table.get(1); await waitFor(() => Table.primaryStore.getEntry(1)?.value, { message: 'source fill should be stored' }); await Table.primaryStore.committed; - assert.strictEqual(Table.primaryStore.getEntry(1).expiresAt, expiresAt); - assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + assert.strictEqual(Table.primaryStore.getEntry(1).expiresAt, contextExpiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(contextExpiresAt)], [1]); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); await runSweep(); assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); From a5714a7236a594943e1edfd62369195edb67f890 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 21:25:20 -0600 Subject: [PATCH 22/43] Close expiration lifecycle races --- resources/Table.ts | 24 +++++------ resources/databases.ts | 22 +++++++++- resources/search.ts | 11 ++--- .../resources/closeLoadedDatabases.test.js | 40 +++++++++++++++++++ unitTests/resources/expirationWarning.test.js | 28 +++++++++++++ .../resources/expiresAtAttribute.test.js | 22 ++++++++++ 6 files changed, 128 insertions(+), 19 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 013ec6a417..65da661c41 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2961,7 +2961,7 @@ export function makeTable(options) { recordToStore, transaction && { transaction }, expiresAt, - priorStaged?.expiresAt ?? existingEntry?.expiresAt + effectiveExpirationIndexValue(priorStaged ?? existingEntry, existingRecord) ); writeCommit(true); @@ -3150,7 +3150,7 @@ export function makeTable(options) { null, transaction && { transaction }, undefined, - priorStaged?.expiresAt ?? existingEntry?.expiresAt + effectiveExpirationIndexValue(priorStaged ?? existingEntry, existingRecord) ); if (audit || trackDeletes) { updateRecord( @@ -5487,6 +5487,11 @@ export function makeTable(options) { } return hasChanges; } + function effectiveExpirationIndexValue(entry: Partial | undefined, record = entry?.value): number | undefined { + return entry?.expiresAt === undefined + ? expirationTimestamp(record?.[expiresAtProperty?.name]) + : expirationTimestamp(entry.expiresAt); + } function checkValidId(id) { switch (typeof id) { case 'number': @@ -6045,10 +6050,10 @@ export function makeTable(options) { if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord }; if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id; if ( + !missingSourceExpirationReported && sourceContext.expiresAt === undefined && expiresAtProperty && - expirationTimestamp(updatedRecord[expiresAtProperty.name]) !== undefined && - !missingSourceExpirationReported + expirationTimestamp(updatedRecord[expiresAtProperty.name]) !== undefined ) { missingSourceExpirationReported = true; logger.warn?.( @@ -6131,7 +6136,7 @@ export function makeTable(options) { updatedRecord, transaction && { transaction }, sourceContext.expiresAt, - existingEntry?.expiresAt + effectiveExpirationIndexValue(existingEntry, existingRecord) ); if (updatedRecord) { if (existingEntry) { @@ -6600,11 +6605,6 @@ export function makeTable(options) { function runRecordExpirationEviction(testHooks?: ExpirationSweepTestHooks, schedule = true) { if (getWorkerIndex() !== 0 || cleanupClosed) return; const expiresAtName = expiresAtProperty.name; - const indexedExpiration = (entry: Entry): number | undefined => - entry.expiresAt === undefined - ? expirationTimestamp(entry.value?.[expiresAtName]) - : expirationTimestamp(entry.expiresAt); - async function sweepRocks(index: any, cutoff: number) { const batcher = createEvictionBatcher(() => cleanupClosed, testHooks); let after: any[] | undefined; @@ -6631,7 +6631,7 @@ export function makeTable(options) { index, }); } else { - const currentExpiration = indexedExpiration(recordEntry); + const currentExpiration = effectiveExpirationIndexValue(recordEntry); if (currentExpiration !== undefined && currentExpiration < cutoff) { if (recordEntry.metadataFlags & HAS_BLOBS) { await testHooks?.beforeEvict?.(); @@ -6698,7 +6698,7 @@ export function makeTable(options) { if (recordEntry?.value == null) { operation = primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); } else { - const currentExpiration = indexedExpiration(recordEntry); + const currentExpiration = effectiveExpirationIndexValue(recordEntry); if (currentExpiration !== undefined && currentExpiration < cutoff) { await testHooks?.beforeEvict?.(); operation = TableResource.evict(id, recordEntry.value, recordEntry.version, currentExpiration); diff --git a/resources/databases.ts b/resources/databases.ts index 8425ef009c..fb685068ff 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -98,6 +98,7 @@ const DEFAULT_DATABASE_NAME = 'data'; const DEFINED_TABLES = Symbol('defined-tables'); const DEFAULT_DATABASE_CLOSE_TIMEOUT = 10_000; let databaseCloseTimeout = DEFAULT_DATABASE_CLOSE_TIMEOUT; +const unavailableDatabases = new Set(); export function setDatabaseCloseTimeoutForTests(timeout = DEFAULT_DATABASE_CLOSE_TIMEOUT): void { databaseCloseTimeout = timeout; @@ -1145,6 +1146,9 @@ export function resolveDatabasePath(databaseName: string): string { */ export function database({ database: databaseName, table: tableName }) { if (!databaseName) databaseName = DEFAULT_DATABASE_NAME; + if (unavailableDatabases.has(databaseName)) { + throw new Error(`Database ${databaseName} is closing and cannot be opened`); + } getDatabases(); ensureDB(databaseName); const definedDatabase = definedDatabases.get(databaseName); @@ -1241,8 +1245,15 @@ function lockDatabaseForDrop(dbPath: string, databaseName: string, held: Restore */ export async function dropDatabase(databaseName) { if (!databases[databaseName]) throw new Error('Database does not exist'); + if (unavailableDatabases.has(databaseName)) throw new Error(`Database ${databaseName} is already closing`); const dbTables = databases[databaseName]; let rootStore; + for (const tableName in dbTables) { + rootStore = dbTables[tableName]?.primaryStore?.rootStore; + if (rootStore) break; + } + if (!rootStore) rootStore = database({ database: databaseName, table: null }); + unavailableDatabases.add(databaseName); // Hold the per-database restore lock across the entire drop so its file deletion can never // interleave with a restore's purge-and-copy on the same directory — a destroy landing after a @@ -1309,7 +1320,6 @@ export async function dropDatabase(databaseName) { } } } else { - rootStore = database({ database: databaseName, table: null }); // a tableless database resolves its root store here rather than in the loop above, so take // the drop lock now (still before any destructive step) if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); @@ -1325,6 +1335,7 @@ export async function dropDatabase(databaseName) { await deleteRootBlobPathsForDB(rootStore); } finally { for (const lock of restoreLocks) releaseRestoreLock(lock); + unavailableDatabases.delete(databaseName); } } @@ -1340,7 +1351,14 @@ const closingDatabases = new Map>(); export function closeDatabase(databaseName: string): Promise { const activeClose = closingDatabases.get(databaseName); if (activeClose) return activeClose; - const completion = closeDatabaseOnce(databaseName).finally(() => closingDatabases.delete(databaseName)); + if (unavailableDatabases.has(databaseName)) { + return Promise.reject(new Error(`Database ${databaseName} is already closing`)); + } + unavailableDatabases.add(databaseName); + const completion = closeDatabaseOnce(databaseName).finally(() => { + closingDatabases.delete(databaseName); + unavailableDatabases.delete(databaseName); + }); closingDatabases.set(databaseName, completion); return completion; } diff --git a/resources/search.ts b/resources/search.ts index 4e57859097..264df89d1b 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -483,14 +483,15 @@ export function searchByIndex( if (isPrimaryKey) { const results = index.getRange(rangeOptions).map( filter - ? function ({ key, value }) { - if (this?.isSync) return value && filter(value) ? key : SKIP; + ? function (entry) { + const { key, value } = entry; + if (this?.isSync) return value && filter(value, entry) ? key : SKIP; // for filter operations, we intentionally yield the event turn so that scanning queries // do not hog resources return new Promise((resolve, reject) => setImmediate(() => { try { - resolve(value && filter(value) ? key : SKIP); + resolve(value && filter(value, entry) ? key : SKIP); } catch (error) { reject(error); } @@ -569,7 +570,7 @@ export function searchByIndex( const { key, value } = entry; if (this.isSync) { recordRead(entry); - return value && filter(value) ? key : SKIP; + return value && filter(value, entry) ? key : SKIP; } // for filter operations, we intentionally yield the event turn so that scanning queries // do not hog resources @@ -577,7 +578,7 @@ export function searchByIndex( setImmediate(() => { try { recordRead(entry); - resolve(value && filter(value) ? key : SKIP); + resolve(value && filter(value, entry) ? key : SKIP); } catch (error) { reject(error); } diff --git a/unitTests/resources/closeLoadedDatabases.test.js b/unitTests/resources/closeLoadedDatabases.test.js index 1d78450466..7a0c7c8231 100644 --- a/unitTests/resources/closeLoadedDatabases.test.js +++ b/unitTests/resources/closeLoadedDatabases.test.js @@ -106,6 +106,35 @@ describe('RocksDB handle release', function () { assert.strictEqual(refCountFor(rootStore.path), 0); }); + it('rejects a lazy table open while the database is closing', async function () { + const databaseName = 'closeopening6'; + const Table = table({ + table: 'existing', + database: databaseName, + attributes: [{ attribute: 'id', isPrimaryKey: true }], + }); + const originalCleanup = Table.cleanup; + let releaseCleanup; + Table.cleanup = () => new Promise((resolve) => (releaseCleanup = resolve)); + + const close = closeDatabase(databaseName); + try { + assert.throws( + () => + table({ + table: 'late', + database: databaseName, + attributes: [{ attribute: 'id', isPrimaryKey: true }], + }), + /Database .* is closing/ + ); + } finally { + Table.cleanup = originalCleanup; + releaseCleanup(); + } + await close; + }); + it('fails closed and resumes cleanup when close times out', async function () { this.timeout(30000); const Table = table({ @@ -163,6 +192,17 @@ describe('RocksDB handle release', function () { assert.strictEqual(Table.cleanupStateForTests().closed, true); }); + it('drops a tableless database while blocking a concurrent reopen', async function () { + const databaseName = 'droptableless7'; + const rootStore = database({ database: databaseName }); + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + + const drop = dropDatabase(databaseName); + assert.throws(() => database({ database: databaseName }), /Database .* is closing/); + await drop; + assert.strictEqual(refCountFor(rootStore.path), 0); + }); + it('fails closed and resumes cleanup when drop times out', async function () { this.timeout(30000); const Table = table({ diff --git a/unitTests/resources/expirationWarning.test.js b/unitTests/resources/expirationWarning.test.js index fbffa09629..372eebc624 100644 --- a/unitTests/resources/expirationWarning.test.js +++ b/unitTests/resources/expirationWarning.test.js @@ -132,4 +132,32 @@ describe('Per-record expiresAt without scheduled cleanup (#1339)', () => { await new Promise((resolve) => setTimeout(resolve, 20)); assert.strictEqual(sourceWarningsFor(tableName).length, 1); }); + + it('uses a table expiration without warning about a returned @expiresAt field', async function () { + const tableName = 'SourcedTableExpirationFallback'; + const SourcedTable = table({ + table: tableName, + database: 'test', + expiration: 60, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + SourcedTable.sourcedFrom( + class extends Resource { + get() { + return { id: this.getId(), expiresAt: Date.now() + 120000 }; + } + } + ); + + const before = Date.now() + 60000; + await SourcedTable.get(1); + await waitFor(() => SourcedTable.primaryStore.getEntry(1)?.expiresAt !== undefined); + const storedExpiration = SourcedTable.primaryStore.getEntry(1).expiresAt; + assert.ok(storedExpiration >= before); + assert.deepStrictEqual([...SourcedTable.indices.expiresAt.getValues(storedExpiration)], [1]); + assert.strictEqual(sourceWarningsFor(tableName).length, 0); + }); }); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 63561692db..fd49d9b325 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -101,6 +101,14 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } assert.strictEqual(matchingOverride.length, 1); assert.strictEqual(matchingOverride[0].expiresAt, fieldExpiresAt); + const fullScanMismatch = []; + for await (const record of Table.search({ + allowFullScan: true, + conditions: [{ attribute: 'expiresAt', comparator: 'ne', value: optionExpiresAt }], + })) { + fullScanMismatch.push(record); + } + assert.deepStrictEqual(fullScanMismatch, []); }); it('keeps the field value across a patch that does not touch it', async function () { @@ -191,6 +199,20 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); + it('removes a legacy field-derived index entry when the row is updated', async function () { + const Table = makeTable('ExpiresAtLegacyUpdate'); + const oldExpiresAt = Date.now() + 3_600_000; + const newExpiresAt = oldExpiresAt + 3_600_000; + Table.primaryStore.putSync(1, { id: 1, expiresAt: new Date(oldExpiresAt).toISOString() }); + Table.indices.expiresAt.put(oldExpiresAt, 1); + + await Table.put(1, { id: 1, expiresAt: newExpiresAt }); + await Table.primaryStore.committed; + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(oldExpiresAt)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(newExpiresAt)], [1]); + Table.cleanup(); + }); + it('uses source context rather than the returned field for cache expiration', async function () { const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSourceFill', 3_600)); const contextExpiresAt = Date.now() - 1_000; From de8cb4a6060c6e9b234463880fa530f3eed9fc6f Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 21:34:06 -0600 Subject: [PATCH 23/43] Make expiration teardown atomic --- resources/Table.ts | 20 +++++----- resources/databases.ts | 40 ++++++++++--------- resources/search.ts | 9 +++-- .../resources/closeLoadedDatabases.test.js | 17 ++++++++ 4 files changed, 54 insertions(+), 32 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 65da661c41..23960ba759 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2100,11 +2100,10 @@ export function makeTable(options) { // removed the entry entirely, but first cleanup indices let lmdbCompletion: MaybePromise; if (primaryStore.ifVersion) { - // lmdb: the index cleanup and the record removal are both version-guarded optimistic writes. - // Capture both promises so a real write failure on either resolves through evict()'s catch - // below rather than escaping as an unhandled rejection from the fire-and-forget callers. + // LMDB batches every write in this callback behind one version check, so the indices and + // primary record can not diverge on a conflict or partial commit. const removalEntry = entry ?? primaryStore.getEntry(id); - const indexCleanup = primaryStore.ifVersion(id, existingVersion, () => { + lmdbCompletion = primaryStore.ifVersion(id, existingVersion, () => { updateIndices( id, existingRecord, @@ -2113,9 +2112,8 @@ export function makeTable(options) { undefined, existingExpiresAtIndexValue ?? removalEntry?.expiresAt ); + return removeEntry(primaryStore, removalEntry); }); - const removal = removeEntry(primaryStore, removalEntry, existingVersion); - lmdbCompletion = Promise.all([indexCleanup, removal]); } else { const removalEntry = entry ?? currentEntry ?? primaryStore.getEntry(id); if (!removalEntry || removalEntry.version !== existingVersion) return; @@ -5488,8 +5486,9 @@ export function makeTable(options) { return hasChanges; } function effectiveExpirationIndexValue(entry: Partial | undefined, record = entry?.value): number | undefined { + if (!expiresAtProperty) return entry?.expiresAt; return entry?.expiresAt === undefined - ? expirationTimestamp(record?.[expiresAtProperty?.name]) + ? expirationTimestamp(record?.[expiresAtProperty.name]) : expirationTimestamp(entry.expiresAt); } function checkValidId(id) { @@ -6677,6 +6676,7 @@ export function makeTable(options) { async function sweepLmdb(index: any, cutoff: number) { const operationTracker = createCleanupOperationTracker(); + let entriesSinceYield = 0; for (const key of index.getRange({ start: true, values: false, end: cutoff, snapshot: false })) { if (cleanupClosed) break; let afterId: any; @@ -6712,9 +6712,11 @@ export function makeTable(options) { } const backpressure = operationTracker.add(operation); if (backpressure) await backpressure; + if (++entriesSinceYield >= 10) { + entriesSinceYield = 0; + await rest(); + } } - await operationTracker.drain(); - await rest(); if (ids.length < EVICTION_BATCH_SIZE) break; } await rest(); diff --git a/resources/databases.ts b/resources/databases.ts index fb685068ff..2e7666a8dd 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1243,7 +1243,11 @@ function lockDatabaseForDrop(dbPath: string, databaseName: string, held: Restore * Delete the database * @param databaseName */ -export async function dropDatabase(databaseName) { +const droppingDatabases = new Map>(); + +export function dropDatabase(databaseName): Promise { + const activeDrop = droppingDatabases.get(databaseName); + if (activeDrop) return activeDrop; if (!databases[databaseName]) throw new Error('Database does not exist'); if (unavailableDatabases.has(databaseName)) throw new Error(`Database ${databaseName} is already closing`); const dbTables = databases[databaseName]; @@ -1254,7 +1258,15 @@ export async function dropDatabase(databaseName) { } if (!rootStore) rootStore = database({ database: databaseName, table: null }); unavailableDatabases.add(databaseName); + const completion = dropDatabaseOnce(databaseName, dbTables, rootStore).finally(() => { + droppingDatabases.delete(databaseName); + unavailableDatabases.delete(databaseName); + }); + droppingDatabases.set(databaseName, completion); + return completion; +} +async function dropDatabaseOnce(databaseName, dbTables, rootStore): Promise { // Hold the per-database restore lock across the entire drop so its file deletion can never // interleave with a restore's purge-and-copy on the same directory — a destroy landing after a // restore's copy would gut a "successful" restore, and vice versa. Restore takes the same lock @@ -1264,9 +1276,11 @@ export async function dropDatabase(databaseName) { try { for (const tableName in dbTables) { const table = dbTables[tableName]; - rootStore = table.primaryStore.rootStore; - if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); + const tableRootStore = table.primaryStore.rootStore; + if (tableRootStore instanceof RocksDatabase) lockDatabaseForDrop(tableRootStore.path, databaseName, restoreLocks); } + if (restoreLocks.length === 0 && rootStore instanceof RocksDatabase) + lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); const cleanupCompletions: Promise[] = []; for (const tableName in dbTables) { @@ -1309,24 +1323,11 @@ export async function dropDatabase(databaseName) { databaseEventsEmitter.emit('dropDatabase', databaseName); - if (rootStore) { - if (rootStore.status === 'open') { - if (rootStore instanceof RocksDatabase) { - rootStore.close(); - rootStore.destroy(); - } else { - await rootStore.close(); - await unlink(rootStore.path); - } - } - } else { - // a tableless database resolves its root store here rather than in the loop above, so take - // the drop lock now (still before any destructive step) - if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); + if (rootStore.status === 'open') { if (rootStore instanceof RocksDatabase) { rootStore.close(); rootStore.destroy(); - } else if (rootStore.status === 'open') { + } else { await rootStore.close(); await unlink(rootStore.path); } @@ -1335,7 +1336,6 @@ export async function dropDatabase(databaseName) { await deleteRootBlobPathsForDB(rootStore); } finally { for (const lock of restoreLocks) releaseRestoreLock(lock); - unavailableDatabases.delete(databaseName); } } @@ -1351,6 +1351,8 @@ const closingDatabases = new Map>(); export function closeDatabase(databaseName: string): Promise { const activeClose = closingDatabases.get(databaseName); if (activeClose) return activeClose; + const activeDrop = droppingDatabases.get(databaseName); + if (activeDrop) return activeDrop.then(() => true); if (unavailableDatabases.has(databaseName)) { return Promise.reject(new Error(`Database ${databaseName} is already closing`)); } diff --git a/resources/search.ts b/resources/search.ts index 264df89d1b..9e8cfa77a2 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -634,10 +634,11 @@ function joinTo(rightIterable, attribute, store, isManyToMany, joined: Map !filter(record))) continue; + if ((joined as any).filters?.some((filter) => !filter(record, storedEntry))) continue; if (isManyToMany) { for (let i = 0; i < leftKey.length; i++) { addEntry(leftKey[i], entry); @@ -714,8 +715,8 @@ function joinFrom(rightIterable, attribute, store, joined: Map, sear for (const id of rightIterable) { if ((joined as any).filters) { // if additional filters are defined, we need to check them - const record = store.getSync(id); - if ((joined as any).filters.some((filter) => !filter(record))) continue; + const entry = store.getEntry(id); + if ((joined as any).filters.some((filter) => !filter(entry?.value, entry))) continue; } ids.add(id); // TODO: Re-enable this when async iteration is used, and do so with manually iterating so that we don't need to do an await on every iteration diff --git a/unitTests/resources/closeLoadedDatabases.test.js b/unitTests/resources/closeLoadedDatabases.test.js index 7a0c7c8231..06a762dacd 100644 --- a/unitTests/resources/closeLoadedDatabases.test.js +++ b/unitTests/resources/closeLoadedDatabases.test.js @@ -20,6 +20,7 @@ const { } = require('#src/resources/databases'); const { registryStatus, RocksDatabase } = require('@harperfast/rocksdb-js'); const { createBlob } = require('#src/resources/blob'); +const { acquireRestoreLock, releaseRestoreLock } = require('#src/dataLayer/restoreMarker'); const { waitFor } = require('../waitFor.js'); describe('RocksDB handle release', function () { @@ -199,10 +200,26 @@ describe('RocksDB handle release', function () { const drop = dropDatabase(databaseName); assert.throws(() => database({ database: databaseName }), /Database .* is closing/); + const close = closeDatabase(databaseName); await drop; + assert.strictEqual(await close, true); assert.strictEqual(refCountFor(rootStore.path), 0); }); + it('refuses to drop a tableless database while its restore lock is held', async function () { + const databaseName = 'droplockedtableless8'; + const rootStore = database({ database: databaseName }); + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const restoreLock = acquireRestoreLock(rootStore.path); + try { + await assert.rejects(dropDatabase(databaseName)); + assert.strictEqual(rootStore.status, 'open'); + } finally { + releaseRestoreLock(restoreLock); + } + await dropDatabase(databaseName); + }); + it('fails closed and resumes cleanup when drop times out', async function () { this.timeout(30000); const Table = table({ From a741492a03e54506c2becc2176dfbecc7eadb1a7 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 21:45:08 -0600 Subject: [PATCH 24/43] Finish expiration lifecycle safeguards --- resources/Table.ts | 25 ++++++++++++++++--------- resources/databases.ts | 21 +++++++++++++-------- resources/search.ts | 16 +++++++++++----- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 23960ba759..6a0e4524a1 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -1426,15 +1426,20 @@ export function makeTable(options) { // next startup (or on a same-name create). delete databases[databaseName][tableName]; const cleanupCompletion = TableResource.cleanup(); - let cleanupTimer: NodeJS.Timeout; + let cleanupTimer: NodeJS.Timeout | undefined; const cleanupTimedOut = Symbol('cleanupTimedOut'); - const cleanupResult = await Promise.race([ - cleanupCompletion, - new Promise((resolve) => { - cleanupTimer = setTimeout(() => resolve(cleanupTimedOut), LOCK_TIMEOUT); - }), - ]); - clearTimeout(cleanupTimer!); + let cleanupResult: void | typeof cleanupTimedOut; + try { + cleanupResult = await Promise.race([ + cleanupCompletion, + new Promise((resolve) => { + cleanupTimer = setTimeout(() => resolve(cleanupTimedOut), LOCK_TIMEOUT); + cleanupTimer.unref(); + }), + ]); + } finally { + if (cleanupTimer) clearTimeout(cleanupTimer); + } if (cleanupResult === cleanupTimedOut) { const message = `dropTable() timed out waiting for cleanup on ${tableName}; the table is unloaded and its drop will complete on restart or same-name create.`; logger.warn?.(message); @@ -2103,6 +2108,7 @@ export function makeTable(options) { // LMDB batches every write in this callback behind one version check, so the indices and // primary record can not diverge on a conflict or partial commit. const removalEntry = entry ?? primaryStore.getEntry(id); + let removal: MaybePromise; lmdbCompletion = primaryStore.ifVersion(id, existingVersion, () => { updateIndices( id, @@ -2112,8 +2118,9 @@ export function makeTable(options) { undefined, existingExpiresAtIndexValue ?? removalEntry?.expiresAt ); - return removeEntry(primaryStore, removalEntry); + removal = removeEntry(primaryStore, removalEntry); }); + lmdbCompletion = Promise.all([lmdbCompletion, removal]); } else { const removalEntry = entry ?? currentEntry ?? primaryStore.getEntry(id); if (!removalEntry || removalEntry.version !== existingVersion) return; diff --git a/resources/databases.ts b/resources/databases.ts index 2e7666a8dd..41e59e53cb 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -364,6 +364,15 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo const lmdbDatabaseEnvs = new Map(); const rocksdbDatabaseEnvs = new Map(); +function forgetDatabaseEnvironment(rootStore: any): void { + for (const [path, store] of lmdbDatabaseEnvs) { + if (store === rootStore || path === rootStore.path) lmdbDatabaseEnvs.delete(path); + } + for (const [path, store] of rocksdbDatabaseEnvs) { + if (store === rootStore || path === rootStore.path) rocksdbDatabaseEnvs.delete(path); + } +} + // set the following in both global and exports _assignPackageExport('databases', databases); _assignPackageExport('tables', tables); @@ -1245,7 +1254,7 @@ function lockDatabaseForDrop(dbPath: string, databaseName: string, held: Restore */ const droppingDatabases = new Map>(); -export function dropDatabase(databaseName): Promise { +export async function dropDatabase(databaseName): Promise { const activeDrop = droppingDatabases.get(databaseName); if (activeDrop) return activeDrop; if (!databases[databaseName]) throw new Error('Database does not exist'); @@ -1303,11 +1312,7 @@ async function dropDatabaseOnce(databaseName, dbTables, rootStore): Promise { for (const rootStore of rootStores) { closeStore(rootStore.dbisDb, 'attributes store'); closeStore(rootStore, 'root store'); - lmdbDatabaseEnvs.delete(rootStore.path); - rocksdbDatabaseEnvs.delete(rootStore.path); + forgetDatabaseEnvironment(rootStore); } const definedDatabase = definedDatabases?.get(databaseName); if (definedDatabase) (definedDatabase as any).rootStore = undefined; diff --git a/resources/search.ts b/resources/search.ts index 9e8cfa77a2..be2603e1f9 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -634,11 +634,16 @@ function joinTo(rightIterable, attribute, store, isManyToMany, joined: Map !filter(record, storedEntry))) continue; + if (filters?.some((filter) => !filter(record, storedEntry))) continue; if (isManyToMany) { for (let i = 0; i < leftKey.length; i++) { addEntry(leftKey[i], entry); @@ -712,10 +717,11 @@ function joinFrom(rightIterable, attribute, store, joined: Map, sear }; //let i = 0; // get all the ids of the related records - for (const id of rightIterable) { + for (const idOrEntry of rightIterable) { + const id = idOrEntry?.key ?? idOrEntry; if ((joined as any).filters) { // if additional filters are defined, we need to check them - const entry = store.getEntry(id); + const entry = idOrEntry?.value !== undefined ? idOrEntry : store.getEntry(id); if ((joined as any).filters.some((filter) => !filter(entry?.value, entry))) continue; } ids.add(id); From 745e7a2d6319df1a659c3563898b5923bb1b0205 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 14 Aug 2026 21:49:02 -0600 Subject: [PATCH 25/43] Preserve multi-root teardown contracts --- resources/databases.ts | 4 +++- resources/search.ts | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index 41e59e53cb..41924b43dc 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1313,6 +1313,9 @@ async function dropDatabaseOnce(databaseName, dbTables, rootStore): Promise !filter(record, storedEntry))) continue; From f004866658cd8e422be4cb0976154420aa485900 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 14:24:47 -0600 Subject: [PATCH 26/43] Harden expiration eviction lifecycle --- dataLayer/harperBridge/ResourceBridge.ts | 1 - .../database/expiration-drop-quiesce.test.ts | 100 +++ .../expiration-drop-quiesce/config.yaml | 5 + .../expiration-drop-quiesce/resources.js | 31 + .../expiration-drop-quiesce/schema.graphql | 9 + resources/DatabaseTransaction.ts | 98 +++ resources/LMDBTransaction.ts | 5 + resources/RecordEncoder.ts | 6 + resources/Table.ts | 783 +++++++++++------- resources/auditStore.ts | 1 + resources/databases.ts | 381 +++++++-- resources/replayLogs.ts | 11 +- resources/search.ts | 13 +- .../certificateVerificationSource.ts | 1 + .../crlVerification.ts | 2 + server/itc/serverHandlers.js | 176 +++- server/jobs/jobRunner.ts | 2 + server/threads/itc.js | 33 +- server/threads/manageThreads.js | 116 ++- storage-format.md | 24 +- .../resources/auditEntryRecordFlags.test.js | 21 +- .../resources/closeLoadedDatabases.test.js | 71 ++ unitTests/resources/databases.test.js | 202 ++++- unitTests/resources/dropTableGhost.test.js | 10 +- unitTests/resources/expirationWarning.test.js | 4 +- .../resources/expiresAtAttribute.test.js | 279 ++++++- unitTests/resources/recordEncoder.test.js | 22 +- unitTests/resources/transaction.test.js | 33 +- .../certificateVerificationSource.test.js | 6 +- unitTests/server/itc/serverHandlers.test.js | 241 ++++++ .../server/threads/acknowledgement-fixture.js | 14 + .../threads/broadcastAcknowledgement.test.js | 151 ++++ unitTests/utility/signalling.test.js | 119 +++ utility/signalling.ts | 186 +++++ 34 files changed, 2698 insertions(+), 459 deletions(-) create mode 100644 integrationTests/database/expiration-drop-quiesce.test.ts create mode 100644 integrationTests/database/expiration-drop-quiesce/config.yaml create mode 100644 integrationTests/database/expiration-drop-quiesce/resources.js create mode 100644 integrationTests/database/expiration-drop-quiesce/schema.graphql create mode 100644 unitTests/server/threads/acknowledgement-fixture.js create mode 100644 unitTests/server/threads/broadcastAcknowledgement.test.js diff --git a/dataLayer/harperBridge/ResourceBridge.ts b/dataLayer/harperBridge/ResourceBridge.ts index 847ac6e196..046c7899ad 100644 --- a/dataLayer/harperBridge/ResourceBridge.ts +++ b/dataLayer/harperBridge/ResourceBridge.ts @@ -186,7 +186,6 @@ export class ResourceBridge extends BridgeMethods { async dropSchema(dropSchemaObj) { await dropDatabase(dropSchemaObj.schema); - signalling.signalSchemaChange(new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, dropSchemaObj.schema)); } async updateRecords(updateObj) { diff --git a/integrationTests/database/expiration-drop-quiesce.test.ts b/integrationTests/database/expiration-drop-quiesce.test.ts new file mode 100644 index 0000000000..6ed94f6e05 --- /dev/null +++ b/integrationTests/database/expiration-drop-quiesce.test.ts @@ -0,0 +1,100 @@ +import { after, before, suite, test } from 'node:test'; +import { strictEqual } from 'node:assert'; +import { existsSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; +import request from 'supertest'; +import { createApiClient } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'expiration-drop-quiesce'); +const CONTROL_DIRECTORY = mkdtempSync(join(tmpdir(), 'expiration-drop-quiesce-')); + +async function waitFor(predicate: () => boolean, message: string) { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + if (predicate()) return; + await delay(20); + } + throw new Error(message); +} + +suite('cross-worker expiration cleanup quiesces destructive DDL', (ctx: ContextWithHarper) => { + let client: ReturnType; + + before(async () => { + await setupHarperWithFixture(ctx, FIXTURE_PATH, { + config: { threads: { count: 2 }, logging: { level: 'error' } }, + env: { EXPIRATION_QUIESCE_CONTROL: CONTROL_DIRECTORY }, + } as any); + client = createApiClient(ctx.harper); + let ready = false; + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + try { + const response = await request(client.restURL) + .post('/QuiesceControl/') + .set(client.headers) + .timeout(2_000) + .send({ action: 'probe' }); + if (response.status !== 404) { + ready = true; + break; + } + } catch { + // Workers are still loading the pre-installed component. + } + await delay(250); + } + strictEqual(ready, true, 'QuiesceControl resource did not become ready'); + }); + + after(async () => { + await teardownHarper(ctx); + }); + + async function postControl(body: Record) { + return request(client.restURL).post('/QuiesceControl/').set(client.headers).timeout(120_000).send(body); + } + + async function proveDropWaitsForWorker(kind: 'indexed' | 'primary') { + const indexed = kind === 'indexed'; + const database = indexed ? 'quiesce_indexed' : 'quiesce_primary'; + const table = indexed ? 'IndexedExpiry' : 'PrimaryExpiry'; + const runId = `${kind}-${Date.now()}`; + await postControl({ action: 'seed', kind, database, table, id: runId }).then((response) => + strictEqual(response.status, 200) + ); + const started = join(CONTROL_DIRECTORY, `${runId}.started`); + const release = join(CONTROL_DIRECTORY, `${runId}.release`); + const sweep = postControl({ action: 'sweep', kind, database, table, runId }); + await waitFor(() => existsSync(started), `${kind} sweep did not reach its blocked commit`); + + let dropSettled = false; + const drop = client + .req() + .timeout(120_000) + .send({ operation: 'drop_table', schema: database, table }) + .then((response) => { + dropSettled = true; + return response; + }); + const described = await client.req().timeout(10_000).send({ operation: 'describe_table', schema: database, table }); + strictEqual(described.status, 200, 'the table must remain visible before cross-worker quiescence completes'); + strictEqual(dropSettled, false, 'physical drop must wait for the blocked cleanup worker'); + + writeFileSync(release, 'release'); + const dropped = await drop; + strictEqual(dropped.status, 200); + await sweep; + } + + test('drop_table waits for a remote indexed expiration sweep', async () => { + await proveDropWaitsForWorker('indexed'); + }); + + test('drop_table waits for a remote primary cleanup scan', async () => { + await proveDropWaitsForWorker('primary'); + }); +}); diff --git a/integrationTests/database/expiration-drop-quiesce/config.yaml b/integrationTests/database/expiration-drop-quiesce/config.yaml new file mode 100644 index 0000000000..efffc0833f --- /dev/null +++ b/integrationTests/database/expiration-drop-quiesce/config.yaml @@ -0,0 +1,5 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true diff --git a/integrationTests/database/expiration-drop-quiesce/resources.js b/integrationTests/database/expiration-drop-quiesce/resources.js new file mode 100644 index 0000000000..7851bab9d7 --- /dev/null +++ b/integrationTests/database/expiration-drop-quiesce/resources.js @@ -0,0 +1,31 @@ +import { existsSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; + +const controlDirectory = process.env.EXPIRATION_QUIESCE_CONTROL; + +export class QuiesceControl extends Resource { + static loadAsInstance = false; + + async post(_query, body) { + const Table = databases[body.database]?.[body.table]; + if (!Table) return { available: false }; + if (body.action === 'seed') { + const expiresAt = Date.now() - 10_000; + if (body.kind === 'indexed') await Table.put(body.id, { id: body.id, expiresAt }); + else await Table.put(body.id, { id: body.id, value: 'expired' }, { expiresAt }); + return { seeded: true }; + } + const started = join(controlDirectory, `${body.runId}.started`); + const release = join(controlDirectory, `${body.runId}.release`); + const hooks = { + beforeBatchCommit: async () => { + writeFileSync(started, 'started'); + while (!existsSync(release)) await delay(20); + }, + }; + if (body.kind === 'indexed') await Table.runRecordExpirationSweepForTests(hooks); + else await Table.runPrimaryCleanupScanForTests(hooks); + return { completed: true }; + } +} diff --git a/integrationTests/database/expiration-drop-quiesce/schema.graphql b/integrationTests/database/expiration-drop-quiesce/schema.graphql new file mode 100644 index 0000000000..becff7fccf --- /dev/null +++ b/integrationTests/database/expiration-drop-quiesce/schema.graphql @@ -0,0 +1,9 @@ +type IndexedExpiry @table(database: "quiesce_indexed") @export { + id: ID! @primaryKey + expiresAt: Float @expiresAt +} + +type PrimaryExpiry @table(database: "quiesce_primary", expiration: 1, eviction: 1) @export { + id: ID! @primaryKey + value: String +} diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 92bf948015..582cac7938 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -239,6 +239,100 @@ export type CommitOptions = { transaction?: RocksTransaction; }; +export const TABLE_COMMIT_ADMISSION = Symbol.for('harper.table-commit-admission'); +const ACTIVE_TABLE_COMMIT_ADMISSION = Symbol('active-table-commit-admission'); + +type CommitAdmissionContext = { + stores: any[]; + releases: Array<() => unknown>; + transactions: DatabaseTransaction[]; +}; + +function admitTransactionStores(transaction: DatabaseTransaction, context: CommitAdmissionContext): void { + for (let link: DatabaseTransaction = transaction; link; link = link.next) { + if ((link as any)[ACTIVE_TABLE_COMMIT_ADMISSION] !== context) { + (link as any)[ACTIVE_TABLE_COMMIT_ADMISSION] = context; + context.transactions.push(link); + } + for (const write of link.writes) { + const store = write?.store; + if (!store || context.stores.includes(store)) continue; + const admit = store[TABLE_COMMIT_ADMISSION]; + if (admit) { + context.stores.push(store); + context.releases.push(admit()); + } + } + } +} + +function releaseTransactionStores(context: CommitAdmissionContext): Promise | void { + for (const transaction of context.transactions) { + if ((transaction as any)[ACTIVE_TABLE_COMMIT_ADMISSION] === context) + delete (transaction as any)[ACTIVE_TABLE_COMMIT_ADMISSION]; + } + let completions: Promise[] | undefined; + for (const release of context.releases) { + try { + const completion = release(); + if ((completion as any)?.then) (completions ??= []).push(Promise.resolve(completion)); + } catch (error) { + (completions ??= []).push(Promise.reject(error)); + } + } + context.stores.length = 0; + context.releases.length = 0; + context.transactions.length = 0; + if (completions) return Promise.allSettled(completions).then(() => undefined); +} + +/** + * Hold every table touched by a transaction chain in the submitted-commit set until the complete + * commit (including retries and chained stores) settles. Recursive commit rounds reuse the same + * context and admit any stores added while asynchronous pre-commit work was running. + */ +export function withTableCommitAdmission( + transaction: DatabaseTransaction, + options: CommitOptions, + commit: (options: CommitOptions) => T +): T { + let context = (transaction as any)[ACTIVE_TABLE_COMMIT_ADMISSION] as CommitAdmissionContext | undefined; + if (context) { + admitTransactionStores(transaction, context); + return commit(options); + } + context = { stores: [], releases: [], transactions: [] }; + try { + admitTransactionStores(transaction, context); + const resolution: any = commit(options); + if (resolution?.then) { + return resolution.then( + (value) => { + const release = releaseTransactionStores(context); + return release ? release.then(() => value) : value; + }, + (error) => { + const release = releaseTransactionStores(context); + if (release) + return release.then(() => { + throw error; + }); + throw error; + } + ) as T; + } + const release = releaseTransactionStores(context); + return (release ? release.then(() => resolution) : resolution) as T; + } catch (error) { + const release = releaseTransactionStores(context); + if (release) + return release.then(() => { + throw error; + }) as T; + throw error; + } +} + type ReadTransaction = (LMDBTransaction | RocksTransaction) & { openTimer?: number; retryRisk?: number; @@ -663,6 +757,10 @@ export class DatabaseTransaction implements Transaction { * Resolves with information on the timestamp and success of the commit */ commit(options: CommitOptions = {}): MaybePromise { + return withTableCommitAdmission(this, options, (admittedOptions) => this.commitRocksAdmitted(admittedOptions)); + } + + private commitRocksAdmitted(options: CommitOptions): MaybePromise { if (this.timedOut) throw transactionOpenTooLongError(); // reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain — // but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction diff --git a/resources/LMDBTransaction.ts b/resources/LMDBTransaction.ts index 7b978f4c52..fab431bdbd 100644 --- a/resources/LMDBTransaction.ts +++ b/resources/LMDBTransaction.ts @@ -1,5 +1,6 @@ import { DatabaseTransaction, + withTableCommitAdmission, transactionOpenTooLongError, type CommitOptions, type TransactionWrite, @@ -119,6 +120,10 @@ export class LMDBTransaction extends DatabaseTransaction { * Resolves with information on the timestamp and success of the commit */ commit(options: CommitOptions = {}): any { + return withTableCommitAdmission(this, options, (admittedOptions) => this.commitLmdbAdmitted(admittedOptions)); + } + + private commitLmdbAdmitted(options: CommitOptions): any { if (this.timedOut) throw transactionOpenTooLongError(); options = options || {}; let txnTime = this.timestamp; diff --git a/resources/RecordEncoder.ts b/resources/RecordEncoder.ts index a229f83cb9..f6113d8af0 100644 --- a/resources/RecordEncoder.ts +++ b/resources/RecordEncoder.ts @@ -16,6 +16,7 @@ import { ACTION_32_BIT, HAS_ADDITIONAL_AUDIT_REFS as HAS_ADDITIONAL_AUDIT_REFS_AUDIT, LOCAL_ONLY, + HAS_EXPIRATION_DECISION, } from './auditStore.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; @@ -728,6 +729,10 @@ export function recordUpdater(store, tableId, auditStore) { : TIMESTAMP_ASSIGN_NEW | 0x4000 // or just assign a new one : NO_TIMESTAMP; const expiresAt = options?.expiresAt; + if (options?.expirationDecisionPresent) { + if (assignMetadata < 0) assignMetadata = 0; + assignMetadata |= HAS_EXPIRATION_DECISION; + } if (expiresAt >= 0) assignMetadata |= HAS_EXPIRATION; metadataInNextEncoding = assignMetadata; expiresAtNextEncoding = expiresAt; @@ -777,6 +782,7 @@ export function recordUpdater(store, tableId, auditStore) { if (!previousResidencyId) previousResidencyId = 0; } if (assignMetadata & HAS_EXPIRATION) extendedType |= HAS_EXPIRATION_EXTENDED_TYPE; // we need to record the expiration in the audit log + if (assignMetadata & HAS_EXPIRATION_DECISION) extendedType |= HAS_EXPIRATION_DECISION; if (options?.originatingOperation) extendedType |= HAS_ORIGINATING_OPERATION; // we use resolveRecord outside of transaction, so must explicitly make it conditional if (resolveRecord) putOptions.ifVersion = ifVersion = existingEntry?.version ?? null; diff --git a/resources/Table.ts b/resources/Table.ts index 6a0e4524a1..f0c16aa094 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -36,6 +36,7 @@ import { DatabaseTransaction, ImmediateTransaction, priorStagedWrite, + TABLE_COMMIT_ADMISSION, TRANSACTION_STATE, } from './DatabaseTransaction.ts'; import * as envMngr from '../utility/environment/environmentManager.ts'; @@ -66,7 +67,7 @@ import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericT import { transaction, contextStorage } from './transaction.ts'; import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary'; import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; -import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts'; +import { HAS_BLOBS, HAS_EXPIRATION_DECISION, auditRetention, removeAuditEntry } from './auditStore.ts'; import { buildEmbedBefore, createDefaultEmbedder, type EmbedAttribute, type Embedder } from './models/embedHook.ts'; import { autoCast, autoCastBooleanStrict } from '../utility/common_utils.ts'; import { recordUpdater, removeEntry, PENDING_LOCAL_TIME, RecordObject, type Entry, entryMap } from './RecordEncoder.ts'; @@ -135,6 +136,7 @@ const EVICTION_BATCH_SIZE = 100; // letting an unbounded number of open transactions (and their snapshots) accumulate. const MAX_INFLIGHT_EVICTION_BATCHES = 4; const MAX_CLEANUP_CONCURRENCY = 50; +const MIN_SCHEDULED_CLEANUP_INTERVAL = 1000; const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]); envMngr.initSync(); const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); @@ -159,6 +161,17 @@ export function expirationTimestamp(value: any): number | undefined { } return Number.isFinite(timestamp) && timestamp >= 0 ? timestamp : undefined; } + +export function effectiveExpirationTimestamp( + entry: Pick, 'expiresAt' | 'metadataFlags'> | undefined, + record: any, + expiresAtAttributeName: string | undefined +): number | undefined { + if (!expiresAtAttributeName) return entry?.expiresAt; + if (entry?.expiresAt !== undefined) return expirationTimestamp(entry.expiresAt); + if ((entry?.metadataFlags ?? 0) & HAS_EXPIRATION_DECISION) return; + return expirationTimestamp(record?.[expiresAtAttributeName]); +} // A frozen record we may need to copy-on-mutate before stamping it (records are immutable — decoded // records are frozen and 5.2 record caching relies on it). Only plain/record objects qualify: never // a Buffer/typed-array (spreading would corrupt the binary into a {0:.., 1:..} object) or a primitive @@ -421,6 +434,9 @@ export function makeTable(options) { // in-flight commit promises here so dropTable() can drain them first, and stop admitting // new ones (droppingTable) once a drop has actually started. const pendingSourceCommits = new Set>(); + let pendingTableCommitCount = 0; + let pendingTableCommitWaiter: Promise | undefined; + let resolvePendingTableCommits: (() => void) | undefined; let droppingTable = false; let createdTimeProperty: Attribute | undefined, updatedTimeProperty: Attribute | undefined, @@ -441,8 +457,30 @@ export function makeTable(options) { let cleanupPriority = 0; let lastCleanupInterval: number; let cleanupTimer: NodeJS.Timeout | undefined; + let cleanupNextScheduled: number | undefined; + let cleanupGeneration = 0; let cleanupClosed = false; let cleanupWasScheduled = false; + let dropQuiescing = false; + (primaryStore as any)[TABLE_COMMIT_ADMISSION] = () => { + assertTableWritable(); + pendingTableCommitCount++; + let released = false; + return () => { + if (released) return; + released = true; + if (--pendingTableCommitCount === 0 && resolvePendingTableCommits) { + const resolve = resolvePendingTableCommits; + pendingTableCommitWaiter = undefined; + resolvePendingTableCommits = undefined; + resolve(); + } + }; + }; + function waitForPendingTableCommits(): Promise | undefined { + if (pendingTableCommitCount === 0) return; + return (pendingTableCommitWaiter ??= new Promise((resolve) => (resolvePendingTableCommits = resolve))); + } let recordExpirationInterval: NodeJS.Timeout | undefined; let missingExpirationIndexReported = false; let missingSourceExpirationReported = false; @@ -701,6 +739,7 @@ export function makeTable(options) { viaNodeId: event.viaNodeId, // use per-event expiresAt: batched txn context only holds the first event's expiration expiresAt: event.expiresAt, + expirationDecisionPresent: event.expirationDecisionPresent, // bulk base-copy snapshot frame: apply current-state directly, without an audit/transaction-log // entry or out-of-order resequencing (harper-pro#480). Only set for copy frames (between // COPY_START and COPY_COMPLETE); post-copy audit-replay frames apply normally. @@ -1386,182 +1425,149 @@ export function makeTable(options) { } static async dropTable() { - if (databaseName === databasePath) { - // Persist a drop tombstone on the primary catalog entry BEFORE any - // destructive work. If the process dies or a column family drop fails - // partway through, the tombstone survives with the catalog rows, and - // the next startup (or a same-name create) completes the drop via - // completeInterruptedDrop in databases.ts instead of resurrecting - // the table. - const primaryCatalogKey = TableResource.tableName + '/'; - const primaryMeta = (dbisDb as any).getSync(primaryCatalogKey); - if (primaryMeta && !primaryMeta.dropping) { - primaryMeta.dropping = true; - // Stamps this drop's identity so the interrupted-drop retry budget in - // databases.ts can be scoped to THIS drop rather than the table name: a - // worker that exhausts the budget for a table can observe the catalog - // mid-flight between this drop's completion and a same-name recreate's - // own drop, without ever seeing a non-tombstoned row to reset on. Keying - // the budget by generation instead makes the new drop's tombstone carry - // its own fresh key regardless of what any worker last observed. - primaryMeta.dropGeneration = randomUUID(); - // put is rebound to putSync on RocksDB stores; on LMDB it returns - // a promise, so await it to make the tombstone durable before the - // destructive work below - const tombstoneWrite = (dbisDb as any).put(primaryCatalogKey, primaryMeta); - if (tombstoneWrite?.then) await tombstoneWrite; - } - } - // A get() against a sourcedFrom table resolves to its caller before the resolved - // record's cache write has committed (see getFromSource) - the write lands "in the - // background" for latency reasons. Flip this BEFORE removing the table from the - // schema below: getFromSource() checks it and skips caching (treats the load as - // noCacheStore) for any call it admits from here on, including one that slipped in - // through a stale reference to this Table between the two steps. - droppingTable = true; - // Remove the table from the in-memory schema immediately so concurrent - // requests get "table does not exist" instead of racing the column - // family drops below. If a drop fails past this point the table stays - // invisible, and the tombstone guarantees the drop completes on the - // next startup (or on a same-name create). - delete databases[databaseName][tableName]; - const cleanupCompletion = TableResource.cleanup(); - let cleanupTimer: NodeJS.Timeout | undefined; - const cleanupTimedOut = Symbol('cleanupTimedOut'); - let cleanupResult: void | typeof cleanupTimedOut; + let quiesceMessage: any; try { - cleanupResult = await Promise.race([ - cleanupCompletion, - new Promise((resolve) => { - cleanupTimer = setTimeout(() => resolve(cleanupTimedOut), LOCK_TIMEOUT); - cleanupTimer.unref(); - }), - ]); - } finally { - if (cleanupTimer) clearTimeout(cleanupTimer); - } - if (cleanupResult === cleanupTimedOut) { - const message = `dropTable() timed out waiting for cleanup on ${tableName}; the table is unloaded and its drop will complete on restart or same-name create.`; - logger.warn?.(message); - throw new Error(message); - } - // The above stops new source-fill writes from starting, but a write from a get() - // that already returned to its caller may still be in flight. Dropping the column - // families out from under that write is a genuine invariant violation, not just a - // benign race: RocksDB rejects the still-open write batch with "Invalid column - // family specified in write batch" (or "Could not access column family N"), which - // can also abort this drop before it removes the tombstoned catalog rows - leaving - // the table stuck "dropping" for completeInterruptedDrop to retry (and fail - // identically) on every subsequent load (harper#1381). Drain any in-flight commits - // before the blob sweep below (so it observes every row a drain-caught write just - // committed) and before touching a single column family. - // - // Bounded, and fails CLOSED: the tracked promise covers the whole source round-trip - // plus the local commit (see getFromSource), so a hung/slow source or a slow commit - // (e.g. a large blob write) could otherwise wedge this drop forever. Rather than - // give up and drop anyway - which would reopen exactly the race this drain exists to - // close, just less often - a timeout FAILS the drop. The tombstone written above is - // already durable, so completeInterruptedDrop picks the drop back up on the next - // load, once the stuck write has had time to finish. - if (pendingSourceCommits.size) { - const pending = [...pendingSourceCommits]; - let timer: NodeJS.Timeout; - const timedOut = Symbol('timedOut'); - const result = await Promise.race([ - Promise.allSettled(pending), - new Promise((resolve) => { - timer = setTimeout(() => resolve(timedOut), LOCK_TIMEOUT); - }), - ]); - clearTimeout(timer); - if (result === timedOut) { - throw new Error( - `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.length} in-flight source-populated cache write(s) on ${tableName} to settle; refusing to drop the column families out from under a write that may still be staged. The drop tombstone is durable, so this will be retried on the next load.` - ); - } + quiesceMessage = await signalling.quiesceSchemaChange( + new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName) + ); + } catch (error) { + throw error; } - for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) { - if (entry.metadataFlags & HAS_BLOBS && entry.value) { - deleteBlobsInObject(entry.value); + let destructiveStarted = false; + let destructiveSucceeded = false; + let commitAttempted = false; + try { + commitAttempted = true; + await signalling.commitSchemaChange(quiesceMessage); + if (databaseName === databasePath) { + // Persist a drop tombstone on the primary catalog entry BEFORE any + // destructive work. If the process dies or a column family drop fails + // partway through, the tombstone survives with the catalog rows, and + // the next startup (or a same-name create) completes the drop via + // completeInterruptedDrop in databases.ts instead of resurrecting + // the table. + const primaryCatalogKey = TableResource.tableName + '/'; + const primaryMeta = (dbisDb as any).getSync(primaryCatalogKey); + if (primaryMeta && !primaryMeta.dropping) { + primaryMeta.dropping = true; + // Stamps this drop's identity so the interrupted-drop retry budget in + // databases.ts can be scoped to THIS drop rather than the table name: a + // worker that exhausts the budget for a table can observe the catalog + // mid-flight between this drop's completion and a same-name recreate's + // own drop, without ever seeing a non-tombstoned row to reset on. Keying + // the budget by generation instead makes the new drop's tombstone carry + // its own fresh key regardless of what any worker last observed. + primaryMeta.dropGeneration = randomUUID(); + // put is rebound to putSync on RocksDB stores; on LMDB it returns + // a promise, so await it to make the tombstone durable before the + // destructive work below + const tombstoneWrite = (dbisDb as any).put(primaryCatalogKey, primaryMeta); + if (tombstoneWrite?.then) await tombstoneWrite; + } } - } - if (databaseName === databasePath) { - // part of a database. - // Drop the column families, then remove the catalog metadata - never - // the reverse: a removed-then-failed drop orphans a "ghost" column - // family that poisons same-name recreates, so a genuine drop failure - // must surface and leave the tombstoned catalog rows for the reconcile. - // - // A drop is broadcast to every worker thread, and each holds its own - // handle to the same underlying column family, so a concurrent worker - // (or completeInterruptedDrop) may already have dropped it - surfaced - // as "Column family already dropped!". That is the intended end state, - // not a failure, so tolerate it. The catalog rows are removed only if - // this drop's tombstone is still the live primary row: a concurrent - // same-name create completes the interrupted drop and writes fresh - // catalog rows, and clobbering those would orphan the new table. - const removeTombstonedCatalog = () => { - const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); - if (!currentPrimary?.dropping) return false; - for (const attribute of attributes) { - dbisDb.remove(TableResource.tableName + '/' + attribute.name); + destructiveStarted = true; + droppingTable = true; + // Remove the table from the in-memory schema immediately so concurrent + // requests get "table does not exist" instead of racing the column + // family drops below. If a drop fails past this point the table stays + // invisible, and the tombstone guarantees the drop completes on the + // next startup (or on a same-name create). + delete databases[databaseName][tableName]; + for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) { + if (entry.metadataFlags & HAS_BLOBS && entry.value) { + deleteBlobsInObject(entry.value); } - dbisDb.remove(TableResource.tableName + '/'); - return true; - }; - const rootStore = primaryStore.rootStore; - if (rootStore instanceof RocksDatabase) { - // Serialize the drops + catalog removal against a concurrent - // same-name create (and completeInterruptedDrop) under the database's - // 'update-attributes' exclusive lock - the same lock the create path - // holds. It is a synchronous spin lock that blocks the event loop, so - // the locked section MUST stay synchronous: drop with dropSync (as - // completeInterruptedDrop does), never an awaited drop(), or a - // concurrent create's spin would deadlock waiting on a drop that the - // blocked event loop can never resolve. - while (!rootStore.tryLock('update-attributes')) {} - let removed = false; - try { + } + if (databaseName === databasePath) { + // part of a database. + // Drop the column families, then remove the catalog metadata - never + // the reverse: a removed-then-failed drop orphans a "ghost" column + // family that poisons same-name recreates, so a genuine drop failure + // must surface and leave the tombstoned catalog rows for the reconcile. + // + // A drop is broadcast to every worker thread, and each holds its own + // handle to the same underlying column family, so a concurrent worker + // (or completeInterruptedDrop) may already have dropped it - surfaced + // as "Column family already dropped!". That is the intended end state, + // not a failure, so tolerate it. The catalog rows are removed only if + // this drop's tombstone is still the live primary row: a concurrent + // same-name create completes the interrupted drop and writes fresh + // catalog rows, and clobbering those would orphan the new table. + const removeTombstonedCatalog = () => { + const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); + if (!currentPrimary?.dropping) return false; for (const attribute of attributes) { - const index = indices[attribute.name]; - if (index) - try { - index.dropSync(); - } catch (error) { - ignoreAlreadyDropped(error); - } + dbisDb.remove(TableResource.tableName + '/' + attribute.name); } + dbisDb.remove(TableResource.tableName + '/'); + return true; + }; + const rootStore = primaryStore.rootStore; + if (rootStore instanceof RocksDatabase) { + // Serialize the drops + catalog removal against a concurrent + // same-name create (and completeInterruptedDrop) under the database's + // 'update-attributes' exclusive lock - the same lock the create path + // holds. It is a synchronous spin lock that blocks the event loop, so + // the locked section MUST stay synchronous: drop with dropSync (as + // completeInterruptedDrop does), never an awaited drop(), or a + // concurrent create's spin would deadlock waiting on a drop that the + // blocked event loop can never resolve. + while (!rootStore.tryLock('update-attributes')) {} + let removed = false; try { - primaryStore.dropSync(); - } catch (error) { - ignoreAlreadyDropped(error); + for (const attribute of attributes) { + const index = indices[attribute.name]; + if (index) + try { + index.dropSync(); + } catch (error) { + ignoreAlreadyDropped(error); + } + } + try { + primaryStore.dropSync(); + } catch (error) { + ignoreAlreadyDropped(error); + } + removed = removeTombstonedCatalog(); + } finally { + rootStore.unlock('update-attributes'); + } + if (removed) await dbisDb.committed; + } else { + // LMDB: no shared column-family double-drop, and its engine lock is + // transactional rather than this spin lock, so keep the awaited drop + // plus the same tombstone-guarded catalog removal. + const drops = []; + for (const attribute of attributes) { + const index = indices[attribute.name]; + if (index) drops.push(index.drop().catch(ignoreAlreadyDropped)); } - removed = removeTombstonedCatalog(); - } finally { - rootStore.unlock('update-attributes'); + drops.push(primaryStore.drop().catch(ignoreAlreadyDropped)); + await Promise.all(drops); + if (removeTombstonedCatalog()) await dbisDb.committed; } - if (removed) await dbisDb.committed; } else { - // LMDB: no shared column-family double-drop, and its engine lock is - // transactional rather than this spin lock, so keep the awaited drop - // plus the same tombstone-guarded catalog removal. - const drops = []; - for (const attribute of attributes) { - const index = indices[attribute.name]; - if (index) drops.push(index.drop().catch(ignoreAlreadyDropped)); + // legacy table per database + await primaryStore.close(); + fs.unlinkSync(primaryStore.path); + } + destructiveSucceeded = true; + } finally { + if (!destructiveStarted) { + if (!commitAttempted) await signalling.abortSchemaQuiesce(quiesceMessage); + else { + const message: any = new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName); + message.quiesceId = quiesceMessage.quiesceId; + await signalling.reconcileSchemaChange(message); } - drops.push(primaryStore.drop().catch(ignoreAlreadyDropped)); - await Promise.all(drops); - if (removeTombstonedCatalog()) await dbisDb.committed; + } else { + const message: any = new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName); + message.quiesceId = quiesceMessage.quiesceId; + if (destructiveSucceeded) await signalling.finalizeSchemaChange(message); + else await signalling.reconcileSchemaChange(message); } - } else { - // legacy table per database - await primaryStore.close(); - fs.unlinkSync(primaryStore.path); } - signalling.signalSchemaChange( - new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName) - ); } // #section: read-path /** @@ -1947,11 +1953,13 @@ export function makeTable(options) { invalidated: true, entry: this.#entry, commit: (txnTime, existingEntry, _retry, transaction: any) => { + assertTableWritable(); write.skipped = false; // reset on each retry; cleanup happens after commit if still true if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) { write.skipped = true; return; } + if (isFrozenRecordObject(partialRecord)) partialRecord = { ...partialRecord }; partialRecord ??= null; for (const name in indices) { if (!partialRecord) partialRecord = {}; @@ -1971,6 +1979,13 @@ export function makeTable(options) { { user: (context as any)?.user, residencyId: options?.residencyId, + expiresAt: + options?.expirationDecisionPresent === true || options?.expiresAt !== undefined + ? options.expiresAt + : existingEntry?.expiresAt, + expirationDecisionPresent: + options?.expirationDecisionPresent === true || + Boolean(existingEntry?.metadataFlags & HAS_EXPIRATION_DECISION), nodeId: options?.nodeId, viaNodeId: options?.viaNodeId, transaction, @@ -1998,6 +2013,7 @@ export function makeTable(options) { ? (this.constructor as any).source.relocate.bind((this.constructor as any).source, id, undefined, context) : undefined, commit: (txnTime, existingEntry, _retry, transaction: any) => { + assertTableWritable(); if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) return; const residency = TableResource.getResidencyRecord(options.residencyId); let metadata = 0; @@ -2029,6 +2045,7 @@ export function makeTable(options) { nodeId: options.nodeId, viaNodeId: options?.viaNodeId, expiresAt: options.expiresAt, + expirationDecisionPresent: options.expirationDecisionPresent, transaction, }, 'relocate', @@ -2065,7 +2082,13 @@ export function makeTable(options) { existingEntry.version, // version number should not change metadata, true, - { residencyId, expiresAt: entry.expiresAt, transaction: txnForContext(context).transaction }, + { + residencyId, + expiresAt: entry.expiresAt, + expirationDecisionPresent: + entry.expirationDecisionPresent === true || Boolean(entry.metadataFlags & HAS_EXPIRATION_DECISION), + transaction: txnForContext(context).transaction, + }, 'relocate', false, null // the audit record value should be empty since there are no changes to the actual data @@ -2075,7 +2098,7 @@ export function makeTable(options) { /** * Evicting a record will remove it from a caching table. This is not considered a canonical data change, and it is assumed that retrieving this record from the source will still yield the same record, this is only removing the local copy of the record. */ - static evict(id, existingRecord, existingVersion, existingExpiresAtIndexValue?: number) { + static evict(id, existingRecord, existingVersion, encounteredExpirationIndexValue?: number) { let entry; let deleteBlobsAfterCommit: (() => void) | undefined; let currentEntry; @@ -2109,29 +2132,28 @@ export function makeTable(options) { // primary record can not diverge on a conflict or partial commit. const removalEntry = entry ?? primaryStore.getEntry(id); let removal: MaybePromise; + let encounteredIndexRemoval: MaybePromise; lmdbCompletion = primaryStore.ifVersion(id, existingVersion, () => { - updateIndices( - id, - existingRecord, - null, - undefined, - undefined, - existingExpiresAtIndexValue ?? removalEntry?.expiresAt - ); + const currentExpiration = effectiveExpirationIndexValue(removalEntry, existingRecord); + updateIndices(id, existingRecord, null, undefined, undefined, currentExpiration); + if ( + encounteredExpirationIndexValue !== undefined && + compareKeys(encounteredExpirationIndexValue, currentExpiration) !== 0 + ) + encounteredIndexRemoval = indices[expiresAtIndexName]?.remove(encounteredExpirationIndexValue, id); removal = removeEntry(primaryStore, removalEntry); }); - lmdbCompletion = Promise.all([lmdbCompletion, removal]); + lmdbCompletion = Promise.all([lmdbCompletion, removal, encounteredIndexRemoval]); } else { const removalEntry = entry ?? currentEntry ?? primaryStore.getEntry(id); if (!removalEntry || removalEntry.version !== existingVersion) return; - updateIndices( - id, - existingRecord, - null, - options, - undefined, - existingExpiresAtIndexValue ?? removalEntry.expiresAt - ); + const currentExpiration = effectiveExpirationIndexValue(removalEntry, existingRecord); + updateIndices(id, existingRecord, null, options, undefined, currentExpiration); + if ( + encounteredExpirationIndexValue !== undefined && + compareKeys(encounteredExpirationIndexValue, currentExpiration) !== 0 + ) + indices[expiresAtIndexName]?.remove(encounteredExpirationIndexValue, id, options); if (removalEntry.metadataFlags & HAS_BLOBS) { if (!existingRecord) return; primaryStore.remove(removalEntry.key, options); @@ -2449,6 +2471,7 @@ export function makeTable(options) { }, before: writeToSource(), commit: (txnTime: number, existingEntry: Entry, retry: boolean, transaction: any) => { + assertTableWritable(); // Whether a prior attempt of THIS write appended its own audit entry (sticky, set in // save(); log entries are not part of the aborted rocks transaction, so they survive). // Only such a write can find its own orphaned entry in the dedup lookups below and must @@ -2509,6 +2532,9 @@ export function makeTable(options) { // (resolved below, once recordToStore is merged) overrides the table default in both // directions; the table default is the final fallback. -1 means no expiration. let expiresAt: number | undefined = options?.expiresAt ?? context?.expiresAt; + let expirationDecisionPresent = Boolean( + expiresAtProperty && (options?.expirationDecisionPresent === true || expiresAt !== undefined) + ); const additionalAuditRefs: Array<{ version: number; nodeId: number }> = []; // track additional audit refs to store // Bulk base-copy snapshot apply: store current-state directly with no audit/transaction-log entry // and no out-of-order resequencing/dedup (the source of the O(n) keyed-lookup spin in @@ -2925,7 +2951,13 @@ export function makeTable(options) { } residencyId = getResidencyId(residency); } - if (expiresAt == undefined) { + expirationDecisionPresent = Boolean( + expiresAtProperty && + (options?.expirationDecisionPresent === true || !options?.isNotification || expiresAt !== undefined) + ); + if (expiresAt == undefined && options?.expirationDecisionPresent === true) { + expiresAt = -1; + } else if (expiresAt == undefined) { // A schema @expiresAt attribute makes the record field authoritative over the table // default, in both directions: stamp it into the stored expiry metadata that governs // read-hiding and the cleanup sweep, not just the separate index-pruning sweep (which @@ -3004,6 +3036,7 @@ export function makeTable(options) { user: (context as any)?.user, residencyId, expiresAt, + expirationDecisionPresent, nodeId: options?.nodeId, viaNodeId: options?.viaNodeId, originatingOperation: (context as any)?.originatingOperation, @@ -3022,7 +3055,11 @@ export function makeTable(options) { // transaction (an audit-only commit stored no record, so it stages nothing and the // earlier staged record, if any, remains the basis) if (storeRecord) { - write.stagedEntry = { value: recordToStore, expiresAt }; + write.stagedEntry = { + value: recordToStore, + expiresAt, + metadataFlags: expirationDecisionPresent ? HAS_EXPIRATION_DECISION : 0, + }; // blobs this write saved are referenced by its audit entry (if it wrote one), which // then owns their lifetime; and any record an earlier write in this transaction // stored is now replaced, so mark those writes for the superseded-blob cleanup @@ -3129,6 +3166,7 @@ export function makeTable(options) { ? (this.constructor as any).source.delete.bind((this.constructor as any).source, id, undefined, context) : undefined, commit: (txnTime, existingEntry, retry, transaction: any) => { + assertTableWritable(); write.stagedEntry = undefined; // reset per round; set below once the removal is applied write.superseded = false; // reset per round, as in the update path // what a preceding write in this transaction left for this key is what gets removed @@ -4123,6 +4161,7 @@ export function makeTable(options) { try { let type = auditRecord.type; let value; + let expirationState = auditRecord; if (type === 'message' || request.rawEvents) { // we only send the full message, this are individual messages that can be sent out of order // TODO: Do we want to have a limit to how far out-of-order we are willing to send? @@ -4143,6 +4182,7 @@ export function makeTable(options) { if (entry) { if (entry.version !== auditRecord.version) return; // out of order event, with old update, don't send anything value = entry.value; + expirationState = entry; type = entry.metadataFlags & INVALIDATED ? 'invalidate' : value ? 'put' : 'delete'; } else { type = 'delete'; @@ -4155,6 +4195,7 @@ export function makeTable(options) { version: auditRecord.version, type, beginTxn, + ...expirationEventProperties(expirationState), }; // Queued events are filtered when the queue drains through send() below; events sent // directly (queue already drained) are filtered here. Each event is filtered once. @@ -4224,6 +4265,7 @@ export function makeTable(options) { version: auditRecord.version, type: auditRecord.type, size: auditRecord.size, + ...expirationEventProperties(auditRecord), }) ) return; @@ -4271,6 +4313,7 @@ export function makeTable(options) { value, version: auditRecord.version, type: auditRecord.type, + ...expirationEventProperties(auditRecord), }; // Filter rows before they consume a previousCount slot. if (allowsEvent && !allowsEvent(historyEntry)) { @@ -4314,12 +4357,13 @@ export function makeTable(options) { // if a post-subscribe write hits a key the cursor also visits. This is // idempotent for "current state then live updates" — both deliveries land at // the same final state. We don't dedupe. - for (const { key: id, value, version, localTime, size } of primaryStore.getRange({ + for (const entry of primaryStore.getRange({ start: thisId ?? false, end: thisId == null ? undefined : [thisId, MAXIMUM_KEY], versions: true, snapshot: false, // no need for a snapshot, just want the latest data })) { + const { key: id, value, version, localTime, size } = entry; if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) { recordsSinceYield = 0; await rest(); @@ -4330,7 +4374,8 @@ export function makeTable(options) { const t = localTime ?? version; if (t > cursorMaxTime) cursorMaxTime = t; if (!value) continue; - if (!send({ id, localTime, value, version, type: 'put', size })) return; + if (!send({ id, localTime, value, version, type: 'put', size, ...expirationEventProperties(entry) })) + return; if (subscription.queue?.length > EVENT_HIGH_WATER_MARK) { // if we have too many messages, we need to pause and let the client catch up if ((await subscription.waitForDrain()) === false) return; @@ -4389,6 +4434,7 @@ export function makeTable(options) { value, localTime: nextTime, ...auditRecord, + ...expirationEventProperties(auditRecord), }; if (!allowsEvent || allowsEvent(historyEntry)) { request.omitCurrent = true; @@ -4411,6 +4457,7 @@ export function makeTable(options) { id: thisId, ...entry, type: 'put', + ...expirationEventProperties(entry), }) ) return; @@ -4455,23 +4502,24 @@ export function makeTable(options) { const isCollection = request.isCollection ?? thisId == null; if (isCollection) { let sinceYield = 0; - for (const { key: id, value, version, localTime, size } of primaryStore.getRange({ + for (const entry of primaryStore.getRange({ start: thisId ?? false, end: thisId == null ? undefined : [thisId, MAXIMUM_KEY], versions: true, snapshot: false, // no need for a snapshot, just want the latest data })) { + const { key: id, value, version, localTime, size } = entry; if (++sinceYield >= REPLAY_YIELD_INTERVAL) { sinceYield = 0; await rest(); if (!isActive()) return; } if (!value) continue; // skip tombstones - yield { id, localTime, value, version, type: 'put', size }; + yield { id, localTime, value, version, type: 'put', size, ...expirationEventProperties(entry) }; } } else { const entry = primaryStore.getEntry(thisId); - if (entry?.value) yield { id: thisId, ...entry, type: 'put' }; + if (entry?.value) yield { id: thisId, ...entry, type: 'put', ...expirationEventProperties(entry) }; } } // Drain the current scope into the subscription with the same back-pressure as the live path. @@ -4588,6 +4636,7 @@ export function makeTable(options) { ? (this.constructor as any).source.publish.bind((this.constructor as any).source, id, message, context) : undefined, commit: (txnTime, existingEntry, _retry, transaction: any) => { + assertTableWritable(); // just need to update the version number of the record so it points to the latest audit record // but have to update the version number of the record // TODO: would be faster to use getBinaryFast here and not have the record loaded @@ -4608,7 +4657,8 @@ export function makeTable(options) { { user: (context as any)?.user, residencyId: options?.residencyId, - expiresAt: context?.expiresAt, + expiresAt: existingEntry?.expiresAt, + expirationDecisionPresent: Boolean(existingEntry?.metadataFlags & HAS_EXPIRATION_DECISION), nodeId: options?.nodeId, viaNodeId: options?.viaNodeId, transaction, @@ -5338,9 +5388,11 @@ export function makeTable(options) { static cleanup() { deleteCallbackHandle?.remove(); cleanupClosed = true; + cleanupGeneration++; if (cleanupTimer) { clearTimeout(cleanupTimer); cleanupTimer = undefined; + cleanupNextScheduled = undefined; } if (recordExpirationInterval) { clearInterval(recordExpirationInterval); @@ -5348,9 +5400,24 @@ export function makeTable(options) { } return Promise.all([lastEvictionCompletion, recordExpirationCompletion]).then(() => undefined); } + static async quiesceForDrop() { + dropQuiescing = true; + await this.cleanup(); + const tableCommitWaiter = waitForPendingTableCommits(); + await Promise.allSettled(tableCommitWaiter ? [...pendingSourceCommits, tableCommitWaiter] : pendingSourceCommits); + } + static abortDropQuiesce() { + if (!dropQuiescing) return; + dropQuiescing = false; + this.resumeCleanup(); + } + static isDropQuiescing() { + return dropQuiescing; + } static resumeCleanup() { if (!cleanupClosed) return; cleanupClosed = false; + cleanupGeneration++; if (cleanupWasScheduled) { lastCleanupInterval = undefined; scheduleCleanup(); @@ -5359,15 +5426,22 @@ export function makeTable(options) { if (audit) addDeleteRemoval(); } static runRecordExpirationSweepForTests(testHooks?: ExpirationSweepTestHooks) { - return runRecordExpirationEviction(testHooks, false) ?? Promise.resolve(); + return runRecordExpirationEviction(testHooks, false, true) ?? Promise.resolve(); } static runPrimaryCleanupScanForTests(testHooks?: ExpirationSweepTestHooks) { - return (lastEvictionCompletion = lastEvictionCompletion.then(() => runPrimaryCleanupScan(testHooks))); + return queuePrimaryCleanup(testHooks); + } + static scheduleCleanupForTests(priority?: number) { + return scheduleCleanup(priority); + } + static cleanupNextScheduledForTests() { + return cleanupNextScheduled; } static cleanupStateForTests() { return { closed: cleanupClosed, cleanupScheduled: !cleanupClosed && !!cleanupTimer, + nextCleanupScheduled: cleanupNextScheduled, expirationScheduled: !cleanupClosed && !!recordExpirationInterval, }; } @@ -5493,10 +5567,15 @@ export function makeTable(options) { return hasChanges; } function effectiveExpirationIndexValue(entry: Partial | undefined, record = entry?.value): number | undefined { - if (!expiresAtProperty) return entry?.expiresAt; - return entry?.expiresAt === undefined - ? expirationTimestamp(record?.[expiresAtProperty.name]) - : expirationTimestamp(entry.expiresAt); + return effectiveExpirationTimestamp(entry, record, expiresAtIndexName); + } + function expirationEventProperties(entry: any) { + if (!expiresAtProperty || !entry) return {}; + const metadataFlags = entry.metadataFlags ?? entry.extendedType ?? 0; + if (metadataFlags & HAS_EXPIRATION_DECISION) { + return { expiresAt: entry.expiresAt, expirationDecisionPresent: true }; + } + return entry.expiresAt === undefined ? {} : { expiresAt: entry.expiresAt }; } function checkValidId(id) { switch (typeof id) { @@ -5528,6 +5607,11 @@ export function makeTable(options) { if (length > MAX_KEY_BYTES) throw new ClientError('Primary key size is too large: ' + id.length, 400); return true; } + function assertTableWritable() { + if (dropQuiescing || droppingTable) { + throw new ServerError(`Table ${databaseName}.${tableName} is being dropped`, 503); + } + } function requestTargetToId(target: RequestTargetOrId): Id { return typeof target === 'object' && target ? (target as any).id : (target as Id); } @@ -5967,7 +6051,7 @@ export function makeTable(options) { // Once dropTable() has started, no new source-fill write may begin (dropTable() // only drains writes already in flight - see there); still resolve the caller's // read with fresh source data, just don't cache it into a table that's going away. - noCacheStore: droppingTable, + noCacheStore: droppingTable || dropQuiescing, source: null, transaction: undefined, expiresAt: undefined, @@ -6002,7 +6086,6 @@ export function makeTable(options) { recordAction(resolveDuration, 'cache-resolution', tableName, null, 'success'); if (responseHeaders) appendHeader(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true); - if (expirationMs && sourceContext.expiresAt == undefined) sourceContext.expiresAt = Date.now() + expirationMs; if (updatedRecord) { if (typeof updatedRecord !== 'object') throw new Error('Only objects can be cached and stored in tables'); if (updatedRecord.status > 0 && updatedRecord.headers) { @@ -6067,13 +6150,14 @@ export function makeTable(options) { ); } } + if (expirationMs && sourceContext.expiresAt == undefined) sourceContext.expiresAt = Date.now() + expirationMs; resolved = true; const resolvedEntry: Entry = { key: id, version, value: updatedRecord, expiresAt: sourceContext.expiresAt, - metadataFlags: 0, + metadataFlags: expiresAtProperty ? HAS_EXPIRATION_DECISION : 0, size: 0, localTime: 0, nodeId: 0, @@ -6115,7 +6199,7 @@ export function makeTable(options) { sourceContext.transaction.abort(); return; } - if (context?.noCacheStore || sourceContext.noCacheStore || droppingTable) { + if (context?.noCacheStore || sourceContext.noCacheStore || droppingTable || dropQuiescing) { // abort before we write any change. droppingTable is re-checked live (not just // the noCacheStore snapshot taken at call start) because a call admitted before // dropTable() began can still be sitting here after it started - the await above @@ -6131,6 +6215,10 @@ export function makeTable(options) { nodeName: 'source', commit: (txnTime, existingEntry, _retry, transaction: any) => { sourceWrite.skipped = false; // reset on each retry; cleanup happens after commit if still true + if (droppingTable || dropQuiescing) { + sourceWrite.skipped = true; + return; + } if (existingEntry?.version !== existingVersion) { // don't do anything if the version has changed sourceWrite.skipped = true; @@ -6214,6 +6302,7 @@ export function makeTable(options) { { user: (sourceContext as any)?.user, expiresAt: sourceContext.expiresAt, + expirationDecisionPresent: Boolean(expiresAtProperty), residencyId, transaction, tableToTrack: tableName, @@ -6259,7 +6348,7 @@ export function makeTable(options) { TableResource.userEmbedders ); if (embedBefore) await embedBefore(); - if (droppingTable) { + if (droppingTable || dropQuiescing) { // Re-check right before staging the write: dropTable() may have started // while we were awaiting the embed step above (harper#1381). sourceContext.transaction.abort(); @@ -6307,10 +6396,35 @@ export function makeTable(options) { type ExpirationSweepTestHooks = { beforeEvict?: () => MaybePromise; beforeBatchCommit?: () => MaybePromise; + afterBatchQueued?: () => MaybePromise; }; + function createInFlightLimiter(maxInFlight: number) { + const inFlight = new Set>(); + return { + track(operation: Promise): Promise | void { + const tracked = operation.then( + () => undefined, + () => undefined + ); + inFlight.add(tracked); + tracked.then(() => inFlight.delete(tracked)); + if (inFlight.size >= maxInFlight) return Promise.race(inFlight); + }, + drain(): Promise { + return Promise.all(inFlight); + }, + }; + } function createEvictionBatcher(isCancelled: () => boolean = () => false, testHooks?: ExpirationSweepTestHooks) { type EvictItem = - | { type: 'evict'; key: any; version: number; indexedExpiration?: number } + | { + type: 'evict'; + key: any; + version: number; + encounteredExpiration?: number; + metadataExpiration?: number; + currentExpiration?: number; + } | { type: 'tombstone'; key: any; version: number } | { type: 'dangling-index'; key: any; indexedValue: any; index: any } | { @@ -6323,7 +6437,7 @@ export function makeTable(options) { index: any; }; let pending: EvictItem[] = []; - const inFlight = new Set>(); + const limiter = createInFlightLimiter(MAX_INFLIGHT_EVICTION_BATCHES); // Apply a batch's removals to the given transaction, re-reading each record fresh and skipping // any that changed since the scan. Returns the number of removals actually staged. @@ -6373,7 +6487,21 @@ export function makeTable(options) { if (entry.value == null) continue; // already removed if (entry.metadataFlags & HAS_BLOBS) continue; // per-record evict() owns blob/commit ordering if (hasSourceGet && primaryStore.hasLock(item.key, entry.version)) continue; // resolution in progress - updateIndices(item.key, entry.value, null, options, undefined, item.indexedExpiration ?? entry.expiresAt); + let currentExpiration = entry.expiresAt; + if (item.encounteredExpiration !== undefined) { + currentExpiration = effectiveExpirationIndexValue(entry); + if ( + compareKeys(entry.expiresAt, item.metadataExpiration) !== 0 || + compareKeys(currentExpiration, item.currentExpiration) !== 0 + ) + continue; + } + updateIndices(item.key, entry.value, null, options, undefined, currentExpiration); + if ( + item.encounteredExpiration !== undefined && + compareKeys(item.encounteredExpiration, currentExpiration) !== 0 + ) + indices[expiresAtIndexName]?.remove(item.encounteredExpiration, item.key, options); } removeEntry(primaryStore, entry, options); staged++; @@ -6425,19 +6553,15 @@ export function makeTable(options) { // Track an in-flight commit and, once the cap is reached, return a promise the caller can await // for backpressure (resolves as soon as any in-flight commit finishes). - function track(commit: Promise): Promise | void { - const tracked = commit.finally(() => inFlight.delete(tracked)); - inFlight.add(tracked); - if (inFlight.size >= MAX_INFLIGHT_EVICTION_BATCHES) return Promise.race(inFlight); - } - return { add(item: EvictItem): Promise | void { pending.push(item); if (pending.length >= EVICTION_BATCH_SIZE) { const items = pending; pending = []; - return track(commitItems(items)); + const backpressure = limiter.track(commitItems(items)); + const queued = testHooks?.afterBatchQueued?.(); + if (queued || backpressure) return Promise.all([queued, backpressure]).then(() => undefined); } }, async drain(): Promise { @@ -6445,50 +6569,51 @@ export function makeTable(options) { if (pending.length > 0) { const items = pending; pending = []; - track(commitItems(items)); + limiter.track(commitItems(items)); } - await Promise.all(inFlight); + await limiter.drain(); }, }; } function createCleanupOperationTracker() { - const inFlight = new Set>(); + const limiter = createInFlightLimiter(MAX_CLEANUP_CONCURRENCY); return { add(operation: MaybePromise | void): Promise | void { if (operation == null) return; - const tracked = Promise.resolve(operation) - .then( - () => undefined, - (error) => { - logger.error?.('Cleanup error', error); - } - ) - .finally(() => inFlight.delete(tracked)); - inFlight.add(tracked); - if (inFlight.size >= MAX_CLEANUP_CONCURRENCY) return Promise.race(inFlight); + const tracked = Promise.resolve(operation).then( + () => undefined, + (error) => { + logger.error?.('Cleanup error', error); + } + ); + return limiter.track(tracked); }, drain(): Promise { - return Promise.all(inFlight); + return limiter.drain(); }, }; } - async function runPrimaryCleanupScan(testHooks?: ExpirationSweepTestHooks): Promise { + async function runPrimaryCleanupScan( + testHooks?: ExpirationSweepTestHooks, + priority = cleanupPriority + ): Promise { if (cleanupClosed) return; const rootStore = primaryStore.rootStore; if (rootStore.status !== 'open') return; const operationTracker = createCleanupOperationTracker(); + const batcher = isRocksDB ? createEvictionBatcher(() => cleanupClosed, testHooks) : undefined; const evictThreshold = - Math.pow(cleanupPriority, 8) * (envMngr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_EVICTIONFACTOR) ?? 100000); - const adjustedEviction = evictionMs / Math.pow(Math.max(cleanupPriority, 1), 4); + Math.pow(priority, 8) * (envMngr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_EVICTIONFACTOR) ?? 100000); + const adjustedEviction = evictionMs / Math.pow(Math.max(priority, 1), 4); logger.debug?.( `Starting cleanup scan for ${tableName}, evict threshold ${evictThreshold}, adjusted eviction ${adjustedEviction}ms` ); function shouldEvict(expiresAt: number, version: number, metadataFlags: number, record: any) { const evictWhen = expiresAt + adjustedEviction - Date.now(); if (evictWhen < 0) return true; - if (cleanupPriority) { + if (priority) { let size = primaryStore.lastSize; if (metadataFlags & HAS_BLOBS) { findBlobsInObject(record, (blob) => { @@ -6506,7 +6631,6 @@ export function makeTable(options) { try { let count = 0; const removeDeletedRecords = !audit || isRocksDB; - const batcher = isRocksDB ? createEvictionBatcher(() => cleanupClosed, testHooks) : undefined; for (const entry of primaryStore.getRange({ start: false, snapshot: false, // avoid holding a read snapshot for the duration of a full-table scan @@ -6541,66 +6665,76 @@ export function makeTable(options) { } await rest(); } - if (batcher) await batcher.drain(); - await operationTracker.drain(); logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); } catch (error) { logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); + } finally { + if (batcher) await batcher.drain(); + await operationTracker.drain(); } } + function queuePrimaryCleanup(testHooks?: ExpirationSweepTestHooks, priority = cleanupPriority): Promise { + return (lastEvictionCompletion = lastEvictionCompletion.then(() => runPrimaryCleanupScan(testHooks, priority))); + } + + function armCleanupTimer(nextScheduled: number, generation: number) { + if (cleanupClosed || generation !== cleanupGeneration) return; + cleanupNextScheduled = nextScheduled; + logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`); + const timer = setTimeout( + () => { + if (cleanupClosed || generation !== cleanupGeneration || cleanupTimer !== timer) return; + cleanupTimer = undefined; + cleanupNextScheduled = undefined; + if (primaryStore.rootStore.status !== 'open') return; + armCleanupTimer(Math.max(nextScheduled + cleanupInterval, Date.now()), generation); + const queuedPriority = cleanupPriority; + queuePrimaryCleanup(undefined, queuedPriority).then(() => { + if (cleanupPriority === queuedPriority) cleanupPriority = 0; + }); + }, + Math.min(nextScheduled - Date.now(), MAX_SET_TIMEOUT_MS) + ); + cleanupTimer = timer; + timer.unref(); + } + function scheduleCleanup(priority?: number): Promise | void { - if (cleanupClosed) return; - let runImmediately = false; - if (priority) { - // run immediately if there is a big increase in priority - if (priority - cleanupPriority > 1) runImmediately = true; - cleanupPriority = priority; - } - // Periodically evict expired records and deleted records searching for records who expiresAt timestamp is before now - if (cleanupInterval === lastCleanupInterval && !runImmediately) return; - lastCleanupInterval = cleanupInterval; - if (getWorkerIndex() === getWorkerCount() - 1) { - cleanupWasScheduled = true; - // run on the last thread so we aren't overloading lower-numbered threads - if (cleanupTimer) clearTimeout(cleanupTimer); - if (!cleanupInterval) return; - return new Promise((resolve) => { + if (cleanupClosed || getWorkerIndex() !== getWorkerCount() - 1) return; + cleanupWasScheduled = true; + if (priority) cleanupPriority = Math.max(cleanupPriority, priority); + if (cleanupInterval !== lastCleanupInterval || priority) { + lastCleanupInterval = cleanupInterval; + cleanupGeneration++; + if (cleanupTimer) { + clearTimeout(cleanupTimer); + cleanupTimer = undefined; + cleanupNextScheduled = undefined; + } + if (cleanupInterval) { const startOfYear = new Date(); startOfYear.setMonth(0); startOfYear.setDate(1); - startOfYear.setHours(0); - startOfYear.setMinutes(0); - startOfYear.setSeconds(0); - const nextInterval = cleanupInterval / (1 + cleanupPriority); - // find the next scheduled run based on regular cycles from the beginning of the year (if we restart, this enables a good continuation of scheduling) - const nextScheduled = runImmediately - ? Date.now() - : Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime(); - const startNextTimer = (nextScheduled) => { - if (cleanupClosed) return; - logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`); - // noinspection JSVoidFunctionReturnValueUsed - cleanupTimer = setTimeout( - () => - (lastEvictionCompletion = lastEvictionCompletion.then(async () => { - if (cleanupClosed) return; - if (primaryStore.rootStore.status !== 'open') { - if (cleanupTimer) clearTimeout(cleanupTimer); - cleanupTimer = undefined; - return; - } - // schedule the next run for when the next cleanup interval should occur (or now if it is in the past) - startNextTimer(Math.max(nextScheduled + cleanupInterval, Date.now())); - await runPrimaryCleanupScan(); - resolve(undefined); - cleanupPriority = 0; // reset the priority - })), - Math.min(nextScheduled - Date.now(), MAX_SET_TIMEOUT_MS) // make sure it can fit in 32-bit signed number - ).unref(); // don't let this prevent closing the thread - }; - startNextTimer(nextScheduled); + startOfYear.setHours(0, 0, 0, 0); + // A full disk produces an Infinity reclamation priority. Keep that value for the + // immediate eviction scan, but never let it collapse the recurring timer to NaN/0. + const schedulingPriority = Number.isFinite(cleanupPriority) ? Math.max(cleanupPriority, 0) : 0; + const nextInterval = Math.max(cleanupInterval / (1 + schedulingPriority), MIN_SCHEDULED_CLEANUP_INTERVAL); + const nextScheduled = + Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime(); + armCleanupTimer(nextScheduled, cleanupGeneration); + } + } else { + return; + } + if (priority) { + const queuedPriority = cleanupPriority; + const completion = queuePrimaryCleanup(undefined, queuedPriority); + completion.then(() => { + if (cleanupPriority === queuedPriority) cleanupPriority = 0; }); + return completion; } } function addDeleteRemoval() { @@ -6608,8 +6742,8 @@ export function makeTable(options) { primaryStore.remove(id, version); }); } - function runRecordExpirationEviction(testHooks?: ExpirationSweepTestHooks, schedule = true) { - if (getWorkerIndex() !== 0 || cleanupClosed) return; + function runRecordExpirationEviction(testHooks?: ExpirationSweepTestHooks, schedule = true, forceWorker = false) { + if ((!forceWorker && getWorkerIndex() !== 0) || cleanupClosed) return; const expiresAtName = expiresAtProperty.name; async function sweepRocks(index: any, cutoff: number) { const batcher = createEvictionBatcher(() => cleanupClosed, testHooks); @@ -6641,13 +6775,15 @@ export function makeTable(options) { if (currentExpiration !== undefined && currentExpiration < cutoff) { if (recordEntry.metadataFlags & HAS_BLOBS) { await testHooks?.beforeEvict?.(); - await TableResource.evict(entry.value, recordEntry.value, recordEntry.version, currentExpiration); + await TableResource.evict(entry.value, recordEntry.value, recordEntry.version, entry.key); } else { backpressure = batcher.add({ type: 'evict', key: entry.value, version: recordEntry.version, - indexedExpiration: currentExpiration, + encounteredExpiration: entry.key, + metadataExpiration: recordEntry.expiresAt, + currentExpiration, }); } } else if (compareKeys(entry.key, currentExpiration) !== 0) { @@ -6684,51 +6820,54 @@ export function makeTable(options) { async function sweepLmdb(index: any, cutoff: number) { const operationTracker = createCleanupOperationTracker(); let entriesSinceYield = 0; - for (const key of index.getRange({ start: true, values: false, end: cutoff, snapshot: false })) { - if (cleanupClosed) break; - let afterId: any; - for (;;) { + try { + for (const key of index.getRange({ start: true, values: false, end: cutoff, snapshot: false })) { if (cleanupClosed) break; - const ids = [ - ...index.getValues(key, { - start: afterId, - exclusiveStart: afterId !== undefined, - limit: EVICTION_BATCH_SIZE, - }), - ]; - if (ids.length === 0) break; - afterId = ids.at(-1); - for (const id of ids) { + let afterId: any; + for (;;) { if (cleanupClosed) break; - const recordEntry = primaryStore.getEntry(id); - let operation: MaybePromise | void; - if (recordEntry?.value == null) { - operation = primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); - } else { - const currentExpiration = effectiveExpirationIndexValue(recordEntry); - if (currentExpiration !== undefined && currentExpiration < cutoff) { - await testHooks?.beforeEvict?.(); - operation = TableResource.evict(id, recordEntry.value, recordEntry.version, currentExpiration); - } else if (compareKeys(key, currentExpiration) !== 0) { - operation = primaryStore.ifVersion(id, recordEntry.version, () => { - const removals = [index.remove(key, id)]; - if (currentExpiration !== undefined) removals.push(index.put(currentExpiration, id)); - return Promise.all(removals); - }); + const ids = [ + ...index.getValues(key, { + start: afterId, + exclusiveStart: afterId !== undefined, + limit: EVICTION_BATCH_SIZE, + }), + ]; + if (ids.length === 0) break; + afterId = ids.at(-1); + for (const id of ids) { + if (cleanupClosed) break; + const recordEntry = primaryStore.getEntry(id); + let operation: MaybePromise | void; + if (recordEntry?.value == null) { + operation = primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); + } else { + const currentExpiration = effectiveExpirationIndexValue(recordEntry); + if (currentExpiration !== undefined && currentExpiration < cutoff) { + await testHooks?.beforeEvict?.(); + operation = TableResource.evict(id, recordEntry.value, recordEntry.version, key); + } else if (compareKeys(key, currentExpiration) !== 0) { + operation = primaryStore.ifVersion(id, recordEntry.version, () => { + const removals = [index.remove(key, id)]; + if (currentExpiration !== undefined) removals.push(index.put(currentExpiration, id)); + return Promise.all(removals); + }); + } + } + const backpressure = operationTracker.add(operation); + if (backpressure) await backpressure; + if (++entriesSinceYield >= 10) { + entriesSinceYield = 0; + await rest(); } } - const backpressure = operationTracker.add(operation); - if (backpressure) await backpressure; - if (++entriesSinceYield >= 10) { - entriesSinceYield = 0; - await rest(); - } + if (ids.length < EVICTION_BATCH_SIZE) break; } - if (ids.length < EVICTION_BATCH_SIZE) break; + await rest(); } - await rest(); + } finally { + await operationTracker.drain(); } - await operationTracker.drain(); } async function sweep() { diff --git a/resources/auditStore.ts b/resources/auditStore.ts index b57b88665e..9739436e39 100644 --- a/resources/auditStore.ts +++ b/resources/auditStore.ts @@ -360,6 +360,7 @@ export const HAS_ADDITIONAL_AUDIT_REFS = 0x4000; * lower-byte action region (which extendedType forbids) and within the always-32-bit metadata form. */ export const LOCAL_ONLY = 0x8000; +export const HAS_EXPIRATION_DECISION = 0x10000; const EVENT_TYPES = { put: PUT | HAS_RECORD, [PUT]: 'put', diff --git a/resources/databases.ts b/resources/databases.ts index 41924b43dc..76ea7aa60e 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -9,13 +9,14 @@ import { getBaseSchemaPath, getTransactionAuditStoreBasePath, } from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; -import { expirationTimestamp, makeTable, ignoreAlreadyDropped } from './Table.ts'; +import { effectiveExpirationTimestamp, makeTable, ignoreAlreadyDropped } from './Table.ts'; import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.ts'; import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME, MIGRATING_DIR_SUFFIX, + OPERATIONS_ENUM, RESERVED_DATABASE_NAMES, } from '../utility/hdbTerms.ts'; import { getConfigPath } from '../config/configUtils.ts'; @@ -99,23 +100,54 @@ const DEFINED_TABLES = Symbol('defined-tables'); const DEFAULT_DATABASE_CLOSE_TIMEOUT = 10_000; let databaseCloseTimeout = DEFAULT_DATABASE_CLOSE_TIMEOUT; const unavailableDatabases = new Set(); +const SCHEMA_QUIESCE_LEASE_MS = 120_000; +type SchemaQuiesceMessage = { + originator?: number; + operation: string; + schema: string; + table?: string; + quiesceId: string; + leaseUntil?: number; + phase?: string; +}; +type SchemaQuiesceState = { + message: SchemaQuiesceMessage; + table?: any; + tables?: any[]; + lease?: NodeJS.Timeout; + wasLoaded?: boolean; + completion?: Promise; + abortRequested?: boolean; + finalizing?: boolean; + committed?: boolean; + localOwner?: boolean; +}; +const schemaQuiescence = new Map(); +type SchemaQuiesceOwners = { schema?: string; tables: Map }; +const quiescedSchemas = new Map(); +const MAX_RETIRED_SCHEMA_QUIESCENCES = 1024; +const retiredSchemaQuiescences = new Set(); export function setDatabaseCloseTimeoutForTests(timeout = DEFAULT_DATABASE_CLOSE_TIMEOUT): void { databaseCloseTimeout = timeout; } -async function waitForTableCleanup(completions: Promise[]): Promise { +async function waitForTableCleanup(completions: Promise[]): Promise[] | false> { let timeout: NodeJS.Timeout | undefined; const timedOut = Symbol('timedOut'); const result = await Promise.race([ - Promise.all(completions), + Promise.allSettled(completions), new Promise((resolve) => { timeout = setTimeout(() => resolve(timedOut), databaseCloseTimeout); timeout.unref(); }), ]); if (timeout) clearTimeout(timeout); - return result !== timedOut; + return result === timedOut ? false : result; +} + +function cleanupFailures(results: PromiseSettledResult[]): unknown[] { + return results.filter((result) => result.status === 'rejected').map((result) => result.reason); } const DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages initSync(); @@ -1254,6 +1286,238 @@ function lockDatabaseForDrop(dbPath: string, databaseName: string, held: Restore */ const droppingDatabases = new Map>(); +function schemaQuiesceTarget(message: SchemaQuiesceMessage): string { + return message.operation === OPERATIONS_ENUM.DROP_TABLE + ? `table:${message.schema}.${message.table}` + : `schema:${message.schema}`; +} + +function claimSchemaQuiesce(message: SchemaQuiesceMessage): string | undefined { + let owners = quiescedSchemas.get(message.schema); + if (!owners) quiescedSchemas.set(message.schema, (owners = { tables: new Map() })); + if (message.operation === OPERATIONS_ENUM.DROP_SCHEMA) { + if (owners.schema && owners.schema !== message.quiesceId) return `schema:${message.schema} is already quiescing`; + for (const owner of owners.tables.values()) { + if (owner !== message.quiesceId) return `schema:${message.schema} has a table already quiescing`; + } + owners.schema = message.quiesceId; + } else { + if (owners.schema && owners.schema !== message.quiesceId) return `schema:${message.schema} is already quiescing`; + const owner = owners.tables.get(message.table!); + if (owner && owner !== message.quiesceId) return `table:${message.schema}.${message.table} is already quiescing`; + owners.tables.set(message.table!, message.quiesceId); + } +} + +function releaseSchemaQuiesce(message: SchemaQuiesceMessage): void { + const owners = quiescedSchemas.get(message.schema); + if (!owners) return; + if (owners.schema === message.quiesceId) owners.schema = undefined; + if (message.table && owners.tables.get(message.table) === message.quiesceId) owners.tables.delete(message.table); + if (!owners.schema && owners.tables.size === 0) quiescedSchemas.delete(message.schema); +} + +function schemaQuiesceOwner(message: SchemaQuiesceMessage): string | undefined { + const owners = quiescedSchemas.get(message.schema); + return message.operation === OPERATIONS_ENUM.DROP_SCHEMA ? owners?.schema : owners?.tables.get(message.table!); +} + +function clearSchemaQuiesce(state: SchemaQuiesceState): void { + if (state.lease) clearTimeout(state.lease); + schemaQuiescence.delete(state.message.quiesceId); + releaseSchemaQuiesce(state.message); +} + +function retireSchemaQuiesce(quiesceId: string): void { + retiredSchemaQuiescences.add(quiesceId); + while (retiredSchemaQuiescences.size > MAX_RETIRED_SCHEMA_QUIESCENCES) { + const oldestId = retiredSchemaQuiescences.values().next().value; + retiredSchemaQuiescences.delete(oldestId); + } +} + +function armSchemaQuiesceLease(state: SchemaQuiesceState): void { + if (state.lease) clearTimeout(state.lease); + const leaseUntil = Math.max(state.message.leaseUntil ?? 0, Date.now() + SCHEMA_QUIESCE_LEASE_MS); + state.lease = setTimeout(() => { + if (state.committed) { + logger.warn(`Schema quiesce ${state.message.quiesceId} expired after its commit boundary; remaining fail-closed`); + armSchemaQuiesceLease(state); + return; + } + reconcileSchemaQuiesce(state).catch((error) => { + logger.warn('Could not reconcile expired schema quiesce:', error); + if (schemaQuiescence.get(state.message.quiesceId) === state) armSchemaQuiesceLease(state); + }); + }, leaseUntil - Date.now()); + state.lease.unref(); +} + +function resetQuiescedDatabase(databaseName: string): void { + delete databases[databaseName]; + const definedDatabase = definedDatabases?.get(databaseName); + if (definedDatabase) (definedDatabase as any).rootStore = undefined; + resetDatabases(); +} + +async function reconcileSchemaQuiesce(state: SchemaQuiesceState): Promise { + state.abortRequested = true; + // A peer may already be closing handles or may have removed the target from its registry. Do not + // reopen it until that attempt settles. Origin-owned quiescence only drains work and can abort + // immediately—the cleanup generation prevents its blocked scan from rearming. + if (!state.localOwner) await state.completion?.catch(() => undefined); + if (schemaQuiescence.get(state.message.quiesceId) !== state) return; + if (state.message.operation === OPERATIONS_ENUM.DROP_TABLE) { + const { schema, table } = state.message; + const dbTables = databases[schema]; + if (!state.localOwner) { + if (state.table && dbTables && !dbTables[table]) dbTables[table] = state.table; + resetDatabases(); + } + if (state.table && (state.localOwner || databases[schema]?.[table] === state.table)) state.table.abortDropQuiesce(); + } else { + if (state.localOwner) { + for (const table of state.tables ?? []) table.abortDropQuiesce?.(); + } else resetQuiescedDatabase(state.message.schema); + } + clearSchemaQuiesce(state); + retireSchemaQuiesce(state.message.quiesceId); + if (state.message.operation === OPERATIONS_ENUM.DROP_SCHEMA) unavailableDatabases.delete(state.message.schema); +} + +export async function quiesceSchemaTarget( + message: SchemaQuiesceMessage & { originLocal?: boolean } +): Promise<{ quiesced: boolean; reason?: string }> { + if (retiredSchemaQuiescences.has(message.quiesceId)) + return { quiesced: false, reason: `Schema quiesce ${message.quiesceId} is no longer active` }; + const target = schemaQuiesceTarget(message); + const existingById = schemaQuiescence.get(message.quiesceId); + if (existingById && schemaQuiesceTarget(existingById.message) !== target) + return { quiesced: false, reason: `Schema quiesce ${message.quiesceId} belongs to another target` }; + const activeId = schemaQuiesceOwner(message); + if (activeId) { + if (activeId !== message.quiesceId) return { quiesced: false, reason: `${target} is already quiescing` }; + const active = schemaQuiescence.get(activeId); + if (!active || active.abortRequested || active.finalizing) + return { quiesced: false, reason: `${target} quiescence is terminating` }; + armSchemaQuiesceLease(active); + await active.completion; + return active.abortRequested ? { quiesced: false, reason: `${target} quiescence was aborted` } : { quiesced: true }; + } + const conflict = claimSchemaQuiesce(message); + if (conflict) return { quiesced: false, reason: conflict }; + const state: SchemaQuiesceState = { message, localOwner: message.originLocal === true }; + schemaQuiescence.set(message.quiesceId, state); + armSchemaQuiesceLease(state); + state.completion = (async () => { + if (message.operation === OPERATIONS_ENUM.DROP_TABLE) { + const dbTables = databases[message.schema]; + const Table = dbTables?.[message.table]; + if (!Table) return; + state.table = Table; + if (!state.localOwner) delete dbTables[message.table]; + await Table.quiesceForDrop(); + } else if (message.operation === OPERATIONS_ENUM.DROP_SCHEMA) { + state.wasLoaded = Boolean(databases[message.schema]); + unavailableDatabases.add(message.schema); + if (state.wasLoaded && state.localOwner) { + state.tables = Object.values(databases[message.schema]); + await Promise.all(state.tables.map((table) => table.quiesceForDrop?.())); + } else if (state.wasLoaded) await closeDatabaseOnce(message.schema); + } else { + throw new Error(`Unsupported schema quiesce operation ${message.operation}`); + } + })(); + try { + const completion = await waitForTableCleanup([state.completion]); + if (!completion) { + if (message.operation === OPERATIONS_ENUM.DROP_SCHEMA) + throw new Error( + `Timed out after ${databaseCloseTimeout}ms waiting for cleanup; refusing to destroy database ${message.schema} while cleanup is active.` + ); + throw new Error( + `Timed out after ${databaseCloseTimeout}ms waiting for ${target} to quiesce; the target remains available.` + ); + } + const failures = cleanupFailures(completion); + if (failures.length) { + if (message.operation === OPERATIONS_ENUM.DROP_SCHEMA) + throw new AggregateError(failures, `Could not quiesce database ${message.schema} for drop`); + throw failures[0]; + } + if (state.abortRequested) return { quiesced: false, reason: `${target} quiescence was aborted` }; + return { quiesced: true }; + } catch (error) { + try { + await reconcileSchemaQuiesce(state); + } catch (abortError) { + logger.warn('Could not restore schema state after quiesce failure:', abortError); + } + return { quiesced: false, reason: error instanceof Error ? error.message : String(error) }; + } +} + +export async function abortSchemaQuiesce(message: Pick): Promise { + const state = schemaQuiescence.get(message.quiesceId); + if (!state) return; + if (state.committed) throw new Error(`Schema quiesce ${message.quiesceId} crossed its commit boundary`); + await reconcileSchemaQuiesce(state); +} + +export async function commitSchemaQuiesce( + message: SchemaQuiesceMessage +): Promise<{ committed: boolean; reason?: string }> { + const state = schemaQuiescence.get(message.quiesceId); + if (!state || schemaQuiesceOwner(message) !== message.quiesceId || state.abortRequested || state.finalizing) + return { committed: false, reason: `Schema quiesce ${message.quiesceId} is not active` }; + await state.completion; + if (state.abortRequested) return { committed: false, reason: `Schema quiesce ${message.quiesceId} was aborted` }; + state.committed = true; + armSchemaQuiesceLease(state); + return { committed: true }; +} + +export function renewSchemaQuiesce(message: SchemaQuiesceMessage): { quiesced: boolean; reason?: string } { + const state = schemaQuiescence.get(message.quiesceId); + if (!state || state.abortRequested || state.finalizing) + return { quiesced: false, reason: `Schema quiesce ${message.quiesceId} is not active` }; + state.message.leaseUntil = message.leaseUntil; + armSchemaQuiesceLease(state); + return { quiesced: true }; +} + +export function finishSchemaQuiesce(message: SchemaQuiesceMessage): boolean { + if (retiredSchemaQuiescences.has(message.quiesceId)) return false; + const activeId = schemaQuiesceOwner(message); + if (activeId === undefined) return message.originator === process.pid; + if (activeId !== message.quiesceId) return false; + const state = schemaQuiescence.get(message.quiesceId); + if (!state || state.abortRequested) return false; + if (message.phase === 'finalize-quiesce' && !state.committed) return false; + state.finalizing = true; + if (state.lease) { + clearTimeout(state.lease); + state.lease = undefined; + } + return true; +} + +export function completeSchemaQuiesce(message: SchemaQuiesceMessage): void { + const state = schemaQuiescence.get(message.quiesceId); + if (state) { + clearSchemaQuiesce(state); + if (state.message.operation === OPERATIONS_ENUM.DROP_SCHEMA) unavailableDatabases.delete(state.message.schema); + } + retireSchemaQuiesce(message.quiesceId); +} + +export function failSchemaQuiesceFinalization(message: Pick): void { + const state = schemaQuiescence.get(message.quiesceId); + if (!state) return; + state.finalizing = false; + armSchemaQuiesceLease(state); +} + export async function dropDatabase(databaseName): Promise { const activeDrop = droppingDatabases.get(databaseName); if (activeDrop) return activeDrop; @@ -1267,15 +1531,33 @@ export async function dropDatabase(databaseName): Promise { } if (!rootStore) rootStore = database({ database: databaseName, table: null }); unavailableDatabases.add(databaseName); - const completion = dropDatabaseOnce(databaseName, dbTables, rootStore).finally(() => { - droppingDatabases.delete(databaseName); - unavailableDatabases.delete(databaseName); - }); + let quiesceMessage: any; + let dropSucceeded = false; + const completion = dropDatabaseOnce(databaseName, dbTables, rootStore, (message) => { + quiesceMessage = message; + }) + .then(() => { + dropSucceeded = true; + }) + .finally(async () => { + droppingDatabases.delete(databaseName); + if (!quiesceMessage) { + for (const tableName in dbTables) dbTables[tableName].abortDropQuiesce?.(); + unavailableDatabases.delete(databaseName); + } + if (quiesceMessage) { + const message: any = new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, databaseName); + message.quiesceId = quiesceMessage.quiesceId; + if (dropSucceeded) await signalling.finalizeSchemaChange(message); + else await signalling.reconcileSchemaChange(message); + unavailableDatabases.delete(databaseName); + } + }); droppingDatabases.set(databaseName, completion); return completion; } -async function dropDatabaseOnce(databaseName, dbTables, rootStore): Promise { +async function dropDatabaseOnce(databaseName, dbTables, rootStore, onQuiesced: (message: any) => void): Promise { // Hold the per-database restore lock across the entire drop so its file deletion can never // interleave with a restore's purge-and-copy on the same directory — a destroy landing after a // restore's copy would gut a "successful" restore, and vice versa. Restore takes the same lock @@ -1291,26 +1573,14 @@ async function dropDatabaseOnce(databaseName, dbTables, rootStore): Promise[] = []; - for (const tableName in dbTables) { - try { - const completion = dbTables[tableName].cleanup?.(); - if (typeof completion?.then === 'function') { - cleanupCompletions.push( - completion.catch((error) => - logger.warn(`Error awaiting cleanup for table ${databaseName}.${tableName}:`, error) - ) - ); - } - } catch (error) { - logger.warn(`Error cleaning up table ${databaseName}.${tableName} while dropping database:`, error); - } - } - if (!(await waitForTableCleanup(cleanupCompletions))) { - for (const tableName in dbTables) dbTables[tableName].resumeCleanup?.(); - throw new Error( - `Timed out after ${databaseCloseTimeout}ms waiting for cleanup; refusing to destroy database ${databaseName} while cleanup is active.` + try { + const quiesceMessage = await signalling.quiesceSchemaChange( + new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, databaseName) ); + onQuiesced(quiesceMessage); + await signalling.commitSchemaChange(quiesceMessage); + } catch (error) { + throw error; } forgetDatabaseEnvironment(rootStore); for (const tableName in dbTables) { @@ -1365,10 +1635,19 @@ export function closeDatabase(databaseName: string): Promise { return Promise.reject(new Error(`Database ${databaseName} is already closing`)); } unavailableDatabases.add(databaseName); - const completion = closeDatabaseOnce(databaseName).finally(() => { - closingDatabases.delete(databaseName); - unavailableDatabases.delete(databaseName); - }); + const completion = closeDatabaseOnce(databaseName) + .catch((error) => { + try { + resetQuiescedDatabase(databaseName); + } catch (recoveryError) { + throw new AggregateError([error, recoveryError], `Could not close or recover database ${databaseName}`); + } + throw error; + }) + .finally(() => { + closingDatabases.delete(databaseName); + unavailableDatabases.delete(databaseName); + }); closingDatabases.set(databaseName, completion); return completion; } @@ -1380,35 +1659,34 @@ async function closeDatabaseOnce(databaseName: string): Promise { for (const tableName in dbTables) tableEntries.push([tableName, dbTables[tableName]]); const rootStores = new Set(); const cleanupCompletions: Promise[] = []; + const closeErrors: unknown[] = []; const closeStore = (store: any, description: string) => { try { store?.close?.(); } catch (error) { - logger.warn(`Error closing ${description} while closing database ${databaseName}:`, error); + closeErrors.push( + new Error(`Error closing ${description} while closing database ${databaseName}`, { cause: error }) + ); } }; - for (const [tableName, table] of tableEntries) { + for (const [, table] of tableEntries) { if (!table?.primaryStore) continue; - try { - const completion = table.cleanup?.(); - if (typeof completion?.then === 'function') { - cleanupCompletions.push( - completion.catch((error) => - logger.warn(`Error awaiting cleanup for table ${databaseName}.${tableName}:`, error) - ) - ); - } - } catch (error) { - logger.warn(`Error cleaning up table ${databaseName}.${tableName} while closing database:`, error); - } + const completion = table.quiesceForDrop?.() ?? table.cleanup?.(); + if (typeof completion?.then === 'function') cleanupCompletions.push(completion); if (table.primaryStore.rootStore) rootStores.add(table.primaryStore.rootStore); } - if (!(await waitForTableCleanup(cleanupCompletions))) { - for (const [, table] of tableEntries) table.resumeCleanup?.(); + const cleanupResults = await waitForTableCleanup(cleanupCompletions); + if (!cleanupResults) { + for (const [, table] of tableEntries) table.abortDropQuiesce?.(); throw new Error( `Timed out after ${databaseCloseTimeout}ms waiting for cleanup; refusing to close database ${databaseName} while cleanup is active.` ); } + const failures = cleanupFailures(cleanupResults); + if (failures.length) { + for (const [, table] of tableEntries) table.abortDropQuiesce?.(); + throw new AggregateError(failures, `Could not quiesce database ${databaseName} for close`); + } for (const [tableName, table] of tableEntries) { if (!table?.primaryStore) continue; for (const indexName in table.indices || {}) { @@ -1426,6 +1704,7 @@ async function closeDatabaseOnce(databaseName: string): Promise { closeStore(rootStore, 'root store'); forgetDatabaseEnvironment(rootStore); } + if (closeErrors.length) throw new AggregateError(closeErrors, `Could not close database ${databaseName}`); const definedDatabase = definedDatabases?.get(databaseName); if (definedDatabase) (definedDatabase as any).rootStore = undefined; if (databaseName === 'data') { @@ -2216,7 +2495,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { } let outstanding = 0; // this means that a new attribute has been introduced that needs to be indexed - for (const { key, value: record, expiresAt } of Table.primaryStore.getRange({ + for (const { key, value: record, expiresAt, metadataFlags } of Table.primaryStore.getRange({ start, lazy: attributesLength < 4, versions: true, @@ -2241,9 +2520,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { try { const resolver = attribute.resolve; const value = attribute.expiresAt - ? expiresAt === undefined - ? expirationTimestamp(record?.[property]) - : expirationTimestamp(expiresAt) + ? effectiveExpirationTimestamp({ expiresAt, metadataFlags }, record, property) : record && (resolver ? resolver(record) : record[property]); if (index.customIndex) { index.customIndex.index(key, value); diff --git a/resources/replayLogs.ts b/resources/replayLogs.ts index b0caecad24..e4bf877d89 100644 --- a/resources/replayLogs.ts +++ b/resources/replayLogs.ts @@ -13,7 +13,7 @@ import { shouldAbortSlowReplay, REPLAY_WALL_CLOCK_LIMIT_MS, } from './replayLogsGuards.ts'; -import { purgeAgedLogs } from './auditStore.ts'; +import { HAS_EXPIRATION_DECISION, purgeAgedLogs } from './auditStore.ts'; import { get as envGet } from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; @@ -198,7 +198,14 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise transaction.isReplay = true; } context.transaction = transaction; - const options = { context, residencyId, nodeId, originatingOperation }; + const options = { + context, + residencyId, + nodeId, + originatingOperation, + expiresAt, + expirationDecisionPresent: Boolean(extendedType & HAS_EXPIRATION_DECISION), + }; writes++; switch (type) { case 'put': diff --git a/resources/search.ts b/resources/search.ts index 3425a0bed9..1086c085fb 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -2,7 +2,7 @@ import { ClientError, IndexRebuildingError, Violation } from '../utility/errors/ import { OVERFLOW_MARKER, MAX_SEARCH_KEY_LENGTH, SEARCH_TYPES } from '../utility/lmdb/terms.ts'; import { compareKeys, MAXIMUM_KEY, writeKey } from 'ordered-binary'; import { SKIP } from '@harperfast/extended-iterable'; -import { expirationTimestamp, INVALIDATED, EVICTED, freezeRecord } from './Table.ts'; +import { effectiveExpirationTimestamp, expirationTimestamp, INVALIDATED, EVICTED, freezeRecord } from './Table.ts'; import type { DirectCondition, Id } from './ResourceInterface.ts'; import { RequestTarget } from './RequestTarget.ts'; import { lastMetadata, type Entry } from './RecordEncoder.ts'; @@ -1094,12 +1094,11 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar function recordFilter(record: any, entry?: Entry) { // `record` may be null/undefined when called via a nested-path filter // where an intermediate property is missing. - let value = - normalizeRecordExpiration && entry?.expiresAt !== undefined - ? entry.expiresAt - : record == null - ? undefined - : record[attribute]; + let value = normalizeRecordExpiration + ? effectiveExpirationTimestamp(entry, record, attribute) + : record == null + ? undefined + : record[attribute]; if (normalizeRecordExpiration) value = normalizeExpirationSearchValue(value); let matches: boolean; if (typeof value !== 'object' || !value || allowObjectMatching) matches = filter(value); diff --git a/security/certificateVerification/certificateVerificationSource.ts b/security/certificateVerification/certificateVerificationSource.ts index 9f304c2c33..4402f4e8c2 100644 --- a/security/certificateVerification/certificateVerificationSource.ts +++ b/security/certificateVerification/certificateVerificationSource.ts @@ -71,6 +71,7 @@ export class CertificateVerificationSource extends Resource { // Handle result consistently const expiresAt = Date.now() + methodConfig.cacheTtl; + context.expiresAt = expiresAt; return { certificate_id: id, diff --git a/security/certificateVerification/crlVerification.ts b/security/certificateVerification/crlVerification.ts index decef6a2cc..b0ce208e01 100644 --- a/security/certificateVerification/crlVerification.ts +++ b/security/certificateVerification/crlVerification.ts @@ -74,6 +74,7 @@ class CertificateRevocationListSource extends Resource { const crlExpiry = result.next_update; const configExpiry = Date.now() + config.cacheTtl; const expiresAt = Math.min(crlExpiry, configExpiry); + context.expiresAt = expiresAt; return { ...result, @@ -90,6 +91,7 @@ class CertificateRevocationListSource extends Resource { if (config.failureMode === 'fail-closed') { // Cache the error for faster recovery const expiresAt = Date.now() + ERROR_CACHE_TTL; + context.expiresAt = expiresAt; return { crl_id: id, diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.js index 5b2f81d33c..a787e80fc5 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.js @@ -12,7 +12,18 @@ const harperBridge = require('../../dataLayer/harperBridge/harperBridge.ts'); const process = require('process'); const { isMainThread, workerData } = require('worker_threads'); -const { resetDatabases, closeDatabase } = require('../../resources/databases.ts'); +const { + resetDatabases, + closeDatabase, + quiesceSchemaTarget, + abortSchemaQuiesce, + commitSchemaQuiesce, + renewSchemaQuiesce, + finishSchemaQuiesce, + completeSchemaQuiesce, + failSchemaQuiesceFinalization, +} = require('../../resources/databases.ts'); +const { holdWorkerStartsForSchema, releaseWorkerStartsForSchema } = require('../threads/manageThreads.js'); /** * This object/functions are passed to the ITC client instance and dynamically added as event handlers. @@ -38,6 +49,64 @@ const serverItcHandlers = { * @returns {Promise} */ const schemaListeners = []; +const MAX_SCHEMA_TERMINAL_OUTCOMES = 1024; +const schemaTerminalCompletions = new Map(); +const schemaTerminalOutcomes = new Map(); +const schemaWorkerBarrierLeases = new Map(); + +function armSchemaWorkerBarrierLease(message) { + if (!isMainThread) return; + const existing = schemaWorkerBarrierLeases.get(message.quiesceId); + if (existing) clearTimeout(existing); + const delay = Math.max(0, (message.leaseUntil ?? Date.now()) - Date.now()); + const timer = setTimeout(() => { + schemaWorkerBarrierLeases.delete(message.quiesceId); + releaseWorkerStartsForSchema(message.quiesceId); + }, delay); + timer.unref(); + schemaWorkerBarrierLeases.set(message.quiesceId, timer); +} + +function commitSchemaWorkerBarrier(message) { + if (!isMainThread) return; + const timer = schemaWorkerBarrierLeases.get(message.quiesceId); + if (timer) clearTimeout(timer); + schemaWorkerBarrierLeases.delete(message.quiesceId); +} + +function releaseSchemaWorkerBarrier(message) { + if (!isMainThread) return; + commitSchemaWorkerBarrier(message); + releaseWorkerStartsForSchema(message.quiesceId); +} + +function sameSchemaTerminalRequest(entry, message) { + return ( + entry.phase === message.phase && + entry.operation === message.operation && + entry.schema === message.schema && + entry.table === message.table + ); +} + +function schemaTerminalEntry(message, value) { + return { + phase: message.phase, + operation: message.operation, + schema: message.schema, + table: message.table, + value, + }; +} + +function retainSchemaTerminalOutcome(message, result) { + schemaTerminalOutcomes.set(message.quiesceId, schemaTerminalEntry(message, result)); + while (schemaTerminalOutcomes.size > MAX_SCHEMA_TERMINAL_OUTCOMES) { + const oldestId = schemaTerminalOutcomes.keys().next().value; + schemaTerminalOutcomes.delete(oldestId); + } +} + async function schemaHandler(event) { const validate = validateEvent(event); if (validate) { @@ -46,31 +115,91 @@ async function schemaHandler(event) { } hdbLogger.trace(`ITC schemaHandler received schema event:`, event); - // restore_backup: this thread must release its store handles so the restore can purge and - // rewrite the database directory. The rescan below (resetDatabases) skips reloading it while - // the restoring marker is present, and reloads it on the completion signal (marker gone). - if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP && event.message.schema) { - for (let attempt = 0; attempt < 2; attempt++) { + if (event.message?.phase === 'hold-worker-starts') { + holdWorkerStartsForSchema(event.message.quiesceId); + armSchemaWorkerBarrierLease(event.message); + return { held: true }; + } + if (event.message?.phase === 'release-worker-starts') { + releaseSchemaWorkerBarrier(event.message); + return { released: true }; + } + if (event.message?.phase === 'quiesce') return quiesceSchemaTarget(event.message); + if (event.message?.phase === 'renew-quiesce') { + const result = renewSchemaQuiesce(event.message); + if (result.quiesced) armSchemaWorkerBarrierLease(event.message); + return result; + } + if (event.message?.phase === 'commit-quiesce') { + const result = await commitSchemaQuiesce(event.message); + if (result.committed) commitSchemaWorkerBarrier(event.message); + return result; + } + if (event.message?.phase === 'abort-quiesce') { + await abortSchemaQuiesce(event.message); + return { aborted: true }; + } + const terminalPhase = + event.message?.phase === 'finalize-quiesce' + ? 'finalized' + : event.message?.phase === 'reconcile-quiesce' + ? 'reconciled' + : undefined; + if (terminalPhase) { + const quiesceId = event.message.quiesceId; + const outcome = schemaTerminalOutcomes.get(quiesceId); + if (outcome) return sameSchemaTerminalRequest(outcome, event.message) ? outcome.value : { [terminalPhase]: false }; + const active = schemaTerminalCompletions.get(quiesceId); + if (active) return sameSchemaTerminalRequest(active, event.message) ? active.value : { [terminalPhase]: false }; + const completion = applySchemaChange(event, terminalPhase); + schemaTerminalCompletions.set(quiesceId, schemaTerminalEntry(event.message, completion)); + try { + const result = await completion; + if (result?.[terminalPhase] === true) retainSchemaTerminalOutcome(event.message, result); + return result; + } finally { + if (schemaTerminalCompletions.get(quiesceId)?.value === completion) schemaTerminalCompletions.delete(quiesceId); + } + } + return applySchemaChange(event); +} + +async function applySchemaChange(event, terminalPhase) { + if (terminalPhase && !finishSchemaQuiesce(event.message)) return { [terminalPhase]: false }; + try { + // restore_backup: this thread must release its store handles so the restore can purge and + // rewrite the database directory. The rescan below (resetDatabases) skips reloading it while + // the restoring marker is present, and reloads it on the completion signal (marker gone). + if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP && event.message.schema) { + for (let attempt = 0; attempt < 2; attempt++) { + try { + await closeDatabase(event.message.schema); + break; + } catch (error) { + hdbLogger.warn( + `Could not close database ${event.message.schema} for restore${attempt === 0 ? '; retrying once' : ''}:`, + error + ); + if (attempt === 1) return; + } + } + } + await cleanLmdbMap(event.message); + await syncSchemaMetadata(event.message, Boolean(terminalPhase)); + for (let listener of schemaListeners) { try { - await closeDatabase(event.message.schema); - break; - } catch (error) { - hdbLogger.warn( - `Could not close database ${event.message.schema} for restore${attempt === 0 ? '; retrying once' : ''}:`, - error - ); - if (attempt === 1) return; + listener(event?.message); + } catch (err) { + hdbLogger.error(err); } } - } - await cleanLmdbMap(event.message); - await syncSchemaMetadata(event.message); - for (let listener of schemaListeners) { - try { - listener(event?.message); - } catch (err) { - hdbLogger.error(err); + if (terminalPhase) { + completeSchemaQuiesce(event.message); + return { [terminalPhase]: true }; } + } catch (error) { + if (terminalPhase) failSchemaQuiesceFinalization(event.message); + throw error; } } @@ -85,7 +214,7 @@ schemaHandler.addListener = function (listener) { * @param msg * @returns {Promise} */ -async function syncSchemaMetadata(msg) { +async function syncSchemaMetadata(msg, strict = false) { try { // TODO: Eventually should indicate which database/table changed so we don't have to scan everything let databases = resetDatabases(); @@ -93,6 +222,7 @@ async function syncSchemaMetadata(msg) { // wait for a write to finish to ensure all writes have been written await databases[msg.database][msg.table].put(Symbol.for('write-verify'), null); } catch (e) { + if (strict) throw e; hdbLogger.error(e); } } diff --git a/server/jobs/jobRunner.ts b/server/jobs/jobRunner.ts index 4c1d7c9549..84dbfa4001 100644 --- a/server/jobs/jobRunner.ts +++ b/server/jobs/jobRunner.ts @@ -143,6 +143,7 @@ async function runJob(runnerMessage: any, operation: any) { async function launchJobThread(job_id: any) { log.trace('launching job thread:', job_id); if (isMainThread) { + await threadsStart.waitForSchemaWorkerStarts(); threadsStart.startWorker(join(__dirname, './jobProcess.js'), { autoRestart: false, name: 'job', @@ -158,6 +159,7 @@ async function launchJobThread(job_id: any) { if (isMainThread) { onMessageByType(hdbTerms.ITC_EVENT_TYPES.START_JOB, async (message) => { try { + await threadsStart.waitForSchemaWorkerStarts(); threadsStart.startWorker(join(__dirname, './jobProcess.js'), { autoRestart: false, name: 'job', diff --git a/server/threads/itc.js b/server/threads/itc.js index 992237f4e4..5a3cd1b0e6 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.js @@ -3,6 +3,7 @@ const hdbUtils = require('../../utility/common_utils.ts'); const hdbTerms = require('../../utility/hdbTerms.ts'); const { ITC_ERRORS } = require('../../utility/errors/commonErrors.ts'); +const harperLogger = require('../../utility/logging/harper_logger.ts'); const { threadId } = require('worker_threads'); const { onMessageFromWorkers, broadcastWithAcknowledgement } = require('./manageThreads.js'); @@ -13,29 +14,45 @@ module.exports = { UserEventMsg, }; let serverItcHandlers; -onMessageFromWorkers(async (event, sender) => { +onMessageFromWorkers((event, sender) => { + handleItcEvent(event, sender).catch((error) => { + harperLogger.error(`ITC event handler failed for event type ${event?.type}`, error); + sendAcknowledgement(event, sender, { handled: false, reason: error?.message }); + }); +}); +async function handleItcEvent(event, sender) { serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); validateEvent(event); + let result; if (serverItcHandlers[event.type]) { - await serverItcHandlers[event.type](event); + result = await serverItcHandlers[event.type](event); } - if (event.requestId && sender) - sender.postMessage({ + sendAcknowledgement(event, sender, result); +} +function sendAcknowledgement(event, sender, result) { + if (!event?.requestId || !sender) return; + try { + const acknowledgement = { type: 'ack', id: event.requestId, - }); -}); + }; + if (event.includeAcknowledgementResult === true) acknowledgement.result = result; + sender.postMessage(acknowledgement); + } catch (error) { + harperLogger.error(`Unable to acknowledge ITC event type ${event?.type}`, error); + } +} /** * Emits an ITC event to the ITC server. * @param event */ -function sendItcEvent(event) { +function sendItcEvent(event, acknowledgementOptions = undefined) { // Always stamp originator so handlers can send direct responses back. // The main thread's threadId is 0 (worker_threads convention); parentPort.threadId // is set to 0 in workers, so sendToThread(0, ...) routes back to main. if (event.message) event.message.originator = threadId; - return broadcastWithAcknowledgement(event); + return broadcastWithAcknowledgement(event, acknowledgementOptions); } /** diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index 87d829c9c3..604b6deed0 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -72,6 +72,13 @@ const UNREGISTER_PROCESS_GROUP = 'unregister-process-group'; const AWAIT_PROCESS_GROUP_TERMINATION = 'await-process-group-termination'; const PROCESS_GROUP_TERMINATION_CONFIRMED = 'process-group-termination-confirmed'; const THREAD_INFO_REQUEST_TIMEOUT_MS = 1000; +class ITCAcknowledgementError extends Error { + constructor(message, results) { + super(message); + this.code = 'ERR_ITC_ACKNOWLEDGEMENT'; + this.results = results; + } +} let getThreadInfo; let awaitProcessGroupTermination; // Worker-side backstop that force-exits if the graceful shutdown sequence doesn't finish in time. @@ -130,6 +137,24 @@ function restoreShutdownDeadline() { const listenersByType = new Map(); const messagesQueuedByType = new Map(); +const schemaWorkerStartBarriers = new Set(); +let schemaWorkerStartWaiters = []; + +function holdWorkerStartsForSchema(quiesceId) { + if (isMainThread) schemaWorkerStartBarriers.add(quiesceId); +} + +function releaseWorkerStartsForSchema(quiesceId) { + if (!isMainThread || !schemaWorkerStartBarriers.delete(quiesceId) || schemaWorkerStartBarriers.size) return; + const waiters = schemaWorkerStartWaiters; + schemaWorkerStartWaiters = []; + for (const resolve of waiters) resolve(); +} + +function waitForSchemaWorkerStarts() { + if (!isMainThread || schemaWorkerStartBarriers.size === 0) return Promise.resolve(); + return new Promise((resolve) => schemaWorkerStartWaiters.push(resolve)); +} module.exports = { startWorker, @@ -142,6 +167,7 @@ module.exports = { onMessageByType, broadcast, broadcastWithAcknowledgement, + ITCAcknowledgementError, getWorkerIndex, getWorkerCount, getTicketKeys, @@ -149,6 +175,9 @@ module.exports = { setTerminateTimeout, extendShutdownDeadline, restoreShutdownDeadline, + holdWorkerStartsForSchema, + releaseWorkerStartsForSchema, + waitForSchemaWorkerStarts, registerWorkerDataProvider, onThreadExit, registerProcessGroup, @@ -424,7 +453,7 @@ function startWorker(path, options = {}) { // if this wasn't an intentional shutdown, restart now (unless we have tried too many times) if (worker.unexpectedRestarts < MAX_UNEXPECTED_RESTARTS) { options.unexpectedRestarts = worker.unexpectedRestarts + 1; - startWorker(path, options); + waitForSchemaWorkerStarts().then(() => startWorker(path, options)); } else harperLogger.error(`Thread has been restarted ${worker.restarts} times and will not be restarted`); } }); @@ -503,6 +532,7 @@ async function restartWorkers( // replacement, startWorker's unexpected-exit handler must not auto-restart it (that would // leave a duplicate once the replacement is up). Restored below if the replacement fails. worker.wasShutdown = true; + await waitForSchemaWorkerStarts(); let newWorker = worker.startCopy(); // Likewise suppress auto-restart on the replacement *while it boots*: if it fails to come up // we leave the existing worker in place, and a background retry succeeding later would push the @@ -575,7 +605,10 @@ async function restartWorkers( // Overlapping types we couldn't pre-start (Windows/Bun): start the replacement now that the old // worker is releasing its port. server.close() stops accepting immediately, so the port frees up // well before the replacement finishes booting and binds. - if (overlapping && startReplacementThreads && !canPreStartReplacement) worker.startCopy(); + if (overlapping && startReplacementThreads && !canPreStartReplacement) { + await waitForSchemaWorkerStarts(); + worker.startCopy(); + } let whenDone = new Promise((resolve) => { // in case the exit inside the thread doesn't timeout, force it from the outside const armTerminate = (delay) => @@ -612,7 +645,7 @@ async function restartWorkers( const index = waitingToFinish.indexOf(whenDone); if (index > -1) waitingToFinish.splice(index, 1); // non-overlapping types have no advance replacement, so start it once the old one is gone - if (!overlapping && startReplacementThreads) worker.startCopy(); + if (!overlapping && startReplacementThreads) waitForSchemaWorkerStarts().then(() => worker.startCopy()); resolve(); }); }); @@ -690,14 +723,25 @@ async function broadcast(message, includeSelf) { const awaitingResponses = new Map(); let nextId = 1; // Backstop so a wedged-but-alive worker (one whose event loop is blocked and never acks, yet -// whose port hasn't closed) can't hang a mutating admin/DDL op forever. The durable write has -// already succeeded by the time we broadcast, and the health monitor restarts a truly stuck -// worker (its port close fires the same ack handlers), so on timeout we proceed best-effort. +// whose port hasn't closed) can't hang a best-effort broadcast forever. Strict callers receive +// the timeout as a failure instead of proceeding. const DEFAULT_ACK_TIMEOUT_MS = 30000; -function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) { - return new Promise((resolve) => { +function broadcastWithAcknowledgement(message, timeoutOrOptions = DEFAULT_ACK_TIMEOUT_MS) { + const options = + typeof timeoutOrOptions === 'number' ? { timeout: timeoutOrOptions } : timeoutOrOptions || Object.create(null); + const timeout = options.timeout ?? DEFAULT_ACK_TIMEOUT_MS; + const acceptResult = options.acceptResult; + const includeJobWorkers = options.includeJobWorkers === true; + if (acceptResult !== undefined && typeof acceptResult !== 'function') { + throw new TypeError('acceptResult must be a function'); + } + if (acceptResult) message.includeAcknowledgementResult = true; + else delete message.includeAcknowledgementResult; + return new Promise((resolve, reject) => { let waitingCount = 0; let timer; + let prepared = false; + const results = []; // Tracks the handlers still awaiting an ack for THIS broadcast. Doubles as an // idempotency guard: a port's handler runs at most once whether it's driven by an ack, // the close listener, or the timeout below. @@ -707,20 +751,46 @@ function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) clearTimeout(timer); timer = undefined; } - resolve(); + if (!acceptResult) return resolve(); + const failures = results.filter((result) => result.status !== 'accepted'); + if (failures.length === 0) return resolve(results); + reject( + new ITCAcknowledgementError( + `ITC broadcast (type ${message.type}) failed acknowledgement from ${failures.length} worker thread(s)`, + results + ) + ); }; for (let port of connectedPorts) { // Job workers run a single isolated task and exit; they don't participate in // schema-change gossip. Including them causes a deadlock: the broadcast waits for // the job worker's ACK while the job worker's event loop is busy waiting for the // same broadcast to complete (re-entrant schema change triggered by the job op). - if (port.isJobWorker) continue; + if (port.isJobWorker && !includeJobWorkers) continue; try { let requestId = nextId++; - const ackHandler = () => { + const ackHandler = (status, result, error) => { if (!pending.delete(ackHandler)) return; // already settled for this port awaitingResponses.delete(requestId); - if (--waitingCount === 0) { + if (acceptResult) { + if (status === 'acknowledged') { + let accepted = false; + try { + accepted = acceptResult(result) === true; + } catch (acceptError) { + error = acceptError; + } + results.push({ + threadId: port.threadId, + status: accepted ? 'accepted' : 'rejected', + result, + ...(error ? { error } : null), + }); + } else { + results.push({ threadId: port.threadId, status, ...(error ? { error } : null) }); + } + } + if (--waitingCount === 0 && prepared) { finish(); } if (port !== parentPort && --port.refCount === 0) { @@ -732,35 +802,39 @@ function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) port.ref(); port.refCount = (port.refCount || 0) + 1; awaitingResponses.set((message.requestId = requestId), ackHandler); + waitingCount++; if (!port.hasAckCloseListener) { // just set a single close listener that can clean up all the ack handlers for a port that is closed port.hasAckCloseListener = true; port.on(port.close ? 'close' : 'exit', () => { for (let [, ackHandler] of awaitingResponses) { if (ackHandler.port === port) { - ackHandler(); + ackHandler('closed'); } } }); } port.postMessage(message); - waitingCount++; } catch (error) { - harperLogger.error(`Unable to send message to worker`, error); + const ackHandler = awaitingResponses.get(message.requestId); + if (ackHandler?.port === port) ackHandler('transport-error', undefined, error); + if (!acceptResult) harperLogger.error(`Unable to send message to worker`, error); } } - if (waitingCount === 0) return resolve(); + prepared = true; + if (waitingCount === 0) return finish(); if (timeout > 0) { timer = setTimeout(() => { timer = undefined; const stuck = []; for (let ackHandler of [...pending]) { stuck.push(ackHandler.port?.threadId); - ackHandler(); // same cleanup path as an ack/close; drives waitingCount to 0 and resolves + ackHandler('timeout'); // same cleanup path as an ack/close; drives waitingCount to 0 and resolves } - harperLogger.warn( - `ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; proceeding best-effort` - ); + if (!acceptResult) + harperLogger.warn( + `ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; proceeding best-effort` + ); }, timeout); timer.unref?.(); } @@ -1157,7 +1231,7 @@ function addPort(port, keepRef, isJobWorker) { } else if (message.type === ACKNOWLEDGEMENT) { let completion = awaitingResponses.get(message.id); if (completion) { - completion(); + completion('acknowledged', message.result); } } else if (message.type === REMOVE_PORT) { const idx = connectedPorts.findIndex((p) => p.threadId === message.threadId); diff --git a/storage-format.md b/storage-format.md index a10900c178..2ad1e652e6 100644 --- a/storage-format.md +++ b/storage-format.md @@ -86,15 +86,21 @@ Variable-length integer encoding the operation type and extended type flags. **Additional Flags (upper bits):** -| Value | Name | Description | -| ------------- | ---------------------------- | ----------------------------------- | -| 64 (0x40) | HAS_PREVIOUS_VERSION | Previous version timestamp included | -| 128 (0x80) | HAS_EXTENDED_TYPE | Extended type information | -| 512 (0x200) | HAS_CURRENT_RESIDENCY_ID | Current residency ID included | -| 1024 (0x400) | HAS_PREVIOUS_RESIDENCY_ID | Previous residency ID included | -| 2048 (0x800) | HAS_ORIGINATING_OPERATION | Originating operation type included | -| 4096 (0x1000) | HAS_EXPIRATION_EXTENDED_TYPE | Expiration timestamp included | -| 8192 (0x2000) | HAS_BLOBS | Binary blob data included | +| Value | Name | Description | +| --------------- | ---------------------------- | ------------------------------------ | +| 64 (0x40) | HAS_PREVIOUS_VERSION | Previous version timestamp included | +| 128 (0x80) | HAS_EXTENDED_TYPE | Extended type information | +| 512 (0x200) | HAS_CURRENT_RESIDENCY_ID | Current residency ID included | +| 1024 (0x400) | HAS_PREVIOUS_RESIDENCY_ID | Previous residency ID included | +| 2048 (0x800) | HAS_ORIGINATING_OPERATION | Originating operation type included | +| 4096 (0x1000) | HAS_EXPIRATION_EXTENDED_TYPE | Expiration timestamp included | +| 8192 (0x2000) | HAS_BLOBS | Binary blob data included | +| 65536 (0x10000) | HAS_EXPIRATION_DECISION | Expiration decision is authoritative | + +`HAS_EXPIRATION_DECISION` carries no payload. It is mirrored in the primary record metadata and audit +entry so a missing `HAS_EXPIRATION_EXTENDED_TYPE` bit means an explicit no-expiration decision rather +than missing legacy metadata. Records and audit events without the decision bit retain the legacy +fallback to a public `@expiresAt` field during upgrades. ### Variable-Length Integer Encoding diff --git a/unitTests/resources/auditEntryRecordFlags.test.js b/unitTests/resources/auditEntryRecordFlags.test.js index dc002782b1..8330656f9b 100644 --- a/unitTests/resources/auditEntryRecordFlags.test.js +++ b/unitTests/resources/auditEntryRecordFlags.test.js @@ -2,7 +2,7 @@ require('../testUtils'); const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); const { table } = require('#src/resources/databases'); -const { createAuditEntry, readAuditEntry } = require('#src/resources/auditStore'); +const { createAuditEntry, readAuditEntry, HAS_EXPIRATION_DECISION } = require('#src/resources/auditStore'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { transaction } = require('#src/resources/transaction'); @@ -26,6 +26,25 @@ describe('Audit entry record flags match the body (#2153)', () => { expiresAt: 1817915930237, }; + it('round-trips an explicit no-expiration decision without adding payload bytes', () => { + const localOnlyEntry = createAuditEntry({ + ...baseRecord, + extendedType: 0x8000, + expiresAt: undefined, + encodedRecord: Buffer.from([0x80]), + }); + const decisionEntry = createAuditEntry({ + ...baseRecord, + extendedType: HAS_EXPIRATION_DECISION, + expiresAt: undefined, + encodedRecord: Buffer.from([0x80]), + }); + assert.equal(decisionEntry.length, localOnlyEntry.length); + const read = readAuditEntry(Buffer.from(decisionEntry)); + assert.ok(read.extendedType & HAS_EXPIRATION_DECISION); + assert.equal(read.expiresAt, undefined); + }); + it('clears HAS_RECORD on a put minted with no encoded record', () => { const entry = createAuditEntry({ ...baseRecord, encodedRecord: undefined }); const read = readAuditEntry(Buffer.from(entry)); diff --git a/unitTests/resources/closeLoadedDatabases.test.js b/unitTests/resources/closeLoadedDatabases.test.js index 06a762dacd..e6e2f33dfd 100644 --- a/unitTests/resources/closeLoadedDatabases.test.js +++ b/unitTests/resources/closeLoadedDatabases.test.js @@ -179,6 +179,29 @@ describe('RocksDB handle release', function () { await closeDatabase('closetimeout6'); }); + it('fails closed and resumes cleanup when table quiescence rejects', async function () { + const Table = table({ + table: 'expiring', + database: 'closefailure7', + attributes: [{ attribute: 'id', isPrimaryKey: true }], + }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const quiesceForDrop = Table.quiesceForDrop; + Table.quiesceForDrop = async function () { + await quiesceForDrop.call(this); + throw new Error('injected cleanup failure'); + }; + try { + await assert.rejects(closeDatabase('closefailure7'), /Could not quiesce database/); + assert.strictEqual(rootStore.status, 'open'); + assert.strictEqual(Table.cleanupStateForTests().closed, false); + } finally { + Table.quiesceForDrop = quiesceForDrop; + } + await closeDatabase('closefailure7'); + }); + it('dropDatabase runs table cleanup before destroying its stores', async function () { const Table = table({ table: 'expiring', @@ -262,4 +285,52 @@ describe('RocksDB handle release', function () { await sweep; await dropDatabase('droptimeout7'); }); + + it('fails closed and resumes cleanup when drop quiescence rejects', async function () { + const Table = table({ + table: 'expiring', + database: 'dropfailure8', + attributes: [{ attribute: 'id', isPrimaryKey: true }], + }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const quiesceForDrop = Table.quiesceForDrop; + Table.quiesceForDrop = async function () { + await quiesceForDrop.call(this); + throw new Error('injected cleanup failure'); + }; + try { + await assert.rejects(dropDatabase('dropfailure8'), /Could not quiesce database/); + assert.strictEqual(rootStore.status, 'open'); + assert.strictEqual(Table.cleanupStateForTests().closed, false); + } finally { + Table.quiesceForDrop = quiesceForDrop; + } + await dropDatabase('dropfailure8'); + }); + + it('reconciles authoritative state when native database destruction fails', async function () { + const databaseName = 'dropdestroyfailure9'; + const Table = table({ + table: 'records', + database: databaseName, + attributes: [{ attribute: 'id', isPrimaryKey: true }], + }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const destroy = rootStore.destroy; + rootStore.destroy = () => { + throw new Error('injected destroy failure'); + }; + try { + await assert.rejects(dropDatabase(databaseName), /injected destroy failure/); + } finally { + rootStore.destroy = destroy; + } + const Recovered = getDatabases()[databaseName]?.records; + assert.ok(Recovered, 'authoritative reconciliation must reload a database that was not destroyed'); + assert.notStrictEqual(Recovered, Table, 'the closed registration must not be restored'); + assert.doesNotThrow(() => database({ database: databaseName }), 'the recovered database must be available'); + await dropDatabase(databaseName); + }); }); diff --git a/unitTests/resources/databases.test.js b/unitTests/resources/databases.test.js index f75c9b5f34..10b9bf454b 100644 --- a/unitTests/resources/databases.test.js +++ b/unitTests/resources/databases.test.js @@ -3,7 +3,20 @@ const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); const { existsSync, mkdirSync, writeFileSync } = require('node:fs'); const { dirname, join } = require('node:path'); -const { table, flushDatabases, dropDatabase, getDatabases, resetDatabases } = require('#src/resources/databases'); +const { + table, + flushDatabases, + dropDatabase, + closeDatabase, + database, + getDatabases, + resetDatabases, + quiesceSchemaTarget, + abortSchemaQuiesce, + commitSchemaQuiesce, + finishSchemaQuiesce, + completeSchemaQuiesce, +} = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { RocksDatabase } = require('@harperfast/rocksdb-js'); const { beginRestore, completeRestore, RESTORE_META_DIR } = require('#src/dataLayer/restoreMarker'); @@ -152,6 +165,33 @@ describe('dropDatabase restore serialization', () => { await assert.doesNotReject(dropDatabase(MULTI)); }); + it('rebuilds authoritative registrations after a native close failure', async () => { + const CLOSE_FAILURE_DB = 'close-failure-recovery-test'; + const Original = table({ + table: 'Records', + database: CLOSE_FAILURE_DB, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const originalClose = Original.primaryStore.close; + Original.primaryStore.close = () => { + throw new Error('injected close failure'); + }; + try { + await assert.rejects( + closeDatabase(CLOSE_FAILURE_DB), + (error) => + error instanceof AggregateError && + error.errors.some((closeError) => closeError.cause?.message === 'injected close failure') + ); + } finally { + Original.primaryStore.close = originalClose; + } + const Recovered = getDatabases()[CLOSE_FAILURE_DB]?.Records; + assert.ok(Recovered, 'the authoritative catalog should be reloaded after close failure'); + assert.notStrictEqual(Recovered, Original, 'the half-closed table registration must not remain public'); + assert.strictEqual(Recovered.isDropQuiescing(), false, 'the recovered table must be writable again'); + }); + it('never loads the reserved restore-metadata directory as a database', function () { // the API can't create a database with this name (schemaRegex rejects the backtick), but the // scan opens any CURRENT+MANIFEST directory regardless of name, so it must skip the reserved dir @@ -175,3 +215,163 @@ describe('dropDatabase restore serialization', () => { assert.ok(existsSync(reservedDir), 'the reserved dir itself is left in place (used for lifecycle metadata)'); }); }); + +describe('cross-worker schema quiescence', () => { + before(function () { + setupTestDBPath(); + setMainIsWorker(true); + }); + + it('waits for in-flight table quiescence before authoritative abort recovery', async () => { + const DB = 'quiesce-abort-wait-test'; + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const originalQuiesce = Table.quiesceForDrop; + let releaseQuiesce; + Table.quiesceForDrop = () => new Promise((resolve) => (releaseQuiesce = resolve)); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-abort-wait', + leaseUntil: Date.now() + 60_000, + }; + try { + const quiesce = quiesceSchemaTarget(message); + await new Promise(setImmediate); + let abortSettled = false; + const abort = abortSchemaQuiesce(message).then(() => (abortSettled = true)); + await new Promise(setImmediate); + assert.strictEqual(abortSettled, false, 'abort must not reset while quiescence is still closing'); + releaseQuiesce(); + const [result] = await Promise.all([quiesce, abort]); + assert.strictEqual(result.quiesced, false); + assert.ok(getDatabases()[DB]?.Records, 'authoritative reset must restore a table that was not dropped'); + } finally { + Table.quiesceForDrop = originalQuiesce; + } + }); + + it('keeps a schema unavailable until terminal reset completes', async () => { + const DB = 'quiesce-final-gate-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_schema', + schema: DB, + quiesceId: 'q-final-gate', + leaseUntil: Date.now() + 60_000, + }; + const result = await quiesceSchemaTarget(message); + assert.strictEqual(result.quiesced, true); + assert.strictEqual(finishSchemaQuiesce(message), true); + assert.throws(() => database({ database: DB, table: null }), /closing and cannot be opened/); + resetDatabases(); + assert.throws(() => database({ database: DB, table: null }), /closing and cannot be opened/); + completeSchemaQuiesce(message); + assert.doesNotThrow(() => database({ database: DB, table: null })); + assert.strictEqual(finishSchemaQuiesce(message), false, 'completed IDs must not re-enter finalization'); + }); + + it('rejects terminal messages for unknown and aborted quiescence IDs', async () => { + const DB = 'quiesce-stale-id-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-aborted-terminal', + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + await abortSchemaQuiesce(message); + assert.strictEqual(finishSchemaQuiesce(message), false); + assert.strictEqual( + finishSchemaQuiesce({ ...message, quiesceId: 'q-never-quiesced' }), + false, + 'unknown IDs must not authorize a metadata reset' + ); + }); + + it('retains a missing-table quiescence until its terminal message', async () => { + const message = { + operation: 'drop_table', + schema: 'quiesce-missing-table-test', + table: 'Records', + quiesceId: 'q-missing-table', + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual(finishSchemaQuiesce(message), true); + completeSchemaQuiesce(message); + assert.strictEqual(finishSchemaQuiesce(message), false); + }); + + it('serializes schema and table quiescence hierarchically', async () => { + const DB = 'quiesce-hierarchy-test'; + const First = table({ table: 'First', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + table({ table: 'Second', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const originalQuiesce = First.quiesceForDrop; + let releaseFirst; + First.quiesceForDrop = () => new Promise((resolve) => (releaseFirst = resolve)); + const firstMessage = { + operation: 'drop_table', + schema: DB, + table: 'First', + quiesceId: 'q-hierarchy-first', + originLocal: true, + }; + try { + const first = quiesceSchemaTarget(firstMessage); + await new Promise(setImmediate); + const duplicate = await quiesceSchemaTarget({ ...firstMessage, quiesceId: 'q-hierarchy-duplicate' }); + assert.strictEqual(duplicate.quiesced, false); + const schema = await quiesceSchemaTarget({ + operation: 'drop_schema', + schema: DB, + quiesceId: 'q-hierarchy-schema', + originLocal: true, + }); + assert.strictEqual(schema.quiesced, false); + releaseFirst(); + assert.strictEqual((await first).quiesced, true); + await abortSchemaQuiesce(firstMessage); + First.quiesceForDrop = originalQuiesce; + + const schemaMessage = { + operation: 'drop_schema', + schema: DB, + quiesceId: 'q-hierarchy-schema-owner', + originLocal: true, + }; + assert.strictEqual((await quiesceSchemaTarget(schemaMessage)).quiesced, true); + const tableWhileSchema = await quiesceSchemaTarget({ + operation: 'drop_table', + schema: DB, + table: 'Second', + quiesceId: 'q-hierarchy-table-loser', + originLocal: true, + }); + assert.strictEqual(tableWhileSchema.quiesced, false); + await abortSchemaQuiesce(schemaMessage); + } finally { + First.quiesceForDrop = originalQuiesce; + } + }); + + it('stays fail-closed after the commit boundary until a terminal reconcile', async () => { + const DB = 'quiesce-committed-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-committed', + originLocal: true, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + await assert.rejects(() => abortSchemaQuiesce(message), /commit boundary/); + const terminal = { ...message, phase: 'reconcile-quiesce' }; + assert.strictEqual(finishSchemaQuiesce(terminal), true); + completeSchemaQuiesce(terminal); + }); +}); diff --git a/unitTests/resources/dropTableGhost.test.js b/unitTests/resources/dropTableGhost.test.js index 646ec41ab2..4eaae2f168 100644 --- a/unitTests/resources/dropTableGhost.test.js +++ b/unitTests/resources/dropTableGhost.test.js @@ -88,7 +88,7 @@ describe('dropTable ghost regression', () => { await Fresh.dropTable(); }); - it('surfaces a failed column family drop and completes the drop on recreate', async function () { + it('surfaces a failed column family drop and reconciles the authoritative tombstone', async function () { const Doomed = defineTable('GhostFailDrop'); await Doomed.put({ id: 1, str: 'data' }); const restore = stubFailingDrop(Doomed.primaryStore, new Error('injected drop failure')); @@ -99,10 +99,12 @@ describe('dropTable ghost regression', () => { } // the table is gone from the live schema (no half-alive table)... assert.equal(databases[TEST_DB]?.GhostFailDrop, undefined, 'failed drop must still remove the table from memory'); - // ...but the tombstoned catalog entry survives so the drop can complete later - assert.equal(getDbisDb().getSync('GhostFailDrop/')?.dropping, true, 'tombstone must survive a failed drop'); + // Reconciliation rescans authoritative storage. A fresh handle can complete the + // tombstoned drop even though the original handle failed, so no worker retains + // stale catalog state after the failure is reported. + assert.equal(getDbisDb().getSync('GhostFailDrop/'), undefined, 'reconcile must complete the tombstoned drop'); - // recreating the same name completes the interrupted drop and works + // recreating the same name starts clean after reconciliation const Fresh = defineTable('GhostFailDrop'); await Fresh.put({ id: 2, str: 'fresh' }); assert.equal((await Fresh.get(2)).str, 'fresh'); diff --git a/unitTests/resources/expirationWarning.test.js b/unitTests/resources/expirationWarning.test.js index 372eebc624..4fe9a39332 100644 --- a/unitTests/resources/expirationWarning.test.js +++ b/unitTests/resources/expirationWarning.test.js @@ -133,7 +133,7 @@ describe('Per-record expiresAt without scheduled cleanup (#1339)', () => { assert.strictEqual(sourceWarningsFor(tableName).length, 1); }); - it('uses a table expiration without warning about a returned @expiresAt field', async function () { + it('warns before applying a table expiration to a returned @expiresAt field', async function () { const tableName = 'SourcedTableExpirationFallback'; const SourcedTable = table({ table: tableName, @@ -158,6 +158,6 @@ describe('Per-record expiresAt without scheduled cleanup (#1339)', () => { const storedExpiration = SourcedTable.primaryStore.getEntry(1).expiresAt; assert.ok(storedExpiration >= before); assert.deepStrictEqual([...SourcedTable.indices.expiresAt.getValues(storedExpiration)], [1]); - assert.strictEqual(sourceWarningsFor(tableName).length, 0); + assert.strictEqual(sourceWarningsFor(tableName).length, 1); }); }); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index fd49d9b325..8a173bfd56 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -9,6 +9,9 @@ const { createBlob, getFilePathForBlob, setDeletionDelay } = require('#src/resou const { existsSync } = require('node:fs'); const { setTimeout: delay } = require('node:timers/promises'); const { waitFor } = require('../waitFor.js'); +const { HAS_EXPIRATION_DECISION } = require('#src/resources/auditStore'); +const { TABLE_COMMIT_ADMISSION } = require('#src/resources/DatabaseTransaction'); +const { transaction } = require('#src/resources/transaction'); const activeTables = new Set(); const table = (options) => { @@ -166,8 +169,10 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const tableName = 'ExpiresAtCanonicalRebuild'; const Table = makeTable(tableName); const expiresAt = Date.now() + 3_600_000; + const noExpiryField = expiresAt + 60_000; const isoExpiresAt = new Date(expiresAt).toISOString(); await Table.put(1, { id: 1, expiresAt: isoExpiresAt }); + await Table.put(2, { id: 2, expiresAt: noExpiryField }, { expiresAt: -1 }); await Table.primaryStore.committed; const attribute = Table.attributes.find((candidate) => candidate.name === 'expiresAt'); @@ -176,12 +181,15 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Table.dbisDB.put(attribute.key, descriptor); await Table.indices.expiresAt.clear(); await Table.indices.expiresAt.put(isoExpiresAt, 1); + await Table.indices.expiresAt.put(noExpiryField, 2); assert.deepStrictEqual([...Table.indices.expiresAt.getValues(isoExpiresAt)], [1]); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(noExpiryField)], [2]); const Reloaded = makeTable(tableName); await Reloaded.indexingOperation; assert.deepStrictEqual([...Reloaded.indices.expiresAt.getValues(isoExpiresAt)], []); assert.deepStrictEqual([...Reloaded.indices.expiresAt.getValues(expiresAt)], [1]); + assert.deepStrictEqual([...Reloaded.indices.expiresAt.getValues(noExpiryField)], []); Table.cleanup(); }); @@ -235,29 +243,129 @@ describe('@expiresAt attribute is authoritative over the table default', () => { assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); }); + it('does not reinterpret a source no-expiration decision as a legacy field expiration', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSourceNoExpiration')); + const fieldExpiresAt = Date.now() - 1_000; + Table.sourcedFrom({ + get(id) { + return { id, expiresAt: fieldExpiresAt, name: 'source' }; + }, + }); + + await Table.get(1); + await waitFor(() => Table.primaryStore.getEntry(1)?.value, { message: 'source fill should be stored' }); + await Table.primaryStore.committed; + const sourceEntry = Table.primaryStore.getEntry(1); + assert.strictEqual(sourceEntry.expiresAt, undefined); + assert.ok(sourceEntry.metadataFlags & HAS_EXPIRATION_DECISION); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + const fullScanResults = []; + for await (const record of Table.search({ + allowFullScan: true, + conditions: [ + { attribute: 'id', value: 1 }, + { attribute: 'expiresAt', value: fieldExpiresAt }, + ], + })) { + fullScanResults.push(record.id); + } + assert.deepStrictEqual(fullScanResults, []); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.name, 'source'); + + await Table.put(1, { id: 1, expiresAt: fieldExpiresAt, name: 'local' }); + await Table.primaryStore.committed; + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], [1]); + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + }); + + it('propagates an explicit no-expiration decision through current and live subscriptions', async function () { + const Source = makeTable('ExpiresAtDecisionSource', undefined, { audit: true }); + const fieldExpiresAt = Date.now() - 1_000; + await Source.put(1, { id: 1, expiresAt: fieldExpiresAt }, { expiresAt: -1 }); + await Source.primaryStore.committed; + + const currentSubscription = await Source.subscribe({ isCollection: true }); + const currentEvents = []; + currentSubscription.on('data', (event) => currentEvents.push(event)); + await waitFor(() => currentEvents.some((event) => event.id === 1), { message: 'current event should arrive' }); + const currentEvent = currentEvents.find((event) => event.id === 1); + assert.strictEqual(currentEvent.expirationDecisionPresent, true); + assert.strictEqual(currentEvent.expiresAt, undefined); + await currentSubscription.return?.(); + + const Target = makeTable('ExpiresAtDecisionTarget'); + Target.sourcedFrom(Source, { intermediateSource: true }); + await Source.put(2, { id: 2, expiresAt: fieldExpiresAt }, { expiresAt: -1 }); + await waitFor(() => Target.primaryStore.getEntry(2)?.value, { message: 'live event should be applied' }); + await Target.primaryStore.committed; + const targetEntry = Target.primaryStore.getEntry(2); + assert.strictEqual(targetEntry.expiresAt, undefined); + assert.ok(targetEntry.metadataFlags & HAS_EXPIRATION_DECISION); + assert.deepStrictEqual([...Target.indices.expiresAt.getValues(fieldExpiresAt)], []); + }); + + it('preserves an explicit no-expiration decision across publish and invalidate', async function () { + const Table = makeTable('ExpiresAtDecisionLifecycle', undefined, { audit: true }); + const fieldExpiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt: fieldExpiresAt }, { expiresAt: -1 }); + await Table.publish(1, { message: 'refresh' }); + await Table.primaryStore.committed; + + let entry = Table.primaryStore.getEntry(1); + assert.strictEqual(entry.expiresAt, undefined); + assert.ok(entry.metadataFlags & HAS_EXPIRATION_DECISION); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + + await Table.invalidate(1); + await Table.primaryStore.committed; + entry = Table.primaryStore.getEntry(1); + assert.strictEqual(entry.expiresAt, undefined); + assert.ok(entry.metadataFlags & HAS_EXPIRATION_DECISION); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + }); + + it('invalidates a frozen replay record without mutating it', async function () { + const Table = makeTable('ExpiresAtFrozenInvalidate', undefined, { audit: true }); + const fieldExpiresAt = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt: fieldExpiresAt }); + const context = {}; + const resource = await Table.getResource(1, context, { ensureLoaded: true }); + const replayRecord = Object.freeze({ id: 1, expiresAt: fieldExpiresAt }); + await transaction(context, () => resource._writeInvalidate(1, replayRecord)); + assert.strictEqual(Object.isFrozen(replayRecord), true); + assert.ok(Table.primaryStore.getEntry(1).metadataFlags); + }); + it('re-arms both expiration sweeps after cleanup is resumed', async function () { const Table = makeTable('ExpiresAtResumeCleanup', undefined, { scanInterval: 100 }); assert.deepStrictEqual(Table.cleanupStateForTests(), { closed: false, cleanupScheduled: true, + nextCleanupScheduled: Table.cleanupStateForTests().nextCleanupScheduled, expirationScheduled: true, }); await Table.cleanup(); assert.deepStrictEqual(Table.cleanupStateForTests(), { closed: true, cleanupScheduled: false, + nextCleanupScheduled: undefined, expirationScheduled: false, }); await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); assert.deepStrictEqual(Table.cleanupStateForTests(), { closed: true, cleanupScheduled: false, + nextCleanupScheduled: undefined, expirationScheduled: false, }); Table.resumeCleanup(); assert.deepStrictEqual(Table.cleanupStateForTests(), { closed: false, cleanupScheduled: true, + nextCleanupScheduled: Table.cleanupStateForTests().nextCleanupScheduled, expirationScheduled: true, }); }); @@ -464,10 +572,58 @@ describe('@expiresAt attribute is authoritative over the table default', () => { Table.cleanup(); }); + it('removes both the encountered and canonical expiration keys when evicting a record', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtExactKeyEviction')); + const canonical = Date.now() - 1_000; + const encountered = canonical - 1_000; + await Table.put(1, { id: 1, expiresAt: canonical }); + await Table.primaryStore.committed; + Table.indices.expiresAt.put(encountered, 1); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(encountered)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(canonical)], []); + }); + + it('removes both expiration keys when physically evicting a blob record', async function () { + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'ExpiresAtExactBlobKeyEviction', + database: 'expires-at-exact-blob-key-eviction', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + const canonical = Date.now() - 1_000; + const encountered = canonical - 1_000; + const blob = createBlob(Buffer.alloc(20_000, 12)); + await Table.put(1, { id: 1, expiresAt: canonical, payload: blob }); + await Table.primaryStore.committed; + Table.indices.expiresAt.put(encountered, 1); + + setDeletionDelay(0); + try { + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(encountered)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(canonical)], []); + await waitFor(() => !existsSync(getFilePathForBlob(blob)), { + timeout: 5_000, + message: 'physical eviction should unlink the blob', + }); + } finally { + setDeletionDelay(500); + } + }); + it('keeps a blob when eviction lacks the record and unlinks it after a committed eviction', async function () { const Table = table({ table: 'ExpiresAtBlobFailSafe', - database: 'test', + database: 'expires-at-blob-fail-safe', attributes: [ { name: 'id', isPrimaryKey: true }, { name: 'expiresAt', expiresAt: true, indexed: true }, @@ -488,7 +644,10 @@ describe('@expiresAt attribute is authoritative over the table default', () => { try { await Table.evict(1, entry.value, entry.version); assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); - await waitFor(() => !existsSync(filePath), { message: 'committed eviction should unlink the blob' }); + await waitFor(() => !existsSync(filePath), { + timeout: 5_000, + message: 'committed eviction should unlink the blob', + }); } finally { setDeletionDelay(500); } @@ -673,6 +832,108 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } }); + it('drains an in-flight batch when the primary scan exits with an error', async function () { + const Table = table({ + table: 'PrimaryCleanupErrorDrain', + database: 'test', + expiration: 1, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const expiresAt = Date.now() - 1_000; + await Promise.all(Array.from({ length: 100 }, (_, id) => Table.put(id, { id }, { expiresAt }))); + await Table.primaryStore.committed; + + let releaseCommit; + let commitStarted = false; + const blockedCommit = new Promise((resolve) => (releaseCommit = resolve)); + let scanResolved = false; + try { + const scan = Table.runPrimaryCleanupScanForTests({ + beforeBatchCommit: async () => { + commitStarted = true; + await blockedCommit; + }, + afterBatchQueued() { + throw new Error('injected scan failure'); + }, + }).then(() => (scanResolved = true)); + await waitFor(() => commitStarted, { message: 'the eviction batch should start committing' }); + await delay(10); + assert.strictEqual(scanResolved, false, 'the scan must drain its batch after the injected failure'); + releaseCommit(); + await scan; + } finally { + releaseCommit(); + } + }); + + it('settles an immediate cleanup request when cleanup cancels its timer', async function () { + const Table = table({ + table: 'CleanupTimerSettlement', + database: 'test', + expiration: 60, + scanInterval: 3_600, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const scheduled = Table.scheduleCleanupForTests(2); + assert(scheduled, 'an explicit reclamation request should return its queued scan'); + const cleanup = Table.cleanup(); + const settled = await Promise.race([scheduled.then(() => true), delay(100).then(() => false)]); + assert.strictEqual(settled, true, 'cleanup must not strand a promise owned by a cleared timer'); + await cleanup; + }); + + it('keeps the recurring timer finite for full-disk reclamation priority', async function () { + const Table = table({ + table: 'FullDiskCleanupSchedule', + database: 'test', + expiration: 60, + scanInterval: 3_600, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const scheduled = Table.scheduleCleanupForTests(Infinity); + assert(scheduled, 'a full-disk reclamation request should queue an immediate scan'); + assert( + Number.isFinite(Table.cleanupStateForTests().nextCleanupScheduled), + 'the recurring cleanup deadline must stay finite' + ); + await scheduled; + }); + + it('waits for an admitted ordinary commit before closing a database', async function () { + const database = 'OrdinaryCommitCloseDrain'; + const Table = table({ + table: 'records', + database, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const originalAdmission = Table.primaryStore[TABLE_COMMIT_ADMISSION]; + let admitted = false; + let releaseCommit; + const blockedRelease = new Promise((resolve) => (releaseCommit = resolve)); + Table.primaryStore[TABLE_COMMIT_ADMISSION] = () => { + const release = originalAdmission(); + admitted = true; + return async () => { + await blockedRelease; + release(); + }; + }; + try { + const put = Table.put(1, { id: 1 }); + await waitFor(() => admitted, { message: 'the ordinary write should enter the table commit barrier' }); + let closeResolved = false; + const close = closeDatabase(database).then(() => (closeResolved = true)); + await delay(10); + assert.strictEqual(closeResolved, false, 'close must wait for the admitted ordinary commit'); + releaseCommit(); + await Promise.all([put, close]); + } finally { + releaseCommit(); + Table.primaryStore[TABLE_COMMIT_ADMISSION] = originalAdmission; + } + }); + it('waits for an active expiration sweep before destroying a database', async function () { const database = 'ExpiresAtDatabaseDrop'; const { Table, runSweep } = captureExpirationSweep(() => @@ -862,6 +1123,17 @@ describe('LMDB @expiresAt cleanup draining', function () { }; }; + it('persists an explicit no-expiration decision without indexing the public field', async function () { + const { Table } = captureExpirationSweep('LmdbExpirationDecision'); + const fieldExpiresAt = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt: fieldExpiresAt }, { expiresAt: -1 }); + await Table.primaryStore.committed; + const entry = Table.primaryStore.getEntry(1); + assert.strictEqual(entry.expiresAt, undefined); + assert.ok(entry.metadataFlags & HAS_EXPIRATION_DECISION); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + }); + it('evicts ISO expirations and repairs stale keys', async function () { const { Table, runSweep } = captureExpirationSweep('LmdbExpirationCorrectness'); const expired = Date.now() - 1_000; @@ -869,11 +1141,14 @@ describe('LMDB @expiresAt cleanup draining', function () { await Table.put(1, { id: 1, expiresAt: new Date(expired).toISOString() }); await Table.put(2, { id: 2, expiresAt: current }); await Table.primaryStore.committed; + const encountered = expired - 1_000; await Table.indices.expiresAt.put(expired, 2); + await Table.indices.expiresAt.put(encountered, 1); await runSweep(); assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expired)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(encountered)], []); assert.strictEqual(Table.primaryStore.getEntry(2)?.value.expiresAt, current); assert.deepStrictEqual([...Table.indices.expiresAt.getValues(current)], [2]); Table.cleanup(); diff --git a/unitTests/resources/recordEncoder.test.js b/unitTests/resources/recordEncoder.test.js index cbb97414b7..bc6ce113e5 100644 --- a/unitTests/resources/recordEncoder.test.js +++ b/unitTests/resources/recordEncoder.test.js @@ -1,6 +1,13 @@ require('../testUtils'); const assert = require('assert'); -const { RecordEncoder, RecordObject, isMissingStructureError } = require('#src/resources/RecordEncoder'); +const { + RecordEncoder, + RecordObject, + isMissingStructureError, + setNextEncoding, + clearNextEncoding, +} = require('#src/resources/RecordEncoder'); +const { HAS_EXPIRATION_DECISION } = require('#src/resources/auditStore'); const harperLogger = require('#src/utility/logging/harper_logger'); const { Encoder } = require('msgpackr'); @@ -31,6 +38,19 @@ function makeEncoder(randomAccessStructure, store) { const record = { name: 'price', type: 'Float', indexed: true }; +describe('RecordEncoder expiration decision metadata', () => { + afterEach(() => clearNextEncoding()); + + it('adds no bytes when the decision has no expiration payload', () => { + const encoder = makeEncoder(true, sharedStore()); + setNextEncoding(1000, 0); + const withoutDecision = Buffer.from(encoder.encode(record)); + setNextEncoding(1000, HAS_EXPIRATION_DECISION); + const withDecision = Buffer.from(encoder.encode(record)); + assert.equal(withDecision.length, withoutDecision.length); + }); +}); + describe('RecordEncoder struct-mode gating', () => { it('non-primary (randomAccessStructure off) writes records mode and bails the struct write hook', () => { const store = sharedStore(); diff --git a/unitTests/resources/transaction.test.js b/unitTests/resources/transaction.test.js index ad5baefb71..9e204ddbe1 100644 --- a/unitTests/resources/transaction.test.js +++ b/unitTests/resources/transaction.test.js @@ -5,13 +5,44 @@ const { setupTestDBPath } = require('../testUtils'); const { table } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { transaction } = require('#src/resources/transaction'); -const { DatabaseTransaction } = require('#src/resources/DatabaseTransaction'); +const { + DatabaseTransaction, + TABLE_COMMIT_ADMISSION, + withTableCommitAdmission, +} = require('#src/resources/DatabaseTransaction'); const { IterableEventQueue } = require('#src/resources/IterableEventQueue'); const { RocksDatabase } = require('@harperfast/rocksdb-js'); const harperLogger = require('#src/utility/logging/harper_logger'); const { resetReplayedWritesWarning } = require('#src/resources/DatabaseTransaction'); const isLMDB = process.env.HARPER_STORAGE_ENGINE === 'lmdb'; +describe('Table commit admission', () => { + it('admits every existing transaction link before committing and preserves dynamic admission', async () => { + const events = []; + const store = (name) => ({ + [TABLE_COMMIT_ADMISSION]() { + events.push(`admit:${name}`); + return () => events.push(`release:${name}`); + }, + }); + const storeA = store('a'); + const storeB = store('b'); + const storeC = store('c'); + const tail = { writes: [{ store: storeB }], next: undefined }; + const head = { writes: [{ store: storeA }], next: tail }; + + await withTableCommitAdmission(head, {}, (options) => { + assert.deepStrictEqual(events, ['admit:a', 'admit:b']); + tail.writes.push({ store: storeC }); + return withTableCommitAdmission(tail, options, async () => { + assert.deepStrictEqual(events, ['admit:a', 'admit:b', 'admit:c']); + }); + }); + + assert.deepStrictEqual(events, ['admit:a', 'admit:b', 'admit:c', 'release:a', 'release:b', 'release:c']); + }); +}); + // The package blocks deep imports of its package.json, so walk up from the resolved entry point. function installedRocksdbVersion() { const { existsSync, readFileSync } = require('node:fs'); diff --git a/unitTests/security/certificateVerification/certificateVerificationSource.test.js b/unitTests/security/certificateVerification/certificateVerificationSource.test.js index 965ac361e5..2052cd7e10 100644 --- a/unitTests/security/certificateVerification/certificateVerificationSource.test.js +++ b/unitTests/security/certificateVerification/certificateVerificationSource.test.js @@ -87,7 +87,7 @@ describe('certificateVerification/certificateVerificationSource.ts', function () const cacheTtl = 3600000; // 1 hour const beforeTime = Date.now(); - sinon.stub(source, 'getContext').returns({ + const context = { requestContext: { certPem: 'cert', issuerPem: 'issuer', @@ -95,7 +95,8 @@ describe('certificateVerification/certificateVerificationSource.ts', function () crl: { cacheTtl, timeout: 10000, failureMode: 'fail-closed', gracePeriod: 86400000 }, }, }, - }); + }; + sinon.stub(source, 'getContext').returns(context); performCRLCheckStub.resolves({ status: 'good' }); @@ -105,6 +106,7 @@ describe('certificateVerification/certificateVerificationSource.ts', function () // expiresAt should be approximately now + cacheTtl assert.ok(result.expiresAt >= beforeTime + cacheTtl); assert.ok(result.expiresAt <= afterTime + cacheTtl + 100); // Allow 100ms tolerance + assert.strictEqual(context.expiresAt, result.expiresAt); }); }); diff --git a/unitTests/server/itc/serverHandlers.test.js b/unitTests/server/itc/serverHandlers.test.js index 7e45cf6577..38849c8276 100644 --- a/unitTests/server/itc/serverHandlers.test.js +++ b/unitTests/server/itc/serverHandlers.test.js @@ -134,6 +134,247 @@ describe('Test hdbChildIpcHandler module', () => { expect(log_error_stub).to.have.been.called; }); + it('returns the explicit quiesce result without resetting databases', async () => { + const expectedResult = { quiesced: true }; + const quiesceStub = sandbox.stub().resolves(expectedResult); + const cleanStub = sandbox.stub().resolves(); + const resetStub = sandbox.stub().returns({}); + const restoreQuiesce = server_itc_handlers.__set__('quiesceSchemaTarget', quiesceStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + const restoreReset = server_itc_handlers.__set__('resetDatabases', resetStub); + try { + const message = { + originator: 12345, + operation: 'drop_table', + phase: 'quiesce', + schema: 'test', + table: 'records', + quiesceId: 'q-1', + }; + const result = await schema_handler({ type: 'schema', message }); + expect(result).to.equal(expectedResult); + expect(quiesceStub).to.have.been.calledOnceWithExactly(message); + expect(cleanStub).not.to.have.been.called; + expect(resetStub).not.to.have.been.called; + } finally { + restoreReset(); + restoreClean(); + restoreQuiesce(); + } + }); + + it('aborts quiescence without resetting databases', async () => { + const abortStub = sandbox.stub().resolves(); + const cleanStub = sandbox.stub().resolves(); + const resetStub = sandbox.stub().returns({}); + const restoreAbort = server_itc_handlers.__set__('abortSchemaQuiesce', abortStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + const restoreReset = server_itc_handlers.__set__('resetDatabases', resetStub); + try { + const message = { + originator: 12345, + operation: 'drop_table', + phase: 'abort-quiesce', + schema: 'test', + table: 'records', + quiesceId: 'q-2', + }; + const result = await schema_handler({ type: 'schema', message }); + expect(result).to.deep.equal({ aborted: true }); + expect(abortStub).to.have.been.calledOnceWithExactly(message); + expect(cleanStub).not.to.have.been.called; + expect(resetStub).not.to.have.been.called; + } finally { + restoreReset(); + restoreClean(); + restoreAbort(); + } + }); + + it('finishes quiescence before the normal database reset', async () => { + const calls = []; + const finishStub = sandbox.stub().callsFake(() => calls.push('finish')); + const cleanStub = sandbox.stub().callsFake(async () => calls.push('clean')); + const resetStub = sandbox.stub().callsFake(() => { + calls.push('reset'); + return {}; + }); + const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); + const completeStub = sandbox.stub(); + const restoreComplete = server_itc_handlers.__set__('completeSchemaQuiesce', completeStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + const restoreReset = server_itc_handlers.__set__('resetDatabases', resetStub); + try { + const message = { + originator: 12345, + operation: 'drop_table', + phase: 'finalize-quiesce', + schema: 'test', + quiesceId: 'q-3', + }; + const result = await schema_handler({ type: 'schema', message }); + expect(finishStub).to.have.been.calledOnceWithExactly(message); + expect(cleanStub).to.have.been.calledOnceWithExactly(message); + expect(resetStub).to.have.been.calledOnce; + expect(completeStub).to.have.been.calledOnceWithExactly(message); + expect(result).to.deep.equal({ finalized: true }); + expect(calls).to.deep.equal(['finish', 'clean', 'reset']); + } finally { + restoreReset(); + restoreClean(); + restoreComplete(); + restoreFinish(); + } + }); + + it('coalesces concurrent terminal retries for the same quiescence', async () => { + let releaseReset; + const resetBlocked = new Promise((resolve) => (releaseReset = resolve)); + const finishStub = sandbox.stub().returns(true); + const cleanStub = sandbox.stub().resolves(); + const syncStub = sandbox.stub().callsFake(() => resetBlocked); + const completeStub = sandbox.stub(); + const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + const restoreSync = server_itc_handlers.__set__('syncSchemaMetadata', syncStub); + const restoreComplete = server_itc_handlers.__set__('completeSchemaQuiesce', completeStub); + try { + const message = { + originator: 12345, + operation: 'drop_schema', + phase: 'finalize-quiesce', + schema: 'test', + quiesceId: 'q-concurrent-finalize', + }; + const first = schema_handler({ type: 'schema', message }); + const retry = schema_handler({ type: 'schema', message }); + releaseReset(); + expect(await Promise.all([first, retry])).to.deep.equal([{ finalized: true }, { finalized: true }]); + expect(await schema_handler({ type: 'schema', message })).to.deep.equal({ finalized: true }); + expect(finishStub).to.have.been.calledOnceWithExactly(message); + expect(cleanStub).to.have.been.calledOnceWithExactly(message); + expect(syncStub).to.have.been.calledOnceWithExactly(message, true); + expect(completeStub).to.have.been.calledOnceWithExactly(message); + } finally { + restoreComplete(); + restoreSync(); + restoreClean(); + restoreFinish(); + } + }); + + it('rejects a mismatched terminal request that reuses a completed ID', async () => { + const finishStub = sandbox.stub().returns(true); + const syncStub = sandbox.stub().resolves(); + const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); + const restoreSync = server_itc_handlers.__set__('syncSchemaMetadata', syncStub); + try { + const message = { + originator: 12345, + operation: 'drop_schema', + phase: 'finalize-quiesce', + schema: 'test', + quiesceId: 'q-terminal-identity', + }; + expect(await schema_handler({ type: 'schema', message })).to.deep.equal({ finalized: true }); + expect( + await schema_handler({ + type: 'schema', + message: { ...message, schema: 'other' }, + }) + ).to.deep.equal({ finalized: false }); + expect(finishStub).to.have.been.calledOnce; + expect(syncStub).to.have.been.calledOnce; + } finally { + restoreSync(); + restoreFinish(); + } + }); + + it('bounds retained terminal outcomes', () => { + const outcomes = server_itc_handlers.__get__('schemaTerminalOutcomes'); + const retain = server_itc_handlers.__get__('retainSchemaTerminalOutcome'); + const maximum = server_itc_handlers.__get__('MAX_SCHEMA_TERMINAL_OUTCOMES'); + outcomes.clear(); + for (let index = 0; index <= maximum; index++) { + retain( + { + operation: 'drop_schema', + phase: 'finalize-quiesce', + schema: `test-${index}`, + quiesceId: `q-bounded-${index}`, + }, + { finalized: true } + ); + } + expect(outcomes.size).to.equal(maximum); + expect(outcomes.has('q-bounded-0')).to.equal(false); + expect(outcomes.has(`q-bounded-${maximum}`)).to.equal(true); + outcomes.clear(); + }); + + it('does not acknowledge finalization when the strict reset fails', async () => { + const finishStub = sandbox.stub().returns(true); + const syncStub = sandbox.stub().rejects(new Error('reset failed')); + const failStub = sandbox.stub(); + const completeStub = sandbox.stub(); + const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); + const restoreSync = server_itc_handlers.__set__('syncSchemaMetadata', syncStub); + const restoreFail = server_itc_handlers.__set__('failSchemaQuiesceFinalization', failStub); + const restoreComplete = server_itc_handlers.__set__('completeSchemaQuiesce', completeStub); + try { + const message = { + originator: 12345, + operation: 'drop_schema', + phase: 'finalize-quiesce', + schema: 'test', + quiesceId: 'q-reset-failure', + }; + let failure; + try { + await schema_handler({ type: 'schema', message }); + } catch (error) { + failure = error; + } + expect(failure?.message).to.equal('reset failed'); + expect(failStub).to.have.been.calledOnceWithExactly(message); + expect(completeStub).not.to.have.been.called; + } finally { + restoreComplete(); + restoreFail(); + restoreSync(); + restoreFinish(); + } + }); + + it('returns an explicit result after authoritative reconciliation', async () => { + const finishStub = sandbox.stub().returns(true); + const syncStub = sandbox.stub().resolves(); + const completeStub = sandbox.stub(); + const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); + const restoreSync = server_itc_handlers.__set__('syncSchemaMetadata', syncStub); + const restoreComplete = server_itc_handlers.__set__('completeSchemaQuiesce', completeStub); + try { + const message = { + originator: 12345, + operation: 'drop_table', + phase: 'reconcile-quiesce', + schema: 'test', + table: 'records', + quiesceId: 'q-reconcile', + }; + const result = await schema_handler({ type: 'schema', message }); + expect(result).to.deep.equal({ reconciled: true }); + expect(finishStub).to.have.been.calledOnceWithExactly(message); + expect(syncStub).to.have.been.calledOnceWithExactly(message, true); + expect(completeStub).to.have.been.calledOnceWithExactly(message); + } finally { + restoreComplete(); + restoreSync(); + restoreFinish(); + } + }); + it('retries a restore close failure and still acknowledges through the schema work', async () => { const closeStub = sandbox.stub(); closeStub.onFirstCall().rejects(new Error('cleanup still active')); diff --git a/unitTests/server/threads/acknowledgement-fixture.js b/unitTests/server/threads/acknowledgement-fixture.js new file mode 100644 index 0000000000..bb75d358b4 --- /dev/null +++ b/unitTests/server/threads/acknowledgement-fixture.js @@ -0,0 +1,14 @@ +'use strict'; + +const { parentPort } = require('node:worker_threads'); + +parentPort.on('message', (message) => { + if (message.type !== 'acknowledgement-test') return; + if (message.action === 'timeout') return; + if (message.action === 'close') return parentPort.close(); + parentPort.postMessage({ + type: 'ack', + id: message.requestId, + ...(message.includeAcknowledgementResult === true ? { result: message.result } : null), + }); +}); diff --git a/unitTests/server/threads/broadcastAcknowledgement.test.js b/unitTests/server/threads/broadcastAcknowledgement.test.js new file mode 100644 index 0000000000..48fa558124 --- /dev/null +++ b/unitTests/server/threads/broadcastAcknowledgement.test.js @@ -0,0 +1,151 @@ +'use strict'; + +const assert = require('node:assert'); +const path = require('node:path'); +const { + broadcastWithAcknowledgement, + ITCAcknowledgementError, + startWorker, + holdWorkerStartsForSchema, + releaseWorkerStartsForSchema, + waitForSchemaWorkerStarts, +} = require('#js/server/threads/manageThreads'); + +const FIXTURE = path.join(__dirname, 'acknowledgement-fixture.js'); +const ACCEPT_QUIESCED = (result) => result?.quiesced === true; + +describe('broadcastWithAcknowledgement', () => { + let worker; + let jobWorker; + + function startTestWorker(name) { + return new Promise((resolve, reject) => { + const spawned = startWorker(FIXTURE, { + name, + autoRestart: false, + onStarted(started) { + started.once('online', () => resolve(spawned)); + started.once('error', reject); + started.once('exit', (code) => reject(new Error(`Worker exited before online (code ${code})`))); + }, + }); + }); + } + + beforeEach(async function () { + this.timeout(30000); + worker = await startTestWorker('acknowledgement-test'); + }); + + afterEach(async () => { + releaseWorkerStartsForSchema('test-barrier-1'); + releaseWorkerStartsForSchema('test-barrier-2'); + for (const spawned of [worker, jobWorker]) { + if (!spawned) continue; + spawned.wasShutdown = true; + await spawned.terminate(); + } + worker = undefined; + jobWorker = undefined; + }); + + it('holds worker starts until every schema barrier is released', async () => { + holdWorkerStartsForSchema('test-barrier-1'); + holdWorkerStartsForSchema('test-barrier-2'); + let released = false; + const waiting = waitForSchemaWorkerStarts().then(() => (released = true)); + await new Promise(setImmediate); + assert.strictEqual(released, false); + releaseWorkerStartsForSchema('test-barrier-1'); + await new Promise(setImmediate); + assert.strictEqual(released, false); + releaseWorkerStartsForSchema('test-barrier-2'); + await waiting; + assert.strictEqual(released, true); + }); + + it('returns accepted handler results in strict mode', async () => { + const results = await broadcastWithAcknowledgement( + { type: 'acknowledgement-test', action: 'result', result: { quiesced: true } }, + { timeout: 1000, acceptResult: ACCEPT_QUIESCED } + ); + assert.strictEqual(results.length, 1); + assert.strictEqual(results[0].status, 'accepted'); + assert.deepStrictEqual(results[0].result, { quiesced: true }); + }); + + it('retains a negative handler payload in strict failures', async () => { + await assert.rejects( + broadcastWithAcknowledgement( + { type: 'acknowledgement-test', action: 'result', result: { quiesced: false, reason: 'busy' } }, + { timeout: 1000, acceptResult: ACCEPT_QUIESCED } + ), + (error) => { + assert.strictEqual(error instanceof ITCAcknowledgementError, true); + assert.strictEqual(error.results[0].status, 'rejected'); + assert.deepStrictEqual(error.results[0].result, { quiesced: false, reason: 'busy' }); + return true; + } + ); + }); + + it('distinguishes strict acknowledgement timeouts', async () => { + await assert.rejects( + broadcastWithAcknowledgement( + { type: 'acknowledgement-test', action: 'timeout' }, + { timeout: 10, acceptResult: ACCEPT_QUIESCED } + ), + (error) => { + assert.strictEqual(error.results[0].status, 'timeout'); + return true; + } + ); + }); + + it('distinguishes a port closing before acknowledgement', async () => { + await assert.rejects( + broadcastWithAcknowledgement( + { type: 'acknowledgement-test', action: 'close' }, + { timeout: 1000, acceptResult: ACCEPT_QUIESCED } + ), + (error) => { + assert.strictEqual(error.results[0].status, 'closed'); + return true; + } + ); + }); + + it('distinguishes synchronous transport failures', async () => { + await assert.rejects( + broadcastWithAcknowledgement( + { type: 'acknowledgement-test', result: () => {} }, + { timeout: 1000, acceptResult: ACCEPT_QUIESCED } + ), + (error) => { + assert.strictEqual(error.results[0].status, 'transport-error'); + return true; + } + ); + }); + + it('preserves legacy void resolution without requiring a result', async () => { + const result = await broadcastWithAcknowledgement({ type: 'acknowledgement-test', action: 'legacy' }, 1000); + assert.strictEqual(result, undefined); + }); + + it('includes active job workers only when explicitly requested', async () => { + jobWorker = await startTestWorker('job'); + const message = { type: 'acknowledgement-test', action: 'result', result: { quiesced: true } }; + const defaultResults = await broadcastWithAcknowledgement(message, { + timeout: 1000, + acceptResult: ACCEPT_QUIESCED, + }); + assert.strictEqual(defaultResults.length, 1); + const ddlResults = await broadcastWithAcknowledgement(message, { + timeout: 1000, + acceptResult: ACCEPT_QUIESCED, + includeJobWorkers: true, + }); + assert.strictEqual(ddlResults.length, 2); + }); +}); diff --git a/unitTests/utility/signalling.test.js b/unitTests/utility/signalling.test.js index 2f654f5664..8c8ce825b7 100644 --- a/unitTests/utility/signalling.test.js +++ b/unitTests/utility/signalling.test.js @@ -26,6 +26,7 @@ describe('Test signalling module', () => { }); afterEach(() => { + send_itc_event_stub.resetBehavior(); send_itc_event_stub.returns(); sandbox.resetHistory(); }); @@ -72,4 +73,122 @@ describe('Test signalling module', () => { signalling.signalUserChange('message'); expect(log_error_stub.lastCall.args[0].name).to.equal(TEST_ERROR); }); + + it('strictly quiesces job workers and stops renewal when aborted', async () => { + send_itc_event_stub.resolves(); + const localSchemaHandler = sandbox.stub().callsFake(async (event) => { + const resultByPhase = { + 'hold-worker-starts': 'held', + 'quiesce': 'quiesced', + 'abort-quiesce': 'aborted', + 'release-worker-starts': 'released', + }; + return { [resultByPhase[event.message.phase]]: true }; + }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + try { + const message = await signalling.quiesceSchemaChange({ + operation: 'drop_table', + schema: 'unit_test', + table: 'records', + }); + const quiesceCall = send_itc_event_stub.getCalls().find((call) => call.args[0].message.phase === 'quiesce'); + expect(quiesceCall.args[1].includeJobWorkers).to.equal(true); + expect(quiesceCall.args[1].acceptResult({ quiesced: true })).to.equal(true); + await signalling.abortSchemaQuiesce(message); + const abortCall = send_itc_event_stub.getCalls().find((call) => call.args[0].message.phase === 'abort-quiesce'); + expect(abortCall.args[1].includeJobWorkers).to.equal(true); + expect(abortCall.args[1].acceptResult({ aborted: true })).to.equal(true); + } finally { + restoreHandlers(); + } + }); + + it('retries strict terminal finalization and requires explicit worker results', async () => { + const localSchemaHandler = sandbox.stub().resolves({ finalized: true }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + send_itc_event_stub.onFirstCall().rejects(new Error('first terminal failure')); + send_itc_event_stub.onSecondCall().rejects(new Error('second terminal failure')); + send_itc_event_stub.onThirdCall().resolves(); + send_itc_event_stub.onCall(3).resolves(); + try { + await signalling.finalizeSchemaChange({ + operation: 'drop_schema', + schema: 'unit_test', + quiesceId: 'q-final', + }); + expect(localSchemaHandler.firstCall.args[0].message.phase).to.equal('finalize-quiesce'); + const terminalCalls = send_itc_event_stub + .getCalls() + .filter((call) => call.args[0].message.phase === 'finalize-quiesce'); + expect(terminalCalls).to.have.length(3); + for (const call of terminalCalls) { + const options = call.args[1]; + expect(options.includeJobWorkers).to.equal(true); + expect(options.acceptResult({ finalized: true })).to.equal(true); + } + } finally { + restoreHandlers(); + } + }); + + it('requires explicit reconciliation results from local and job workers', async () => { + const localSchemaHandler = sandbox.stub().resolves({ reconciled: true }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + send_itc_event_stub.resolves(); + try { + await signalling.reconcileSchemaChange({ + operation: 'drop_table', + schema: 'unit_test', + table: 'records', + quiesceId: 'q-reconcile', + }); + const event = localSchemaHandler.firstCall.args[0]; + expect(event.message.phase).to.equal('reconcile-quiesce'); + const terminalCall = send_itc_event_stub + .getCalls() + .find((call) => call.args[0].message.phase === 'reconcile-quiesce'); + const options = terminalCall.args[1]; + expect(options.includeJobWorkers).to.equal(true); + expect(options.acceptResult({ reconciled: true })).to.equal(true); + expect(options.acceptResult({ finalized: true })).to.equal(false); + } finally { + restoreHandlers(); + } + }); + + it('reconciles and releases the worker barrier after a partial commit failure', async () => { + const localSchemaHandler = sandbox.stub().callsFake(async (event) => { + if (event.message.phase === 'commit-quiesce') return { committed: true }; + if (event.message.phase === 'reconcile-quiesce') return { reconciled: true }; + if (event.message.phase === 'release-worker-starts') return { released: true }; + }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + const message = { + operation: 'drop_table', + schema: 'unit_test', + table: 'records', + quiesceId: 'q-partial-commit', + }; + send_itc_event_stub.rejects(new Error('commit acknowledgement lost')); + try { + let commitError; + try { + await signalling.commitSchemaChange(message); + } catch (error) { + commitError = error; + } + expect(commitError).to.exist; + expect( + localSchemaHandler.getCalls().some((call) => call.args[0].message.phase === 'release-worker-starts') + ).to.equal(false); + send_itc_event_stub.resolves(); + await signalling.reconcileSchemaChange(message); + expect( + localSchemaHandler.getCalls().some((call) => call.args[0].message.phase === 'release-worker-starts') + ).to.equal(true); + } finally { + restoreHandlers(); + } + }); }); diff --git a/utility/signalling.ts b/utility/signalling.ts index aae59d81cd..b75dc5bbda 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -3,6 +3,7 @@ import * as hdbTerms from './hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import ITCEventObject from '../server/itc/utility/ITCEventObject.js'; +import { randomUUID } from 'node:crypto'; let serverItcHandlers; import { sendItcEvent } from '../server/threads/itc.js'; @@ -23,6 +24,191 @@ export async function signalSchemaChange(message: any) { } } +const SCHEMA_QUIESCE_TIMEOUT_MS = 30_000; +const SCHEMA_QUIESCE_LEASE_MS = 120_000; +const SCHEMA_QUIESCE_RENEW_MS = 40_000; +const SCHEMA_FINALIZE_ATTEMPTS = 3; +const schemaQuiesceRenewals = new Map(); + +function stopSchemaQuiesceRenewal(quiesceId: string) { + const renewal = schemaQuiesceRenewals.get(quiesceId); + if (renewal) clearInterval(renewal); + schemaQuiesceRenewals.delete(quiesceId); +} + +function startSchemaQuiesceRenewal(message: any) { + stopSchemaQuiesceRenewal(message.quiesceId); + const renewal = setInterval(() => { + const renewalMessage = { + ...message, + phase: 'renew-quiesce', + leaseUntil: Date.now() + SCHEMA_QUIESCE_LEASE_MS, + }; + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, renewalMessage); + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + Promise.all([ + serverItcHandlers.schema(event).then((result) => { + if (result?.quiesced !== true) throw new Error(`Could not renew local schema quiesce ${message.quiesceId}`); + }), + sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.quiesced === true, + includeJobWorkers: true, + }), + ]).catch((error) => hdbLogger.warn('Could not renew schema quiesce lease:', error)); + }, SCHEMA_QUIESCE_RENEW_MS); + renewal.unref(); + schemaQuiesceRenewals.set(message.quiesceId, renewal); +} + +export async function quiesceSchemaChange(message: any) { + const quiesceMessage = { + ...message, + phase: 'quiesce', + quiesceId: randomUUID(), + leaseUntil: Date.now() + SCHEMA_QUIESCE_LEASE_MS, + }; + try { + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + const holdEvent = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, { + ...quiesceMessage, + phase: 'hold-worker-starts', + }); + const localHold = await serverItcHandlers.schema(holdEvent); + if (localHold?.held !== true) throw new Error(`Could not hold local worker starts for ${quiesceMessage.quiesceId}`); + await sendItcEvent(holdEvent, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.held === true, + includeJobWorkers: true, + }); + const localEvent = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, { + ...quiesceMessage, + originLocal: true, + }); + const localResult = await serverItcHandlers.schema(localEvent); + if (localResult?.quiesced !== true) + throw new Error(localResult?.reason ?? `Could not quiesce local schema target ${quiesceMessage.quiesceId}`); + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, quiesceMessage); + await sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.quiesced === true, + includeJobWorkers: true, + }); + startSchemaQuiesceRenewal(quiesceMessage); + return quiesceMessage; + } catch (error) { + await abortSchemaQuiesce(quiesceMessage); + throw error; + } +} + +export async function abortSchemaQuiesce(message: any) { + stopSchemaQuiesceRenewal(message.quiesceId); + try { + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, { + ...message, + phase: 'abort-quiesce', + }); + const localResult = await serverItcHandlers.schema(event); + if (localResult?.aborted !== true) throw new Error(`Could not abort local schema quiesce ${message.quiesceId}`); + await sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.aborted === true, + includeJobWorkers: true, + }); + } catch (error) { + hdbLogger.warn('Could not confirm schema quiesce abort on every worker:', error); + } finally { + await releaseSchemaWorkerBarrier(message); + } +} + +async function releaseSchemaWorkerBarrier(message: any) { + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, { + ...message, + phase: 'release-worker-starts', + }); + await serverItcHandlers.schema(event); + try { + await sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.released === true, + includeJobWorkers: true, + }); + } catch (error) { + hdbLogger.warn('Could not confirm worker-start barrier release:', error); + } +} + +export async function commitSchemaChange(message: any) { + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + const commitMessage = { ...message, phase: 'commit-quiesce' }; + const localEvent = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, commitMessage); + const localResult = await serverItcHandlers.schema(localEvent); + if (localResult?.committed !== true) + throw new Error(localResult?.reason ?? `Could not commit local schema quiesce ${message.quiesceId}`); + let lastError: unknown; + for (let attempt = 0; attempt < SCHEMA_FINALIZE_ATTEMPTS; attempt++) { + try { + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, commitMessage); + await sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.committed === true, + includeJobWorkers: true, + }); + return; + } catch (error) { + lastError = error; + } + } + throw new Error(`Could not commit schema quiesce ${message.quiesceId} on every worker; remaining fail-closed`, { + cause: lastError, + }); +} + +async function completeSchemaChange(message: any, phase: string, resultProperty: string) { + let completed = false; + try { + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + const finalMessage = { ...message, phase }; + const localEvent = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, finalMessage); + const localResult = await serverItcHandlers.schema(localEvent); + if (localResult?.[resultProperty] !== true) + throw new Error(`Could not ${resultProperty} local schema quiesce ${message.quiesceId}`); + let lastError: unknown; + for (let attempt = 0; attempt < SCHEMA_FINALIZE_ATTEMPTS; attempt++) { + try { + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, finalMessage); + await sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.[resultProperty] === true, + includeJobWorkers: true, + }); + completed = true; + return; + } catch (error) { + lastError = error; + } + } + throw new Error(`Could not ${resultProperty} schema quiesce ${message.quiesceId} on every worker`, { + cause: lastError, + }); + } finally { + stopSchemaQuiesceRenewal(message.quiesceId); + if (completed) await releaseSchemaWorkerBarrier(message); + } +} + +export function finalizeSchemaChange(message: any) { + return completeSchemaChange(message, 'finalize-quiesce', 'finalized'); +} + +export function reconcileSchemaChange(message: any) { + return completeSchemaChange(message, 'reconcile-quiesce', 'reconciled'); +} + /** * Notify local listeners that JS resources have just been registered (resources.js loaded). This is * deliberately local-only — no ITC broadcast — because every worker registers its own JS resources, From 8c6a04aa9ab18ff2855cdf6b8da7d336f92e4901 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 14:59:27 -0600 Subject: [PATCH 27/43] Close quiescence recovery gaps --- .../database/expiration-drop-quiesce.test.ts | 53 +++-- .../expiration-drop-quiesce/resources.js | 104 +++++++-- resources/DatabaseTransaction.ts | 70 +++--- resources/Table.ts | 208 ++++++++++++------ resources/databases.ts | 102 ++++++++- resources/transaction.ts | 11 +- storage-format.md | 17 ++ unitTests/resources/databases.test.js | 86 +++++++- .../resources/expiresAtAttribute.test.js | 120 +++++++++- unitTests/resources/transaction.test.js | 29 ++- unitTests/utility/signalling.test.js | 33 +++ utility/signalling.ts | 4 +- 12 files changed, 676 insertions(+), 161 deletions(-) diff --git a/integrationTests/database/expiration-drop-quiesce.test.ts b/integrationTests/database/expiration-drop-quiesce.test.ts index 6ed94f6e05..99f0248dbf 100644 --- a/integrationTests/database/expiration-drop-quiesce.test.ts +++ b/integrationTests/database/expiration-drop-quiesce.test.ts @@ -1,6 +1,6 @@ import { after, before, suite, test } from 'node:test'; import { strictEqual } from 'node:assert'; -import { existsSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; @@ -22,6 +22,7 @@ async function waitFor(predicate: () => boolean, message: string) { suite('cross-worker expiration cleanup quiesces destructive DDL', (ctx: ContextWithHarper) => { let client: ReturnType; + let workerIds: number[]; before(async () => { await setupHarperWithFixture(ctx, FIXTURE_PATH, { @@ -48,6 +49,9 @@ suite('cross-worker expiration cleanup quiesces destructive DDL', (ctx: ContextW await delay(250); } strictEqual(ready, true, 'QuiesceControl resource did not become ready'); + const probe = await postControl({ action: 'probe' }); + workerIds = probe.body.workerIds; + strictEqual(workerIds.length, 2, 'the fixture must expose both HTTP workers for deterministic pinning'); }); after(async () => { @@ -58,36 +62,53 @@ suite('cross-worker expiration cleanup quiesces destructive DDL', (ctx: ContextW return request(client.restURL).post('/QuiesceControl/').set(client.headers).timeout(120_000).send(body); } + async function postControlOnWorker(body: Record, targetThreadId: number) { + const response = await postControl({ ...body, targetThreadId }); + strictEqual(response.body?.threadId, targetThreadId, `control action did not execute on worker ${targetThreadId}`); + return response; + } + async function proveDropWaitsForWorker(kind: 'indexed' | 'primary') { const indexed = kind === 'indexed'; const database = indexed ? 'quiesce_indexed' : 'quiesce_primary'; const table = indexed ? 'IndexedExpiry' : 'PrimaryExpiry'; const runId = `${kind}-${Date.now()}`; + const [sweepWorkerId, ddlWorkerId] = workerIds; await postControl({ action: 'seed', kind, database, table, id: runId }).then((response) => strictEqual(response.status, 200) ); - const started = join(CONTROL_DIRECTORY, `${runId}.started`); + const sweepRunId = `${runId}-sweep`; + const dropRunId = `${runId}-drop`; + const started = join(CONTROL_DIRECTORY, `${sweepRunId}.started`); const release = join(CONTROL_DIRECTORY, `${runId}.release`); - const sweep = postControl({ action: 'sweep', kind, database, table, runId }); + const sweep = postControlOnWorker( + { action: 'sweep', kind, database, table, runId: sweepRunId, releaseRunId: runId }, + sweepWorkerId + ); await waitFor(() => existsSync(started), `${kind} sweep did not reach its blocked commit`); + strictEqual(JSON.parse(readFileSync(started, 'utf8')).threadId, sweepWorkerId); let dropSettled = false; - const drop = client - .req() - .timeout(120_000) - .send({ operation: 'drop_table', schema: database, table }) - .then((response) => { + const dropStarted = join(CONTROL_DIRECTORY, `${dropRunId}.started`); + const drop = postControlOnWorker({ action: 'drop', database, table, runId: dropRunId }, ddlWorkerId).then( + (response) => { dropSettled = true; return response; - }); - const described = await client.req().timeout(10_000).send({ operation: 'describe_table', schema: database, table }); - strictEqual(described.status, 200, 'the table must remain visible before cross-worker quiescence completes'); - strictEqual(dropSettled, false, 'physical drop must wait for the blocked cleanup worker'); - - writeFileSync(release, 'release'); - const dropped = await drop; + } + ); + let dropped; + try { + await waitFor(() => existsSync(dropStarted), `${kind} drop did not start on its pinned worker`); + strictEqual(JSON.parse(readFileSync(dropStarted, 'utf8')).threadId, ddlWorkerId); + strictEqual(sweepWorkerId === ddlWorkerId, false, 'DDL and cleanup must execute on different workers'); + await delay(100); + strictEqual(dropSettled, false, 'physical drop must wait for the blocked cleanup worker'); + } finally { + writeFileSync(release, 'release'); + [dropped] = await Promise.all([drop, sweep]); + } strictEqual(dropped.status, 200); - await sweep; + strictEqual(dropped.body.threadId, ddlWorkerId); } test('drop_table waits for a remote indexed expiration sweep', async () => { diff --git a/integrationTests/database/expiration-drop-quiesce/resources.js b/integrationTests/database/expiration-drop-quiesce/resources.js index 7851bab9d7..ca2cd5b050 100644 --- a/integrationTests/database/expiration-drop-quiesce/resources.js +++ b/integrationTests/database/expiration-drop-quiesce/resources.js @@ -1,31 +1,99 @@ import { existsSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; +import { threadId } from 'node:worker_threads'; const controlDirectory = process.env.EXPIRATION_QUIESCE_CONTROL; +const CONTROL_REQUEST = 'expiration-quiesce-control-request'; +const CONTROL_RESPONSE = 'expiration-quiesce-control-response'; +const pendingRequests = new Map(); +let nextRequestId = 1; + +async function performControl(body) { + const Table = databases[body.database]?.[body.table]; + if (!Table) return { available: false, threadId }; + if (body.action === 'seed') { + const expiresAt = Date.now() - 10_000; + if (body.kind === 'indexed') await Table.put(body.id, { id: body.id, expiresAt }); + else await Table.put(body.id, { id: body.id, value: 'expired' }, { expiresAt }); + return { seeded: true, threadId }; + } + const started = join(controlDirectory, `${body.runId}.started`); + const release = join(controlDirectory, `${body.releaseRunId ?? body.runId}.release`); + if (body.action === 'drop') { + writeFileSync(started, JSON.stringify({ threadId })); + await Table.dropTable(); + return { completed: true, threadId }; + } + const hooks = { + beforeBatchCommit: async () => { + writeFileSync(started, JSON.stringify({ threadId })); + while (!existsSync(release)) await delay(20); + }, + }; + if (body.kind === 'indexed') await Table.runRecordExpirationSweepForTests(hooks); + else await Table.runPrimaryCleanupScanForTests(hooks); + return { completed: true, threadId }; +} + +threads.onMessageByType(CONTROL_REQUEST, async (message) => { + if (message.targetThreadId !== threadId) return; + try { + const result = await performControl(message.body); + threads.sendToThread(message.originThreadId, { type: CONTROL_RESPONSE, requestId: message.requestId, result }); + } catch (error) { + threads.sendToThread(message.originThreadId, { + type: CONTROL_RESPONSE, + requestId: message.requestId, + error: error?.stack ?? error?.message ?? String(error), + }); + } +}); + +threads.onMessageByType(CONTROL_RESPONSE, (message) => { + const pending = pendingRequests.get(message.requestId); + if (!pending) return; + pendingRequests.delete(message.requestId); + clearTimeout(pending.timer); + if (message.error) pending.reject(new Error(message.error)); + else pending.resolve(message.result); +}); + +function performControlOnWorker(body, targetThreadId) { + if (targetThreadId === threadId) return performControl(body); + return new Promise((resolve, reject) => { + const requestId = `${threadId}:${nextRequestId++}`; + const timer = setTimeout(() => { + pendingRequests.delete(requestId); + reject(new Error(`Timed out dispatching expiration quiesce control to worker ${targetThreadId}`)); + }, 120_000); + timer.unref(); + pendingRequests.set(requestId, { resolve, reject, timer }); + if ( + !threads.sendToThread(targetThreadId, { + type: CONTROL_REQUEST, + requestId, + originThreadId: threadId, + targetThreadId, + body, + }) + ) { + clearTimeout(timer); + pendingRequests.delete(requestId); + reject(new Error(`Worker ${targetThreadId} is not reachable`)); + } + }); +} export class QuiesceControl extends Resource { static loadAsInstance = false; async post(_query, body) { - const Table = databases[body.database]?.[body.table]; - if (!Table) return { available: false }; - if (body.action === 'seed') { - const expiresAt = Date.now() - 10_000; - if (body.kind === 'indexed') await Table.put(body.id, { id: body.id, expiresAt }); - else await Table.put(body.id, { id: body.id, value: 'expired' }, { expiresAt }); - return { seeded: true }; + if (body.action === 'probe') { + const workerIds = [threadId, ...threads.map((port) => port.threadId)].filter((id) => id > 0); + return { ready: true, threadId, workerIds: [...new Set(workerIds)] }; } - const started = join(controlDirectory, `${body.runId}.started`); - const release = join(controlDirectory, `${body.runId}.release`); - const hooks = { - beforeBatchCommit: async () => { - writeFileSync(started, 'started'); - while (!existsSync(release)) await delay(20); - }, - }; - if (body.kind === 'indexed') await Table.runRecordExpirationSweepForTests(hooks); - else await Table.runPrimaryCleanupScanForTests(hooks); - return { completed: true }; + if (body.targetThreadId !== undefined) return performControlOnWorker(body, body.targetThreadId); + return performControl(body); } } diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 582cac7938..69976cc4de 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -240,49 +240,46 @@ export type CommitOptions = { }; export const TABLE_COMMIT_ADMISSION = Symbol.for('harper.table-commit-admission'); -const ACTIVE_TABLE_COMMIT_ADMISSION = Symbol('active-table-commit-admission'); +export const TABLE_COMMIT_RELEASE = Symbol.for('harper.table-commit-release'); -type CommitAdmissionContext = { - stores: any[]; - releases: Array<() => unknown>; - transactions: DatabaseTransaction[]; -}; +function hasAdmittedStore(owner: DatabaseTransaction, store: any): boolean { + return owner.admittedTableStore === store || owner.additionalAdmittedTableStores?.includes(store) === true; +} + +function addAdmittedStore(owner: DatabaseTransaction, store: any): void { + if (owner.admittedTableStore === undefined) owner.admittedTableStore = store; + else (owner.additionalAdmittedTableStores ??= []).push(store); +} -function admitTransactionStores(transaction: DatabaseTransaction, context: CommitAdmissionContext): void { +function admitTransactionStores(transaction: DatabaseTransaction, owner: DatabaseTransaction): void { for (let link: DatabaseTransaction = transaction; link; link = link.next) { - if ((link as any)[ACTIVE_TABLE_COMMIT_ADMISSION] !== context) { - (link as any)[ACTIVE_TABLE_COMMIT_ADMISSION] = context; - context.transactions.push(link); - } + link.tableCommitAdmissionOwner = owner; for (const write of link.writes) { const store = write?.store; - if (!store || context.stores.includes(store)) continue; + if (!store || hasAdmittedStore(owner, store)) continue; const admit = store[TABLE_COMMIT_ADMISSION]; - if (admit) { - context.stores.push(store); - context.releases.push(admit()); - } + if (admit && admit() !== false) addAdmittedStore(owner, store); } } } -function releaseTransactionStores(context: CommitAdmissionContext): Promise | void { - for (const transaction of context.transactions) { - if ((transaction as any)[ACTIVE_TABLE_COMMIT_ADMISSION] === context) - delete (transaction as any)[ACTIVE_TABLE_COMMIT_ADMISSION]; - } +function releaseTransactionStores(owner: DatabaseTransaction): Promise | void { let completions: Promise[] | undefined; - for (const release of context.releases) { + const releaseStore = (store: any) => { try { - const completion = release(); + const completion = store[TABLE_COMMIT_RELEASE]?.(); if ((completion as any)?.then) (completions ??= []).push(Promise.resolve(completion)); } catch (error) { (completions ??= []).push(Promise.reject(error)); } + }; + if (owner.admittedTableStore !== undefined) releaseStore(owner.admittedTableStore); + for (const store of owner.additionalAdmittedTableStores ?? []) releaseStore(store); + for (let link: DatabaseTransaction = owner; link; link = link.next) { + if (link.tableCommitAdmissionOwner === owner) link.tableCommitAdmissionOwner = undefined; } - context.stores.length = 0; - context.releases.length = 0; - context.transactions.length = 0; + owner.admittedTableStore = undefined; + owner.additionalAdmittedTableStores = undefined; if (completions) return Promise.allSettled(completions).then(() => undefined); } @@ -296,23 +293,23 @@ export function withTableCommitAdmission( options: CommitOptions, commit: (options: CommitOptions) => T ): T { - let context = (transaction as any)[ACTIVE_TABLE_COMMIT_ADMISSION] as CommitAdmissionContext | undefined; - if (context) { - admitTransactionStores(transaction, context); + const activeOwner = transaction.tableCommitAdmissionOwner; + if (activeOwner) { + admitTransactionStores(transaction, activeOwner); return commit(options); } - context = { stores: [], releases: [], transactions: [] }; + transaction.tableCommitAdmissionOwner = transaction; try { - admitTransactionStores(transaction, context); + admitTransactionStores(transaction, transaction); const resolution: any = commit(options); if (resolution?.then) { return resolution.then( (value) => { - const release = releaseTransactionStores(context); + const release = releaseTransactionStores(transaction); return release ? release.then(() => value) : value; }, (error) => { - const release = releaseTransactionStores(context); + const release = releaseTransactionStores(transaction); if (release) return release.then(() => { throw error; @@ -321,10 +318,10 @@ export function withTableCommitAdmission( } ) as T; } - const release = releaseTransactionStores(context); + const release = releaseTransactionStores(transaction); return (release ? release.then(() => resolution) : resolution) as T; } catch (error) { - const release = releaseTransactionStores(context); + const release = releaseTransactionStores(transaction); if (release) return release.then(() => { throw error; @@ -434,6 +431,9 @@ export class DatabaseTransaction implements Transaction { timestamp = 0; retries = 0; declare next: DatabaseTransaction; + declare tableCommitAdmissionOwner?: DatabaseTransaction; + declare admittedTableStore?: any; + declare additionalAdmittedTableStores?: any[]; declare stale: boolean; // Whether this read handle's base reference (readTxnsUsed starts at 1 in getReadTxn) has been // consumed by a commit round; iterator references are consumed only by doneReadTxn(). diff --git a/resources/Table.ts b/resources/Table.ts index f0c16aa094..9418a93eaa 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -37,6 +37,7 @@ import { ImmediateTransaction, priorStagedWrite, TABLE_COMMIT_ADMISSION, + TABLE_COMMIT_RELEASE, TRANSACTION_STATE, } from './DatabaseTransaction.ts'; import * as envMngr from '../utility/environment/environmentManager.ts'; @@ -162,6 +163,12 @@ export function expirationTimestamp(value: any): number | undefined { return Number.isFinite(timestamp) && timestamp >= 0 ? timestamp : undefined; } +function expirationOverrideTimestamp(value: any): number | undefined { + const timestamp = expirationTimestamp(value); + if (timestamp !== undefined) return timestamp; + if ((typeof value === 'number' || typeof value === 'bigint') && Number(value) < 0) return -1; +} + export function effectiveExpirationTimestamp( entry: Pick, 'expiresAt' | 'metadataFlags'> | undefined, record: any, @@ -462,20 +469,21 @@ export function makeTable(options) { let cleanupClosed = false; let cleanupWasScheduled = false; let dropQuiescing = false; - (primaryStore as any)[TABLE_COMMIT_ADMISSION] = () => { - assertTableWritable(); + (primaryStore as any)[TABLE_COMMIT_ADMISSION] = (skipIfQuiescing = false) => { + if (dropQuiescing || droppingTable) { + if (skipIfQuiescing) return false; + assertTableWritable(); + } pendingTableCommitCount++; - let released = false; - return () => { - if (released) return; - released = true; - if (--pendingTableCommitCount === 0 && resolvePendingTableCommits) { - const resolve = resolvePendingTableCommits; - pendingTableCommitWaiter = undefined; - resolvePendingTableCommits = undefined; - resolve(); - } - }; + return true; + }; + (primaryStore as any)[TABLE_COMMIT_RELEASE] = () => { + if (--pendingTableCommitCount === 0 && resolvePendingTableCommits) { + const resolve = resolvePendingTableCommits; + pendingTableCommitWaiter = undefined; + resolvePendingTableCommits = undefined; + resolve(); + } }; function waitForPendingTableCommits(): Promise | undefined { if (pendingTableCommitCount === 0) return; @@ -738,7 +746,7 @@ export function makeTable(options) { nodeId: event.nodeId, viaNodeId: event.viaNodeId, // use per-event expiresAt: batched txn context only holds the first event's expiration - expiresAt: event.expiresAt, + expiresAt: expirationOverrideTimestamp(event.expiresAt), expirationDecisionPresent: event.expirationDecisionPresent, // bulk base-copy snapshot frame: apply current-state directly, without an audit/transaction-log // entry or out-of-order resequencing (harper-pro#480). Only set for copy frames (between @@ -1954,6 +1962,7 @@ export function makeTable(options) { entry: this.#entry, commit: (txnTime, existingEntry, _retry, transaction: any) => { assertTableWritable(); + const optionExpiresAt = expirationOverrideTimestamp(options?.expiresAt); write.skipped = false; // reset on each retry; cleanup happens after commit if still true if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) { write.skipped = true; @@ -1980,9 +1989,9 @@ export function makeTable(options) { user: (context as any)?.user, residencyId: options?.residencyId, expiresAt: - options?.expirationDecisionPresent === true || options?.expiresAt !== undefined - ? options.expiresAt - : existingEntry?.expiresAt, + options?.expirationDecisionPresent === true + ? (optionExpiresAt ?? -1) + : (optionExpiresAt ?? existingEntry?.expiresAt), expirationDecisionPresent: options?.expirationDecisionPresent === true || Boolean(existingEntry?.metadataFlags & HAS_EXPIRATION_DECISION), @@ -2044,7 +2053,10 @@ export function makeTable(options) { residencyId: options.residencyId, nodeId: options.nodeId, viaNodeId: options?.viaNodeId, - expiresAt: options.expiresAt, + expiresAt: + options.expirationDecisionPresent === true + ? (expirationOverrideTimestamp(options.expiresAt) ?? -1) + : expirationOverrideTimestamp(options.expiresAt), expirationDecisionPresent: options.expirationDecisionPresent, transaction, }, @@ -2084,7 +2096,10 @@ export function makeTable(options) { true, { residencyId, - expiresAt: entry.expiresAt, + expiresAt: + entry.expirationDecisionPresent === true || Boolean(entry.metadataFlags & HAS_EXPIRATION_DECISION) + ? (expirationOverrideTimestamp(entry.expiresAt) ?? -1) + : expirationOverrideTimestamp(entry.expiresAt), expirationDecisionPresent: entry.expirationDecisionPresent === true || Boolean(entry.metadataFlags & HAS_EXPIRATION_DECISION), transaction: txnForContext(context).transaction, @@ -2099,34 +2114,52 @@ export function makeTable(options) { * Evicting a record will remove it from a caching table. This is not considered a canonical data change, and it is assumed that retrieving this record from the source will still yield the same record, this is only removing the local copy of the record. */ static evict(id, existingRecord, existingVersion, encounteredExpirationIndexValue?: number) { + let admitted: unknown; + try { + admitted = (primaryStore as any)[TABLE_COMMIT_ADMISSION]?.(true); + } catch (error) { + logger.warn?.('Error admitting record eviction', id, error); + return Promise.resolve(); + } + if (admitted === false) return Promise.resolve(); let entry; let deleteBlobsAfterCommit: (() => void) | undefined; let currentEntry; - const lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction() }); - let transaction = lmdbTransaction.getReadTxn(); - let options = { transaction }; - let committed = false; + let lmdbTransaction: any; + let transaction: any; + let options: any; + let completionOwnsAdmission = false; + let lmdbCompletion: MaybePromise; + const releaseAdmission = () => (primaryStore as any)[TABLE_COMMIT_RELEASE]?.(); + const abortEviction = () => { + try { + if (primaryStore.ifVersion) lmdbTransaction?.abort?.(); + else transaction?.abort?.(); + } catch {} + }; try { + lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction() }); + transaction = lmdbTransaction.getReadTxn(); + options = { transaction }; // A raw Rocks read transaction can reuse an older snapshot. Reject an already-refreshed // record before consulting it; commit-time conflict detection guards later races. if (!primaryStore.ifVersion) { currentEntry = primaryStore.getEntry(id, { lazy: true }); - if (!currentEntry || currentEntry.version !== existingVersion) return; + if (!currentEntry || currentEntry.version !== existingVersion) return Promise.resolve(); } if (hasSourceGet || audit) { - if (!existingRecord) return; + if (!existingRecord) return Promise.resolve(); entry = primaryStore.getEntry(id, options); - if (!entry || !existingRecord) return; - if (entry.version !== existingVersion) return; + if (!entry || !existingRecord) return Promise.resolve(); + if (entry.version !== existingVersion) return Promise.resolve(); } if (hasSourceGet) { // if there is a resolution in-progress, abandon the eviction - if (primaryStore.hasLock(id, entry.version)) return; + if (primaryStore.hasLock(id, entry.version)) return Promise.resolve(); } // evictions never go in the audit log, so we can not record a deletion entry for the eviction // as there is no corresponding audit entry and it would never get cleaned up. So we must simply // removed the entry entirely, but first cleanup indices - let lmdbCompletion: MaybePromise; if (primaryStore.ifVersion) { // LMDB batches every write in this callback behind one version check, so the indices and // primary record can not diverge on a conflict or partial commit. @@ -2146,7 +2179,7 @@ export function makeTable(options) { lmdbCompletion = Promise.all([lmdbCompletion, removal, encounteredIndexRemoval]); } else { const removalEntry = entry ?? currentEntry ?? primaryStore.getEntry(id); - if (!removalEntry || removalEntry.version !== existingVersion) return; + if (!removalEntry || removalEntry.version !== existingVersion) return Promise.resolve(); const currentExpiration = effectiveExpirationIndexValue(removalEntry, existingRecord); updateIndices(id, existingRecord, null, options, undefined, currentExpiration); if ( @@ -2155,14 +2188,13 @@ export function makeTable(options) { ) indices[expiresAtIndexName]?.remove(encounteredExpirationIndexValue, id, options); if (removalEntry.metadataFlags & HAS_BLOBS) { - if (!existingRecord) return; + if (!existingRecord) return Promise.resolve(); primaryStore.remove(removalEntry.key, options); deleteBlobsAfterCommit = prepareBlobDeletion(existingRecord); } else { removeEntry(primaryStore, removalEntry, options); } } - committed = true; // Eviction is best-effort cleanup, run fire-and-forget from the record-expiration sweep and the // read path as well as the concurrency-limited cleanup scan. A concurrent write to the same record // makes the commit conflict — that is expected, not a failure: lazy-expiry-on-read keeps queries @@ -2175,41 +2207,49 @@ export function makeTable(options) { // a plain resolution object rather than a promise — return the store's write promises instead, so // the caller gets a real thenable that resolves once the removal is durable. (lmdbTransaction as any).commit(); - return Promise.resolve(lmdbCompletion).catch((error) => { - logger.warn?.('Error evicting record', id, error); - }); + completionOwnsAdmission = true; + return Promise.resolve(lmdbCompletion) + .catch((error) => logger.warn?.('Error evicting record', id, error)) + .finally(releaseAdmission); } // RocksDB: eviction writes went directly into the raw transaction via options; commit it directly, // as DatabaseTransaction.commit() would abort it (no tracked writes). The raw commit bypasses // DatabaseTransaction's ERR_BUSY retry, so a concurrent-write conflict rejects here — swallow it // (abandon the eviction) and log anything unexpected, rather than letting it crash the process. - return (transaction as any).commit().then( - () => { - try { - deleteBlobsAfterCommit?.(); - } catch (error) { - logger.warn?.('Error deleting blobs for evicted record', id, error); + const commitCompletion = (transaction as any).commit(); + completionOwnsAdmission = true; + return Promise.resolve(commitCompletion) + .then( + () => { + try { + deleteBlobsAfterCommit?.(); + } catch (error) { + logger.warn?.('Error deleting blobs for evicted record', id, error); + } + }, + (error) => { + abortEviction(); + if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); + else logger.warn?.('Error evicting record', id, error); } - }, - (error) => { - // The commit failed, so the read-snapshot/transaction handle is still open — release it, as the - // batched-eviction path does on its own commit failures. committed===true skips the finally abort. - try { - (transaction as any).abort(); - } catch {} - if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); - else logger.warn?.('Error evicting record', id, error); - } - ); + ) + .finally(releaseAdmission); + } catch (error) { + abortEviction(); + logger.warn?.('Error evicting record', id, error); + if (lmdbCompletion) { + completionOwnsAdmission = true; + return Promise.resolve(lmdbCompletion) + .catch((completionError) => logger.warn?.('Error evicting record', id, completionError)) + .finally(releaseAdmission); + } + return Promise.resolve(); } finally { - if (!committed) { + if (!completionOwnsAdmission) { // Skip path or thrown error: abort instead of committing so we don't apply // partial work and the txn handle is released. - if (primaryStore.ifVersion) { - (lmdbTransaction as any).abort?.(); - } else { - (transaction as any)?.abort?.(); - } + abortEviction(); + releaseAdmission(); } } } @@ -2531,7 +2571,10 @@ export function makeTable(options) { // options/context expiresAt are the most specific overrides; a record @expiresAt field // (resolved below, once recordToStore is merged) overrides the table default in both // directions; the table default is the final fallback. -1 means no expiration. - let expiresAt: number | undefined = options?.expiresAt ?? context?.expiresAt; + let expiresAt: number | undefined = + options?.expirationDecisionPresent === true + ? expirationOverrideTimestamp(options?.expiresAt) + : (expirationOverrideTimestamp(options?.expiresAt) ?? expirationOverrideTimestamp(context?.expiresAt)); let expirationDecisionPresent = Boolean( expiresAtProperty && (options?.expirationDecisionPresent === true || expiresAt !== undefined) ); @@ -4637,6 +4680,20 @@ export function makeTable(options) { : undefined, commit: (txnTime, existingEntry, _retry, transaction: any) => { assertTableWritable(); + const optionExpiresAt = expirationOverrideTimestamp(options?.expiresAt); + const contextExpiresAt = expirationOverrideTimestamp((context as any)?.expiresAt); + const hasExpirationOverride = + options?.expirationDecisionPresent === true || + optionExpiresAt !== undefined || + contextExpiresAt !== undefined; + const expiresAt = + options?.expirationDecisionPresent === true + ? (optionExpiresAt ?? -1) + : (optionExpiresAt ?? contextExpiresAt ?? existingEntry?.expiresAt); + const expirationDecisionPresent = + options?.expirationDecisionPresent === true || + Boolean(existingEntry?.metadataFlags & HAS_EXPIRATION_DECISION) || + Boolean(expiresAtProperty && hasExpirationOverride); // just need to update the version number of the record so it points to the latest audit record // but have to update the version number of the record // TODO: would be faster to use getBinaryFast here and not have the record loaded @@ -4644,6 +4701,16 @@ export function makeTable(options) { if (existingEntry === undefined && trackDeletes && !audit) { scheduleCleanup(); } + if (expiresAtProperty && hasExpirationOverride && existingEntry) { + updateIndices( + id, + existingEntry.value, + existingEntry.value, + { transaction }, + expiresAt, + effectiveExpirationIndexValue(existingEntry) + ); + } logger.trace?.(`Publishing message to id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`); // always audit this, but don't change existing version // TODO: Use direct writes in the future (copying binary data is hard because it invalidates the cache) @@ -4657,8 +4724,8 @@ export function makeTable(options) { { user: (context as any)?.user, residencyId: options?.residencyId, - expiresAt: existingEntry?.expiresAt, - expirationDecisionPresent: Boolean(existingEntry?.metadataFlags & HAS_EXPIRATION_DECISION), + expiresAt, + expirationDecisionPresent, nodeId: options?.nodeId, viaNodeId: options?.viaNodeId, transaction, @@ -6079,6 +6146,7 @@ export function makeTable(options) { let hasChanges, invalidated; try { updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry); + sourceContext.expiresAt = expirationOverrideTimestamp(sourceContext.expiresAt); invalidated = metadataFlags & INVALIDATED; let version = sourceContext.lastModified || (invalidated && existingVersion); hasChanges = invalidated || version > existingVersion || !existingRecord; @@ -6138,16 +6206,18 @@ export function makeTable(options) { // 5.2 record caching relies on it — so we must not write through the frozen object). if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord }; if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id; - if ( - !missingSourceExpirationReported && + const ignoredSourceExpiration = sourceContext.expiresAt === undefined && expiresAtProperty && - expirationTimestamp(updatedRecord[expiresAtProperty.name]) !== undefined - ) { - missingSourceExpirationReported = true; - logger.warn?.( - `Source for table "${tableName}" returned an @expiresAt field without setting context.expiresAt; the field does not set cache expiration.` - ); + expirationTimestamp(updatedRecord[expiresAtProperty.name]) !== undefined; + if (ignoredSourceExpiration) { + recordActionBinary(true, 'cache-source-expiration-ignored', tableName); + if (!missingSourceExpirationReported) { + missingSourceExpirationReported = true; + logger.warn?.( + `Source for table "${tableName}" returned an @expiresAt field without setting context.expiresAt; the field does not set cache expiration.` + ); + } } } if (expirationMs && sourceContext.expiresAt == undefined) sourceContext.expiresAt = Date.now() + expirationMs; diff --git a/resources/databases.ts b/resources/databases.ts index 76ea7aa60e..aeb0449e37 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -121,12 +121,16 @@ type SchemaQuiesceState = { finalizing?: boolean; committed?: boolean; localOwner?: boolean; + rootPaths?: string[]; + recoveryAttempts?: number; }; const schemaQuiescence = new Map(); type SchemaQuiesceOwners = { schema?: string; tables: Map }; const quiescedSchemas = new Map(); const MAX_RETIRED_SCHEMA_QUIESCENCES = 1024; const retiredSchemaQuiescences = new Set(); +const recoveringSchemaQuiescences = new Set(); +const MAX_COMMITTED_QUIESCE_RECOVERY_ATTEMPTS = 3; export function setDatabaseCloseTimeoutForTests(timeout = DEFAULT_DATABASE_CLOSE_TIMEOUT): void { databaseCloseTimeout = timeout; @@ -744,6 +748,8 @@ function initStores( auditPath?: string, isLegacy?: boolean ) { + const schemaOwner = quiescedSchemas.get(databaseName)?.schema; + if (schemaOwner && !recoveringSchemaQuiescences.has(schemaOwner)) return rootStore; const envInit = new OpenEnvironmentObject(path, isReadOnlyMode()); const internalDbiInit = createOpenDBIObject(false); let attributesDbi = rootStore.dbisDb; @@ -811,6 +817,12 @@ function initStores( if (attribute_name != null) tableDef.attributes.push(value); Object.defineProperty(value, 'key', { value: key, configurable: true }); } + const tableOwners = quiescedSchemas.get(databaseName)?.tables; + if (tableOwners) { + for (const [tableName, owner] of tableOwners) { + if (!recoveringSchemaQuiescences.has(owner)) tablesToLoad.delete(tableName); + } + } // Complete any drops that were interrupted mid-flight. dropTable persists a // `dropping` tombstone on the table's primary catalog entry before removing @@ -1341,8 +1353,9 @@ function armSchemaQuiesceLease(state: SchemaQuiesceState): void { const leaseUntil = Math.max(state.message.leaseUntil ?? 0, Date.now() + SCHEMA_QUIESCE_LEASE_MS); state.lease = setTimeout(() => { if (state.committed) { - logger.warn(`Schema quiesce ${state.message.quiesceId} expired after its commit boundary; remaining fail-closed`); - armSchemaQuiesceLease(state); + recoverCommittedSchemaQuiesce(state).catch((error) => { + logger.error(`Could not recover committed schema quiesce ${state.message.quiesceId}`, error); + }); return; } reconcileSchemaQuiesce(state).catch((error) => { @@ -1353,6 +1366,67 @@ function armSchemaQuiesceLease(state: SchemaQuiesceState): void { state.lease.unref(); } +async function recoverCommittedSchemaQuiesce(state: SchemaQuiesceState): Promise { + if (schemaQuiescence.get(state.message.quiesceId) !== state || !state.committed) return false; + if ((state.recoveryAttempts ?? 0) >= MAX_COMMITTED_QUIESCE_RECOVERY_ATTEMPTS) return false; + state.recoveryAttempts = (state.recoveryAttempts ?? 0) + 1; + const locks: RestoreLock[] = []; + try { + for (const path of state.rootPaths ?? []) locks.push(acquireRestoreLock(path)); + } catch (error) { + for (const lock of locks) releaseRestoreLock(lock); + if (state.recoveryAttempts < MAX_COMMITTED_QUIESCE_RECOVERY_ATTEMPTS) { + logger.warn( + `Committed schema quiesce ${state.message.quiesceId} is still protected by an active drop; recovery attempt ${state.recoveryAttempts} deferred` + ); + armSchemaQuiesceLease(state); + } else { + logger.error( + `Committed schema quiesce ${state.message.quiesceId} could not be recovered after ${state.recoveryAttempts} attempts; target remains fail-closed`, + error + ); + } + return false; + } + for (const lock of locks) releaseRestoreLock(lock); + + const { schema, table } = state.message; + recoveringSchemaQuiescences.add(state.message.quiesceId); + try { + if (state.message.operation === OPERATIONS_ENUM.DROP_TABLE) { + if (databases[schema] && table) delete databases[schema][table]; + resetDatabases(); + const descriptor = table && state.table?.dbisDB?.getSync?.(`${table}/`); + if (descriptor?.dropping) + throw new Error(`Durable drop tombstone for ${schema}.${table} could not be reconciled`); + } else { + resetQuiescedDatabase(schema); + } + clearSchemaQuiesce(state); + retireSchemaQuiesce(state.message.quiesceId); + if (state.message.operation === OPERATIONS_ENUM.DROP_SCHEMA) unavailableDatabases.delete(schema); + logger.warn( + `Recovered expired committed schema quiesce ${state.message.quiesceId} from authoritative storage state` + ); + return true; + } catch (error) { + if (state.recoveryAttempts < MAX_COMMITTED_QUIESCE_RECOVERY_ATTEMPTS) armSchemaQuiesceLease(state); + else + logger.error( + `Committed schema quiesce ${state.message.quiesceId} exhausted authoritative recovery attempts; target remains fail-closed`, + error + ); + throw error; + } finally { + recoveringSchemaQuiescences.delete(state.message.quiesceId); + } +} + +export function recoverCommittedSchemaQuiesceForTests(quiesceId: string): Promise { + const state = schemaQuiescence.get(quiesceId); + return state ? recoverCommittedSchemaQuiesce(state) : Promise.resolve(false); +} + function resetQuiescedDatabase(databaseName: string): void { delete databases[databaseName]; const definedDatabase = definedDatabases?.get(databaseName); @@ -1415,11 +1489,23 @@ export async function quiesceSchemaTarget( const Table = dbTables?.[message.table]; if (!Table) return; state.table = Table; + if (Table.primaryStore?.rootStore?.path) state.rootPaths = [Table.primaryStore.rootStore.path]; if (!state.localOwner) delete dbTables[message.table]; await Table.quiesceForDrop(); } else if (message.operation === OPERATIONS_ENUM.DROP_SCHEMA) { state.wasLoaded = Boolean(databases[message.schema]); unavailableDatabases.add(message.schema); + if (state.wasLoaded) { + const definedRootPath = (definedDatabases?.get(message.schema) as any)?.rootStore?.path; + state.rootPaths = [ + ...new Set( + [ + definedRootPath, + ...Object.values(databases[message.schema]).map((table: any) => table?.primaryStore?.rootStore?.path), + ].filter(Boolean) + ), + ]; + } if (state.wasLoaded && state.localOwner) { state.tables = Object.values(databases[message.schema]); await Promise.all(state.tables.map((table) => table.quiesceForDrop?.())); @@ -1489,12 +1575,20 @@ export function renewSchemaQuiesce(message: SchemaQuiesceMessage): { quiesced: b export function finishSchemaQuiesce(message: SchemaQuiesceMessage): boolean { if (retiredSchemaQuiescences.has(message.quiesceId)) return false; const activeId = schemaQuiesceOwner(message); - if (activeId === undefined) return message.originator === process.pid; + // A worker started after quiescence (or one that never loaded this target) has no local state, + // but must still perform the authoritative terminal rescan. Retired IDs above remain rejected, + // preventing an aborted/completed operation from being replayed. `originator` is a worker thread + // id stamped by sendItcEvent, not an OS pid, so comparing it with process.pid incorrectly rejected + // every legitimate no-state worker. + if (activeId === undefined) return true; if (activeId !== message.quiesceId) return false; const state = schemaQuiescence.get(message.quiesceId); if (!state || state.abortRequested) return false; if (message.phase === 'finalize-quiesce' && !state.committed) return false; state.finalizing = true; + // The terminal handler is the authoritative rescan for this operation. Let only this owner pass + // the catalog fence while it applies durable state; ordinary concurrent schema rescans remain blocked. + recoveringSchemaQuiescences.add(message.quiesceId); if (state.lease) { clearTimeout(state.lease); state.lease = undefined; @@ -1503,6 +1597,7 @@ export function finishSchemaQuiesce(message: SchemaQuiesceMessage): boolean { } export function completeSchemaQuiesce(message: SchemaQuiesceMessage): void { + recoveringSchemaQuiescences.delete(message.quiesceId); const state = schemaQuiescence.get(message.quiesceId); if (state) { clearSchemaQuiesce(state); @@ -1512,6 +1607,7 @@ export function completeSchemaQuiesce(message: SchemaQuiesceMessage): void { } export function failSchemaQuiesceFinalization(message: Pick): void { + recoveringSchemaQuiescences.delete(message.quiesceId); const state = schemaQuiescence.get(message.quiesceId); if (!state) return; state.finalizing = false; diff --git a/resources/transaction.ts b/resources/transaction.ts index 837455bd64..8cb78d9338 100644 --- a/resources/transaction.ts +++ b/resources/transaction.ts @@ -57,11 +57,14 @@ export function transaction( return onComplete(result); // when the transaction function completes, run this to commit the transaction function onComplete(result) { - const committed = transaction.commit({ doneWriting: true }); + let committed; + try { + committed = transaction.commit({ doneWriting: true }); + } catch (error) { + return onError(error); + } if ((committed as any).then) { - return (committed as any).then(() => { - return result; - }); + return (committed as any).then(() => result, onError); } else { return result; } diff --git a/storage-format.md b/storage-format.md index 2ad1e652e6..82b405c321 100644 --- a/storage-format.md +++ b/storage-format.md @@ -102,6 +102,23 @@ entry so a missing `HAS_EXPIRATION_EXTENDED_TYPE` bit means an explicit no-expir than missing legacy metadata. Records and audit events without the decision bit retain the legacy fallback to a public `@expiresAt` field during upgrades. +#### `@expiresAt` migration and mixed-version compatibility + +Source-backed tables now treat the source request context as authoritative for cache expiration. A +source that intends a returned record to expire must set `context.expiresAt`; an `@expiresAt` field in +the returned record is application data and no longer schedules expiration by itself. Omitting +`context.expiresAt` records an explicit no-expiration decision, even when the returned record contains +an `@expiresAt` field. This is a breaking change for source adapters that previously relied on that +returned field: update them to set `context.expiresAt` before upgrading. + +New readers remain compatible with records written before `HAS_EXPIRATION_DECISION`: when the decision +bit is absent, they fall back to the public `@expiresAt` field. The reverse is not safe during a rolling +mixed-version deployment. An older worker does not understand an explicit no-expiration decision and +can reinterpret the public field as an expiration. Do not run mixed versions for source-backed tables +that return an `@expiresAt` field while intentionally omitting `context.expiresAt`; upgrade those +workers together, or remove/neutralize the returned field until every worker understands the decision +bit. + ### Variable-Length Integer Encoding Integers (Node ID, Table ID, lengths) use a variable-length encoding scheme: diff --git a/unitTests/resources/databases.test.js b/unitTests/resources/databases.test.js index 10b9bf454b..a272b0df30 100644 --- a/unitTests/resources/databases.test.js +++ b/unitTests/resources/databases.test.js @@ -16,10 +16,17 @@ const { commitSchemaQuiesce, finishSchemaQuiesce, completeSchemaQuiesce, + recoverCommittedSchemaQuiesceForTests, } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { RocksDatabase } = require('@harperfast/rocksdb-js'); -const { beginRestore, completeRestore, RESTORE_META_DIR } = require('#src/dataLayer/restoreMarker'); +const { + acquireRestoreLock, + releaseRestoreLock, + beginRestore, + completeRestore, + RESTORE_META_DIR, +} = require('#src/dataLayer/restoreMarker'); describe('flushDatabases', () => { before(async function () { @@ -271,7 +278,7 @@ describe('cross-worker schema quiescence', () => { assert.strictEqual(finishSchemaQuiesce(message), false, 'completed IDs must not re-enter finalization'); }); - it('rejects terminal messages for unknown and aborted quiescence IDs', async () => { + it('accepts a terminal rescan with no local state while rejecting aborted IDs', async () => { const DB = 'quiesce-stale-id-test'; table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); const message = { @@ -285,12 +292,30 @@ describe('cross-worker schema quiescence', () => { await abortSchemaQuiesce(message); assert.strictEqual(finishSchemaQuiesce(message), false); assert.strictEqual( - finishSchemaQuiesce({ ...message, quiesceId: 'q-never-quiesced' }), - false, - 'unknown IDs must not authorize a metadata reset' + finishSchemaQuiesce({ ...message, quiesceId: 'q-never-quiesced', phase: 'finalize-quiesce' }), + true, + 'a worker that joined after quiescence must accept the authoritative terminal rescan' ); }); + it('does not re-register a table while resetDatabases runs during quiescence', async () => { + const DB = 'quiesce-reset-registration-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-reset-registration', + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual(getDatabases()[DB]?.Records, undefined); + resetDatabases(); + assert.strictEqual(getDatabases()[DB]?.Records, undefined, 'catalog rescan must not resurrect a quiesced table'); + await abortSchemaQuiesce(message); + assert.ok(getDatabases()[DB]?.Records, 'abort recovery should reload the authoritative live table'); + }); + it('retains a missing-table quiescence until its terminal message', async () => { const message = { operation: 'drop_table', @@ -374,4 +399,55 @@ describe('cross-worker schema quiescence', () => { assert.strictEqual(finishSchemaQuiesce(terminal), true); completeSchemaQuiesce(terminal); }); + + it('recovers an expired committed quiescence from the durable live catalog', async () => { + const DB = 'quiesce-committed-recovery-test'; + const Original = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-committed-recovery', + originLocal: true, + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + assert.strictEqual(await recoverCommittedSchemaQuiesceForTests(message.quiesceId), true); + const Recovered = getDatabases()[DB]?.Records; + assert.ok(Recovered, 'the durable non-tombstoned catalog should be restored'); + assert.notStrictEqual(Recovered, Original, 'recovery must not republish the quiesced table instance'); + assert.strictEqual(Recovered.isDropQuiescing(), false); + assert.strictEqual(finishSchemaQuiesce({ ...message, phase: 'finalize-quiesce' }), false); + }); + + it('bounds committed recovery attempts while an active drop lock is still held', async () => { + const DB = 'quiesce-committed-recovery-bound-test'; + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-committed-recovery-bound', + originLocal: true, + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + const lock = acquireRestoreLock(Table.primaryStore.rootStore.path); + try { + for (let attempt = 0; attempt < 3; attempt++) + assert.strictEqual(await recoverCommittedSchemaQuiesceForTests(message.quiesceId), false); + } finally { + releaseRestoreLock(lock); + } + assert.strictEqual( + await recoverCommittedSchemaQuiesceForTests(message.quiesceId), + false, + 'the recovery budget must not restart after it is exhausted' + ); + const terminal = { ...message, phase: 'reconcile-quiesce' }; + assert.strictEqual(finishSchemaQuiesce(terminal), true); + completeSchemaQuiesce(terminal); + }); }); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 8a173bfd56..c8f6abfafe 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -10,7 +10,8 @@ const { existsSync } = require('node:fs'); const { setTimeout: delay } = require('node:timers/promises'); const { waitFor } = require('../waitFor.js'); const { HAS_EXPIRATION_DECISION } = require('#src/resources/auditStore'); -const { TABLE_COMMIT_ADMISSION } = require('#src/resources/DatabaseTransaction'); +const { TABLE_COMMIT_ADMISSION, TABLE_COMMIT_RELEASE } = require('#src/resources/DatabaseTransaction'); +const { LMDBTransaction } = require('#src/resources/LMDBTransaction'); const { transaction } = require('#src/resources/transaction'); const activeTables = new Set(); @@ -227,7 +228,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const fieldExpiresAt = contextExpiresAt + 3_600_000; Table.sourcedFrom({ get(id, context) { - context.expiresAt = contextExpiresAt; + context.expiresAt = new Date(contextExpiresAt).toISOString(); return { id, expiresAt: new Date(fieldExpiresAt).toISOString() }; }, }); @@ -327,6 +328,27 @@ describe('@expiresAt attribute is authoritative over the table default', () => { assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); }); + it('normalizes publish expiration overrides and updates the expiration index', async function () { + const Table = makeTable('ExpiresAtPublishOverride', undefined, { audit: true }); + const originalExpiresAt = Date.now() + 60_000; + const optionExpiresAt = originalExpiresAt + 60_000; + const contextExpiresAt = optionExpiresAt + 60_000; + await Table.put(1, { id: 1, expiresAt: originalExpiresAt }); + await Table.publish(1, { message: 'option' }, { expiresAt: new Date(optionExpiresAt).toISOString() }); + await Table.primaryStore.committed; + assert.strictEqual(Table.primaryStore.getEntry(1).expiresAt, optionExpiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(originalExpiresAt)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(optionExpiresAt)], [1]); + + const context = { expiresAt: new Date(contextExpiresAt) }; + const resource = new Table(1, context); + await transaction(context, () => resource._writePublish(1, { message: 'context' })); + await Table.primaryStore.committed; + assert.strictEqual(Table.primaryStore.getEntry(1).expiresAt, contextExpiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(optionExpiresAt)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(contextExpiresAt)], [1]); + }); + it('invalidates a frozen replay record without mutating it', async function () { const Table = makeTable('ExpiresAtFrozenInvalidate', undefined, { audit: true }); const fieldExpiresAt = Date.now() + 60_000; @@ -908,16 +930,18 @@ describe('@expiresAt attribute is authoritative over the table default', () => { attributes: [{ name: 'id', isPrimaryKey: true }], }); const originalAdmission = Table.primaryStore[TABLE_COMMIT_ADMISSION]; + const originalRelease = Table.primaryStore[TABLE_COMMIT_RELEASE]; let admitted = false; let releaseCommit; const blockedRelease = new Promise((resolve) => (releaseCommit = resolve)); Table.primaryStore[TABLE_COMMIT_ADMISSION] = () => { - const release = originalAdmission(); + originalAdmission(); admitted = true; - return async () => { - await blockedRelease; - release(); - }; + return true; + }; + Table.primaryStore[TABLE_COMMIT_RELEASE] = async () => { + await blockedRelease; + originalRelease(); }; try { const put = Table.put(1, { id: 1 }); @@ -931,6 +955,46 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } finally { releaseCommit(); Table.primaryStore[TABLE_COMMIT_ADMISSION] = originalAdmission; + Table.primaryStore[TABLE_COMMIT_RELEASE] = originalRelease; + } + }); + + it('joins read-path eviction to drop quiescence and skips eviction after quiescence starts', async function () { + const Table = makeTable('ExpiresAtReadEvictionQuiesce'); + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.primaryStore.committed; + const entry = Table.primaryStore.getEntry(1); + const originalAdmission = Table.primaryStore[TABLE_COMMIT_ADMISSION]; + const originalRelease = Table.primaryStore[TABLE_COMMIT_RELEASE]; + let admitted = false; + let releaseEviction; + const blockedRelease = new Promise((resolve) => (releaseEviction = resolve)); + Table.primaryStore[TABLE_COMMIT_ADMISSION] = (...args) => { + const result = originalAdmission(...args); + if (result !== false) admitted = true; + return result; + }; + Table.primaryStore[TABLE_COMMIT_RELEASE] = async () => { + await blockedRelease; + originalRelease(); + }; + try { + const eviction = Table.evict(1, entry.value, entry.version); + await waitFor(() => admitted, { message: 'read-path eviction should enter the commit barrier' }); + let quiesced = false; + const quiesce = Table.quiesceForDrop().then(() => (quiesced = true)); + await delay(10); + assert.strictEqual(quiesced, false, 'drop quiescence must wait for an admitted eviction'); + releaseEviction(); + await Promise.all([eviction, quiesce]); + + await assert.doesNotReject(Table.evict(1, entry.value, entry.version)); + } finally { + releaseEviction(); + Table.primaryStore[TABLE_COMMIT_ADMISSION] = originalAdmission; + Table.primaryStore[TABLE_COMMIT_RELEASE] = originalRelease; + Table.abortDropQuiesce(); } }); @@ -1134,6 +1198,48 @@ describe('LMDB @expiresAt cleanup draining', function () { assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); }); + it('settles read-path eviction when admission throws', async function () { + const { Table } = captureExpirationSweep('LmdbExpirationAdmissionFailure'); + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.primaryStore.committed; + const entry = Table.primaryStore.getEntry(1); + const originalAdmission = Table.primaryStore[TABLE_COMMIT_ADMISSION]; + Table.primaryStore[TABLE_COMMIT_ADMISSION] = () => { + throw new Error('injected admission failure'); + }; + try { + await assert.doesNotReject(Table.evict(1, entry.value, entry.version)); + } finally { + Table.primaryStore[TABLE_COMMIT_ADMISSION] = originalAdmission; + } + }); + + it('aborts and settles read-path eviction when LMDB commit throws synchronously', async function () { + const { Table } = captureExpirationSweep('LmdbExpirationCommitFailure'); + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.primaryStore.committed; + const entry = Table.primaryStore.getEntry(1); + const originalCommit = LMDBTransaction.prototype.commit; + const originalAbort = LMDBTransaction.prototype.abort; + let aborted = false; + LMDBTransaction.prototype.commit = () => { + throw new Error('injected commit failure'); + }; + LMDBTransaction.prototype.abort = function () { + aborted = true; + return originalAbort.call(this); + }; + try { + await assert.doesNotReject(Table.evict(1, entry.value, entry.version)); + assert.strictEqual(aborted, true); + } finally { + LMDBTransaction.prototype.commit = originalCommit; + LMDBTransaction.prototype.abort = originalAbort; + } + }); + it('evicts ISO expirations and repairs stale keys', async function () { const { Table, runSweep } = captureExpirationSweep('LmdbExpirationCorrectness'); const expired = Date.now() - 1_000; diff --git a/unitTests/resources/transaction.test.js b/unitTests/resources/transaction.test.js index 9e204ddbe1..47241446ae 100644 --- a/unitTests/resources/transaction.test.js +++ b/unitTests/resources/transaction.test.js @@ -8,6 +8,7 @@ const { transaction } = require('#src/resources/transaction'); const { DatabaseTransaction, TABLE_COMMIT_ADMISSION, + TABLE_COMMIT_RELEASE, withTableCommitAdmission, } = require('#src/resources/DatabaseTransaction'); const { IterableEventQueue } = require('#src/resources/IterableEventQueue'); @@ -22,7 +23,10 @@ describe('Table commit admission', () => { const store = (name) => ({ [TABLE_COMMIT_ADMISSION]() { events.push(`admit:${name}`); - return () => events.push(`release:${name}`); + return true; + }, + [TABLE_COMMIT_RELEASE]() { + events.push(`release:${name}`); }, }); const storeA = store('a'); @@ -105,6 +109,29 @@ describe('Transactions', () => { assert.equal(answer.name, 'the answer'); assert.equal(answer.computed, 'the answer computed'); }); + it('aborts and releases the context when final commit admission is refused', async function () { + const context = {}; + const id = 43; + const originalAdmission = TxnTest.primaryStore[TABLE_COMMIT_ADMISSION]; + TxnTest.primaryStore[TABLE_COMMIT_ADMISSION] = () => { + const error = new Error('table is quiescing'); + error.statusCode = 503; + throw error; + }; + try { + assert.throws( + () => + transaction(context, () => { + TxnTest.put(id, { name: 'must not commit' }, context); + }), + (error) => error?.statusCode === 503 + ); + assert.strictEqual(context.transaction, null); + assert.strictEqual(TxnTest.primaryStore.getEntry(id), undefined); + } finally { + TxnTest.primaryStore[TABLE_COMMIT_ADMISSION] = originalAdmission; + } + }); it('Can run txn with three tables and two databases', async function () { const context = {}; let start = Date.now(); diff --git a/unitTests/utility/signalling.test.js b/unitTests/utility/signalling.test.js index 8c8ce825b7..7ce85dcb5d 100644 --- a/unitTests/utility/signalling.test.js +++ b/unitTests/utility/signalling.test.js @@ -157,6 +157,39 @@ describe('Test signalling module', () => { } }); + it('releases the worker-start barrier when terminal acknowledgements are exhausted', async () => { + const localSchemaHandler = sandbox.stub().callsFake(async (event) => { + if (event.message.phase === 'finalize-quiesce') return { finalized: true }; + if (event.message.phase === 'release-worker-starts') return { released: true }; + }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + send_itc_event_stub.callsFake(async (event) => { + if (event.message.phase === 'finalize-quiesce') throw new Error('terminal acknowledgement lost'); + }); + try { + let terminalError; + try { + await signalling.finalizeSchemaChange({ + operation: 'drop_schema', + schema: 'unit_test', + quiesceId: 'q-terminal-exhausted', + }); + } catch (error) { + terminalError = error; + } + expect(terminalError?.message).to.include('Could not finalized schema quiesce'); + expect( + localSchemaHandler.getCalls().some((call) => call.args[0].message.phase === 'release-worker-starts') + ).to.equal(true); + const terminalAttempts = send_itc_event_stub + .getCalls() + .filter((call) => call.args[0].message.phase === 'finalize-quiesce'); + expect(terminalAttempts).to.have.length(3); + } finally { + restoreHandlers(); + } + }); + it('reconciles and releases the worker barrier after a partial commit failure', async () => { const localSchemaHandler = sandbox.stub().callsFake(async (event) => { if (event.message.phase === 'commit-quiesce') return { committed: true }; diff --git a/utility/signalling.ts b/utility/signalling.ts index b75dc5bbda..015ddef7ea 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -169,7 +169,6 @@ export async function commitSchemaChange(message: any) { } async function completeSchemaChange(message: any, phase: string, resultProperty: string) { - let completed = false; try { serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); const finalMessage = { ...message, phase }; @@ -186,7 +185,6 @@ async function completeSchemaChange(message: any, phase: string, resultProperty: acceptResult: (result) => result?.[resultProperty] === true, includeJobWorkers: true, }); - completed = true; return; } catch (error) { lastError = error; @@ -197,7 +195,7 @@ async function completeSchemaChange(message: any, phase: string, resultProperty: }); } finally { stopSchemaQuiesceRenewal(message.quiesceId); - if (completed) await releaseSchemaWorkerBarrier(message); + await releaseSchemaWorkerBarrier(message); } } From 6b24bdcf8149b92792b2886302ad2dcd68ad52e9 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 15:14:28 -0600 Subject: [PATCH 28/43] Finalize quiescence recovery fencing --- .../database/expiration-drop-quiesce.test.ts | 35 ++++--- .../expiration-drop-quiesce/resources.js | 12 ++- resources/DatabaseTransaction.ts | 21 +++- resources/databases.ts | 64 +++++++++--- unitTests/resources/databases.test.js | 97 ++++++++++++++++++- unitTests/resources/transaction.test.js | 25 +++++ unitTests/utility/signalling.test.js | 8 +- utility/signalling.ts | 6 +- 8 files changed, 231 insertions(+), 37 deletions(-) diff --git a/integrationTests/database/expiration-drop-quiesce.test.ts b/integrationTests/database/expiration-drop-quiesce.test.ts index 99f0248dbf..e122ec2e31 100644 --- a/integrationTests/database/expiration-drop-quiesce.test.ts +++ b/integrationTests/database/expiration-drop-quiesce.test.ts @@ -85,28 +85,41 @@ suite('cross-worker expiration cleanup quiesces destructive DDL', (ctx: ContextW { action: 'sweep', kind, database, table, runId: sweepRunId, releaseRunId: runId }, sweepWorkerId ); - await waitFor(() => existsSync(started), `${kind} sweep did not reach its blocked commit`); - strictEqual(JSON.parse(readFileSync(started, 'utf8')).threadId, sweepWorkerId); - let dropSettled = false; const dropStarted = join(CONTROL_DIRECTORY, `${dropRunId}.started`); - const drop = postControlOnWorker({ action: 'drop', database, table, runId: dropRunId }, ddlWorkerId).then( - (response) => { - dropSettled = true; - return response; - } - ); + let drop: ReturnType | undefined; let dropped; + let primaryError: unknown; + const cleanupErrors: unknown[] = []; try { + await waitFor(() => existsSync(started), `${kind} sweep did not reach its blocked commit`); + strictEqual(JSON.parse(readFileSync(started, 'utf8')).threadId, sweepWorkerId); + drop = postControlOnWorker({ action: 'drop', database, table, runId: dropRunId }, ddlWorkerId).then( + (response) => { + dropSettled = true; + return response; + } + ); await waitFor(() => existsSync(dropStarted), `${kind} drop did not start on its pinned worker`); strictEqual(JSON.parse(readFileSync(dropStarted, 'utf8')).threadId, ddlWorkerId); strictEqual(sweepWorkerId === ddlWorkerId, false, 'DDL and cleanup must execute on different workers'); await delay(100); strictEqual(dropSettled, false, 'physical drop must wait for the blocked cleanup worker'); + } catch (error) { + primaryError = error; } finally { - writeFileSync(release, 'release'); - [dropped] = await Promise.all([drop, sweep]); + try { + writeFileSync(release, 'release'); + } catch (error) { + cleanupErrors.push(error); + } + const completions = drop ? [drop, sweep] : [sweep]; + const results = await Promise.allSettled(completions); + if (drop && results[0].status === 'fulfilled') dropped = results[0].value; + for (const result of results) if (result.status === 'rejected') cleanupErrors.push(result.reason); } + if (primaryError) throw primaryError; + if (cleanupErrors.length) throw new AggregateError(cleanupErrors, `${kind} quiescence test cleanup failed`); strictEqual(dropped.status, 200); strictEqual(dropped.body.threadId, ddlWorkerId); } diff --git a/integrationTests/database/expiration-drop-quiesce/resources.js b/integrationTests/database/expiration-drop-quiesce/resources.js index ca2cd5b050..873de99eff 100644 --- a/integrationTests/database/expiration-drop-quiesce/resources.js +++ b/integrationTests/database/expiration-drop-quiesce/resources.js @@ -1,4 +1,4 @@ -import { existsSync, writeFileSync } from 'node:fs'; +import { existsSync, renameSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; import { threadId } from 'node:worker_threads'; @@ -9,6 +9,12 @@ const CONTROL_RESPONSE = 'expiration-quiesce-control-response'; const pendingRequests = new Map(); let nextRequestId = 1; +function publishWorkerMarker(path) { + const temporaryPath = `${path}.${threadId}.tmp`; + writeFileSync(temporaryPath, JSON.stringify({ threadId })); + renameSync(temporaryPath, path); +} + async function performControl(body) { const Table = databases[body.database]?.[body.table]; if (!Table) return { available: false, threadId }; @@ -21,13 +27,13 @@ async function performControl(body) { const started = join(controlDirectory, `${body.runId}.started`); const release = join(controlDirectory, `${body.releaseRunId ?? body.runId}.release`); if (body.action === 'drop') { - writeFileSync(started, JSON.stringify({ threadId })); + publishWorkerMarker(started); await Table.dropTable(); return { completed: true, threadId }; } const hooks = { beforeBatchCommit: async () => { - writeFileSync(started, JSON.stringify({ threadId })); + publishWorkerMarker(started); while (!existsSync(release)) await delay(20); }, }; diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 69976cc4de..e376460064 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -251,9 +251,17 @@ function addAdmittedStore(owner: DatabaseTransaction, store: any): void { else (owner.additionalAdmittedTableStores ??= []).push(store); } +function stampTransactionLink(owner: DatabaseTransaction, link: DatabaseTransaction): void { + if (link.tableCommitAdmissionOwner === owner) return; + link.tableCommitAdmissionOwner = owner; + if (link === owner) return; + if (owner.admittedTransactionLink === undefined) owner.admittedTransactionLink = link; + else (owner.additionalAdmittedTransactionLinks ??= []).push(link); +} + function admitTransactionStores(transaction: DatabaseTransaction, owner: DatabaseTransaction): void { for (let link: DatabaseTransaction = transaction; link; link = link.next) { - link.tableCommitAdmissionOwner = owner; + stampTransactionLink(owner, link); for (const write of link.writes) { const store = write?.store; if (!store || hasAdmittedStore(owner, store)) continue; @@ -275,11 +283,16 @@ function releaseTransactionStores(owner: DatabaseTransaction): Promise | v }; if (owner.admittedTableStore !== undefined) releaseStore(owner.admittedTableStore); for (const store of owner.additionalAdmittedTableStores ?? []) releaseStore(store); - for (let link: DatabaseTransaction = owner; link; link = link.next) { + const clearAdmissionOwner = (link: DatabaseTransaction) => { if (link.tableCommitAdmissionOwner === owner) link.tableCommitAdmissionOwner = undefined; - } + }; + clearAdmissionOwner(owner); + if (owner.admittedTransactionLink) clearAdmissionOwner(owner.admittedTransactionLink); + for (const link of owner.additionalAdmittedTransactionLinks ?? []) clearAdmissionOwner(link); owner.admittedTableStore = undefined; owner.additionalAdmittedTableStores = undefined; + owner.admittedTransactionLink = undefined; + owner.additionalAdmittedTransactionLinks = undefined; if (completions) return Promise.allSettled(completions).then(() => undefined); } @@ -434,6 +447,8 @@ export class DatabaseTransaction implements Transaction { declare tableCommitAdmissionOwner?: DatabaseTransaction; declare admittedTableStore?: any; declare additionalAdmittedTableStores?: any[]; + declare admittedTransactionLink?: DatabaseTransaction; + declare additionalAdmittedTransactionLinks?: DatabaseTransaction[]; declare stale: boolean; // Whether this read handle's base reference (readTxnsUsed starts at 1 in getReadTxn) has been // consumed by a commit round; iterator references are consumed only by doneReadTxn(). diff --git a/resources/databases.ts b/resources/databases.ts index aeb0449e37..f382ea55c1 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -693,6 +693,12 @@ export function readMetaDb( auditPath?: string, isLegacy?: boolean ) { + const schemaOwner = quiescedSchemas.get(databaseName)?.schema; + if (schemaOwner && !recoveringSchemaQuiescences.has(schemaOwner)) { + const existing = lmdbDatabaseEnvs.get(path); + if (existing) existing.needsDeletion = false; + return existing; + } const envInit = new OpenEnvironmentObject(path, isReadOnlyMode()); try { let rootStore = lmdbDatabaseEnvs.get(path); @@ -711,6 +717,8 @@ export function readMetaDb( } function readRocksMetaDb(path: string, defaultTable?: string, databaseName: string = DEFAULT_DATABASE_NAME) { + const schemaOwner = quiescedSchemas.get(databaseName)?.schema; + if (schemaOwner && !recoveringSchemaQuiescences.has(schemaOwner)) return rocksdbDatabaseEnvs.get(path); try { logger.trace(`loading rocksdb database: ${path}`); @@ -1391,13 +1399,21 @@ async function recoverCommittedSchemaQuiesce(state: SchemaQuiesceState): Promise for (const lock of locks) releaseRestoreLock(lock); const { schema, table } = state.message; - recoveringSchemaQuiescences.add(state.message.quiesceId); try { + // Inspect the durable decision before changing public registrations or bypassing the + // quiescence catalog fence. This distinguishes a live catalog row (destruction never + // started), a drop tombstone (finish interrupted destruction), and an absent row + // (destruction completed) without letting resetDatabases manufacture the answer. + const durableDescriptor = + state.message.operation === OPERATIONS_ENUM.DROP_TABLE && table + ? state.table?.dbisDB?.getSync?.(`${table}/`) + : undefined; + recoveringSchemaQuiescences.add(state.message.quiesceId); if (state.message.operation === OPERATIONS_ENUM.DROP_TABLE) { if (databases[schema] && table) delete databases[schema][table]; resetDatabases(); - const descriptor = table && state.table?.dbisDB?.getSync?.(`${table}/`); - if (descriptor?.dropping) + const reconciledDescriptor = table && state.table?.dbisDB?.getSync?.(`${table}/`); + if (durableDescriptor?.dropping && reconciledDescriptor?.dropping) throw new Error(`Durable drop tombstone for ${schema}.${table} could not be reconciled`); } else { resetQuiescedDatabase(schema); @@ -1452,7 +1468,14 @@ async function reconcileSchemaQuiesce(state: SchemaQuiesceState): Promise } else { if (state.localOwner) { for (const table of state.tables ?? []) table.abortDropQuiesce?.(); - } else resetQuiescedDatabase(state.message.schema); + } else { + recoveringSchemaQuiescences.add(state.message.quiesceId); + try { + resetQuiescedDatabase(state.message.schema); + } finally { + recoveringSchemaQuiescences.delete(state.message.quiesceId); + } + } } clearSchemaQuiesce(state); retireSchemaQuiesce(state.message.quiesceId); @@ -1644,9 +1667,20 @@ export async function dropDatabase(databaseName): Promise { if (quiesceMessage) { const message: any = new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, databaseName); message.quiesceId = quiesceMessage.quiesceId; - if (dropSucceeded) await signalling.finalizeSchemaChange(message); - else await signalling.reconcileSchemaChange(message); - unavailableDatabases.delete(databaseName); + if (dropSucceeded) { + try { + await signalling.finalizeSchemaChange(message); + } finally { + // finalizeSchemaChange applies the local terminal rescan before broadcasting it. + // A later remote acknowledgement failure must not permanently poison this + // physically-destroyed name on the origin. If local terminal work failed, its + // quiescence state remains and the unavailable fence must stay fail-closed. + if (!schemaQuiescence.has(quiesceMessage.quiesceId)) unavailableDatabases.delete(databaseName); + } + } else { + await signalling.reconcileSchemaChange(message); + unavailableDatabases.delete(databaseName); + } } }); droppingDatabases.set(databaseName, completion); @@ -1669,15 +1703,11 @@ async function dropDatabaseOnce(databaseName, dbTables, rootStore, onQuiesced: ( if (restoreLocks.length === 0 && rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); - try { - const quiesceMessage = await signalling.quiesceSchemaChange( - new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, databaseName) - ); - onQuiesced(quiesceMessage); - await signalling.commitSchemaChange(quiesceMessage); - } catch (error) { - throw error; - } + const quiesceMessage = await signalling.quiesceSchemaChange( + new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, databaseName) + ); + onQuiesced(quiesceMessage); + await signalling.commitSchemaChange(quiesceMessage); forgetDatabaseEnvironment(rootStore); for (const tableName in dbTables) { forgetDatabaseEnvironment(dbTables[tableName].primaryStore.rootStore); @@ -1694,6 +1724,8 @@ async function dropDatabaseOnce(databaseName, dbTables, rootStore, onQuiesced: ( delete tables[DEFINED_TABLES]; } delete databases[databaseName]; + const definedDatabase = definedDatabases?.get(databaseName); + if (definedDatabase) (definedDatabase as any).rootStore = undefined; databaseEventsEmitter.emit('dropDatabase', databaseName); diff --git a/unitTests/resources/databases.test.js b/unitTests/resources/databases.test.js index a272b0df30..ede1fb9c15 100644 --- a/unitTests/resources/databases.test.js +++ b/unitTests/resources/databases.test.js @@ -20,6 +20,7 @@ const { } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { RocksDatabase } = require('@harperfast/rocksdb-js'); +const signalling = require('#src/utility/signalling'); const { acquireRestoreLock, releaseRestoreLock, @@ -316,6 +317,25 @@ describe('cross-worker schema quiescence', () => { assert.ok(getDatabases()[DB]?.Records, 'abort recovery should reload the authoritative live table'); }); + it('does not reopen a root store or replay logs while its schema is quiesced', async () => { + const DB = 'quiesce-reset-root-handle-test'; + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const rootStore = Table.primaryStore.rootStore; + const message = { + operation: 'drop_schema', + schema: DB, + quiesceId: 'q-reset-root-handle', + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual(rootStore.status, 'closed'); + resetDatabases(); + assert.strictEqual(rootStore.status, 'closed', 'catalog scan must not reopen the quiesced root store'); + assert.strictEqual(getDatabases()[DB], undefined, 'schema scan must remain fenced before replay/registration'); + await abortSchemaQuiesce(message); + assert.ok(getDatabases()[DB]?.Records); + }); + it('retains a missing-table quiescence until its terminal message', async () => { const message = { operation: 'drop_table', @@ -413,7 +433,20 @@ describe('cross-worker schema quiescence', () => { }; assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); - assert.strictEqual(await recoverCommittedSchemaQuiesceForTests(message.quiesceId), true); + const registry = getDatabases(); + const originalGetSync = Original.dbisDB.getSync; + let descriptorReadBeforeMutation = false; + Original.dbisDB.getSync = function (key) { + if (key === 'Records/' && !descriptorReadBeforeMutation) + descriptorReadBeforeMutation = registry[DB]?.Records === Original; + return originalGetSync.call(this, key); + }; + try { + assert.strictEqual(await recoverCommittedSchemaQuiesceForTests(message.quiesceId), true); + } finally { + Original.dbisDB.getSync = originalGetSync; + } + assert.strictEqual(descriptorReadBeforeMutation, true, 'durable descriptor must be read before registry mutation'); const Recovered = getDatabases()[DB]?.Records; assert.ok(Recovered, 'the durable non-tombstoned catalog should be restored'); assert.notStrictEqual(Recovered, Original, 'recovery must not republish the quiesced table instance'); @@ -421,6 +454,68 @@ describe('cross-worker schema quiescence', () => { assert.strictEqual(finishSchemaQuiesce({ ...message, phase: 'finalize-quiesce' }), false); }); + it('recovers an expired committed schema and clears its unavailable fence', async () => { + const DB = 'quiesce-committed-schema-recovery-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_schema', + schema: DB, + quiesceId: 'q-committed-schema-recovery', + originLocal: true, + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + assert.throws(() => database({ database: DB, table: null }), /closing and cannot be opened/); + assert.strictEqual(await recoverCommittedSchemaQuiesceForTests(message.quiesceId), true); + assert.doesNotThrow(() => database({ database: DB, table: null })); + assert.ok(getDatabases()[DB]?.Records); + }); + + it('clears the origin unavailable fence when only terminal broadcast acknowledgement fails', async function () { + this.timeout(30000); + const DB = 'drop-terminal-broadcast-failure-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const originalFinalize = signalling.finalizeSchemaChange; + signalling.finalizeSchemaChange = async (message) => { + completeSchemaQuiesce({ ...message, phase: 'finalize-quiesce' }); + throw new Error('injected remote terminal acknowledgement failure'); + }; + try { + await assert.rejects(dropDatabase(DB), /injected remote terminal acknowledgement failure/); + assert.doesNotThrow(() => database({ database: DB, table: null })); + assert.ok( + table({ table: 'Recreated', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }), + 'physically destroyed schema name must be reusable after local terminal completion' + ); + } finally { + signalling.finalizeSchemaChange = originalFinalize; + } + }); + + it('retains the origin unavailable fence when local terminal state is unresolved', async function () { + this.timeout(30000); + const DB = 'drop-terminal-local-failure-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const originalFinalize = signalling.finalizeSchemaChange; + let terminalMessage; + signalling.finalizeSchemaChange = async (message) => { + terminalMessage = message; + throw new Error('injected local terminal failure'); + }; + try { + await assert.rejects(dropDatabase(DB), /injected local terminal failure/); + assert.throws( + () => database({ database: DB, table: null }), + /closing and cannot be opened/, + 'uncertain local terminal state must remain fail-closed' + ); + } finally { + signalling.finalizeSchemaChange = originalFinalize; + if (terminalMessage) completeSchemaQuiesce({ ...terminalMessage, phase: 'reconcile-quiesce' }); + } + }); + it('bounds committed recovery attempts while an active drop lock is still held', async () => { const DB = 'quiesce-committed-recovery-bound-test'; const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); diff --git a/unitTests/resources/transaction.test.js b/unitTests/resources/transaction.test.js index 47241446ae..7ccfd476e1 100644 --- a/unitTests/resources/transaction.test.js +++ b/unitTests/resources/transaction.test.js @@ -45,6 +45,31 @@ describe('Table commit admission', () => { assert.deepStrictEqual(events, ['admit:a', 'admit:b', 'admit:c', 'release:a', 'release:b', 'release:c']); }); + + it('releases a stamped transaction link after it is detached from the chain', async () => { + let pendingCommits = 0; + const store = { + [TABLE_COMMIT_ADMISSION]() { + pendingCommits++; + return true; + }, + [TABLE_COMMIT_RELEASE]() { + pendingCommits--; + }, + }; + const tail = { writes: [{ store }], next: undefined }; + const head = { writes: [], next: tail }; + + await withTableCommitAdmission(head, {}, () => { + head.next = undefined; + }); + assert.strictEqual(tail.tableCommitAdmissionOwner, undefined); + assert.strictEqual(pendingCommits, 0); + + await withTableCommitAdmission(tail, {}, () => undefined); + assert.strictEqual(tail.tableCommitAdmissionOwner, undefined); + assert.strictEqual(pendingCommits, 0, 'the detached link must release admission on its next commit'); + }); }); // The package blocks deep imports of its package.json, so walk up from the resolved entry point. diff --git a/unitTests/utility/signalling.test.js b/unitTests/utility/signalling.test.js index 7ce85dcb5d..f81d6093a3 100644 --- a/unitTests/utility/signalling.test.js +++ b/unitTests/utility/signalling.test.js @@ -157,10 +157,10 @@ describe('Test signalling module', () => { } }); - it('releases the worker-start barrier when terminal acknowledgements are exhausted', async () => { + it('preserves terminal failure while attempting barrier release on every worker', async () => { const localSchemaHandler = sandbox.stub().callsFake(async (event) => { if (event.message.phase === 'finalize-quiesce') return { finalized: true }; - if (event.message.phase === 'release-worker-starts') return { released: true }; + if (event.message.phase === 'release-worker-starts') throw new Error('local barrier release failed'); }); const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); send_itc_event_stub.callsFake(async (event) => { @@ -178,9 +178,13 @@ describe('Test signalling module', () => { terminalError = error; } expect(terminalError?.message).to.include('Could not finalized schema quiesce'); + expect(terminalError?.message).not.to.include('local barrier release failed'); expect( localSchemaHandler.getCalls().some((call) => call.args[0].message.phase === 'release-worker-starts') ).to.equal(true); + expect( + send_itc_event_stub.getCalls().some((call) => call.args[0].message.phase === 'release-worker-starts') + ).to.equal(true); const terminalAttempts = send_itc_event_stub .getCalls() .filter((call) => call.args[0].message.phase === 'finalize-quiesce'); diff --git a/utility/signalling.ts b/utility/signalling.ts index 015ddef7ea..6814a0e06b 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -130,7 +130,11 @@ async function releaseSchemaWorkerBarrier(message: any) { ...message, phase: 'release-worker-starts', }); - await serverItcHandlers.schema(event); + try { + await serverItcHandlers.schema(event); + } catch (error) { + hdbLogger.warn('Could not release the local worker-start barrier:', error); + } try { await sendItcEvent(event, { timeout: SCHEMA_QUIESCE_TIMEOUT_MS, From 0ded803b97e71ed89bed99c6d83b28e4129a4327 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 15:23:02 -0600 Subject: [PATCH 29/43] Restore availability after reconcile timeout --- resources/DatabaseTransaction.ts | 1 + resources/databases.ts | 9 +++++++-- unitTests/resources/databases.test.js | 26 ++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index e376460064..c2b752612d 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -308,6 +308,7 @@ export function withTableCommitAdmission( ): T { const activeOwner = transaction.tableCommitAdmissionOwner; if (activeOwner) { + // Retry and recursive commit rounds remain owned by the original head until its promise settles. admitTransactionStores(transaction, activeOwner); return commit(options); } diff --git a/resources/databases.ts b/resources/databases.ts index f382ea55c1..21465f6799 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1678,8 +1678,13 @@ export async function dropDatabase(databaseName): Promise { if (!schemaQuiescence.has(quiesceMessage.quiesceId)) unavailableDatabases.delete(databaseName); } } else { - await signalling.reconcileSchemaChange(message); - unavailableDatabases.delete(databaseName); + try { + await signalling.reconcileSchemaChange(message); + } finally { + // Reconciliation applies locally before waiting for peer acknowledgements. Once + // local state is resolved, a remote timeout must not keep intact storage fenced. + if (!schemaQuiescence.has(quiesceMessage.quiesceId)) unavailableDatabases.delete(databaseName); + } } } }); diff --git a/unitTests/resources/databases.test.js b/unitTests/resources/databases.test.js index ede1fb9c15..50e65f8cac 100644 --- a/unitTests/resources/databases.test.js +++ b/unitTests/resources/databases.test.js @@ -516,6 +516,32 @@ describe('cross-worker schema quiescence', () => { } }); + it('clears the origin unavailable fence when failed-drop reconciliation only loses remote acknowledgement', async function () { + this.timeout(30000); + const DB = 'drop-reconcile-broadcast-failure-test'; + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const originalDestroy = rootStore.destroy; + const originalReconcile = signalling.reconcileSchemaChange; + rootStore.destroy = () => { + throw new Error('injected destroy failure'); + }; + signalling.reconcileSchemaChange = async (message) => { + await originalReconcile(message); + throw new Error('injected remote reconciliation acknowledgement failure'); + }; + try { + await assert.rejects(dropDatabase(DB), /injected remote reconciliation acknowledgement failure/); + assert.doesNotThrow(() => database({ database: DB, table: null })); + assert.ok(getDatabases()[DB]?.Records, 'locally reconciled intact storage must be available'); + } finally { + rootStore.destroy = originalDestroy; + signalling.reconcileSchemaChange = originalReconcile; + await dropDatabase(DB); + } + }); + it('bounds committed recovery attempts while an active drop lock is still held', async () => { const DB = 'quiesce-committed-recovery-bound-test'; const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); From 19ba24adbce17489ca40a559dfa21992b18c7a43 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 15:43:33 -0600 Subject: [PATCH 30/43] Refresh cached worker preloads after config changes --- server/threads/manageThreads.js | 42 +++++++++++-------- server/threads/resolvePreload.ts | 3 +- .../server/threads/preloadSafeMode.test.js | 16 +++++++ 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index 604b6deed0..fdc9bc74ec 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -22,25 +22,31 @@ const { _assignPackageExport } = require('../../globals.js'); const { PACKAGE_ROOT } = require('../../utility/packageUtils.js'); const { resolvePreloadModules } = require('./resolvePreload.ts'); const { getConfigPath } = require('../../config/configUtils.ts'); -let importModules; -function getImportModules() { - if (importModules === undefined) - importModules = resolvePreloadModules( - envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_PRELOAD), - getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), - 'threads.preload' +const importModuleCache = { modules: [] }; +const requireModuleCache = { modules: [] }; +function getPreloadModules(configParam, configKey, cache) { + try { + const configured = envMgr.get(configParam); + const componentsRoot = getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT); + const key = JSON.stringify([configured, componentsRoot, process.env.RUN_HDB_APP]); + if (cache.key !== key) { + const modules = resolvePreloadModules(configured, componentsRoot, configKey); + cache.key = key; + cache.modules = modules; + } + } catch (error) { + harperLogger.error( + `Unable to resolve ${configKey} modules for worker startup; keeping the prior resolution`, + error ); - return importModules; + } + return cache.modules; +} +function getImportModules() { + return getPreloadModules(hdbTerms.CONFIG_PARAMS.THREADS_PRELOAD, 'threads.preload', importModuleCache); } -let requireModules; function getRequireModules() { - if (requireModules === undefined) - requireModules = resolvePreloadModules( - envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE), - getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), - 'threads.preloadRequire' - ); - return requireModules; + return getPreloadModules(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE, 'threads.preloadRequire', requireModuleCache); } const chokidar = require('chokidar'); const isBun = typeof globalThis.Bun !== 'undefined'; @@ -385,8 +391,8 @@ function startWorker(path, options = {}) { if (!isBun && envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_HEAPSNAPSHOTNEARLIMIT)) execArgv.push('--heapsnapshot-near-heap-limit=1'); // Preload configured modules (e.g. an APM agent like dd-trace) before the worker's entry - // script so they can instrument all subsequent Harper and app module loads. Resolved once - // (config and installed components are fixed for the process lifetime). `threads.preload` + // script so they can instrument all subsequent Harper and app module loads. Resolution is + // cached until its configuration or resolution roots change. `threads.preload` // uses --import (ESM/loader-hook registration, e.g. dd-trace/register.js — the entry that // instruments worker threads); `threads.preloadRequire` uses --require for CJS agents that // document that path (e.g. dd-trace/init, Dynatrace OneAgent). --import is URL-based, so diff --git a/server/threads/resolvePreload.ts b/server/threads/resolvePreload.ts index fab3a8a42b..7af71a8172 100644 --- a/server/threads/resolvePreload.ts +++ b/server/threads/resolvePreload.ts @@ -12,8 +12,7 @@ import { PACKAGE_ROOT } from '../../utility/packageUtils.js'; // for `--import`, or `dd-trace/init` for `--require`) bundled in a deployed component can be // preloaded before any Harper or app module in a worker thread — the only point early enough for // an APM agent to instrument subsequent module loads. `configKey` only labels warnings. The -// caller memoizes the result since the config and installed components are fixed for the process -// lifetime. +// caller memoizes the result until the config or resolution roots change. export function resolvePreloadModules( configured: unknown, componentsRoot: string | undefined, diff --git a/unitTests/server/threads/preloadSafeMode.test.js b/unitTests/server/threads/preloadSafeMode.test.js index 0e1dffcb36..b02b54c0db 100644 --- a/unitTests/server/threads/preloadSafeMode.test.js +++ b/unitTests/server/threads/preloadSafeMode.test.js @@ -71,4 +71,20 @@ describe('worker preloads in safe mode', () => { assert.strictEqual(report.execArgv.includes('--import'), true); assert.strictEqual(report.execArgv.includes('--require'), true); }); + + it('refreshes resolved preloads when configuration changes after a worker starts', async function () { + this.timeout(30000); + delete process.env.HARPER_SAFE_MODE; + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOAD, null); + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE, null); + const before = await getWorkerReport(); + assert.strictEqual(before.importLoaded, false); + assert.strictEqual(before.requireLoaded, false); + + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOAD, path.join(FIXTURES, 'import.cjs')); + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE, path.join(FIXTURES, 'require.cjs')); + const after = await getWorkerReport(); + assert.strictEqual(after.importLoaded, true); + assert.strictEqual(after.requireLoaded, true); + }); }); From 4d7a4df944baa85275d9329ff2d06d18705eb4d4 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 15:46:35 -0600 Subject: [PATCH 31/43] Retry unresolved worker preloads --- server/threads/manageThreads.js | 7 +++++-- .../server/threads/preloadSafeMode.test.js | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index fdc9bc74ec..842f398bf8 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -31,12 +31,15 @@ function getPreloadModules(configParam, configKey, cache) { const key = JSON.stringify([configured, componentsRoot, process.env.RUN_HDB_APP]); if (cache.key !== key) { const modules = resolvePreloadModules(configured, componentsRoot, configKey); - cache.key = key; cache.modules = modules; + const specifierCount = (Array.isArray(configured) ? configured : [configured]).filter( + (specifier) => typeof specifier === 'string' && specifier.length > 0 + ).length; + if (modules.length === specifierCount) cache.key = key; } } catch (error) { harperLogger.error( - `Unable to resolve ${configKey} modules for worker startup; keeping the prior resolution`, + `Unable to resolve ${configKey} modules for worker startup; keeping the last known resolution`, error ); } diff --git a/unitTests/server/threads/preloadSafeMode.test.js b/unitTests/server/threads/preloadSafeMode.test.js index b02b54c0db..85df18336e 100644 --- a/unitTests/server/threads/preloadSafeMode.test.js +++ b/unitTests/server/threads/preloadSafeMode.test.js @@ -1,6 +1,8 @@ 'use strict'; const assert = require('node:assert'); +const { mkdtempSync, rmSync, writeFileSync } = require('node:fs'); +const { tmpdir } = require('node:os'); const path = require('node:path'); const envMgr = require('#src/utility/environment/environmentManager'); @@ -87,4 +89,22 @@ describe('worker preloads in safe mode', () => { assert.strictEqual(after.importLoaded, true); assert.strictEqual(after.requireLoaded, true); }); + + it('retries an unresolved preload when the module becomes available', async function () { + this.timeout(30000); + delete process.env.HARPER_SAFE_MODE; + const fixtureDirectory = mkdtempSync(path.join(tmpdir(), 'preload-late-install-')); + const importPath = path.join(fixtureDirectory, 'import.cjs'); + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOAD, importPath); + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE, null); + try { + const before = await getWorkerReport(); + assert.strictEqual(before.importLoaded, false); + writeFileSync(importPath, 'globalThis.__harperImportPreloaded = true;\n'); + const after = await getWorkerReport(); + assert.strictEqual(after.importLoaded, true); + } finally { + rmSync(fixtureDirectory, { recursive: true, force: true }); + } + }); }); From 7079798d789576646b69db3c693b8a89c985f09a Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 16:01:41 -0600 Subject: [PATCH 32/43] Release read snapshots after eviction --- resources/DatabaseTransaction.ts | 7 ++++++- resources/Table.ts | 6 +++++- unitTests/resources/expiresAtAttribute.test.js | 11 ++++++++++- unitTests/resources/indexOrphanSameKeyTxn.test.js | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index c2b752612d..4d21e8d94f 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -1118,7 +1118,12 @@ export class DatabaseTransaction implements Transaction { ); } abort(): void { - while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it + // A failed final commit has already closed this wrapper and may have replayed its writes on a + // separate native transaction while iterators retain the original read handle. Clean up the + // failed writes below, but leave that read handle to the iterators that still own it. + if (this.open !== TRANSACTION_STATE.CLOSED) { + while (this.readTxnsUsed > 0) this.doneReadTxn(); + } this.open = TRANSACTION_STATE.CLOSED; for (const write of this.writes) { if (write?.savedBlobs) diff --git a/resources/Table.ts b/resources/Table.ts index 9418a93eaa..7f7e617eb3 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2136,6 +2136,7 @@ export function makeTable(options) { if (primaryStore.ifVersion) lmdbTransaction?.abort?.(); else transaction?.abort?.(); } catch {} + if (!primaryStore.ifVersion) lmdbTransaction?.releaseReadTxn?.(); }; try { lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction() }); @@ -2233,7 +2234,10 @@ export function makeTable(options) { else logger.warn?.('Error evicting record', id, error); } ) - .finally(releaseAdmission); + .finally(() => { + lmdbTransaction.releaseReadTxn(); + releaseAdmission(); + }); } catch (error) { abortEviction(); logger.warn?.('Error evicting record', id, error); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index c8f6abfafe..9da34c726a 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -10,7 +10,11 @@ const { existsSync } = require('node:fs'); const { setTimeout: delay } = require('node:timers/promises'); const { waitFor } = require('../waitFor.js'); const { HAS_EXPIRATION_DECISION } = require('#src/resources/auditStore'); -const { TABLE_COMMIT_ADMISSION, TABLE_COMMIT_RELEASE } = require('#src/resources/DatabaseTransaction'); +const { + TABLE_COMMIT_ADMISSION, + TABLE_COMMIT_RELEASE, + setTxnExpiration, +} = require('#src/resources/DatabaseTransaction'); const { LMDBTransaction } = require('#src/resources/LMDBTransaction'); const { transaction } = require('#src/resources/transaction'); @@ -425,7 +429,12 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const Table = makeTable('ExpiresAtReadHide'); // no table-level expiration await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); await Table.primaryStore.committed; + const trackedTransactions = setTxnExpiration(30_000); + const trackedBeforeRead = trackedTransactions.size; assert.strictEqual(await Table.get(1), null); + await waitFor(() => trackedTransactions.size === trackedBeforeRead, { + message: 'read-path eviction should release its internal Rocks transaction', + }); }); it('enumerates exact Rocks index values for the expiration sweep', async function () { diff --git a/unitTests/resources/indexOrphanSameKeyTxn.test.js b/unitTests/resources/indexOrphanSameKeyTxn.test.js index 78d4e278b4..ec63354db2 100644 --- a/unitTests/resources/indexOrphanSameKeyTxn.test.js +++ b/unitTests/resources/indexOrphanSameKeyTxn.test.js @@ -204,7 +204,7 @@ describe('secondary index vs. two writes to the same key in one transaction', () setDeletionDelay(0); const BlobTbl = table({ table: 'IndexOrphanBlob', - database: 'test', + database: 'index-orphan-blob', attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'blob' }], audit: false, }); From b2986f70a6a2fb1726f493c6f24091dbb4f66e26 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 16:06:47 -0600 Subject: [PATCH 33/43] Close every eviction transaction path --- resources/DatabaseTransaction.ts | 13 +++++++++++-- resources/Table.ts | 5 ++--- unitTests/resources/expiresAtAttribute.test.js | 18 ++++++++++++++---- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 4d21e8d94f..a835325870 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -555,13 +555,18 @@ export class DatabaseTransaction implements Transaction { * abort (an in-flight replay commit owns them), only the snapshot's lifetime is enforced. */ releaseReadTxn(): void { - trackedTxns.delete(this); - this.readTxnsUsed = 0; // doneReadTxn() no-ops for the remaining iterators (guarded on this.transaction) try { this.transaction?.abort(); } catch (error) { harperLogger.debug?.('releasing timed-out read transaction', error); } + this.detachReadTxn(); + } + + /** Release wrapper bookkeeping after its native transaction was committed directly. */ + detachReadTxn(): void { + trackedTxns.delete(this); + this.readTxnsUsed = 0; // doneReadTxn() no-ops for the remaining iterators (guarded on this.transaction) this.transaction = null; this.completeDeferredContextRelease(); } @@ -1367,6 +1372,10 @@ export function resetReplayedWritesWarning() { replayedWritesWarned = false; } +export function trackedTransactionCountForTests(): number { + return trackedTxns.size; +} + export function setTxnExpiration(ms) { clearInterval(timer); txnExpiration = ms; diff --git a/resources/Table.ts b/resources/Table.ts index 7f7e617eb3..ab717a4498 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2134,9 +2134,8 @@ export function makeTable(options) { const abortEviction = () => { try { if (primaryStore.ifVersion) lmdbTransaction?.abort?.(); - else transaction?.abort?.(); + else lmdbTransaction?.releaseReadTxn?.(); } catch {} - if (!primaryStore.ifVersion) lmdbTransaction?.releaseReadTxn?.(); }; try { lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction() }); @@ -2235,7 +2234,7 @@ export function makeTable(options) { } ) .finally(() => { - lmdbTransaction.releaseReadTxn(); + lmdbTransaction.detachReadTxn(); releaseAdmission(); }); } catch (error) { diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 9da34c726a..66d5856dc8 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -13,7 +13,7 @@ const { HAS_EXPIRATION_DECISION } = require('#src/resources/auditStore'); const { TABLE_COMMIT_ADMISSION, TABLE_COMMIT_RELEASE, - setTxnExpiration, + trackedTransactionCountForTests, } = require('#src/resources/DatabaseTransaction'); const { LMDBTransaction } = require('#src/resources/LMDBTransaction'); const { transaction } = require('#src/resources/transaction'); @@ -429,12 +429,22 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const Table = makeTable('ExpiresAtReadHide'); // no table-level expiration await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); await Table.primaryStore.committed; - const trackedTransactions = setTxnExpiration(30_000); - const trackedBeforeRead = trackedTransactions.size; + const trackedBeforeRead = trackedTransactionCountForTests(); assert.strictEqual(await Table.get(1), null); - await waitFor(() => trackedTransactions.size === trackedBeforeRead, { + await waitFor(() => trackedTransactionCountForTests() === trackedBeforeRead, { message: 'read-path eviction should release its internal Rocks transaction', }); + + await Table.put(2, { id: 2, expiresAt: Date.now() + 60_000 }); + await Table.primaryStore.committed; + const staleEntry = Table.primaryStore.getEntry(2); + await Table.patch(2, { refreshed: true }); + await Table.evict(2, staleEntry.value, staleEntry.version); + assert.strictEqual( + trackedTransactionCountForTests(), + trackedBeforeRead, + 'a version-mismatched eviction should release its internal Rocks transaction' + ); }); it('enumerates exact Rocks index values for the expiration sweep', async function () { From 0c0088b50954702b9f1224020c0e43c8afb827f8 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 16:09:49 -0600 Subject: [PATCH 34/43] Pin eviction leak regression to RocksDB --- unitTests/resources/expiresAtAttribute.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 66d5856dc8..ea592b8273 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -439,6 +439,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { await Table.primaryStore.committed; const staleEntry = Table.primaryStore.getEntry(2); await Table.patch(2, { refreshed: true }); + assert.strictEqual(Table.primaryStore.ifVersion, undefined, 'premise: this regression must exercise RocksDB'); await Table.evict(2, staleEntry.value, staleEntry.version); assert.strictEqual( trackedTransactionCountForTests(), From c68e36d2c4c4829896c7d6ed2a1de2b301710655 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 16:26:09 -0600 Subject: [PATCH 35/43] Await database handle closure during quiescence --- resources/databases.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index 21465f6799..35068e0e51 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1793,9 +1793,9 @@ async function closeDatabaseOnce(databaseName: string): Promise { const rootStores = new Set(); const cleanupCompletions: Promise[] = []; const closeErrors: unknown[] = []; - const closeStore = (store: any, description: string) => { + const closeStore = async (store: any, description: string) => { try { - store?.close?.(); + await store?.close?.(); } catch (error) { closeErrors.push( new Error(`Error closing ${description} while closing database ${databaseName}`, { cause: error }) @@ -1823,9 +1823,9 @@ async function closeDatabaseOnce(databaseName: string): Promise { for (const [tableName, table] of tableEntries) { if (!table?.primaryStore) continue; for (const indexName in table.indices || {}) { - closeStore(table.indices[indexName], `index ${tableName}.${indexName}`); + await closeStore(table.indices[indexName], `index ${tableName}.${indexName}`); } - closeStore(table.primaryStore, `table ${tableName}`); + await closeStore(table.primaryStore, `table ${tableName}`); } // a database with no tables (an empty schema, or one whose tables were all dropped) still holds // an open root store, tracked only on the defined-database entry rather than any table — include @@ -1833,8 +1833,8 @@ async function closeDatabaseOnce(databaseName: string): Promise { const definedRoot = (definedDatabases?.get(databaseName) as any)?.rootStore; if (definedRoot) rootStores.add(definedRoot); for (const rootStore of rootStores) { - closeStore(rootStore.dbisDb, 'attributes store'); - closeStore(rootStore, 'root store'); + await closeStore(rootStore.dbisDb, 'attributes store'); + await closeStore(rootStore, 'root store'); forgetDatabaseEnvironment(rootStore); } if (closeErrors.length) throw new AggregateError(closeErrors, `Could not close database ${databaseName}`); From b5d41aecfcbebbc98c68bf1d89a8920c3d34dc24 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 16:33:34 -0600 Subject: [PATCH 36/43] Stabilize cleanup error drain regression --- unitTests/resources/expiresAtAttribute.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index ea592b8273..e01e50bcad 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -879,6 +879,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { table: 'PrimaryCleanupErrorDrain', database: 'test', expiration: 1, + scanInterval: 3_600, attributes: [{ name: 'id', isPrimaryKey: true }], }); const expiresAt = Date.now() - 1_000; From ef2d5d01d7509f4e6f6269efbe817c2c4975e816 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sun, 16 Aug 2026 16:46:15 -0600 Subject: [PATCH 37/43] Skip cross-worker quiescence test on Windows --- integrationTests/database/expiration-drop-quiesce.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integrationTests/database/expiration-drop-quiesce.test.ts b/integrationTests/database/expiration-drop-quiesce.test.ts index e122ec2e31..3a9dd1f767 100644 --- a/integrationTests/database/expiration-drop-quiesce.test.ts +++ b/integrationTests/database/expiration-drop-quiesce.test.ts @@ -10,6 +10,7 @@ import { createApiClient } from '../apiTests/utils/client.mjs'; const FIXTURE_PATH = resolve(import.meta.dirname, 'expiration-drop-quiesce'); const CONTROL_DIRECTORY = mkdtempSync(join(tmpdir(), 'expiration-drop-quiesce-')); +const skipSuite = process.platform === 'win32'; // Windows cannot expose two HTTP workers on one port. async function waitFor(predicate: () => boolean, message: string) { const deadline = Date.now() + 30_000; @@ -20,7 +21,7 @@ async function waitFor(predicate: () => boolean, message: string) { throw new Error(message); } -suite('cross-worker expiration cleanup quiesces destructive DDL', (ctx: ContextWithHarper) => { +suite('cross-worker expiration cleanup quiesces destructive DDL', { skip: skipSuite }, (ctx: ContextWithHarper) => { let client: ReturnType; let workerIds: number[]; From 11d91709a78e40b02e9b17291bd33bd045e3222a Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Mon, 17 Aug 2026 15:15:41 -0600 Subject: [PATCH 38/43] Release timed-out RocksDB transaction handles --- resources/DatabaseTransaction.ts | 23 +++++++++--------- unitTests/resources/txn-tracking.test.js | 30 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index a835325870..9c0ecd214e 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -1144,6 +1144,16 @@ export class DatabaseTransaction implements Transaction { // make that check see `undefined?.timedOut` and take the "start fresh" branch instead. this.releaseContext(!this.timedOut); } + private abortNativeTransaction(nativeTransaction: RocksTransaction | undefined, logMessage: string): void { + this.transaction = null; + this.readTxnsUsed = 0; + trackedTxns.delete(this); + try { + nativeTransaction?.abort(); + } catch (abortError) { + harperLogger.debug?.(logMessage, abortError); + } + } /** * Give up on a chain of linked transactions after exhausting conflict retries: poison every link * first, then abort each link's native transaction and release its DatabaseTransaction-level @@ -1159,17 +1169,7 @@ export class DatabaseTransaction implements Transaction { } for (let txn: DatabaseTransaction = this; txn; txn = txn.next) { const nativeTxn = txn === this ? headTransaction : txn.transaction; - // Clear the native handle and read-snapshot bookkeeping so the abort() below only performs - // non-native cleanup (blobs, writes) and can't double-abort (RocksTransaction.abort() throws - // on an already-aborted handle) or spin abort()'s doneReadTxn loop on a nulled handle. - txn.transaction = null; - txn.readTxnsUsed = 0; - trackedTxns.delete(txn); - try { - nativeTxn?.abort(); - } catch (abortError) { - harperLogger.debug?.('aborting conflicted transaction in chain after exhausting retries', abortError); - } + txn.abortNativeTransaction(nativeTxn, 'aborting conflicted transaction in chain after exhausting retries'); try { // abort() synchronously walks savedBlobs and can call write.store.getEntry(), which can throw // (closed store, decode error). Catch and continue so one link's wrapper-cleanup failure can't @@ -1209,6 +1209,7 @@ export class DatabaseTransaction implements Transaction { txn.open = TRANSACTION_STATE.CLOSED; } for (let txn: DatabaseTransaction = this; txn; txn = txn.next) { + txn.abortNativeTransaction(txn.transaction, 'aborting timed-out transaction in chain'); try { txn.abort(); } catch (error) { diff --git a/unitTests/resources/txn-tracking.test.js b/unitTests/resources/txn-tracking.test.js index 66a23ad98e..6c68735b16 100644 --- a/unitTests/resources/txn-tracking.test.js +++ b/unitTests/resources/txn-tracking.test.js @@ -172,6 +172,36 @@ describe('Write txn timeout', () => { } }); + it('immediately releases every native read handle when timeout poisoning a transaction chain', function () { + if (isLMDB) this.skip(); + const trackedTxns = setTxnExpiration(30_000); + const head = new DatabaseTransaction(); + const next = new DatabaseTransaction(); + head.next = next; + let abortedHandles = 0; + for (const [index, txn] of [head, next].entries()) { + txn.transaction = { + abort() { + abortedHandles++; + if (index === 0) throw new Error('expected abort failure'); + }, + }; + txn.readTxnsUsed = 2; + trackedTxns.add(txn); + } + + head.abortDueToTimeout(); + + assert.equal(abortedHandles, 2); + for (const txn of [head, next]) { + assert.strictEqual(txn.transaction, null); + assert.equal(txn.readTxnsUsed, 0); + assert.ok(!trackedTxns.has(txn)); + assert.strictEqual(txn.timedOut, true); + assert.equal(txn.open, TRANSACTION_STATE.CLOSED); + } + }); + // A handler that keeps reading must not extend the limit once it is holding uncommitted writes: // those hold write intents other writers' commits park on (harper#2001). The read-only arm below // pins the other half — reads alone still re-arm. RocksDB-only: LMDBTransaction.getReadTxn() From 879f243c3be5cddd868428d1a512da3628a66d87 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Tue, 18 Aug 2026 09:32:25 -0600 Subject: [PATCH 39/43] Complete schema quiescence recovery safely --- resources/databases.ts | 3 ++- server/itc/serverHandlers.js | 10 +++++----- server/threads/manageThreads.js | 11 +++++++++-- unitTests/resources/databases.test.js | 7 ++++--- unitTests/server/itc/serverHandlers.test.js | 19 ++++++++++++++++++- 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index 35068e0e51..6c7b79d240 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1619,9 +1619,10 @@ export function finishSchemaQuiesce(message: SchemaQuiesceMessage): boolean { return true; } -export function completeSchemaQuiesce(message: SchemaQuiesceMessage): void { +export function completeSchemaQuiesce(message: SchemaQuiesceMessage): void | Promise { recoveringSchemaQuiescences.delete(message.quiesceId); const state = schemaQuiescence.get(message.quiesceId); + if (state && message.phase === 'reconcile-quiesce') return reconcileSchemaQuiesce(state); if (state) { clearSchemaQuiesce(state); if (state.message.operation === OPERATIONS_ENUM.DROP_SCHEMA) unavailableDatabases.delete(state.message.schema); diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.js index a787e80fc5..eb359d9f86 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.js @@ -69,14 +69,14 @@ function armSchemaWorkerBarrierLease(message) { function commitSchemaWorkerBarrier(message) { if (!isMainThread) return; - const timer = schemaWorkerBarrierLeases.get(message.quiesceId); - if (timer) clearTimeout(timer); - schemaWorkerBarrierLeases.delete(message.quiesceId); + armSchemaWorkerBarrierLease(message); } function releaseSchemaWorkerBarrier(message) { if (!isMainThread) return; - commitSchemaWorkerBarrier(message); + const timer = schemaWorkerBarrierLeases.get(message.quiesceId); + if (timer) clearTimeout(timer); + schemaWorkerBarrierLeases.delete(message.quiesceId); releaseWorkerStartsForSchema(message.quiesceId); } @@ -194,7 +194,7 @@ async function applySchemaChange(event, terminalPhase) { } } if (terminalPhase) { - completeSchemaQuiesce(event.message); + await completeSchemaQuiesce(event.message); return { [terminalPhase]: true }; } } catch (error) { diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index 842f398bf8..cf148a1aa0 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -165,6 +165,12 @@ function waitForSchemaWorkerStarts() { return new Promise((resolve) => schemaWorkerStartWaiters.push(resolve)); } +function startAfterSchemaWorkerBarrier(start, description) { + waitForSchemaWorkerStarts() + .then(start) + .catch((error) => harperLogger.error(`Could not ${description} after the schema worker-start barrier:`, error)); +} + module.exports = { startWorker, restartWorkers, @@ -462,7 +468,7 @@ function startWorker(path, options = {}) { // if this wasn't an intentional shutdown, restart now (unless we have tried too many times) if (worker.unexpectedRestarts < MAX_UNEXPECTED_RESTARTS) { options.unexpectedRestarts = worker.unexpectedRestarts + 1; - waitForSchemaWorkerStarts().then(() => startWorker(path, options)); + startAfterSchemaWorkerBarrier(() => startWorker(path, options), `restart worker ${options.workerIndex}`); } else harperLogger.error(`Thread has been restarted ${worker.restarts} times and will not be restarted`); } }); @@ -654,7 +660,8 @@ async function restartWorkers( const index = waitingToFinish.indexOf(whenDone); if (index > -1) waitingToFinish.splice(index, 1); // non-overlapping types have no advance replacement, so start it once the old one is gone - if (!overlapping && startReplacementThreads) waitForSchemaWorkerStarts().then(() => worker.startCopy()); + if (!overlapping && startReplacementThreads) + startAfterSchemaWorkerBarrier(() => worker.startCopy(), `start replacement worker ${worker.threadId}`); resolve(); }); }); diff --git a/unitTests/resources/databases.test.js b/unitTests/resources/databases.test.js index 50e65f8cac..e3790826c3 100644 --- a/unitTests/resources/databases.test.js +++ b/unitTests/resources/databases.test.js @@ -404,7 +404,7 @@ describe('cross-worker schema quiescence', () => { it('stays fail-closed after the commit boundary until a terminal reconcile', async () => { const DB = 'quiesce-committed-test'; - table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); const message = { operation: 'drop_table', schema: DB, @@ -417,7 +417,8 @@ describe('cross-worker schema quiescence', () => { await assert.rejects(() => abortSchemaQuiesce(message), /commit boundary/); const terminal = { ...message, phase: 'reconcile-quiesce' }; assert.strictEqual(finishSchemaQuiesce(terminal), true); - completeSchemaQuiesce(terminal); + await completeSchemaQuiesce(terminal); + assert.strictEqual(Table.isDropQuiescing(), false); }); it('recovers an expired committed quiescence from the durable live catalog', async () => { @@ -569,6 +570,6 @@ describe('cross-worker schema quiescence', () => { ); const terminal = { ...message, phase: 'reconcile-quiesce' }; assert.strictEqual(finishSchemaQuiesce(terminal), true); - completeSchemaQuiesce(terminal); + await completeSchemaQuiesce(terminal); }); }); diff --git a/unitTests/server/itc/serverHandlers.test.js b/unitTests/server/itc/serverHandlers.test.js index 38849c8276..a5a667ec80 100644 --- a/unitTests/server/itc/serverHandlers.test.js +++ b/unitTests/server/itc/serverHandlers.test.js @@ -163,6 +163,23 @@ describe('Test hdbChildIpcHandler module', () => { } }); + it('retains a lease for the worker-start barrier after the commit boundary', () => { + const leases = server_itc_handlers.__get__('schemaWorkerBarrierLeases'); + const armBarrier = server_itc_handlers.__get__('armSchemaWorkerBarrierLease'); + const commitBarrier = server_itc_handlers.__get__('commitSchemaWorkerBarrier'); + const releaseBarrier = server_itc_handlers.__get__('releaseSchemaWorkerBarrier'); + const message = { quiesceId: 'q-committed-barrier', leaseUntil: Date.now() + 60_000 }; + try { + armBarrier(message); + const firstLease = leases.get(message.quiesceId); + commitBarrier(message); + expect(leases.has(message.quiesceId)).to.equal(true); + expect(leases.get(message.quiesceId)).not.to.equal(firstLease); + } finally { + releaseBarrier(message); + } + }); + it('aborts quiescence without resetting databases', async () => { const abortStub = sandbox.stub().resolves(); const cleanStub = sandbox.stub().resolves(); @@ -200,7 +217,7 @@ describe('Test hdbChildIpcHandler module', () => { return {}; }); const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); - const completeStub = sandbox.stub(); + const completeStub = sandbox.stub().resolves(); const restoreComplete = server_itc_handlers.__set__('completeSchemaQuiesce', completeStub); const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); const restoreReset = server_itc_handlers.__set__('resetDatabases', resetStub); From 9e54d24216d27a235e43e0f3cadca19771a66c47 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Tue, 18 Aug 2026 09:43:37 -0600 Subject: [PATCH 40/43] Refresh schema quiesce lease at commit --- unitTests/utility/signalling.test.js | 22 ++++++++++++++++++++++ utility/signalling.ts | 6 +++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/unitTests/utility/signalling.test.js b/unitTests/utility/signalling.test.js index f81d6093a3..d3dc3efcd4 100644 --- a/unitTests/utility/signalling.test.js +++ b/unitTests/utility/signalling.test.js @@ -228,4 +228,26 @@ describe('Test signalling module', () => { restoreHandlers(); } }); + + it('refreshes a stale lease at the schema commit boundary', async () => { + const localSchemaHandler = sandbox.stub().resolves({ committed: true }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + send_itc_event_stub.resolves(); + const beforeCommit = Date.now(); + try { + await signalling.commitSchemaChange({ + operation: 'drop_table', + schema: 'unit_test', + table: 'records', + quiesceId: 'q-stale-commit-lease', + leaseUntil: 1, + }); + const localMessage = localSchemaHandler.firstCall.args[0].message; + const remoteMessage = send_itc_event_stub.firstCall.args[0].message; + expect(localMessage.leaseUntil).to.be.at.least(beforeCommit + 120_000); + expect(remoteMessage.leaseUntil).to.equal(localMessage.leaseUntil); + } finally { + restoreHandlers(); + } + }); }); diff --git a/utility/signalling.ts b/utility/signalling.ts index 6814a0e06b..9165970cf7 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -148,7 +148,11 @@ async function releaseSchemaWorkerBarrier(message: any) { export async function commitSchemaChange(message: any) { serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); - const commitMessage = { ...message, phase: 'commit-quiesce' }; + const commitMessage = { + ...message, + phase: 'commit-quiesce', + leaseUntil: Date.now() + SCHEMA_QUIESCE_LEASE_MS, + }; const localEvent = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, commitMessage); const localResult = await serverItcHandlers.schema(localEvent); if (localResult?.committed !== true) From c9db0c68eefad0d84e48bc4679834a1e62c7e206 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Tue, 18 Aug 2026 10:32:04 -0600 Subject: [PATCH 41/43] Keep schema teardown fenced while origin is live --- resources/databases.ts | 33 ++++++++++++----- server/itc/serverHandlers.js | 25 ++++++++++--- server/threads/manageThreads.js | 6 +++ unitTests/resources/databases.test.js | 41 +++++++++++++++++++++ unitTests/server/itc/serverHandlers.test.js | 39 ++++++++++++++++++++ unitTests/utility/signalling.test.js | 2 + utility/signalling.ts | 2 + 7 files changed, 133 insertions(+), 15 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index 6c7b79d240..5afb1d3576 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1359,19 +1359,27 @@ function retireSchemaQuiesce(quiesceId: string): void { function armSchemaQuiesceLease(state: SchemaQuiesceState): void { if (state.lease) clearTimeout(state.lease); const leaseUntil = Math.max(state.message.leaseUntil ?? 0, Date.now() + SCHEMA_QUIESCE_LEASE_MS); - state.lease = setTimeout(() => { - if (state.committed) { - recoverCommittedSchemaQuiesce(state).catch((error) => { - logger.error(`Could not recover committed schema quiesce ${state.message.quiesceId}`, error); - }); + state.lease = setTimeout(() => expireSchemaQuiesceLease(state), leaseUntil - Date.now()); + state.lease.unref(); +} + +function expireSchemaQuiesceLease(state: SchemaQuiesceState): void { + if (state.committed) { + // A synchronous RocksDB drop can outlive the timer that its blocked origin would renew. + // A connected owner may still be destructing storage, so peers must remain fail-closed. + if (!state.localOwner && manageThreads.isThreadConnected(state.message.originator)) { + armSchemaQuiesceLease(state); return; } - reconcileSchemaQuiesce(state).catch((error) => { - logger.warn('Could not reconcile expired schema quiesce:', error); - if (schemaQuiescence.get(state.message.quiesceId) === state) armSchemaQuiesceLease(state); + recoverCommittedSchemaQuiesce(state).catch((error) => { + logger.error(`Could not recover committed schema quiesce ${state.message.quiesceId}`, error); }); - }, leaseUntil - Date.now()); - state.lease.unref(); + return; + } + reconcileSchemaQuiesce(state).catch((error) => { + logger.warn('Could not reconcile expired schema quiesce:', error); + if (schemaQuiescence.get(state.message.quiesceId) === state) armSchemaQuiesceLease(state); + }); } async function recoverCommittedSchemaQuiesce(state: SchemaQuiesceState): Promise { @@ -1443,6 +1451,11 @@ export function recoverCommittedSchemaQuiesceForTests(quiesceId: string): Promis return state ? recoverCommittedSchemaQuiesce(state) : Promise.resolve(false); } +export function expireSchemaQuiesceLeaseForTests(quiesceId: string): void { + const state = schemaQuiescence.get(quiesceId); + if (state) expireSchemaQuiesceLease(state); +} + function resetQuiescedDatabase(databaseName: string): void { delete databases[databaseName]; const definedDatabase = definedDatabases?.get(databaseName); diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.js index eb359d9f86..0344a1df5f 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.js @@ -23,7 +23,11 @@ const { completeSchemaQuiesce, failSchemaQuiesceFinalization, } = require('../../resources/databases.ts'); -const { holdWorkerStartsForSchema, releaseWorkerStartsForSchema } = require('../threads/manageThreads.js'); +const { + holdWorkerStartsForSchema, + releaseWorkerStartsForSchema, + isThreadConnected, +} = require('../threads/manageThreads.js'); /** * This object/functions are passed to the ITC client instance and dynamically added as event handlers. @@ -53,16 +57,27 @@ const MAX_SCHEMA_TERMINAL_OUTCOMES = 1024; const schemaTerminalCompletions = new Map(); const schemaTerminalOutcomes = new Map(); const schemaWorkerBarrierLeases = new Map(); +const SCHEMA_WORKER_BARRIER_RECHECK_MS = 1_000; + +function expireSchemaWorkerBarrierLease(message) { + // The origin can legitimately block its event loop during synchronous RocksDB destruction. + // Keep replacement workers fenced until that owner disconnects or sends a terminal message. + if (isThreadConnected(message.originator)) { + const timer = setTimeout(() => expireSchemaWorkerBarrierLease(message), SCHEMA_WORKER_BARRIER_RECHECK_MS); + timer.unref(); + schemaWorkerBarrierLeases.set(message.quiesceId, timer); + return; + } + schemaWorkerBarrierLeases.delete(message.quiesceId); + releaseWorkerStartsForSchema(message.quiesceId); +} function armSchemaWorkerBarrierLease(message) { if (!isMainThread) return; const existing = schemaWorkerBarrierLeases.get(message.quiesceId); if (existing) clearTimeout(existing); const delay = Math.max(0, (message.leaseUntil ?? Date.now()) - Date.now()); - const timer = setTimeout(() => { - schemaWorkerBarrierLeases.delete(message.quiesceId); - releaseWorkerStartsForSchema(message.quiesceId); - }, delay); + const timer = setTimeout(() => expireSchemaWorkerBarrierLease(message), delay); timer.unref(); schemaWorkerBarrierLeases.set(message.quiesceId, timer); } diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index cf148a1aa0..9a285d4c2d 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -165,6 +165,11 @@ function waitForSchemaWorkerStarts() { return new Promise((resolve) => schemaWorkerStartWaiters.push(resolve)); } +function isThreadConnected(ownerThreadId) { + if (ownerThreadId === threadId || ownerThreadId === 0) return true; + return connectedPorts.some((port) => port.threadId === ownerThreadId); +} + function startAfterSchemaWorkerBarrier(start, description) { waitForSchemaWorkerStarts() .then(start) @@ -193,6 +198,7 @@ module.exports = { holdWorkerStartsForSchema, releaseWorkerStartsForSchema, waitForSchemaWorkerStarts, + isThreadConnected, registerWorkerDataProvider, onThreadExit, registerProcessGroup, diff --git a/unitTests/resources/databases.test.js b/unitTests/resources/databases.test.js index e3790826c3..6dcda230ac 100644 --- a/unitTests/resources/databases.test.js +++ b/unitTests/resources/databases.test.js @@ -14,8 +14,10 @@ const { quiesceSchemaTarget, abortSchemaQuiesce, commitSchemaQuiesce, + renewSchemaQuiesce, finishSchemaQuiesce, completeSchemaQuiesce, + expireSchemaQuiesceLeaseForTests, recoverCommittedSchemaQuiesceForTests, } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); @@ -421,6 +423,45 @@ describe('cross-worker schema quiescence', () => { assert.strictEqual(Table.isDropQuiescing(), false); }); + it('does not recover a committed peer quiescence while its origin is connected', async () => { + const message = { + originator: require('node:worker_threads').threadId, + operation: 'drop_table', + schema: 'quiesce-live-origin-test', + table: 'Missing', + quiesceId: 'q-live-origin', + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + expireSchemaQuiesceLeaseForTests(message.quiesceId); + assert.strictEqual( + renewSchemaQuiesce({ ...message, leaseUntil: Date.now() + 60_000 }).quiesced, + true, + 'a connected origin must keep the committed peer fence active' + ); + const terminal = { ...message, phase: 'reconcile-quiesce' }; + assert.strictEqual(finishSchemaQuiesce(terminal), true); + await completeSchemaQuiesce(terminal); + }); + + it('recovers a committed peer quiescence after its origin disconnects', async () => { + const message = { + originator: Number.MAX_SAFE_INTEGER, + operation: 'drop_table', + schema: 'quiesce-disconnected-origin-test', + table: 'Missing', + quiesceId: 'q-disconnected-origin', + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + expireSchemaQuiesceLeaseForTests(message.quiesceId); + assert.strictEqual( + renewSchemaQuiesce({ ...message, leaseUntil: Date.now() + 60_000 }).quiesced, + false, + 'a disconnected origin must allow authoritative recovery' + ); + }); + it('recovers an expired committed quiescence from the durable live catalog', async () => { const DB = 'quiesce-committed-recovery-test'; const Original = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); diff --git a/unitTests/server/itc/serverHandlers.test.js b/unitTests/server/itc/serverHandlers.test.js index a5a667ec80..a8bc22ee24 100644 --- a/unitTests/server/itc/serverHandlers.test.js +++ b/unitTests/server/itc/serverHandlers.test.js @@ -13,6 +13,8 @@ const harperBridge = require('#src/dataLayer/harperBridge/harperBridge').default // for testing validation logic, not for replacing dependencies with mocks const server_itc_handlers = rewire('#js/server/itc/serverHandlers'); const { resetResources } = require('#src/resources/Resources'); +const { threadId } = require('node:worker_threads'); +const { waitForSchemaWorkerStarts } = require('#js/server/threads/manageThreads'); describe('Test hdbChildIpcHandler module', () => { const TEST_ERR = 'The roof is on fire'; @@ -180,6 +182,43 @@ describe('Test hdbChildIpcHandler module', () => { } }); + it('retains the worker-start barrier while the committed origin is connected', async () => { + const message = { + originator: threadId, + operation: 'drop_table', + phase: 'hold-worker-starts', + schema: 'test', + table: 'records', + quiesceId: 'q-live-origin-barrier', + leaseUntil: Date.now(), + }; + await schema_handler({ type: 'schema', message }); + let released = false; + const waiting = waitForSchemaWorkerStarts().then(() => (released = true)); + try { + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(released).to.equal(false); + } finally { + await schema_handler({ type: 'schema', message: { ...message, phase: 'release-worker-starts' } }); + } + await waiting; + expect(released).to.equal(true); + }); + + it('releases an expired worker-start barrier after its origin disconnects', async () => { + const message = { + originator: Number.MAX_SAFE_INTEGER, + operation: 'drop_table', + phase: 'hold-worker-starts', + schema: 'test', + table: 'records', + quiesceId: 'q-disconnected-origin-barrier', + leaseUntil: Date.now(), + }; + await schema_handler({ type: 'schema', message }); + await waitForSchemaWorkerStarts(); + }); + it('aborts quiescence without resetting databases', async () => { const abortStub = sandbox.stub().resolves(); const cleanStub = sandbox.stub().resolves(); diff --git a/unitTests/utility/signalling.test.js b/unitTests/utility/signalling.test.js index d3dc3efcd4..3e4bf9a330 100644 --- a/unitTests/utility/signalling.test.js +++ b/unitTests/utility/signalling.test.js @@ -3,6 +3,7 @@ const chai = require('chai'); const sinon = require('sinon'); const rewire = require('rewire'); +const { threadId } = require('node:worker_threads'); const { expect } = chai; const sinon_chai = require('sinon-chai').default; chai.use(sinon_chai); @@ -93,6 +94,7 @@ describe('Test signalling module', () => { table: 'records', }); const quiesceCall = send_itc_event_stub.getCalls().find((call) => call.args[0].message.phase === 'quiesce'); + expect(message.originator).to.equal(threadId); expect(quiesceCall.args[1].includeJobWorkers).to.equal(true); expect(quiesceCall.args[1].acceptResult({ quiesced: true })).to.equal(true); await signalling.abortSchemaQuiesce(message); diff --git a/utility/signalling.ts b/utility/signalling.ts index 9165970cf7..dc367bda8c 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -4,6 +4,7 @@ import * as hdbTerms from './hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import ITCEventObject from '../server/itc/utility/ITCEventObject.js'; import { randomUUID } from 'node:crypto'; +import { threadId } from 'node:worker_threads'; let serverItcHandlers; import { sendItcEvent } from '../server/threads/itc.js'; @@ -64,6 +65,7 @@ function startSchemaQuiesceRenewal(message: any) { export async function quiesceSchemaChange(message: any) { const quiesceMessage = { ...message, + originator: threadId, phase: 'quiesce', quiesceId: randomUUID(), leaseUntil: Date.now() + SCHEMA_QUIESCE_LEASE_MS, From 8b89fbc452d791d1e29ad3707286cce90125d981 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Thu, 20 Aug 2026 09:38:33 -0600 Subject: [PATCH 42/43] Harden eviction metadata and commit admission --- resources/Table.ts | 16 +++- .../resources/expiresAtAttribute.test.js | 82 +++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index cb084e556b..55e3ac1845 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -479,6 +479,10 @@ export function makeTable(options) { return true; }; (primaryStore as any)[TABLE_COMMIT_RELEASE] = () => { + if (pendingTableCommitCount === 0) { + harperLogger.warn?.(`Ignored unmatched table commit release for ${tableName}`); + return; + } if (--pendingTableCommitCount === 0 && resolvePendingTableCommits) { const resolve = resolvePendingTableCommits; pendingTableCommitWaiter = undefined; @@ -2145,8 +2149,16 @@ export function makeTable(options) { // A raw Rocks read transaction can reuse an older snapshot. Reject an already-refreshed // record before consulting it; commit-time conflict detection guards later races. if (!primaryStore.ifVersion) { - currentEntry = primaryStore.getEntry(id, { lazy: true }); - if (!currentEntry || currentEntry.version !== existingVersion) return Promise.resolve(); + const lazyEntry = primaryStore.getEntry(id, { lazy: true }); + if (!lazyEntry || lazyEntry.version !== existingVersion) return Promise.resolve(); + // A lazy Rocks entry can reuse its decode buffer during the index updates below. Snapshot + // every field eviction still needs before another store operation can recycle that buffer. + currentEntry = { + key: id, + version: lazyEntry.version, + expiresAt: lazyEntry.expiresAt, + metadataFlags: lazyEntry.metadataFlags, + }; } if (hasSourceGet || audit) { if (!existingRecord) return Promise.resolve(); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index e01e50bcad..3af75f838d 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -17,6 +17,7 @@ const { } = require('#src/resources/DatabaseTransaction'); const { LMDBTransaction } = require('#src/resources/LMDBTransaction'); const { transaction } = require('#src/resources/transaction'); +const harperLogger = require('#src/utility/logging/harper_logger'); const activeTables = new Set(); const table = (options) => { @@ -662,6 +663,64 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } }); + it('snapshots lazy Rocks metadata before eviction updates recycle the decode buffer', async function () { + const Table = table({ + table: 'ExpiresAtLazyBlobEviction', + database: 'expires-at-lazy-blob-eviction', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }); + const blob = createBlob(Buffer.alloc(20_000, 13)); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: blob }); + await Table.primaryStore.committed; + const storedEntry = Table.primaryStore.getEntry(1); + const filePath = getFilePathForBlob(blob); + const lazyState = { + version: storedEntry.version, + expiresAt: storedEntry.expiresAt, + metadataFlags: storedEntry.metadataFlags, + }; + const originalGetEntry = Table.primaryStore.getEntry.bind(Table.primaryStore); + const originalIndexRemove = Table.indices.expiresAt.remove.bind(Table.indices.expiresAt); + Table.primaryStore.getEntry = (id, options) => + options?.lazy + ? { + get version() { + return lazyState.version; + }, + get expiresAt() { + return lazyState.expiresAt; + }, + get metadataFlags() { + return lazyState.metadataFlags; + }, + get key() { + return id; + }, + } + : originalGetEntry(id, options); + Table.indices.expiresAt.remove = (...args) => { + lazyState.metadataFlags = 0; + return originalIndexRemove(...args); + }; + + setDeletionDelay(0); + try { + await Table.evict(1, storedEntry.value, storedEntry.version); + await waitFor(() => !existsSync(filePath), { + timeout: 5_000, + message: 'eviction should use the snapshotted blob metadata after index writes', + }); + } finally { + Table.primaryStore.getEntry = originalGetEntry; + Table.indices.expiresAt.remove = originalIndexRemove; + setDeletionDelay(500); + } + }); + it('keeps a blob when eviction lacks the record and unlinks it after a committed eviction', async function () { const Table = table({ table: 'ExpiresAtBlobFailSafe', @@ -980,6 +1039,29 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } }); + it('ignores an unmatched commit release without wedging later drop quiescence', async function () { + const Table = makeTable('ExpiresAtCommitReleaseFloor'); + const admit = Table.primaryStore[TABLE_COMMIT_ADMISSION]; + const release = Table.primaryStore[TABLE_COMMIT_RELEASE]; + const originalWarn = harperLogger.warn; + const warnings = []; + harperLogger.warn = (...args) => warnings.push(args); + try { + release(); + assert.match(warnings[0][0], /unmatched table commit release/); + assert.strictEqual(admit(), true); + let quiesced = false; + const quiesce = Table.quiesceForDrop().then(() => (quiesced = true)); + await delay(10); + assert.strictEqual(quiesced, false, 'quiescence must still wait for the balanced admission'); + release(); + await quiesce; + } finally { + harperLogger.warn = originalWarn; + Table.abortDropQuiesce(); + } + }); + it('joins read-path eviction to drop quiescence and skips eviction after quiescence starts', async function () { const Table = makeTable('ExpiresAtReadEvictionQuiesce'); const expiresAt = Date.now() - 1_000; From 33e9beadab1c1f9b44130f244d1887a1f651a4e4 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Thu, 20 Aug 2026 09:42:28 -0600 Subject: [PATCH 43/43] Improve commit admission diagnostics --- resources/Table.ts | 9 ++++++++- unitTests/resources/expiresAtAttribute.test.js | 14 +++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 55e3ac1845..fea6168f19 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -445,6 +445,7 @@ export function makeTable(options) { let pendingTableCommitCount = 0; let pendingTableCommitWaiter: Promise | undefined; let resolvePendingTableCommits: (() => void) | undefined; + let unmatchedTableCommitReleaseReported = false; let droppingTable = false; let createdTimeProperty: Attribute | undefined, updatedTimeProperty: Attribute | undefined, @@ -480,7 +481,12 @@ export function makeTable(options) { }; (primaryStore as any)[TABLE_COMMIT_RELEASE] = () => { if (pendingTableCommitCount === 0) { - harperLogger.warn?.(`Ignored unmatched table commit release for ${tableName}`); + // Admission is synchronous and always precedes release, so this can only be duplicate + // accounting. Stay at the safe floor and report the first caller with a stack. + if (!unmatchedTableCommitReleaseReported) { + unmatchedTableCommitReleaseReported = true; + harperLogger.error?.(`Ignored unmatched table commit release for ${tableName}`, new Error()); + } return; } if (--pendingTableCommitCount === 0 && resolvePendingTableCommits) { @@ -2153,6 +2159,7 @@ export function makeTable(options) { if (!lazyEntry || lazyEntry.version !== existingVersion) return Promise.resolve(); // A lazy Rocks entry can reuse its decode buffer during the index updates below. Snapshot // every field eviction still needs before another store operation can recycle that buffer. + // Value is deliberately omitted: blobs use existingRecord; non-blob removal only needs key. currentEntry = { key: id, version: lazyEntry.version, diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 3af75f838d..7890caf9cd 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -683,6 +683,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { expiresAt: storedEntry.expiresAt, metadataFlags: storedEntry.metadataFlags, }; + let lazyMetadataReads = 0; const originalGetEntry = Table.primaryStore.getEntry.bind(Table.primaryStore); const originalIndexRemove = Table.indices.expiresAt.remove.bind(Table.indices.expiresAt); Table.primaryStore.getEntry = (id, options) => @@ -695,6 +696,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { return lazyState.expiresAt; }, get metadataFlags() { + lazyMetadataReads++; return lazyState.metadataFlags; }, get key() { @@ -709,7 +711,9 @@ describe('@expiresAt attribute is authoritative over the table default', () => { setDeletionDelay(0); try { + assert.strictEqual(Table.primaryStore.ifVersion, undefined, 'premise: this must exercise RocksDB eviction'); await Table.evict(1, storedEntry.value, storedEntry.version); + assert.strictEqual(lazyMetadataReads, 1, 'eviction must consume lazy metadata only while creating the snapshot'); await waitFor(() => !existsSync(filePath), { timeout: 5_000, message: 'eviction should use the snapshotted blob metadata after index writes', @@ -1043,12 +1047,12 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const Table = makeTable('ExpiresAtCommitReleaseFloor'); const admit = Table.primaryStore[TABLE_COMMIT_ADMISSION]; const release = Table.primaryStore[TABLE_COMMIT_RELEASE]; - const originalWarn = harperLogger.warn; - const warnings = []; - harperLogger.warn = (...args) => warnings.push(args); + const originalError = harperLogger.error; + const errors = []; + harperLogger.error = (...args) => errors.push(args); try { release(); - assert.match(warnings[0][0], /unmatched table commit release/); + assert.ok(errors.some(([message]) => /unmatched table commit release/.test(message))); assert.strictEqual(admit(), true); let quiesced = false; const quiesce = Table.quiesceForDrop().then(() => (quiesced = true)); @@ -1057,7 +1061,7 @@ describe('@expiresAt attribute is authoritative over the table default', () => { release(); await quiesce; } finally { - harperLogger.warn = originalWarn; + harperLogger.error = originalError; Table.abortDropQuiesce(); } });