diff --git a/DESIGN.md b/DESIGN.md index 8b8a19dcbd..187727dbdb 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -53,6 +53,8 @@ The mitigations live in three places: When adding a new commit-handler early-return path: reset `write.skipped = false` at the top of the handler if you don't already, then set `write.skipped = true` immediately before the `return`. Decide first whether the audit log will reference the blob (via `auditRecordToStore`) — if it does, leave `skipped` unset. `cleanupOrphans` is the periodic safety net; don't rely on it for transactional correctness. +RocksDB eviction has the inverse ordering constraint: removing a record that already references a blob must not unlink the file until the raw transaction commits. A transactional `store.remove()` is synchronous because it only stages the delete, so `removeEntry()` cannot infer durability from its return value on this path. `TableResource.evict()` therefore carries the blob-bearing record through the commit and calls `deleteBlobsInObject()` only from the commit-success handler. Shared eviction batches must continue excluding blob-bearing records because one commit covers many staged removals and can conflict as a unit. + **Source-unavailable blobs must not abort the commit.** `startPreCommitBlobsForRecord().complete()` awaits each blob's `saving` promise; a rejection there propagates up and aborts the record's apply (the replication subscription loop catches and logs it as `error in subscription handler`). For a blob the replication source can no longer provide — evicted/expired at the origin, the receiver having flagged the rejection `sourceBlobUnavailable` (harper-pro#403) — that abort permanently wedged a replication copy stream on an expiration cache table whose TTL-evicted blobs are gone everywhere: every orphaned record's apply re-threw, the copy never advanced, and backpressure pinned at ~100%. `complete()` therefore tolerates a `sourceBlobUnavailable` rejection (`isSourceBlobUnavailable`): the record commits with a diverged blob reference, left for proactive backfill (harper-pro#388). Local/transient save faults stay unmarked and still reject, so the write aborts and a reconnect retries it — no silent loss. This is the apply/commit-side complement to the replication receiver's resume-cursor advance (harper-pro#403/#405), which handles the durability-watermark side of the same missing blob. ## Over-time transactions are aborted, not force-committed (`DatabaseTransaction`/`LMDBTransaction`) diff --git a/dataLayer/harperBridge/ResourceBridge.ts b/dataLayer/harperBridge/ResourceBridge.ts index 847ac6e196..046c7899ad 100644 --- a/dataLayer/harperBridge/ResourceBridge.ts +++ b/dataLayer/harperBridge/ResourceBridge.ts @@ -186,7 +186,6 @@ export class ResourceBridge extends BridgeMethods { async dropSchema(dropSchemaObj) { await dropDatabase(dropSchemaObj.schema); - signalling.signalSchemaChange(new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, dropSchemaObj.schema)); } async updateRecords(updateObj) { diff --git a/integrationTests/database/expiration-drop-quiesce.test.ts b/integrationTests/database/expiration-drop-quiesce.test.ts new file mode 100644 index 0000000000..3a9dd1f767 --- /dev/null +++ b/integrationTests/database/expiration-drop-quiesce.test.ts @@ -0,0 +1,135 @@ +import { after, before, suite, test } from 'node:test'; +import { strictEqual } from 'node:assert'; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; +import request from 'supertest'; +import { createApiClient } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'expiration-drop-quiesce'); +const CONTROL_DIRECTORY = mkdtempSync(join(tmpdir(), 'expiration-drop-quiesce-')); +const skipSuite = process.platform === 'win32'; // Windows cannot expose two HTTP workers on one port. + +async function waitFor(predicate: () => boolean, message: string) { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + if (predicate()) return; + await delay(20); + } + throw new Error(message); +} + +suite('cross-worker expiration cleanup quiesces destructive DDL', { skip: skipSuite }, (ctx: ContextWithHarper) => { + let client: ReturnType; + let workerIds: number[]; + + before(async () => { + await setupHarperWithFixture(ctx, FIXTURE_PATH, { + config: { threads: { count: 2 }, logging: { level: 'error' } }, + env: { EXPIRATION_QUIESCE_CONTROL: CONTROL_DIRECTORY }, + } as any); + client = createApiClient(ctx.harper); + let ready = false; + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + try { + const response = await request(client.restURL) + .post('/QuiesceControl/') + .set(client.headers) + .timeout(2_000) + .send({ action: 'probe' }); + if (response.status !== 404) { + ready = true; + break; + } + } catch { + // Workers are still loading the pre-installed component. + } + await delay(250); + } + strictEqual(ready, true, 'QuiesceControl resource did not become ready'); + const probe = await postControl({ action: 'probe' }); + workerIds = probe.body.workerIds; + strictEqual(workerIds.length, 2, 'the fixture must expose both HTTP workers for deterministic pinning'); + }); + + after(async () => { + await teardownHarper(ctx); + }); + + async function postControl(body: Record) { + return request(client.restURL).post('/QuiesceControl/').set(client.headers).timeout(120_000).send(body); + } + + async function postControlOnWorker(body: Record, targetThreadId: number) { + const response = await postControl({ ...body, targetThreadId }); + strictEqual(response.body?.threadId, targetThreadId, `control action did not execute on worker ${targetThreadId}`); + return response; + } + + async function proveDropWaitsForWorker(kind: 'indexed' | 'primary') { + const indexed = kind === 'indexed'; + const database = indexed ? 'quiesce_indexed' : 'quiesce_primary'; + const table = indexed ? 'IndexedExpiry' : 'PrimaryExpiry'; + const runId = `${kind}-${Date.now()}`; + const [sweepWorkerId, ddlWorkerId] = workerIds; + await postControl({ action: 'seed', kind, database, table, id: runId }).then((response) => + strictEqual(response.status, 200) + ); + const sweepRunId = `${runId}-sweep`; + const dropRunId = `${runId}-drop`; + const started = join(CONTROL_DIRECTORY, `${sweepRunId}.started`); + const release = join(CONTROL_DIRECTORY, `${runId}.release`); + const sweep = postControlOnWorker( + { action: 'sweep', kind, database, table, runId: sweepRunId, releaseRunId: runId }, + sweepWorkerId + ); + let dropSettled = false; + const dropStarted = join(CONTROL_DIRECTORY, `${dropRunId}.started`); + let drop: ReturnType | undefined; + let dropped; + let primaryError: unknown; + const cleanupErrors: unknown[] = []; + try { + await waitFor(() => existsSync(started), `${kind} sweep did not reach its blocked commit`); + strictEqual(JSON.parse(readFileSync(started, 'utf8')).threadId, sweepWorkerId); + drop = postControlOnWorker({ action: 'drop', database, table, runId: dropRunId }, ddlWorkerId).then( + (response) => { + dropSettled = true; + return response; + } + ); + await waitFor(() => existsSync(dropStarted), `${kind} drop did not start on its pinned worker`); + strictEqual(JSON.parse(readFileSync(dropStarted, 'utf8')).threadId, ddlWorkerId); + strictEqual(sweepWorkerId === ddlWorkerId, false, 'DDL and cleanup must execute on different workers'); + await delay(100); + strictEqual(dropSettled, false, 'physical drop must wait for the blocked cleanup worker'); + } catch (error) { + primaryError = error; + } finally { + try { + writeFileSync(release, 'release'); + } catch (error) { + cleanupErrors.push(error); + } + const completions = drop ? [drop, sweep] : [sweep]; + const results = await Promise.allSettled(completions); + if (drop && results[0].status === 'fulfilled') dropped = results[0].value; + for (const result of results) if (result.status === 'rejected') cleanupErrors.push(result.reason); + } + if (primaryError) throw primaryError; + if (cleanupErrors.length) throw new AggregateError(cleanupErrors, `${kind} quiescence test cleanup failed`); + strictEqual(dropped.status, 200); + strictEqual(dropped.body.threadId, ddlWorkerId); + } + + test('drop_table waits for a remote indexed expiration sweep', async () => { + await proveDropWaitsForWorker('indexed'); + }); + + test('drop_table waits for a remote primary cleanup scan', async () => { + await proveDropWaitsForWorker('primary'); + }); +}); diff --git a/integrationTests/database/expiration-drop-quiesce/config.yaml b/integrationTests/database/expiration-drop-quiesce/config.yaml new file mode 100644 index 0000000000..efffc0833f --- /dev/null +++ b/integrationTests/database/expiration-drop-quiesce/config.yaml @@ -0,0 +1,5 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true diff --git a/integrationTests/database/expiration-drop-quiesce/resources.js b/integrationTests/database/expiration-drop-quiesce/resources.js new file mode 100644 index 0000000000..873de99eff --- /dev/null +++ b/integrationTests/database/expiration-drop-quiesce/resources.js @@ -0,0 +1,105 @@ +import { existsSync, renameSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { threadId } from 'node:worker_threads'; + +const controlDirectory = process.env.EXPIRATION_QUIESCE_CONTROL; +const CONTROL_REQUEST = 'expiration-quiesce-control-request'; +const CONTROL_RESPONSE = 'expiration-quiesce-control-response'; +const pendingRequests = new Map(); +let nextRequestId = 1; + +function publishWorkerMarker(path) { + const temporaryPath = `${path}.${threadId}.tmp`; + writeFileSync(temporaryPath, JSON.stringify({ threadId })); + renameSync(temporaryPath, path); +} + +async function performControl(body) { + const Table = databases[body.database]?.[body.table]; + if (!Table) return { available: false, threadId }; + if (body.action === 'seed') { + const expiresAt = Date.now() - 10_000; + if (body.kind === 'indexed') await Table.put(body.id, { id: body.id, expiresAt }); + else await Table.put(body.id, { id: body.id, value: 'expired' }, { expiresAt }); + return { seeded: true, threadId }; + } + const started = join(controlDirectory, `${body.runId}.started`); + const release = join(controlDirectory, `${body.releaseRunId ?? body.runId}.release`); + if (body.action === 'drop') { + publishWorkerMarker(started); + await Table.dropTable(); + return { completed: true, threadId }; + } + const hooks = { + beforeBatchCommit: async () => { + publishWorkerMarker(started); + while (!existsSync(release)) await delay(20); + }, + }; + if (body.kind === 'indexed') await Table.runRecordExpirationSweepForTests(hooks); + else await Table.runPrimaryCleanupScanForTests(hooks); + return { completed: true, threadId }; +} + +threads.onMessageByType(CONTROL_REQUEST, async (message) => { + if (message.targetThreadId !== threadId) return; + try { + const result = await performControl(message.body); + threads.sendToThread(message.originThreadId, { type: CONTROL_RESPONSE, requestId: message.requestId, result }); + } catch (error) { + threads.sendToThread(message.originThreadId, { + type: CONTROL_RESPONSE, + requestId: message.requestId, + error: error?.stack ?? error?.message ?? String(error), + }); + } +}); + +threads.onMessageByType(CONTROL_RESPONSE, (message) => { + const pending = pendingRequests.get(message.requestId); + if (!pending) return; + pendingRequests.delete(message.requestId); + clearTimeout(pending.timer); + if (message.error) pending.reject(new Error(message.error)); + else pending.resolve(message.result); +}); + +function performControlOnWorker(body, targetThreadId) { + if (targetThreadId === threadId) return performControl(body); + return new Promise((resolve, reject) => { + const requestId = `${threadId}:${nextRequestId++}`; + const timer = setTimeout(() => { + pendingRequests.delete(requestId); + reject(new Error(`Timed out dispatching expiration quiesce control to worker ${targetThreadId}`)); + }, 120_000); + timer.unref(); + pendingRequests.set(requestId, { resolve, reject, timer }); + if ( + !threads.sendToThread(targetThreadId, { + type: CONTROL_REQUEST, + requestId, + originThreadId: threadId, + targetThreadId, + body, + }) + ) { + clearTimeout(timer); + pendingRequests.delete(requestId); + reject(new Error(`Worker ${targetThreadId} is not reachable`)); + } + }); +} + +export class QuiesceControl extends Resource { + static loadAsInstance = false; + + async post(_query, body) { + if (body.action === 'probe') { + const workerIds = [threadId, ...threads.map((port) => port.threadId)].filter((id) => id > 0); + return { ready: true, threadId, workerIds: [...new Set(workerIds)] }; + } + if (body.targetThreadId !== undefined) return performControlOnWorker(body, body.targetThreadId); + return performControl(body); + } +} diff --git a/integrationTests/database/expiration-drop-quiesce/schema.graphql b/integrationTests/database/expiration-drop-quiesce/schema.graphql new file mode 100644 index 0000000000..becff7fccf --- /dev/null +++ b/integrationTests/database/expiration-drop-quiesce/schema.graphql @@ -0,0 +1,9 @@ +type IndexedExpiry @table(database: "quiesce_indexed") @export { + id: ID! @primaryKey + expiresAt: Float @expiresAt +} + +type PrimaryExpiry @table(database: "quiesce_primary", expiration: 1, eviction: 1) @export { + id: ID! @primaryKey + value: String +} diff --git a/resources/DESIGN.md b/resources/DESIGN.md index fffde552d6..9f062df316 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -105,6 +105,36 @@ One giant `makeTable()` factory that returns a `TableResource extends Resource` --- +## Record expiration sweeps + +An `@expiresAt` attribute is a public timestamp field backed by the record encoder's expiration metadata. +Its index stores one canonical, non-negative epoch-millisecond number derived from that metadata, regardless of +whether the API value was a number, numeric string, ISO string, or `Date`. Writes, index rebuilds, index searches, +and scan filters apply the same normalization. `expirationIndexVersion` participates in the normal resumable +schema reindex flow so stores created before canonicalization are rebuilt without a separate open-time scan. +Rows created before expiration metadata was stored fall back to their public field during that rebuild and while +sweeping; an explicit no-expiration metadata sentinel never falls back. This preserves upgrade behavior without +making every open scan the table. + +The expiration index represents the effective stored expiration, not an ordinary field-value index. An explicit +`options.expiresAt` or `context.expiresAt` override can therefore differ from the serialized field while remaining +authoritative for index searches and reclamation. Source/cache fills do not infer TTL from the returned field; the +source must set `sourceContext.expiresAt`, with table expiration providing the fallback. + +RocksDB `@expiresAt` sweeps walk one bounded composite-index range at a time with a fixed cutoff and an +owned `(expiresAt, primaryKey)` cursor. A sweep never holds an iterator snapshot across an `await`, and it +continues until every index entry at or before that cutoff has been considered. This keeps memory and native +iterator lifetime bounded without leaving a permanent backlog when more than one chunk expires together. + +Dangling-index cleanup reads and writes the primary key in the same RocksDB transaction as the index removal. +The primary write is a conflict guard against concurrent resurrection: absent keys receive a transactional +remove, while retained audit tombstones are rewritten byte-for-byte so their version and retention semantics do +not change. An `ERR_BUSY` abort leaves the newly written record and index authoritative for the next sweep. + +Table cleanup is a join point for both the primary reclamation scan and the `@expiresAt` index sweep. Database +close and drop wait for that join before closing or destroying stores. A timeout fails closed: handles and +registries remain live and cleanup scheduling is resumed, so active work is never raced by teardown. + ## Path routing & parameterised routes `Resources.ts` is the registry that maps URL paths to `Resource` classes. Resources are registered (`jsResource.ts`) from a component's exports: diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index ac8a68e770..bbe8a3549d 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -244,6 +244,111 @@ export type CommitOptions = { transaction?: RocksTransaction; }; +export const TABLE_COMMIT_ADMISSION = Symbol.for('harper.table-commit-admission'); +export const TABLE_COMMIT_RELEASE = Symbol.for('harper.table-commit-release'); + +function hasAdmittedStore(owner: DatabaseTransaction, store: any): boolean { + return owner.admittedTableStore === store || owner.additionalAdmittedTableStores?.includes(store) === true; +} + +function addAdmittedStore(owner: DatabaseTransaction, store: any): void { + if (owner.admittedTableStore === undefined) owner.admittedTableStore = store; + else (owner.additionalAdmittedTableStores ??= []).push(store); +} + +function stampTransactionLink(owner: DatabaseTransaction, link: DatabaseTransaction): void { + if (link.tableCommitAdmissionOwner === owner) return; + link.tableCommitAdmissionOwner = owner; + if (link === owner) return; + if (owner.admittedTransactionLink === undefined) owner.admittedTransactionLink = link; + else (owner.additionalAdmittedTransactionLinks ??= []).push(link); +} + +function admitTransactionStores(transaction: DatabaseTransaction, owner: DatabaseTransaction): void { + for (let link: DatabaseTransaction = transaction; link; link = link.next) { + stampTransactionLink(owner, link); + for (const write of link.writes) { + const store = write?.store; + if (!store || hasAdmittedStore(owner, store)) continue; + const admit = store[TABLE_COMMIT_ADMISSION]; + if (admit && admit() !== false) addAdmittedStore(owner, store); + } + } +} + +function releaseTransactionStores(owner: DatabaseTransaction): Promise | void { + let completions: Promise[] | undefined; + const releaseStore = (store: any) => { + try { + const completion = store[TABLE_COMMIT_RELEASE]?.(); + if ((completion as any)?.then) (completions ??= []).push(Promise.resolve(completion)); + } catch (error) { + (completions ??= []).push(Promise.reject(error)); + } + }; + if (owner.admittedTableStore !== undefined) releaseStore(owner.admittedTableStore); + for (const store of owner.additionalAdmittedTableStores ?? []) releaseStore(store); + const clearAdmissionOwner = (link: DatabaseTransaction) => { + if (link.tableCommitAdmissionOwner === owner) link.tableCommitAdmissionOwner = undefined; + }; + clearAdmissionOwner(owner); + if (owner.admittedTransactionLink) clearAdmissionOwner(owner.admittedTransactionLink); + for (const link of owner.additionalAdmittedTransactionLinks ?? []) clearAdmissionOwner(link); + owner.admittedTableStore = undefined; + owner.additionalAdmittedTableStores = undefined; + owner.admittedTransactionLink = undefined; + owner.additionalAdmittedTransactionLinks = undefined; + if (completions) return Promise.allSettled(completions).then(() => undefined); +} + +/** + * Hold every table touched by a transaction chain in the submitted-commit set until the complete + * commit (including retries and chained stores) settles. Recursive commit rounds reuse the same + * context and admit any stores added while asynchronous pre-commit work was running. + */ +export function withTableCommitAdmission( + transaction: DatabaseTransaction, + options: CommitOptions, + commit: (options: CommitOptions) => T +): T { + const activeOwner = transaction.tableCommitAdmissionOwner; + if (activeOwner) { + // Retry and recursive commit rounds remain owned by the original head until its promise settles. + admitTransactionStores(transaction, activeOwner); + return commit(options); + } + transaction.tableCommitAdmissionOwner = transaction; + try { + admitTransactionStores(transaction, transaction); + const resolution: any = commit(options); + if (resolution?.then) { + return resolution.then( + (value) => { + const release = releaseTransactionStores(transaction); + return release ? release.then(() => value) : value; + }, + (error) => { + const release = releaseTransactionStores(transaction); + if (release) + return release.then(() => { + throw error; + }); + throw error; + } + ) as T; + } + const release = releaseTransactionStores(transaction); + return (release ? release.then(() => resolution) : resolution) as T; + } catch (error) { + const release = releaseTransactionStores(transaction); + if (release) + return release.then(() => { + throw error; + }) as T; + throw error; + } +} + type ReadTransaction = (LMDBTransaction | RocksTransaction) & { openTimer?: number; retryRisk?: number; @@ -346,6 +451,11 @@ export class DatabaseTransaction implements Transaction { timestamp = 0; retries = 0; declare next: DatabaseTransaction; + declare tableCommitAdmissionOwner?: DatabaseTransaction; + declare admittedTableStore?: any; + declare additionalAdmittedTableStores?: any[]; + declare admittedTransactionLink?: DatabaseTransaction; + declare additionalAdmittedTransactionLinks?: DatabaseTransaction[]; // The head of this multi-store chain, set when the link is created; absent on the head itself. declare root?: DatabaseTransaction; // Whether this link is why its chain root is write-supervised (see endWriteSupervision). @@ -431,11 +541,6 @@ export class DatabaseTransaction implements Transaction { this.baseReadRefConsumed = false; } - /** - * Drop this link's supervision claim, and the root's with it once no link in the chain still holds - * one. Membership is keyed on the root but claimed per link, so removing it on any link's detach - * would unsupervise a logical transaction still holding writes elsewhere in the chain. - */ /** * Give up on the whole chain: release any handle its links still hold, then drop the supervision * that was the only remaining way to find them. Clearing the bookkeeping alone would strand a live @@ -452,6 +557,11 @@ export class DatabaseTransaction implements Transaction { supervisedWriteRoots.delete(root); } + /** + * Drop this link's supervision claim, and the root's with it once no link in the chain still holds + * one. Membership is keyed on the root but claimed per link, so removing it on any link's detach + * would unsupervise a logical transaction still holding writes elsewhere in the chain. + */ private endWriteSupervision(): void { if (!this.writeSupervised) return; this.writeSupervised = false; @@ -517,6 +627,12 @@ export class DatabaseTransaction implements Transaction { this.completeDeferredContextRelease(); } + /** Release wrapper bookkeeping after its native transaction was committed directly. */ + detachReadTxn(): void { + this.detachOwnedTransaction(); + this.completeDeferredContextRelease(); + } + /** * Complete a context release that releaseContext() deferred because outstanding read iterators * were still using this transaction (see releaseContext()) — called once the last one drains, @@ -785,6 +901,10 @@ export class DatabaseTransaction implements Transaction { * Resolves with information on the timestamp and success of the commit */ commit(options: CommitOptions = {}): MaybePromise { + return withTableCommitAdmission(this, options, (admittedOptions) => this.commitRocksAdmitted(admittedOptions)); + } + + private commitRocksAdmitted(options: CommitOptions): MaybePromise { if (this.timedOut) throw transactionOpenTooLongError(); // reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain — // but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction @@ -1472,6 +1592,10 @@ export function resetReplayedWritesWarning() { replayedWritesWarned = false; } +export function trackedTransactionCountForTests(): number { + return trackedTxns.size; +} + /** Test seam: whether the monitor supervises this logical transaction for its writes. */ export function isWriteSupervised(txn: DatabaseTransaction): boolean { return supervisedWriteRoots.has(txn); diff --git a/resources/LMDBTransaction.ts b/resources/LMDBTransaction.ts index a290445415..a6f73a7b8e 100644 --- a/resources/LMDBTransaction.ts +++ b/resources/LMDBTransaction.ts @@ -1,5 +1,6 @@ import { DatabaseTransaction, + withTableCommitAdmission, transactionOpenTooLongError, type CommitOptions, type TransactionWrite, @@ -118,6 +119,10 @@ export class LMDBTransaction extends DatabaseTransaction { * Resolves with information on the timestamp and success of the commit */ commit(options: CommitOptions = {}): any { + return withTableCommitAdmission(this, options, (admittedOptions) => this.commitLmdbAdmitted(admittedOptions)); + } + + private commitLmdbAdmitted(options: CommitOptions): any { if (this.timedOut) throw transactionOpenTooLongError(); options = options || {}; let txnTime = this.timestamp; diff --git a/resources/RecordEncoder.ts b/resources/RecordEncoder.ts index a229f83cb9..f6113d8af0 100644 --- a/resources/RecordEncoder.ts +++ b/resources/RecordEncoder.ts @@ -16,6 +16,7 @@ import { ACTION_32_BIT, HAS_ADDITIONAL_AUDIT_REFS as HAS_ADDITIONAL_AUDIT_REFS_AUDIT, LOCAL_ONLY, + HAS_EXPIRATION_DECISION, } from './auditStore.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; @@ -728,6 +729,10 @@ export function recordUpdater(store, tableId, auditStore) { : TIMESTAMP_ASSIGN_NEW | 0x4000 // or just assign a new one : NO_TIMESTAMP; const expiresAt = options?.expiresAt; + if (options?.expirationDecisionPresent) { + if (assignMetadata < 0) assignMetadata = 0; + assignMetadata |= HAS_EXPIRATION_DECISION; + } if (expiresAt >= 0) assignMetadata |= HAS_EXPIRATION; metadataInNextEncoding = assignMetadata; expiresAtNextEncoding = expiresAt; @@ -777,6 +782,7 @@ export function recordUpdater(store, tableId, auditStore) { if (!previousResidencyId) previousResidencyId = 0; } if (assignMetadata & HAS_EXPIRATION) extendedType |= HAS_EXPIRATION_EXTENDED_TYPE; // we need to record the expiration in the audit log + if (assignMetadata & HAS_EXPIRATION_DECISION) extendedType |= HAS_EXPIRATION_DECISION; if (options?.originatingOperation) extendedType |= HAS_ORIGINATING_OPERATION; // we use resolveRecord outside of transaction, so must explicitly make it conditional if (resolveRecord) putOptions.ifVersion = ifVersion = existingEntry?.version ?? null; diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index c223f84090..4da0b120d8 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -6,7 +6,14 @@ import { RocksDatabase, } from '@harperfast/rocksdb-js'; import { Id } from './ResourceInterface.ts'; -import { MAXIMUM_KEY } from 'ordered-binary'; +import { compareKeys, MAXIMUM_KEY } from 'ordered-binary'; +import { ExtendedIterable } from '@harperfast/extended-iterable'; + +function cloneKey(key: any): any { + if (Array.isArray(key)) return key.map(cloneKey); + if (key instanceof Uint8Array) return key.slice(); + return key; +} declare module '@harperfast/rocksdb-js' { interface DBI { @@ -34,12 +41,94 @@ export class RocksIndexStore extends RocksDatabase { if ((reverse ? !inclusiveEnd : inclusiveEnd) && end !== undefined) { end = [end, MAXIMUM_KEY]; } - const translatedOptions = { ...options, start, end }; + const translatedOptions: StoreIteratorOptions & { offset?: number } = { ...options, start, end }; + if (options.values === false) { + const { limit, offset = 0, ...keyOptions } = translatedOptions; + const getKeys = () => super.getKeys(keyOptions); + return new ExtendedIterable({ + *[Symbol.iterator]() { + if (limit !== undefined && limit <= 0) return; + let first = true; + let previous: any; + let skipped = 0; + let yielded = 0; + for (const key of getKeys()) { + const indexedValue = key[0]; + if (!first && compareKeys(previous, indexedValue) === 0) continue; + first = false; + previous = cloneKey(indexedValue); + if (skipped < Math.max(0, offset)) { + skipped++; + continue; + } + yield cloneKey(indexedValue); + yielded++; + if (limit !== undefined && yielded >= limit) return; + } + }, + }); + } return super.getRange(translatedOptions).map(({ key }) => { return { key: key[0], value: key.length > 2 ? key.slice(1) : key[1] }; }); } + getCompositeRange(options: { after?: any[]; end: any; limit: number }): Iterable<{ + key: any; + value: Id; + cursor: any[]; + }> { + const keyOptions: StoreIteratorOptions & { snapshot?: boolean } = { + start: options.after ?? true, + end: [options.end, MAXIMUM_KEY], + limit: options.limit, + snapshot: false, + ...(options.after === undefined ? {} : { exclusiveStart: true }), + }; + return super.getKeys(keyOptions).map((key) => { + const cursor = cloneKey(key) as any[]; + return { + key: cursor[0], + value: cursor.length > 2 ? cursor.slice(1) : cursor[1], + cursor, + }; + }); + } + + getValues(indexedValue: any, options: Omit = {}): Iterable { + const { + limit, + offset = 0, + ...keyOptions + } = options as Omit & { + offset?: number; + }; + const bounds = keyOptions.reverse + ? { start: [indexedValue, MAXIMUM_KEY], end: [indexedValue], inclusiveEnd: true } + : { start: [indexedValue], end: [indexedValue, MAXIMUM_KEY] }; + const primaryKey = (key: any[]) => (key.length > 2 ? key.slice(1).map(cloneKey) : cloneKey(key[1])); + if (Math.max(0, offset) === 0 && (limit === undefined || limit > 0)) { + return super.getKeys({ ...keyOptions, ...bounds, limit }).map(primaryKey); + } + const getKeys = () => super.getKeys({ ...keyOptions, ...bounds }); + return new ExtendedIterable({ + *[Symbol.iterator]() { + if (limit !== undefined && limit <= 0) return; + let skipped = 0; + let yielded = 0; + for (const key of getKeys()) { + if (skipped < Math.max(0, offset)) { + skipped++; + continue; + } + yield primaryKey(key); + yielded++; + if (limit !== undefined && yielded >= limit) return; + } + }, + }); + } + /** * Translate a put with indexed value and primary key to an underlying put * @param indexedValue - ignored, only used by LMDB @@ -72,7 +161,7 @@ export class RocksIndexStore extends RocksDatabase { */ DBI.prototype.getValuesCount = function getValuesCount(indexedValue: any) { if (this instanceof RocksIndexStore) { - return this.store.getCount(this._context, { start: indexedValue, end: [indexedValue, MAXIMUM_KEY] }); + return this.store.getCount(this._context, { start: [indexedValue], end: [indexedValue, MAXIMUM_KEY] }); } throw new Error('getValuesCount is only supported if dupSort=true'); }; diff --git a/resources/Table.ts b/resources/Table.ts index dc4a2836e4..fea6168f19 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -11,7 +11,7 @@ import { SYSTEM_SCHEMA_NAME, MAX_SET_TIMEOUT_MS, } from '../utility/hdbTerms.ts'; -import { type Database } from 'lmdb'; +import { asBinary, type Database } from 'lmdb'; import { Script } from 'node:vm'; import { randomUUID } from 'node:crypto'; import { getIndexedValues } from '../utility/lmdb/commonUtility.ts'; @@ -36,6 +36,8 @@ import { DatabaseTransaction, ImmediateTransaction, priorStagedWrite, + TABLE_COMMIT_ADMISSION, + TABLE_COMMIT_RELEASE, isReleasedTransaction, TRANSACTION_STATE, } from './DatabaseTransaction.ts'; @@ -67,7 +69,7 @@ import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericT import { transaction, contextStorage } from './transaction.ts'; import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary'; import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; -import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts'; +import { HAS_BLOBS, HAS_EXPIRATION_DECISION, auditRetention, removeAuditEntry } from './auditStore.ts'; import { buildEmbedBefore, createDefaultEmbedder, type EmbedAttribute, type Embedder } from './models/embedHook.ts'; import { autoCast, autoCastBooleanStrict } from '../utility/common_utils.ts'; import { recordUpdater, removeEntry, PENDING_LOCAL_TIME, RecordObject, type Entry, entryMap } from './RecordEncoder.ts'; @@ -75,7 +77,13 @@ import { recordAction, recordActionBinary } from './analytics/write.ts'; import { rebuildUpdateBefore } from './crdt.ts'; import { appendHeader } from '../server/serverHelpers/Headers.ts'; import fs from 'node:fs'; -import { Blob, deleteBlobsInObject, findBlobsInObject, startPreCommitBlobsForRecord } from './blob.ts'; +import { + Blob, + deleteBlobsInObject, + findBlobsInObject, + prepareBlobDeletion, + startPreCommitBlobsForRecord, +} from './blob.ts'; import { onStorageReclamation, getStorageSpaceStats } from '../server/storageReclamation.ts'; import { RequestTarget } from './RequestTarget.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; @@ -105,6 +113,7 @@ export type Attribute = { computedFromExpression?: any; embed?: { source: string; model: string }; version?: any; + expirationIndexVersion?: number; properties?: Array; elements?: Attribute; sealed?: boolean; @@ -128,6 +137,8 @@ const EVICTION_BATCH_SIZE = 100; // Cap on eviction-batch commits in flight at once, so commit I/O overlaps scan/staging without // letting an unbounded number of open transactions (and their snapshots) accumulate. const MAX_INFLIGHT_EVICTION_BATCHES = 4; +const MAX_CLEANUP_CONCURRENCY = 50; +const MIN_SCHEDULED_CLEANUP_INTERVAL = 1000; const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]); envMngr.initSync(); const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); @@ -141,6 +152,34 @@ export function ignoreAlreadyDropped(error: any): void { if (error?.message?.includes('Column family already dropped')) return; throw error; } + +export function expirationTimestamp(value: any): number | undefined { + let timestamp = NaN; + if (typeof value === 'number' || typeof value === 'bigint') timestamp = Number(value); + else if (value instanceof Date) timestamp = value.getTime(); + else if (typeof value === 'string' && value.trim() !== '') { + const numeric = Number(value); + timestamp = Number.isFinite(numeric) ? numeric : Date.parse(value); + } + return Number.isFinite(timestamp) && timestamp >= 0 ? timestamp : undefined; +} + +function expirationOverrideTimestamp(value: any): number | undefined { + const timestamp = expirationTimestamp(value); + if (timestamp !== undefined) return timestamp; + if ((typeof value === 'number' || typeof value === 'bigint') && Number(value) < 0) return -1; +} + +export function effectiveExpirationTimestamp( + entry: Pick, 'expiresAt' | 'metadataFlags'> | undefined, + record: any, + expiresAtAttributeName: string | undefined +): number | undefined { + if (!expiresAtAttributeName) return entry?.expiresAt; + if (entry?.expiresAt !== undefined) return expirationTimestamp(entry.expiresAt); + if ((entry?.metadataFlags ?? 0) & HAS_EXPIRATION_DECISION) return; + return expirationTimestamp(record?.[expiresAtAttributeName]); +} // A frozen record we may need to copy-on-mutate before stamping it (records are immutable — decoded // records are frozen and 5.2 record caching relies on it). Only plain/record objects qualify: never // a Buffer/typed-array (spreading would corrupt the binary into a {0:.., 1:..} object) or a primitive @@ -403,6 +442,10 @@ export function makeTable(options) { // in-flight commit promises here so dropTable() can drain them first, and stop admitting // new ones (droppingTable) once a drop has actually started. const pendingSourceCommits = new Set>(); + let pendingTableCommitCount = 0; + let pendingTableCommitWaiter: Promise | undefined; + let resolvePendingTableCommits: (() => void) | undefined; + let unmatchedTableCommitReleaseReported = false; let droppingTable = false; let createdTimeProperty: Attribute | undefined, updatedTimeProperty: Attribute | undefined, @@ -413,6 +456,7 @@ export function makeTable(options) { if (attribute.expiresAt) expiresAtProperty = attribute; if (attribute.isPrimaryKey) primaryKeyAttribute = attribute; } + const expiresAtIndexName = expiresAtProperty?.name; let deleteCallbackHandle: { remove: () => void }; let prefetchIds = []; let prefetchCallbacks = []; @@ -421,7 +465,44 @@ export function makeTable(options) { let cleanupInterval = 86400000; let cleanupPriority = 0; let lastCleanupInterval: number; - let cleanupTimer: NodeJS.Timeout; + let cleanupTimer: NodeJS.Timeout | undefined; + let cleanupNextScheduled: number | undefined; + let cleanupGeneration = 0; + let cleanupClosed = false; + let cleanupWasScheduled = false; + let dropQuiescing = false; + (primaryStore as any)[TABLE_COMMIT_ADMISSION] = (skipIfQuiescing = false) => { + if (dropQuiescing || droppingTable) { + if (skipIfQuiescing) return false; + assertTableWritable(); + } + pendingTableCommitCount++; + return true; + }; + (primaryStore as any)[TABLE_COMMIT_RELEASE] = () => { + if (pendingTableCommitCount === 0) { + // Admission is synchronous and always precedes release, so this can only be duplicate + // accounting. Stay at the safe floor and report the first caller with a stack. + if (!unmatchedTableCommitReleaseReported) { + unmatchedTableCommitReleaseReported = true; + harperLogger.error?.(`Ignored unmatched table commit release for ${tableName}`, new Error()); + } + return; + } + if (--pendingTableCommitCount === 0 && resolvePendingTableCommits) { + const resolve = resolvePendingTableCommits; + pendingTableCommitWaiter = undefined; + resolvePendingTableCommits = undefined; + resolve(); + } + }; + function waitForPendingTableCommits(): Promise | undefined { + if (pendingTableCommitCount === 0) return; + return (pendingTableCommitWaiter ??= new Promise((resolve) => (resolvePendingTableCommits = resolve))); + } + let recordExpirationInterval: NodeJS.Timeout | undefined; + let missingExpirationIndexReported = false; + let missingSourceExpirationReported = false; // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup let expirationScanScheduled = false; // set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table @@ -442,6 +523,7 @@ export function makeTable(options) { // it, so such a table takes the guarded serialization path rather than the raw fast path. let hasSurfacedComputed = false; let runningRecordExpiration: boolean; + let recordExpirationCompletion: Promise = Promise.resolve(); const isRocksDB = primaryStore instanceof RocksDatabase; type BigInt64ArrayAndMaxSafeId = BigInt64Array & { maxSafeId: number }; let idIncrementer: BigInt64ArrayAndMaxSafeId; @@ -570,6 +652,7 @@ export function makeTable(options) { static databasePath = databasePath; static databaseName = databaseName; static attributes = attributes; + static expiresAtAttributeName = expiresAtIndexName; static description = description; static properties = properties; static hidden = hidden; @@ -674,7 +757,8 @@ export function makeTable(options) { nodeId: event.nodeId, viaNodeId: event.viaNodeId, // use per-event expiresAt: batched txn context only holds the first event's expiration - expiresAt: event.expiresAt, + expiresAt: expirationOverrideTimestamp(event.expiresAt), + expirationDecisionPresent: event.expirationDecisionPresent, // bulk base-copy snapshot frame: apply current-state directly, without an audit/transaction-log // entry or out-of-order resequencing (harper-pro#480). Only set for copy frames (between // COPY_START and COPY_COMPLETE); post-copy audit-replay frames apply normally. @@ -1360,162 +1444,149 @@ export function makeTable(options) { } static async dropTable() { - if (databaseName === databasePath) { - // Persist a drop tombstone on the primary catalog entry BEFORE any - // destructive work. If the process dies or a column family drop fails - // partway through, the tombstone survives with the catalog rows, and - // the next startup (or a same-name create) completes the drop via - // completeInterruptedDrop in databases.ts instead of resurrecting - // the table. - const primaryCatalogKey = TableResource.tableName + '/'; - const primaryMeta = (dbisDb as any).getSync(primaryCatalogKey); - if (primaryMeta && !primaryMeta.dropping) { - primaryMeta.dropping = true; - // Stamps this drop's identity so the interrupted-drop retry budget in - // databases.ts can be scoped to THIS drop rather than the table name: a - // worker that exhausts the budget for a table can observe the catalog - // mid-flight between this drop's completion and a same-name recreate's - // own drop, without ever seeing a non-tombstoned row to reset on. Keying - // the budget by generation instead makes the new drop's tombstone carry - // its own fresh key regardless of what any worker last observed. - primaryMeta.dropGeneration = randomUUID(); - // put is rebound to putSync on RocksDB stores; on LMDB it returns - // a promise, so await it to make the tombstone durable before the - // destructive work below - const tombstoneWrite = (dbisDb as any).put(primaryCatalogKey, primaryMeta); - if (tombstoneWrite?.then) await tombstoneWrite; - } - } - // A get() against a sourcedFrom table resolves to its caller before the resolved - // record's cache write has committed (see getFromSource) - the write lands "in the - // background" for latency reasons. Flip this BEFORE removing the table from the - // schema below: getFromSource() checks it and skips caching (treats the load as - // noCacheStore) for any call it admits from here on, including one that slipped in - // through a stale reference to this Table between the two steps. - droppingTable = true; - // Remove the table from the in-memory schema immediately so concurrent - // requests get "table does not exist" instead of racing the column - // family drops below. If a drop fails past this point the table stays - // invisible, and the tombstone guarantees the drop completes on the - // next startup (or on a same-name create). - delete databases[databaseName][tableName]; - // The above stops new source-fill writes from starting, but a write from a get() - // that already returned to its caller may still be in flight. Dropping the column - // families out from under that write is a genuine invariant violation, not just a - // benign race: RocksDB rejects the still-open write batch with "Invalid column - // family specified in write batch" (or "Could not access column family N"), which - // can also abort this drop before it removes the tombstoned catalog rows - leaving - // the table stuck "dropping" for completeInterruptedDrop to retry (and fail - // identically) on every subsequent load (harper#1381). Drain any in-flight commits - // before the blob sweep below (so it observes every row a drain-caught write just - // committed) and before touching a single column family. - // - // Bounded, and fails CLOSED: the tracked promise covers the whole source round-trip - // plus the local commit (see getFromSource), so a hung/slow source or a slow commit - // (e.g. a large blob write) could otherwise wedge this drop forever. Rather than - // give up and drop anyway - which would reopen exactly the race this drain exists to - // close, just less often - a timeout FAILS the drop. The tombstone written above is - // already durable, so completeInterruptedDrop picks the drop back up on the next - // load, once the stuck write has had time to finish. - if (pendingSourceCommits.size) { - const pending = [...pendingSourceCommits]; - let timer: NodeJS.Timeout; - const timedOut = Symbol('timedOut'); - const result = await Promise.race([ - Promise.allSettled(pending), - new Promise((resolve) => { - timer = setTimeout(() => resolve(timedOut), LOCK_TIMEOUT); - }), - ]); - clearTimeout(timer); - if (result === timedOut) { - throw new Error( - `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.length} in-flight source-populated cache write(s) on ${tableName} to settle; refusing to drop the column families out from under a write that may still be staged. The drop tombstone is durable, so this will be retried on the next load.` - ); - } + let quiesceMessage: any; + try { + quiesceMessage = await signalling.quiesceSchemaChange( + new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName) + ); + } catch (error) { + throw error; } - for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) { - if (entry.metadataFlags & HAS_BLOBS && entry.value) { - deleteBlobsInObject(entry.value); + let destructiveStarted = false; + let destructiveSucceeded = false; + let commitAttempted = false; + try { + commitAttempted = true; + await signalling.commitSchemaChange(quiesceMessage); + if (databaseName === databasePath) { + // Persist a drop tombstone on the primary catalog entry BEFORE any + // destructive work. If the process dies or a column family drop fails + // partway through, the tombstone survives with the catalog rows, and + // the next startup (or a same-name create) completes the drop via + // completeInterruptedDrop in databases.ts instead of resurrecting + // the table. + const primaryCatalogKey = TableResource.tableName + '/'; + const primaryMeta = (dbisDb as any).getSync(primaryCatalogKey); + if (primaryMeta && !primaryMeta.dropping) { + primaryMeta.dropping = true; + // Stamps this drop's identity so the interrupted-drop retry budget in + // databases.ts can be scoped to THIS drop rather than the table name: a + // worker that exhausts the budget for a table can observe the catalog + // mid-flight between this drop's completion and a same-name recreate's + // own drop, without ever seeing a non-tombstoned row to reset on. Keying + // the budget by generation instead makes the new drop's tombstone carry + // its own fresh key regardless of what any worker last observed. + primaryMeta.dropGeneration = randomUUID(); + // put is rebound to putSync on RocksDB stores; on LMDB it returns + // a promise, so await it to make the tombstone durable before the + // destructive work below + const tombstoneWrite = (dbisDb as any).put(primaryCatalogKey, primaryMeta); + if (tombstoneWrite?.then) await tombstoneWrite; + } } - } - if (databaseName === databasePath) { - // part of a database. - // Drop the column families, then remove the catalog metadata - never - // the reverse: a removed-then-failed drop orphans a "ghost" column - // family that poisons same-name recreates, so a genuine drop failure - // must surface and leave the tombstoned catalog rows for the reconcile. - // - // A drop is broadcast to every worker thread, and each holds its own - // handle to the same underlying column family, so a concurrent worker - // (or completeInterruptedDrop) may already have dropped it - surfaced - // as "Column family already dropped!". That is the intended end state, - // not a failure, so tolerate it. The catalog rows are removed only if - // this drop's tombstone is still the live primary row: a concurrent - // same-name create completes the interrupted drop and writes fresh - // catalog rows, and clobbering those would orphan the new table. - const removeTombstonedCatalog = () => { - const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); - if (!currentPrimary?.dropping) return false; - for (const attribute of attributes) { - dbisDb.remove(TableResource.tableName + '/' + attribute.name); + destructiveStarted = true; + droppingTable = true; + // Remove the table from the in-memory schema immediately so concurrent + // requests get "table does not exist" instead of racing the column + // family drops below. If a drop fails past this point the table stays + // invisible, and the tombstone guarantees the drop completes on the + // next startup (or on a same-name create). + delete databases[databaseName][tableName]; + for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) { + if (entry.metadataFlags & HAS_BLOBS && entry.value) { + deleteBlobsInObject(entry.value); } - dbisDb.remove(TableResource.tableName + '/'); - return true; - }; - const rootStore = primaryStore.rootStore; - if (rootStore instanceof RocksDatabase) { - // Serialize the drops + catalog removal against a concurrent - // same-name create (and completeInterruptedDrop) under the database's - // 'update-attributes' exclusive lock - the same lock the create path - // holds. It is a synchronous spin lock that blocks the event loop, so - // the locked section MUST stay synchronous: drop with dropSync (as - // completeInterruptedDrop does), never an awaited drop(), or a - // concurrent create's spin would deadlock waiting on a drop that the - // blocked event loop can never resolve. - while (!rootStore.tryLock('update-attributes')) {} - let removed = false; - try { + } + if (databaseName === databasePath) { + // part of a database. + // Drop the column families, then remove the catalog metadata - never + // the reverse: a removed-then-failed drop orphans a "ghost" column + // family that poisons same-name recreates, so a genuine drop failure + // must surface and leave the tombstoned catalog rows for the reconcile. + // + // A drop is broadcast to every worker thread, and each holds its own + // handle to the same underlying column family, so a concurrent worker + // (or completeInterruptedDrop) may already have dropped it - surfaced + // as "Column family already dropped!". That is the intended end state, + // not a failure, so tolerate it. The catalog rows are removed only if + // this drop's tombstone is still the live primary row: a concurrent + // same-name create completes the interrupted drop and writes fresh + // catalog rows, and clobbering those would orphan the new table. + const removeTombstonedCatalog = () => { + const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); + if (!currentPrimary?.dropping) return false; for (const attribute of attributes) { - const index = indices[attribute.name]; - if (index) - try { - index.dropSync(); - } catch (error) { - ignoreAlreadyDropped(error); - } + dbisDb.remove(TableResource.tableName + '/' + attribute.name); } + dbisDb.remove(TableResource.tableName + '/'); + return true; + }; + const rootStore = primaryStore.rootStore; + if (rootStore instanceof RocksDatabase) { + // Serialize the drops + catalog removal against a concurrent + // same-name create (and completeInterruptedDrop) under the database's + // 'update-attributes' exclusive lock - the same lock the create path + // holds. It is a synchronous spin lock that blocks the event loop, so + // the locked section MUST stay synchronous: drop with dropSync (as + // completeInterruptedDrop does), never an awaited drop(), or a + // concurrent create's spin would deadlock waiting on a drop that the + // blocked event loop can never resolve. + while (!rootStore.tryLock('update-attributes')) {} + let removed = false; try { - primaryStore.dropSync(); - } catch (error) { - ignoreAlreadyDropped(error); + for (const attribute of attributes) { + const index = indices[attribute.name]; + if (index) + try { + index.dropSync(); + } catch (error) { + ignoreAlreadyDropped(error); + } + } + try { + primaryStore.dropSync(); + } catch (error) { + ignoreAlreadyDropped(error); + } + removed = removeTombstonedCatalog(); + } finally { + rootStore.unlock('update-attributes'); + } + if (removed) await dbisDb.committed; + } else { + // LMDB: no shared column-family double-drop, and its engine lock is + // transactional rather than this spin lock, so keep the awaited drop + // plus the same tombstone-guarded catalog removal. + const drops = []; + for (const attribute of attributes) { + const index = indices[attribute.name]; + if (index) drops.push(index.drop().catch(ignoreAlreadyDropped)); } - removed = removeTombstonedCatalog(); - } finally { - rootStore.unlock('update-attributes'); + drops.push(primaryStore.drop().catch(ignoreAlreadyDropped)); + await Promise.all(drops); + if (removeTombstonedCatalog()) await dbisDb.committed; } - if (removed) await dbisDb.committed; } else { - // LMDB: no shared column-family double-drop, and its engine lock is - // transactional rather than this spin lock, so keep the awaited drop - // plus the same tombstone-guarded catalog removal. - const drops = []; - for (const attribute of attributes) { - const index = indices[attribute.name]; - if (index) drops.push(index.drop().catch(ignoreAlreadyDropped)); + // legacy table per database + await primaryStore.close(); + fs.unlinkSync(primaryStore.path); + } + destructiveSucceeded = true; + } finally { + if (!destructiveStarted) { + if (!commitAttempted) await signalling.abortSchemaQuiesce(quiesceMessage); + else { + const message: any = new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName); + message.quiesceId = quiesceMessage.quiesceId; + await signalling.reconcileSchemaChange(message); } - drops.push(primaryStore.drop().catch(ignoreAlreadyDropped)); - await Promise.all(drops); - if (removeTombstonedCatalog()) await dbisDb.committed; + } else { + const message: any = new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName); + message.quiesceId = quiesceMessage.quiesceId; + if (destructiveSucceeded) await signalling.finalizeSchemaChange(message); + else await signalling.reconcileSchemaChange(message); } - } else { - // legacy table per database - await primaryStore.close(); - fs.unlinkSync(primaryStore.path); } - signalling.signalSchemaChange( - new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName) - ); } // #section: read-path /** @@ -1901,11 +1972,14 @@ export function makeTable(options) { invalidated: true, entry: this.#entry, commit: (txnTime, existingEntry, _retry, transaction: any) => { + assertTableWritable(); + const optionExpiresAt = expirationOverrideTimestamp(options?.expiresAt); write.skipped = false; // reset on each retry; cleanup happens after commit if still true if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) { write.skipped = true; return; } + if (isFrozenRecordObject(partialRecord)) partialRecord = { ...partialRecord }; partialRecord ??= null; for (const name in indices) { if (!partialRecord) partialRecord = {}; @@ -1925,6 +1999,13 @@ export function makeTable(options) { { user: (context as any)?.user, residencyId: options?.residencyId, + expiresAt: + options?.expirationDecisionPresent === true + ? (optionExpiresAt ?? -1) + : (optionExpiresAt ?? existingEntry?.expiresAt), + expirationDecisionPresent: + options?.expirationDecisionPresent === true || + Boolean(existingEntry?.metadataFlags & HAS_EXPIRATION_DECISION), nodeId: options?.nodeId, viaNodeId: options?.viaNodeId, transaction, @@ -1952,6 +2033,7 @@ export function makeTable(options) { ? (this.constructor as any).source.relocate.bind((this.constructor as any).source, id, undefined, context) : undefined, commit: (txnTime, existingEntry, _retry, transaction: any) => { + assertTableWritable(); if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) <= 0) return; const residency = TableResource.getResidencyRecord(options.residencyId); let metadata = 0; @@ -1982,7 +2064,11 @@ export function makeTable(options) { residencyId: options.residencyId, nodeId: options.nodeId, viaNodeId: options?.viaNodeId, - expiresAt: options.expiresAt, + expiresAt: + options.expirationDecisionPresent === true + ? (expirationOverrideTimestamp(options.expiresAt) ?? -1) + : expirationOverrideTimestamp(options.expiresAt), + expirationDecisionPresent: options.expirationDecisionPresent, transaction, }, 'relocate', @@ -2019,7 +2105,16 @@ export function makeTable(options) { existingEntry.version, // version number should not change metadata, true, - { residencyId, expiresAt: entry.expiresAt, transaction: txnForContext(context).transaction }, + { + residencyId, + expiresAt: + entry.expirationDecisionPresent === true || Boolean(entry.metadataFlags & HAS_EXPIRATION_DECISION) + ? (expirationOverrideTimestamp(entry.expiresAt) ?? -1) + : expirationOverrideTimestamp(entry.expiresAt), + expirationDecisionPresent: + entry.expirationDecisionPresent === true || Boolean(entry.metadataFlags & HAS_EXPIRATION_DECISION), + transaction: txnForContext(context).transaction, + }, 'relocate', false, null // the audit record value should be empty since there are no changes to the actual data @@ -2029,41 +2124,97 @@ export function makeTable(options) { /** * Evicting a record will remove it from a caching table. This is not considered a canonical data change, and it is assumed that retrieving this record from the source will still yield the same record, this is only removing the local copy of the record. */ - static evict(id, existingRecord, existingVersion) { + static evict(id, existingRecord, existingVersion, encounteredExpirationIndexValue?: number) { + let admitted: unknown; + try { + admitted = (primaryStore as any)[TABLE_COMMIT_ADMISSION]?.(true); + } catch (error) { + logger.warn?.('Error admitting record eviction', id, error); + return Promise.resolve(); + } + if (admitted === false) return Promise.resolve(); let entry; - const lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction() }); - let transaction = lmdbTransaction.getReadTxn(); - let options = { transaction }; - let committed = false; + let deleteBlobsAfterCommit: (() => void) | undefined; + let currentEntry; + let lmdbTransaction: any; + let transaction: any; + let options: any; + let completionOwnsAdmission = false; + let lmdbCompletion: MaybePromise; + const releaseAdmission = () => (primaryStore as any)[TABLE_COMMIT_RELEASE]?.(); + const abortEviction = () => { + try { + if (primaryStore.ifVersion) lmdbTransaction?.abort?.(); + else lmdbTransaction?.releaseReadTxn?.(); + } catch {} + }; try { + lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction() }); + transaction = lmdbTransaction.getReadTxn(); + options = { transaction }; + // A raw Rocks read transaction can reuse an older snapshot. Reject an already-refreshed + // record before consulting it; commit-time conflict detection guards later races. + if (!primaryStore.ifVersion) { + const lazyEntry = primaryStore.getEntry(id, { lazy: true }); + if (!lazyEntry || lazyEntry.version !== existingVersion) return Promise.resolve(); + // A lazy Rocks entry can reuse its decode buffer during the index updates below. Snapshot + // every field eviction still needs before another store operation can recycle that buffer. + // Value is deliberately omitted: blobs use existingRecord; non-blob removal only needs key. + currentEntry = { + key: id, + version: lazyEntry.version, + expiresAt: lazyEntry.expiresAt, + metadataFlags: lazyEntry.metadataFlags, + }; + } if (hasSourceGet || audit) { - if (!existingRecord) return; + if (!existingRecord) return Promise.resolve(); entry = primaryStore.getEntry(id, options); - if (!entry || !existingRecord) return; - if (entry.version !== existingVersion) return; + if (!entry || !existingRecord) return Promise.resolve(); + if (entry.version !== existingVersion) return Promise.resolve(); } if (hasSourceGet) { // if there is a resolution in-progress, abandon the eviction - if (primaryStore.hasLock(id, entry.version)) return; + if (primaryStore.hasLock(id, entry.version)) return Promise.resolve(); } // evictions never go in the audit log, so we can not record a deletion entry for the eviction // as there is no corresponding audit entry and it would never get cleaned up. So we must simply // removed the entry entirely, but first cleanup indices - let lmdbCompletion: MaybePromise; if (primaryStore.ifVersion) { - // lmdb: the index cleanup and the record removal are both version-guarded optimistic writes. - // Capture both promises so a real write failure on either resolves through evict()'s catch - // below rather than escaping as an unhandled rejection from the fire-and-forget callers. - const indexCleanup = primaryStore.ifVersion(id, existingVersion, () => { - updateIndices(id, existingRecord, null); + // LMDB batches every write in this callback behind one version check, so the indices and + // primary record can not diverge on a conflict or partial commit. + const removalEntry = entry ?? primaryStore.getEntry(id); + let removal: MaybePromise; + let encounteredIndexRemoval: MaybePromise; + lmdbCompletion = primaryStore.ifVersion(id, existingVersion, () => { + const currentExpiration = effectiveExpirationIndexValue(removalEntry, existingRecord); + updateIndices(id, existingRecord, null, undefined, undefined, currentExpiration); + if ( + encounteredExpirationIndexValue !== undefined && + compareKeys(encounteredExpirationIndexValue, currentExpiration) !== 0 + ) + encounteredIndexRemoval = indices[expiresAtIndexName]?.remove(encounteredExpirationIndexValue, id); + removal = removeEntry(primaryStore, removalEntry); }); - const removal = removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion); - lmdbCompletion = Promise.all([indexCleanup, removal]); + lmdbCompletion = Promise.all([lmdbCompletion, removal, encounteredIndexRemoval]); } else { - updateIndices(id, existingRecord, null, options); - removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options); + const removalEntry = entry ?? currentEntry ?? primaryStore.getEntry(id); + if (!removalEntry || removalEntry.version !== existingVersion) return Promise.resolve(); + const currentExpiration = effectiveExpirationIndexValue(removalEntry, existingRecord); + updateIndices(id, existingRecord, null, options, undefined, currentExpiration); + if ( + encounteredExpirationIndexValue !== undefined && + compareKeys(encounteredExpirationIndexValue, currentExpiration) !== 0 + ) + indices[expiresAtIndexName]?.remove(encounteredExpirationIndexValue, id, options); + if (removalEntry.metadataFlags & HAS_BLOBS) { + if (!existingRecord) return Promise.resolve(); + primaryStore.remove(removalEntry.key, options); + deleteBlobsAfterCommit = prepareBlobDeletion(existingRecord); + } else { + removeEntry(primaryStore, removalEntry, options); + } } - committed = true; // Eviction is best-effort cleanup, run fire-and-forget from the record-expiration sweep and the // read path as well as the concurrency-limited cleanup scan. A concurrent write to the same record // makes the commit conflict — that is expected, not a failure: lazy-expiry-on-read keeps queries @@ -2076,32 +2227,52 @@ export function makeTable(options) { // a plain resolution object rather than a promise — return the store's write promises instead, so // the caller gets a real thenable that resolves once the removal is durable. (lmdbTransaction as any).commit(); - return Promise.resolve(lmdbCompletion).catch((error) => { - logger.warn?.('Error evicting record', id, error); - }); + completionOwnsAdmission = true; + return Promise.resolve(lmdbCompletion) + .catch((error) => logger.warn?.('Error evicting record', id, error)) + .finally(releaseAdmission); } // RocksDB: eviction writes went directly into the raw transaction via options; commit it directly, // as DatabaseTransaction.commit() would abort it (no tracked writes). The raw commit bypasses // DatabaseTransaction's ERR_BUSY retry, so a concurrent-write conflict rejects here — swallow it // (abandon the eviction) and log anything unexpected, rather than letting it crash the process. - return (transaction as any).commit().catch((error) => { - // The commit failed, so the read-snapshot/transaction handle is still open — release it, as the - // batched-eviction path does on its own commit failures. committed===true skips the finally abort. - try { - (transaction as any).abort(); - } catch {} - if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); - else logger.warn?.('Error evicting record', id, error); - }); + const commitCompletion = (transaction as any).commit(); + completionOwnsAdmission = true; + return Promise.resolve(commitCompletion) + .then( + () => { + try { + deleteBlobsAfterCommit?.(); + } catch (error) { + logger.warn?.('Error deleting blobs for evicted record', id, error); + } + }, + (error) => { + abortEviction(); + if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); + else logger.warn?.('Error evicting record', id, error); + } + ) + .finally(() => { + lmdbTransaction.detachReadTxn(); + releaseAdmission(); + }); + } catch (error) { + abortEviction(); + logger.warn?.('Error evicting record', id, error); + if (lmdbCompletion) { + completionOwnsAdmission = true; + return Promise.resolve(lmdbCompletion) + .catch((completionError) => logger.warn?.('Error evicting record', id, completionError)) + .finally(releaseAdmission); + } + return Promise.resolve(); } finally { - if (!committed) { + if (!completionOwnsAdmission) { // Skip path or thrown error: abort instead of committing so we don't apply // partial work and the txn handle is released. - if (primaryStore.ifVersion) { - (lmdbTransaction as any).abort?.(); - } else { - (transaction as any)?.abort?.(); - } + abortEviction(); + releaseAdmission(); } } } @@ -2363,6 +2534,7 @@ export function makeTable(options) { }, before: writeToSource(), commit: (txnTime: number, existingEntry: Entry, retry: boolean, transaction: any) => { + assertTableWritable(); // Whether a prior attempt of THIS write appended its own audit entry (sticky, set in // save(); log entries are not part of the aborted rocks transaction, so they survive). // Only such a write can find its own orphaned entry in the dedup lookups below and must @@ -2422,7 +2594,13 @@ export function makeTable(options) { // options/context expiresAt are the most specific overrides; a record @expiresAt field // (resolved below, once recordToStore is merged) overrides the table default in both // directions; the table default is the final fallback. -1 means no expiration. - let expiresAt: number | undefined = options?.expiresAt ?? context?.expiresAt; + let expiresAt: number | undefined = + options?.expirationDecisionPresent === true + ? expirationOverrideTimestamp(options?.expiresAt) + : (expirationOverrideTimestamp(options?.expiresAt) ?? expirationOverrideTimestamp(context?.expiresAt)); + let expirationDecisionPresent = Boolean( + expiresAtProperty && (options?.expirationDecisionPresent === true || expiresAt !== undefined) + ); const additionalAuditRefs: Array<{ version: number; nodeId: number }> = []; // track additional audit refs to store // Bulk base-copy snapshot apply: store current-state directly with no audit/transaction-log entry // and no out-of-order resequencing/dedup (the source of the O(n) keyed-lookup spin in @@ -2839,7 +3017,13 @@ export function makeTable(options) { } residencyId = getResidencyId(residency); } - if (expiresAt == undefined) { + expirationDecisionPresent = Boolean( + expiresAtProperty && + (options?.expirationDecisionPresent === true || !options?.isNotification || expiresAt !== undefined) + ); + if (expiresAt == undefined && options?.expirationDecisionPresent === true) { + expiresAt = -1; + } else if (expiresAt == undefined) { // A schema @expiresAt attribute makes the record field authoritative over the table // default, in both directions: stamp it into the stored expiry metadata that governs // read-hiding and the cleanup sweep, not just the separate index-pruning sweep (which @@ -2847,26 +3031,12 @@ export function makeTable(options) { // Read from recordToStore so the metadata matches exactly what the pruning sweep later // reads back. Falls back to the table default when the field is unset or not a timestamp. const fieldExpiresAt = expiresAtProperty ? recordToStore?.[expiresAtProperty.name] : undefined; - // Coerce only genuine timestamp shapes: a number/bigint epoch, a Date, or a numeric/ISO - // string. Booleans, empty/whitespace strings, and null/undefined fall through to NaN so a - // nonsensical field value uses the table default rather than expiring the record at epoch 0. - let fieldExpiresAtMs = NaN; - if (typeof fieldExpiresAt === 'number' || typeof fieldExpiresAt === 'bigint') - fieldExpiresAtMs = Number(fieldExpiresAt); - else if (fieldExpiresAt instanceof Date) fieldExpiresAtMs = fieldExpiresAt.getTime(); - else if (typeof fieldExpiresAt === 'string' && fieldExpiresAt.trim() !== '') { - const numeric = Number(fieldExpiresAt); - fieldExpiresAtMs = Number.isFinite(numeric) ? numeric : Date.parse(fieldExpiresAt); - } + const fieldExpiresAtMs = expirationTimestamp(fieldExpiresAt); // Only a finite, non-negative epoch counts: negatives collide with the -1 "no expiration" // sentinel (the encoder omits HAS_EXPIRATION for <0, but the field sweep would still evict a // negative field value), so treat a negative/NaN field as unset and use the table default. expiresAt = - Number.isFinite(fieldExpiresAtMs) && fieldExpiresAtMs >= 0 - ? fieldExpiresAtMs - : expirationMs - ? expirationMs + Date.now() - : -1; + fieldExpiresAtMs !== undefined ? fieldExpiresAtMs : expirationMs ? expirationMs + Date.now() : -1; } if (!fullUpdate) { // we use our own data as the basis for the audit record, which will include information about the incremental updates, even if it was overwritten by CRDT resolution @@ -2888,7 +3058,14 @@ export function makeTable(options) { } })() ); - updateIndices(id, existingRecord, recordToStore, transaction && { transaction }); + updateIndices( + id, + existingRecord, + recordToStore, + transaction && { transaction }, + expiresAt, + effectiveExpirationIndexValue(priorStaged ?? existingEntry, existingRecord) + ); writeCommit(true); if (expiresAt >= 0) { @@ -2925,6 +3102,7 @@ export function makeTable(options) { user: (context as any)?.user, residencyId, expiresAt, + expirationDecisionPresent, nodeId: options?.nodeId, viaNodeId: options?.viaNodeId, originatingOperation: (context as any)?.originatingOperation, @@ -2943,7 +3121,11 @@ export function makeTable(options) { // transaction (an audit-only commit stored no record, so it stages nothing and the // earlier staged record, if any, remains the basis) if (storeRecord) { - write.stagedEntry = { value: recordToStore }; + write.stagedEntry = { + value: recordToStore, + expiresAt, + metadataFlags: expirationDecisionPresent ? HAS_EXPIRATION_DECISION : 0, + }; // blobs this write saved are referenced by its audit entry (if it wrote one), which // then owns their lifetime; and any record an earlier write in this transaction // stored is now replaced, so mark those writes for the superseded-blob cleanup @@ -3050,6 +3232,7 @@ export function makeTable(options) { ? (this.constructor as any).source.delete.bind((this.constructor as any).source, id, undefined, context) : undefined, commit: (txnTime, existingEntry, retry, transaction: any) => { + assertTableWritable(); write.stagedEntry = undefined; // reset per round; set below once the removal is applied write.superseded = false; // reset per round, as in the update path // what a preceding write in this transaction left for this key is what gets removed @@ -3070,7 +3253,14 @@ export function makeTable(options) { if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) { return; } - updateIndices(id, existingRecord, null, transaction && { transaction }); + updateIndices( + id, + existingRecord, + null, + transaction && { transaction }, + undefined, + effectiveExpirationIndexValue(priorStaged ?? existingEntry, existingRecord) + ); if (audit || trackDeletes) { updateRecord( id, @@ -4037,6 +4227,7 @@ export function makeTable(options) { try { let type = auditRecord.type; let value; + let expirationState = auditRecord; if (type === 'message' || request.rawEvents) { // we only send the full message, this are individual messages that can be sent out of order // TODO: Do we want to have a limit to how far out-of-order we are willing to send? @@ -4057,6 +4248,7 @@ export function makeTable(options) { if (entry) { if (entry.version !== auditRecord.version) return; // out of order event, with old update, don't send anything value = entry.value; + expirationState = entry; type = entry.metadataFlags & INVALIDATED ? 'invalidate' : value ? 'put' : 'delete'; } else { type = 'delete'; @@ -4069,6 +4261,7 @@ export function makeTable(options) { version: auditRecord.version, type, beginTxn, + ...expirationEventProperties(expirationState), }; // Queued events are filtered when the queue drains through send() below; events sent // directly (queue already drained) are filtered here. Each event is filtered once. @@ -4138,6 +4331,7 @@ export function makeTable(options) { version: auditRecord.version, type: auditRecord.type, size: auditRecord.size, + ...expirationEventProperties(auditRecord), }) ) return; @@ -4185,6 +4379,7 @@ export function makeTable(options) { value, version: auditRecord.version, type: auditRecord.type, + ...expirationEventProperties(auditRecord), }; // Filter rows before they consume a previousCount slot. if (allowsEvent && !allowsEvent(historyEntry)) { @@ -4228,12 +4423,13 @@ export function makeTable(options) { // if a post-subscribe write hits a key the cursor also visits. This is // idempotent for "current state then live updates" — both deliveries land at // the same final state. We don't dedupe. - for (const { key: id, value, version, localTime, size } of primaryStore.getRange({ + for (const entry of primaryStore.getRange({ start: thisId ?? false, end: thisId == null ? undefined : [thisId, MAXIMUM_KEY], versions: true, snapshot: false, // no need for a snapshot, just want the latest data })) { + const { key: id, value, version, localTime, size } = entry; if (++recordsSinceYield >= REPLAY_YIELD_INTERVAL) { recordsSinceYield = 0; await rest(); @@ -4244,7 +4440,8 @@ export function makeTable(options) { const t = localTime ?? version; if (t > cursorMaxTime) cursorMaxTime = t; if (!value) continue; - if (!send({ id, localTime, value, version, type: 'put', size })) return; + if (!send({ id, localTime, value, version, type: 'put', size, ...expirationEventProperties(entry) })) + return; if (subscription.queue?.length > EVENT_HIGH_WATER_MARK) { // if we have too many messages, we need to pause and let the client catch up if ((await subscription.waitForDrain()) === false) return; @@ -4303,6 +4500,7 @@ export function makeTable(options) { value, localTime: nextTime, ...auditRecord, + ...expirationEventProperties(auditRecord), }; if (!allowsEvent || allowsEvent(historyEntry)) { request.omitCurrent = true; @@ -4325,6 +4523,7 @@ export function makeTable(options) { id: thisId, ...entry, type: 'put', + ...expirationEventProperties(entry), }) ) return; @@ -4369,23 +4568,24 @@ export function makeTable(options) { const isCollection = request.isCollection ?? thisId == null; if (isCollection) { let sinceYield = 0; - for (const { key: id, value, version, localTime, size } of primaryStore.getRange({ + for (const entry of primaryStore.getRange({ start: thisId ?? false, end: thisId == null ? undefined : [thisId, MAXIMUM_KEY], versions: true, snapshot: false, // no need for a snapshot, just want the latest data })) { + const { key: id, value, version, localTime, size } = entry; if (++sinceYield >= REPLAY_YIELD_INTERVAL) { sinceYield = 0; await rest(); if (!isActive()) return; } if (!value) continue; // skip tombstones - yield { id, localTime, value, version, type: 'put', size }; + yield { id, localTime, value, version, type: 'put', size, ...expirationEventProperties(entry) }; } } else { const entry = primaryStore.getEntry(thisId); - if (entry?.value) yield { id: thisId, ...entry, type: 'put' }; + if (entry?.value) yield { id: thisId, ...entry, type: 'put', ...expirationEventProperties(entry) }; } } // Drain the current scope into the subscription with the same back-pressure as the live path. @@ -4502,6 +4702,21 @@ export function makeTable(options) { ? (this.constructor as any).source.publish.bind((this.constructor as any).source, id, message, context) : undefined, commit: (txnTime, existingEntry, _retry, transaction: any) => { + assertTableWritable(); + const optionExpiresAt = expirationOverrideTimestamp(options?.expiresAt); + const contextExpiresAt = expirationOverrideTimestamp((context as any)?.expiresAt); + const hasExpirationOverride = + options?.expirationDecisionPresent === true || + optionExpiresAt !== undefined || + contextExpiresAt !== undefined; + const expiresAt = + options?.expirationDecisionPresent === true + ? (optionExpiresAt ?? -1) + : (optionExpiresAt ?? contextExpiresAt ?? existingEntry?.expiresAt); + const expirationDecisionPresent = + options?.expirationDecisionPresent === true || + Boolean(existingEntry?.metadataFlags & HAS_EXPIRATION_DECISION) || + Boolean(expiresAtProperty && hasExpirationOverride); // just need to update the version number of the record so it points to the latest audit record // but have to update the version number of the record // TODO: would be faster to use getBinaryFast here and not have the record loaded @@ -4509,6 +4724,16 @@ export function makeTable(options) { if (existingEntry === undefined && trackDeletes && !audit) { scheduleCleanup(); } + if (expiresAtProperty && hasExpirationOverride && existingEntry) { + updateIndices( + id, + existingEntry.value, + existingEntry.value, + { transaction }, + expiresAt, + effectiveExpirationIndexValue(existingEntry) + ); + } logger.trace?.(`Publishing message to id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`); // always audit this, but don't change existing version // TODO: Use direct writes in the future (copying binary data is hard because it invalidates the cache) @@ -4522,7 +4747,8 @@ export function makeTable(options) { { user: (context as any)?.user, residencyId: options?.residencyId, - expiresAt: context?.expiresAt, + expiresAt, + expirationDecisionPresent, nodeId: options?.nodeId, viaNodeId: options?.viaNodeId, transaction, @@ -5252,6 +5478,63 @@ export function makeTable(options) { } static cleanup() { deleteCallbackHandle?.remove(); + cleanupClosed = true; + cleanupGeneration++; + if (cleanupTimer) { + clearTimeout(cleanupTimer); + cleanupTimer = undefined; + cleanupNextScheduled = undefined; + } + if (recordExpirationInterval) { + clearInterval(recordExpirationInterval); + recordExpirationInterval = undefined; + } + return Promise.all([lastEvictionCompletion, recordExpirationCompletion]).then(() => undefined); + } + static async quiesceForDrop() { + dropQuiescing = true; + await this.cleanup(); + const tableCommitWaiter = waitForPendingTableCommits(); + await Promise.allSettled(tableCommitWaiter ? [...pendingSourceCommits, tableCommitWaiter] : pendingSourceCommits); + } + static abortDropQuiesce() { + if (!dropQuiescing) return; + dropQuiescing = false; + this.resumeCleanup(); + } + static isDropQuiescing() { + return dropQuiescing; + } + static resumeCleanup() { + if (!cleanupClosed) return; + cleanupClosed = false; + cleanupGeneration++; + if (cleanupWasScheduled) { + lastCleanupInterval = undefined; + scheduleCleanup(); + } + if (expiresAtProperty) runRecordExpirationEviction(); + if (audit) addDeleteRemoval(); + } + static runRecordExpirationSweepForTests(testHooks?: ExpirationSweepTestHooks) { + return runRecordExpirationEviction(testHooks, false, true) ?? Promise.resolve(); + } + static runPrimaryCleanupScanForTests(testHooks?: ExpirationSweepTestHooks) { + return queuePrimaryCleanup(testHooks); + } + static scheduleCleanupForTests(priority?: number) { + return scheduleCleanup(priority); + } + static cleanupNextScheduledForTests() { + return cleanupNextScheduled; + } + static cleanupStateForTests() { + return { + closed: cleanupClosed, + cleanupScheduled: !cleanupClosed && !!cleanupTimer, + nextCleanupScheduled: cleanupNextScheduled, + expirationScheduled: !cleanupClosed && !!recordExpirationInterval, + }; } static _readTxnForContext(context) { return txnForContext(context).getReadTxn(); @@ -5277,7 +5560,14 @@ export function makeTable(options) { if (expirationMs) TableResource.setTTLExpiration(expirationMs / 1000); if (expiresAtProperty) runRecordExpirationEviction(); return TableResource; - function updateIndices(id: any, existingRecord: any, record: any, options?: any) { + function updateIndices( + id: any, + existingRecord: any, + record: any, + options?: any, + expiresAtIndexValue?: number, + existingExpiresAtIndexValue?: number + ) { let hasChanges; // iterate the entries from the record // for-in is about 5x as fast as for-of Object.entries, and this is extremely time sensitive since it can be @@ -5294,8 +5584,26 @@ export function makeTable(options) { // the real [value, id] entry was removed, orphaning it against the now-deleted record. A record // that is present but whose attribute is null is a different case (record is a truthy object) and // still indexes under null. See harper#1894 (F-149). - const value = record == null ? undefined : resolver ? resolver(record) : record[key]; - const existingValue = existingRecord && (resolver ? resolver(existingRecord) : existingRecord[key]); + const expirationIndex = key === expiresAtIndexName; + const value = + record == null + ? undefined + : expirationIndex + ? expiresAtIndexValue !== undefined && expiresAtIndexValue >= 0 + ? expiresAtIndexValue + : undefined + : resolver + ? resolver(record) + : record[key]; + const existingValue = + existingRecord && + (expirationIndex + ? existingExpiresAtIndexValue !== undefined && existingExpiresAtIndexValue >= 0 + ? existingExpiresAtIndexValue + : undefined + : resolver + ? resolver(existingRecord) + : existingRecord[key]); if (value === existingValue && !isIndexing) { continue; } @@ -5349,6 +5657,17 @@ export function makeTable(options) { } return hasChanges; } + function effectiveExpirationIndexValue(entry: Partial | undefined, record = entry?.value): number | undefined { + return effectiveExpirationTimestamp(entry, record, expiresAtIndexName); + } + function expirationEventProperties(entry: any) { + if (!expiresAtProperty || !entry) return {}; + const metadataFlags = entry.metadataFlags ?? entry.extendedType ?? 0; + if (metadataFlags & HAS_EXPIRATION_DECISION) { + return { expiresAt: entry.expiresAt, expirationDecisionPresent: true }; + } + return entry.expiresAt === undefined ? {} : { expiresAt: entry.expiresAt }; + } function checkValidId(id) { switch (typeof id) { case 'number': @@ -5379,6 +5698,11 @@ export function makeTable(options) { if (length > MAX_KEY_BYTES) throw new ClientError('Primary key size is too large: ' + id.length, 400); return true; } + function assertTableWritable() { + if (dropQuiescing || droppingTable) { + throw new ServerError(`Table ${databaseName}.${tableName} is being dropped`, 503); + } + } function requestTargetToId(target: RequestTargetOrId): Id { return typeof target === 'object' && target ? (target as any).id : (target as Id); } @@ -5823,7 +6147,7 @@ export function makeTable(options) { // Once dropTable() has started, no new source-fill write may begin (dropTable() // only drains writes already in flight - see there); still resolve the caller's // read with fresh source data, just don't cache it into a table that's going away. - noCacheStore: droppingTable, + noCacheStore: droppingTable || dropQuiescing, source: null, transaction: undefined, expiresAt: undefined, @@ -5851,6 +6175,7 @@ export function makeTable(options) { let hasChanges, invalidated; try { updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry); + sourceContext.expiresAt = expirationOverrideTimestamp(sourceContext.expiresAt); invalidated = metadataFlags & INVALIDATED; let version = sourceContext.lastModified || (invalidated && existingVersion); hasChanges = invalidated || version > existingVersion || !existingRecord; @@ -5858,7 +6183,6 @@ export function makeTable(options) { recordAction(resolveDuration, 'cache-resolution', tableName, null, 'success'); if (responseHeaders) appendHeader(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true); - if (expirationMs && sourceContext.expiresAt == undefined) sourceContext.expiresAt = Date.now() + expirationMs; if (updatedRecord) { if (typeof updatedRecord !== 'object') throw new Error('Only objects can be cached and stored in tables'); if (updatedRecord.status > 0 && updatedRecord.headers) { @@ -5911,14 +6235,28 @@ export function makeTable(options) { // 5.2 record caching relies on it — so we must not write through the frozen object). if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord }; if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id; + const ignoredSourceExpiration = + sourceContext.expiresAt === undefined && + expiresAtProperty && + expirationTimestamp(updatedRecord[expiresAtProperty.name]) !== undefined; + if (ignoredSourceExpiration) { + recordActionBinary(true, 'cache-source-expiration-ignored', tableName); + if (!missingSourceExpirationReported) { + missingSourceExpirationReported = true; + logger.warn?.( + `Source for table "${tableName}" returned an @expiresAt field without setting context.expiresAt; the field does not set cache expiration.` + ); + } + } } + if (expirationMs && sourceContext.expiresAt == undefined) sourceContext.expiresAt = Date.now() + expirationMs; resolved = true; const resolvedEntry: Entry = { key: id, version, value: updatedRecord, expiresAt: sourceContext.expiresAt, - metadataFlags: 0, + metadataFlags: expiresAtProperty ? HAS_EXPIRATION_DECISION : 0, size: 0, localTime: 0, nodeId: 0, @@ -5960,7 +6298,7 @@ export function makeTable(options) { sourceContext.transaction.abort(); return; } - if (context?.noCacheStore || sourceContext.noCacheStore || droppingTable) { + if (context?.noCacheStore || sourceContext.noCacheStore || droppingTable || dropQuiescing) { // abort before we write any change. droppingTable is re-checked live (not just // the noCacheStore snapshot taken at call start) because a call admitted before // dropTable() began can still be sitting here after it started - the await above @@ -5976,12 +6314,23 @@ export function makeTable(options) { nodeName: 'source', commit: (txnTime, existingEntry, _retry, transaction: any) => { sourceWrite.skipped = false; // reset on each retry; cleanup happens after commit if still true + if (droppingTable || dropQuiescing) { + sourceWrite.skipped = true; + return; + } if (existingEntry?.version !== existingVersion) { // don't do anything if the version has changed sourceWrite.skipped = true; return; } - updateIndices(id, existingRecord, updatedRecord, transaction && { transaction }); + updateIndices( + id, + existingRecord, + updatedRecord, + transaction && { transaction }, + sourceContext.expiresAt, + effectiveExpirationIndexValue(existingEntry, existingRecord) + ); if (updatedRecord) { if (existingEntry) { context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId); @@ -6052,6 +6401,7 @@ export function makeTable(options) { { user: (sourceContext as any)?.user, expiresAt: sourceContext.expiresAt, + expirationDecisionPresent: Boolean(expiresAtProperty), residencyId, transaction, tableToTrack: tableName, @@ -6097,7 +6447,7 @@ export function makeTable(options) { TableResource.userEmbedders ); if (embedBefore) await embedBefore(); - if (droppingTable) { + if (droppingTable || dropQuiescing) { // Re-check right before staging the write: dropTable() may have started // while we were awaiting the embed step above (harper#1381). sourceContext.transaction.abort(); @@ -6137,17 +6487,56 @@ export function makeTable(options) { return true; } // RocksDB-only: coalesces eviction/tombstone removals into shared transactions so the cleanup - // scan pays one commit per batch instead of one per record. Descriptors hold only the decoded - // primary key and the version seen during the scan (both stable primitives — the scanned record - // value lives in a reused iterator buffer, so it is re-read fresh at commit time). Each record is - // version-guarded inside the commit transaction, and RocksDB's optimistic conflict detection + // scan pays one commit per batch instead of one per record. Descriptors hold stable, owned keys. + // Each record is guarded inside the commit transaction, and RocksDB's optimistic conflict detection // catches anything modified between staging and commit: on conflict (ERR_BUSY) we re-stage once // into a fresh transaction (dropping the now-changed record) and otherwise skip the batch, leaving // those records for the next cleanup cycle. - function createEvictionBatcher() { - type EvictItem = { type: 'evict' | 'tombstone'; key: any; version: number }; - let pending: EvictItem[] = []; + type ExpirationSweepTestHooks = { + beforeEvict?: () => MaybePromise; + beforeBatchCommit?: () => MaybePromise; + afterBatchQueued?: () => MaybePromise; + }; + function createInFlightLimiter(maxInFlight: number) { const inFlight = new Set>(); + return { + track(operation: Promise): Promise | void { + const tracked = operation.then( + () => undefined, + () => undefined + ); + inFlight.add(tracked); + tracked.then(() => inFlight.delete(tracked)); + if (inFlight.size >= maxInFlight) return Promise.race(inFlight); + }, + drain(): Promise { + return Promise.all(inFlight); + }, + }; + } + function createEvictionBatcher(isCancelled: () => boolean = () => false, testHooks?: ExpirationSweepTestHooks) { + type EvictItem = + | { + type: 'evict'; + key: any; + version: number; + encounteredExpiration?: number; + metadataExpiration?: number; + currentExpiration?: number; + } + | { type: 'tombstone'; key: any; version: number } + | { type: 'dangling-index'; key: any; indexedValue: any; index: any } + | { + type: 'stale-index'; + key: any; + version: number; + indexedValue: any; + metadataExpiration: number | undefined; + currentExpiration: number | undefined; + index: any; + }; + let pending: EvictItem[] = []; + const limiter = createInFlightLimiter(MAX_INFLIGHT_EVICTION_BATCHES); // Apply a batch's removals to the given transaction, re-reading each record fresh and skipping // any that changed since the scan. Returns the number of removals actually staged. @@ -6155,14 +6544,63 @@ export function makeTable(options) { const options = { transaction }; let staged = 0; for (const item of items) { + if (item.type === 'dangling-index') { + const entry = primaryStore.getEntry(item.key, options); + if (entry?.value != null) continue; + if (entry) { + const encodedTombstone = primaryStore.getBinarySync(item.key, options); + if (encodedTombstone === undefined) continue; + // Deliberate no-op write: make resurrection conflict with dangling-index cleanup. + primaryStore.putSync(item.key, asBinary(encodedTombstone.slice()), options); + } else { + primaryStore.removeSync(item.key, options); + } + item.index.remove(item.indexedValue, item.key, options); + staged++; + continue; + } + if (item.type === 'stale-index') { + const entry = primaryStore.getEntry(item.key, options); + if ( + !entry || + entry.value == null || + entry.version !== item.version || + compareKeys(entry.expiresAt, item.metadataExpiration) !== 0 || + compareKeys(item.indexedValue, item.currentExpiration) === 0 + ) + continue; + const encodedRecord = primaryStore.getBinarySync(item.key, options); + if (encodedRecord === undefined) continue; + primaryStore.putSync(item.key, asBinary(encodedRecord.slice()), options); + item.index.remove(item.indexedValue, item.key, options); + if (item.currentExpiration !== undefined && item.currentExpiration >= 0) + item.index.put(item.currentExpiration, item.key, options); + staged++; + continue; + } const entry = primaryStore.getEntry(item.key, options); if (!entry || entry.version !== item.version) continue; // gone or changed since the scan; leave for next cycle if (item.type === 'tombstone') { if (entry.value != null) continue; // resurrected since the scan } else { if (entry.value == null) continue; // already removed + if (entry.metadataFlags & HAS_BLOBS) continue; // per-record evict() owns blob/commit ordering if (hasSourceGet && primaryStore.hasLock(item.key, entry.version)) continue; // resolution in progress - updateIndices(item.key, entry.value, null, options); + let currentExpiration = entry.expiresAt; + if (item.encounteredExpiration !== undefined) { + currentExpiration = effectiveExpirationIndexValue(entry); + if ( + compareKeys(entry.expiresAt, item.metadataExpiration) !== 0 || + compareKeys(currentExpiration, item.currentExpiration) !== 0 + ) + continue; + } + updateIndices(item.key, entry.value, null, options, undefined, currentExpiration); + if ( + item.encounteredExpiration !== undefined && + compareKeys(item.encounteredExpiration, currentExpiration) !== 0 + ) + indices[expiresAtIndexName]?.remove(item.encounteredExpiration, item.key, options); } removeEntry(primaryStore, entry, options); staged++; @@ -6172,6 +6610,7 @@ export function makeTable(options) { async function commitItems(items: EvictItem[]) { for (let attempt = 0; attempt < 2; attempt++) { + if (isCancelled() || primaryStore.rootStore.status !== 'open') return; // Create the transaction inside the try: if the store is closing mid-scan, the constructor // can throw, and this promise is not always awaited (in-flight under the cap), so an // uncaught throw here would surface as an unhandled rejection. @@ -6194,6 +6633,7 @@ export function makeTable(options) { return; } try { + await testHooks?.beforeBatchCommit?.(); await transaction.commit(); return; } catch (error: any) { @@ -6212,157 +6652,188 @@ export function makeTable(options) { // Track an in-flight commit and, once the cap is reached, return a promise the caller can await // for backpressure (resolves as soon as any in-flight commit finishes). - function track(commit: Promise): Promise | void { - const tracked = commit.finally(() => inFlight.delete(tracked)); - inFlight.add(tracked); - if (inFlight.size >= MAX_INFLIGHT_EVICTION_BATCHES) return Promise.race(inFlight); - } - return { - add(type: 'evict' | 'tombstone', key: any, version: number): Promise | void { - pending.push({ type, key, version }); + add(item: EvictItem): Promise | void { + pending.push(item); if (pending.length >= EVICTION_BATCH_SIZE) { const items = pending; pending = []; - return track(commitItems(items)); + const backpressure = limiter.track(commitItems(items)); + const queued = testHooks?.afterBatchQueued?.(); + if (queued || backpressure) return Promise.all([queued, backpressure]).then(() => undefined); } }, async drain(): Promise { + if (isCancelled() || primaryStore.rootStore.status !== 'open') pending = []; if (pending.length > 0) { const items = pending; pending = []; - track(commitItems(items)); + limiter.track(commitItems(items)); } - await Promise.all(inFlight); + await limiter.drain(); }, }; } - function scheduleCleanup(priority?: number): Promise | void { - let runImmediately = false; - if (priority) { - // run immediately if there is a big increase in priority - if (priority - cleanupPriority > 1) runImmediately = true; - cleanupPriority = priority; + + function createCleanupOperationTracker() { + const limiter = createInFlightLimiter(MAX_CLEANUP_CONCURRENCY); + return { + add(operation: MaybePromise | void): Promise | void { + if (operation == null) return; + const tracked = Promise.resolve(operation).then( + () => undefined, + (error) => { + logger.error?.('Cleanup error', error); + } + ); + return limiter.track(tracked); + }, + drain(): Promise { + return limiter.drain(); + }, + }; + } + + async function runPrimaryCleanupScan( + testHooks?: ExpirationSweepTestHooks, + priority = cleanupPriority + ): Promise { + if (cleanupClosed) return; + const rootStore = primaryStore.rootStore; + if (rootStore.status !== 'open') return; + const operationTracker = createCleanupOperationTracker(); + const batcher = isRocksDB ? createEvictionBatcher(() => cleanupClosed, testHooks) : undefined; + const evictThreshold = + Math.pow(priority, 8) * (envMngr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_EVICTIONFACTOR) ?? 100000); + const adjustedEviction = evictionMs / Math.pow(Math.max(priority, 1), 4); + logger.debug?.( + `Starting cleanup scan for ${tableName}, evict threshold ${evictThreshold}, adjusted eviction ${adjustedEviction}ms` + ); + function shouldEvict(expiresAt: number, version: number, metadataFlags: number, record: any) { + const evictWhen = expiresAt + adjustedEviction - Date.now(); + if (evictWhen < 0) return true; + if (priority) { + let size = primaryStore.lastSize; + if (metadataFlags & HAS_BLOBS) { + findBlobsInObject(record, (blob) => { + if (blob.size) size += blob.size; + }); + } + logger.trace?.( + `shouldEvict adjusted ${evictWhen} ${size}, ${(evictWhen * (expiresAt - version)) / size} < ${evictThreshold}` + ); + return (evictWhen * (expiresAt - version)) / size < evictThreshold; + } + return false; } - // Periodically evict expired records and deleted records searching for records who expiresAt timestamp is before now - if (cleanupInterval === lastCleanupInterval && !runImmediately) return; - lastCleanupInterval = cleanupInterval; - if (getWorkerIndex() === getWorkerCount() - 1) { - // run on the last thread so we aren't overloading lower-numbered threads - if (cleanupTimer) clearTimeout(cleanupTimer); - if (!cleanupInterval) return; - return new Promise((resolve) => { + + try { + let count = 0; + const removeDeletedRecords = !audit || isRocksDB; + for (const entry of primaryStore.getRange({ + start: false, + snapshot: false, // avoid holding a read snapshot for the duration of a full-table scan + versions: true, + lazy: true, + })) { + if (cleanupClosed) break; + const { key, value: record, version, expiresAt, metadataFlags } = entry; + let action: 'tombstone' | 'evict' | undefined; + if (record === null && removeDeletedRecords && version + auditRetention < Date.now()) { + action = 'tombstone'; + } else if (expiresAt != undefined && shouldEvict(expiresAt, version, metadataFlags, record)) { + action = 'evict'; + count++; + } + if (action) { + // Blob deletion is a non-transactional side effect, so blob records stay on evict()'s + // per-record path where unlinking waits for the record removal to commit. + if (batcher && !(action === 'evict' && metadataFlags & HAS_BLOBS)) { + await batcher.add({ type: action, key, version }); + } else { + if (action === 'evict') await testHooks?.beforeEvict?.(); + const resolution = + action === 'tombstone' + ? removeEntry(primaryStore, entry, version) + : TableResource.evict(key, record, version); + if (resolution) { + const backpressure = operationTracker.add(resolution); + if (backpressure) await backpressure; + } + } + } + await rest(); + } + logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); + } catch (error) { + logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); + } finally { + if (batcher) await batcher.drain(); + await operationTracker.drain(); + } + } + + function queuePrimaryCleanup(testHooks?: ExpirationSweepTestHooks, priority = cleanupPriority): Promise { + return (lastEvictionCompletion = lastEvictionCompletion.then(() => runPrimaryCleanupScan(testHooks, priority))); + } + + function armCleanupTimer(nextScheduled: number, generation: number) { + if (cleanupClosed || generation !== cleanupGeneration) return; + cleanupNextScheduled = nextScheduled; + logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`); + const timer = setTimeout( + () => { + if (cleanupClosed || generation !== cleanupGeneration || cleanupTimer !== timer) return; + cleanupTimer = undefined; + cleanupNextScheduled = undefined; + if (primaryStore.rootStore.status !== 'open') return; + armCleanupTimer(Math.max(nextScheduled + cleanupInterval, Date.now()), generation); + const queuedPriority = cleanupPriority; + queuePrimaryCleanup(undefined, queuedPriority).then(() => { + if (cleanupPriority === queuedPriority) cleanupPriority = 0; + }); + }, + Math.min(nextScheduled - Date.now(), MAX_SET_TIMEOUT_MS) + ); + cleanupTimer = timer; + timer.unref(); + } + + function scheduleCleanup(priority?: number): Promise | void { + if (cleanupClosed || getWorkerIndex() !== getWorkerCount() - 1) return; + cleanupWasScheduled = true; + if (priority) cleanupPriority = Math.max(cleanupPriority, priority); + if (cleanupInterval !== lastCleanupInterval || priority) { + lastCleanupInterval = cleanupInterval; + cleanupGeneration++; + if (cleanupTimer) { + clearTimeout(cleanupTimer); + cleanupTimer = undefined; + cleanupNextScheduled = undefined; + } + if (cleanupInterval) { const startOfYear = new Date(); startOfYear.setMonth(0); startOfYear.setDate(1); - startOfYear.setHours(0); - startOfYear.setMinutes(0); - startOfYear.setSeconds(0); - const nextInterval = cleanupInterval / (1 + cleanupPriority); - // find the next scheduled run based on regular cycles from the beginning of the year (if we restart, this enables a good continuation of scheduling) - const nextScheduled = runImmediately - ? Date.now() - : Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime(); - const startNextTimer = (nextScheduled) => { - logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`); - // noinspection JSVoidFunctionReturnValueUsed - cleanupTimer = setTimeout( - () => - (lastEvictionCompletion = lastEvictionCompletion.then(async () => { - // schedule the next run for when the next cleanup interval should occur (or now if it is in the past) - startNextTimer(Math.max(nextScheduled + cleanupInterval, Date.now())); - const rootStore = primaryStore.rootStore; - if (rootStore.status !== 'open') { - clearTimeout(cleanupTimer); - return; - } - const MAX_CLEANUP_CONCURRENCY = 50; - const outstandingCleanupOperations = new Array(MAX_CLEANUP_CONCURRENCY); - let cleanupIndex = 0; - const evictThreshold = - Math.pow(cleanupPriority, 8) * - (envMngr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_EVICTIONFACTOR) ?? 100000); - const adjustedEviction = evictionMs / Math.pow(Math.max(cleanupPriority, 1), 4); - logger.debug?.( - `Starting cleanup scan for ${tableName}, evict threshold ${evictThreshold}, adjusted eviction ${adjustedEviction}ms` - ); - function shouldEvict(expiresAt: number, version: number, metadataFlags: number, record: any) { - const evictWhen = expiresAt + adjustedEviction - Date.now(); - if (evictWhen < 0) return true; - else if (cleanupPriority) { - let size = primaryStore.lastSize; - if (metadataFlags & HAS_BLOBS) { - findBlobsInObject(record, (blob) => { - if (blob.size) size += blob.size; - }); - } - logger.trace?.( - `shouldEvict adjusted ${evictWhen} ${size}, ${(evictWhen * (expiresAt - version)) / size} < ${evictThreshold}` - ); - // heuristic to determine if we should perform early eviction based on priority - return (evictWhen * (expiresAt - version)) / size < evictThreshold; - } - return false; - } - - try { - let count = 0; - let removeDeletedRecords = !audit || isRocksDB; - // RocksDB coalesces eviction/tombstone removals into shared transactions to amortize - // the per-record commit cost; LMDB keeps the per-record path (eventTurnBatching already - // coalesces async writes per event turn). - const batcher = isRocksDB ? createEvictionBatcher() : undefined; - // iterate through all entries to find expired records and deleted records - for (const entry of primaryStore.getRange({ - start: false, - snapshot: false, // we don't want to keep read transaction snapshots open - versions: true, - lazy: true, // only want to access metadata most of the time - })) { - const { key, value: record, version, expiresAt, metadataFlags } = entry; - // if there is no auditing cleanup and we are tracking deletion, need to do cleanup of - // these deletion entries (LMDB audit cleanup has its own scheduled job for this) - let action: 'tombstone' | 'evict' | undefined; - if (record === null && removeDeletedRecords && version + auditRetention < Date.now()) { - action = 'tombstone'; - } else if (expiresAt != undefined && shouldEvict(expiresAt, version, metadataFlags, record)) { - action = 'evict'; - count++; - } - if (action) { - // Blob-bearing records delete their blob files as a non-transactional side effect, so - // they stay on the per-record evict() path that preserves the existing blob/commit ordering. - if (batcher && !(action === 'evict' && metadataFlags & HAS_BLOBS)) { - await batcher.add(action, key, version); - } else { - const resolution = - action === 'tombstone' - ? removeEntry(primaryStore, entry, version) - : TableResource.evict(key, record, version); - if (resolution) { - await outstandingCleanupOperations[cleanupIndex]; - outstandingCleanupOperations[cleanupIndex] = resolution.catch((error) => { - logger.error?.('Cleanup error', error); - }); - if (++cleanupIndex >= MAX_CLEANUP_CONCURRENCY) cleanupIndex = 0; - } - } - } - await rest(); - } - if (batcher) await batcher.drain(); - logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); - } catch (error) { - logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); - } - resolve(undefined); - cleanupPriority = 0; // reset the priority - })), - Math.min(nextScheduled - Date.now(), MAX_SET_TIMEOUT_MS) // make sure it can fit in 32-bit signed number - ).unref(); // don't let this prevent closing the thread - }; - startNextTimer(nextScheduled); + startOfYear.setHours(0, 0, 0, 0); + // A full disk produces an Infinity reclamation priority. Keep that value for the + // immediate eviction scan, but never let it collapse the recurring timer to NaN/0. + const schedulingPriority = Number.isFinite(cleanupPriority) ? Math.max(cleanupPriority, 0) : 0; + const nextInterval = Math.max(cleanupInterval / (1 + schedulingPriority), MIN_SCHEDULED_CLEANUP_INTERVAL); + const nextScheduled = + Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime(); + armCleanupTimer(nextScheduled, cleanupGeneration); + } + } else { + return; + } + if (priority) { + const queuedPriority = cleanupPriority; + const completion = queuePrimaryCleanup(undefined, queuedPriority); + completion.then(() => { + if (cleanupPriority === queuedPriority) cleanupPriority = 0; }); + return completion; } } function addDeleteRemoval() { @@ -6370,44 +6841,164 @@ export function makeTable(options) { primaryStore.remove(id, version); }); } - function runRecordExpirationEviction() { - // Periodically evict expired records, searching for records who expiresAt timestamp is before now - if (getWorkerIndex() === 0) { - // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting - setInterval(async () => { - // go through each database and table and then search for expired entries - // find any entries that are set to expire before now - if (runningRecordExpiration) return; - runningRecordExpiration = true; - try { - const expiresAtName = expiresAtProperty.name; - const index = indices[expiresAtName]; - if (!index) throw new Error(`expiresAt attribute ${expiresAtProperty} must be indexed`); - for (const key of index.getRange({ - start: true, - values: false, - end: Date.now(), - snapshot: false, - })) { - for (const id of index.getValues(key)) { + function runRecordExpirationEviction(testHooks?: ExpirationSweepTestHooks, schedule = true, forceWorker = false) { + if ((!forceWorker && getWorkerIndex() !== 0) || cleanupClosed) return; + const expiresAtName = expiresAtProperty.name; + async function sweepRocks(index: any, cutoff: number) { + const batcher = createEvictionBatcher(() => cleanupClosed, testHooks); + let after: any[] | undefined; + try { + while (!cleanupClosed && primaryStore.rootStore.status === 'open') { + const previousAfter = after; + const entries = [...index.getCompositeRange({ after, end: cutoff, limit: EVICTION_BATCH_SIZE })]; + if (entries.length === 0) break; + let completedChunk = true; + let entriesSinceYield = 0; + for (const entry of entries) { + if (cleanupClosed || primaryStore.rootStore.status !== 'open') { + completedChunk = false; + break; + } + after = entry.cursor; + const recordEntry = primaryStore.getEntry(entry.value); + let backpressure: Promise | void; + if (recordEntry?.value == null) { + backpressure = batcher.add({ + type: 'dangling-index', + key: entry.value, + indexedValue: entry.key, + index, + }); + } else { + const currentExpiration = effectiveExpirationIndexValue(recordEntry); + if (currentExpiration !== undefined && currentExpiration < cutoff) { + if (recordEntry.metadataFlags & HAS_BLOBS) { + await testHooks?.beforeEvict?.(); + await TableResource.evict(entry.value, recordEntry.value, recordEntry.version, entry.key); + } else { + backpressure = batcher.add({ + type: 'evict', + key: entry.value, + version: recordEntry.version, + encounteredExpiration: entry.key, + metadataExpiration: recordEntry.expiresAt, + currentExpiration, + }); + } + } else if (compareKeys(entry.key, currentExpiration) !== 0) { + backpressure = batcher.add({ + type: 'stale-index', + key: entry.value, + version: recordEntry.version, + indexedValue: entry.key, + metadataExpiration: recordEntry.expiresAt, + currentExpiration, + index, + }); + } + } + if (backpressure) await backpressure; + if (++entriesSinceYield >= 10) { + entriesSinceYield = 0; + await rest(); + } + } + if (!completedChunk) break; + if (previousAfter && compareKeys(after, previousAfter) <= 0) { + logger.warn?.(`Record expiration cursor did not advance for ${tableName}`); + break; + } + await rest(); + if (entries.length < EVICTION_BATCH_SIZE) break; + } + } finally { + await batcher.drain(); + } + } + + async function sweepLmdb(index: any, cutoff: number) { + const operationTracker = createCleanupOperationTracker(); + let entriesSinceYield = 0; + try { + for (const key of index.getRange({ start: true, values: false, end: cutoff, snapshot: false })) { + if (cleanupClosed) break; + let afterId: any; + for (;;) { + if (cleanupClosed) break; + const ids = [ + ...index.getValues(key, { + start: afterId, + exclusiveStart: afterId !== undefined, + limit: EVICTION_BATCH_SIZE, + }), + ]; + if (ids.length === 0) break; + afterId = ids.at(-1); + for (const id of ids) { + if (cleanupClosed) break; const recordEntry = primaryStore.getEntry(id); - if (!recordEntry?.value) { - // cleanup the index if the record is gone - primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); - } else if (recordEntry.value[expiresAtName] < Date.now()) { - // make sure the record hasn't changed and won't change while removing - TableResource.evict(id, recordEntry.value, recordEntry.version); + let operation: MaybePromise | void; + if (recordEntry?.value == null) { + operation = primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); + } else { + const currentExpiration = effectiveExpirationIndexValue(recordEntry); + if (currentExpiration !== undefined && currentExpiration < cutoff) { + await testHooks?.beforeEvict?.(); + operation = TableResource.evict(id, recordEntry.value, recordEntry.version, key); + } else if (compareKeys(key, currentExpiration) !== 0) { + operation = primaryStore.ifVersion(id, recordEntry.version, () => { + const removals = [index.remove(key, id)]; + if (currentExpiration !== undefined) removals.push(index.put(currentExpiration, id)); + return Promise.all(removals); + }); + } + } + const backpressure = operationTracker.add(operation); + if (backpressure) await backpressure; + if (++entriesSinceYield >= 10) { + entriesSinceYield = 0; + await rest(); } } - await rest(); + if (ids.length < EVICTION_BATCH_SIZE) break; } - } catch (error) { - logger.error?.('Error in evicting old records', error); - } finally { - runningRecordExpiration = false; + await rest(); + } + } finally { + await operationTracker.drain(); + } + } + + async function sweep() { + if (runningRecordExpiration) return recordExpirationCompletion; + if (cleanupClosed) return; + runningRecordExpiration = true; + try { + if (primaryStore.rootStore.status !== 'open') return; + const index = indices[expiresAtName]; + if (!index) { + if (!missingExpirationIndexReported) { + missingExpirationIndexReported = true; + logger.warn?.(`expiresAt attribute ${expiresAtName} must be indexed for ${tableName}`); + } + return; } - }, RECORD_PRUNING_INTERVAL).unref(); + if (index.isIndexing) return; + missingExpirationIndexReported = false; + const cutoff = Date.now(); + if (isRocksDB) await sweepRocks(index, cutoff); + else await sweepLmdb(index, cutoff); + } catch (error) { + logger.error?.('Error in evicting old records', error); + } finally { + runningRecordExpiration = false; + } } + + const runSweep = () => (recordExpirationCompletion = sweep()); + if (!schedule) return runSweep(); + recordExpirationInterval = setInterval(runSweep, RECORD_PRUNING_INTERVAL); + recordExpirationInterval.unref(); } function residencyFromFunction(shardOrResidencyList: ResidencyDefinition): string[] | void { if (shardOrResidencyList == undefined) return; diff --git a/resources/auditStore.ts b/resources/auditStore.ts index b57b88665e..9739436e39 100644 --- a/resources/auditStore.ts +++ b/resources/auditStore.ts @@ -360,6 +360,7 @@ export const HAS_ADDITIONAL_AUDIT_REFS = 0x4000; * lower-byte action region (which extendedType forbids) and within the always-32-bit metadata form. */ export const LOCAL_ONLY = 0x8000; +export const HAS_EXPIRATION_DECISION = 0x10000; const EVENT_TYPES = { put: PUT | HAS_RECORD, [PUT]: 'put', diff --git a/resources/blob.ts b/resources/blob.ts index f68a04028d..7c5c32bc27 100644 --- a/resources/blob.ts +++ b/resources/blob.ts @@ -1099,7 +1099,6 @@ function resetDrainedQueue(): void { * @param blob */ export function deleteBlob(blob: Blob): void { - // do we even need to check for completion here? const filePath = getFilePathForBlob(blob as any); if (!filePath) { return; @@ -1229,6 +1228,14 @@ function runReclamation(): void { if (earliest !== Infinity) scheduleReclamation(earliest); } +export function prepareBlobDeletion(object: any): () => void { + const blobs: Blob[] = []; + findBlobsInObject(object, (blob) => { + if (getFilePathForBlob(blob as any)) blobs.push(blob); + }); + return () => blobs.forEach(deleteBlob); +} + /** * Test knob: the delay between a blob being superseded and its file being reclaimed. `undefined` * restores the configured value. diff --git a/resources/databases.ts b/resources/databases.ts index 55cd4fba2b..47d1a15dc4 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -9,13 +9,14 @@ import { getBaseSchemaPath, getTransactionAuditStoreBasePath, } from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; -import { makeTable, ignoreAlreadyDropped } from './Table.ts'; +import { effectiveExpirationTimestamp, makeTable, ignoreAlreadyDropped } from './Table.ts'; import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.ts'; import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME, MIGRATING_DIR_SUFFIX, + OPERATIONS_ENUM, RESERVED_DATABASE_NAMES, } from '../utility/hdbTerms.ts'; import { getConfigPath } from '../config/configUtils.ts'; @@ -96,6 +97,62 @@ const logger = forComponent('storage'); const DEFAULT_DATABASE_NAME = 'data'; const DEFINED_TABLES = Symbol('defined-tables'); +const DEFAULT_DATABASE_CLOSE_TIMEOUT = 10_000; +let databaseCloseTimeout = DEFAULT_DATABASE_CLOSE_TIMEOUT; +const unavailableDatabases = new Set(); +const SCHEMA_QUIESCE_LEASE_MS = 120_000; +type SchemaQuiesceMessage = { + originator?: number; + operation: string; + schema: string; + table?: string; + quiesceId: string; + leaseUntil?: number; + phase?: string; +}; +type SchemaQuiesceState = { + message: SchemaQuiesceMessage; + table?: any; + tables?: any[]; + lease?: NodeJS.Timeout; + wasLoaded?: boolean; + completion?: Promise; + abortRequested?: boolean; + finalizing?: boolean; + committed?: boolean; + localOwner?: boolean; + rootPaths?: string[]; + recoveryAttempts?: number; +}; +const schemaQuiescence = new Map(); +type SchemaQuiesceOwners = { schema?: string; tables: Map }; +const quiescedSchemas = new Map(); +const MAX_RETIRED_SCHEMA_QUIESCENCES = 1024; +const retiredSchemaQuiescences = new Set(); +const recoveringSchemaQuiescences = new Set(); +const MAX_COMMITTED_QUIESCE_RECOVERY_ATTEMPTS = 3; + +export function setDatabaseCloseTimeoutForTests(timeout = DEFAULT_DATABASE_CLOSE_TIMEOUT): void { + databaseCloseTimeout = timeout; +} + +async function waitForTableCleanup(completions: Promise[]): Promise[] | false> { + let timeout: NodeJS.Timeout | undefined; + const timedOut = Symbol('timedOut'); + const result = await Promise.race([ + Promise.allSettled(completions), + new Promise((resolve) => { + timeout = setTimeout(() => resolve(timedOut), databaseCloseTimeout); + timeout.unref(); + }), + ]); + if (timeout) clearTimeout(timeout); + return result === timedOut ? false : result; +} + +function cleanupFailures(results: PromiseSettledResult[]): unknown[] { + return results.filter((result) => result.status === 'rejected').map((result) => result.reason); +} const DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages initSync(); /** @@ -343,6 +400,15 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo const lmdbDatabaseEnvs = new Map(); const rocksdbDatabaseEnvs = new Map(); +function forgetDatabaseEnvironment(rootStore: any): void { + for (const [path, store] of lmdbDatabaseEnvs) { + if (store === rootStore || path === rootStore.path) lmdbDatabaseEnvs.delete(path); + } + for (const [path, store] of rocksdbDatabaseEnvs) { + if (store === rootStore || path === rootStore.path) rocksdbDatabaseEnvs.delete(path); + } +} + // set the following in both global and exports _assignPackageExport('databases', databases); _assignPackageExport('tables', tables); @@ -627,6 +693,12 @@ export function readMetaDb( auditPath?: string, isLegacy?: boolean ) { + const schemaOwner = quiescedSchemas.get(databaseName)?.schema; + if (schemaOwner && !recoveringSchemaQuiescences.has(schemaOwner)) { + const existing = lmdbDatabaseEnvs.get(path); + if (existing) existing.needsDeletion = false; + return existing; + } const envInit = new OpenEnvironmentObject(path, isReadOnlyMode()); try { let rootStore = lmdbDatabaseEnvs.get(path); @@ -645,6 +717,8 @@ export function readMetaDb( } function readRocksMetaDb(path: string, defaultTable?: string, databaseName: string = DEFAULT_DATABASE_NAME) { + const schemaOwner = quiescedSchemas.get(databaseName)?.schema; + if (schemaOwner && !recoveringSchemaQuiescences.has(schemaOwner)) return rocksdbDatabaseEnvs.get(path); try { logger.trace(`loading rocksdb database: ${path}`); @@ -682,6 +756,8 @@ function initStores( auditPath?: string, isLegacy?: boolean ) { + const schemaOwner = quiescedSchemas.get(databaseName)?.schema; + if (schemaOwner && !recoveringSchemaQuiescences.has(schemaOwner)) return rootStore; const envInit = new OpenEnvironmentObject(path, isReadOnlyMode()); const internalDbiInit = createOpenDBIObject(false); let attributesDbi = rootStore.dbisDb; @@ -749,6 +825,12 @@ function initStores( if (attribute_name != null) tableDef.attributes.push(value); Object.defineProperty(value, 'key', { value: key, configurable: true }); } + const tableOwners = quiescedSchemas.get(databaseName)?.tables; + if (tableOwners) { + for (const [tableName, owner] of tableOwners) { + if (!recoveringSchemaQuiescences.has(owner)) tablesToLoad.delete(tableName); + } + } // Complete any drops that were interrupted mid-flight. dropTable persists a // `dropping` tombstone on the table's primary catalog entry before removing @@ -967,6 +1049,7 @@ function initStores( table.updatedAttributes(); } } else { + if (recreateForEngineChange) table.cleanup?.(); table = setTable( tables, tableName, @@ -1124,6 +1207,9 @@ export function resolveDatabasePath(databaseName: string): string { */ export function database({ database: databaseName, table: tableName }) { if (!databaseName) databaseName = DEFAULT_DATABASE_NAME; + if (unavailableDatabases.has(databaseName)) { + throw new Error(`Database ${databaseName} is closing and cannot be opened`); + } getDatabases(); ensureDB(databaseName); const definedDatabase = definedDatabases.get(databaseName); @@ -1218,11 +1304,410 @@ function lockDatabaseForDrop(dbPath: string, databaseName: string, held: Restore * Delete the database * @param databaseName */ -export async function dropDatabase(databaseName) { +const droppingDatabases = new Map>(); + +function schemaQuiesceTarget(message: SchemaQuiesceMessage): string { + return message.operation === OPERATIONS_ENUM.DROP_TABLE + ? `table:${message.schema}.${message.table}` + : `schema:${message.schema}`; +} + +function claimSchemaQuiesce(message: SchemaQuiesceMessage): string | undefined { + let owners = quiescedSchemas.get(message.schema); + if (!owners) quiescedSchemas.set(message.schema, (owners = { tables: new Map() })); + if (message.operation === OPERATIONS_ENUM.DROP_SCHEMA) { + if (owners.schema && owners.schema !== message.quiesceId) return `schema:${message.schema} is already quiescing`; + for (const owner of owners.tables.values()) { + if (owner !== message.quiesceId) return `schema:${message.schema} has a table already quiescing`; + } + owners.schema = message.quiesceId; + } else { + if (owners.schema && owners.schema !== message.quiesceId) return `schema:${message.schema} is already quiescing`; + const owner = owners.tables.get(message.table!); + if (owner && owner !== message.quiesceId) return `table:${message.schema}.${message.table} is already quiescing`; + owners.tables.set(message.table!, message.quiesceId); + } +} + +function releaseSchemaQuiesce(message: SchemaQuiesceMessage): void { + const owners = quiescedSchemas.get(message.schema); + if (!owners) return; + if (owners.schema === message.quiesceId) owners.schema = undefined; + if (message.table && owners.tables.get(message.table) === message.quiesceId) owners.tables.delete(message.table); + if (!owners.schema && owners.tables.size === 0) quiescedSchemas.delete(message.schema); +} + +function schemaQuiesceOwner(message: SchemaQuiesceMessage): string | undefined { + const owners = quiescedSchemas.get(message.schema); + return message.operation === OPERATIONS_ENUM.DROP_SCHEMA ? owners?.schema : owners?.tables.get(message.table!); +} + +function clearSchemaQuiesce(state: SchemaQuiesceState): void { + if (state.lease) clearTimeout(state.lease); + schemaQuiescence.delete(state.message.quiesceId); + releaseSchemaQuiesce(state.message); +} + +function retireSchemaQuiesce(quiesceId: string): void { + retiredSchemaQuiescences.add(quiesceId); + while (retiredSchemaQuiescences.size > MAX_RETIRED_SCHEMA_QUIESCENCES) { + const oldestId = retiredSchemaQuiescences.values().next().value; + retiredSchemaQuiescences.delete(oldestId); + } +} + +function armSchemaQuiesceLease(state: SchemaQuiesceState): void { + if (state.lease) clearTimeout(state.lease); + const leaseUntil = Math.max(state.message.leaseUntil ?? 0, Date.now() + SCHEMA_QUIESCE_LEASE_MS); + state.lease = setTimeout(() => expireSchemaQuiesceLease(state), leaseUntil - Date.now()); + state.lease.unref(); +} + +function expireSchemaQuiesceLease(state: SchemaQuiesceState): void { + if (state.committed) { + // A synchronous RocksDB drop can outlive the timer that its blocked origin would renew. + // A connected owner may still be destructing storage, so peers must remain fail-closed. + if (!state.localOwner && manageThreads.isThreadConnected(state.message.originator)) { + armSchemaQuiesceLease(state); + return; + } + recoverCommittedSchemaQuiesce(state).catch((error) => { + logger.error(`Could not recover committed schema quiesce ${state.message.quiesceId}`, error); + }); + return; + } + reconcileSchemaQuiesce(state).catch((error) => { + logger.warn('Could not reconcile expired schema quiesce:', error); + if (schemaQuiescence.get(state.message.quiesceId) === state) armSchemaQuiesceLease(state); + }); +} + +async function recoverCommittedSchemaQuiesce(state: SchemaQuiesceState): Promise { + if (schemaQuiescence.get(state.message.quiesceId) !== state || !state.committed) return false; + if ((state.recoveryAttempts ?? 0) >= MAX_COMMITTED_QUIESCE_RECOVERY_ATTEMPTS) return false; + state.recoveryAttempts = (state.recoveryAttempts ?? 0) + 1; + const locks: RestoreLock[] = []; + try { + for (const path of state.rootPaths ?? []) locks.push(acquireRestoreLock(path)); + } catch (error) { + for (const lock of locks) releaseRestoreLock(lock); + if (state.recoveryAttempts < MAX_COMMITTED_QUIESCE_RECOVERY_ATTEMPTS) { + logger.warn( + `Committed schema quiesce ${state.message.quiesceId} is still protected by an active drop; recovery attempt ${state.recoveryAttempts} deferred` + ); + armSchemaQuiesceLease(state); + } else { + logger.error( + `Committed schema quiesce ${state.message.quiesceId} could not be recovered after ${state.recoveryAttempts} attempts; target remains fail-closed`, + error + ); + } + return false; + } + for (const lock of locks) releaseRestoreLock(lock); + + const { schema, table } = state.message; + try { + // Inspect the durable decision before changing public registrations or bypassing the + // quiescence catalog fence. This distinguishes a live catalog row (destruction never + // started), a drop tombstone (finish interrupted destruction), and an absent row + // (destruction completed) without letting resetDatabases manufacture the answer. + const durableDescriptor = + state.message.operation === OPERATIONS_ENUM.DROP_TABLE && table + ? state.table?.dbisDB?.getSync?.(`${table}/`) + : undefined; + recoveringSchemaQuiescences.add(state.message.quiesceId); + if (state.message.operation === OPERATIONS_ENUM.DROP_TABLE) { + if (databases[schema] && table) delete databases[schema][table]; + resetDatabases(); + const reconciledDescriptor = table && state.table?.dbisDB?.getSync?.(`${table}/`); + if (durableDescriptor?.dropping && reconciledDescriptor?.dropping) + throw new Error(`Durable drop tombstone for ${schema}.${table} could not be reconciled`); + } else { + resetQuiescedDatabase(schema); + } + clearSchemaQuiesce(state); + retireSchemaQuiesce(state.message.quiesceId); + if (state.message.operation === OPERATIONS_ENUM.DROP_SCHEMA) unavailableDatabases.delete(schema); + logger.warn( + `Recovered expired committed schema quiesce ${state.message.quiesceId} from authoritative storage state` + ); + return true; + } catch (error) { + if (state.recoveryAttempts < MAX_COMMITTED_QUIESCE_RECOVERY_ATTEMPTS) armSchemaQuiesceLease(state); + else + logger.error( + `Committed schema quiesce ${state.message.quiesceId} exhausted authoritative recovery attempts; target remains fail-closed`, + error + ); + throw error; + } finally { + recoveringSchemaQuiescences.delete(state.message.quiesceId); + } +} + +export function recoverCommittedSchemaQuiesceForTests(quiesceId: string): Promise { + const state = schemaQuiescence.get(quiesceId); + return state ? recoverCommittedSchemaQuiesce(state) : Promise.resolve(false); +} + +export function expireSchemaQuiesceLeaseForTests(quiesceId: string): void { + const state = schemaQuiescence.get(quiesceId); + if (state) expireSchemaQuiesceLease(state); +} + +function resetQuiescedDatabase(databaseName: string): void { + delete databases[databaseName]; + const definedDatabase = definedDatabases?.get(databaseName); + if (definedDatabase) (definedDatabase as any).rootStore = undefined; + resetDatabases(); +} + +async function reconcileSchemaQuiesce(state: SchemaQuiesceState): Promise { + state.abortRequested = true; + // A peer may already be closing handles or may have removed the target from its registry. Do not + // reopen it until that attempt settles. Origin-owned quiescence only drains work and can abort + // immediately—the cleanup generation prevents its blocked scan from rearming. + if (!state.localOwner) await state.completion?.catch(() => undefined); + if (schemaQuiescence.get(state.message.quiesceId) !== state) return; + if (state.message.operation === OPERATIONS_ENUM.DROP_TABLE) { + const { schema, table } = state.message; + const dbTables = databases[schema]; + if (!state.localOwner) { + if (state.table && dbTables && !dbTables[table]) dbTables[table] = state.table; + resetDatabases(); + } + if (state.table && (state.localOwner || databases[schema]?.[table] === state.table)) state.table.abortDropQuiesce(); + } else { + if (state.localOwner) { + for (const table of state.tables ?? []) table.abortDropQuiesce?.(); + } else { + recoveringSchemaQuiescences.add(state.message.quiesceId); + try { + resetQuiescedDatabase(state.message.schema); + } finally { + recoveringSchemaQuiescences.delete(state.message.quiesceId); + } + } + } + clearSchemaQuiesce(state); + retireSchemaQuiesce(state.message.quiesceId); + if (state.message.operation === OPERATIONS_ENUM.DROP_SCHEMA) unavailableDatabases.delete(state.message.schema); +} + +export async function quiesceSchemaTarget( + message: SchemaQuiesceMessage & { originLocal?: boolean } +): Promise<{ quiesced: boolean; reason?: string }> { + if (retiredSchemaQuiescences.has(message.quiesceId)) + return { quiesced: false, reason: `Schema quiesce ${message.quiesceId} is no longer active` }; + const target = schemaQuiesceTarget(message); + const existingById = schemaQuiescence.get(message.quiesceId); + if (existingById && schemaQuiesceTarget(existingById.message) !== target) + return { quiesced: false, reason: `Schema quiesce ${message.quiesceId} belongs to another target` }; + const activeId = schemaQuiesceOwner(message); + if (activeId) { + if (activeId !== message.quiesceId) return { quiesced: false, reason: `${target} is already quiescing` }; + const active = schemaQuiescence.get(activeId); + if (!active || active.abortRequested || active.finalizing) + return { quiesced: false, reason: `${target} quiescence is terminating` }; + armSchemaQuiesceLease(active); + await active.completion; + return active.abortRequested ? { quiesced: false, reason: `${target} quiescence was aborted` } : { quiesced: true }; + } + const conflict = claimSchemaQuiesce(message); + if (conflict) return { quiesced: false, reason: conflict }; + const state: SchemaQuiesceState = { message, localOwner: message.originLocal === true }; + schemaQuiescence.set(message.quiesceId, state); + armSchemaQuiesceLease(state); + state.completion = (async () => { + if (message.operation === OPERATIONS_ENUM.DROP_TABLE) { + const dbTables = databases[message.schema]; + const Table = dbTables?.[message.table]; + if (!Table) return; + state.table = Table; + if (Table.primaryStore?.rootStore?.path) state.rootPaths = [Table.primaryStore.rootStore.path]; + if (!state.localOwner) delete dbTables[message.table]; + await Table.quiesceForDrop(); + } else if (message.operation === OPERATIONS_ENUM.DROP_SCHEMA) { + state.wasLoaded = Boolean(databases[message.schema]); + unavailableDatabases.add(message.schema); + if (state.wasLoaded) { + const definedRootPath = (definedDatabases?.get(message.schema) as any)?.rootStore?.path; + state.rootPaths = [ + ...new Set( + [ + definedRootPath, + ...Object.values(databases[message.schema]).map((table: any) => table?.primaryStore?.rootStore?.path), + ].filter(Boolean) + ), + ]; + } + if (state.wasLoaded && state.localOwner) { + state.tables = Object.values(databases[message.schema]); + await Promise.all(state.tables.map((table) => table.quiesceForDrop?.())); + } else if (state.wasLoaded) await closeDatabaseOnce(message.schema); + } else { + throw new Error(`Unsupported schema quiesce operation ${message.operation}`); + } + })(); + try { + const completion = await waitForTableCleanup([state.completion]); + if (!completion) { + if (message.operation === OPERATIONS_ENUM.DROP_SCHEMA) + throw new Error( + `Timed out after ${databaseCloseTimeout}ms waiting for cleanup; refusing to destroy database ${message.schema} while cleanup is active.` + ); + throw new Error( + `Timed out after ${databaseCloseTimeout}ms waiting for ${target} to quiesce; the target remains available.` + ); + } + const failures = cleanupFailures(completion); + if (failures.length) { + if (message.operation === OPERATIONS_ENUM.DROP_SCHEMA) + throw new AggregateError(failures, `Could not quiesce database ${message.schema} for drop`); + throw failures[0]; + } + if (state.abortRequested) return { quiesced: false, reason: `${target} quiescence was aborted` }; + return { quiesced: true }; + } catch (error) { + try { + await reconcileSchemaQuiesce(state); + } catch (abortError) { + logger.warn('Could not restore schema state after quiesce failure:', abortError); + } + return { quiesced: false, reason: error instanceof Error ? error.message : String(error) }; + } +} + +export async function abortSchemaQuiesce(message: Pick): Promise { + const state = schemaQuiescence.get(message.quiesceId); + if (!state) return; + if (state.committed) throw new Error(`Schema quiesce ${message.quiesceId} crossed its commit boundary`); + await reconcileSchemaQuiesce(state); +} + +export async function commitSchemaQuiesce( + message: SchemaQuiesceMessage +): Promise<{ committed: boolean; reason?: string }> { + const state = schemaQuiescence.get(message.quiesceId); + if (!state || schemaQuiesceOwner(message) !== message.quiesceId || state.abortRequested || state.finalizing) + return { committed: false, reason: `Schema quiesce ${message.quiesceId} is not active` }; + await state.completion; + if (state.abortRequested) return { committed: false, reason: `Schema quiesce ${message.quiesceId} was aborted` }; + state.committed = true; + armSchemaQuiesceLease(state); + return { committed: true }; +} + +export function renewSchemaQuiesce(message: SchemaQuiesceMessage): { quiesced: boolean; reason?: string } { + const state = schemaQuiescence.get(message.quiesceId); + if (!state || state.abortRequested || state.finalizing) + return { quiesced: false, reason: `Schema quiesce ${message.quiesceId} is not active` }; + state.message.leaseUntil = message.leaseUntil; + armSchemaQuiesceLease(state); + return { quiesced: true }; +} + +export function finishSchemaQuiesce(message: SchemaQuiesceMessage): boolean { + if (retiredSchemaQuiescences.has(message.quiesceId)) return false; + const activeId = schemaQuiesceOwner(message); + // A worker started after quiescence (or one that never loaded this target) has no local state, + // but must still perform the authoritative terminal rescan. Retired IDs above remain rejected, + // preventing an aborted/completed operation from being replayed. `originator` is a worker thread + // id stamped by sendItcEvent, not an OS pid, so comparing it with process.pid incorrectly rejected + // every legitimate no-state worker. + if (activeId === undefined) return true; + if (activeId !== message.quiesceId) return false; + const state = schemaQuiescence.get(message.quiesceId); + if (!state || state.abortRequested) return false; + if (message.phase === 'finalize-quiesce' && !state.committed) return false; + state.finalizing = true; + // The terminal handler is the authoritative rescan for this operation. Let only this owner pass + // the catalog fence while it applies durable state; ordinary concurrent schema rescans remain blocked. + recoveringSchemaQuiescences.add(message.quiesceId); + if (state.lease) { + clearTimeout(state.lease); + state.lease = undefined; + } + return true; +} + +export function completeSchemaQuiesce(message: SchemaQuiesceMessage): void | Promise { + recoveringSchemaQuiescences.delete(message.quiesceId); + const state = schemaQuiescence.get(message.quiesceId); + if (state && message.phase === 'reconcile-quiesce') return reconcileSchemaQuiesce(state); + if (state) { + clearSchemaQuiesce(state); + if (state.message.operation === OPERATIONS_ENUM.DROP_SCHEMA) unavailableDatabases.delete(state.message.schema); + } + retireSchemaQuiesce(message.quiesceId); +} + +export function failSchemaQuiesceFinalization(message: Pick): void { + recoveringSchemaQuiescences.delete(message.quiesceId); + const state = schemaQuiescence.get(message.quiesceId); + if (!state) return; + state.finalizing = false; + state.abortRequested = false; + armSchemaQuiesceLease(state); +} + +export async function dropDatabase(databaseName): Promise { + const activeDrop = droppingDatabases.get(databaseName); + if (activeDrop) return activeDrop; if (!databases[databaseName]) throw new Error('Database does not exist'); + if (unavailableDatabases.has(databaseName)) throw new Error(`Database ${databaseName} is already closing`); const dbTables = databases[databaseName]; let rootStore; + for (const tableName in dbTables) { + rootStore = dbTables[tableName]?.primaryStore?.rootStore; + if (rootStore) break; + } + if (!rootStore) rootStore = database({ database: databaseName, table: null }); + unavailableDatabases.add(databaseName); + let quiesceMessage: any; + let dropSucceeded = false; + const completion = dropDatabaseOnce(databaseName, dbTables, rootStore, (message) => { + quiesceMessage = message; + }) + .then(() => { + dropSucceeded = true; + }) + .finally(async () => { + droppingDatabases.delete(databaseName); + if (!quiesceMessage) { + for (const tableName in dbTables) dbTables[tableName].abortDropQuiesce?.(); + unavailableDatabases.delete(databaseName); + } + if (quiesceMessage) { + const message: any = new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, databaseName); + message.quiesceId = quiesceMessage.quiesceId; + if (dropSucceeded) { + try { + await signalling.finalizeSchemaChange(message); + } finally { + // finalizeSchemaChange applies the local terminal rescan before broadcasting it. + // A later remote acknowledgement failure must not permanently poison this + // physically-destroyed name on the origin. If local terminal work failed, its + // quiescence state remains and the unavailable fence must stay fail-closed. + if (!schemaQuiescence.has(quiesceMessage.quiesceId)) unavailableDatabases.delete(databaseName); + } + } else { + try { + await signalling.reconcileSchemaChange(message); + } finally { + // Reconciliation applies locally before waiting for peer acknowledgements. Once + // local state is resolved, a remote timeout must not keep intact storage fenced. + if (!schemaQuiescence.has(quiesceMessage.quiesceId)) unavailableDatabases.delete(databaseName); + } + } + } + }); + droppingDatabases.set(databaseName, completion); + return completion; +} +async function dropDatabaseOnce(databaseName, dbTables, rootStore, onQuiesced: (message: any) => void): Promise { // Hold the per-database restore lock across the entire drop so its file deletion can never // interleave with a restore's purge-and-copy on the same directory — a destroy landing after a // restore's copy would gut a "successful" restore, and vice versa. Restore takes the same lock @@ -1232,10 +1717,20 @@ export async function dropDatabase(databaseName) { try { for (const tableName in dbTables) { const table = dbTables[tableName]; - rootStore = table.primaryStore.rootStore; - if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); - lmdbDatabaseEnvs.delete(rootStore.path); - rocksdbDatabaseEnvs.delete(rootStore.path); + const tableRootStore = table.primaryStore.rootStore; + if (tableRootStore instanceof RocksDatabase) lockDatabaseForDrop(tableRootStore.path, databaseName, restoreLocks); + } + if (restoreLocks.length === 0 && rootStore instanceof RocksDatabase) + lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); + + const quiesceMessage = await signalling.quiesceSchemaChange( + new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, databaseName) + ); + onQuiesced(quiesceMessage); + await signalling.commitSchemaChange(quiesceMessage); + forgetDatabaseEnvironment(rootStore); + for (const tableName in dbTables) { + forgetDatabaseEnvironment(dbTables[tableName].primaryStore.rootStore); } for (const tableName in dbTables) { @@ -1249,28 +1744,16 @@ export async function dropDatabase(databaseName) { delete tables[DEFINED_TABLES]; } delete databases[databaseName]; + const definedDatabase = definedDatabases?.get(databaseName); + if (definedDatabase) (definedDatabase as any).rootStore = undefined; databaseEventsEmitter.emit('dropDatabase', databaseName); - if (rootStore) { - if (rootStore.status === 'open') { - if (rootStore instanceof RocksDatabase) { - rootStore.close(); - rootStore.destroy(); - } else { - await rootStore.close(); - await unlink(rootStore.path); - } - } - } else { - rootStore = database({ database: databaseName, table: null }); - // a tableless database resolves its root store here rather than in the loop above, so take - // the drop lock now (still before any destructive step) - if (rootStore instanceof RocksDatabase) lockDatabaseForDrop(rootStore.path, databaseName, restoreLocks); + if (rootStore.status === 'open') { if (rootStore instanceof RocksDatabase) { rootStore.close(); rootStore.destroy(); - } else if (rootStore.status === 'open') { + } else { await rootStore.close(); await unlink(rootStore.path); } @@ -1289,25 +1772,75 @@ export async function dropDatabase(databaseName) { * `resetDatabases()`/`getDatabases()` rescan reloads it (or skips it while a restore is in * progress, per the restore marker checks in the scan). */ -export function closeDatabase(databaseName: string): boolean { +const closingDatabases = new Map>(); + +export function closeDatabase(databaseName: string): Promise { + const activeClose = closingDatabases.get(databaseName); + if (activeClose) return activeClose; + const activeDrop = droppingDatabases.get(databaseName); + if (activeDrop) return activeDrop.then(() => true); + if (unavailableDatabases.has(databaseName)) { + return Promise.reject(new Error(`Database ${databaseName} is already closing`)); + } + unavailableDatabases.add(databaseName); + const completion = closeDatabaseOnce(databaseName) + .catch((error) => { + try { + resetQuiescedDatabase(databaseName); + } catch (recoveryError) { + throw new AggregateError([error, recoveryError], `Could not close or recover database ${databaseName}`); + } + throw error; + }) + .finally(() => { + closingDatabases.delete(databaseName); + unavailableDatabases.delete(databaseName); + }); + closingDatabases.set(databaseName, completion); + return completion; +} + +async function closeDatabaseOnce(databaseName: string): Promise { const dbTables = databases[databaseName]; if (!dbTables) return false; + const tableEntries: [string, any][] = []; + for (const tableName in dbTables) tableEntries.push([tableName, dbTables[tableName]]); const rootStores = new Set(); - const closeStore = (store: any, description: string) => { + const cleanupCompletions: Promise[] = []; + const closeErrors: unknown[] = []; + const closeStore = async (store: any, description: string) => { try { - store?.close?.(); + await store?.close?.(); } catch (error) { - logger.warn(`Error closing ${description} while closing database ${databaseName}:`, error); + closeErrors.push( + new Error(`Error closing ${description} while closing database ${databaseName}`, { cause: error }) + ); } }; - for (const tableName in dbTables) { - const table: any = dbTables[tableName]; + for (const [, table] of tableEntries) { if (!table?.primaryStore) continue; + const completion = table.quiesceForDrop?.() ?? table.cleanup?.(); + if (typeof completion?.then === 'function') cleanupCompletions.push(completion); if (table.primaryStore.rootStore) rootStores.add(table.primaryStore.rootStore); + } + const cleanupResults = await waitForTableCleanup(cleanupCompletions); + if (!cleanupResults) { + for (const [, table] of tableEntries) table.abortDropQuiesce?.(); + throw new Error( + `Timed out after ${databaseCloseTimeout}ms waiting for cleanup; refusing to close database ${databaseName} while cleanup is active.` + ); + } + const failures = cleanupFailures(cleanupResults); + if (failures.length) { + for (const [, table] of tableEntries) table.abortDropQuiesce?.(); + throw new AggregateError(failures, `Could not quiesce database ${databaseName} for close`); + } + for (const [tableName, table] of tableEntries) { + if (!table?.primaryStore) continue; for (const indexName in table.indices || {}) { - closeStore(table.indices[indexName], `index ${tableName}.${indexName}`); + await closeStore(table.indices[indexName], `index ${tableName}.${indexName}`); } - closeStore(table.primaryStore, `table ${tableName}`); + await closeStore(table.primaryStore, `table ${tableName}`); } // a database with no tables (an empty schema, or one whose tables were all dropped) still holds // an open root store, tracked only on the defined-database entry rather than any table — include @@ -1315,11 +1848,11 @@ export function closeDatabase(databaseName: string): boolean { const definedRoot = (definedDatabases?.get(databaseName) as any)?.rootStore; if (definedRoot) rootStores.add(definedRoot); for (const rootStore of rootStores) { - closeStore(rootStore.dbisDb, 'attributes store'); - closeStore(rootStore, 'root store'); - lmdbDatabaseEnvs.delete(rootStore.path); - rocksdbDatabaseEnvs.delete(rootStore.path); + await closeStore(rootStore.dbisDb, 'attributes store'); + await closeStore(rootStore, 'root store'); + forgetDatabaseEnvironment(rootStore); } + if (closeErrors.length) throw new AggregateError(closeErrors, `Could not close database ${databaseName}`); const definedDatabase = definedDatabases?.get(databaseName); if (definedDatabase) (definedDatabase as any).rootStore = undefined; if (databaseName === 'data') { @@ -1343,9 +1876,10 @@ export function closeDatabase(databaseName: string): boolean { * handles linger process-wide (and, e.g., block an online `restore_backup` from confirming the * database is closed). The `system` database is intentionally left open: it is non-enumerable here * (skipped by the loop), is never restored online, and the exiting worker may still touch the job - * table during teardown. Best-effort: closing failures are swallowed inside `closeDatabase`. + * table during teardown. A cleanup timeout is logged here and leaves the handles open rather than + * closing them while writes are active. */ -export function closeLoadedDatabases(): void { +export async function closeLoadedDatabases(): Promise { // snapshot the names first: closeDatabase() deletes from `databases` as it goes for (const databaseName of Object.keys(databases)) { const dbTables = databases[databaseName]; @@ -1362,7 +1896,13 @@ export function closeLoadedDatabases(): void { if (!isRocks && (definedDatabases?.get(databaseName) as any)?.rootStore instanceof RocksDatabase) { isRocks = true; } - if (isRocks) closeDatabase(databaseName); + if (isRocks) { + try { + await closeDatabase(databaseName); + } catch (error) { + logger.warn(`Could not close database ${databaseName} during worker shutdown:`, error); + } + } } } // HNSW_NO_AUTOVERSION kill-switch: when set, a NEW index initializes as legacy rather than @@ -1552,7 +2092,10 @@ export function table(tableDefinition: TableDefinition): Tabl attribute.name = attribute.attribute; attribute.indexed = true; } else attribute.attribute = attribute.name; - if (attribute.expiresAt) attribute.indexed = true; + if (attribute.expiresAt) { + attribute.indexed = true; + attribute.expirationIndexVersion = 1; + } } let hasChanges; let releaseExclusiveLock: () => void; @@ -1844,6 +2387,7 @@ export function table(tableDefinition: TableDefinition): Tabl const commonChanged = !attributeDescriptor || attributeDescriptor.type !== attribute.type || + attributeDescriptor.expirationIndexVersion !== attribute.expirationIndexVersion || attributeDescriptor.nullable !== attribute.nullable || attributeDescriptor.version !== attribute.version || attributeDescriptor.enumerable !== attribute.enumerable || @@ -2099,7 +2643,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { } let outstanding = 0; // this means that a new attribute has been introduced that needs to be indexed - for (const { key, value: record } of Table.primaryStore.getRange({ + for (const { key, value: record, expiresAt, metadataFlags } of Table.primaryStore.getRange({ start, lazy: attributesLength < 4, versions: true, @@ -2123,7 +2667,9 @@ async function runIndexing(Table, attributes, indicesToRemove) { const index = attribute.dbi; try { const resolver = attribute.resolve; - const value = record && (resolver ? resolver(record) : record[property]); + const value = attribute.expiresAt + ? effectiveExpirationTimestamp({ expiresAt, metadataFlags }, record, property) + : record && (resolver ? resolver(record) : record[property]); if (index.customIndex) { index.customIndex.index(key, value); didSynchronousIndexing = true; diff --git a/resources/replayLogs.ts b/resources/replayLogs.ts index b0caecad24..e4bf877d89 100644 --- a/resources/replayLogs.ts +++ b/resources/replayLogs.ts @@ -13,7 +13,7 @@ import { shouldAbortSlowReplay, REPLAY_WALL_CLOCK_LIMIT_MS, } from './replayLogsGuards.ts'; -import { purgeAgedLogs } from './auditStore.ts'; +import { HAS_EXPIRATION_DECISION, purgeAgedLogs } from './auditStore.ts'; import { get as envGet } from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; @@ -198,7 +198,14 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise transaction.isReplay = true; } context.transaction = transaction; - const options = { context, residencyId, nodeId, originatingOperation }; + const options = { + context, + residencyId, + nodeId, + originatingOperation, + expiresAt, + expirationDecisionPresent: Boolean(extendedType & HAS_EXPIRATION_DECISION), + }; writes++; switch (type) { case 'put': diff --git a/resources/search.ts b/resources/search.ts index c1f7dcafdf..1086c085fb 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -2,10 +2,10 @@ import { ClientError, IndexRebuildingError, Violation } from '../utility/errors/ import { OVERFLOW_MARKER, MAX_SEARCH_KEY_LENGTH, SEARCH_TYPES } from '../utility/lmdb/terms.ts'; import { compareKeys, MAXIMUM_KEY, writeKey } from 'ordered-binary'; import { SKIP } from '@harperfast/extended-iterable'; -import { INVALIDATED, EVICTED, freezeRecord } from './Table.ts'; +import { effectiveExpirationTimestamp, expirationTimestamp, INVALIDATED, EVICTED, freezeRecord } from './Table.ts'; import type { DirectCondition, Id } from './ResourceInterface.ts'; import { RequestTarget } from './RequestTarget.ts'; -import { lastMetadata } from './RecordEncoder.ts'; +import { lastMetadata, type Entry } from './RecordEncoder.ts'; import { recordAction } from './analytics/write'; import { RocksDatabase } from '@harperfast/rocksdb-js'; @@ -14,6 +14,11 @@ const OPEN_RANGE_ESTIMATE = 0.3; const BETWEEN_ESTIMATE = 0.1; const STARTS_WITH_ESTIMATE = 0.05; +function normalizeExpirationSearchValue(value: any): any { + if (Array.isArray(value)) return value.map(normalizeExpirationSearchValue); + return expirationTimestamp(value) ?? value; +} + function getStringPrefixUpperBound(prefix: string): Uint8Array { const maximumEncodedLength = prefix.length * 3 + 3; const encodedPrefix = new Uint8Array((maximumEncodedLength + 7) & ~3); @@ -327,6 +332,9 @@ export function searchByIndex( } const isPrimaryKey = attribute_name === Table.primaryKey || attribute_name == null; const index = isPrimaryKey ? Table.primaryStore : Table.indices[attribute_name]; + if (!isPrimaryKey && Table.expiresAtAttributeName !== undefined && Table.expiresAtAttributeName === attribute_name) { + value = normalizeExpirationSearchValue(value); + } let start; let end, inclusiveEnd, exclusiveStart, stringPrefix; if (value instanceof Date) value = value.getTime(); @@ -475,14 +483,15 @@ export function searchByIndex( if (isPrimaryKey) { const results = index.getRange(rangeOptions).map( filter - ? function ({ key, value }) { - if (this?.isSync) return value && filter(value) ? key : SKIP; + ? function (entry) { + const { key, value } = entry; + if (this?.isSync) return value && filter(value, entry) ? key : SKIP; // for filter operations, we intentionally yield the event turn so that scanning queries // do not hog resources return new Promise((resolve, reject) => setImmediate(() => { try { - resolve(value && filter(value) ? key : SKIP); + resolve(value && filter(value, entry) ? key : SKIP); } catch (error) { reject(error); } @@ -561,7 +570,7 @@ export function searchByIndex( const { key, value } = entry; if (this.isSync) { recordRead(entry); - return value && filter(value) ? key : SKIP; + return value && filter(value, entry) ? key : SKIP; } // for filter operations, we intentionally yield the event turn so that scanning queries // do not hog resources @@ -569,7 +578,7 @@ export function searchByIndex( setImmediate(() => { try { recordRead(entry); - resolve(value && filter(value) ? key : SKIP); + resolve(value && filter(value, entry) ? key : SKIP); } catch (error) { reject(error); } @@ -622,13 +631,23 @@ function joinTo(rightIterable, attribute, store, isManyToMany, joined: Map !filter(record))) continue; + if (filters?.some((filter) => !filter(record, storedEntry))) continue; if (isManyToMany) { for (let i = 0; i < leftKey.length; i++) { addEntry(leftKey[i], entry); @@ -702,11 +721,12 @@ function joinFrom(rightIterable, attribute, store, joined: Map, sear }; //let i = 0; // get all the ids of the related records - for (const id of rightIterable) { + for (const idOrEntry of rightIterable) { + const id = idOrEntry?.key ?? idOrEntry; if ((joined as any).filters) { // if additional filters are defined, we need to check them - const record = store.getSync(id); - if ((joined as any).filters.some((filter) => !filter(record))) continue; + const entry = idOrEntry?.value !== undefined ? idOrEntry : store.getEntry(id); + if ((joined as any).filters.some((filter) => !filter(entry?.value, entry))) continue; } ids.add(id); // TODO: Re-enable this when async iteration is used, and do so with manually iterating so that we don't need to do an await on every iteration @@ -947,6 +967,9 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar return recordFilter; } } + const normalizeExpirationValue = + !isPrimaryKey && Table?.expiresAtAttributeName !== undefined && Table.expiresAtAttributeName === attribute; + if (normalizeExpirationValue) value = normalizeExpirationSearchValue(value); if (value instanceof Date) value = value.getTime(); let baseFilter; @@ -1051,6 +1074,8 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar canUseIndex?: boolean, allowObjectMatching?: boolean ) { + const normalizeRecordExpiration = + !isPrimaryKey && Table?.expiresAtAttributeName !== undefined && Table.expiresAtAttributeName === attribute; let thresholdRemainingMisses: number; canUseIndex = canUseIndex && // is it a comparator that makes sense to use index @@ -1066,10 +1091,15 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar } let misses = 0; let filteredSoFar = 3; // what we use to calculate miss rate; we give some buffer so we don't jump to indexed retrieval too quickly - function recordFilter(record: any) { + function recordFilter(record: any, entry?: Entry) { // `record` may be null/undefined when called via a nested-path filter // where an intermediate property is missing. - const value = record == null ? undefined : record[attribute]; + let value = normalizeRecordExpiration + ? effectiveExpirationTimestamp(entry, record, attribute) + : record == null + ? undefined + : record[attribute]; + if (normalizeRecordExpiration) value = normalizeExpirationSearchValue(value); let matches: boolean; if (typeof value !== 'object' || !value || allowObjectMatching) matches = filter(value); else if (Array.isArray(value)) matches = value.some(filter); @@ -1168,7 +1198,10 @@ export function estimateCondition(table) { } else { // we only attempt to estimate count on equals operator because that's really all that LMDB supports (some other key-value stores like libmdbx could be considered if we need to do estimated counts of ranges at some point) const index = table.indices[attribute_name]; - condition.estimated_count = index ? index.getValuesCount(condition[1] ?? condition.value) : Infinity; + let value = condition[1] ?? condition.value; + if (table.expiresAtAttributeName !== undefined && table.expiresAtAttributeName === attribute_name) + value = normalizeExpirationSearchValue(value); + condition.estimated_count = index ? index.getValuesCount(value) : Infinity; } } else if (searchType === 'contains' || searchType === 'ends_with' || searchType === 'ne') { const attribute_name = condition[0] ?? condition.attribute; @@ -1183,8 +1216,10 @@ export function estimateCondition(table) { if (Array.isArray(condition.value) && index) { // Sum of per-value matches (over-counts duplicates but is a fine ceiling) let estimate = 0; + const normalizeExpiration = + table.expiresAtAttributeName !== undefined && table.expiresAtAttributeName === attribute_name; for (const item of condition.value) { - estimate += index.getValuesCount(item); + estimate += index.getValuesCount(normalizeExpiration ? normalizeExpirationSearchValue(item) : item); } condition.estimated_count = estimate; } else if (Array.isArray(condition.value)) { diff --git a/resources/transaction.ts b/resources/transaction.ts index b2dd2abcdf..7550165581 100644 --- a/resources/transaction.ts +++ b/resources/transaction.ts @@ -66,11 +66,14 @@ export function transaction( return onComplete(result); // when the transaction function completes, run this to commit the transaction function onComplete(result) { - const committed = transaction.commit({ doneWriting: true }); + let committed; + try { + committed = transaction.commit({ doneWriting: true }); + } catch (error) { + return onError(error); + } if ((committed as any).then) { - return (committed as any).then(() => { - return result; - }); + return (committed as any).then(() => result, onError); } else { return result; } diff --git a/schema.graphql b/schema.graphql index 3820724af5..03eded2414 100644 --- a/schema.graphql +++ b/schema.graphql @@ -163,7 +163,8 @@ Allows enumeration over a computed field, causing it to be included in serialize directive @enumerable on FIELD_DEFINITION """ -Flags the field as containing the expiration time of the entry. +Flags the field as containing the expiration time of the entry. The index follows the effective stored expiration; +an explicit write-context expiration can override the serialized field value. """ directive @expiresAt on FIELD_DEFINITION diff --git a/security/certificateVerification/certificateVerificationSource.ts b/security/certificateVerification/certificateVerificationSource.ts index 9f304c2c33..4402f4e8c2 100644 --- a/security/certificateVerification/certificateVerificationSource.ts +++ b/security/certificateVerification/certificateVerificationSource.ts @@ -71,6 +71,7 @@ export class CertificateVerificationSource extends Resource { // Handle result consistently const expiresAt = Date.now() + methodConfig.cacheTtl; + context.expiresAt = expiresAt; return { certificate_id: id, diff --git a/security/certificateVerification/crlVerification.ts b/security/certificateVerification/crlVerification.ts index decef6a2cc..b0ce208e01 100644 --- a/security/certificateVerification/crlVerification.ts +++ b/security/certificateVerification/crlVerification.ts @@ -74,6 +74,7 @@ class CertificateRevocationListSource extends Resource { const crlExpiry = result.next_update; const configExpiry = Date.now() + config.cacheTtl; const expiresAt = Math.min(crlExpiry, configExpiry); + context.expiresAt = expiresAt; return { ...result, @@ -90,6 +91,7 @@ class CertificateRevocationListSource extends Resource { if (config.failureMode === 'fail-closed') { // Cache the error for faster recovery const expiresAt = Date.now() + ERROR_CACHE_TTL; + context.expiresAt = expiresAt; return { crl_id: id, diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.js index 9b48ef3266..0344a1df5f 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.js @@ -12,7 +12,22 @@ const harperBridge = require('../../dataLayer/harperBridge/harperBridge.ts'); const process = require('process'); const { isMainThread, workerData } = require('worker_threads'); -const { resetDatabases, closeDatabase } = require('../../resources/databases.ts'); +const { + resetDatabases, + closeDatabase, + quiesceSchemaTarget, + abortSchemaQuiesce, + commitSchemaQuiesce, + renewSchemaQuiesce, + finishSchemaQuiesce, + completeSchemaQuiesce, + failSchemaQuiesceFinalization, +} = require('../../resources/databases.ts'); +const { + holdWorkerStartsForSchema, + releaseWorkerStartsForSchema, + isThreadConnected, +} = require('../threads/manageThreads.js'); /** * This object/functions are passed to the ITC client instance and dynamically added as event handlers. @@ -38,6 +53,75 @@ const serverItcHandlers = { * @returns {Promise} */ const schemaListeners = []; +const MAX_SCHEMA_TERMINAL_OUTCOMES = 1024; +const schemaTerminalCompletions = new Map(); +const schemaTerminalOutcomes = new Map(); +const schemaWorkerBarrierLeases = new Map(); +const SCHEMA_WORKER_BARRIER_RECHECK_MS = 1_000; + +function expireSchemaWorkerBarrierLease(message) { + // The origin can legitimately block its event loop during synchronous RocksDB destruction. + // Keep replacement workers fenced until that owner disconnects or sends a terminal message. + if (isThreadConnected(message.originator)) { + const timer = setTimeout(() => expireSchemaWorkerBarrierLease(message), SCHEMA_WORKER_BARRIER_RECHECK_MS); + timer.unref(); + schemaWorkerBarrierLeases.set(message.quiesceId, timer); + return; + } + schemaWorkerBarrierLeases.delete(message.quiesceId); + releaseWorkerStartsForSchema(message.quiesceId); +} + +function armSchemaWorkerBarrierLease(message) { + if (!isMainThread) return; + const existing = schemaWorkerBarrierLeases.get(message.quiesceId); + if (existing) clearTimeout(existing); + const delay = Math.max(0, (message.leaseUntil ?? Date.now()) - Date.now()); + const timer = setTimeout(() => expireSchemaWorkerBarrierLease(message), delay); + timer.unref(); + schemaWorkerBarrierLeases.set(message.quiesceId, timer); +} + +function commitSchemaWorkerBarrier(message) { + if (!isMainThread) return; + armSchemaWorkerBarrierLease(message); +} + +function releaseSchemaWorkerBarrier(message) { + if (!isMainThread) return; + const timer = schemaWorkerBarrierLeases.get(message.quiesceId); + if (timer) clearTimeout(timer); + schemaWorkerBarrierLeases.delete(message.quiesceId); + releaseWorkerStartsForSchema(message.quiesceId); +} + +function sameSchemaTerminalRequest(entry, message) { + return ( + entry.phase === message.phase && + entry.operation === message.operation && + entry.schema === message.schema && + entry.table === message.table + ); +} + +function schemaTerminalEntry(message, value) { + return { + phase: message.phase, + operation: message.operation, + schema: message.schema, + table: message.table, + value, + }; +} + +function retainSchemaTerminalOutcome(message, result) { + schemaTerminalOutcomes.set(message.quiesceId, schemaTerminalEntry(message, result)); + while (schemaTerminalOutcomes.size > MAX_SCHEMA_TERMINAL_OUTCOMES) { + const oldestId = schemaTerminalOutcomes.keys().next().value; + schemaTerminalOutcomes.delete(oldestId); + } +} + async function schemaHandler(event) { const validate = validateEvent(event); if (validate) { @@ -46,21 +130,92 @@ async function schemaHandler(event) { } hdbLogger.trace(`ITC schemaHandler received schema event:`, event); - // restore_backup: this thread must release its store handles so the restore can purge and - // rewrite the database directory. The rescan below (resetDatabases) skips reloading it while - // the restoring marker is present, and reloads it on the completion signal (marker gone). - if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP && event.message.schema) { - closeDatabase(event.message.schema); + if (event.message?.phase === 'hold-worker-starts') { + holdWorkerStartsForSchema(event.message.quiesceId); + armSchemaWorkerBarrierLease(event.message); + return { held: true }; + } + if (event.message?.phase === 'release-worker-starts') { + releaseSchemaWorkerBarrier(event.message); + return { released: true }; + } + if (event.message?.phase === 'quiesce') return quiesceSchemaTarget(event.message); + if (event.message?.phase === 'renew-quiesce') { + const result = renewSchemaQuiesce(event.message); + if (result.quiesced) armSchemaWorkerBarrierLease(event.message); + return result; } - await cleanLmdbMap(event.message); - await syncSchemaMetadata(event.message); - for (let listener of schemaListeners) { + if (event.message?.phase === 'commit-quiesce') { + const result = await commitSchemaQuiesce(event.message); + if (result.committed) commitSchemaWorkerBarrier(event.message); + return result; + } + if (event.message?.phase === 'abort-quiesce') { + await abortSchemaQuiesce(event.message); + return { aborted: true }; + } + const terminalPhase = + event.message?.phase === 'finalize-quiesce' + ? 'finalized' + : event.message?.phase === 'reconcile-quiesce' + ? 'reconciled' + : undefined; + if (terminalPhase) { + const quiesceId = event.message.quiesceId; + const outcome = schemaTerminalOutcomes.get(quiesceId); + if (outcome) return sameSchemaTerminalRequest(outcome, event.message) ? outcome.value : { [terminalPhase]: false }; + const active = schemaTerminalCompletions.get(quiesceId); + if (active) return sameSchemaTerminalRequest(active, event.message) ? active.value : { [terminalPhase]: false }; + const completion = applySchemaChange(event, terminalPhase); + schemaTerminalCompletions.set(quiesceId, schemaTerminalEntry(event.message, completion)); try { - listener(event?.message); - } catch (err) { - hdbLogger.error(err); + const result = await completion; + if (result?.[terminalPhase] === true) retainSchemaTerminalOutcome(event.message, result); + return result; + } finally { + if (schemaTerminalCompletions.get(quiesceId)?.value === completion) schemaTerminalCompletions.delete(quiesceId); } } + return applySchemaChange(event); +} + +async function applySchemaChange(event, terminalPhase) { + if (terminalPhase && !finishSchemaQuiesce(event.message)) return { [terminalPhase]: false }; + try { + // restore_backup: this thread must release its store handles so the restore can purge and + // rewrite the database directory. The rescan below (resetDatabases) skips reloading it while + // the restoring marker is present, and reloads it on the completion signal (marker gone). + if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP && event.message.schema) { + for (let attempt = 0; attempt < 2; attempt++) { + try { + await closeDatabase(event.message.schema); + break; + } catch (error) { + hdbLogger.warn( + `Could not close database ${event.message.schema} for restore${attempt === 0 ? '; retrying once' : ''}:`, + error + ); + if (attempt === 1) return; + } + } + } + await cleanLmdbMap(event.message); + await syncSchemaMetadata(event.message, Boolean(terminalPhase)); + for (let listener of schemaListeners) { + try { + listener(event?.message); + } catch (err) { + hdbLogger.error(err); + } + } + if (terminalPhase) { + await completeSchemaQuiesce(event.message); + return { [terminalPhase]: true }; + } + } catch (error) { + if (terminalPhase) failSchemaQuiesceFinalization(event.message); + throw error; + } } schemaHandler.addListener = function (listener) { @@ -74,7 +229,7 @@ schemaHandler.addListener = function (listener) { * @param msg * @returns {Promise} */ -async function syncSchemaMetadata(msg) { +async function syncSchemaMetadata(msg, strict = false) { try { // TODO: Eventually should indicate which database/table changed so we don't have to scan everything let databases = resetDatabases(); @@ -82,6 +237,7 @@ async function syncSchemaMetadata(msg) { // wait for a write to finish to ensure all writes have been written await databases[msg.database][msg.table].put(Symbol.for('write-verify'), null); } catch (e) { + if (strict) throw e; hdbLogger.error(e); } } diff --git a/server/jobs/jobProcess.ts b/server/jobs/jobProcess.ts index 9688a3c32b..c02127507c 100644 --- a/server/jobs/jobProcess.ts +++ b/server/jobs/jobProcess.ts @@ -91,7 +91,7 @@ const JOB_ID = JOB_NAME.substring(4); // effort — never let cleanup mask the job result. try { const { closeLoadedDatabases } = await import('../../resources/databases.ts'); - closeLoadedDatabases(); + await closeLoadedDatabases(); } catch (closeErr) { harperLogger.warn('Error releasing database handles on job worker exit:', closeErr); } diff --git a/server/jobs/jobRunner.ts b/server/jobs/jobRunner.ts index 4c1d7c9549..84dbfa4001 100644 --- a/server/jobs/jobRunner.ts +++ b/server/jobs/jobRunner.ts @@ -143,6 +143,7 @@ async function runJob(runnerMessage: any, operation: any) { async function launchJobThread(job_id: any) { log.trace('launching job thread:', job_id); if (isMainThread) { + await threadsStart.waitForSchemaWorkerStarts(); threadsStart.startWorker(join(__dirname, './jobProcess.js'), { autoRestart: false, name: 'job', @@ -158,6 +159,7 @@ async function launchJobThread(job_id: any) { if (isMainThread) { onMessageByType(hdbTerms.ITC_EVENT_TYPES.START_JOB, async (message) => { try { + await threadsStart.waitForSchemaWorkerStarts(); threadsStart.startWorker(join(__dirname, './jobProcess.js'), { autoRestart: false, name: 'job', diff --git a/server/threads/itc.js b/server/threads/itc.js index 992237f4e4..5a3cd1b0e6 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.js @@ -3,6 +3,7 @@ const hdbUtils = require('../../utility/common_utils.ts'); const hdbTerms = require('../../utility/hdbTerms.ts'); const { ITC_ERRORS } = require('../../utility/errors/commonErrors.ts'); +const harperLogger = require('../../utility/logging/harper_logger.ts'); const { threadId } = require('worker_threads'); const { onMessageFromWorkers, broadcastWithAcknowledgement } = require('./manageThreads.js'); @@ -13,29 +14,45 @@ module.exports = { UserEventMsg, }; let serverItcHandlers; -onMessageFromWorkers(async (event, sender) => { +onMessageFromWorkers((event, sender) => { + handleItcEvent(event, sender).catch((error) => { + harperLogger.error(`ITC event handler failed for event type ${event?.type}`, error); + sendAcknowledgement(event, sender, { handled: false, reason: error?.message }); + }); +}); +async function handleItcEvent(event, sender) { serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); validateEvent(event); + let result; if (serverItcHandlers[event.type]) { - await serverItcHandlers[event.type](event); + result = await serverItcHandlers[event.type](event); } - if (event.requestId && sender) - sender.postMessage({ + sendAcknowledgement(event, sender, result); +} +function sendAcknowledgement(event, sender, result) { + if (!event?.requestId || !sender) return; + try { + const acknowledgement = { type: 'ack', id: event.requestId, - }); -}); + }; + if (event.includeAcknowledgementResult === true) acknowledgement.result = result; + sender.postMessage(acknowledgement); + } catch (error) { + harperLogger.error(`Unable to acknowledge ITC event type ${event?.type}`, error); + } +} /** * Emits an ITC event to the ITC server. * @param event */ -function sendItcEvent(event) { +function sendItcEvent(event, acknowledgementOptions = undefined) { // Always stamp originator so handlers can send direct responses back. // The main thread's threadId is 0 (worker_threads convention); parentPort.threadId // is set to 0 in workers, so sendToThread(0, ...) routes back to main. if (event.message) event.message.originator = threadId; - return broadcastWithAcknowledgement(event); + return broadcastWithAcknowledgement(event, acknowledgementOptions); } /** diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index 87d829c9c3..4e2cd658c5 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -22,25 +22,34 @@ const { _assignPackageExport } = require('../../globals.js'); const { PACKAGE_ROOT } = require('../../utility/packageUtils.js'); const { resolvePreloadModules } = require('./resolvePreload.ts'); const { getConfigPath } = require('../../config/configUtils.ts'); -let importModules; -function getImportModules() { - if (importModules === undefined) - importModules = resolvePreloadModules( - envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_PRELOAD), - getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), - 'threads.preload' +const importModuleCache = { modules: [] }; +const requireModuleCache = { modules: [] }; +function getPreloadModules(configParam, configKey, cache) { + try { + const configured = envMgr.get(configParam); + const componentsRoot = getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT); + const key = JSON.stringify([configured, componentsRoot, process.env.RUN_HDB_APP]); + if (cache.key !== key) { + const modules = resolvePreloadModules(configured, componentsRoot, configKey); + cache.modules = modules; + const specifierCount = (Array.isArray(configured) ? configured : [configured]).filter( + (specifier) => typeof specifier === 'string' && specifier.length > 0 + ).length; + if (modules.length === specifierCount) cache.key = key; + } + } catch (error) { + harperLogger.error( + `Unable to resolve ${configKey} modules for worker startup; keeping the last known resolution`, + error ); - return importModules; + } + return cache.modules; +} +function getImportModules() { + return getPreloadModules(hdbTerms.CONFIG_PARAMS.THREADS_PRELOAD, 'threads.preload', importModuleCache); } -let requireModules; function getRequireModules() { - if (requireModules === undefined) - requireModules = resolvePreloadModules( - envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE), - getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), - 'threads.preloadRequire' - ); - return requireModules; + return getPreloadModules(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE, 'threads.preloadRequire', requireModuleCache); } const chokidar = require('chokidar'); const isBun = typeof globalThis.Bun !== 'undefined'; @@ -72,6 +81,13 @@ const UNREGISTER_PROCESS_GROUP = 'unregister-process-group'; const AWAIT_PROCESS_GROUP_TERMINATION = 'await-process-group-termination'; const PROCESS_GROUP_TERMINATION_CONFIRMED = 'process-group-termination-confirmed'; const THREAD_INFO_REQUEST_TIMEOUT_MS = 1000; +class ITCAcknowledgementError extends Error { + constructor(message, results) { + super(message); + this.code = 'ERR_ITC_ACKNOWLEDGEMENT'; + this.results = results; + } +} let getThreadInfo; let awaitProcessGroupTermination; // Worker-side backstop that force-exits if the graceful shutdown sequence doesn't finish in time. @@ -130,6 +146,39 @@ function restoreShutdownDeadline() { const listenersByType = new Map(); const messagesQueuedByType = new Map(); +const schemaWorkerStartBarriers = new Set(); +let schemaWorkerStartWaiters = []; + +function holdWorkerStartsForSchema(quiesceId) { + if (isMainThread) schemaWorkerStartBarriers.add(quiesceId); +} + +function releaseWorkerStartsForSchema(quiesceId) { + if (!isMainThread || !schemaWorkerStartBarriers.delete(quiesceId) || schemaWorkerStartBarriers.size) return; + const waiters = schemaWorkerStartWaiters; + schemaWorkerStartWaiters = []; + for (const resolve of waiters) resolve(); +} + +function waitForSchemaWorkerStarts() { + if (!isMainThread || schemaWorkerStartBarriers.size === 0) return Promise.resolve(); + return new Promise((resolve) => schemaWorkerStartWaiters.push(resolve)); +} + +function isThreadConnected(ownerThreadId) { + // Schema teardown callers need a synchronous local-port check because their event loop can be + // blocked by native destruction. Every worker is connected to its siblings; an unstamped legacy + // message must remain fenced rather than guessing that an unknown owner has exited. + if (!Number.isInteger(ownerThreadId)) return true; + if (ownerThreadId === threadId || ownerThreadId === 0) return true; + return connectedPorts.some((port) => port.threadId === ownerThreadId); +} + +function startAfterSchemaWorkerBarrier(start, description) { + waitForSchemaWorkerStarts() + .then(start) + .catch((error) => harperLogger.error(`Could not ${description} after the schema worker-start barrier:`, error)); +} module.exports = { startWorker, @@ -142,6 +191,7 @@ module.exports = { onMessageByType, broadcast, broadcastWithAcknowledgement, + ITCAcknowledgementError, getWorkerIndex, getWorkerCount, getTicketKeys, @@ -149,6 +199,10 @@ module.exports = { setTerminateTimeout, extendShutdownDeadline, restoreShutdownDeadline, + holdWorkerStartsForSchema, + releaseWorkerStartsForSchema, + waitForSchemaWorkerStarts, + isThreadConnected, registerWorkerDataProvider, onThreadExit, registerProcessGroup, @@ -356,8 +410,8 @@ function startWorker(path, options = {}) { if (!isBun && envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_HEAPSNAPSHOTNEARLIMIT)) execArgv.push('--heapsnapshot-near-heap-limit=1'); // Preload configured modules (e.g. an APM agent like dd-trace) before the worker's entry - // script so they can instrument all subsequent Harper and app module loads. Resolved once - // (config and installed components are fixed for the process lifetime). `threads.preload` + // script so they can instrument all subsequent Harper and app module loads. Resolution is + // cached until its configuration or resolution roots change. `threads.preload` // uses --import (ESM/loader-hook registration, e.g. dd-trace/register.js — the entry that // instruments worker threads); `threads.preloadRequire` uses --require for CJS agents that // document that path (e.g. dd-trace/init, Dynatrace OneAgent). --import is URL-based, so @@ -424,7 +478,7 @@ function startWorker(path, options = {}) { // if this wasn't an intentional shutdown, restart now (unless we have tried too many times) if (worker.unexpectedRestarts < MAX_UNEXPECTED_RESTARTS) { options.unexpectedRestarts = worker.unexpectedRestarts + 1; - startWorker(path, options); + startAfterSchemaWorkerBarrier(() => startWorker(path, options), `restart worker ${options.workerIndex}`); } else harperLogger.error(`Thread has been restarted ${worker.restarts} times and will not be restarted`); } }); @@ -503,6 +557,7 @@ async function restartWorkers( // replacement, startWorker's unexpected-exit handler must not auto-restart it (that would // leave a duplicate once the replacement is up). Restored below if the replacement fails. worker.wasShutdown = true; + await waitForSchemaWorkerStarts(); let newWorker = worker.startCopy(); // Likewise suppress auto-restart on the replacement *while it boots*: if it fails to come up // we leave the existing worker in place, and a background retry succeeding later would push the @@ -575,7 +630,10 @@ async function restartWorkers( // Overlapping types we couldn't pre-start (Windows/Bun): start the replacement now that the old // worker is releasing its port. server.close() stops accepting immediately, so the port frees up // well before the replacement finishes booting and binds. - if (overlapping && startReplacementThreads && !canPreStartReplacement) worker.startCopy(); + if (overlapping && startReplacementThreads && !canPreStartReplacement) { + await waitForSchemaWorkerStarts(); + worker.startCopy(); + } let whenDone = new Promise((resolve) => { // in case the exit inside the thread doesn't timeout, force it from the outside const armTerminate = (delay) => @@ -612,7 +670,8 @@ async function restartWorkers( const index = waitingToFinish.indexOf(whenDone); if (index > -1) waitingToFinish.splice(index, 1); // non-overlapping types have no advance replacement, so start it once the old one is gone - if (!overlapping && startReplacementThreads) worker.startCopy(); + if (!overlapping && startReplacementThreads) + startAfterSchemaWorkerBarrier(() => worker.startCopy(), `start replacement worker ${worker.threadId}`); resolve(); }); }); @@ -690,14 +749,25 @@ async function broadcast(message, includeSelf) { const awaitingResponses = new Map(); let nextId = 1; // Backstop so a wedged-but-alive worker (one whose event loop is blocked and never acks, yet -// whose port hasn't closed) can't hang a mutating admin/DDL op forever. The durable write has -// already succeeded by the time we broadcast, and the health monitor restarts a truly stuck -// worker (its port close fires the same ack handlers), so on timeout we proceed best-effort. +// whose port hasn't closed) can't hang a best-effort broadcast forever. Strict callers receive +// the timeout as a failure instead of proceeding. const DEFAULT_ACK_TIMEOUT_MS = 30000; -function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) { - return new Promise((resolve) => { +function broadcastWithAcknowledgement(message, timeoutOrOptions = DEFAULT_ACK_TIMEOUT_MS) { + const options = + typeof timeoutOrOptions === 'number' ? { timeout: timeoutOrOptions } : timeoutOrOptions || Object.create(null); + const timeout = options.timeout ?? DEFAULT_ACK_TIMEOUT_MS; + const acceptResult = options.acceptResult; + const includeJobWorkers = options.includeJobWorkers === true; + if (acceptResult !== undefined && typeof acceptResult !== 'function') { + throw new TypeError('acceptResult must be a function'); + } + if (acceptResult) message.includeAcknowledgementResult = true; + else delete message.includeAcknowledgementResult; + return new Promise((resolve, reject) => { let waitingCount = 0; let timer; + let prepared = false; + const results = []; // Tracks the handlers still awaiting an ack for THIS broadcast. Doubles as an // idempotency guard: a port's handler runs at most once whether it's driven by an ack, // the close listener, or the timeout below. @@ -707,20 +777,46 @@ function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) clearTimeout(timer); timer = undefined; } - resolve(); + if (!acceptResult) return resolve(); + const failures = results.filter((result) => result.status !== 'accepted'); + if (failures.length === 0) return resolve(results); + reject( + new ITCAcknowledgementError( + `ITC broadcast (type ${message.type}) failed acknowledgement from ${failures.length} worker thread(s)`, + results + ) + ); }; for (let port of connectedPorts) { // Job workers run a single isolated task and exit; they don't participate in // schema-change gossip. Including them causes a deadlock: the broadcast waits for // the job worker's ACK while the job worker's event loop is busy waiting for the // same broadcast to complete (re-entrant schema change triggered by the job op). - if (port.isJobWorker) continue; + if (port.isJobWorker && !includeJobWorkers) continue; try { let requestId = nextId++; - const ackHandler = () => { + const ackHandler = (status, result, error) => { if (!pending.delete(ackHandler)) return; // already settled for this port awaitingResponses.delete(requestId); - if (--waitingCount === 0) { + if (acceptResult) { + if (status === 'acknowledged') { + let accepted = false; + try { + accepted = acceptResult(result) === true; + } catch (acceptError) { + error = acceptError; + } + results.push({ + threadId: port.threadId, + status: accepted ? 'accepted' : 'rejected', + result, + ...(error ? { error } : null), + }); + } else { + results.push({ threadId: port.threadId, status, ...(error ? { error } : null) }); + } + } + if (--waitingCount === 0 && prepared) { finish(); } if (port !== parentPort && --port.refCount === 0) { @@ -732,35 +828,39 @@ function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) port.ref(); port.refCount = (port.refCount || 0) + 1; awaitingResponses.set((message.requestId = requestId), ackHandler); + waitingCount++; if (!port.hasAckCloseListener) { // just set a single close listener that can clean up all the ack handlers for a port that is closed port.hasAckCloseListener = true; port.on(port.close ? 'close' : 'exit', () => { for (let [, ackHandler] of awaitingResponses) { if (ackHandler.port === port) { - ackHandler(); + ackHandler('closed'); } } }); } port.postMessage(message); - waitingCount++; } catch (error) { - harperLogger.error(`Unable to send message to worker`, error); + const ackHandler = awaitingResponses.get(message.requestId); + if (ackHandler?.port === port) ackHandler('transport-error', undefined, error); + if (!acceptResult) harperLogger.error(`Unable to send message to worker`, error); } } - if (waitingCount === 0) return resolve(); + prepared = true; + if (waitingCount === 0) return finish(); if (timeout > 0) { timer = setTimeout(() => { timer = undefined; const stuck = []; for (let ackHandler of [...pending]) { stuck.push(ackHandler.port?.threadId); - ackHandler(); // same cleanup path as an ack/close; drives waitingCount to 0 and resolves + ackHandler('timeout'); // same cleanup path as an ack/close; drives waitingCount to 0 and resolves } - harperLogger.warn( - `ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; proceeding best-effort` - ); + if (!acceptResult) + harperLogger.warn( + `ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; proceeding best-effort` + ); }, timeout); timer.unref?.(); } @@ -1157,7 +1257,7 @@ function addPort(port, keepRef, isJobWorker) { } else if (message.type === ACKNOWLEDGEMENT) { let completion = awaitingResponses.get(message.id); if (completion) { - completion(); + completion('acknowledged', message.result); } } else if (message.type === REMOVE_PORT) { const idx = connectedPorts.findIndex((p) => p.threadId === message.threadId); diff --git a/server/threads/resolvePreload.ts b/server/threads/resolvePreload.ts index fab3a8a42b..7af71a8172 100644 --- a/server/threads/resolvePreload.ts +++ b/server/threads/resolvePreload.ts @@ -12,8 +12,7 @@ import { PACKAGE_ROOT } from '../../utility/packageUtils.js'; // for `--import`, or `dd-trace/init` for `--require`) bundled in a deployed component can be // preloaded before any Harper or app module in a worker thread — the only point early enough for // an APM agent to instrument subsequent module loads. `configKey` only labels warnings. The -// caller memoizes the result since the config and installed components are fixed for the process -// lifetime. +// caller memoizes the result until the config or resolution roots change. export function resolvePreloadModules( configured: unknown, componentsRoot: string | undefined, diff --git a/storage-format.md b/storage-format.md index a10900c178..82b405c321 100644 --- a/storage-format.md +++ b/storage-format.md @@ -86,15 +86,38 @@ Variable-length integer encoding the operation type and extended type flags. **Additional Flags (upper bits):** -| Value | Name | Description | -| ------------- | ---------------------------- | ----------------------------------- | -| 64 (0x40) | HAS_PREVIOUS_VERSION | Previous version timestamp included | -| 128 (0x80) | HAS_EXTENDED_TYPE | Extended type information | -| 512 (0x200) | HAS_CURRENT_RESIDENCY_ID | Current residency ID included | -| 1024 (0x400) | HAS_PREVIOUS_RESIDENCY_ID | Previous residency ID included | -| 2048 (0x800) | HAS_ORIGINATING_OPERATION | Originating operation type included | -| 4096 (0x1000) | HAS_EXPIRATION_EXTENDED_TYPE | Expiration timestamp included | -| 8192 (0x2000) | HAS_BLOBS | Binary blob data included | +| Value | Name | Description | +| --------------- | ---------------------------- | ------------------------------------ | +| 64 (0x40) | HAS_PREVIOUS_VERSION | Previous version timestamp included | +| 128 (0x80) | HAS_EXTENDED_TYPE | Extended type information | +| 512 (0x200) | HAS_CURRENT_RESIDENCY_ID | Current residency ID included | +| 1024 (0x400) | HAS_PREVIOUS_RESIDENCY_ID | Previous residency ID included | +| 2048 (0x800) | HAS_ORIGINATING_OPERATION | Originating operation type included | +| 4096 (0x1000) | HAS_EXPIRATION_EXTENDED_TYPE | Expiration timestamp included | +| 8192 (0x2000) | HAS_BLOBS | Binary blob data included | +| 65536 (0x10000) | HAS_EXPIRATION_DECISION | Expiration decision is authoritative | + +`HAS_EXPIRATION_DECISION` carries no payload. It is mirrored in the primary record metadata and audit +entry so a missing `HAS_EXPIRATION_EXTENDED_TYPE` bit means an explicit no-expiration decision rather +than missing legacy metadata. Records and audit events without the decision bit retain the legacy +fallback to a public `@expiresAt` field during upgrades. + +#### `@expiresAt` migration and mixed-version compatibility + +Source-backed tables now treat the source request context as authoritative for cache expiration. A +source that intends a returned record to expire must set `context.expiresAt`; an `@expiresAt` field in +the returned record is application data and no longer schedules expiration by itself. Omitting +`context.expiresAt` records an explicit no-expiration decision, even when the returned record contains +an `@expiresAt` field. This is a breaking change for source adapters that previously relied on that +returned field: update them to set `context.expiresAt` before upgrading. + +New readers remain compatible with records written before `HAS_EXPIRATION_DECISION`: when the decision +bit is absent, they fall back to the public `@expiresAt` field. The reverse is not safe during a rolling +mixed-version deployment. An older worker does not understand an explicit no-expiration decision and +can reinterpret the public field as an expiration. Do not run mixed versions for source-backed tables +that return an `@expiresAt` field while intentionally omitting `context.expiresAt`; upgrade those +workers together, or remove/neutralize the returned field until every worker understands the decision +bit. ### Variable-Length Integer Encoding diff --git a/unitTests/resources/auditEntryRecordFlags.test.js b/unitTests/resources/auditEntryRecordFlags.test.js index dc002782b1..8330656f9b 100644 --- a/unitTests/resources/auditEntryRecordFlags.test.js +++ b/unitTests/resources/auditEntryRecordFlags.test.js @@ -2,7 +2,7 @@ require('../testUtils'); const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); const { table } = require('#src/resources/databases'); -const { createAuditEntry, readAuditEntry } = require('#src/resources/auditStore'); +const { createAuditEntry, readAuditEntry, HAS_EXPIRATION_DECISION } = require('#src/resources/auditStore'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { transaction } = require('#src/resources/transaction'); @@ -26,6 +26,25 @@ describe('Audit entry record flags match the body (#2153)', () => { expiresAt: 1817915930237, }; + it('round-trips an explicit no-expiration decision without adding payload bytes', () => { + const localOnlyEntry = createAuditEntry({ + ...baseRecord, + extendedType: 0x8000, + expiresAt: undefined, + encodedRecord: Buffer.from([0x80]), + }); + const decisionEntry = createAuditEntry({ + ...baseRecord, + extendedType: HAS_EXPIRATION_DECISION, + expiresAt: undefined, + encodedRecord: Buffer.from([0x80]), + }); + assert.equal(decisionEntry.length, localOnlyEntry.length); + const read = readAuditEntry(Buffer.from(decisionEntry)); + assert.ok(read.extendedType & HAS_EXPIRATION_DECISION); + assert.equal(read.expiresAt, undefined); + }); + it('clears HAS_RECORD on a put minted with no encoded record', () => { const entry = createAuditEntry({ ...baseRecord, encodedRecord: undefined }); const read = readAuditEntry(Buffer.from(entry)); diff --git a/unitTests/resources/closeLoadedDatabases.test.js b/unitTests/resources/closeLoadedDatabases.test.js index 176395b7c2..e6e2f33dfd 100644 --- a/unitTests/resources/closeLoadedDatabases.test.js +++ b/unitTests/resources/closeLoadedDatabases.test.js @@ -9,8 +9,19 @@ require('../testUtils'); const assert = require('node:assert'); const { setupTestDBPath } = require('../testUtils'); -const { table, database, getDatabases, closeDatabase, closeLoadedDatabases } = require('#src/resources/databases'); +const { + table, + database, + getDatabases, + closeDatabase, + closeLoadedDatabases, + dropDatabase, + setDatabaseCloseTimeoutForTests, +} = require('#src/resources/databases'); const { registryStatus, RocksDatabase } = require('@harperfast/rocksdb-js'); +const { createBlob } = require('#src/resources/blob'); +const { acquireRestoreLock, releaseRestoreLock } = require('#src/dataLayer/restoreMarker'); +const { waitFor } = require('../waitFor.js'); describe('RocksDB handle release', function () { before(function () { @@ -38,7 +49,7 @@ describe('RocksDB handle release', function () { const dbPath = rootStore.path; assert.ok(refCountFor(dbPath) > 0, 'database should be open before close'); - closeDatabase('closerelease1'); + await closeDatabase('closerelease1'); assert.strictEqual(refCountFor(dbPath), 0, 'no native handles should remain after closeDatabase'); }); @@ -50,7 +61,7 @@ describe('RocksDB handle release', function () { if (!(a instanceof RocksDatabase)) return this.skip(); assert.ok(refCountFor(a.path) > 0 && refCountFor(b.path) > 0, 'both databases should be open'); - closeLoadedDatabases(); + await closeLoadedDatabases(); assert.strictEqual(refCountFor(a.path), 0, 'database a should be released'); assert.strictEqual(refCountFor(b.path), 0, 'database b should be released'); @@ -65,8 +76,261 @@ describe('RocksDB handle release', function () { const dbPath = rootStore.path; assert.ok(refCountFor(dbPath) > 0, 'tableless database should be open'); - closeLoadedDatabases(); + await closeLoadedDatabases(); assert.strictEqual(refCountFor(dbPath), 0, 'tableless database should be released'); }); + + it('closeDatabase runs table cleanup before closing its stores', async function () { + const Table = table({ + table: 'expiring', + database: 'closerelease4', + attributes: [ + { attribute: 'id', isPrimaryKey: true }, + { attribute: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + await closeDatabase('closerelease4'); + + assert.strictEqual(Table.cleanupStateForTests().closed, true); + }); + + it('coalesces concurrent closes of the same database', async function () { + const rootStore = openRocksDb('closerelease5'); + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + + const first = closeDatabase('closerelease5'); + const second = closeDatabase('closerelease5'); + assert.strictEqual(second, first, 'concurrent callers should share one close operation'); + await first; + + assert.strictEqual(refCountFor(rootStore.path), 0); + }); + + it('rejects a lazy table open while the database is closing', async function () { + const databaseName = 'closeopening6'; + const Table = table({ + table: 'existing', + database: databaseName, + attributes: [{ attribute: 'id', isPrimaryKey: true }], + }); + const originalCleanup = Table.cleanup; + let releaseCleanup; + Table.cleanup = () => new Promise((resolve) => (releaseCleanup = resolve)); + + const close = closeDatabase(databaseName); + try { + assert.throws( + () => + table({ + table: 'late', + database: databaseName, + attributes: [{ attribute: 'id', isPrimaryKey: true }], + }), + /Database .* is closing/ + ); + } finally { + Table.cleanup = originalCleanup; + releaseCleanup(); + } + await close; + }); + + it('fails closed and resumes cleanup when close times out', async function () { + this.timeout(30000); + const Table = table({ + table: 'expiring', + database: 'closetimeout6', + attributes: [ + { attribute: 'id', isPrimaryKey: true }, + { attribute: 'expiresAt', expiresAt: true, indexed: true }, + { attribute: 'payload', type: 'Blob' }, + ], + }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + await Table.put(1, { + id: 1, + expiresAt: Date.now() - 1_000, + payload: createBlob(Buffer.alloc(20_000, 1)), + }); + await Table.primaryStore.committed; + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + const sweep = Table.runRecordExpirationSweepForTests({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'the expiration sweep should start' }); + setDatabaseCloseTimeoutForTests(25); + try { + await assert.rejects(closeDatabase('closetimeout6'), /refusing to close database/); + assert.ok(refCountFor(rootStore.path) > 0, 'timed-out close must retain native handles'); + assert.strictEqual(rootStore.status, 'open'); + assert.strictEqual(Table.cleanupStateForTests().closed, false); + } finally { + setDatabaseCloseTimeoutForTests(); + releaseEviction(); + } + await sweep; + await closeDatabase('closetimeout6'); + }); + + it('fails closed and resumes cleanup when table quiescence rejects', async function () { + const Table = table({ + table: 'expiring', + database: 'closefailure7', + attributes: [{ attribute: 'id', isPrimaryKey: true }], + }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const quiesceForDrop = Table.quiesceForDrop; + Table.quiesceForDrop = async function () { + await quiesceForDrop.call(this); + throw new Error('injected cleanup failure'); + }; + try { + await assert.rejects(closeDatabase('closefailure7'), /Could not quiesce database/); + assert.strictEqual(rootStore.status, 'open'); + assert.strictEqual(Table.cleanupStateForTests().closed, false); + } finally { + Table.quiesceForDrop = quiesceForDrop; + } + await closeDatabase('closefailure7'); + }); + + it('dropDatabase runs table cleanup before destroying its stores', async function () { + const Table = table({ + table: 'expiring', + database: 'droprelease6', + attributes: [ + { attribute: 'id', isPrimaryKey: true }, + { attribute: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + await dropDatabase('droprelease6'); + + assert.strictEqual(Table.cleanupStateForTests().closed, true); + }); + + it('drops a tableless database while blocking a concurrent reopen', async function () { + const databaseName = 'droptableless7'; + const rootStore = database({ database: databaseName }); + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + + const drop = dropDatabase(databaseName); + assert.throws(() => database({ database: databaseName }), /Database .* is closing/); + const close = closeDatabase(databaseName); + await drop; + assert.strictEqual(await close, true); + assert.strictEqual(refCountFor(rootStore.path), 0); + }); + + it('refuses to drop a tableless database while its restore lock is held', async function () { + const databaseName = 'droplockedtableless8'; + const rootStore = database({ database: databaseName }); + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const restoreLock = acquireRestoreLock(rootStore.path); + try { + await assert.rejects(dropDatabase(databaseName)); + assert.strictEqual(rootStore.status, 'open'); + } finally { + releaseRestoreLock(restoreLock); + } + await dropDatabase(databaseName); + }); + + it('fails closed and resumes cleanup when drop times out', async function () { + this.timeout(30000); + const Table = table({ + table: 'expiring', + database: 'droptimeout7', + attributes: [ + { attribute: 'id', isPrimaryKey: true }, + { attribute: 'expiresAt', expiresAt: true, indexed: true }, + { attribute: 'payload', type: 'Blob' }, + ], + }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + await Table.put(1, { + id: 1, + expiresAt: Date.now() - 1_000, + payload: createBlob(Buffer.alloc(20_000, 2)), + }); + await Table.primaryStore.committed; + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + const sweep = Table.runRecordExpirationSweepForTests({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'the expiration sweep should start' }); + setDatabaseCloseTimeoutForTests(25); + try { + await assert.rejects(dropDatabase('droptimeout7'), /refusing to destroy database/); + assert.ok(refCountFor(rootStore.path) > 0, 'timed-out drop must retain native handles'); + assert.strictEqual(rootStore.status, 'open'); + assert.strictEqual(Table.cleanupStateForTests().closed, false); + } finally { + setDatabaseCloseTimeoutForTests(); + releaseEviction(); + } + await sweep; + await dropDatabase('droptimeout7'); + }); + + it('fails closed and resumes cleanup when drop quiescence rejects', async function () { + const Table = table({ + table: 'expiring', + database: 'dropfailure8', + attributes: [{ attribute: 'id', isPrimaryKey: true }], + }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const quiesceForDrop = Table.quiesceForDrop; + Table.quiesceForDrop = async function () { + await quiesceForDrop.call(this); + throw new Error('injected cleanup failure'); + }; + try { + await assert.rejects(dropDatabase('dropfailure8'), /Could not quiesce database/); + assert.strictEqual(rootStore.status, 'open'); + assert.strictEqual(Table.cleanupStateForTests().closed, false); + } finally { + Table.quiesceForDrop = quiesceForDrop; + } + await dropDatabase('dropfailure8'); + }); + + it('reconciles authoritative state when native database destruction fails', async function () { + const databaseName = 'dropdestroyfailure9'; + const Table = table({ + table: 'records', + database: databaseName, + attributes: [{ attribute: 'id', isPrimaryKey: true }], + }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const destroy = rootStore.destroy; + rootStore.destroy = () => { + throw new Error('injected destroy failure'); + }; + try { + await assert.rejects(dropDatabase(databaseName), /injected destroy failure/); + } finally { + rootStore.destroy = destroy; + } + const Recovered = getDatabases()[databaseName]?.records; + assert.ok(Recovered, 'authoritative reconciliation must reload a database that was not destroyed'); + assert.notStrictEqual(Recovered, Table, 'the closed registration must not be restored'); + assert.doesNotThrow(() => database({ database: databaseName }), 'the recovered database must be available'); + await dropDatabase(databaseName); + }); }); diff --git a/unitTests/resources/databases.test.js b/unitTests/resources/databases.test.js index f75c9b5f34..2fb87dfec0 100644 --- a/unitTests/resources/databases.test.js +++ b/unitTests/resources/databases.test.js @@ -3,10 +3,34 @@ const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); const { existsSync, mkdirSync, writeFileSync } = require('node:fs'); const { dirname, join } = require('node:path'); -const { table, flushDatabases, dropDatabase, getDatabases, resetDatabases } = require('#src/resources/databases'); +const { + table, + flushDatabases, + dropDatabase, + closeDatabase, + database, + getDatabases, + resetDatabases, + quiesceSchemaTarget, + abortSchemaQuiesce, + commitSchemaQuiesce, + renewSchemaQuiesce, + finishSchemaQuiesce, + completeSchemaQuiesce, + failSchemaQuiesceFinalization, + expireSchemaQuiesceLeaseForTests, + recoverCommittedSchemaQuiesceForTests, +} = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { RocksDatabase } = require('@harperfast/rocksdb-js'); -const { beginRestore, completeRestore, RESTORE_META_DIR } = require('#src/dataLayer/restoreMarker'); +const signalling = require('#src/utility/signalling'); +const { + acquireRestoreLock, + releaseRestoreLock, + beginRestore, + completeRestore, + RESTORE_META_DIR, +} = require('#src/dataLayer/restoreMarker'); describe('flushDatabases', () => { before(async function () { @@ -152,6 +176,33 @@ describe('dropDatabase restore serialization', () => { await assert.doesNotReject(dropDatabase(MULTI)); }); + it('rebuilds authoritative registrations after a native close failure', async () => { + const CLOSE_FAILURE_DB = 'close-failure-recovery-test'; + const Original = table({ + table: 'Records', + database: CLOSE_FAILURE_DB, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const originalClose = Original.primaryStore.close; + Original.primaryStore.close = () => { + throw new Error('injected close failure'); + }; + try { + await assert.rejects( + closeDatabase(CLOSE_FAILURE_DB), + (error) => + error instanceof AggregateError && + error.errors.some((closeError) => closeError.cause?.message === 'injected close failure') + ); + } finally { + Original.primaryStore.close = originalClose; + } + const Recovered = getDatabases()[CLOSE_FAILURE_DB]?.Records; + assert.ok(Recovered, 'the authoritative catalog should be reloaded after close failure'); + assert.notStrictEqual(Recovered, Original, 'the half-closed table registration must not remain public'); + assert.strictEqual(Recovered.isDropQuiescing(), false, 'the recovered table must be writable again'); + }); + it('never loads the reserved restore-metadata directory as a database', function () { // the API can't create a database with this name (schemaRegex rejects the backtick), but the // scan opens any CURRENT+MANIFEST directory regardless of name, so it must skip the reserved dir @@ -175,3 +226,425 @@ describe('dropDatabase restore serialization', () => { assert.ok(existsSync(reservedDir), 'the reserved dir itself is left in place (used for lifecycle metadata)'); }); }); + +describe('cross-worker schema quiescence', () => { + before(function () { + setupTestDBPath(); + setMainIsWorker(true); + }); + + it('waits for in-flight table quiescence before authoritative abort recovery', async () => { + const DB = 'quiesce-abort-wait-test'; + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const originalQuiesce = Table.quiesceForDrop; + let releaseQuiesce; + Table.quiesceForDrop = () => new Promise((resolve) => (releaseQuiesce = resolve)); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-abort-wait', + leaseUntil: Date.now() + 60_000, + }; + try { + const quiesce = quiesceSchemaTarget(message); + await new Promise(setImmediate); + let abortSettled = false; + const abort = abortSchemaQuiesce(message).then(() => (abortSettled = true)); + await new Promise(setImmediate); + assert.strictEqual(abortSettled, false, 'abort must not reset while quiescence is still closing'); + releaseQuiesce(); + const [result] = await Promise.all([quiesce, abort]); + assert.strictEqual(result.quiesced, false); + assert.ok(getDatabases()[DB]?.Records, 'authoritative reset must restore a table that was not dropped'); + } finally { + Table.quiesceForDrop = originalQuiesce; + } + }); + + it('keeps a schema unavailable until terminal reset completes', async () => { + const DB = 'quiesce-final-gate-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_schema', + schema: DB, + quiesceId: 'q-final-gate', + leaseUntil: Date.now() + 60_000, + }; + const result = await quiesceSchemaTarget(message); + assert.strictEqual(result.quiesced, true); + assert.strictEqual(finishSchemaQuiesce(message), true); + assert.throws(() => database({ database: DB, table: null }), /closing and cannot be opened/); + resetDatabases(); + assert.throws(() => database({ database: DB, table: null }), /closing and cannot be opened/); + completeSchemaQuiesce(message); + assert.doesNotThrow(() => database({ database: DB, table: null })); + assert.strictEqual(finishSchemaQuiesce(message), false, 'completed IDs must not re-enter finalization'); + }); + + it('accepts a terminal rescan with no local state while rejecting aborted IDs', async () => { + const DB = 'quiesce-stale-id-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-aborted-terminal', + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + await abortSchemaQuiesce(message); + assert.strictEqual(finishSchemaQuiesce(message), false); + assert.strictEqual( + finishSchemaQuiesce({ ...message, quiesceId: 'q-never-quiesced', phase: 'finalize-quiesce' }), + true, + 'a worker that joined after quiescence must accept the authoritative terminal rescan' + ); + }); + + it('does not re-register a table while resetDatabases runs during quiescence', async () => { + const DB = 'quiesce-reset-registration-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-reset-registration', + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual(getDatabases()[DB]?.Records, undefined); + resetDatabases(); + assert.strictEqual(getDatabases()[DB]?.Records, undefined, 'catalog rescan must not resurrect a quiesced table'); + await abortSchemaQuiesce(message); + assert.ok(getDatabases()[DB]?.Records, 'abort recovery should reload the authoritative live table'); + }); + + it('does not reopen a root store or replay logs while its schema is quiesced', async () => { + const DB = 'quiesce-reset-root-handle-test'; + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const rootStore = Table.primaryStore.rootStore; + const message = { + operation: 'drop_schema', + schema: DB, + quiesceId: 'q-reset-root-handle', + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual(rootStore.status, 'closed'); + resetDatabases(); + assert.strictEqual(rootStore.status, 'closed', 'catalog scan must not reopen the quiesced root store'); + assert.strictEqual(getDatabases()[DB], undefined, 'schema scan must remain fenced before replay/registration'); + await abortSchemaQuiesce(message); + assert.ok(getDatabases()[DB]?.Records); + }); + + it('retains a missing-table quiescence until its terminal message', async () => { + const message = { + operation: 'drop_table', + schema: 'quiesce-missing-table-test', + table: 'Records', + quiesceId: 'q-missing-table', + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual(finishSchemaQuiesce(message), true); + completeSchemaQuiesce(message); + assert.strictEqual(finishSchemaQuiesce(message), false); + }); + + it('serializes schema and table quiescence hierarchically', async () => { + const DB = 'quiesce-hierarchy-test'; + const First = table({ table: 'First', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + table({ table: 'Second', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const originalQuiesce = First.quiesceForDrop; + let releaseFirst; + First.quiesceForDrop = () => new Promise((resolve) => (releaseFirst = resolve)); + const firstMessage = { + operation: 'drop_table', + schema: DB, + table: 'First', + quiesceId: 'q-hierarchy-first', + originLocal: true, + }; + try { + const first = quiesceSchemaTarget(firstMessage); + await new Promise(setImmediate); + const duplicate = await quiesceSchemaTarget({ ...firstMessage, quiesceId: 'q-hierarchy-duplicate' }); + assert.strictEqual(duplicate.quiesced, false); + const schema = await quiesceSchemaTarget({ + operation: 'drop_schema', + schema: DB, + quiesceId: 'q-hierarchy-schema', + originLocal: true, + }); + assert.strictEqual(schema.quiesced, false); + releaseFirst(); + assert.strictEqual((await first).quiesced, true); + await abortSchemaQuiesce(firstMessage); + First.quiesceForDrop = originalQuiesce; + + const schemaMessage = { + operation: 'drop_schema', + schema: DB, + quiesceId: 'q-hierarchy-schema-owner', + originLocal: true, + }; + assert.strictEqual((await quiesceSchemaTarget(schemaMessage)).quiesced, true); + const tableWhileSchema = await quiesceSchemaTarget({ + operation: 'drop_table', + schema: DB, + table: 'Second', + quiesceId: 'q-hierarchy-table-loser', + originLocal: true, + }); + assert.strictEqual(tableWhileSchema.quiesced, false); + await abortSchemaQuiesce(schemaMessage); + } finally { + First.quiesceForDrop = originalQuiesce; + } + }); + + it('stays fail-closed after the commit boundary until a terminal reconcile', async () => { + const DB = 'quiesce-committed-test'; + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-committed', + originLocal: true, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + await assert.rejects(() => abortSchemaQuiesce(message), /commit boundary/); + const terminal = { ...message, phase: 'reconcile-quiesce' }; + assert.strictEqual(finishSchemaQuiesce(terminal), true); + await completeSchemaQuiesce(terminal); + assert.strictEqual(Table.isDropQuiescing(), false); + }); + + it('can retry terminal reconciliation after a transient restore failure', async () => { + const DB = 'quiesce-reconcile-retry-test'; + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-reconcile-retry', + originLocal: true, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + const terminal = { ...message, phase: 'reconcile-quiesce' }; + const originalAbort = Table.abortDropQuiesce; + Table.abortDropQuiesce = () => { + throw new Error('injected restore failure'); + }; + try { + assert.strictEqual(finishSchemaQuiesce(terminal), true); + await assert.rejects(completeSchemaQuiesce(terminal), /injected restore failure/); + failSchemaQuiesceFinalization(terminal); + assert.strictEqual( + finishSchemaQuiesce(terminal), + true, + 'a transient failure must leave reconciliation retryable' + ); + } finally { + Table.abortDropQuiesce = originalAbort; + } + await completeSchemaQuiesce(terminal); + assert.strictEqual(Table.isDropQuiescing(), false); + }); + + it('does not recover a committed peer quiescence while its origin is connected', async () => { + const message = { + originator: require('node:worker_threads').threadId, + operation: 'drop_table', + schema: 'quiesce-live-origin-test', + table: 'Missing', + quiesceId: 'q-live-origin', + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + expireSchemaQuiesceLeaseForTests(message.quiesceId); + assert.strictEqual( + renewSchemaQuiesce({ ...message, leaseUntil: Date.now() + 60_000 }).quiesced, + true, + 'a connected origin must keep the committed peer fence active' + ); + const terminal = { ...message, phase: 'reconcile-quiesce' }; + assert.strictEqual(finishSchemaQuiesce(terminal), true); + await completeSchemaQuiesce(terminal); + }); + + it('recovers a committed peer quiescence after its origin disconnects', async () => { + const message = { + originator: Number.MAX_SAFE_INTEGER, + operation: 'drop_table', + schema: 'quiesce-disconnected-origin-test', + table: 'Missing', + quiesceId: 'q-disconnected-origin', + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + expireSchemaQuiesceLeaseForTests(message.quiesceId); + assert.strictEqual( + renewSchemaQuiesce({ ...message, leaseUntil: Date.now() + 60_000 }).quiesced, + false, + 'a disconnected origin must allow authoritative recovery' + ); + }); + + it('recovers an expired committed quiescence from the durable live catalog', async () => { + const DB = 'quiesce-committed-recovery-test'; + const Original = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-committed-recovery', + originLocal: true, + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + const registry = getDatabases(); + const originalGetSync = Original.dbisDB.getSync; + let descriptorReadBeforeMutation = false; + Original.dbisDB.getSync = function (key) { + if (key === 'Records/' && !descriptorReadBeforeMutation) + descriptorReadBeforeMutation = registry[DB]?.Records === Original; + return originalGetSync.call(this, key); + }; + try { + assert.strictEqual(await recoverCommittedSchemaQuiesceForTests(message.quiesceId), true); + } finally { + Original.dbisDB.getSync = originalGetSync; + } + assert.strictEqual(descriptorReadBeforeMutation, true, 'durable descriptor must be read before registry mutation'); + const Recovered = getDatabases()[DB]?.Records; + assert.ok(Recovered, 'the durable non-tombstoned catalog should be restored'); + assert.notStrictEqual(Recovered, Original, 'recovery must not republish the quiesced table instance'); + assert.strictEqual(Recovered.isDropQuiescing(), false); + assert.strictEqual(finishSchemaQuiesce({ ...message, phase: 'finalize-quiesce' }), false); + }); + + it('recovers an expired committed schema and clears its unavailable fence', async () => { + const DB = 'quiesce-committed-schema-recovery-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_schema', + schema: DB, + quiesceId: 'q-committed-schema-recovery', + originLocal: true, + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + assert.throws(() => database({ database: DB, table: null }), /closing and cannot be opened/); + assert.strictEqual(await recoverCommittedSchemaQuiesceForTests(message.quiesceId), true); + assert.doesNotThrow(() => database({ database: DB, table: null })); + assert.ok(getDatabases()[DB]?.Records); + }); + + it('clears the origin unavailable fence when only terminal broadcast acknowledgement fails', async function () { + this.timeout(30000); + const DB = 'drop-terminal-broadcast-failure-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const originalFinalize = signalling.finalizeSchemaChange; + signalling.finalizeSchemaChange = async (message) => { + completeSchemaQuiesce({ ...message, phase: 'finalize-quiesce' }); + throw new Error('injected remote terminal acknowledgement failure'); + }; + try { + await assert.rejects(dropDatabase(DB), /injected remote terminal acknowledgement failure/); + assert.doesNotThrow(() => database({ database: DB, table: null })); + assert.ok( + table({ table: 'Recreated', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }), + 'physically destroyed schema name must be reusable after local terminal completion' + ); + } finally { + signalling.finalizeSchemaChange = originalFinalize; + } + }); + + it('retains the origin unavailable fence when local terminal state is unresolved', async function () { + this.timeout(30000); + const DB = 'drop-terminal-local-failure-test'; + table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const originalFinalize = signalling.finalizeSchemaChange; + let terminalMessage; + signalling.finalizeSchemaChange = async (message) => { + terminalMessage = message; + throw new Error('injected local terminal failure'); + }; + try { + await assert.rejects(dropDatabase(DB), /injected local terminal failure/); + assert.throws( + () => database({ database: DB, table: null }), + /closing and cannot be opened/, + 'uncertain local terminal state must remain fail-closed' + ); + } finally { + signalling.finalizeSchemaChange = originalFinalize; + if (terminalMessage) await completeSchemaQuiesce({ ...terminalMessage, phase: 'reconcile-quiesce' }); + } + }); + + it('clears the origin unavailable fence when failed-drop reconciliation only loses remote acknowledgement', async function () { + this.timeout(30000); + const DB = 'drop-reconcile-broadcast-failure-test'; + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const rootStore = Table.primaryStore.rootStore; + if (!(rootStore instanceof RocksDatabase)) return this.skip(); + const originalDestroy = rootStore.destroy; + const originalReconcile = signalling.reconcileSchemaChange; + rootStore.destroy = () => { + throw new Error('injected destroy failure'); + }; + signalling.reconcileSchemaChange = async (message) => { + await originalReconcile(message); + throw new Error('injected remote reconciliation acknowledgement failure'); + }; + try { + await assert.rejects(dropDatabase(DB), /injected remote reconciliation acknowledgement failure/); + assert.doesNotThrow(() => database({ database: DB, table: null })); + assert.ok(getDatabases()[DB]?.Records, 'locally reconciled intact storage must be available'); + } finally { + rootStore.destroy = originalDestroy; + signalling.reconcileSchemaChange = originalReconcile; + await dropDatabase(DB); + } + }); + + it('bounds committed recovery attempts while an active drop lock is still held', async () => { + const DB = 'quiesce-committed-recovery-bound-test'; + const Table = table({ table: 'Records', database: DB, attributes: [{ name: 'id', isPrimaryKey: true }] }); + const message = { + operation: 'drop_table', + schema: DB, + table: 'Records', + quiesceId: 'q-committed-recovery-bound', + originLocal: true, + leaseUntil: Date.now() + 60_000, + }; + assert.strictEqual((await quiesceSchemaTarget(message)).quiesced, true); + assert.strictEqual((await commitSchemaQuiesce(message)).committed, true); + const lock = acquireRestoreLock(Table.primaryStore.rootStore.path); + try { + for (let attempt = 0; attempt < 3; attempt++) + assert.strictEqual(await recoverCommittedSchemaQuiesceForTests(message.quiesceId), false); + } finally { + releaseRestoreLock(lock); + } + assert.strictEqual( + await recoverCommittedSchemaQuiesceForTests(message.quiesceId), + false, + 'the recovery budget must not restart after it is exhausted' + ); + const terminal = { ...message, phase: 'reconcile-quiesce' }; + assert.strictEqual(finishSchemaQuiesce(terminal), true); + await completeSchemaQuiesce(terminal); + }); +}); diff --git a/unitTests/resources/dropTableGhost.test.js b/unitTests/resources/dropTableGhost.test.js index 646ec41ab2..4eaae2f168 100644 --- a/unitTests/resources/dropTableGhost.test.js +++ b/unitTests/resources/dropTableGhost.test.js @@ -88,7 +88,7 @@ describe('dropTable ghost regression', () => { await Fresh.dropTable(); }); - it('surfaces a failed column family drop and completes the drop on recreate', async function () { + it('surfaces a failed column family drop and reconciles the authoritative tombstone', async function () { const Doomed = defineTable('GhostFailDrop'); await Doomed.put({ id: 1, str: 'data' }); const restore = stubFailingDrop(Doomed.primaryStore, new Error('injected drop failure')); @@ -99,10 +99,12 @@ describe('dropTable ghost regression', () => { } // the table is gone from the live schema (no half-alive table)... assert.equal(databases[TEST_DB]?.GhostFailDrop, undefined, 'failed drop must still remove the table from memory'); - // ...but the tombstoned catalog entry survives so the drop can complete later - assert.equal(getDbisDb().getSync('GhostFailDrop/')?.dropping, true, 'tombstone must survive a failed drop'); + // Reconciliation rescans authoritative storage. A fresh handle can complete the + // tombstoned drop even though the original handle failed, so no worker retains + // stale catalog state after the failure is reported. + assert.equal(getDbisDb().getSync('GhostFailDrop/'), undefined, 'reconcile must complete the tombstoned drop'); - // recreating the same name completes the interrupted drop and works + // recreating the same name starts clean after reconciliation const Fresh = defineTable('GhostFailDrop'); await Fresh.put({ id: 2, str: 'fresh' }); assert.equal((await Fresh.get(2)).str, 'fresh'); diff --git a/unitTests/resources/evictionBatch.test.js b/unitTests/resources/evictionBatch.test.js index afeedb0523..0858b9483f 100644 --- a/unitTests/resources/evictionBatch.test.js +++ b/unitTests/resources/evictionBatch.test.js @@ -21,7 +21,6 @@ const { setMainIsWorker } = require('#js/server/threads/manageThreads'); describe('Batched eviction (RocksDB)', () => { if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; let EvictTable; - let RetryTable; const makeSource = () => class extends Resource { @@ -72,12 +71,6 @@ describe('Batched eviction (RocksDB)', () => { ], }); EvictTable.sourcedFrom(makeSource()); - RetryTable = table({ - table: 'EvictRetryTable', - database: 'test', - attributes: [{ name: 'id', isPrimaryKey: true }], - }); - RetryTable.sourcedFrom(makeSource()); }); it('physically evicts >EVICTION_BATCH_SIZE records across multiple batches and cleans indices', async function () { @@ -103,36 +96,4 @@ describe('Batched eviction (RocksDB)', () => { 'index entries must be removed with the records' ); }); - - it('recovers from an optimistic commit conflict (ERR_BUSY) and still evicts', async function () { - const { Transaction } = require('@harperfast/rocksdb-js'); - const originalCommit = Transaction.prototype.commit; - let injected = false; - let armed = false; - // Inject a single ERR_BUSY into the first commit after arming, simulating an optimistic - // write-write conflict. The batcher must abort, re-stage into a fresh transaction, and commit. - Transaction.prototype.commit = async function (...args) { - if (armed && !injected) { - injected = true; - const error = new Error('injected optimistic conflict'); - error.code = 'ERR_BUSY'; - throw error; - } - return originalCommit.apply(this, args); - }; - - const ids = [1000, 1001, 1002, 1003, 1004]; - try { - RetryTable.setTTLExpiration(HOLD); // keep records resident (and un-evicted) until the conflict is armed - for (const id of ids) await RetryTable.get(id); - assert.equal(await waitForResident(RetryTable, ids), ids.length, 'records should be resident after warming'); - - armed = true; // the next commit (an eviction-batch commit) will hit the injected conflict - const resident = await evictAndWait(RetryTable, ids); - assert(injected, 'the injected ERR_BUSY conflict should have fired on a commit'); - assert.equal(resident, 0, 'eviction should still complete after retrying the conflicting batch'); - } finally { - Transaction.prototype.commit = originalCommit; - } - }); }); diff --git a/unitTests/resources/expirationWarning.test.js b/unitTests/resources/expirationWarning.test.js index 8f27f049f0..4fe9a39332 100644 --- a/unitTests/resources/expirationWarning.test.js +++ b/unitTests/resources/expirationWarning.test.js @@ -20,6 +20,13 @@ describe('Per-record expiresAt without scheduled cleanup (#1339)', () => { const matchesWarning = (message) => typeof message === 'string' && message.includes('per-record expiresAt'); const warningsFor = (tableName) => warnings.filter(([message]) => matchesWarning(message) && message.includes(`"${tableName}"`)); + const sourceWarningsFor = (tableName) => + warnings.filter( + ([message]) => + typeof message === 'string' && + message.includes(`table "${tableName}"`) && + message.includes('@expiresAt field without setting context.expiresAt') + ); before(function () { setupTestDBPath(); @@ -97,4 +104,60 @@ describe('Per-record expiresAt without scheduled cleanup (#1339)', () => { await new Promise((resolve) => setTimeout(resolve, 50)); assert.strictEqual(warningsFor('SourcedExpiresAtTable').length, 0); }); + + it('warns once when a source returns an @expiresAt field without setting cache expiration', async function () { + const tableName = 'SourcedFieldOnlyExpiresAtTable'; + const SourcedTable = table({ + table: tableName, + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + SourcedTable.sourcedFrom( + class extends Resource { + get() { + return { id: this.getId(), expiresAt: Date.now() + 60000 }; + } + } + ); + + await SourcedTable.get(1); + await waitFor(() => sourceWarningsFor(tableName).length === 1); + assert.strictEqual(SourcedTable.primaryStore.getEntry(1)?.expiresAt, undefined); + assert.deepStrictEqual([...SourcedTable.indices.expiresAt.getRange({ start: true })], []); + + await SourcedTable.get(2); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.strictEqual(sourceWarningsFor(tableName).length, 1); + }); + + it('warns before applying a table expiration to a returned @expiresAt field', async function () { + const tableName = 'SourcedTableExpirationFallback'; + const SourcedTable = table({ + table: tableName, + database: 'test', + expiration: 60, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + SourcedTable.sourcedFrom( + class extends Resource { + get() { + return { id: this.getId(), expiresAt: Date.now() + 120000 }; + } + } + ); + + const before = Date.now() + 60000; + await SourcedTable.get(1); + await waitFor(() => SourcedTable.primaryStore.getEntry(1)?.expiresAt !== undefined); + const storedExpiration = SourcedTable.primaryStore.getEntry(1).expiresAt; + assert.ok(storedExpiration >= before); + assert.deepStrictEqual([...SourcedTable.indices.expiresAt.getValues(storedExpiration)], [1]); + assert.strictEqual(sourceWarningsFor(tableName).length, 1); + }); }); diff --git a/unitTests/resources/expiresAtAttribute.test.js b/unitTests/resources/expiresAtAttribute.test.js index 76e2726f6f..7890caf9cd 100644 --- a/unitTests/resources/expiresAtAttribute.test.js +++ b/unitTests/resources/expiresAtAttribute.test.js @@ -1,8 +1,36 @@ require('../testUtils'); const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); -const { table } = require('#src/resources/databases'); +const { table: createTable, closeDatabase, dropDatabase } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); +const { Transaction: RocksTransaction } = require('@harperfast/rocksdb-js'); +const { asBinary } = require('lmdb'); +const { createBlob, getFilePathForBlob, setDeletionDelay } = require('#src/resources/blob'); +const { existsSync } = require('node:fs'); +const { setTimeout: delay } = require('node:timers/promises'); +const { waitFor } = require('../waitFor.js'); +const { HAS_EXPIRATION_DECISION } = require('#src/resources/auditStore'); +const { + TABLE_COMMIT_ADMISSION, + TABLE_COMMIT_RELEASE, + trackedTransactionCountForTests, +} = require('#src/resources/DatabaseTransaction'); +const { LMDBTransaction } = require('#src/resources/LMDBTransaction'); +const { transaction } = require('#src/resources/transaction'); +const harperLogger = require('#src/utility/logging/harper_logger'); + +const activeTables = new Set(); +const table = (options) => { + const Table = createTable(options); + activeTables.add(Table); + return Table; +}; + +afterEach(async function () { + const tables = [...activeTables]; + activeTables.clear(); + await Promise.all(tables.map((Table) => Table.cleanup())); +}); // A schema @expiresAt attribute must be authoritative over the table-level expiration default, in both // directions. Previously the field only armed a separate index-pruning sweep (which can only remove @@ -17,10 +45,11 @@ describe('@expiresAt attribute is authoritative over the table default', () => { setMainIsWorker(true); }); - const makeTable = (name, expirationSeconds) => + const makeTable = (name, expirationSeconds, options = {}) => table({ table: name, database: 'test', + ...options, ...(expirationSeconds == null ? {} : { expiration: expirationSeconds }), attributes: [ { name: 'id', isPrimaryKey: true }, @@ -33,6 +62,14 @@ describe('@expiresAt attribute is authoritative over the table default', () => { return Table.primaryStore.getEntry(id)?.expiresAt; }; + const captureExpirationSweep = (createTable) => { + const Table = createTable(); + return { + Table, + runSweep: (testHooks) => Table.runRecordExpirationSweepForTests(testHooks), + }; + }; + it('extends: a far-future field overrides a short table default', async function () { const Table = makeTable('ExpiresAtExtend', 3); const fieldExpiresAt = Date.now() + 3_600_000; // 1h, far past the 3s table default @@ -59,8 +96,28 @@ describe('@expiresAt attribute is authoritative over the table default', () => { it('lets an explicit options.expiresAt override the field', async function () { const Table = makeTable('ExpiresAtOptionsOverride', 3); const optionExpiresAt = Date.now() + 60_000; - await Table.put(1, { id: 1, expiresAt: Date.now() + 3_600_000 }, { expiresAt: optionExpiresAt }); + const fieldExpiresAt = optionExpiresAt + 3_600_000; + await Table.put(1, { id: 1, expiresAt: fieldExpiresAt }, { expiresAt: optionExpiresAt }); assert.strictEqual(await storedExpiresAt(Table, 1), optionExpiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(optionExpiresAt)], [1]); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + const matchingOverride = []; + for await (const record of Table.search({ + allowFullScan: false, + conditions: [{ attribute: 'expiresAt', value: optionExpiresAt }], + })) { + matchingOverride.push(record); + } + assert.strictEqual(matchingOverride.length, 1); + assert.strictEqual(matchingOverride[0].expiresAt, fieldExpiresAt); + const fullScanMismatch = []; + for await (const record of Table.search({ + allowFullScan: true, + conditions: [{ attribute: 'expiresAt', comparator: 'ne', value: optionExpiresAt }], + })) { + fullScanMismatch.push(record); + } + assert.deepStrictEqual(fullScanMismatch, []); }); it('keeps the field value across a patch that does not touch it', async function () { @@ -93,6 +150,253 @@ describe('@expiresAt attribute is authoritative over the table default', () => { assert.strictEqual(await storedExpiresAt(Table, 1), fieldExpiresAt); }); + it('canonicalizes supported expiration shapes in the index and in searches', async function () { + const Table = makeTable('ExpiresAtCanonicalIndex'); + const expiresAt = Date.now() + 3_600_000; + const values = [expiresAt, String(expiresAt), new Date(expiresAt).toISOString(), new Date(expiresAt)]; + for (let id = 0; id < values.length; id++) await Table.put(id, { id, expiresAt: values[id] }); + await Table.primaryStore.committed; + + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [0, 1, 2, 3]); + for (const value of values) { + const ids = []; + for await (const record of Table.search({ + allowFullScan: false, + conditions: [{ attribute: 'expiresAt', value }], + })) { + ids.push(record.id); + } + assert.deepStrictEqual(ids, [0, 1, 2, 3]); + } + Table.cleanup(); + }); + + it('rebuilds a legacy expiration index into the canonical format', async function () { + const tableName = 'ExpiresAtCanonicalRebuild'; + const Table = makeTable(tableName); + const expiresAt = Date.now() + 3_600_000; + const noExpiryField = expiresAt + 60_000; + const isoExpiresAt = new Date(expiresAt).toISOString(); + await Table.put(1, { id: 1, expiresAt: isoExpiresAt }); + await Table.put(2, { id: 2, expiresAt: noExpiryField }, { expiresAt: -1 }); + await Table.primaryStore.committed; + + const attribute = Table.attributes.find((candidate) => candidate.name === 'expiresAt'); + const descriptor = Table.dbisDB.getSync(attribute.key); + delete descriptor.expirationIndexVersion; + await Table.dbisDB.put(attribute.key, descriptor); + await Table.indices.expiresAt.clear(); + await Table.indices.expiresAt.put(isoExpiresAt, 1); + await Table.indices.expiresAt.put(noExpiryField, 2); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(isoExpiresAt)], [1]); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(noExpiryField)], [2]); + + const Reloaded = makeTable(tableName); + await Reloaded.indexingOperation; + assert.deepStrictEqual([...Reloaded.indices.expiresAt.getValues(isoExpiresAt)], []); + assert.deepStrictEqual([...Reloaded.indices.expiresAt.getValues(expiresAt)], [1]); + assert.deepStrictEqual([...Reloaded.indices.expiresAt.getValues(noExpiryField)], []); + Table.cleanup(); + }); + + it('preserves expiration for legacy rows that have a field but no stored metadata', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtLegacyMetadata')); + const expiresAt = Date.now() - 1_000; + const record = { id: 1, expiresAt: new Date(expiresAt).toISOString() }; + Table.primaryStore.putSync(1, record); + Table.indices.expiresAt.put(expiresAt, 1); + assert.strictEqual(Table.primaryStore.getEntry(1).expiresAt, undefined); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + Table.cleanup(); + }); + + it('removes a legacy field-derived index entry when the row is updated', async function () { + const Table = makeTable('ExpiresAtLegacyUpdate'); + const oldExpiresAt = Date.now() + 3_600_000; + const newExpiresAt = oldExpiresAt + 3_600_000; + Table.primaryStore.putSync(1, { id: 1, expiresAt: new Date(oldExpiresAt).toISOString() }); + Table.indices.expiresAt.put(oldExpiresAt, 1); + + await Table.put(1, { id: 1, expiresAt: newExpiresAt }); + await Table.primaryStore.committed; + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(oldExpiresAt)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(newExpiresAt)], [1]); + Table.cleanup(); + }); + + it('uses source context rather than the returned field for cache expiration', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSourceFill', 3_600)); + const contextExpiresAt = Date.now() - 1_000; + const fieldExpiresAt = contextExpiresAt + 3_600_000; + Table.sourcedFrom({ + get(id, context) { + context.expiresAt = new Date(contextExpiresAt).toISOString(); + return { id, expiresAt: new Date(fieldExpiresAt).toISOString() }; + }, + }); + + await Table.get(1); + await waitFor(() => Table.primaryStore.getEntry(1)?.value, { message: 'source fill should be stored' }); + await Table.primaryStore.committed; + assert.strictEqual(Table.primaryStore.getEntry(1).expiresAt, contextExpiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(contextExpiresAt)], [1]); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + }); + + it('does not reinterpret a source no-expiration decision as a legacy field expiration', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSourceNoExpiration')); + const fieldExpiresAt = Date.now() - 1_000; + Table.sourcedFrom({ + get(id) { + return { id, expiresAt: fieldExpiresAt, name: 'source' }; + }, + }); + + await Table.get(1); + await waitFor(() => Table.primaryStore.getEntry(1)?.value, { message: 'source fill should be stored' }); + await Table.primaryStore.committed; + const sourceEntry = Table.primaryStore.getEntry(1); + assert.strictEqual(sourceEntry.expiresAt, undefined); + assert.ok(sourceEntry.metadataFlags & HAS_EXPIRATION_DECISION); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + const fullScanResults = []; + for await (const record of Table.search({ + allowFullScan: true, + conditions: [ + { attribute: 'id', value: 1 }, + { attribute: 'expiresAt', value: fieldExpiresAt }, + ], + })) { + fullScanResults.push(record.id); + } + assert.deepStrictEqual(fullScanResults, []); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.name, 'source'); + + await Table.put(1, { id: 1, expiresAt: fieldExpiresAt, name: 'local' }); + await Table.primaryStore.committed; + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], [1]); + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + }); + + it('propagates an explicit no-expiration decision through current and live subscriptions', async function () { + const Source = makeTable('ExpiresAtDecisionSource', undefined, { audit: true }); + const fieldExpiresAt = Date.now() - 1_000; + await Source.put(1, { id: 1, expiresAt: fieldExpiresAt }, { expiresAt: -1 }); + await Source.primaryStore.committed; + + const currentSubscription = await Source.subscribe({ isCollection: true }); + const currentEvents = []; + currentSubscription.on('data', (event) => currentEvents.push(event)); + await waitFor(() => currentEvents.some((event) => event.id === 1), { message: 'current event should arrive' }); + const currentEvent = currentEvents.find((event) => event.id === 1); + assert.strictEqual(currentEvent.expirationDecisionPresent, true); + assert.strictEqual(currentEvent.expiresAt, undefined); + await currentSubscription.return?.(); + + const Target = makeTable('ExpiresAtDecisionTarget'); + Target.sourcedFrom(Source, { intermediateSource: true }); + await Source.put(2, { id: 2, expiresAt: fieldExpiresAt }, { expiresAt: -1 }); + await waitFor(() => Target.primaryStore.getEntry(2)?.value, { message: 'live event should be applied' }); + await Target.primaryStore.committed; + const targetEntry = Target.primaryStore.getEntry(2); + assert.strictEqual(targetEntry.expiresAt, undefined); + assert.ok(targetEntry.metadataFlags & HAS_EXPIRATION_DECISION); + assert.deepStrictEqual([...Target.indices.expiresAt.getValues(fieldExpiresAt)], []); + }); + + it('preserves an explicit no-expiration decision across publish and invalidate', async function () { + const Table = makeTable('ExpiresAtDecisionLifecycle', undefined, { audit: true }); + const fieldExpiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt: fieldExpiresAt }, { expiresAt: -1 }); + await Table.publish(1, { message: 'refresh' }); + await Table.primaryStore.committed; + + let entry = Table.primaryStore.getEntry(1); + assert.strictEqual(entry.expiresAt, undefined); + assert.ok(entry.metadataFlags & HAS_EXPIRATION_DECISION); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + + await Table.invalidate(1); + await Table.primaryStore.committed; + entry = Table.primaryStore.getEntry(1); + assert.strictEqual(entry.expiresAt, undefined); + assert.ok(entry.metadataFlags & HAS_EXPIRATION_DECISION); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + }); + + it('normalizes publish expiration overrides and updates the expiration index', async function () { + const Table = makeTable('ExpiresAtPublishOverride', undefined, { audit: true }); + const originalExpiresAt = Date.now() + 60_000; + const optionExpiresAt = originalExpiresAt + 60_000; + const contextExpiresAt = optionExpiresAt + 60_000; + await Table.put(1, { id: 1, expiresAt: originalExpiresAt }); + await Table.publish(1, { message: 'option' }, { expiresAt: new Date(optionExpiresAt).toISOString() }); + await Table.primaryStore.committed; + assert.strictEqual(Table.primaryStore.getEntry(1).expiresAt, optionExpiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(originalExpiresAt)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(optionExpiresAt)], [1]); + + const context = { expiresAt: new Date(contextExpiresAt) }; + const resource = new Table(1, context); + await transaction(context, () => resource._writePublish(1, { message: 'context' })); + await Table.primaryStore.committed; + assert.strictEqual(Table.primaryStore.getEntry(1).expiresAt, contextExpiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(optionExpiresAt)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(contextExpiresAt)], [1]); + }); + + it('invalidates a frozen replay record without mutating it', async function () { + const Table = makeTable('ExpiresAtFrozenInvalidate', undefined, { audit: true }); + const fieldExpiresAt = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt: fieldExpiresAt }); + const context = {}; + const resource = await Table.getResource(1, context, { ensureLoaded: true }); + const replayRecord = Object.freeze({ id: 1, expiresAt: fieldExpiresAt }); + await transaction(context, () => resource._writeInvalidate(1, replayRecord)); + assert.strictEqual(Object.isFrozen(replayRecord), true); + assert.ok(Table.primaryStore.getEntry(1).metadataFlags); + }); + + it('re-arms both expiration sweeps after cleanup is resumed', async function () { + const Table = makeTable('ExpiresAtResumeCleanup', undefined, { scanInterval: 100 }); + assert.deepStrictEqual(Table.cleanupStateForTests(), { + closed: false, + cleanupScheduled: true, + nextCleanupScheduled: Table.cleanupStateForTests().nextCleanupScheduled, + expirationScheduled: true, + }); + await Table.cleanup(); + assert.deepStrictEqual(Table.cleanupStateForTests(), { + closed: true, + cleanupScheduled: false, + nextCleanupScheduled: undefined, + expirationScheduled: false, + }); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); + assert.deepStrictEqual(Table.cleanupStateForTests(), { + closed: true, + cleanupScheduled: false, + nextCleanupScheduled: undefined, + expirationScheduled: false, + }); + Table.resumeCleanup(); + assert.deepStrictEqual(Table.cleanupStateForTests(), { + closed: false, + cleanupScheduled: true, + nextCleanupScheduled: Table.cleanupStateForTests().nextCleanupScheduled, + expirationScheduled: true, + }); + }); + it('ignores non-timestamp field values (boolean / empty string) and uses the table default', async function () { const Bool = makeTable('ExpiresAtBool', 100); const Empty = makeTable('ExpiresAtEmpty', 100); @@ -105,6 +409,19 @@ describe('@expiresAt attribute is authoritative over the table default', () => { } }); + it('does not index or evict a field-only record without a valid expiration', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtNoExpiration')); + await Table.put(1, { id: 1 }); + await Table.put(2, { id: 2, expiresAt: -1 }); + await Table.primaryStore.committed; + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(-1)], []); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.id, 1); + assert.strictEqual(Table.primaryStore.getEntry(2)?.value.id, 2); + Table.cleanup(); + }); + // End-to-end: stamping the field into the expiry metadata makes read-hiding enforce it on a // field-only table with no table default — the record is no longer served past its field time. // This also covers the RocksDB correctness half of #1481 (the field sweep is LMDB-only, so @@ -113,6 +430,978 @@ describe('@expiresAt attribute is authoritative over the table default', () => { const Table = makeTable('ExpiresAtReadHide'); // no table-level expiration await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); await Table.primaryStore.committed; + const trackedBeforeRead = trackedTransactionCountForTests(); assert.strictEqual(await Table.get(1), null); + await waitFor(() => trackedTransactionCountForTests() === trackedBeforeRead, { + message: 'read-path eviction should release its internal Rocks transaction', + }); + + await Table.put(2, { id: 2, expiresAt: Date.now() + 60_000 }); + await Table.primaryStore.committed; + const staleEntry = Table.primaryStore.getEntry(2); + await Table.patch(2, { refreshed: true }); + assert.strictEqual(Table.primaryStore.ifVersion, undefined, 'premise: this regression must exercise RocksDB'); + await Table.evict(2, staleEntry.value, staleEntry.version); + assert.strictEqual( + trackedTransactionCountForTests(), + trackedBeforeRead, + 'a version-mismatched eviction should release its internal Rocks transaction' + ); + }); + + it('enumerates exact Rocks index values for the expiration sweep', async function () { + const Table = makeTable('ExpiresAtIndexValues'); + const expiresAt = Date.now() + 60_000; + await Table.put(0, { id: 0, expiresAt: expiresAt - 1 }); + await Table.put(5, { id: 5, expiresAt: expiresAt - 1 }); + await Table.put(1, { id: 1, expiresAt }); + await Table.put(2, { id: 2, expiresAt }); + await Table.put(3, { id: 3, expiresAt: expiresAt + 1 }); + await Table.put([4, 'part'], { id: [4, 'part'], expiresAt: expiresAt + 2 }); + await Table.primaryStore.committed; + + const index = Table.indices.expiresAt; + const expirationKeys = index.getRange({ start: true, values: false, end: expiresAt + 1, snapshot: false }); + assert.deepStrictEqual([...expirationKeys], [expiresAt - 1, expiresAt]); + assert.deepStrictEqual([...expirationKeys], [expiresAt - 1, expiresAt]); + assert.deepStrictEqual([...expirationKeys.map((value) => value)], [expiresAt - 1, expiresAt]); + assert.deepStrictEqual( + [...index.getRange({ start: true, values: false, end: expiresAt + 2, limit: 1, snapshot: false })], + [expiresAt - 1] + ); + assert.deepStrictEqual( + [...index.getRange({ start: true, values: false, end: expiresAt + 2, offset: 1, limit: 2, snapshot: false })], + [expiresAt, expiresAt + 1] + ); + const sweepEntries = []; + for (const key of expirationKeys) { + for (const id of index.getValues(key)) sweepEntries.push([key, id]); + } + assert.deepStrictEqual(sweepEntries, [ + [expiresAt - 1, 0], + [expiresAt - 1, 5], + [expiresAt, 1], + [expiresAt, 2], + ]); + assert.deepStrictEqual( + [...index.getValues(expiresAt)].sort((left, right) => left - right), + [1, 2] + ); + assert.deepStrictEqual([...index.getValues(expiresAt + 2)], [[4, 'part']]); + + const firstChunk = [...index.getCompositeRange({ end: expiresAt + 2, limit: 2 })]; + const cursor = firstChunk.at(-1).cursor; + const savedCursor = structuredClone(cursor); + assert.deepStrictEqual( + [...index.getCompositeRange({ end: expiresAt + 2, limit: 20 })].map((entry) => entry.value), + [0, 5, 1, 2, 3, [4, 'part']] + ); + assert.deepStrictEqual(cursor, savedCursor); + assert.deepStrictEqual( + [...index.getCompositeRange({ after: cursor, end: expiresAt + 2, limit: 20 })].map((entry) => entry.value), + [1, 2, 3, [4, 'part']] + ); + }); + + it('keeps exact Rocks index matching distinct for similar values', async function () { + const Table = table({ + table: 'SimilarIndexValues', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'label', indexed: true }, + ], + }); + await Table.put(1, { id: 1, label: 'hello' }); + await Table.put(4, { id: 4, label: 'hello' }); + await Table.put(2, { id: 2, label: 'hello world' }); + await Table.put(3, { id: 3, label: 'hell' }); + await Table.put([5, 'part'], { id: [5, 'part'], label: 'hello' }); + await Table.primaryStore.committed; + + const index = Table.indices.label; + assert.deepStrictEqual([...index.getValues('hello')], [1, 4, [5, 'part']]); + assert.deepStrictEqual([...index.getValues('hello', { offset: 1, limit: 1 })], [4]); + assert.deepStrictEqual([...index.getValues('hello', { reverse: true })], [[5, 'part'], 4, 1]); + assert.deepStrictEqual([...index.getValues('hello', { reverse: true, offset: 1, limit: 1 })], [4]); + assert.deepStrictEqual([...index.getValues('hello world')], [2]); + assert.deepStrictEqual([...index.getValues('hell')], [3]); + assert.strictEqual(index.getValuesCount('hello'), 3); + assert.strictEqual(index.getValuesCount('hello world'), 1); + assert.strictEqual(index.getValuesCount('hell'), 1); + + const searchResults = []; + for await (const record of Table.search({ + allowFullScan: false, + conditions: [{ attribute: 'label', value: 'hello' }], + })) { + searchResults.push(record.id); + } + assert.deepStrictEqual(searchResults, [1, 4, [5, 'part']]); + }); + + it('physically evicts expired records and transactionally cleans dangling index entries', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSweep')); + + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.put(2, { id: 2, expiresAt }); + await Table.put([3, 'part'], { id: [3, 'part'], expiresAt }); + await Table.primaryStore.committed; + Table.primaryStore.removeSync(2); + Table.primaryStore.removeSync([3, 'part']); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1, 2, [3, 'part']]); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + Table.cleanup(); + }); + + it('retries a conflicting Rocks eviction batch', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtBatchConflict')); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); + await Table.primaryStore.committed; + let commitAttempts = 0; + + await runSweep({ + beforeBatchCommit() { + commitAttempts++; + if (commitAttempts === 1) throw Object.assign(new Error('injected optimistic conflict'), { code: 'ERR_BUSY' }); + }, + }); + + assert.strictEqual(commitAttempts, 2, 'the conflicting batch should be re-staged and committed once'); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + }); + + it('continues an expiration sweep beyond one RocksDB chunk', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSweepContinuation')); + const expiresAt = Date.now() - 1_000; + await Promise.all(Array.from({ length: 505 }, (_, id) => Table.put(id, { id, expiresAt }))); + await Table.primaryStore.committed; + + await runSweep(); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + for (let id = 0; id < 505; id++) assert.strictEqual(Table.primaryStore.getEntry(id)?.value, undefined); + Table.cleanup(); + }); + + it('physically evicts an expired ISO-string value', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtIsoSweep')); + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt: new Date(expiresAt).toISOString() }); + await Table.primaryStore.committed; + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + Table.cleanup(); + }); + + it('removes a stale expired key while preserving a refreshed record and its current key', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtStaleKey')); + const expired = Date.now() - 1_000; + const current = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt: current }); + await Table.primaryStore.committed; + Table.indices.expiresAt.put(expired, 1); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.expiresAt, current); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expired)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(current)], [1]); + Table.cleanup(); + }); + + it('removes both the encountered and canonical expiration keys when evicting a record', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtExactKeyEviction')); + const canonical = Date.now() - 1_000; + const encountered = canonical - 1_000; + await Table.put(1, { id: 1, expiresAt: canonical }); + await Table.primaryStore.committed; + Table.indices.expiresAt.put(encountered, 1); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(encountered)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(canonical)], []); + }); + + it('removes both expiration keys when physically evicting a blob record', async function () { + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'ExpiresAtExactBlobKeyEviction', + database: 'expires-at-exact-blob-key-eviction', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + const canonical = Date.now() - 1_000; + const encountered = canonical - 1_000; + const blob = createBlob(Buffer.alloc(20_000, 12)); + await Table.put(1, { id: 1, expiresAt: canonical, payload: blob }); + await Table.primaryStore.committed; + Table.indices.expiresAt.put(encountered, 1); + + setDeletionDelay(0); + try { + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(encountered)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(canonical)], []); + await waitFor(() => !existsSync(getFilePathForBlob(blob)), { + timeout: 5_000, + message: 'physical eviction should unlink the blob', + }); + } finally { + setDeletionDelay(500); + } + }); + + it('snapshots lazy Rocks metadata before eviction updates recycle the decode buffer', async function () { + const Table = table({ + table: 'ExpiresAtLazyBlobEviction', + database: 'expires-at-lazy-blob-eviction', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }); + const blob = createBlob(Buffer.alloc(20_000, 13)); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: blob }); + await Table.primaryStore.committed; + const storedEntry = Table.primaryStore.getEntry(1); + const filePath = getFilePathForBlob(blob); + const lazyState = { + version: storedEntry.version, + expiresAt: storedEntry.expiresAt, + metadataFlags: storedEntry.metadataFlags, + }; + let lazyMetadataReads = 0; + const originalGetEntry = Table.primaryStore.getEntry.bind(Table.primaryStore); + const originalIndexRemove = Table.indices.expiresAt.remove.bind(Table.indices.expiresAt); + Table.primaryStore.getEntry = (id, options) => + options?.lazy + ? { + get version() { + return lazyState.version; + }, + get expiresAt() { + return lazyState.expiresAt; + }, + get metadataFlags() { + lazyMetadataReads++; + return lazyState.metadataFlags; + }, + get key() { + return id; + }, + } + : originalGetEntry(id, options); + Table.indices.expiresAt.remove = (...args) => { + lazyState.metadataFlags = 0; + return originalIndexRemove(...args); + }; + + setDeletionDelay(0); + try { + assert.strictEqual(Table.primaryStore.ifVersion, undefined, 'premise: this must exercise RocksDB eviction'); + await Table.evict(1, storedEntry.value, storedEntry.version); + assert.strictEqual(lazyMetadataReads, 1, 'eviction must consume lazy metadata only while creating the snapshot'); + await waitFor(() => !existsSync(filePath), { + timeout: 5_000, + message: 'eviction should use the snapshotted blob metadata after index writes', + }); + } finally { + Table.primaryStore.getEntry = originalGetEntry; + Table.indices.expiresAt.remove = originalIndexRemove; + setDeletionDelay(500); + } + }); + + it('keeps a blob when eviction lacks the record and unlinks it after a committed eviction', async function () { + const Table = table({ + table: 'ExpiresAtBlobFailSafe', + database: 'expires-at-blob-fail-safe', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }); + const blob = createBlob(Buffer.alloc(20_000, 7)); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: blob }); + await Table.primaryStore.committed; + const filePath = getFilePathForBlob(blob); + const entry = Table.primaryStore.getEntry(1); + + await Table.evict(1, undefined, entry.version); + assert(Table.primaryStore.getEntry(1)?.value, 'eviction without the record must fail safe'); + assert(existsSync(filePath), 'fail-safe eviction must keep the referenced blob'); + + setDeletionDelay(0); + try { + await Table.evict(1, entry.value, entry.version); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + await waitFor(() => !existsSync(filePath), { + timeout: 5_000, + message: 'committed eviction should unlink the blob', + }); + } finally { + setDeletionDelay(500); + } + }); + + it('preserves a blob record refreshed before its eviction transaction starts', async function () { + const Table = table({ + table: 'ExpiresAtBlobRefreshRace', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }); + const oldBlob = createBlob(Buffer.alloc(20_000, 5)); + const freshBlob = createBlob(Buffer.alloc(20_000, 6)); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: oldBlob }); + await Table.primaryStore.committed; + const expiredEntry = Table.primaryStore.getEntry(1); + + const freshExpiresAt = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt: freshExpiresAt, payload: freshBlob }); + await Table.primaryStore.committed; + assert.notStrictEqual(Table.primaryStore.getEntry(1).version, expiredEntry.version); + await Table.evict(1, expiredEntry.value, expiredEntry.version); + + const refreshed = Table.primaryStore.getEntry(1)?.value; + assert.strictEqual(refreshed.expiresAt, freshExpiresAt); + assert.strictEqual(refreshed.payload.id, freshBlob.id); + assert(existsSync(getFilePathForBlob(freshBlob)), 'the refreshed record must retain its blob file'); + Table.cleanup(); + }); + + it('stops a running expiration sweep when the table is cleaned up', async function () { + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'ExpiresAtSweepCancellation', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + const expiresAt = Date.now() - 1_000; + await Table.put('a', { id: 'a', expiresAt, payload: createBlob(Buffer.alloc(20_000, 3)) }); + await Table.put('b', { id: 'b', expiresAt, payload: createBlob(Buffer.alloc(20_000, 4)) }); + await Table.primaryStore.committed; + const [, secondEntry] = [...Table.indices.expiresAt.getCompositeRange({ end: Date.now(), limit: 2 })]; + assert(secondEntry, 'the sweep should have a second entry to skip'); + + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + try { + const sweep = runSweep({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'the first eviction should start' }); + Table.cleanup(); + releaseEviction(); + await sweep; + } finally { + releaseEviction(); + } + + assert(Table.primaryStore.getEntry(secondEntry.value)?.value, 'cleanup must stop the sweep before its next entry'); + }); + + it('waits for an active expiration sweep before dropping a table', async function () { + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'ExpiresAtTableDrop', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: createBlob(Buffer.alloc(20_000, 8)) }); + await Table.primaryStore.committed; + + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + try { + const sweep = runSweep({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'the eviction should start' }); + let dropResolved = false; + const drop = Table.dropTable().then(() => (dropResolved = true)); + await delay(10); + assert.strictEqual(dropResolved, false, 'drop must wait for the active sweep'); + releaseEviction(); + await Promise.all([sweep, drop]); + } finally { + releaseEviction(); + } + }); + + it('waits for an active expiration sweep before closing a database', async function () { + const database = 'ExpiresAtDatabaseClose'; + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'expiring', + database, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: createBlob(Buffer.alloc(20_000, 9)) }); + await Table.primaryStore.committed; + + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + try { + const sweep = runSweep({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'the eviction should start' }); + let closeResolved = false; + const close = closeDatabase(database).then(() => (closeResolved = true)); + await delay(10); + assert.strictEqual(closeResolved, false, 'close must wait for the active sweep'); + releaseEviction(); + await Promise.all([sweep, close]); + } finally { + releaseEviction(); + } + }); + + it('cleanup drains the primary cleanup scan', async function () { + const Table = table({ + table: 'PrimaryCleanupDrain', + database: 'test', + expiration: 1, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'payload', type: 'Blob' }, + ], + }); + await Table.put(1, { id: 1, payload: createBlob(Buffer.alloc(20_000, 11)) }, { expiresAt: Date.now() - 1_000 }); + await Table.primaryStore.committed; + + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + try { + const scan = Table.runPrimaryCleanupScanForTests({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'the primary cleanup scan should start eviction' }); + let cleanupResolved = false; + const cleanup = Table.cleanup().then(() => (cleanupResolved = true)); + await delay(10); + assert.strictEqual(cleanupResolved, false); + releaseEviction(); + await Promise.all([scan, cleanup]); + } finally { + releaseEviction(); + } + }); + + it('drains an in-flight batch when the primary scan exits with an error', async function () { + const Table = table({ + table: 'PrimaryCleanupErrorDrain', + database: 'test', + expiration: 1, + scanInterval: 3_600, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const expiresAt = Date.now() - 1_000; + await Promise.all(Array.from({ length: 100 }, (_, id) => Table.put(id, { id }, { expiresAt }))); + await Table.primaryStore.committed; + + let releaseCommit; + let commitStarted = false; + const blockedCommit = new Promise((resolve) => (releaseCommit = resolve)); + let scanResolved = false; + try { + const scan = Table.runPrimaryCleanupScanForTests({ + beforeBatchCommit: async () => { + commitStarted = true; + await blockedCommit; + }, + afterBatchQueued() { + throw new Error('injected scan failure'); + }, + }).then(() => (scanResolved = true)); + await waitFor(() => commitStarted, { message: 'the eviction batch should start committing' }); + await delay(10); + assert.strictEqual(scanResolved, false, 'the scan must drain its batch after the injected failure'); + releaseCommit(); + await scan; + } finally { + releaseCommit(); + } + }); + + it('settles an immediate cleanup request when cleanup cancels its timer', async function () { + const Table = table({ + table: 'CleanupTimerSettlement', + database: 'test', + expiration: 60, + scanInterval: 3_600, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const scheduled = Table.scheduleCleanupForTests(2); + assert(scheduled, 'an explicit reclamation request should return its queued scan'); + const cleanup = Table.cleanup(); + const settled = await Promise.race([scheduled.then(() => true), delay(100).then(() => false)]); + assert.strictEqual(settled, true, 'cleanup must not strand a promise owned by a cleared timer'); + await cleanup; + }); + + it('keeps the recurring timer finite for full-disk reclamation priority', async function () { + const Table = table({ + table: 'FullDiskCleanupSchedule', + database: 'test', + expiration: 60, + scanInterval: 3_600, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const scheduled = Table.scheduleCleanupForTests(Infinity); + assert(scheduled, 'a full-disk reclamation request should queue an immediate scan'); + assert( + Number.isFinite(Table.cleanupStateForTests().nextCleanupScheduled), + 'the recurring cleanup deadline must stay finite' + ); + await scheduled; + }); + + it('waits for an admitted ordinary commit before closing a database', async function () { + const database = 'OrdinaryCommitCloseDrain'; + const Table = table({ + table: 'records', + database, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const originalAdmission = Table.primaryStore[TABLE_COMMIT_ADMISSION]; + const originalRelease = Table.primaryStore[TABLE_COMMIT_RELEASE]; + let admitted = false; + let releaseCommit; + const blockedRelease = new Promise((resolve) => (releaseCommit = resolve)); + Table.primaryStore[TABLE_COMMIT_ADMISSION] = () => { + originalAdmission(); + admitted = true; + return true; + }; + Table.primaryStore[TABLE_COMMIT_RELEASE] = async () => { + await blockedRelease; + originalRelease(); + }; + try { + const put = Table.put(1, { id: 1 }); + await waitFor(() => admitted, { message: 'the ordinary write should enter the table commit barrier' }); + let closeResolved = false; + const close = closeDatabase(database).then(() => (closeResolved = true)); + await delay(10); + assert.strictEqual(closeResolved, false, 'close must wait for the admitted ordinary commit'); + releaseCommit(); + await Promise.all([put, close]); + } finally { + releaseCommit(); + Table.primaryStore[TABLE_COMMIT_ADMISSION] = originalAdmission; + Table.primaryStore[TABLE_COMMIT_RELEASE] = originalRelease; + } + }); + + it('ignores an unmatched commit release without wedging later drop quiescence', async function () { + const Table = makeTable('ExpiresAtCommitReleaseFloor'); + const admit = Table.primaryStore[TABLE_COMMIT_ADMISSION]; + const release = Table.primaryStore[TABLE_COMMIT_RELEASE]; + const originalError = harperLogger.error; + const errors = []; + harperLogger.error = (...args) => errors.push(args); + try { + release(); + assert.ok(errors.some(([message]) => /unmatched table commit release/.test(message))); + assert.strictEqual(admit(), true); + let quiesced = false; + const quiesce = Table.quiesceForDrop().then(() => (quiesced = true)); + await delay(10); + assert.strictEqual(quiesced, false, 'quiescence must still wait for the balanced admission'); + release(); + await quiesce; + } finally { + harperLogger.error = originalError; + Table.abortDropQuiesce(); + } + }); + + it('joins read-path eviction to drop quiescence and skips eviction after quiescence starts', async function () { + const Table = makeTable('ExpiresAtReadEvictionQuiesce'); + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.primaryStore.committed; + const entry = Table.primaryStore.getEntry(1); + const originalAdmission = Table.primaryStore[TABLE_COMMIT_ADMISSION]; + const originalRelease = Table.primaryStore[TABLE_COMMIT_RELEASE]; + let admitted = false; + let releaseEviction; + const blockedRelease = new Promise((resolve) => (releaseEviction = resolve)); + Table.primaryStore[TABLE_COMMIT_ADMISSION] = (...args) => { + const result = originalAdmission(...args); + if (result !== false) admitted = true; + return result; + }; + Table.primaryStore[TABLE_COMMIT_RELEASE] = async () => { + await blockedRelease; + originalRelease(); + }; + try { + const eviction = Table.evict(1, entry.value, entry.version); + await waitFor(() => admitted, { message: 'read-path eviction should enter the commit barrier' }); + let quiesced = false; + const quiesce = Table.quiesceForDrop().then(() => (quiesced = true)); + await delay(10); + assert.strictEqual(quiesced, false, 'drop quiescence must wait for an admitted eviction'); + releaseEviction(); + await Promise.all([eviction, quiesce]); + + await assert.doesNotReject(Table.evict(1, entry.value, entry.version)); + } finally { + releaseEviction(); + Table.primaryStore[TABLE_COMMIT_ADMISSION] = originalAdmission; + Table.primaryStore[TABLE_COMMIT_RELEASE] = originalRelease; + Table.abortDropQuiesce(); + } + }); + + it('waits for an active expiration sweep before destroying a database', async function () { + const database = 'ExpiresAtDatabaseDrop'; + const { Table, runSweep } = captureExpirationSweep(() => + table({ + table: 'expiring', + database, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + { name: 'payload', type: 'Blob' }, + ], + }) + ); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000, payload: createBlob(Buffer.alloc(20_000, 10)) }); + await Table.primaryStore.committed; + + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + try { + const sweep = runSweep({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'the eviction should start' }); + let dropResolved = false; + const drop = dropDatabase(database).then(() => (dropResolved = true)); + await delay(10); + assert.strictEqual(dropResolved, false, 'drop must wait for the active sweep'); + releaseEviction(); + await Promise.all([sweep, drop]); + } finally { + releaseEviction(); + } + }); + + it('preserves retained tombstones while removing their dangling expiration index entries', async function () { + const { Table, runSweep } = captureExpirationSweep(() => + makeTable('ExpiresAtTombstoneSweep', undefined, { audit: true }) + ); + + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.delete(1); + await Table.primaryStore.committed; + const tombstone = Table.primaryStore.getEntry(1); + assert(tombstone); + assert.strictEqual(tombstone.value, null); + const tombstoneMetadata = { + version: tombstone.version, + expiresAt: tombstone.expiresAt, + metadataFlags: tombstone.metadataFlags, + residencyId: tombstone.residencyId, + }; + Table.indices.expiresAt.put(expiresAt, 1); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + + await runSweep(); + const retained = Table.primaryStore.getEntry(1); + assert(retained); + assert.strictEqual(retained.value, null); + assert.deepStrictEqual( + { + version: retained.version, + expiresAt: retained.expiresAt, + metadataFlags: retained.metadataFlags, + residencyId: retained.residencyId, + }, + tombstoneMetadata + ); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + Table.cleanup(); + }); + + it('conflicts dangling cleanup with a concurrent primary resurrection', async function () { + const Table = makeTable('ExpiresAtDanglingConflict'); + const expiresAt = Date.now() + 60_000; + Table.indices.expiresAt.put(expiresAt, 1); + + const transaction = new RocksTransaction(Table.primaryStore.store); + const options = { transaction }; + assert.strictEqual(Table.primaryStore.getEntry(1, options), undefined); + Table.primaryStore.removeSync(1, options); + Table.indices.expiresAt.remove(expiresAt, 1, options); + await Table.put(1, { id: 1, expiresAt }); + await assert.rejects(transaction.commit(), (error) => error?.code === 'ERR_BUSY'); + + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.id, 1); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.expiresAt, expiresAt); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + Table.cleanup(); + }); + + it('keeps a resurrection and its index when the real sweep cleanup conflicts', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtSweepConflict')); + const expiresAt = Date.now() - 1_000; + Table.indices.expiresAt.put(expiresAt, 1); + + let injected = false; + await runSweep({ + beforeBatchCommit: async () => { + if (injected) return; + injected = true; + await Table.put(1, { id: 1, expiresAt, name: 'resurrected' }); + }, + }); + + assert(injected); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.name, 'resurrected'); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + Table.cleanup(); + }); + + it('does not remove a refreshed expiration key when stale-key cleanup conflicts', async function () { + const { Table, runSweep } = captureExpirationSweep(() => makeTable('ExpiresAtStaleConflict')); + const expired = Date.now() - 1_000; + const current = Date.now() + 60_000; + const refreshed = current + 60_000; + await Table.put(1, { id: 1, expiresAt: current }); + await Table.primaryStore.committed; + Table.indices.expiresAt.put(expired, 1); + + let injected = false; + await runSweep({ + beforeBatchCommit: async () => { + if (injected) return; + injected = true; + await Table.put(1, { id: 1, expiresAt: refreshed }); + }, + }); + + assert(injected); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.expiresAt, refreshed); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(refreshed)], [1]); + await runSweep(); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expired)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(refreshed)], [1]); + Table.cleanup(); + }); + + it('conflicts retained-tombstone cleanup with a concurrent primary resurrection', async function () { + const Table = makeTable('ExpiresAtTombstoneConflict', undefined, { audit: true }); + const expiresAt = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.delete(1); + await Table.primaryStore.committed; + Table.indices.expiresAt.put(expiresAt, 1); + + const transaction = new RocksTransaction(Table.primaryStore.store); + const options = { transaction }; + const tombstone = Table.primaryStore.getEntry(1, options); + assert.strictEqual(tombstone.value, null); + const encodedTombstone = Table.primaryStore.getBinarySync(1, options); + Table.primaryStore.putSync(1, asBinary(encodedTombstone), options); + Table.indices.expiresAt.remove(expiresAt, 1, options); + await Table.put(1, { id: 1, expiresAt, name: 'resurrected' }); + await assert.rejects(transaction.commit(), (error) => error?.code === 'ERR_BUSY'); + + assert.strictEqual(Table.primaryStore.getEntry(1)?.value.name, 'resurrected'); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], [1]); + Table.cleanup(); + }); +}); + +describe('LMDB @expiresAt cleanup draining', function () { + if (process.env.HARPER_STORAGE_ENGINE !== 'lmdb') return; + + before(function () { + setupTestDBPath(); + setMainIsWorker(true); + }); + + const captureExpirationSweep = (name) => { + const Table = table({ + table: name, + database: 'lmdb-expiration-drain', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + return { + Table, + runSweep: (testHooks) => Table.runRecordExpirationSweepForTests(testHooks), + }; + }; + + it('persists an explicit no-expiration decision without indexing the public field', async function () { + const { Table } = captureExpirationSweep('LmdbExpirationDecision'); + const fieldExpiresAt = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt: fieldExpiresAt }, { expiresAt: -1 }); + await Table.primaryStore.committed; + const entry = Table.primaryStore.getEntry(1); + assert.strictEqual(entry.expiresAt, undefined); + assert.ok(entry.metadataFlags & HAS_EXPIRATION_DECISION); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(fieldExpiresAt)], []); + }); + + it('settles read-path eviction when admission throws', async function () { + const { Table } = captureExpirationSweep('LmdbExpirationAdmissionFailure'); + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.primaryStore.committed; + const entry = Table.primaryStore.getEntry(1); + const originalAdmission = Table.primaryStore[TABLE_COMMIT_ADMISSION]; + Table.primaryStore[TABLE_COMMIT_ADMISSION] = () => { + throw new Error('injected admission failure'); + }; + try { + await assert.doesNotReject(Table.evict(1, entry.value, entry.version)); + } finally { + Table.primaryStore[TABLE_COMMIT_ADMISSION] = originalAdmission; + } + }); + + it('aborts and settles read-path eviction when LMDB commit throws synchronously', async function () { + const { Table } = captureExpirationSweep('LmdbExpirationCommitFailure'); + const expiresAt = Date.now() - 1_000; + await Table.put(1, { id: 1, expiresAt }); + await Table.primaryStore.committed; + const entry = Table.primaryStore.getEntry(1); + const originalCommit = LMDBTransaction.prototype.commit; + const originalAbort = LMDBTransaction.prototype.abort; + let aborted = false; + LMDBTransaction.prototype.commit = () => { + throw new Error('injected commit failure'); + }; + LMDBTransaction.prototype.abort = function () { + aborted = true; + return originalAbort.call(this); + }; + try { + await assert.doesNotReject(Table.evict(1, entry.value, entry.version)); + assert.strictEqual(aborted, true); + } finally { + LMDBTransaction.prototype.commit = originalCommit; + LMDBTransaction.prototype.abort = originalAbort; + } + }); + + it('evicts ISO expirations and repairs stale keys', async function () { + const { Table, runSweep } = captureExpirationSweep('LmdbExpirationCorrectness'); + const expired = Date.now() - 1_000; + const current = Date.now() + 60_000; + await Table.put(1, { id: 1, expiresAt: new Date(expired).toISOString() }); + await Table.put(2, { id: 2, expiresAt: current }); + await Table.primaryStore.committed; + const encountered = expired - 1_000; + await Table.indices.expiresAt.put(expired, 2); + await Table.indices.expiresAt.put(encountered, 1); + + await runSweep(); + assert.strictEqual(Table.primaryStore.getEntry(1)?.value, undefined); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expired)], []); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(encountered)], []); + assert.strictEqual(Table.primaryStore.getEntry(2)?.value.expiresAt, current); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(current)], [2]); + Table.cleanup(); + }); + + it('paginates duplicate expiration values across bounded chunks', async function () { + const { Table, runSweep } = captureExpirationSweep('LmdbExpirationPagination'); + const expiresAt = Date.now() - 1_000; + for (let id = 0; id < 205; id++) await Table.put(id, { id, expiresAt }); + await Table.primaryStore.committed; + + await runSweep(); + assert.deepStrictEqual([...Table.indices.expiresAt.getValues(expiresAt)], []); + for (let id = 0; id < 205; id++) assert.strictEqual(Table.primaryStore.getEntry(id)?.value, undefined); + }); + + it('does not resolve cleanup while LMDB eviction writes are active', async function () { + const { Table, runSweep } = captureExpirationSweep('LmdbExpirationDrain'); + await Table.put(1, { id: 1, expiresAt: Date.now() - 1_000 }); + await Table.primaryStore.committed; + + let releaseEviction; + let evictionStarted = false; + const blockedEviction = new Promise((resolve) => (releaseEviction = resolve)); + try { + const sweep = runSweep({ + beforeEvict: async () => { + evictionStarted = true; + await blockedEviction; + }, + }); + await waitFor(() => evictionStarted, { message: 'LMDB eviction should start' }); + let cleanupResolved = false; + const cleanup = Table.cleanup().then(() => (cleanupResolved = true)); + await delay(10); + assert.strictEqual(cleanupResolved, false); + releaseEviction(); + await Promise.all([sweep, cleanup]); + } finally { + releaseEviction(); + } }); }); diff --git a/unitTests/resources/indexOrphanSameKeyTxn.test.js b/unitTests/resources/indexOrphanSameKeyTxn.test.js index 78d4e278b4..ec63354db2 100644 --- a/unitTests/resources/indexOrphanSameKeyTxn.test.js +++ b/unitTests/resources/indexOrphanSameKeyTxn.test.js @@ -204,7 +204,7 @@ describe('secondary index vs. two writes to the same key in one transaction', () setDeletionDelay(0); const BlobTbl = table({ table: 'IndexOrphanBlob', - database: 'test', + database: 'index-orphan-blob', attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'blob' }], audit: false, }); diff --git a/unitTests/resources/recordEncoder.test.js b/unitTests/resources/recordEncoder.test.js index cbb97414b7..bc6ce113e5 100644 --- a/unitTests/resources/recordEncoder.test.js +++ b/unitTests/resources/recordEncoder.test.js @@ -1,6 +1,13 @@ require('../testUtils'); const assert = require('assert'); -const { RecordEncoder, RecordObject, isMissingStructureError } = require('#src/resources/RecordEncoder'); +const { + RecordEncoder, + RecordObject, + isMissingStructureError, + setNextEncoding, + clearNextEncoding, +} = require('#src/resources/RecordEncoder'); +const { HAS_EXPIRATION_DECISION } = require('#src/resources/auditStore'); const harperLogger = require('#src/utility/logging/harper_logger'); const { Encoder } = require('msgpackr'); @@ -31,6 +38,19 @@ function makeEncoder(randomAccessStructure, store) { const record = { name: 'price', type: 'Float', indexed: true }; +describe('RecordEncoder expiration decision metadata', () => { + afterEach(() => clearNextEncoding()); + + it('adds no bytes when the decision has no expiration payload', () => { + const encoder = makeEncoder(true, sharedStore()); + setNextEncoding(1000, 0); + const withoutDecision = Buffer.from(encoder.encode(record)); + setNextEncoding(1000, HAS_EXPIRATION_DECISION); + const withDecision = Buffer.from(encoder.encode(record)); + assert.equal(withDecision.length, withoutDecision.length); + }); +}); + describe('RecordEncoder struct-mode gating', () => { it('non-primary (randomAccessStructure off) writes records mode and bails the struct write hook', () => { const store = sharedStore(); diff --git a/unitTests/resources/rocksCompressionOpen.test.js b/unitTests/resources/rocksCompressionOpen.test.js index 958221f727..faabd0d31d 100644 --- a/unitTests/resources/rocksCompressionOpen.test.js +++ b/unitTests/resources/rocksCompressionOpen.test.js @@ -37,8 +37,8 @@ describe('storage.rocks.compression reaches a real RocksDB open', function () { mkdirSync(dir, { recursive: true }); }); - afterEach(function () { - closeDatabase(databaseName); + afterEach(async function () { + await closeDatabase(databaseName); rmSync(dir, { recursive: true, force: true }); }); @@ -90,13 +90,13 @@ describe('storage.rocks.compression reaches a real RocksDB open', function () { resetRocksCompression(); const initial = database({ database: databaseName }); assert.strictEqual(initial.compression.algorithm, 'none'); - closeDatabase(databaseName); + await closeDatabase(databaseName); setProperty(CONFIG_PARAMS.STORAGE_ROCKS_COMPRESSION, 'zstd'); resetRocksCompression(); const upgraded = database({ database: databaseName }); assert.strictEqual(upgraded.compression.algorithm, 'zstd'); - closeDatabase(databaseName); + await closeDatabase(databaseName); const reopenedSibling = RocksDatabase.open(dir, { name: 'sibling' }); try { diff --git a/unitTests/resources/transaction.test.js b/unitTests/resources/transaction.test.js index 38dd5024ca..c47f777ad4 100644 --- a/unitTests/resources/transaction.test.js +++ b/unitTests/resources/transaction.test.js @@ -5,15 +5,77 @@ const { setupTestDBPath } = require('../testUtils'); const { table } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { transaction, contextStorage } = require('#src/resources/transaction'); -const serverUtilities = require('#src/server/serverHelpers/serverUtilities'); -const { DatabaseTransaction, RELEASED_TRANSACTION, TRANSACTION_STATE } = require('#src/resources/DatabaseTransaction'); +const { + DatabaseTransaction, + RELEASED_TRANSACTION, + TABLE_COMMIT_ADMISSION, + TABLE_COMMIT_RELEASE, + TRANSACTION_STATE, + withTableCommitAdmission, +} = require('#src/resources/DatabaseTransaction'); const { LMDBTransaction } = require('#src/resources/LMDBTransaction'); +const serverUtilities = require('#src/server/serverHelpers/serverUtilities'); const { IterableEventQueue } = require('#src/resources/IterableEventQueue'); const { RocksDatabase } = require('@harperfast/rocksdb-js'); const harperLogger = require('#src/utility/logging/harper_logger'); const { resetReplayedWritesWarning } = require('#src/resources/DatabaseTransaction'); const isLMDB = process.env.HARPER_STORAGE_ENGINE === 'lmdb'; +describe('Table commit admission', () => { + it('admits every existing transaction link before committing and preserves dynamic admission', async () => { + const events = []; + const store = (name) => ({ + [TABLE_COMMIT_ADMISSION]() { + events.push(`admit:${name}`); + return true; + }, + [TABLE_COMMIT_RELEASE]() { + events.push(`release:${name}`); + }, + }); + const storeA = store('a'); + const storeB = store('b'); + const storeC = store('c'); + const tail = { writes: [{ store: storeB }], next: undefined }; + const head = { writes: [{ store: storeA }], next: tail }; + + await withTableCommitAdmission(head, {}, (options) => { + assert.deepStrictEqual(events, ['admit:a', 'admit:b']); + tail.writes.push({ store: storeC }); + return withTableCommitAdmission(tail, options, async () => { + assert.deepStrictEqual(events, ['admit:a', 'admit:b', 'admit:c']); + }); + }); + + assert.deepStrictEqual(events, ['admit:a', 'admit:b', 'admit:c', 'release:a', 'release:b', 'release:c']); + }); + + it('releases a stamped transaction link after it is detached from the chain', async () => { + let pendingCommits = 0; + const store = { + [TABLE_COMMIT_ADMISSION]() { + pendingCommits++; + return true; + }, + [TABLE_COMMIT_RELEASE]() { + pendingCommits--; + }, + }; + const tail = { writes: [{ store }], next: undefined }; + const head = { writes: [], next: tail }; + + await withTableCommitAdmission(head, {}, () => { + head.next = undefined; + }); + assert.strictEqual(tail.tableCommitAdmissionOwner, undefined); + assert.strictEqual(pendingCommits, 0); + + await withTableCommitAdmission(tail, {}, () => undefined); + assert.strictEqual(tail.tableCommitAdmissionOwner, undefined); + assert.strictEqual(pendingCommits, 0, 'the detached link must release admission on its next commit'); + }); +}); + // The package blocks deep imports of its package.json, so walk up from the resolved entry point. function installedRocksdbVersion() { const { existsSync, readFileSync } = require('node:fs'); @@ -76,6 +138,29 @@ describe('Transactions', () => { assert.equal(answer.name, 'the answer'); assert.equal(answer.computed, 'the answer computed'); }); + it('aborts and releases the context when final commit admission is refused', async function () { + const context = {}; + const id = 43; + const originalAdmission = TxnTest.primaryStore[TABLE_COMMIT_ADMISSION]; + TxnTest.primaryStore[TABLE_COMMIT_ADMISSION] = () => { + const error = new Error('table is quiescing'); + error.statusCode = 503; + throw error; + }; + try { + assert.throws( + () => + transaction(context, () => { + TxnTest.put(id, { name: 'must not commit' }, context); + }), + (error) => error?.statusCode === 503 + ); + assert.strictEqual(context.transaction, RELEASED_TRANSACTION); + assert.strictEqual(TxnTest.primaryStore.getEntry(id), undefined); + } finally { + TxnTest.primaryStore[TABLE_COMMIT_ADMISSION] = originalAdmission; + } + }); it('waits for promise-returning commit callbacks on RocksDB', async function () { if (isLMDB) return this.skip(); const transaction = new DatabaseTransaction(); diff --git a/unitTests/resources/txn-tracking.test.js b/unitTests/resources/txn-tracking.test.js index 66a23ad98e..6c68735b16 100644 --- a/unitTests/resources/txn-tracking.test.js +++ b/unitTests/resources/txn-tracking.test.js @@ -172,6 +172,36 @@ describe('Write txn timeout', () => { } }); + it('immediately releases every native read handle when timeout poisoning a transaction chain', function () { + if (isLMDB) this.skip(); + const trackedTxns = setTxnExpiration(30_000); + const head = new DatabaseTransaction(); + const next = new DatabaseTransaction(); + head.next = next; + let abortedHandles = 0; + for (const [index, txn] of [head, next].entries()) { + txn.transaction = { + abort() { + abortedHandles++; + if (index === 0) throw new Error('expected abort failure'); + }, + }; + txn.readTxnsUsed = 2; + trackedTxns.add(txn); + } + + head.abortDueToTimeout(); + + assert.equal(abortedHandles, 2); + for (const txn of [head, next]) { + assert.strictEqual(txn.transaction, null); + assert.equal(txn.readTxnsUsed, 0); + assert.ok(!trackedTxns.has(txn)); + assert.strictEqual(txn.timedOut, true); + assert.equal(txn.open, TRANSACTION_STATE.CLOSED); + } + }); + // A handler that keeps reading must not extend the limit once it is holding uncommitted writes: // those hold write intents other writers' commits park on (harper#2001). The read-only arm below // pins the other half — reads alone still re-arm. RocksDB-only: LMDBTransaction.getReadTxn() diff --git a/unitTests/security/certificateVerification/certificateVerificationSource.test.js b/unitTests/security/certificateVerification/certificateVerificationSource.test.js index 965ac361e5..2052cd7e10 100644 --- a/unitTests/security/certificateVerification/certificateVerificationSource.test.js +++ b/unitTests/security/certificateVerification/certificateVerificationSource.test.js @@ -87,7 +87,7 @@ describe('certificateVerification/certificateVerificationSource.ts', function () const cacheTtl = 3600000; // 1 hour const beforeTime = Date.now(); - sinon.stub(source, 'getContext').returns({ + const context = { requestContext: { certPem: 'cert', issuerPem: 'issuer', @@ -95,7 +95,8 @@ describe('certificateVerification/certificateVerificationSource.ts', function () crl: { cacheTtl, timeout: 10000, failureMode: 'fail-closed', gracePeriod: 86400000 }, }, }, - }); + }; + sinon.stub(source, 'getContext').returns(context); performCRLCheckStub.resolves({ status: 'good' }); @@ -105,6 +106,7 @@ describe('certificateVerification/certificateVerificationSource.ts', function () // expiresAt should be approximately now + cacheTtl assert.ok(result.expiresAt >= beforeTime + cacheTtl); assert.ok(result.expiresAt <= afterTime + cacheTtl + 100); // Allow 100ms tolerance + assert.strictEqual(context.expiresAt, result.expiresAt); }); }); diff --git a/unitTests/server/itc/serverHandlers.test.js b/unitTests/server/itc/serverHandlers.test.js index 4b0c693394..851dde7776 100644 --- a/unitTests/server/itc/serverHandlers.test.js +++ b/unitTests/server/itc/serverHandlers.test.js @@ -13,6 +13,9 @@ const harperBridge = require('#src/dataLayer/harperBridge/harperBridge').default // for testing validation logic, not for replacing dependencies with mocks const server_itc_handlers = rewire('#js/server/itc/serverHandlers'); const { resetResources } = require('#src/resources/Resources'); +const { threadId } = require('node:worker_threads'); +const { isThreadConnected, waitForSchemaWorkerStarts } = require('#js/server/threads/manageThreads'); +const { waitFor } = require('../../waitFor'); describe('Test hdbChildIpcHandler module', () => { const TEST_ERR = 'The roof is on fire'; @@ -133,6 +136,352 @@ describe('Test hdbChildIpcHandler module', () => { await schema_handler(test_event); expect(log_error_stub).to.have.been.called; }); + + it('returns the explicit quiesce result without resetting databases', async () => { + const expectedResult = { quiesced: true }; + const quiesceStub = sandbox.stub().resolves(expectedResult); + const cleanStub = sandbox.stub().resolves(); + const resetStub = sandbox.stub().returns({}); + const restoreQuiesce = server_itc_handlers.__set__('quiesceSchemaTarget', quiesceStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + const restoreReset = server_itc_handlers.__set__('resetDatabases', resetStub); + try { + const message = { + originator: 12345, + operation: 'drop_table', + phase: 'quiesce', + schema: 'test', + table: 'records', + quiesceId: 'q-1', + }; + const result = await schema_handler({ type: 'schema', message }); + expect(result).to.equal(expectedResult); + expect(quiesceStub).to.have.been.calledOnceWithExactly(message); + expect(cleanStub).not.to.have.been.called; + expect(resetStub).not.to.have.been.called; + } finally { + restoreReset(); + restoreClean(); + restoreQuiesce(); + } + }); + + it('retains a lease for the worker-start barrier after the commit boundary', () => { + const leases = server_itc_handlers.__get__('schemaWorkerBarrierLeases'); + const armBarrier = server_itc_handlers.__get__('armSchemaWorkerBarrierLease'); + const commitBarrier = server_itc_handlers.__get__('commitSchemaWorkerBarrier'); + const releaseBarrier = server_itc_handlers.__get__('releaseSchemaWorkerBarrier'); + const message = { quiesceId: 'q-committed-barrier', leaseUntil: Date.now() + 60_000 }; + try { + armBarrier(message); + const firstLease = leases.get(message.quiesceId); + commitBarrier(message); + expect(leases.has(message.quiesceId)).to.equal(true); + expect(leases.get(message.quiesceId)).not.to.equal(firstLease); + } finally { + releaseBarrier(message); + } + }); + + it('retains the worker-start barrier while the committed origin is connected', async () => { + const message = { + originator: threadId, + operation: 'drop_table', + phase: 'hold-worker-starts', + schema: 'test', + table: 'records', + quiesceId: 'q-live-origin-barrier', + leaseUntil: Date.now(), + }; + await schema_handler({ type: 'schema', message }); + let released = false; + const waiting = waitForSchemaWorkerStarts().then(() => (released = true)); + try { + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(released).to.equal(false); + } finally { + await schema_handler({ type: 'schema', message: { ...message, phase: 'release-worker-starts' } }); + } + await waiting; + expect(released).to.equal(true); + }); + + it('keeps an unstamped schema teardown fenced', () => { + expect(isThreadConnected(undefined)).to.equal(true); + }); + + it('releases an expired worker-start barrier after its origin disconnects', async () => { + const message = { + originator: Number.MAX_SAFE_INTEGER, + operation: 'drop_table', + phase: 'hold-worker-starts', + schema: 'test', + table: 'records', + quiesceId: 'q-disconnected-origin-barrier', + leaseUntil: Date.now(), + }; + await schema_handler({ type: 'schema', message }); + let released = false; + waitForSchemaWorkerStarts().then(() => (released = true)); + await waitFor(() => released, { + timeout: 2_000, + message: 'worker-start barrier was not released after its origin disconnected', + }); + }); + + it('aborts quiescence without resetting databases', async () => { + const abortStub = sandbox.stub().resolves(); + const cleanStub = sandbox.stub().resolves(); + const resetStub = sandbox.stub().returns({}); + const restoreAbort = server_itc_handlers.__set__('abortSchemaQuiesce', abortStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + const restoreReset = server_itc_handlers.__set__('resetDatabases', resetStub); + try { + const message = { + originator: 12345, + operation: 'drop_table', + phase: 'abort-quiesce', + schema: 'test', + table: 'records', + quiesceId: 'q-2', + }; + const result = await schema_handler({ type: 'schema', message }); + expect(result).to.deep.equal({ aborted: true }); + expect(abortStub).to.have.been.calledOnceWithExactly(message); + expect(cleanStub).not.to.have.been.called; + expect(resetStub).not.to.have.been.called; + } finally { + restoreReset(); + restoreClean(); + restoreAbort(); + } + }); + + it('finishes quiescence before the normal database reset', async () => { + const calls = []; + const finishStub = sandbox.stub().callsFake(() => calls.push('finish')); + const cleanStub = sandbox.stub().callsFake(async () => calls.push('clean')); + const resetStub = sandbox.stub().callsFake(() => { + calls.push('reset'); + return {}; + }); + const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); + const completeStub = sandbox.stub().resolves(); + const restoreComplete = server_itc_handlers.__set__('completeSchemaQuiesce', completeStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + const restoreReset = server_itc_handlers.__set__('resetDatabases', resetStub); + try { + const message = { + originator: 12345, + operation: 'drop_table', + phase: 'finalize-quiesce', + schema: 'test', + quiesceId: 'q-3', + }; + const result = await schema_handler({ type: 'schema', message }); + expect(finishStub).to.have.been.calledOnceWithExactly(message); + expect(cleanStub).to.have.been.calledOnceWithExactly(message); + expect(resetStub).to.have.been.calledOnce; + expect(completeStub).to.have.been.calledOnceWithExactly(message); + expect(result).to.deep.equal({ finalized: true }); + expect(calls).to.deep.equal(['finish', 'clean', 'reset']); + } finally { + restoreReset(); + restoreClean(); + restoreComplete(); + restoreFinish(); + } + }); + + it('coalesces concurrent terminal retries for the same quiescence', async () => { + let releaseReset; + const resetBlocked = new Promise((resolve) => (releaseReset = resolve)); + const finishStub = sandbox.stub().returns(true); + const cleanStub = sandbox.stub().resolves(); + const syncStub = sandbox.stub().callsFake(() => resetBlocked); + const completeStub = sandbox.stub(); + const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + const restoreSync = server_itc_handlers.__set__('syncSchemaMetadata', syncStub); + const restoreComplete = server_itc_handlers.__set__('completeSchemaQuiesce', completeStub); + try { + const message = { + originator: 12345, + operation: 'drop_schema', + phase: 'finalize-quiesce', + schema: 'test', + quiesceId: 'q-concurrent-finalize', + }; + const first = schema_handler({ type: 'schema', message }); + const retry = schema_handler({ type: 'schema', message }); + releaseReset(); + expect(await Promise.all([first, retry])).to.deep.equal([{ finalized: true }, { finalized: true }]); + expect(await schema_handler({ type: 'schema', message })).to.deep.equal({ finalized: true }); + expect(finishStub).to.have.been.calledOnceWithExactly(message); + expect(cleanStub).to.have.been.calledOnceWithExactly(message); + expect(syncStub).to.have.been.calledOnceWithExactly(message, true); + expect(completeStub).to.have.been.calledOnceWithExactly(message); + } finally { + restoreComplete(); + restoreSync(); + restoreClean(); + restoreFinish(); + } + }); + + it('rejects a mismatched terminal request that reuses a completed ID', async () => { + const finishStub = sandbox.stub().returns(true); + const syncStub = sandbox.stub().resolves(); + const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); + const restoreSync = server_itc_handlers.__set__('syncSchemaMetadata', syncStub); + try { + const message = { + originator: 12345, + operation: 'drop_schema', + phase: 'finalize-quiesce', + schema: 'test', + quiesceId: 'q-terminal-identity', + }; + expect(await schema_handler({ type: 'schema', message })).to.deep.equal({ finalized: true }); + expect( + await schema_handler({ + type: 'schema', + message: { ...message, schema: 'other' }, + }) + ).to.deep.equal({ finalized: false }); + expect(finishStub).to.have.been.calledOnce; + expect(syncStub).to.have.been.calledOnce; + } finally { + restoreSync(); + restoreFinish(); + } + }); + + it('bounds retained terminal outcomes', () => { + const outcomes = server_itc_handlers.__get__('schemaTerminalOutcomes'); + const retain = server_itc_handlers.__get__('retainSchemaTerminalOutcome'); + const maximum = server_itc_handlers.__get__('MAX_SCHEMA_TERMINAL_OUTCOMES'); + outcomes.clear(); + for (let index = 0; index <= maximum; index++) { + retain( + { + operation: 'drop_schema', + phase: 'finalize-quiesce', + schema: `test-${index}`, + quiesceId: `q-bounded-${index}`, + }, + { finalized: true } + ); + } + expect(outcomes.size).to.equal(maximum); + expect(outcomes.has('q-bounded-0')).to.equal(false); + expect(outcomes.has(`q-bounded-${maximum}`)).to.equal(true); + outcomes.clear(); + }); + + it('does not acknowledge finalization when the strict reset fails', async () => { + const finishStub = sandbox.stub().returns(true); + const syncStub = sandbox.stub().rejects(new Error('reset failed')); + const failStub = sandbox.stub(); + const completeStub = sandbox.stub(); + const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); + const restoreSync = server_itc_handlers.__set__('syncSchemaMetadata', syncStub); + const restoreFail = server_itc_handlers.__set__('failSchemaQuiesceFinalization', failStub); + const restoreComplete = server_itc_handlers.__set__('completeSchemaQuiesce', completeStub); + try { + const message = { + originator: 12345, + operation: 'drop_schema', + phase: 'finalize-quiesce', + schema: 'test', + quiesceId: 'q-reset-failure', + }; + let failure; + try { + await schema_handler({ type: 'schema', message }); + } catch (error) { + failure = error; + } + expect(failure?.message).to.equal('reset failed'); + expect(failStub).to.have.been.calledOnceWithExactly(message); + expect(completeStub).not.to.have.been.called; + } finally { + restoreComplete(); + restoreFail(); + restoreSync(); + restoreFinish(); + } + }); + + it('returns an explicit result after authoritative reconciliation', async () => { + const finishStub = sandbox.stub().returns(true); + const syncStub = sandbox.stub().resolves(); + const completeStub = sandbox.stub(); + const restoreFinish = server_itc_handlers.__set__('finishSchemaQuiesce', finishStub); + const restoreSync = server_itc_handlers.__set__('syncSchemaMetadata', syncStub); + const restoreComplete = server_itc_handlers.__set__('completeSchemaQuiesce', completeStub); + try { + const message = { + originator: 12345, + operation: 'drop_table', + phase: 'reconcile-quiesce', + schema: 'test', + table: 'records', + quiesceId: 'q-reconcile', + }; + const result = await schema_handler({ type: 'schema', message }); + expect(result).to.deep.equal({ reconciled: true }); + expect(finishStub).to.have.been.calledOnceWithExactly(message); + expect(syncStub).to.have.been.calledOnceWithExactly(message, true); + expect(completeStub).to.have.been.calledOnceWithExactly(message); + } finally { + restoreComplete(); + restoreSync(); + restoreFinish(); + } + }); + + it('retries a restore close failure and still acknowledges through the schema work', async () => { + const closeStub = sandbox.stub(); + closeStub.onFirstCall().rejects(new Error('cleanup still active')); + closeStub.onSecondCall().resolves(true); + const cleanStub = sandbox.stub().resolves(); + const syncStub = sandbox.stub().resolves(); + const restoreClose = server_itc_handlers.__set__('closeDatabase', closeStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + const restoreSync = server_itc_handlers.__set__('syncSchemaMetadata', syncStub); + try { + await schema_handler({ + type: 'schema', + message: { originator: 12345, operation: 'restore_backup', schema: 'test' }, + }); + expect(closeStub).to.have.been.calledTwice; + expect(cleanStub).to.have.been.calledOnce; + expect(syncStub).to.have.been.calledOnce; + } finally { + restoreSync(); + restoreClean(); + restoreClose(); + } + }); + + it('resolves a restore close failure without rescanning an open database', async () => { + const closeStub = sandbox.stub().rejects(new Error('cleanup still active')); + const cleanStub = sandbox.stub().resolves(); + const restoreClose = server_itc_handlers.__set__('closeDatabase', closeStub); + const restoreClean = server_itc_handlers.__set__('cleanLmdbMap', cleanStub); + try { + await schema_handler({ + type: 'schema', + message: { originator: 12345, operation: 'restore_backup', schema: 'test' }, + }); + expect(closeStub).to.have.been.calledTwice; + expect(cleanStub).not.to.have.been.called; + } finally { + restoreClean(); + restoreClose(); + } + }); }); describe('Test componentStatusRequestHandler function', () => { diff --git a/unitTests/server/threads/acknowledgement-fixture.js b/unitTests/server/threads/acknowledgement-fixture.js new file mode 100644 index 0000000000..bb75d358b4 --- /dev/null +++ b/unitTests/server/threads/acknowledgement-fixture.js @@ -0,0 +1,14 @@ +'use strict'; + +const { parentPort } = require('node:worker_threads'); + +parentPort.on('message', (message) => { + if (message.type !== 'acknowledgement-test') return; + if (message.action === 'timeout') return; + if (message.action === 'close') return parentPort.close(); + parentPort.postMessage({ + type: 'ack', + id: message.requestId, + ...(message.includeAcknowledgementResult === true ? { result: message.result } : null), + }); +}); diff --git a/unitTests/server/threads/broadcastAcknowledgement.test.js b/unitTests/server/threads/broadcastAcknowledgement.test.js new file mode 100644 index 0000000000..48fa558124 --- /dev/null +++ b/unitTests/server/threads/broadcastAcknowledgement.test.js @@ -0,0 +1,151 @@ +'use strict'; + +const assert = require('node:assert'); +const path = require('node:path'); +const { + broadcastWithAcknowledgement, + ITCAcknowledgementError, + startWorker, + holdWorkerStartsForSchema, + releaseWorkerStartsForSchema, + waitForSchemaWorkerStarts, +} = require('#js/server/threads/manageThreads'); + +const FIXTURE = path.join(__dirname, 'acknowledgement-fixture.js'); +const ACCEPT_QUIESCED = (result) => result?.quiesced === true; + +describe('broadcastWithAcknowledgement', () => { + let worker; + let jobWorker; + + function startTestWorker(name) { + return new Promise((resolve, reject) => { + const spawned = startWorker(FIXTURE, { + name, + autoRestart: false, + onStarted(started) { + started.once('online', () => resolve(spawned)); + started.once('error', reject); + started.once('exit', (code) => reject(new Error(`Worker exited before online (code ${code})`))); + }, + }); + }); + } + + beforeEach(async function () { + this.timeout(30000); + worker = await startTestWorker('acknowledgement-test'); + }); + + afterEach(async () => { + releaseWorkerStartsForSchema('test-barrier-1'); + releaseWorkerStartsForSchema('test-barrier-2'); + for (const spawned of [worker, jobWorker]) { + if (!spawned) continue; + spawned.wasShutdown = true; + await spawned.terminate(); + } + worker = undefined; + jobWorker = undefined; + }); + + it('holds worker starts until every schema barrier is released', async () => { + holdWorkerStartsForSchema('test-barrier-1'); + holdWorkerStartsForSchema('test-barrier-2'); + let released = false; + const waiting = waitForSchemaWorkerStarts().then(() => (released = true)); + await new Promise(setImmediate); + assert.strictEqual(released, false); + releaseWorkerStartsForSchema('test-barrier-1'); + await new Promise(setImmediate); + assert.strictEqual(released, false); + releaseWorkerStartsForSchema('test-barrier-2'); + await waiting; + assert.strictEqual(released, true); + }); + + it('returns accepted handler results in strict mode', async () => { + const results = await broadcastWithAcknowledgement( + { type: 'acknowledgement-test', action: 'result', result: { quiesced: true } }, + { timeout: 1000, acceptResult: ACCEPT_QUIESCED } + ); + assert.strictEqual(results.length, 1); + assert.strictEqual(results[0].status, 'accepted'); + assert.deepStrictEqual(results[0].result, { quiesced: true }); + }); + + it('retains a negative handler payload in strict failures', async () => { + await assert.rejects( + broadcastWithAcknowledgement( + { type: 'acknowledgement-test', action: 'result', result: { quiesced: false, reason: 'busy' } }, + { timeout: 1000, acceptResult: ACCEPT_QUIESCED } + ), + (error) => { + assert.strictEqual(error instanceof ITCAcknowledgementError, true); + assert.strictEqual(error.results[0].status, 'rejected'); + assert.deepStrictEqual(error.results[0].result, { quiesced: false, reason: 'busy' }); + return true; + } + ); + }); + + it('distinguishes strict acknowledgement timeouts', async () => { + await assert.rejects( + broadcastWithAcknowledgement( + { type: 'acknowledgement-test', action: 'timeout' }, + { timeout: 10, acceptResult: ACCEPT_QUIESCED } + ), + (error) => { + assert.strictEqual(error.results[0].status, 'timeout'); + return true; + } + ); + }); + + it('distinguishes a port closing before acknowledgement', async () => { + await assert.rejects( + broadcastWithAcknowledgement( + { type: 'acknowledgement-test', action: 'close' }, + { timeout: 1000, acceptResult: ACCEPT_QUIESCED } + ), + (error) => { + assert.strictEqual(error.results[0].status, 'closed'); + return true; + } + ); + }); + + it('distinguishes synchronous transport failures', async () => { + await assert.rejects( + broadcastWithAcknowledgement( + { type: 'acknowledgement-test', result: () => {} }, + { timeout: 1000, acceptResult: ACCEPT_QUIESCED } + ), + (error) => { + assert.strictEqual(error.results[0].status, 'transport-error'); + return true; + } + ); + }); + + it('preserves legacy void resolution without requiring a result', async () => { + const result = await broadcastWithAcknowledgement({ type: 'acknowledgement-test', action: 'legacy' }, 1000); + assert.strictEqual(result, undefined); + }); + + it('includes active job workers only when explicitly requested', async () => { + jobWorker = await startTestWorker('job'); + const message = { type: 'acknowledgement-test', action: 'result', result: { quiesced: true } }; + const defaultResults = await broadcastWithAcknowledgement(message, { + timeout: 1000, + acceptResult: ACCEPT_QUIESCED, + }); + assert.strictEqual(defaultResults.length, 1); + const ddlResults = await broadcastWithAcknowledgement(message, { + timeout: 1000, + acceptResult: ACCEPT_QUIESCED, + includeJobWorkers: true, + }); + assert.strictEqual(ddlResults.length, 2); + }); +}); diff --git a/unitTests/server/threads/preloadSafeMode.test.js b/unitTests/server/threads/preloadSafeMode.test.js index 0e1dffcb36..85df18336e 100644 --- a/unitTests/server/threads/preloadSafeMode.test.js +++ b/unitTests/server/threads/preloadSafeMode.test.js @@ -1,6 +1,8 @@ 'use strict'; const assert = require('node:assert'); +const { mkdtempSync, rmSync, writeFileSync } = require('node:fs'); +const { tmpdir } = require('node:os'); const path = require('node:path'); const envMgr = require('#src/utility/environment/environmentManager'); @@ -71,4 +73,38 @@ describe('worker preloads in safe mode', () => { assert.strictEqual(report.execArgv.includes('--import'), true); assert.strictEqual(report.execArgv.includes('--require'), true); }); + + it('refreshes resolved preloads when configuration changes after a worker starts', async function () { + this.timeout(30000); + delete process.env.HARPER_SAFE_MODE; + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOAD, null); + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE, null); + const before = await getWorkerReport(); + assert.strictEqual(before.importLoaded, false); + assert.strictEqual(before.requireLoaded, false); + + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOAD, path.join(FIXTURES, 'import.cjs')); + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE, path.join(FIXTURES, 'require.cjs')); + const after = await getWorkerReport(); + assert.strictEqual(after.importLoaded, true); + assert.strictEqual(after.requireLoaded, true); + }); + + it('retries an unresolved preload when the module becomes available', async function () { + this.timeout(30000); + delete process.env.HARPER_SAFE_MODE; + const fixtureDirectory = mkdtempSync(path.join(tmpdir(), 'preload-late-install-')); + const importPath = path.join(fixtureDirectory, 'import.cjs'); + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOAD, importPath); + envMgr.setProperty(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE, null); + try { + const before = await getWorkerReport(); + assert.strictEqual(before.importLoaded, false); + writeFileSync(importPath, 'globalThis.__harperImportPreloaded = true;\n'); + const after = await getWorkerReport(); + assert.strictEqual(after.importLoaded, true); + } finally { + rmSync(fixtureDirectory, { recursive: true, force: true }); + } + }); }); diff --git a/unitTests/utility/signalling.test.js b/unitTests/utility/signalling.test.js index 2f654f5664..3e4bf9a330 100644 --- a/unitTests/utility/signalling.test.js +++ b/unitTests/utility/signalling.test.js @@ -3,6 +3,7 @@ const chai = require('chai'); const sinon = require('sinon'); const rewire = require('rewire'); +const { threadId } = require('node:worker_threads'); const { expect } = chai; const sinon_chai = require('sinon-chai').default; chai.use(sinon_chai); @@ -26,6 +27,7 @@ describe('Test signalling module', () => { }); afterEach(() => { + send_itc_event_stub.resetBehavior(); send_itc_event_stub.returns(); sandbox.resetHistory(); }); @@ -72,4 +74,182 @@ describe('Test signalling module', () => { signalling.signalUserChange('message'); expect(log_error_stub.lastCall.args[0].name).to.equal(TEST_ERROR); }); + + it('strictly quiesces job workers and stops renewal when aborted', async () => { + send_itc_event_stub.resolves(); + const localSchemaHandler = sandbox.stub().callsFake(async (event) => { + const resultByPhase = { + 'hold-worker-starts': 'held', + 'quiesce': 'quiesced', + 'abort-quiesce': 'aborted', + 'release-worker-starts': 'released', + }; + return { [resultByPhase[event.message.phase]]: true }; + }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + try { + const message = await signalling.quiesceSchemaChange({ + operation: 'drop_table', + schema: 'unit_test', + table: 'records', + }); + const quiesceCall = send_itc_event_stub.getCalls().find((call) => call.args[0].message.phase === 'quiesce'); + expect(message.originator).to.equal(threadId); + expect(quiesceCall.args[1].includeJobWorkers).to.equal(true); + expect(quiesceCall.args[1].acceptResult({ quiesced: true })).to.equal(true); + await signalling.abortSchemaQuiesce(message); + const abortCall = send_itc_event_stub.getCalls().find((call) => call.args[0].message.phase === 'abort-quiesce'); + expect(abortCall.args[1].includeJobWorkers).to.equal(true); + expect(abortCall.args[1].acceptResult({ aborted: true })).to.equal(true); + } finally { + restoreHandlers(); + } + }); + + it('retries strict terminal finalization and requires explicit worker results', async () => { + const localSchemaHandler = sandbox.stub().resolves({ finalized: true }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + send_itc_event_stub.onFirstCall().rejects(new Error('first terminal failure')); + send_itc_event_stub.onSecondCall().rejects(new Error('second terminal failure')); + send_itc_event_stub.onThirdCall().resolves(); + send_itc_event_stub.onCall(3).resolves(); + try { + await signalling.finalizeSchemaChange({ + operation: 'drop_schema', + schema: 'unit_test', + quiesceId: 'q-final', + }); + expect(localSchemaHandler.firstCall.args[0].message.phase).to.equal('finalize-quiesce'); + const terminalCalls = send_itc_event_stub + .getCalls() + .filter((call) => call.args[0].message.phase === 'finalize-quiesce'); + expect(terminalCalls).to.have.length(3); + for (const call of terminalCalls) { + const options = call.args[1]; + expect(options.includeJobWorkers).to.equal(true); + expect(options.acceptResult({ finalized: true })).to.equal(true); + } + } finally { + restoreHandlers(); + } + }); + + it('requires explicit reconciliation results from local and job workers', async () => { + const localSchemaHandler = sandbox.stub().resolves({ reconciled: true }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + send_itc_event_stub.resolves(); + try { + await signalling.reconcileSchemaChange({ + operation: 'drop_table', + schema: 'unit_test', + table: 'records', + quiesceId: 'q-reconcile', + }); + const event = localSchemaHandler.firstCall.args[0]; + expect(event.message.phase).to.equal('reconcile-quiesce'); + const terminalCall = send_itc_event_stub + .getCalls() + .find((call) => call.args[0].message.phase === 'reconcile-quiesce'); + const options = terminalCall.args[1]; + expect(options.includeJobWorkers).to.equal(true); + expect(options.acceptResult({ reconciled: true })).to.equal(true); + expect(options.acceptResult({ finalized: true })).to.equal(false); + } finally { + restoreHandlers(); + } + }); + + it('preserves terminal failure while attempting barrier release on every worker', async () => { + const localSchemaHandler = sandbox.stub().callsFake(async (event) => { + if (event.message.phase === 'finalize-quiesce') return { finalized: true }; + if (event.message.phase === 'release-worker-starts') throw new Error('local barrier release failed'); + }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + send_itc_event_stub.callsFake(async (event) => { + if (event.message.phase === 'finalize-quiesce') throw new Error('terminal acknowledgement lost'); + }); + try { + let terminalError; + try { + await signalling.finalizeSchemaChange({ + operation: 'drop_schema', + schema: 'unit_test', + quiesceId: 'q-terminal-exhausted', + }); + } catch (error) { + terminalError = error; + } + expect(terminalError?.message).to.include('Could not finalized schema quiesce'); + expect(terminalError?.message).not.to.include('local barrier release failed'); + expect( + localSchemaHandler.getCalls().some((call) => call.args[0].message.phase === 'release-worker-starts') + ).to.equal(true); + expect( + send_itc_event_stub.getCalls().some((call) => call.args[0].message.phase === 'release-worker-starts') + ).to.equal(true); + const terminalAttempts = send_itc_event_stub + .getCalls() + .filter((call) => call.args[0].message.phase === 'finalize-quiesce'); + expect(terminalAttempts).to.have.length(3); + } finally { + restoreHandlers(); + } + }); + + it('reconciles and releases the worker barrier after a partial commit failure', async () => { + const localSchemaHandler = sandbox.stub().callsFake(async (event) => { + if (event.message.phase === 'commit-quiesce') return { committed: true }; + if (event.message.phase === 'reconcile-quiesce') return { reconciled: true }; + if (event.message.phase === 'release-worker-starts') return { released: true }; + }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + const message = { + operation: 'drop_table', + schema: 'unit_test', + table: 'records', + quiesceId: 'q-partial-commit', + }; + send_itc_event_stub.rejects(new Error('commit acknowledgement lost')); + try { + let commitError; + try { + await signalling.commitSchemaChange(message); + } catch (error) { + commitError = error; + } + expect(commitError).to.exist; + expect( + localSchemaHandler.getCalls().some((call) => call.args[0].message.phase === 'release-worker-starts') + ).to.equal(false); + send_itc_event_stub.resolves(); + await signalling.reconcileSchemaChange(message); + expect( + localSchemaHandler.getCalls().some((call) => call.args[0].message.phase === 'release-worker-starts') + ).to.equal(true); + } finally { + restoreHandlers(); + } + }); + + it('refreshes a stale lease at the schema commit boundary', async () => { + const localSchemaHandler = sandbox.stub().resolves({ committed: true }); + const restoreHandlers = signalling.__set__('serverItcHandlers', { schema: localSchemaHandler }); + send_itc_event_stub.resolves(); + const beforeCommit = Date.now(); + try { + await signalling.commitSchemaChange({ + operation: 'drop_table', + schema: 'unit_test', + table: 'records', + quiesceId: 'q-stale-commit-lease', + leaseUntil: 1, + }); + const localMessage = localSchemaHandler.firstCall.args[0].message; + const remoteMessage = send_itc_event_stub.firstCall.args[0].message; + expect(localMessage.leaseUntil).to.be.at.least(beforeCommit + 120_000); + expect(remoteMessage.leaseUntil).to.equal(localMessage.leaseUntil); + } finally { + restoreHandlers(); + } + }); }); diff --git a/utility/signalling.ts b/utility/signalling.ts index aae59d81cd..dc367bda8c 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -3,6 +3,8 @@ import * as hdbTerms from './hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import ITCEventObject from '../server/itc/utility/ITCEventObject.js'; +import { randomUUID } from 'node:crypto'; +import { threadId } from 'node:worker_threads'; let serverItcHandlers; import { sendItcEvent } from '../server/threads/itc.js'; @@ -23,6 +25,198 @@ export async function signalSchemaChange(message: any) { } } +const SCHEMA_QUIESCE_TIMEOUT_MS = 30_000; +const SCHEMA_QUIESCE_LEASE_MS = 120_000; +const SCHEMA_QUIESCE_RENEW_MS = 40_000; +const SCHEMA_FINALIZE_ATTEMPTS = 3; +const schemaQuiesceRenewals = new Map(); + +function stopSchemaQuiesceRenewal(quiesceId: string) { + const renewal = schemaQuiesceRenewals.get(quiesceId); + if (renewal) clearInterval(renewal); + schemaQuiesceRenewals.delete(quiesceId); +} + +function startSchemaQuiesceRenewal(message: any) { + stopSchemaQuiesceRenewal(message.quiesceId); + const renewal = setInterval(() => { + const renewalMessage = { + ...message, + phase: 'renew-quiesce', + leaseUntil: Date.now() + SCHEMA_QUIESCE_LEASE_MS, + }; + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, renewalMessage); + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + Promise.all([ + serverItcHandlers.schema(event).then((result) => { + if (result?.quiesced !== true) throw new Error(`Could not renew local schema quiesce ${message.quiesceId}`); + }), + sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.quiesced === true, + includeJobWorkers: true, + }), + ]).catch((error) => hdbLogger.warn('Could not renew schema quiesce lease:', error)); + }, SCHEMA_QUIESCE_RENEW_MS); + renewal.unref(); + schemaQuiesceRenewals.set(message.quiesceId, renewal); +} + +export async function quiesceSchemaChange(message: any) { + const quiesceMessage = { + ...message, + originator: threadId, + phase: 'quiesce', + quiesceId: randomUUID(), + leaseUntil: Date.now() + SCHEMA_QUIESCE_LEASE_MS, + }; + try { + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + const holdEvent = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, { + ...quiesceMessage, + phase: 'hold-worker-starts', + }); + const localHold = await serverItcHandlers.schema(holdEvent); + if (localHold?.held !== true) throw new Error(`Could not hold local worker starts for ${quiesceMessage.quiesceId}`); + await sendItcEvent(holdEvent, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.held === true, + includeJobWorkers: true, + }); + const localEvent = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, { + ...quiesceMessage, + originLocal: true, + }); + const localResult = await serverItcHandlers.schema(localEvent); + if (localResult?.quiesced !== true) + throw new Error(localResult?.reason ?? `Could not quiesce local schema target ${quiesceMessage.quiesceId}`); + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, quiesceMessage); + await sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.quiesced === true, + includeJobWorkers: true, + }); + startSchemaQuiesceRenewal(quiesceMessage); + return quiesceMessage; + } catch (error) { + await abortSchemaQuiesce(quiesceMessage); + throw error; + } +} + +export async function abortSchemaQuiesce(message: any) { + stopSchemaQuiesceRenewal(message.quiesceId); + try { + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, { + ...message, + phase: 'abort-quiesce', + }); + const localResult = await serverItcHandlers.schema(event); + if (localResult?.aborted !== true) throw new Error(`Could not abort local schema quiesce ${message.quiesceId}`); + await sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.aborted === true, + includeJobWorkers: true, + }); + } catch (error) { + hdbLogger.warn('Could not confirm schema quiesce abort on every worker:', error); + } finally { + await releaseSchemaWorkerBarrier(message); + } +} + +async function releaseSchemaWorkerBarrier(message: any) { + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, { + ...message, + phase: 'release-worker-starts', + }); + try { + await serverItcHandlers.schema(event); + } catch (error) { + hdbLogger.warn('Could not release the local worker-start barrier:', error); + } + try { + await sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.released === true, + includeJobWorkers: true, + }); + } catch (error) { + hdbLogger.warn('Could not confirm worker-start barrier release:', error); + } +} + +export async function commitSchemaChange(message: any) { + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + const commitMessage = { + ...message, + phase: 'commit-quiesce', + leaseUntil: Date.now() + SCHEMA_QUIESCE_LEASE_MS, + }; + const localEvent = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, commitMessage); + const localResult = await serverItcHandlers.schema(localEvent); + if (localResult?.committed !== true) + throw new Error(localResult?.reason ?? `Could not commit local schema quiesce ${message.quiesceId}`); + let lastError: unknown; + for (let attempt = 0; attempt < SCHEMA_FINALIZE_ATTEMPTS; attempt++) { + try { + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, commitMessage); + await sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.committed === true, + includeJobWorkers: true, + }); + return; + } catch (error) { + lastError = error; + } + } + throw new Error(`Could not commit schema quiesce ${message.quiesceId} on every worker; remaining fail-closed`, { + cause: lastError, + }); +} + +async function completeSchemaChange(message: any, phase: string, resultProperty: string) { + try { + serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + const finalMessage = { ...message, phase }; + const localEvent = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, finalMessage); + const localResult = await serverItcHandlers.schema(localEvent); + if (localResult?.[resultProperty] !== true) + throw new Error(`Could not ${resultProperty} local schema quiesce ${message.quiesceId}`); + let lastError: unknown; + for (let attempt = 0; attempt < SCHEMA_FINALIZE_ATTEMPTS; attempt++) { + try { + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, finalMessage); + await sendItcEvent(event, { + timeout: SCHEMA_QUIESCE_TIMEOUT_MS, + acceptResult: (result) => result?.[resultProperty] === true, + includeJobWorkers: true, + }); + return; + } catch (error) { + lastError = error; + } + } + throw new Error(`Could not ${resultProperty} schema quiesce ${message.quiesceId} on every worker`, { + cause: lastError, + }); + } finally { + stopSchemaQuiesceRenewal(message.quiesceId); + await releaseSchemaWorkerBarrier(message); + } +} + +export function finalizeSchemaChange(message: any) { + return completeSchemaChange(message, 'finalize-quiesce', 'finalized'); +} + +export function reconcileSchemaChange(message: any) { + return completeSchemaChange(message, 'reconcile-quiesce', 'reconciled'); +} + /** * Notify local listeners that JS resources have just been registered (resources.js loaded). This is * deliberately local-only — no ITC broadcast — because every worker registers its own JS resources,