From 220c304c02a49c5f42713108a243ae43ac1115fc Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 21:04:02 -0600 Subject: [PATCH 01/49] Serialize same-path opens with database destruction --- AGENTS.md | 7 + src/binding/core/test_seam.h | 4 +- src/binding/database/db_registry.cpp | 227 ++++++++++++++++--------- src/binding/database/db_registry.h | 3 + test/destroy.test.ts | 62 ++++++- test/fixtures/fork-destroy-failure.mts | 25 +++ test/fixtures/fork-destroy-open.mts | 48 ++++++ test/workers/destroy-open-worker.mts | 17 ++ 8 files changed, 308 insertions(+), 85 deletions(-) create mode 100644 test/fixtures/fork-destroy-failure.mts create mode 100644 test/fixtures/fork-destroy-open.mts create mode 100644 test/workers/destroy-open-worker.mts diff --git a/AGENTS.md b/AGENTS.md index cd131043a..1f761650c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -219,6 +219,8 @@ sufficient (env teardown does not honor tsfn acquire counts); see `::getenv`-vs-`process.env` caveat as `ROCKSDB_JS_PARK_TIMEOUT_MS`), so it must be set in the environment a process is started with. `0` disables the window (every rising edge emits); malformed/negative falls back to the default +- `ROCKSDB_JS_DESTROY_DELAY_MS` - Test-only: delay after descriptor teardown and + before physical database destruction (widens same-path reopen races) ## Test Structure @@ -324,6 +326,11 @@ sufficient (env teardown does not honor tsfn acquire counts); see racing close skip the purge (`use_count > 1`), so their state destructors re-run `PurgeIfUnreferenced` after releasing the ref — without that retry the skipped purge is permanent and the entry (plus the open RocksDB) leaks (HarperFast/rocksdb-js#672). + Database destruction is path-global rather than `(path, readOnly)`-scoped: `DestroyDB` closes every + descriptor for the path and keeps the path in `destroyingPaths` until both `rocksdb::DestroyDB` and + directory cleanup finish. `OpenDB` waits on that state before resolving a registry entry and must + re-resolve the map after every condition-variable wake; retaining a map-node reference across an + unlocked wait is a use-after-free when the closer erases that node. 7. **One writable BackupEngine per backup directory (kernel advisory lock)**: each backup op opens its own short-lived `rocksdb::BackupEngine`/`BackupEngineReadOnly` (`src/binding/database/backup.cpp`), and RocksDB only serializes work _within_ a single engine — it has no cross-engine lock on the directory. diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index fa3172e4d..c7e3c15fb 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -9,8 +9,8 @@ // production where the env var is unset. // // Pass the env var name to testDelayMs() at the call site; see -// EventEmitter::notify, TransactionHandle::close, and TransactionHandle::get -// for usage. +// EventEmitter::notify, TransactionHandle::close, TransactionHandle::get, and +// DBRegistry::DestroyDB for usage. inline int testDelayMs(const char* envName) { const char* value = ::getenv(envName); return value ? ::atoi(value) : 0; diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 1900fd7b3..f8e350a1a 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -2,15 +2,54 @@ #include #include "database/db_registry.h" #include "transaction/transaction_handle.h" +#include "core/test_seam.h" #include "napi/macros.h" #include "core/platform.h" #include "core/compression.h" #include "napi/helpers.h" #include "napi/async.h" #include "rocksdb/table.h" +#include +#include namespace rocksdb_js { +namespace { + +constexpr std::chrono::seconds DATABASE_LIFECYCLE_WAIT{30}; + +class DestroyPathGuard final { +public: + DestroyPathGuard( + std::mutex& mutex, + std::condition_variable& condition, + std::unordered_set& destroyingPaths, + std::string path + ) : mutex(mutex), condition(condition), destroyingPaths(destroyingPaths), path(std::move(path)) {} + + ~DestroyPathGuard() { + { + std::lock_guard lock(this->mutex); + this->destroyingPaths.erase(this->path); + } + this->condition.notify_all(); + } + +private: + std::mutex& mutex; + std::condition_variable& condition; + std::unordered_set& destroyingPaths; + std::string path; +}; + +struct ClosingDescriptor final { + DBKey key; + std::shared_ptr descriptor; + std::shared_ptr condition; +}; + +} // namespace + // Initialize the static instance std::unique_ptr DBRegistry::instance; @@ -110,9 +149,8 @@ void DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { std::lock_guard lock(instance->databasesMutex); auto eraseIt = instance->databases.find(key); - // Only erase the entry we claimed. OpenDB's wait predicate may have - // reset the map's descriptor ref to null while we closed; a brand-new - // descriptor cannot appear because OpenDB blocks until we notify below. + // Only erase the entry we claimed. A brand-new descriptor cannot appear + // because OpenDB blocks until we notify below. if (eraseIt != instance->databases.end() && (!eraseIt->second.descriptor || eraseIt->second.descriptor == descriptor)) { instance->databases.erase(eraseIt); @@ -150,76 +188,87 @@ void DBRegistry::DestroyDB(const std::string& path) { } DEBUG_LOG("%p DBRegistry::DestroyDB Destroying \"%s\"\n", instance.get(), path.c_str()); - - std::shared_ptr descriptor; - std::shared_ptr condition; - - // Claim the descriptor under the lock but leave the entry in the map until - // the close completes (same discipline as CloseDB): the entry is how the - // env-cleanup hooks (RemoveListenersByEnv / ReleaseCommitCompletionsByEnv) - // find shared descriptors, so erasing before close would let a worker env - // tear down in that window without scrubbing its tsfns from this - // descriptor — the close's own release pass would then touch freed tsfns. - // It also keeps a concurrent OpenDB waiting on the entry's condition - // instead of re-opening the path while its files are being destroyed. + const auto deadline = std::chrono::steady_clock::now() + DATABASE_LIFECYCLE_WAIT; { - std::lock_guard lock(instance->databasesMutex); - for (auto& [key, entry] : instance->databases) { - if (key.path == path && entry.descriptor) { + std::unique_lock lock(instance->databasesMutex); + if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + return instance->destroyingPaths.find(path) == instance->destroyingPaths.end(); + })) { + throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": another destroy is still in progress"); + } + instance->destroyingPaths.insert(path); + } + // A physical destroy has several throwing stages; the path gate must never + // survive one of them and permanently block every later open. + DestroyPathGuard pathGuard( + instance->databasesMutex, + instance->lifecycleCondition, + instance->destroyingPaths, + path + ); + + while (true) { + std::vector claimed; + std::vector alreadyClosing; + { + std::lock_guard lock(instance->databasesMutex); + for (auto& [key, entry] : instance->databases) { + if (key.path != path || !entry.descriptor) { + continue; + } + ClosingDescriptor closing{key, entry.descriptor, entry.condition}; if (entry.descriptor->beginClose()) { - descriptor = entry.descriptor; - condition = entry.condition; - DEBUG_LOG("%p DBRegistry::DestroyDB Claimed descriptor close (ref count = %ld)\n", - instance.get(), descriptor.use_count()); + claimed.push_back(std::move(closing)); + } else { + alreadyClosing.push_back(std::move(closing)); } - break; } } - } - if (descriptor) { - // Close all closables (iterators, transactions, handles) attached to this descriptor - // This should release all DBHandle references - DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor and all attached resources (ref count = %zu)\n", - instance.get(), descriptor.use_count()); - descriptor->finishClose(); + for (auto& closing : claimed) { + DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", + instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); + closing.descriptor->finishClose(); + } - // Now that the close is complete, remove the path's entries and wake - // any OpenDB waiting on this path. { std::lock_guard lock(instance->databasesMutex); - for (auto it = instance->databases.begin(); it != instance->databases.end(); ) { - if (it->first.path == path) { - it = instance->databases.erase(it); - } else { - ++it; + for (const auto& closing : claimed) { + auto entry = instance->databases.find(closing.key); + if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { + instance->databases.erase(entry); } } } - if (condition) { - condition->notify_all(); + for (const auto& closing : claimed) { + closing.condition->notify_all(); + const size_t refCountAfterClose = closing.descriptor.use_count(); + if (refCountAfterClose > 1) { + std::string errorMsg = "Cannot destroy database: " + std::to_string(refCountAfterClose - 1) + + " reference(s) still held after closing all handles. This may indicate handles not properly closed or JavaScript objects not yet garbage collected."; + DEBUG_LOG("%p DBRegistry::DestroyDB Error: %s\n", instance.get(), errorMsg.c_str()); + throw rocksdb_js::DBException(errorMsg); + } } - // After closing, check if there are still lingering references - // Should only be our local reference (= 1) at this point - size_t refCountAfterClose = descriptor.use_count(); - if (refCountAfterClose > 1) { - std::string errorMsg = "Cannot destroy database: " + std::to_string(refCountAfterClose - 1) + - " reference(s) still held after closing all handles. This may indicate handles not properly closed or JavaScript objects not yet garbage collected."; - DEBUG_LOG("%p DBRegistry::DestroyDB Error: %s\n", instance.get(), errorMsg.c_str()); - throw rocksdb_js::DBException(errorMsg); + if (alreadyClosing.empty()) { + break; + } + for (const auto& closing : alreadyClosing) { + std::unique_lock lock(instance->databasesMutex); + if (!closing.condition->wait_until(lock, deadline, [&]() { + auto entry = instance->databases.find(closing.key); + return entry == instance->databases.end() || entry->second.descriptor != closing.descriptor; + })) { + throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": an open descriptor is still closing"); + } } + } - // Release our reference to the descriptor - // This will trigger the destructor which properly closes the DB - DEBUG_LOG("%p DBRegistry::DestroyDB Releasing descriptor reference\n", instance.get()); - descriptor.reset(); - } else { - // No open descriptor claimed; remove any placeholder entries for the - // path (an entry mid-close is erased by its closer's guarded erase). + { std::lock_guard lock(instance->databasesMutex); - for (auto it = instance->databases.begin(); it != instance->databases.end(); ) { - if (it->first.path == path && !it->second.descriptor) { + for (auto it = instance->databases.begin(); it != instance->databases.end();) { + if (it->first.path == path) { it = instance->databases.erase(it); } else { ++it; @@ -228,6 +277,10 @@ void DBRegistry::DestroyDB(const std::string& path) { } // Now the database lock should be released, safe to destroy + const int destroyDelayMs = testDelayMs("ROCKSDB_JS_DESTROY_DELAY_MS"); + if (destroyDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); + } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); rocksdb::Status status = rocksdb::DestroyDB(path, rocksdb::Options()); if (!status.ok()) { @@ -276,34 +329,42 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons std::string name = options.name.empty() ? "default" : options.name; std::shared_ptr descriptor; std::unique_lock lock(instance->databasesMutex); - - // get or create entry for this path + mode + readOnly combination + const auto deadline = std::chrono::steady_clock::now() + DATABASE_LIFECYCLE_WAIT; DBKey key{path, options.readOnly}; - auto entryIterator = instance->databases.find(key); - if (entryIterator == instance->databases.end()) { - // create entry with empty descriptor and new condition variable - auto [it, inserted] = instance->databases.emplace(key, DBRegistryEntry()); - entryIterator = it; - } + decltype(instance->databases)::iterator entryIterator; + while (true) { + if (!instance->destroyingPaths.empty() && + instance->destroyingPaths.find(path) != instance->destroyingPaths.end() + ) { + DEBUG_LOG("%p DBRegistry::OpenDB Database \"%s\" is being destroyed, waiting\n", instance.get(), path.c_str()); + if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + return instance->destroyingPaths.find(path) == instance->destroyingPaths.end(); + })) { + throw rocksdb_js::DBException("Timed out opening database \"" + path + "\": destruction is still in progress"); + } + continue; + } - auto& entry = entryIterator->second; + entryIterator = instance->databases.find(key); + if (entryIterator == instance->databases.end()) { + entryIterator = instance->databases.emplace(key, DBRegistryEntry()).first; + } + if (!entryIterator->second.descriptor || !entryIterator->second.descriptor->isClosing()) { + break; + } - // wait for any closing database on this specific path to be fully removed before proceeding - entry.condition->wait(lock, [&]() { - if (entry.descriptor) { - if (entry.descriptor->isClosing()) { - DEBUG_LOG("%p DBRegistry::OpenDB Database \"%s\" is closing, waiting for removal\n", instance.get(), path.c_str()); - entry.descriptor.reset(); - return false; // keep waiting - } - return true; // database exists and is not closing + DEBUG_LOG("%p DBRegistry::OpenDB Database \"%s\" is closing, waiting for removal\n", instance.get(), path.c_str()); + auto condition = entryIterator->second.condition; + if (!condition->wait_until(lock, deadline, [&]() { + auto current = instance->databases.find(key); + return current == instance->databases.end() || + !current->second.descriptor || !current->second.descriptor->isClosing(); + })) { + throw rocksdb_js::DBException("Timed out opening database \"" + path + "\": the previous instance is still closing"); } - return true; // database doesn't exist, can proceed - }); + } - // at this point, either: - // 1. descriptor is set to a valid, non-closing database, or - // 2. descriptor is nullptr (database doesn't exist) + auto& entry = entryIterator->second; if (entry.descriptor) { // database exists and is not closing, proceed with existing logic @@ -467,6 +528,10 @@ void DBRegistry::PurgeAll() { uint32_t i = 0; #endif for (auto it = instance->databases.begin(); it != instance->databases.end();) { + if (instance->destroyingPaths.find(it->first.path) != instance->destroyingPaths.end()) { + ++it; + continue; + } auto descriptor = it->second.descriptor; if (descriptor) { DEBUG_LOG("%p DBRegistry::PurgeAll %u) Purging \"%s\" (ref count = %ld)\n", instance.get(), i, it->first.path.c_str(), descriptor.use_count()); @@ -727,7 +792,9 @@ void DBRegistry::Shutdown() { // Collect all descriptors to close for (auto& [_key, entry] : instance->databases) { - if (entry.descriptor) { + if (entry.descriptor && + instance->destroyingPaths.find(entry.descriptor->path) == instance->destroyingPaths.end() + ) { descriptorsToClose.push_back(entry.descriptor); } } diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index 18c010f24..a8444c2fd 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -5,6 +5,7 @@ #include #include #include +#include #include "database/db_descriptor.h" #include "database/db_handle.h" #include "transaction/transaction.h" @@ -78,6 +79,8 @@ class DBRegistry final { * Mutex to protect the databases map. */ std::mutex databasesMutex; + std::condition_variable lifecycleCondition; + std::unordered_set destroyingPaths; /** * The singleton instance of the registry. diff --git a/test/destroy.test.ts b/test/destroy.test.ts index ca3c529a1..0f7cd891a 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,7 +1,41 @@ -import { dbRunner } from './lib/util.ts'; +import { dbRunner, generateDBPath } from './lib/util.ts'; +import { spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; +import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; +const destroyOpenFixture = join(__dirname, 'fixtures', 'fork-destroy-open.mts'); +const destroyFailureFixture = join(__dirname, 'fixtures', 'fork-destroy-failure.mts'); + +function runDestroyFixture( + fixture: string, + dbPath: string, + env?: NodeJS.ProcessEnv +): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [fixture, dbPath], { + env: { ...process.env, ...env }, + }); + let stderr = ''; + child.stderr.on('data', (chunk) => { + stderr += chunk.toString(); + }); + const timeout = setTimeout(() => { + child.kill(); + reject(new Error(`Destroy fixture timed out\n${stderr}`)); + }, 10_000); + child.on('error', reject); + child.on('close', (code, signal) => { + clearTimeout(timeout); + if (code === 0 && signal === null) { + resolve(); + } else { + reject(new Error(`Destroy fixture failed (code=${code}, signal=${signal})\n${stderr}`)); + } + }); + }); +} + describe('Destroy', () => { it('should destroy a closed database', () => dbRunner(async ({ db, dbPath }) => { @@ -25,19 +59,41 @@ describe('Destroy', () => { it('should destroy all related instances', () => dbRunner( - { dbOptions: [{}, { name: 'test' }] }, - async ({ db: db1, dbPath: dbPath1 }, { db: db2, dbPath: dbPath2 }) => { + { dbOptions: [{}, { name: 'test' }, { readOnly: true }] }, + async ( + { db: db1, dbPath: dbPath1 }, + { db: db2, dbPath: dbPath2 }, + { db: readOnly, dbPath: readOnlyPath } + ) => { expect(existsSync(dbPath1)).toBe(true); expect(existsSync(dbPath2)).toBe(true); + expect(existsSync(readOnlyPath)).toBe(true); expect(db1.isOpen()).toBe(true); expect(db2.isOpen()).toBe(true); + expect(readOnly.isOpen()).toBe(true); db1.destroy(); expect(existsSync(dbPath1)).toBe(false); expect(existsSync(dbPath2)).toBe(false); + expect(existsSync(readOnlyPath)).toBe(false); expect(db1.isOpen()).toBe(false); expect(db2.isOpen()).toBe(false); + expect(readOnly.isOpen()).toBe(false); } )); + + it('waits for physical destruction before reopening the same path', async () => { + await runDestroyFixture(destroyOpenFixture, generateDBPath(), { + ROCKSDB_JS_DESTROY_DELAY_MS: '1000', + }); + }, 15_000); + + it.skipIf(process.platform === 'win32')( + 'releases the path gate when physical destruction fails', + async () => { + await runDestroyFixture(destroyFailureFixture, generateDBPath()); + }, + 15_000 + ); }); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts new file mode 100644 index 000000000..6b0d31f07 --- /dev/null +++ b/test/fixtures/fork-destroy-failure.mts @@ -0,0 +1,25 @@ +import { RocksDatabase } from '../../src/index.ts'; +import { chmodSync } from 'node:fs'; + +const path = process.argv[2]; +const db = RocksDatabase.open(path); +db.putSync('key', 'value'); +db.close(); + +chmodSync(path, 0o500); +let destroyFailed = false; +try { + db.destroy(); +} catch { + destroyFailed = true; +} finally { + chmodSync(path, 0o700); +} +if (!destroyFailed) + throw new Error('Expected destroy to fail for a non-writable database directory'); + +try { + RocksDatabase.open(path).close(); +} catch { + // Physical destruction may have partially completed; only gate release is asserted. +} diff --git a/test/fixtures/fork-destroy-open.mts b/test/fixtures/fork-destroy-open.mts new file mode 100644 index 000000000..3bfcfb515 --- /dev/null +++ b/test/fixtures/fork-destroy-open.mts @@ -0,0 +1,48 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { setTimeout as delay } from 'node:timers/promises'; +import { Worker } from 'node:worker_threads'; + +const path = process.argv[2]; +const original = RocksDatabase.open(path); +original.putSync('before-destroy', 'present'); + +const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { + eval: true, + workerData: { path }, +}); + +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} + +const ready = await nextMessage(); +if (!ready.ready) throw new Error(`Destroy worker failed to initialize: ${JSON.stringify(ready)}`); +worker.postMessage({ destroy: true }); +const destroying = await nextMessage(); +if (!destroying.destroying) + throw new Error(`Destroy worker did not start: ${JSON.stringify(destroying)}`); + +const registryDeadline = Date.now() + 5_000; +while (registryStatus().some((entry) => entry.path === path)) { + if (Date.now() >= registryDeadline) throw new Error('Timed out waiting for the destroy window'); + await delay(1); +} + +const destroyResult = nextMessage(); +const startedAt = Date.now(); +const reopened = RocksDatabase.open(path); +const openDuration = Date.now() - startedAt; +const destroyed = await destroyResult; +if (!destroyed.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyed)}`); +if (openDuration < 500) throw new Error(`Reopen did not wait for destroy (${openDuration}ms)`); +if (reopened.getSync('before-destroy') !== undefined) + throw new Error('Reopen observed pre-destroy data'); +reopened.putSync('after-destroy', 'present'); +if (reopened.getSync('after-destroy') !== 'present') + throw new Error('Reopened database is not usable'); +reopened.close(); +await worker.terminate(); diff --git a/test/workers/destroy-open-worker.mts b/test/workers/destroy-open-worker.mts new file mode 100644 index 000000000..7fe2c0585 --- /dev/null +++ b/test/workers/destroy-open-worker.mts @@ -0,0 +1,17 @@ +import { RocksDatabase } from '../../src/index.ts'; +import { parentPort, workerData } from 'node:worker_threads'; + +const db = RocksDatabase.open(workerData.path); +if (!parentPort) throw new Error('Destroy/open worker requires a parent port'); +const port = parentPort; +port.postMessage({ ready: true }); + +port.once('message', () => { + port.postMessage({ destroying: true }); + try { + db.destroy(); + port.postMessage({ destroyed: true }); + } catch (error) { + port.postMessage({ error: error instanceof Error ? error.message : String(error) }); + } +}); From d8823f94d249f366778c2ca5408e900d27145611 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 21:33:10 -0600 Subject: [PATCH 02/49] Address lifecycle review findings --- AGENTS.md | 5 -- README.md | 3 + src/binding/core/test_seam.h | 4 ++ src/binding/database/db_registry.cpp | 93 +++++++++++++++++++++----- src/binding/database/db_settings.cpp | 17 +++++ src/binding/database/db_settings.h | 7 +- src/load-binding.ts | 5 ++ test/destroy.test.ts | 22 +++--- test/fixtures/fork-destroy-failure.mts | 24 +++---- 9 files changed, 133 insertions(+), 47 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1f761650c..a559e9e3f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -326,11 +326,6 @@ sufficient (env teardown does not honor tsfn acquire counts); see racing close skip the purge (`use_count > 1`), so their state destructors re-run `PurgeIfUnreferenced` after releasing the ref — without that retry the skipped purge is permanent and the entry (plus the open RocksDB) leaks (HarperFast/rocksdb-js#672). - Database destruction is path-global rather than `(path, readOnly)`-scoped: `DestroyDB` closes every - descriptor for the path and keeps the path in `destroyingPaths` until both `rocksdb::DestroyDB` and - directory cleanup finish. `OpenDB` waits on that state before resolving a registry entry and must - re-resolve the map after every condition-variable wake; retaining a map-node reference across an - unlocked wait is a use-after-free when the closer erases that node. 7. **One writable BackupEngine per backup directory (kernel advisory lock)**: each backup op opens its own short-lived `rocksdb::BackupEngine`/`BackupEngineReadOnly` (`src/binding/database/backup.cpp`), and RocksDB only serializes work _within_ a single engine — it has no cross-engine lock on the directory. diff --git a/README.md b/README.md index b7103701b..a04d0460f 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,9 @@ Sets global database settings. Defaults to 32MB. Set to `0` (zero) disables block cache for future opened databases. Existing block cache for any opened databases is resized immediately. Negative values throw an error. - `compactOnClose: boolean` When `true`, compacts the database on close. Defaults to `false`. + - `lifecycleWaitSeconds: number` Maximum time a synchronous open or destroy waits for another + lifecycle operation on the same path before throwing a retryable timeout error. Defaults to + `30` seconds and must be a positive integer. - `verificationTableEntries: number` The number of slots in the process-global [Verification Table](#verification-table). Each slot is 8 bytes, so the default of `131072` (128K) slots is 1 MB. Set to `0` to disable the verification table. This must be configured diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index c7e3c15fb..24126370d 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -16,6 +16,10 @@ inline int testDelayMs(const char* envName) { return value ? ::atoi(value) : 0; } +inline bool testFailureEnabled(const char* envName) { + return ::getenv(envName) != nullptr; +} + // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N // transaction commits to fail with TryAgain (the caller rolls back so no data is committed), // reproducing an ERR_TRY_AGAIN that a real memtable flush would cause but that is finicky to diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index f8e350a1a..bd06f2e92 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -2,6 +2,7 @@ #include #include "database/db_registry.h" #include "transaction/transaction_handle.h" +#include "database/db_settings.h" #include "core/test_seam.h" #include "napi/macros.h" #include "core/platform.h" @@ -10,22 +11,21 @@ #include "napi/async.h" #include "rocksdb/table.h" #include +#include #include namespace rocksdb_js { namespace { -constexpr std::chrono::seconds DATABASE_LIFECYCLE_WAIT{30}; - class DestroyPathGuard final { public: DestroyPathGuard( std::mutex& mutex, std::condition_variable& condition, std::unordered_set& destroyingPaths, - std::string path - ) : mutex(mutex), condition(condition), destroyingPaths(destroyingPaths), path(std::move(path)) {} + const std::string& path + ) : mutex(mutex), condition(condition), destroyingPaths(destroyingPaths), path(path) {} ~DestroyPathGuard() { { @@ -39,13 +39,14 @@ class DestroyPathGuard final { std::mutex& mutex; std::condition_variable& condition; std::unordered_set& destroyingPaths; - std::string path; + const std::string& path; }; struct ClosingDescriptor final { DBKey key; std::shared_ptr descriptor; std::shared_ptr condition; + bool closed = false; }; } // namespace @@ -188,7 +189,8 @@ void DBRegistry::DestroyDB(const std::string& path) { } DEBUG_LOG("%p DBRegistry::DestroyDB Destroying \"%s\"\n", instance.get(), path.c_str()); - const auto deadline = std::chrono::steady_clock::now() + DATABASE_LIFECYCLE_WAIT; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(DBSettings::getInstance().getLifecycleWaitSeconds()); { std::unique_lock lock(instance->databasesMutex); if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { @@ -212,6 +214,8 @@ void DBRegistry::DestroyDB(const std::string& path) { std::vector alreadyClosing; { std::lock_guard lock(instance->databasesMutex); + claimed.reserve(instance->databases.size()); + alreadyClosing.reserve(instance->databases.size()); for (auto& [key, entry] : instance->databases) { if (key.path != path || !entry.descriptor) { continue; @@ -225,15 +229,28 @@ void DBRegistry::DestroyDB(const std::string& path) { } } + // Keep entries discoverable while finishClose runs: env cleanup uses the + // registry to remove callbacks owned by a worker that exits mid-close. + std::exception_ptr closeError; for (auto& closing : claimed) { DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); - closing.descriptor->finishClose(); + try { + closing.descriptor->finishClose(); + closing.closed = true; + } catch (...) { + if (!closeError) { + closeError = std::current_exception(); + } + } } { std::lock_guard lock(instance->databasesMutex); for (const auto& closing : claimed) { + if (!closing.closed) { + continue; + } auto entry = instance->databases.find(closing.key); if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { instance->databases.erase(entry); @@ -241,7 +258,17 @@ void DBRegistry::DestroyDB(const std::string& path) { } } for (const auto& closing : claimed) { - closing.condition->notify_all(); + if (closing.closed) { + closing.condition->notify_all(); + } + } + if (closeError) { + std::rethrow_exception(closeError); + } + for (const auto& closing : claimed) { + if (!closing.closed) { + continue; + } const size_t refCountAfterClose = closing.descriptor.use_count(); if (refCountAfterClose > 1) { std::string errorMsg = "Cannot destroy database: " + std::to_string(refCountAfterClose - 1) + @@ -281,6 +308,9 @@ void DBRegistry::DestroyDB(const std::string& path) { if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); } + if (testFailureEnabled("ROCKSDB_JS_DESTROY_FAILURE")) { + throw rocksdb_js::DBException("Injected database destruction failure"); + } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); rocksdb::Status status = rocksdb::DestroyDB(path, rocksdb::Options()); if (!status.ok()) { @@ -329,7 +359,8 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons std::string name = options.name.empty() ? "default" : options.name; std::shared_ptr descriptor; std::unique_lock lock(instance->databasesMutex); - const auto deadline = std::chrono::steady_clock::now() + DATABASE_LIFECYCLE_WAIT; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(DBSettings::getInstance().getLifecycleWaitSeconds()); DBKey key{path, options.readOnly}; decltype(instance->databases)::iterator entryIterator; while (true) { @@ -784,26 +815,52 @@ void DBRegistry::ReleaseParkTimeoutsByEnv(napi_env env) { */ void DBRegistry::Shutdown() { if (instance) { - std::vector> descriptorsToClose; + std::vector descriptorsToClose; { std::lock_guard lock(instance->databasesMutex); DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); + descriptorsToClose.reserve(instance->databases.size()); - // Collect all descriptors to close - for (auto& [_key, entry] : instance->databases) { - if (entry.descriptor && - instance->destroyingPaths.find(entry.descriptor->path) == instance->destroyingPaths.end() + // Claim each close while holding the registry lock so DestroyDB can + // safely wait on the matching erase-and-notify below. + for (auto& [key, entry] : instance->databases) { + if (!entry.descriptor || + instance->destroyingPaths.find(key.path) != instance->destroyingPaths.end() ) { - descriptorsToClose.push_back(entry.descriptor); + continue; + } + ClosingDescriptor closing{key, entry.descriptor, entry.condition}; + if (entry.descriptor->beginClose()) { + descriptorsToClose.push_back(std::move(closing)); } } } // Close all descriptors without holding the lock - for (auto& descriptor : descriptorsToClose) { - DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), descriptor->path.c_str()); - descriptor->close(); + std::exception_ptr closeError; + for (auto& closing : descriptorsToClose) { + DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); + try { + closing.descriptor->finishClose(); + closing.closed = true; + } catch (...) { + if (!closeError) { + closeError = std::current_exception(); + } + continue; + } + { + std::lock_guard lock(instance->databasesMutex); + auto entry = instance->databases.find(closing.key); + if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { + instance->databases.erase(entry); + } + } + closing.condition->notify_all(); + } + if (closeError) { + std::rethrow_exception(closeError); } // Purge the registry diff --git a/src/binding/database/db_settings.cpp b/src/binding/database/db_settings.cpp index 660b50308..ee53c89a1 100644 --- a/src/binding/database/db_settings.cpp +++ b/src/binding/database/db_settings.cpp @@ -1,4 +1,5 @@ #include "database/db_settings.h" +#include #include #include "napi/macros.h" #include "core/platform.h" @@ -30,6 +31,7 @@ DBSettings::DBSettings(): writeBufferManagerAllowStall(false), writeBufferManager(nullptr), compactOnClose(false), + lifecycleWaitSeconds(30), verificationTableEntries(128 * 1024), // 128K slots = 1 MB at 8 bytes per slot verificationTableSeed(generateSeed()), verificationTable(nullptr) @@ -213,6 +215,21 @@ napi_value DBSettings::Config(napi_env env, napi_callback_info info) { NAPI_STATUS_THROWS(rocksdb_js::getProperty(env, params, "compactOnClose", settings.compactOnClose, false)); + int64_t lifecycleWaitSeconds = 0; + status = rocksdb_js::getProperty(env, params, "lifecycleWaitSeconds", lifecycleWaitSeconds, true); + if (status == napi_ok) { + if (lifecycleWaitSeconds <= 0 || + static_cast(lifecycleWaitSeconds) > std::numeric_limits::max() + ) { + ::napi_throw_range_error(env, nullptr, "Lifecycle wait seconds must be a positive integer"); + return nullptr; + } + settings.lifecycleWaitSeconds.store( + static_cast(lifecycleWaitSeconds), + std::memory_order_relaxed + ); + } + int64_t verificationTableEntries = 0; status = rocksdb_js::getProperty(env, params, "verificationTableEntries", verificationTableEntries, true); if (status == napi_ok) { diff --git a/src/binding/database/db_settings.h b/src/binding/database/db_settings.h index 63106ebce..d308ab643 100644 --- a/src/binding/database/db_settings.h +++ b/src/binding/database/db_settings.h @@ -49,6 +49,7 @@ class DBSettings final { std::mutex writeBufferManagerMutex; bool compactOnClose; + std::atomic lifecycleWaitSeconds; // Number of slots requested for the verification table. Default 128K // (1 MB at 8 bytes per slot). 0 disables the table. Configurable via @@ -98,6 +99,10 @@ class DBSettings final { return compactOnClose; } + uint32_t getLifecycleWaitSeconds() const { + return lifecycleWaitSeconds.load(std::memory_order_relaxed); + } + /** * Returns the global verification table, materializing it on first call. * After the first call, the table size is fixed for the process lifetime. @@ -122,4 +127,4 @@ class DBSettings final { } // namespace rocksdb_js -#endif \ No newline at end of file +#endif diff --git a/src/load-binding.ts b/src/load-binding.ts index 04ff17a3e..22dfa2d96 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -510,6 +510,11 @@ export type RocksDatabaseConfig = { */ verificationTableEntries?: number; compactOnClose?: boolean; + /** + * Maximum seconds an open or destroy call waits for another lifecycle + * operation on the same path. Defaults to 30. + */ + lifecycleWaitSeconds?: number; /** * Total memtable memory limit (bytes) shared across every database opened * in this process. When set, RocksDB uses a single `WriteBufferManager` so diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 0f7cd891a..90366e7ed 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,3 +1,4 @@ +import { RocksDatabase } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; @@ -37,6 +38,13 @@ function runDestroyFixture( } describe('Destroy', () => { + it('validates the lifecycle wait configuration', () => { + expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 0 })).toThrow( + 'Lifecycle wait seconds must be a positive integer' + ); + expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 30 })).not.toThrow(); + }); + it('should destroy a closed database', () => dbRunner(async ({ db, dbPath }) => { expect(db.isOpen()).toBe(true); @@ -85,15 +93,13 @@ describe('Destroy', () => { it('waits for physical destruction before reopening the same path', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { - ROCKSDB_JS_DESTROY_DELAY_MS: '1000', + ROCKSDB_JS_DESTROY_DELAY_MS: '2000', }); }, 15_000); - it.skipIf(process.platform === 'win32')( - 'releases the path gate when physical destruction fails', - async () => { - await runDestroyFixture(destroyFailureFixture, generateDBPath()); - }, - 15_000 - ); + it('releases the path gate when physical destruction fails', async () => { + await runDestroyFixture(destroyFailureFixture, generateDBPath(), { + ROCKSDB_JS_DESTROY_FAILURE: '1', + }); + }, 15_000); }); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index 6b0d31f07..c2cc60a97 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -1,25 +1,19 @@ import { RocksDatabase } from '../../src/index.ts'; -import { chmodSync } from 'node:fs'; const path = process.argv[2]; const db = RocksDatabase.open(path); db.putSync('key', 'value'); -db.close(); -chmodSync(path, 0o500); -let destroyFailed = false; +let destroyError: unknown; try { db.destroy(); -} catch { - destroyFailed = true; -} finally { - chmodSync(path, 0o700); +} catch (error) { + destroyError = error; } -if (!destroyFailed) - throw new Error('Expected destroy to fail for a non-writable database directory'); +if (!String(destroyError).includes('Injected database destruction failure')) + throw new Error(`Expected injected destroy failure, received: ${String(destroyError)}`); -try { - RocksDatabase.open(path).close(); -} catch { - // Physical destruction may have partially completed; only gate release is asserted. -} +const reopened = RocksDatabase.open(path); +if (reopened.getSync('key') !== 'value') + throw new Error('Reopen after failed destruction did not preserve the database'); +reopened.close(); From 31a4ab75447868620defc15f111f20298715fc80 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 21:44:52 -0600 Subject: [PATCH 03/49] Contain lifecycle close failures --- src/binding/binding.cpp | 17 +++++-- src/binding/core/test_seam.h | 4 -- src/binding/database/db_registry.cpp | 66 ++++++++++++++++++++++---- src/binding/database/db_registry.h | 4 ++ src/binding/database/db_settings.cpp | 8 ++-- test/destroy.test.ts | 9 ++++ test/fixtures/fork-destroy-failure.mts | 24 +++++++++- 7 files changed, 111 insertions(+), 21 deletions(-) diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 00b15dd38..efb3f5745 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -40,7 +40,12 @@ namespace rocksdb_js { */ napi_value Shutdown(napi_env env, napi_callback_info info) { GlobalEvents::Shutdown(); - DBRegistry::Shutdown(); + try { + DBRegistry::Shutdown(); + } catch (const std::exception& error) { + ::napi_throw_error(env, nullptr, error.what()); + return nullptr; + } napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; @@ -216,9 +221,13 @@ NAPI_MODULE_INIT() { int32_t newRefCount = --moduleRefCount; if (newRefCount == 0) { DEBUG_LOG("Binding::Init Cleaning up last instance, shutting down all databases\n"); - rocksdb_js::GlobalEvents::Shutdown(); - rocksdb_js::TransactionLogStoreRegistry::Shutdown(); - rocksdb_js::DBRegistry::Shutdown(); + try { + rocksdb_js::GlobalEvents::Shutdown(); + rocksdb_js::TransactionLogStoreRegistry::Shutdown(); + rocksdb_js::DBRegistry::Shutdown(); + } catch (const std::exception& error) { + ::fprintf(stderr, "rocksdb-js cleanup failed: %s\n", error.what()); + } DEBUG_LOG("Binding::Init env cleanup done\n"); } else if (newRefCount < 0) { DEBUG_LOG("Binding::Init WARNING: Module ref count went negative!\n"); diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index 24126370d..c7e3c15fb 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -16,10 +16,6 @@ inline int testDelayMs(const char* envName) { return value ? ::atoi(value) : 0; } -inline bool testFailureEnabled(const char* envName) { - return ::getenv(envName) != nullptr; -} - // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N // transaction commits to fail with TryAgain (the caller rolls back so no data is committed), // reproducing an ERR_TRY_AGAIN that a real memtable flush would cause but that is finicky to diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index bd06f2e92..086a44e3e 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -47,6 +47,13 @@ struct ClosingDescriptor final { std::shared_ptr descriptor; std::shared_ptr condition; bool closed = false; + std::string closeError; + + ClosingDescriptor( + const DBKey& key, + std::shared_ptr descriptor, + std::shared_ptr condition + ) : key(key), descriptor(std::move(descriptor)), condition(std::move(condition)) {} }; } // namespace @@ -208,6 +215,9 @@ void DBRegistry::DestroyDB(const std::string& path) { instance->destroyingPaths, path ); + // Test-only failure modes: 1 fails before physical deletion; 2 fails the + // descriptor-close stage so quarantine and waiter behavior are testable. + const int destroyFailureMode = testDelayMs("ROCKSDB_JS_DESTROY_FAILURE"); while (true) { std::vector claimed; @@ -216,6 +226,13 @@ void DBRegistry::DestroyDB(const std::string& path) { std::lock_guard lock(instance->databasesMutex); claimed.reserve(instance->databases.size()); alreadyClosing.reserve(instance->databases.size()); + for (const auto& [key, entry] : instance->databases) { + if (key.path == path && !entry.closeError.empty()) { + throw rocksdb_js::DBException( + "Cannot destroy database \"" + path + "\": previous close failed: " + entry.closeError + ); + } + } for (auto& [key, entry] : instance->databases) { if (key.path != path || !entry.descriptor) { continue; @@ -236,9 +253,18 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); try { + if (destroyFailureMode == 2) { + throw rocksdb_js::DBException("Injected database close failure"); + } closing.descriptor->finishClose(); closing.closed = true; + } catch (const std::exception& error) { + closing.closeError = error.what(); + if (!closeError) { + closeError = std::current_exception(); + } } catch (...) { + closing.closeError = "unknown native close failure"; if (!closeError) { closeError = std::current_exception(); } @@ -248,19 +274,19 @@ void DBRegistry::DestroyDB(const std::string& path) { { std::lock_guard lock(instance->databasesMutex); for (const auto& closing : claimed) { - if (!closing.closed) { + auto entry = instance->databases.find(closing.key); + if (entry == instance->databases.end() || entry->second.descriptor != closing.descriptor) { continue; } - auto entry = instance->databases.find(closing.key); - if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { + if (closing.closed) { instance->databases.erase(entry); + } else { + entry->second.closeError = closing.closeError; } } } for (const auto& closing : claimed) { - if (closing.closed) { - closing.condition->notify_all(); - } + closing.condition->notify_all(); } if (closeError) { std::rethrow_exception(closeError); @@ -308,7 +334,7 @@ void DBRegistry::DestroyDB(const std::string& path) { if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); } - if (testFailureEnabled("ROCKSDB_JS_DESTROY_FAILURE")) { + if (destroyFailureMode == 1) { throw rocksdb_js::DBException("Injected database destruction failure"); } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); @@ -380,6 +406,12 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons if (entryIterator == instance->databases.end()) { entryIterator = instance->databases.emplace(key, DBRegistryEntry()).first; } + if (!entryIterator->second.closeError.empty()) { + throw rocksdb_js::DBException( + "Cannot open database \"" + path + "\": previous close failed: " + + entryIterator->second.closeError + ); + } if (!entryIterator->second.descriptor || !entryIterator->second.descriptor->isClosing()) { break; } @@ -389,6 +421,7 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons if (!condition->wait_until(lock, deadline, [&]() { auto current = instance->databases.find(key); return current == instance->databases.end() || + !current->second.closeError.empty() || !current->second.descriptor || !current->second.descriptor->isClosing(); })) { throw rocksdb_js::DBException("Timed out opening database \"" + path + "\": the previous instance is still closing"); @@ -563,12 +596,18 @@ void DBRegistry::PurgeAll() { ++it; continue; } + if (!it->second.closeError.empty()) { + ++it; + continue; + } + auto condition = it->second.condition; auto descriptor = it->second.descriptor; if (descriptor) { DEBUG_LOG("%p DBRegistry::PurgeAll %u) Purging \"%s\" (ref count = %ld)\n", instance.get(), i, it->first.path.c_str(), descriptor.use_count()); descriptor->close(); } it = instance->databases.erase(it); + condition->notify_all(); #ifdef DEBUG ++i; #endif @@ -844,17 +883,26 @@ void DBRegistry::Shutdown() { try { closing.descriptor->finishClose(); closing.closed = true; + } catch (const std::exception& error) { + closing.closeError = error.what(); + if (!closeError) { + closeError = std::current_exception(); + } } catch (...) { + closing.closeError = "unknown native close failure"; if (!closeError) { closeError = std::current_exception(); } - continue; } { std::lock_guard lock(instance->databasesMutex); auto entry = instance->databases.find(closing.key); if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { - instance->databases.erase(entry); + if (closing.closed) { + instance->databases.erase(entry); + } else { + entry->second.closeError = closing.closeError; + } } } closing.condition->notify_all(); diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index a8444c2fd..abdcf533a 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -41,6 +41,7 @@ struct DBKeyHash { struct DBRegistryEntry final { std::shared_ptr descriptor; std::shared_ptr condition; + std::string closeError; // Default constructor DBRegistryEntry() : condition(std::make_shared()) {} @@ -79,6 +80,9 @@ class DBRegistry final { * Mutex to protect the databases map. */ std::mutex databasesMutex; + // Destruction owns a physical path across every (path, readOnly) entry. + // Waiters must re-resolve databases after every wake because the closer can + // erase the node while the mutex is released. std::condition_variable lifecycleCondition; std::unordered_set destroyingPaths; diff --git a/src/binding/database/db_settings.cpp b/src/binding/database/db_settings.cpp index ee53c89a1..2b6099bcc 100644 --- a/src/binding/database/db_settings.cpp +++ b/src/binding/database/db_settings.cpp @@ -1,4 +1,5 @@ #include "database/db_settings.h" +#include #include #include #include "napi/macros.h" @@ -215,11 +216,12 @@ napi_value DBSettings::Config(napi_env env, napi_callback_info info) { NAPI_STATUS_THROWS(rocksdb_js::getProperty(env, params, "compactOnClose", settings.compactOnClose, false)); - int64_t lifecycleWaitSeconds = 0; + double lifecycleWaitSeconds = 0; status = rocksdb_js::getProperty(env, params, "lifecycleWaitSeconds", lifecycleWaitSeconds, true); if (status == napi_ok) { - if (lifecycleWaitSeconds <= 0 || - static_cast(lifecycleWaitSeconds) > std::numeric_limits::max() + if (!std::isfinite(lifecycleWaitSeconds) || lifecycleWaitSeconds <= 0 || + std::trunc(lifecycleWaitSeconds) != lifecycleWaitSeconds || + lifecycleWaitSeconds > std::numeric_limits::max() ) { ::napi_throw_range_error(env, nullptr, "Lifecycle wait seconds must be a positive integer"); return nullptr; diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 90366e7ed..9d2aa9158 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -42,6 +42,9 @@ describe('Destroy', () => { expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 0 })).toThrow( 'Lifecycle wait seconds must be a positive integer' ); + expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 1.5 })).toThrow( + 'Lifecycle wait seconds must be a positive integer' + ); expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 30 })).not.toThrow(); }); @@ -102,4 +105,10 @@ describe('Destroy', () => { ROCKSDB_JS_DESTROY_FAILURE: '1', }); }, 15_000); + + it('quarantines a descriptor whose native close fails', async () => { + await runDestroyFixture(destroyFailureFixture, generateDBPath(), { + ROCKSDB_JS_DESTROY_FAILURE: '2', + }); + }, 15_000); }); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index c2cc60a97..15d8d0704 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -1,6 +1,7 @@ import { RocksDatabase } from '../../src/index.ts'; const path = process.argv[2]; +const failureMode = process.env.ROCKSDB_JS_DESTROY_FAILURE; const db = RocksDatabase.open(path); db.putSync('key', 'value'); @@ -10,9 +11,30 @@ try { } catch (error) { destroyError = error; } -if (!String(destroyError).includes('Injected database destruction failure')) +const expectedError = + failureMode === '2' ? 'Injected database close failure' : 'Injected database destruction failure'; +if (!String(destroyError).includes(expectedError)) throw new Error(`Expected injected destroy failure, received: ${String(destroyError)}`); +if (failureMode === '2') { + const startedAt = Date.now(); + try { + RocksDatabase.open(path); + throw new Error('Expected the failed descriptor to remain quarantined'); + } catch (error) { + if (!String(error).includes(`previous close failed: ${expectedError}`)) throw error; + } + if (Date.now() - startedAt >= 1_000) + throw new Error('Opening a quarantined descriptor waited instead of failing immediately'); + try { + db.destroy(); + throw new Error('Expected repeated destroy to report the previous close failure'); + } catch (error) { + if (!String(error).includes(`previous close failed: ${expectedError}`)) throw error; + } + process.exit(0); +} + const reopened = RocksDatabase.open(path); if (reopened.getSync('key') !== 'value') throw new Error('Reopen after failed destruction did not preserve the database'); From 3b63c060f86545070b61fe8560d02b1b2076238f Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 21:51:35 -0600 Subject: [PATCH 04/49] Close remaining lifecycle race windows --- src/binding/binding.cpp | 7 ++- src/binding/database/db_registry.cpp | 63 ++++++++++++++++++------- src/binding/database/db_registry.h | 1 + test/destroy.test.ts | 16 ++++++- test/fixtures/fork-close-failure.mts | 22 +++++++++ test/fixtures/fork-destroy-failure.mts | 9 ++-- test/fixtures/fork-shutdown-failure.mts | 28 +++++++++++ 7 files changed, 124 insertions(+), 22 deletions(-) create mode 100644 test/fixtures/fork-close-failure.mts create mode 100644 test/fixtures/fork-shutdown-failure.mts diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index efb3f5745..2a85b06ae 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -39,13 +39,16 @@ namespace rocksdb_js { * Shutdown function to ensure that we write in-memory data from all databases. */ napi_value Shutdown(napi_env env, napi_callback_info info) { - GlobalEvents::Shutdown(); try { DBRegistry::Shutdown(); } catch (const std::exception& error) { ::napi_throw_error(env, nullptr, error.what()); return nullptr; + } catch (...) { + ::napi_throw_error(env, nullptr, "Unknown native database shutdown failure"); + return nullptr; } + GlobalEvents::Shutdown(); napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; @@ -227,6 +230,8 @@ NAPI_MODULE_INIT() { rocksdb_js::DBRegistry::Shutdown(); } catch (const std::exception& error) { ::fprintf(stderr, "rocksdb-js cleanup failed: %s\n", error.what()); + } catch (...) { + ::fprintf(stderr, "rocksdb-js cleanup failed: unknown native error\n"); } DEBUG_LOG("Binding::Init env cleanup done\n"); } else if (newRefCount < 0) { diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 086a44e3e..388017bc5 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -56,6 +56,13 @@ struct ClosingDescriptor final { ) : key(key), descriptor(std::move(descriptor)), condition(std::move(condition)) {} }; +void finishCloseWithTestSeam(const std::shared_ptr& descriptor) { + if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { + throw rocksdb_js::DBException("Injected database close failure"); + } + descriptor->finishClose(); +} + } // namespace // Initialize the static instance @@ -153,15 +160,27 @@ void DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { if (descriptor) { // We claimed the close under the lock via beginClose(); run the actual // teardown now. The local copy keeps the descriptor alive throughout. - descriptor->finishClose(); + std::string closeError; + try { + finishCloseWithTestSeam(descriptor); + } catch (const std::exception& error) { + closeError = error.what(); + } catch (...) { + closeError = "unknown native close failure"; + } std::lock_guard lock(instance->databasesMutex); auto eraseIt = instance->databases.find(key); // Only erase the entry we claimed. A brand-new descriptor cannot appear // because OpenDB blocks until we notify below. - if (eraseIt != instance->databases.end() - && (!eraseIt->second.descriptor || eraseIt->second.descriptor == descriptor)) { - instance->databases.erase(eraseIt); + if (eraseIt != instance->databases.end() && eraseIt->second.descriptor == descriptor) { + if (closeError.empty()) { + instance->databases.erase(eraseIt); + } else { + eraseIt->second.closeError = closeError; + DEBUG_LOG("%p DBRegistry::PurgeIfUnreferenced Quarantined \"%s\": %s\n", + instance.get(), path.c_str(), closeError.c_str()); + } } } @@ -215,10 +234,6 @@ void DBRegistry::DestroyDB(const std::string& path) { instance->destroyingPaths, path ); - // Test-only failure modes: 1 fails before physical deletion; 2 fails the - // descriptor-close stage so quarantine and waiter behavior are testable. - const int destroyFailureMode = testDelayMs("ROCKSDB_JS_DESTROY_FAILURE"); - while (true) { std::vector claimed; std::vector alreadyClosing; @@ -253,10 +268,7 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); try { - if (destroyFailureMode == 2) { - throw rocksdb_js::DBException("Injected database close failure"); - } - closing.descriptor->finishClose(); + finishCloseWithTestSeam(closing.descriptor); closing.closed = true; } catch (const std::exception& error) { closing.closeError = error.what(); @@ -311,7 +323,8 @@ void DBRegistry::DestroyDB(const std::string& path) { std::unique_lock lock(instance->databasesMutex); if (!closing.condition->wait_until(lock, deadline, [&]() { auto entry = instance->databases.find(closing.key); - return entry == instance->databases.end() || entry->second.descriptor != closing.descriptor; + return entry == instance->databases.end() || + !entry->second.closeError.empty() || entry->second.descriptor != closing.descriptor; })) { throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": an open descriptor is still closing"); } @@ -334,7 +347,7 @@ void DBRegistry::DestroyDB(const std::string& path) { if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); } - if (destroyFailureMode == 1) { + if (testDelayMs("ROCKSDB_JS_DESTROY_FAILURE") > 0) { throw rocksdb_js::DBException("Injected database destruction failure"); } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); @@ -402,6 +415,15 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons continue; } + for (const auto& [registeredKey, registeredEntry] : instance->databases) { + if (registeredKey.path == path && !registeredEntry.closeError.empty()) { + throw rocksdb_js::DBException( + "Cannot open database \"" + path + "\": previous close failed: " + + registeredEntry.closeError + ); + } + } + entryIterator = instance->databases.find(key); if (entryIterator == instance->databases.end()) { entryIterator = instance->databases.emplace(key, DBRegistryEntry()).first; @@ -603,6 +625,10 @@ void DBRegistry::PurgeAll() { auto condition = it->second.condition; auto descriptor = it->second.descriptor; if (descriptor) { + if (descriptor->isClosing()) { + ++it; + continue; + } DEBUG_LOG("%p DBRegistry::PurgeAll %u) Purging \"%s\" (ref count = %ld)\n", instance.get(), i, it->first.path.c_str(), descriptor.use_count()); descriptor->close(); } @@ -870,7 +896,11 @@ void DBRegistry::Shutdown() { continue; } ClosingDescriptor closing{key, entry.descriptor, entry.condition}; - if (entry.descriptor->beginClose()) { + if (!entry.closeError.empty() && !entry.closeRetrying) { + entry.closeRetrying = true; + descriptorsToClose.push_back(std::move(closing)); + } else if (entry.closeError.empty() && entry.descriptor->beginClose()) { + entry.closeRetrying = true; descriptorsToClose.push_back(std::move(closing)); } } @@ -881,7 +911,7 @@ void DBRegistry::Shutdown() { for (auto& closing : descriptorsToClose) { DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); try { - closing.descriptor->finishClose(); + finishCloseWithTestSeam(closing.descriptor); closing.closed = true; } catch (const std::exception& error) { closing.closeError = error.what(); @@ -902,6 +932,7 @@ void DBRegistry::Shutdown() { instance->databases.erase(entry); } else { entry->second.closeError = closing.closeError; + entry->second.closeRetrying = false; } } } diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index abdcf533a..b2cd7b9d1 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -42,6 +42,7 @@ struct DBRegistryEntry final { std::shared_ptr descriptor; std::shared_ptr condition; std::string closeError; + bool closeRetrying = false; // Default constructor DBRegistryEntry() : condition(std::make_shared()) {} diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 9d2aa9158..abc9607e3 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -7,6 +7,8 @@ import { describe, expect, it } from 'vitest'; const destroyOpenFixture = join(__dirname, 'fixtures', 'fork-destroy-open.mts'); const destroyFailureFixture = join(__dirname, 'fixtures', 'fork-destroy-failure.mts'); +const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts'); +const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); function runDestroyFixture( fixture: string, @@ -108,7 +110,19 @@ describe('Destroy', () => { it('quarantines a descriptor whose native close fails', async () => { await runDestroyFixture(destroyFailureFixture, generateDBPath(), { - ROCKSDB_JS_DESTROY_FAILURE: '2', + ROCKSDB_JS_CLOSE_FAILURE: '1', + }); + }, 15_000); + + it('surfaces shutdown close failures and quarantines the whole path', async () => { + await runDestroyFixture(shutdownFailureFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FAILURE: '1', + }); + }, 15_000); + + it('quarantines a failed automatic last-handle close', async () => { + await runDestroyFixture(closeFailureFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FAILURE: '1', }); }, 15_000); }); diff --git a/test/fixtures/fork-close-failure.mts b/test/fixtures/fork-close-failure.mts new file mode 100644 index 000000000..765caad7d --- /dev/null +++ b/test/fixtures/fork-close-failure.mts @@ -0,0 +1,22 @@ +import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; + +const path = process.argv[2]; +const db = RocksDatabase.open(path); +db.putSync('key', 'value'); +db.close(); + +const startedAt = Date.now(); +try { + RocksDatabase.open(path); + throw new Error('Expected the failed automatic close to quarantine the path'); +} catch (error) { + if (!String(error).includes('previous close failed: Injected database close failure')) + throw error; +} +if (Date.now() - startedAt >= 1_000) + throw new Error('Open waited instead of reporting the failed automatic close immediately'); + +delete process.env.ROCKSDB_JS_CLOSE_FAILURE; +shutdown(); +if (registryStatus().some((entry) => entry.path === path)) + throw new Error('Shutdown retry did not clear the quarantined automatic close'); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index 15d8d0704..ca3d99eaf 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -1,7 +1,7 @@ import { RocksDatabase } from '../../src/index.ts'; const path = process.argv[2]; -const failureMode = process.env.ROCKSDB_JS_DESTROY_FAILURE; +const closeFailure = process.env.ROCKSDB_JS_CLOSE_FAILURE === '1'; const db = RocksDatabase.open(path); db.putSync('key', 'value'); @@ -11,12 +11,13 @@ try { } catch (error) { destroyError = error; } -const expectedError = - failureMode === '2' ? 'Injected database close failure' : 'Injected database destruction failure'; +const expectedError = closeFailure + ? 'Injected database close failure' + : 'Injected database destruction failure'; if (!String(destroyError).includes(expectedError)) throw new Error(`Expected injected destroy failure, received: ${String(destroyError)}`); -if (failureMode === '2') { +if (closeFailure) { const startedAt = Date.now(); try { RocksDatabase.open(path); diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts new file mode 100644 index 000000000..66fc09d23 --- /dev/null +++ b/test/fixtures/fork-shutdown-failure.mts @@ -0,0 +1,28 @@ +import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; + +const path = process.argv[2]; +const db = RocksDatabase.open(path); +db.putSync('key', 'value'); + +try { + shutdown(); + throw new Error('Expected shutdown to surface the injected close failure'); +} catch (error) { + if (!String(error).includes('Injected database close failure')) throw error; +} + +const startedAt = Date.now(); +try { + RocksDatabase.open(path, { readOnly: true }); + throw new Error('Expected the failed path to remain quarantined across open modes'); +} catch (error) { + if (!String(error).includes('previous close failed: Injected database close failure')) + throw error; +} +if (Date.now() - startedAt >= 1_000) + throw new Error('Cross-mode open waited instead of reporting the quarantined path immediately'); + +delete process.env.ROCKSDB_JS_CLOSE_FAILURE; +shutdown(); +if (registryStatus().some((entry) => entry.path === path)) + throw new Error('Shutdown retry did not clear the quarantined descriptor'); From 8139d8145f207874d8ab2a8d3a2fd20756a85035 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 22:01:09 -0600 Subject: [PATCH 05/49] Make descriptor close retries resumable --- src/binding/binding.cpp | 21 ++++--- src/binding/database/database.cpp | 9 ++- src/binding/database/db_descriptor.cpp | 73 +++++++++++++------------ src/binding/database/db_descriptor.h | 5 ++ src/binding/database/db_registry.cpp | 76 ++++++++++++++++---------- src/binding/database/db_registry.h | 5 +- src/load-binding.ts | 1 + test/fixtures/fork-close-failure.mts | 12 +++- 8 files changed, 127 insertions(+), 75 deletions(-) diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 2a85b06ae..19aecb435 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -224,15 +224,18 @@ NAPI_MODULE_INIT() { int32_t newRefCount = --moduleRefCount; if (newRefCount == 0) { DEBUG_LOG("Binding::Init Cleaning up last instance, shutting down all databases\n"); - try { - rocksdb_js::GlobalEvents::Shutdown(); - rocksdb_js::TransactionLogStoreRegistry::Shutdown(); - rocksdb_js::DBRegistry::Shutdown(); - } catch (const std::exception& error) { - ::fprintf(stderr, "rocksdb-js cleanup failed: %s\n", error.what()); - } catch (...) { - ::fprintf(stderr, "rocksdb-js cleanup failed: unknown native error\n"); - } + auto cleanup = [](const char* name, auto shutdown) { + try { + shutdown(); + } catch (const std::exception& error) { + ::fprintf(stderr, "rocksdb-js %s cleanup failed: %s\n", name, error.what()); + } catch (...) { + ::fprintf(stderr, "rocksdb-js %s cleanup failed: unknown native error\n", name); + } + }; + cleanup("global events", []() { rocksdb_js::GlobalEvents::Shutdown(); }); + cleanup("transaction logs", []() { rocksdb_js::TransactionLogStoreRegistry::Shutdown(); }); + cleanup("database registry", []() { rocksdb_js::DBRegistry::Shutdown(); }); DEBUG_LOG("Binding::Init env cleanup done\n"); } else if (newRefCount < 0) { DEBUG_LOG("Binding::Init WARNING: Module ref count went negative!\n"); diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 5da78bc40..8885a6764 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -190,7 +190,11 @@ napi_value Database::Close(napi_env env, napi_callback_info info) { if (*dbHandle) { DEBUG_LOG("%p Database::Close Closing database: \"%s\"\n", dbHandle->get(), (*dbHandle)->path.c_str()); - DBRegistry::CloseDB(*dbHandle); + std::string closeError = DBRegistry::CloseDB(*dbHandle); + if (!closeError.empty()) { + ::napi_throw_error(env, nullptr, closeError.c_str()); + return nullptr; + } DEBUG_LOG("%p Database::Close Closed database\n", dbHandle->get()); } else { DEBUG_LOG("%p Database::Close Database not opened\n", dbHandle->get()); @@ -447,6 +451,9 @@ napi_value Database::Destroy(napi_env env, napi_callback_info info) { DEBUG_LOG("%p Database::Destroy Error: %s\n", dbHandle->get(), e.what()); ::napi_throw_error(env, nullptr, e.what()); return nullptr; + } catch (...) { + ::napi_throw_error(env, nullptr, "Unknown native database destruction failure"); + return nullptr; } } else { ::napi_throw_error(env, nullptr, "Invalid database handle"); diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 69b1b335c..ae7801930 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -1,5 +1,6 @@ #include "core/background_error.h" #include "core/platform.h" +#include "core/test_seam.h" #include "database/db_descriptor.h" #include "database/db_settings.h" #include "napi/helpers.h" @@ -415,41 +416,42 @@ void DBDescriptor::finishClose() { DEBUG_LOG("%p DBDescriptor::close Closing \"%s\" (mode=%s read-only=%s closables=%zu columns=%zu transactions=%zu)\n", this, this->path.c_str(), this->mode == DBMode::Optimistic ? "optimistic" : "pessimistic", this->readOnly ? "true" : "false", this->closables.size(), this->columns.size(), this->transactions.size()); - // Wait for all in-flight operations to complete before cleanup. - // The closing flag is already set, so new operations will fail with "Database is closing". - // Existing operations will decrement operationsInFlight and notify us when done. - DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); - uint32_t current; - while ((current = this->operationsInFlight.load()) != 0) { - this->operationsInFlight.wait(current); - } - DEBUG_LOG("%p DBDescriptor::close All operations complete \"%s\"\n", this, this->path.c_str()); - - // Drain the commit pipeline before flushing so its data is included in - // the flush. The log lane feeds the commit lane, so it must drain first; - // its final tasks enqueue onto the still-running commit lane (or run - // inline once that lane stops). - this->logWorker.shutdown(); - this->commitWorker.shutdown(); - - // Release any remaining per-env commit-completion tsfns. An in-flight - // commit pins this descriptor (state -> txnHandle -> dbHandle -> descriptor), - // so reaching here means no commit is in flight; only idle (unref'd) tsfns - // for still-living envs can remain, and those envs will issue no further - // commits to this descriptor. Queued completions already handed to a tsfn - // are still delivered (napi_tsfn_release, not abort). - { - std::lock_guard lock(this->commitMutex); - for (auto& [env, completion] : this->commitCompletions) { - if (completion.tsfn) { - ::napi_release_threadsafe_function(completion.tsfn, napi_tsfn_release); + if (!this->closeWorkersStopped) { + // Wait for all in-flight operations to complete before cleanup. + // The closing flag is already set, so new operations will fail with "Database is closing". + // Existing operations will decrement operationsInFlight and notify us when done. + DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); + uint32_t current; + while ((current = this->operationsInFlight.load()) != 0) { + this->operationsInFlight.wait(current); + } + DEBUG_LOG("%p DBDescriptor::close All operations complete \"%s\"\n", this, this->path.c_str()); + + // Drain the commit pipeline before flushing so its data is included in + // the flush. The log lane feeds the commit lane, so it must drain first. + this->logWorker.shutdown(); + this->commitWorker.shutdown(); + + { + std::lock_guard lock(this->commitMutex); + for (auto& [env, completion] : this->commitCompletions) { + if (completion.tsfn) { + ::napi_release_threadsafe_function(completion.tsfn, napi_tsfn_release); + } } + this->commitCompletions.clear(); + this->commitCompletionsClosed = true; } - this->commitCompletions.clear(); - // Block any later registerCommitCompletion (a commit racing this close - // from another env) from re-creating a tsfn that would never be - // released; such commits fall back to the legacy libuv path. - this->commitCompletionsClosed = true; + this->closeWorkersStopped = true; + } + + // Inject after the one-shot pipeline shutdown so retry coverage exercises + // a genuinely partially-completed close rather than an untouched descriptor. + if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { + throw rocksdb_js::DBException("Injected database close failure"); + } + if (!this->db) { + return; } // We want to ensure that all in-memory data is written to disk. Keeps the waiting default on @@ -521,7 +523,10 @@ void DBDescriptor::finishClose() { // Unregister from transaction log store registry - this will clean up stores // when the last descriptor for this path is closed - TransactionLogStoreRegistry::Unregister(this->path); + if (!this->transactionLogsUnregistered) { + TransactionLogStoreRegistry::Unregister(this->path); + this->transactionLogsUnregistered = true; + } this->transactions.clear(); { diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index 5fe5ce32b..bfb874631 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -287,6 +287,11 @@ struct DBDescriptor final : public std::enable_shared_from_this { * descriptor. */ std::atomic closing{false}; + // finishClose() can be retried after a quarantined failure. These guards + // prevent its one-shot stages from running twice while later idempotent + // cleanup resumes from the failed point. + bool closeWorkersStopped = false; + bool transactionLogsUnregistered = false; /** * Counter tracking in-flight database operations. close() uses diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 388017bc5..a26fabda4 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -56,13 +56,6 @@ struct ClosingDescriptor final { ) : key(key), descriptor(std::move(descriptor)), condition(std::move(condition)) {} }; -void finishCloseWithTestSeam(const std::shared_ptr& descriptor) { - if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { - throw rocksdb_js::DBException("Injected database close failure"); - } - descriptor->finishClose(); -} - } // namespace // Initialize the static instance @@ -71,15 +64,15 @@ std::unique_ptr DBRegistry::instance; /** * Close a RocksDB database handle. */ -void DBRegistry::CloseDB(const std::shared_ptr handle) { +std::string DBRegistry::CloseDB(const std::shared_ptr handle) { if (!instance) { DEBUG_LOG("%p DBRegistry::CloseDB Registry not initialized\n", instance.get()); - return; + return {}; } if (!handle) { DEBUG_LOG("%p DBRegistry::CloseDB Invalid handle\n", instance.get()); - return; + return {}; } #ifdef DEBUG @@ -88,7 +81,7 @@ void DBRegistry::CloseDB(const std::shared_ptr handle) { if (!handle->descriptor) { DEBUG_LOG("%p DBRegistry::CloseDB Database not opened\n", instance.get()); - return; + return {}; } DBKey key{handle->descriptor->path, handle->descriptor->readOnly}; @@ -98,7 +91,7 @@ void DBRegistry::CloseDB(const std::shared_ptr handle) { // close the handle, decrements the descriptor ref count handle->close(); - DBRegistry::PurgeIfUnreferenced(key.path, key.readOnly); + return DBRegistry::PurgeIfUnreferenced(key.path, key.readOnly); } /** @@ -133,14 +126,15 @@ void DBRegistry::CloseDB(const std::shared_ptr handle) { * the duration of finishClose(), so a concurrent OpenDB keeps waiting on * the condition rather than re-opening the path mid-close. */ -void DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { +std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { if (!instance) { - return; + return {}; } DBKey key{path, readOnly}; std::shared_ptr descriptor; std::shared_ptr condition; + std::string closeError; { std::lock_guard lock(instance->databasesMutex); auto entryIterator = instance->databases.find(key); @@ -160,9 +154,8 @@ void DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { if (descriptor) { // We claimed the close under the lock via beginClose(); run the actual // teardown now. The local copy keeps the descriptor alive throughout. - std::string closeError; try { - finishCloseWithTestSeam(descriptor); + descriptor->finishClose(); } catch (const std::exception& error) { closeError = error.what(); } catch (...) { @@ -188,6 +181,7 @@ void DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { if (condition) { condition->notify_all(); } + return closeError; } /** @@ -244,7 +238,8 @@ void DBRegistry::DestroyDB(const std::string& path) { for (const auto& [key, entry] : instance->databases) { if (key.path == path && !entry.closeError.empty()) { throw rocksdb_js::DBException( - "Cannot destroy database \"" + path + "\": previous close failed: " + entry.closeError + "Cannot destroy database \"" + path + "\": previous close failed: " + + entry.closeError + ". Call shutdown() to retry cleanup" ); } } @@ -268,7 +263,7 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); try { - finishCloseWithTestSeam(closing.descriptor); + closing.descriptor->finishClose(); closing.closed = true; } catch (const std::exception& error) { closing.closeError = error.what(); @@ -419,7 +414,7 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons if (registeredKey.path == path && !registeredEntry.closeError.empty()) { throw rocksdb_js::DBException( "Cannot open database \"" + path + "\": previous close failed: " + - registeredEntry.closeError + registeredEntry.closeError + ". Call shutdown() to retry cleanup" ); } } @@ -428,12 +423,6 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons if (entryIterator == instance->databases.end()) { entryIterator = instance->databases.emplace(key, DBRegistryEntry()).first; } - if (!entryIterator->second.closeError.empty()) { - throw rocksdb_js::DBException( - "Cannot open database \"" + path + "\": previous close failed: " + - entryIterator->second.closeError - ); - } if (!entryIterator->second.descriptor || !entryIterator->second.descriptor->isClosing()) { break; } @@ -607,7 +596,9 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons */ void DBRegistry::PurgeAll() { if (instance) { - std::lock_guard lock(instance->databasesMutex); + std::exception_ptr closeError; + { + std::lock_guard lock(instance->databasesMutex); #ifdef DEBUG size_t initialSize = instance->databases.size(); DEBUG_LOG("%p DBRegistry::PurgeAll Purging %zu databases:\n", instance.get(), instance->databases.size()); @@ -630,7 +621,21 @@ void DBRegistry::PurgeAll() { continue; } DEBUG_LOG("%p DBRegistry::PurgeAll %u) Purging \"%s\" (ref count = %ld)\n", instance.get(), i, it->first.path.c_str(), descriptor.use_count()); - descriptor->close(); + try { + descriptor->close(); + } catch (const std::exception& error) { + it->second.closeError = error.what(); + condition->notify_all(); + if (!closeError) closeError = std::current_exception(); + ++it; + continue; + } catch (...) { + it->second.closeError = "unknown native close failure"; + condition->notify_all(); + if (!closeError) closeError = std::current_exception(); + ++it; + continue; + } } it = instance->databases.erase(it); condition->notify_all(); @@ -647,6 +652,10 @@ void DBRegistry::PurgeAll() { currentSize ); #endif + } + if (closeError) { + std::rethrow_exception(closeError); + } } } @@ -675,6 +684,16 @@ napi_value DBRegistry::RegistryStatus(napi_env env, napi_callback_info info) { napi_value pathValue; NAPI_STATUS_THROWS(::napi_create_string_utf8(env, key.path.c_str(), key.path.size(), &pathValue)); NAPI_STATUS_THROWS(::napi_set_named_property(env, database, "path", pathValue)); + if (!entry.closeError.empty()) { + napi_value closeErrorValue; + NAPI_STATUS_THROWS(::napi_create_string_utf8( + env, + entry.closeError.c_str(), + entry.closeError.size(), + &closeErrorValue + )); + NAPI_STATUS_THROWS(::napi_set_named_property(env, database, "closeError", closeErrorValue)); + } napi_value modeValue; std::string mode = entry.descriptor->mode == DBMode::Optimistic ? "optimistic" : "pessimistic"; NAPI_STATUS_THROWS(::napi_create_string_utf8(env, mode.c_str(), mode.size(), &modeValue)); @@ -880,6 +899,7 @@ void DBRegistry::ReleaseParkTimeoutsByEnv(napi_env env) { */ void DBRegistry::Shutdown() { if (instance) { + std::lock_guard shutdownLock(instance->shutdownMutex); std::vector descriptorsToClose; { @@ -911,7 +931,7 @@ void DBRegistry::Shutdown() { for (auto& closing : descriptorsToClose) { DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); try { - finishCloseWithTestSeam(closing.descriptor); + closing.descriptor->finishClose(); closing.closed = true; } catch (const std::exception& error) { closing.closeError = error.what(); diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index b2cd7b9d1..cf7486d93 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -81,6 +81,7 @@ class DBRegistry final { * Mutex to protect the databases map. */ std::mutex databasesMutex; + std::mutex shutdownMutex; // Destruction owns a physical path across every (path, readOnly) entry. // Waiters must re-resolve databases after every wake because the closer can // erase the node while the mutex is released. @@ -93,7 +94,7 @@ class DBRegistry final { static std::unique_ptr instance; public: - static void CloseDB(const std::shared_ptr handle); + static std::string CloseDB(const std::shared_ptr handle); #ifdef DEBUG static void DebugLogDescriptorRefs(); #endif @@ -101,7 +102,7 @@ class DBRegistry final { static void Init(napi_env env, napi_value exports); static std::unique_ptr OpenDB(const std::string& path, const DBOptions& options); static void PurgeAll(); - static void PurgeIfUnreferenced(const std::string& path, bool readOnly); + static std::string PurgeIfUnreferenced(const std::string& path, bool readOnly); static napi_value RegistryStatus(napi_env env, napi_callback_info info); static void CloseTransactionsByEnv(napi_env env); static void RemoveListenersByEnv(napi_env env); diff --git a/src/load-binding.ts b/src/load-binding.ts index 22dfa2d96..e08b2b46d 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -633,6 +633,7 @@ export type RegistryStatusTransaction = { export type RegistryStatusDB = { path: string; + closeError?: string; refCount: number; columnFamilies: string[]; transactions: number; diff --git a/test/fixtures/fork-close-failure.mts b/test/fixtures/fork-close-failure.mts index 765caad7d..c2687e7a7 100644 --- a/test/fixtures/fork-close-failure.mts +++ b/test/fixtures/fork-close-failure.mts @@ -3,7 +3,17 @@ import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; const path = process.argv[2]; const db = RocksDatabase.open(path); db.putSync('key', 'value'); -db.close(); +try { + db.close(); + throw new Error('Expected close to surface the injected native failure'); +} catch (error) { + if (!String(error).includes('Injected database close failure')) throw error; +} +if ( + registryStatus().find((entry) => entry.path === path)?.closeError !== + 'Injected database close failure' +) + throw new Error('Registry status did not expose the quarantined close failure'); const startedAt = Date.now(); try { From 94bcdb10b611ec355330c8c939be3b436514ea49 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 22:13:40 -0600 Subject: [PATCH 06/49] Resolve lifecycle review findings --- src/binding/binding.cpp | 11 ++- src/binding/database/database.cpp | 10 ++- src/binding/database/db_descriptor.cpp | 2 +- src/binding/database/db_registry.cpp | 111 ++++++++++++++---------- src/binding/database/db_registry.h | 2 +- test/destroy.test.ts | 9 +- test/fixtures/fork-gc-close-failure.mts | 37 ++++++++ test/fixtures/fork-shutdown-failure.mts | 1 - 8 files changed, 126 insertions(+), 57 deletions(-) create mode 100644 test/fixtures/fork-gc-close-failure.mts diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 19aecb435..674acbb96 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -39,16 +39,19 @@ namespace rocksdb_js { * Shutdown function to ensure that we write in-memory data from all databases. */ napi_value Shutdown(napi_env env, napi_callback_info info) { + std::string shutdownError; try { DBRegistry::Shutdown(); } catch (const std::exception& error) { - ::napi_throw_error(env, nullptr, error.what()); - return nullptr; + shutdownError = error.what(); } catch (...) { - ::napi_throw_error(env, nullptr, "Unknown native database shutdown failure"); - return nullptr; + shutdownError = "Unknown native database shutdown failure"; } GlobalEvents::Shutdown(); + if (!shutdownError.empty()) { + ::napi_throw_error(env, nullptr, shutdownError.c_str()); + return nullptr; + } napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 8885a6764..842907835 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -16,6 +16,7 @@ #include "core/platform.h" #include "napi/helpers.h" #include "napi/async.h" +#include "napi/global_events.h" #include "core/verification_table.h" #include "core/compression.h" @@ -48,7 +49,14 @@ napi_value Database::Constructor(napi_env env, napi_callback_info info) { DEBUG_LOG("Database::Constructor NativeDatabase GC'd dbHandle=%p\n", data); auto* dbHandle = static_cast*>(data); if (*dbHandle) { - DBRegistry::CloseDB(*dbHandle); + std::string path = (*dbHandle)->path; + std::string closeError = DBRegistry::CloseDB(*dbHandle); + if (!closeError.empty() && GlobalEvents::hasListeners()) { + emitGlobalEvent( + "database:closeFailed", + ListenerData::fromStrings({path, closeError}) + ); + } } delete dbHandle; }, diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index ae7801930..842e3f182 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -524,8 +524,8 @@ void DBDescriptor::finishClose() { // Unregister from transaction log store registry - this will clean up stores // when the last descriptor for this path is closed if (!this->transactionLogsUnregistered) { - TransactionLogStoreRegistry::Unregister(this->path); this->transactionLogsUnregistered = true; + TransactionLogStoreRegistry::Unregister(this->path); } this->transactions.clear(); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index a26fabda4..e560c20a7 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -235,11 +235,12 @@ void DBRegistry::DestroyDB(const std::string& path) { std::lock_guard lock(instance->databasesMutex); claimed.reserve(instance->databases.size()); alreadyClosing.reserve(instance->databases.size()); - for (const auto& [key, entry] : instance->databases) { - if (key.path == path && !entry.closeError.empty()) { + for (bool readOnly : {false, true}) { + auto entry = instance->databases.find(DBKey{path, readOnly}); + if (entry != instance->databases.end() && !entry->second.closeError.empty()) { throw rocksdb_js::DBException( "Cannot destroy database \"" + path + "\": previous close failed: " + - entry.closeError + ". Call shutdown() to retry cleanup" + entry->second.closeError + ". Call shutdown() to retry cleanup" ); } } @@ -410,11 +411,12 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons continue; } - for (const auto& [registeredKey, registeredEntry] : instance->databases) { - if (registeredKey.path == path && !registeredEntry.closeError.empty()) { + for (bool readOnly : {false, true}) { + auto registeredEntry = instance->databases.find(DBKey{path, readOnly}); + if (registeredEntry != instance->databases.end() && !registeredEntry->second.closeError.empty()) { throw rocksdb_js::DBException( "Cannot open database \"" + path + "\": previous close failed: " + - registeredEntry.closeError + ". Call shutdown() to retry cleanup" + registeredEntry->second.closeError + ". Call shutdown() to retry cleanup" ); } } @@ -596,62 +598,70 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons */ void DBRegistry::PurgeAll() { if (instance) { + std::vector descriptorsToClose; + std::vector> removedConditions; std::exception_ptr closeError; { std::lock_guard lock(instance->databasesMutex); #ifdef DEBUG - size_t initialSize = instance->databases.size(); - DEBUG_LOG("%p DBRegistry::PurgeAll Purging %zu databases:\n", instance.get(), instance->databases.size()); - uint32_t i = 0; + size_t initialSize = instance->databases.size(); + DEBUG_LOG("%p DBRegistry::PurgeAll Purging %zu databases:\n", instance.get(), initialSize); #endif - for (auto it = instance->databases.begin(); it != instance->databases.end();) { - if (instance->destroyingPaths.find(it->first.path) != instance->destroyingPaths.end()) { - ++it; - continue; - } - if (!it->second.closeError.empty()) { - ++it; - continue; - } - auto condition = it->second.condition; - auto descriptor = it->second.descriptor; - if (descriptor) { - if (descriptor->isClosing()) { + descriptorsToClose.reserve(instance->databases.size()); + for (auto it = instance->databases.begin(); it != instance->databases.end();) { + if (instance->destroyingPaths.find(it->first.path) != instance->destroyingPaths.end() || + !it->second.closeError.empty() + ) { ++it; continue; } - DEBUG_LOG("%p DBRegistry::PurgeAll %u) Purging \"%s\" (ref count = %ld)\n", instance.get(), i, it->first.path.c_str(), descriptor.use_count()); - try { - descriptor->close(); - } catch (const std::exception& error) { - it->second.closeError = error.what(); - condition->notify_all(); - if (!closeError) closeError = std::current_exception(); - ++it; - continue; - } catch (...) { - it->second.closeError = "unknown native close failure"; - condition->notify_all(); - if (!closeError) closeError = std::current_exception(); + auto descriptor = it->second.descriptor; + if (descriptor) { + if (!descriptor->beginClose()) { + ++it; + continue; + } + DEBUG_LOG("%p DBRegistry::PurgeAll Claiming \"%s\" (ref count = %ld)\n", + instance.get(), it->first.path.c_str(), descriptor.use_count()); + descriptorsToClose.emplace_back(it->first, descriptor, it->second.condition); ++it; continue; } + removedConditions.push_back(it->second.condition); + it = instance->databases.erase(it); } - it = instance->databases.erase(it); - condition->notify_all(); #ifdef DEBUG - ++i; + DEBUG_LOG("%p DBRegistry::PurgeAll Claimed %zu of %zu descriptors\n", + instance.get(), descriptorsToClose.size(), initialSize); #endif } -#ifdef DEBUG - size_t currentSize = instance->databases.size(); - DEBUG_LOG( - "%p DBRegistry::PurgeAll Purged %zu unused descriptors (size=%zu)\n", - instance.get(), - initialSize - currentSize, - currentSize - ); -#endif + for (const auto& condition : removedConditions) { + condition->notify_all(); + } + + for (auto& closing : descriptorsToClose) { + try { + closing.descriptor->finishClose(); + closing.closed = true; + } catch (const std::exception& error) { + closing.closeError = error.what(); + if (!closeError) closeError = std::current_exception(); + } catch (...) { + closing.closeError = "unknown native close failure"; + if (!closeError) closeError = std::current_exception(); + } + { + std::lock_guard lock(instance->databasesMutex); + auto entry = instance->databases.find(closing.key); + if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { + if (closing.closed) { + instance->databases.erase(entry); + } else { + entry->second.closeError = closing.closeError; + } + } + } + closing.condition->notify_all(); } if (closeError) { std::rethrow_exception(closeError); @@ -899,7 +909,12 @@ void DBRegistry::ReleaseParkTimeoutsByEnv(napi_env env) { */ void DBRegistry::Shutdown() { if (instance) { - std::lock_guard shutdownLock(instance->shutdownMutex); + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(DBSettings::getInstance().getLifecycleWaitSeconds()); + std::unique_lock shutdownLock(instance->shutdownMutex, std::defer_lock); + if (!shutdownLock.try_lock_until(deadline)) { + throw rocksdb_js::DBException("Timed out waiting for another database shutdown to finish"); + } std::vector descriptorsToClose; { diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index cf7486d93..cbddbbfe0 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -81,7 +81,7 @@ class DBRegistry final { * Mutex to protect the databases map. */ std::mutex databasesMutex; - std::mutex shutdownMutex; + std::timed_mutex shutdownMutex; // Destruction owns a physical path across every (path, readOnly) entry. // Waiters must re-resolve databases after every wake because the closer can // erase the node while the mutex is released. diff --git a/test/destroy.test.ts b/test/destroy.test.ts index abc9607e3..bca814e1e 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -8,6 +8,7 @@ import { describe, expect, it } from 'vitest'; const destroyOpenFixture = join(__dirname, 'fixtures', 'fork-destroy-open.mts'); const destroyFailureFixture = join(__dirname, 'fixtures', 'fork-destroy-failure.mts'); const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts'); +const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure.mts'); const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); function runDestroyFixture( @@ -16,7 +17,7 @@ function runDestroyFixture( env?: NodeJS.ProcessEnv ): Promise { return new Promise((resolve, reject) => { - const child = spawn(process.execPath, [fixture, dbPath], { + const child = spawn(process.execPath, ['--expose-gc', fixture, dbPath], { env: { ...process.env, ...env }, }); let stderr = ''; @@ -121,6 +122,12 @@ describe('Destroy', () => { }, 15_000); it('quarantines a failed automatic last-handle close', async () => { + await runDestroyFixture(gcCloseFailureFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FAILURE: '1', + }); + }, 15_000); + + it('surfaces an explicit close failure and permits a shutdown retry', async () => { await runDestroyFixture(closeFailureFixture, generateDBPath(), { ROCKSDB_JS_CLOSE_FAILURE: '1', }); diff --git a/test/fixtures/fork-gc-close-failure.mts b/test/fixtures/fork-gc-close-failure.mts new file mode 100644 index 000000000..a028fe978 --- /dev/null +++ b/test/fixtures/fork-gc-close-failure.mts @@ -0,0 +1,37 @@ +import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; +import { setTimeout as delay } from 'node:timers/promises'; + +const path = process.argv[2]; +let db: RocksDatabase | undefined = RocksDatabase.open(path); +db.putSync('key', 'value'); + +let resolveCloseFailure!: (args: unknown[]) => void; +const closeFailure = new Promise((resolve) => { + resolveCloseFailure = resolve; +}); +RocksDatabase.on('database:closeFailed', (...args) => resolveCloseFailure(args)); + +db = undefined; +for (let attempt = 0; attempt < 40; attempt++) { + global.gc!(); + await delay(25); +} + +const args = await Promise.race([ + closeFailure, + delay(1_000).then(() => { + throw new Error('Automatic close failure did not emit database:closeFailed'); + }), +]); +if (args[0] !== path || args[1] !== 'Injected database close failure') { + throw new Error(`Unexpected database:closeFailed arguments: ${JSON.stringify(args)}`); +} +if ( + registryStatus().find((entry) => entry.path === path)?.closeError !== + 'Injected database close failure' +) { + throw new Error('Automatic close failure was not quarantined'); +} + +delete process.env.ROCKSDB_JS_CLOSE_FAILURE; +shutdown(); diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts index 66fc09d23..6dc4f858e 100644 --- a/test/fixtures/fork-shutdown-failure.mts +++ b/test/fixtures/fork-shutdown-failure.mts @@ -10,7 +10,6 @@ try { } catch (error) { if (!String(error).includes('Injected database close failure')) throw error; } - const startedAt = Date.now(); try { RocksDatabase.open(path, { readOnly: true }); From c175ae9dc6bdb9542ddaead79259f2f2f9331a5c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 22:32:34 -0600 Subject: [PATCH 07/49] Contain close failures to the affected database --- README.md | 6 ++--- src/binding/binding.cpp | 11 ++++----- src/binding/database/database.cpp | 13 +++-------- src/binding/database/db_descriptor.cpp | 14 +++++++---- src/binding/database/db_registry.cpp | 31 ++++++++++++++----------- src/binding/database/db_settings.cpp | 10 ++++++-- src/database.ts | 4 +++- test/destroy.test.ts | 10 ++++++++ test/fixtures/fork-close-failure.mts | 4 ++++ test/fixtures/fork-destroy-failure.mts | 8 ++----- test/fixtures/fork-destroy-open.mts | 12 +++++++++- test/fixtures/fork-gc-close-failure.mts | 4 ++++ test/fixtures/fork-shutdown-failure.mts | 4 ++++ 13 files changed, 84 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index a04d0460f..c8d5f7c54 100644 --- a/README.md +++ b/README.md @@ -183,9 +183,9 @@ Sets global database settings. Defaults to 32MB. Set to `0` (zero) disables block cache for future opened databases. Existing block cache for any opened databases is resized immediately. Negative values throw an error. - `compactOnClose: boolean` When `true`, compacts the database on close. Defaults to `false`. - - `lifecycleWaitSeconds: number` Maximum time a synchronous open or destroy waits for another - lifecycle operation on the same path before throwing a retryable timeout error. Defaults to - `30` seconds and must be a positive integer. + - `lifecycleWaitSeconds: number` Maximum time a synchronous open, destroy, or shutdown waits for + another lifecycle operation before throwing a retryable timeout error. Defaults to `30` seconds + and must be a positive integer. - `verificationTableEntries: number` The number of slots in the process-global [Verification Table](#verification-table). Each slot is 8 bytes, so the default of `131072` (128K) slots is 1 MB. Set to `0` to disable the verification table. This must be configured diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 674acbb96..19aecb435 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -39,19 +39,16 @@ namespace rocksdb_js { * Shutdown function to ensure that we write in-memory data from all databases. */ napi_value Shutdown(napi_env env, napi_callback_info info) { - std::string shutdownError; try { DBRegistry::Shutdown(); } catch (const std::exception& error) { - shutdownError = error.what(); + ::napi_throw_error(env, nullptr, error.what()); + return nullptr; } catch (...) { - shutdownError = "Unknown native database shutdown failure"; - } - GlobalEvents::Shutdown(); - if (!shutdownError.empty()) { - ::napi_throw_error(env, nullptr, shutdownError.c_str()); + ::napi_throw_error(env, nullptr, "Unknown native database shutdown failure"); return nullptr; } + GlobalEvents::Shutdown(); napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 842907835..da1bc7d51 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -16,7 +16,6 @@ #include "core/platform.h" #include "napi/helpers.h" #include "napi/async.h" -#include "napi/global_events.h" #include "core/verification_table.h" #include "core/compression.h" @@ -49,14 +48,7 @@ napi_value Database::Constructor(napi_env env, napi_callback_info info) { DEBUG_LOG("Database::Constructor NativeDatabase GC'd dbHandle=%p\n", data); auto* dbHandle = static_cast*>(data); if (*dbHandle) { - std::string path = (*dbHandle)->path; - std::string closeError = DBRegistry::CloseDB(*dbHandle); - if (!closeError.empty() && GlobalEvents::hasListeners()) { - emitGlobalEvent( - "database:closeFailed", - ListenerData::fromStrings({path, closeError}) - ); - } + DBRegistry::CloseDB(*dbHandle); } delete dbHandle; }, @@ -200,7 +192,8 @@ napi_value Database::Close(napi_env env, napi_callback_info info) { DEBUG_LOG("%p Database::Close Closing database: \"%s\"\n", dbHandle->get(), (*dbHandle)->path.c_str()); std::string closeError = DBRegistry::CloseDB(*dbHandle); if (!closeError.empty()) { - ::napi_throw_error(env, nullptr, closeError.c_str()); + std::string message = closeError + ". Call destroy() or shutdown() to retry cleanup"; + ::napi_throw_error(env, nullptr, message.c_str()); return nullptr; } DEBUG_LOG("%p Database::Close Closed database\n", dbHandle->get()); diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 842e3f182..2267e0a71 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -454,9 +454,12 @@ void DBDescriptor::finishClose() { return; } - // We want to ensure that all in-memory data is written to disk. Keeps the waiting default on - // purpose: flushing immediately here races transaction-log-store teardown (AGENTS invariant 15). - this->flush(); + // We want to ensure that all in-memory data is written to disk. Keep the waiting default: an + // immediate flush races transaction-log-store teardown (AGENTS invariant 15). + rocksdb::Status status = this->flush(); + if (!status.ok()) { + throw rocksdb_js::DBException("Failed to flush database during close: " + status.ToString()); + } // Trigger manual compaction on all column families to reclaim space from // tombstones before closing @@ -484,7 +487,10 @@ void DBDescriptor::finishClose() { // suggestions of the documentation, this method alone does not seem to // trigger a flush rocksdb::WaitForCompactOptions options; - this->db->WaitForCompact(options); + status = this->db->WaitForCompact(options); + if (!status.ok()) { + throw rocksdb_js::DBException("Failed waiting for database compaction during close: " + status.ToString()); + } std::unique_lock txnsLock(this->txnsMutex); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index e560c20a7..da3ecde4d 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -9,6 +9,7 @@ #include "core/compression.h" #include "napi/helpers.h" #include "napi/async.h" +#include "napi/global_events.h" #include "rocksdb/table.h" #include #include @@ -181,6 +182,9 @@ std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn if (condition) { condition->notify_all(); } + if (!closeError.empty() && GlobalEvents::hasListeners()) { + emitGlobalEvent("database:closeFailed", ListenerData::fromStrings({path, closeError})); + } return closeError; } @@ -235,21 +239,15 @@ void DBRegistry::DestroyDB(const std::string& path) { std::lock_guard lock(instance->databasesMutex); claimed.reserve(instance->databases.size()); alreadyClosing.reserve(instance->databases.size()); - for (bool readOnly : {false, true}) { - auto entry = instance->databases.find(DBKey{path, readOnly}); - if (entry != instance->databases.end() && !entry->second.closeError.empty()) { - throw rocksdb_js::DBException( - "Cannot destroy database \"" + path + "\": previous close failed: " + - entry->second.closeError + ". Call shutdown() to retry cleanup" - ); - } - } for (auto& [key, entry] : instance->databases) { if (key.path != path || !entry.descriptor) { continue; } ClosingDescriptor closing{key, entry.descriptor, entry.condition}; - if (entry.descriptor->beginClose()) { + if (!entry.closeError.empty() && !entry.closeRetrying) { + entry.closeRetrying = true; + claimed.push_back(std::move(closing)); + } else if (entry.descriptor->beginClose()) { claimed.push_back(std::move(closing)); } else { alreadyClosing.push_back(std::move(closing)); @@ -290,6 +288,7 @@ void DBRegistry::DestroyDB(const std::string& path) { instance->databases.erase(entry); } else { entry->second.closeError = closing.closeError; + entry->second.closeRetrying = false; } } } @@ -320,7 +319,8 @@ void DBRegistry::DestroyDB(const std::string& path) { if (!closing.condition->wait_until(lock, deadline, [&]() { auto entry = instance->databases.find(closing.key); return entry == instance->databases.end() || - !entry->second.closeError.empty() || entry->second.descriptor != closing.descriptor; + entry->second.descriptor != closing.descriptor || + (!entry->second.closeError.empty() && !entry->second.closeRetrying); })) { throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": an open descriptor is still closing"); } @@ -416,7 +416,7 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons if (registeredEntry != instance->databases.end() && !registeredEntry->second.closeError.empty()) { throw rocksdb_js::DBException( "Cannot open database \"" + path + "\": previous close failed: " + - registeredEntry->second.closeError + ". Call shutdown() to retry cleanup" + registeredEntry->second.closeError + ". Call destroy() or shutdown() to retry cleanup" ); } } @@ -918,7 +918,12 @@ void DBRegistry::Shutdown() { std::vector descriptorsToClose; { - std::lock_guard lock(instance->databasesMutex); + std::unique_lock lock(instance->databasesMutex); + if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + return instance->destroyingPaths.empty(); + })) { + throw rocksdb_js::DBException("Timed out waiting for database destruction to finish before shutdown"); + } DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); descriptorsToClose.reserve(instance->databases.size()); diff --git a/src/binding/database/db_settings.cpp b/src/binding/database/db_settings.cpp index 2b6099bcc..77a5ee27b 100644 --- a/src/binding/database/db_settings.cpp +++ b/src/binding/database/db_settings.cpp @@ -217,8 +217,14 @@ napi_value DBSettings::Config(napi_env env, napi_callback_info info) { NAPI_STATUS_THROWS(rocksdb_js::getProperty(env, params, "compactOnClose", settings.compactOnClose, false)); double lifecycleWaitSeconds = 0; - status = rocksdb_js::getProperty(env, params, "lifecycleWaitSeconds", lifecycleWaitSeconds, true); - if (status == napi_ok) { + bool lifecycleWaitProvided = false; + NAPI_STATUS_THROWS(::napi_has_named_property(env, params, "lifecycleWaitSeconds", &lifecycleWaitProvided)); + if (lifecycleWaitProvided) { + status = rocksdb_js::getProperty(env, params, "lifecycleWaitSeconds", lifecycleWaitSeconds, true); + if (status != napi_ok) { + ::napi_throw_type_error(env, nullptr, "Lifecycle wait seconds must be a number"); + return nullptr; + } if (!std::isfinite(lifecycleWaitSeconds) || lifecycleWaitSeconds <= 0 || std::trunc(lifecycleWaitSeconds) != lifecycleWaitSeconds || lifecycleWaitSeconds > std::numeric_limits::max() diff --git a/src/database.ts b/src/database.ts index 32542dda7..78bfa3409 100644 --- a/src/database.ts +++ b/src/database.ts @@ -347,7 +347,9 @@ export class RocksDatabase extends DBI { * native binding use namespaced keys (e.g. `'transactionLog:warning'`). * * Listeners are not tied to any specific database — they fire for every - * matching event emitted in this process. + * matching event emitted in this process. Native lifecycle failures use + * `'database:closeFailed'` with `(path, error)` string arguments; the path + * remains quarantined until `destroy()` or `shutdown()` retries cleanup. * * @example * ```typescript diff --git a/test/destroy.test.ts b/test/destroy.test.ts index bca814e1e..2c558cfe5 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -48,6 +48,9 @@ describe('Destroy', () => { expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 1.5 })).toThrow( 'Lifecycle wait seconds must be a positive integer' ); + expect(() => RocksDatabase.config({ lifecycleWaitSeconds: '30' as unknown as number })).toThrow( + 'Lifecycle wait seconds must be a number' + ); expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 30 })).not.toThrow(); }); @@ -103,6 +106,13 @@ describe('Destroy', () => { }); }, 15_000); + it('waits for physical destruction before shutdown completes', async () => { + await runDestroyFixture(destroyOpenFixture, generateDBPath(), { + ROCKSDB_JS_DESTROY_DELAY_MS: '2000', + ROCKSDB_JS_TEST_SHUTDOWN_DURING_DESTROY: '1', + }); + }, 15_000); + it('releases the path gate when physical destruction fails', async () => { await runDestroyFixture(destroyFailureFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_FAILURE: '1', diff --git a/test/fixtures/fork-close-failure.mts b/test/fixtures/fork-close-failure.mts index c2687e7a7..1a830754d 100644 --- a/test/fixtures/fork-close-failure.mts +++ b/test/fixtures/fork-close-failure.mts @@ -30,3 +30,7 @@ delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); if (registryStatus().some((entry) => entry.path === path)) throw new Error('Shutdown retry did not clear the quarantined automatic close'); +const reopened = RocksDatabase.open(path); +if (reopened.getSync('key') !== 'value') + throw new Error('Shutdown recovery did not preserve the database'); +reopened.destroy(); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index ca3d99eaf..9999a4d72 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -27,12 +27,8 @@ if (closeFailure) { } if (Date.now() - startedAt >= 1_000) throw new Error('Opening a quarantined descriptor waited instead of failing immediately'); - try { - db.destroy(); - throw new Error('Expected repeated destroy to report the previous close failure'); - } catch (error) { - if (!String(error).includes(`previous close failed: ${expectedError}`)) throw error; - } + delete process.env.ROCKSDB_JS_CLOSE_FAILURE; + db.destroy(); process.exit(0); } diff --git a/test/fixtures/fork-destroy-open.mts b/test/fixtures/fork-destroy-open.mts index 3bfcfb515..9e15df499 100644 --- a/test/fixtures/fork-destroy-open.mts +++ b/test/fixtures/fork-destroy-open.mts @@ -1,4 +1,4 @@ -import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; import { setTimeout as delay } from 'node:timers/promises'; import { Worker } from 'node:worker_threads'; @@ -34,6 +34,16 @@ while (registryStatus().some((entry) => entry.path === path)) { const destroyResult = nextMessage(); const startedAt = Date.now(); +if (process.env.ROCKSDB_JS_TEST_SHUTDOWN_DURING_DESTROY === '1') { + shutdown(); + const shutdownDuration = Date.now() - startedAt; + const destroyed = await destroyResult; + if (!destroyed.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyed)}`); + if (shutdownDuration < 500) + throw new Error(`Shutdown did not wait for destroy (${shutdownDuration}ms)`); + await worker.terminate(); + process.exit(0); +} const reopened = RocksDatabase.open(path); const openDuration = Date.now() - startedAt; const destroyed = await destroyResult; diff --git a/test/fixtures/fork-gc-close-failure.mts b/test/fixtures/fork-gc-close-failure.mts index a028fe978..cbc73eb36 100644 --- a/test/fixtures/fork-gc-close-failure.mts +++ b/test/fixtures/fork-gc-close-failure.mts @@ -35,3 +35,7 @@ if ( delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); +const reopened = RocksDatabase.open(path); +if (reopened.getSync('key') !== 'value') + throw new Error('Shutdown recovery did not preserve the database'); +reopened.destroy(); diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts index 6dc4f858e..19d868d4e 100644 --- a/test/fixtures/fork-shutdown-failure.mts +++ b/test/fixtures/fork-shutdown-failure.mts @@ -25,3 +25,7 @@ delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); if (registryStatus().some((entry) => entry.path === path)) throw new Error('Shutdown retry did not clear the quarantined descriptor'); +const reopened = RocksDatabase.open(path); +if (reopened.getSync('key') !== 'value') + throw new Error('Shutdown recovery did not preserve the database'); +reopened.destroy(); From b549fc2f72c19735d4a3a333741df29d65175f43 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 22:45:39 -0600 Subject: [PATCH 08/49] Complete teardown after close status errors --- src/binding/database/db_descriptor.cpp | 20 ++++++++++++++------ src/binding/database/db_descriptor.h | 1 + src/binding/database/db_registry.cpp | 12 +++++++++--- test/checkpoint.test.ts | 13 +++++-------- test/drop.test.ts | 7 ++++++- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 2267e0a71..64d7f9786 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -392,9 +392,13 @@ DBDescriptor::DBDescriptor( */ DBDescriptor::~DBDescriptor() { DEBUG_LOG("%p DBDescriptor::~DBDescriptor Closing \"%s\"\n", this, this->path.c_str()); - this->close(); - // Idempotent safety net, matching commitWorker/logWorker's own - // destructor shutdown. + try { + this->close(); + } catch (const std::exception& error) { + DEBUG_LOG("%p DBDescriptor::~DBDescriptor Close failed for \"%s\": %s\n", this, this->path.c_str(), error.what()); + } catch (...) { + DEBUG_LOG("%p DBDescriptor::~DBDescriptor Close failed for \"%s\"\n", this, this->path.c_str()); + } this->parkTimeouts->shutdown(); } @@ -456,9 +460,10 @@ void DBDescriptor::finishClose() { // We want to ensure that all in-memory data is written to disk. Keep the waiting default: an // immediate flush races transaction-log-store teardown (AGENTS invariant 15). + std::string closeError; rocksdb::Status status = this->flush(); if (!status.ok()) { - throw rocksdb_js::DBException("Failed to flush database during close: " + status.ToString()); + closeError = "Failed to flush database during close: " + status.ToString(); } // Trigger manual compaction on all column families to reclaim space from @@ -488,8 +493,8 @@ void DBDescriptor::finishClose() { // trigger a flush rocksdb::WaitForCompactOptions options; status = this->db->WaitForCompact(options); - if (!status.ok()) { - throw rocksdb_js::DBException("Failed waiting for database compaction during close: " + status.ToString()); + if (!status.ok() && closeError.empty()) { + closeError = "Failed waiting for database compaction during close: " + status.ToString(); } std::unique_lock txnsLock(this->txnsMutex); @@ -543,6 +548,9 @@ void DBDescriptor::finishClose() { this->events.releaseAll(); this->db.reset(); + if (!closeError.empty()) { + throw rocksdb_js::DBException(closeError); + } } napi_status DBDescriptor::registerCommitCompletion(napi_env env, napi_threadsafe_function_call_js callJs, bool& closed) { diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index bfb874631..792ea0c60 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -463,6 +463,7 @@ struct DBDescriptor final : public std::enable_shared_from_this { void close(); bool isClosing() const { return this->closing.load(); } + bool isClosed() const { return !this->db; } /** * Atomically transitions the descriptor into the closing state. Returns diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index da3ecde4d..eddb57bcb 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -168,7 +168,7 @@ std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn // Only erase the entry we claimed. A brand-new descriptor cannot appear // because OpenDB blocks until we notify below. if (eraseIt != instance->databases.end() && eraseIt->second.descriptor == descriptor) { - if (closeError.empty()) { + if (closeError.empty() || descriptor->isClosed()) { instance->databases.erase(eraseIt); } else { eraseIt->second.closeError = closeError; @@ -265,13 +265,15 @@ void DBRegistry::DestroyDB(const std::string& path) { closing.descriptor->finishClose(); closing.closed = true; } catch (const std::exception& error) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = error.what(); - if (!closeError) { + if (!closing.closed && !closeError) { closeError = std::current_exception(); } } catch (...) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = "unknown native close failure"; - if (!closeError) { + if (!closing.closed && !closeError) { closeError = std::current_exception(); } } @@ -644,9 +646,11 @@ void DBRegistry::PurgeAll() { closing.descriptor->finishClose(); closing.closed = true; } catch (const std::exception& error) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = error.what(); if (!closeError) closeError = std::current_exception(); } catch (...) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = "unknown native close failure"; if (!closeError) closeError = std::current_exception(); } @@ -954,11 +958,13 @@ void DBRegistry::Shutdown() { closing.descriptor->finishClose(); closing.closed = true; } catch (const std::exception& error) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = error.what(); if (!closeError) { closeError = std::current_exception(); } } catch (...) { + closing.closed = closing.descriptor->isClosed(); closing.closeError = "unknown native close failure"; if (!closeError) { closeError = std::current_exception(); diff --git a/test/checkpoint.test.ts b/test/checkpoint.test.ts index 6424c34a6..5b8ad48d8 100644 --- a/test/checkpoint.test.ts +++ b/test/checkpoint.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase, shutdown } from '../src/index.ts'; +import { RocksDatabase } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { existsSync, mkdirSync, rmSync } from 'node:fs'; import { join } from 'node:path'; @@ -26,13 +26,10 @@ async function writeAll(db: RocksDatabase, count: number, prefix = 'value'): Pro describe('Checkpoints', () => { afterEach(() => { - // The close()/destroy()-during-checkpoint tests can leave a descriptor - // pending registry purge (closing before an in-flight, descriptor-pinned - // checkpoint settles defers the purge — see #672), which keeps the source - // database open and its temp dir locked. That fails cleanup on Windows and - // crashes the Bun worker on exit. Purge the registry first so the locks are - // released before we remove the directories. - shutdown(); + // The checkpoint completion path purges a descriptor that became + // unreferenced while the copy was in flight. Avoid process-global shutdown + // here: test files share a process, so it can close another suite's active + // database while that suite is dropping a column family. for (const dir of tempDirs) { rmSync(dir, { force: true, recursive: true, maxRetries: 3, retryDelay: 500 }); } diff --git a/test/drop.test.ts b/test/drop.test.ts index 67d2bba2f..62e64aaec 100644 --- a/test/drop.test.ts +++ b/test/drop.test.ts @@ -213,7 +213,8 @@ describe('Drop', () => { // pre-existing bug tracked as // https://github.com/HarperFast/rocksdb-js/issues/726 (needs the drop // interlocked against in-flight transactions), so this test asserts only - // atomicity and leaves the handles to dbRunner's per-test database.) + // atomicity and verifies that the poisoned environment is reported while + // still completing native teardown.) it('should not partially apply a pessimistic transaction spanning a dropped column family', () => dbRunner( { @@ -235,6 +236,10 @@ describe('Drop', () => { // the live half must NOT have been applied expect(victim.getSync('live')).toBeUndefined(); + + stale.close(); + doomed.close(); + expect(() => victim.close()).toThrow('Failed to flush database during close'); } )); From 51e168d0a537b4635b813765d97fdea3ef38d7a3 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 22:52:48 -0600 Subject: [PATCH 09/49] Make cross-env lifecycle teardown safe --- README.md | 8 ++++++-- src/binding/database/db_descriptor.cpp | 4 ++-- src/binding/database/db_handle.cpp | 16 ++++++++++------ src/binding/database/db_handle.h | 2 ++ src/database.ts | 5 +++-- test/fixtures/fork-destroy-open.mts | 1 + test/lib/util.ts | 16 +++++++++++++++- 7 files changed, 39 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c8d5f7c54..eb6be97da 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,9 @@ Creates a new database instance. ### `db.close()` Closes a database. This function can be called multiple times and will only close an opened -database. A database instance can be reopened once its closed. +database. A database instance can be reopened once it is closed. A flush or compaction failure is +reported as an exception after native teardown completes. A failure that prevents teardown emits +`database:closeFailed`; same-path opens then fail until `destroy()` or `shutdown()` retries cleanup. ```typescript const db = RocksDatabase.open('foo'); @@ -1981,7 +1983,9 @@ console.log(registryStatus()); The `shutdown()` will flush all in-memory data to disk and wait for any outstanding compactions to finish, for all open databases. It is highly recommended to call this in a `process` `exit` event -listener (on the main thread), to ensure that all data is flushed to disk before the process exits: +listener (on the main thread), to ensure that all data is flushed to disk before the process exits. +It throws the first close failure after attempting every claimed database; call it again to retry +any descriptor whose teardown did not complete: ```typescript import { shutdown } from '@harperfast/rocksdb-js'; diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 64d7f9786..979b26e4d 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -436,6 +436,8 @@ void DBDescriptor::finishClose() { this->logWorker.shutdown(); this->commitWorker.shutdown(); + // An in-flight commit pins this descriptor through its transaction and DB + // handles, so reaching this release pass means only idle per-env TSFNs remain. { std::lock_guard lock(this->commitMutex); for (auto& [env, completion] : this->commitCompletions) { @@ -449,8 +451,6 @@ void DBDescriptor::finishClose() { this->closeWorkersStopped = true; } - // Inject after the one-shot pipeline shutdown so retry coverage exercises - // a genuinely partially-completed close rather than an untouched descriptor. if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { throw rocksdb_js::DBException("Injected database close failure"); } diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index 912839911..60ab1ba29 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -67,7 +67,7 @@ void setTxnlogSummaryStatsOnObject( * Creates a new DBHandle. */ DBHandle::DBHandle(napi_env env, napi_ref exportsRef) - : descriptor(nullptr), env(env), exportsRef(exportsRef) {} + : descriptor(nullptr), env(env), ownerThreadId(std::this_thread::get_id()), exportsRef(exportsRef) {} /** * Close the DBHandle and destroy it. @@ -155,12 +155,16 @@ void DBHandle::close() { this->descriptor.reset(); } - // clean up transaction log references - for (auto& [name, ref] : this->logRefs) { - DEBUG_LOG("%p DBHandle::close Releasing transaction log JS reference \"%s\"\n", this, name.c_str()); - ::napi_delete_reference(this->env, ref); + // N-API references are environment-thread-affine. Destroying a shared + // descriptor can close this handle from another worker; retain the refs in + // that case so the owning environment's later close/finalizer releases them. + if (std::this_thread::get_id() == this->ownerThreadId) { + for (auto& [name, ref] : this->logRefs) { + DEBUG_LOG("%p DBHandle::close Releasing transaction log JS reference \"%s\"\n", this, name.c_str()); + ::napi_delete_reference(this->env, ref); + } + this->logRefs.clear(); } - this->logRefs.clear(); DEBUG_LOG("%p DBHandle::close Handle closed\n", this); } diff --git a/src/binding/database/db_handle.h b/src/binding/database/db_handle.h index 1dffc7af2..625f51e77 100644 --- a/src/binding/database/db_handle.h +++ b/src/binding/database/db_handle.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "rocksdb/db.h" #include "database/db_descriptor.h" @@ -64,6 +65,7 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro * The node environment. */ napi_env env; + std::thread::id ownerThreadId; /** * A reference to the main `rocksdb_js` exports object. This is needed to diff --git a/src/database.ts b/src/database.ts index 78bfa3409..2f94bd1db 100644 --- a/src/database.ts +++ b/src/database.ts @@ -348,8 +348,9 @@ export class RocksDatabase extends DBI { * * Listeners are not tied to any specific database — they fire for every * matching event emitted in this process. Native lifecycle failures use - * `'database:closeFailed'` with `(path, error)` string arguments; the path - * remains quarantined until `destroy()` or `shutdown()` retries cleanup. + * `'database:closeFailed'` with `(path, error)` string arguments. A failure + * that prevents teardown quarantines the path until `destroy()` or + * `shutdown()` retries cleanup. * * @example * ```typescript diff --git a/test/fixtures/fork-destroy-open.mts b/test/fixtures/fork-destroy-open.mts index 9e15df499..ee1145cfc 100644 --- a/test/fixtures/fork-destroy-open.mts +++ b/test/fixtures/fork-destroy-open.mts @@ -6,6 +6,7 @@ import { Worker } from 'node:worker_threads'; const path = process.argv[2]; const original = RocksDatabase.open(path); original.putSync('before-destroy', 'present'); +original.useLog('cross-env-close'); const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { eval: true, diff --git a/test/lib/util.ts b/test/lib/util.ts index 828f2a7e8..898407340 100644 --- a/test/lib/util.ts +++ b/test/lib/util.ts @@ -65,6 +65,10 @@ export async function dbRunner(options: TestOptions | TestFn, test?: TestFn): Pr const dbPath = generateDBPath(); const dbPaths = new Set([dbPath]); const databases: TestDB[] = []; + let testError: unknown; + let closeError: unknown; + let testFailed = false; + let closeFailed = false; try { for (let i = 0; i < testFn.length; i++) { @@ -78,9 +82,17 @@ export async function dbRunner(options: TestOptions | TestFn, test?: TestFn): Pr } await testFn(...databases); + } catch (error) { + testFailed = true; + testError = error; } finally { for (const { db } of databases.reverse()) { - db?.close(); + try { + db?.close(); + } catch (error) { + if (!closeFailed) closeError = error; + closeFailed = true; + } } if (globalThis.gc) { @@ -103,6 +115,8 @@ export async function dbRunner(options: TestOptions | TestFn, test?: TestFn): Pr } } } + if (testFailed) throw testError; + if (closeFailed) throw closeError; } /** From db9e86db38d4b0a824d014b7e1b5b5a4f398de82 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:11:05 -0600 Subject: [PATCH 10/49] Close remaining lifecycle race windows --- README.md | 6 +- src/binding/database/database.cpp | 9 +- src/binding/database/db_descriptor.cpp | 7 + src/binding/database/db_registry.cpp | 204 +++++++++++++++---------- src/binding/database/db_registry.h | 9 +- src/database.ts | 6 +- src/load-binding.ts | 4 +- test/checkpoint.test.ts | 6 +- test/destroy.test.ts | 29 +++- test/drop.test.ts | 4 +- test/fixtures/fork-shutdown-retry.mts | 36 +++++ test/workers/shutdown-retry-worker.mts | 11 ++ 12 files changed, 234 insertions(+), 97 deletions(-) create mode 100644 test/fixtures/fork-shutdown-retry.mts create mode 100644 test/workers/shutdown-retry-worker.mts diff --git a/README.md b/README.md index eb6be97da..d88928f10 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,10 @@ Creates a new database instance. Closes a database. This function can be called multiple times and will only close an opened database. A database instance can be reopened once it is closed. A flush or compaction failure is -reported as an exception after native teardown completes. A failure that prevents teardown emits -`database:closeFailed`; same-path opens then fail until `destroy()` or `shutdown()` retries cleanup. +reported as an exception after native teardown completes. All native close errors emit +`database:closeFailed`; when teardown does not complete, same-path opens also fail until +`destroy()` or `shutdown()` retries cleanup. The quarantine applies to both writable and read-only +opens because both modes share the physical path lifecycle. ```typescript const db = RocksDatabase.open('foo'); diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index da1bc7d51..e3ca632b2 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -190,9 +190,12 @@ napi_value Database::Close(napi_env env, napi_callback_info info) { if (*dbHandle) { DEBUG_LOG("%p Database::Close Closing database: \"%s\"\n", dbHandle->get(), (*dbHandle)->path.c_str()); - std::string closeError = DBRegistry::CloseDB(*dbHandle); - if (!closeError.empty()) { - std::string message = closeError + ". Call destroy() or shutdown() to retry cleanup"; + CloseResult closeResult = DBRegistry::CloseDB(*dbHandle); + if (!closeResult.error.empty()) { + std::string message = closeResult.error; + if (closeResult.quarantined) { + message += ". Call destroy() or shutdown() to retry cleanup"; + } ::napi_throw_error(env, nullptr, message.c_str()); return nullptr; } diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 979b26e4d..99af48e07 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -420,6 +420,7 @@ void DBDescriptor::finishClose() { DEBUG_LOG("%p DBDescriptor::close Closing \"%s\" (mode=%s read-only=%s closables=%zu columns=%zu transactions=%zu)\n", this, this->path.c_str(), this->mode == DBMode::Optimistic ? "optimistic" : "pessimistic", this->readOnly ? "true" : "false", this->closables.size(), this->columns.size(), this->transactions.size()); + const bool retryingClose = this->closeWorkersStopped; if (!this->closeWorkersStopped) { // Wait for all in-flight operations to complete before cleanup. // The closing flag is already set, so new operations will fail with "Database is closing". @@ -450,6 +451,12 @@ void DBDescriptor::finishClose() { } this->closeWorkersStopped = true; } + if (retryingClose) { + const int retryDelayMs = testDelayMs("ROCKSDB_JS_CLOSE_RETRY_DELAY_MS"); + if (retryDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(retryDelayMs)); + } + } if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { throw rocksdb_js::DBException("Injected database close failure"); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index eddb57bcb..feefa8afe 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -65,7 +65,7 @@ std::unique_ptr DBRegistry::instance; /** * Close a RocksDB database handle. */ -std::string DBRegistry::CloseDB(const std::shared_ptr handle) { +CloseResult DBRegistry::CloseDB(const std::shared_ptr handle) { if (!instance) { DEBUG_LOG("%p DBRegistry::CloseDB Registry not initialized\n", instance.get()); return {}; @@ -127,15 +127,16 @@ std::string DBRegistry::CloseDB(const std::shared_ptr handle) { * the duration of finishClose(), so a concurrent OpenDB keeps waiting on * the condition rather than re-opening the path mid-close. */ -std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { +CloseResult DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOnly) { if (!instance) { - return {}; + return CloseResult{}; } DBKey key{path, readOnly}; std::shared_ptr descriptor; std::shared_ptr condition; std::string closeError; + bool quarantined = false; { std::lock_guard lock(instance->databasesMutex); auto entryIterator = instance->databases.find(key); @@ -172,6 +173,7 @@ std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn instance->databases.erase(eraseIt); } else { eraseIt->second.closeError = closeError; + quarantined = true; DEBUG_LOG("%p DBRegistry::PurgeIfUnreferenced Quarantined \"%s\": %s\n", instance.get(), path.c_str(), closeError.c_str()); } @@ -185,7 +187,7 @@ std::string DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn if (!closeError.empty() && GlobalEvents::hasListeners()) { emitGlobalEvent("database:closeFailed", ListenerData::fromStrings({path, closeError})); } - return closeError; + return CloseResult{closeError, quarantined}; } /** @@ -300,19 +302,6 @@ void DBRegistry::DestroyDB(const std::string& path) { if (closeError) { std::rethrow_exception(closeError); } - for (const auto& closing : claimed) { - if (!closing.closed) { - continue; - } - const size_t refCountAfterClose = closing.descriptor.use_count(); - if (refCountAfterClose > 1) { - std::string errorMsg = "Cannot destroy database: " + std::to_string(refCountAfterClose - 1) + - " reference(s) still held after closing all handles. This may indicate handles not properly closed or JavaScript objects not yet garbage collected."; - DEBUG_LOG("%p DBRegistry::DestroyDB Error: %s\n", instance.get(), errorMsg.c_str()); - throw rocksdb_js::DBException(errorMsg); - } - } - if (alreadyClosing.empty()) { break; } @@ -351,11 +340,21 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); rocksdb::Status status = rocksdb::DestroyDB(path, rocksdb::Options()); if (!status.ok()) { - throw rocksdb_js::DBException(status.ToString()); + const std::string error = status.ToString(); + std::lock_guard lock(instance->databasesMutex); + instance->databases[DBKey{path, false}].closeError = error; + throw rocksdb_js::DBException(error); } // remove the database directory including transaction logs - std::filesystem::remove_all(path); + std::error_code cleanupError; + std::filesystem::remove_all(path, cleanupError); + if (cleanupError) { + const std::string error = cleanupError.message(); + std::lock_guard lock(instance->databasesMutex); + instance->databases[DBKey{path, false}].closeError = error; + throw rocksdb_js::DBException("Failed to remove database directory: " + error); + } DEBUG_LOG("%p DBRegistry::DestroyDB Successfully destroyed database at \"%s\"\n", instance.get(), path.c_str()); } @@ -415,13 +414,37 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons for (bool readOnly : {false, true}) { auto registeredEntry = instance->databases.find(DBKey{path, readOnly}); - if (registeredEntry != instance->databases.end() && !registeredEntry->second.closeError.empty()) { + if (registeredEntry != instance->databases.end() && + !registeredEntry->second.closeError.empty() && !registeredEntry->second.closeRetrying + ) { + const bool destroyCleanupFailed = !registeredEntry->second.descriptor; throw rocksdb_js::DBException( - "Cannot open database \"" + path + "\": previous close failed: " + - registeredEntry->second.closeError + ". Call destroy() or shutdown() to retry cleanup" + "Cannot open database \"" + path + "\": previous " + + (destroyCleanupFailed ? "destroy cleanup" : "close") + " failed: " + + registeredEntry->second.closeError + + (destroyCleanupFailed ? ". Call destroy() to retry cleanup" : ". Call destroy() or shutdown() to retry cleanup") ); } } + std::shared_ptr retryCondition; + bool retryReadOnly = false; + for (bool readOnly : {false, true}) { + auto registeredEntry = instance->databases.find(DBKey{path, readOnly}); + if (registeredEntry != instance->databases.end() && registeredEntry->second.closeRetrying) { + retryCondition = registeredEntry->second.condition; + retryReadOnly = readOnly; + break; + } + } + if (retryCondition) { + if (!retryCondition->wait_until(lock, deadline, [&]() { + auto registeredEntry = instance->databases.find(DBKey{path, retryReadOnly}); + return registeredEntry == instance->databases.end() || !registeredEntry->second.closeRetrying; + })) { + throw rocksdb_js::DBException("Timed out opening database \"" + path + "\": close retry is still in progress"); + } + continue; + } entryIterator = instance->databases.find(key); if (entryIterator == instance->databases.end()) { @@ -919,73 +942,94 @@ void DBRegistry::Shutdown() { if (!shutdownLock.try_lock_until(deadline)) { throw rocksdb_js::DBException("Timed out waiting for another database shutdown to finish"); } - std::vector descriptorsToClose; - - { - std::unique_lock lock(instance->databasesMutex); - if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { - return instance->destroyingPaths.empty(); - })) { - throw rocksdb_js::DBException("Timed out waiting for database destruction to finish before shutdown"); + while (true) { + std::vector descriptorsToClose; + std::vector descriptorsToWaitFor; + bool destroysInFlight; + { + std::unique_lock lock(instance->databasesMutex); + destroysInFlight = !instance->destroyingPaths.empty(); + DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); + descriptorsToClose.reserve(instance->databases.size()); + descriptorsToWaitFor.reserve(instance->databases.size()); + + for (auto& [key, entry] : instance->databases) { + if (instance->destroyingPaths.find(key.path) != instance->destroyingPaths.end()) { + continue; + } + if (!entry.descriptor) { + if (!entry.closeError.empty()) { + throw rocksdb_js::DBException( + "Cannot complete shutdown: database \"" + key.path + + "\" requires destroy() cleanup: " + entry.closeError + ); + } + continue; + } + ClosingDescriptor closing{key, entry.descriptor, entry.condition}; + if (!entry.closeError.empty() && !entry.closeRetrying) { + entry.closeRetrying = true; + descriptorsToClose.push_back(std::move(closing)); + } else if (entry.closeError.empty() && entry.descriptor->beginClose()) { + entry.closeRetrying = true; + descriptorsToClose.push_back(std::move(closing)); + } else { + descriptorsToWaitFor.push_back(std::move(closing)); + } + } } - DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); - descriptorsToClose.reserve(instance->databases.size()); - // Claim each close while holding the registry lock so DestroyDB can - // safely wait on the matching erase-and-notify below. - for (auto& [key, entry] : instance->databases) { - if (!entry.descriptor || - instance->destroyingPaths.find(key.path) != instance->destroyingPaths.end() - ) { - continue; + std::exception_ptr closeError; + for (auto& closing : descriptorsToClose) { + DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); + try { + closing.descriptor->finishClose(); + closing.closed = true; + } catch (const std::exception& error) { + closing.closed = closing.descriptor->isClosed(); + closing.closeError = error.what(); + if (!closeError) closeError = std::current_exception(); + } catch (...) { + closing.closed = closing.descriptor->isClosed(); + closing.closeError = "unknown native close failure"; + if (!closeError) closeError = std::current_exception(); } - ClosingDescriptor closing{key, entry.descriptor, entry.condition}; - if (!entry.closeError.empty() && !entry.closeRetrying) { - entry.closeRetrying = true; - descriptorsToClose.push_back(std::move(closing)); - } else if (entry.closeError.empty() && entry.descriptor->beginClose()) { - entry.closeRetrying = true; - descriptorsToClose.push_back(std::move(closing)); + { + std::lock_guard lock(instance->databasesMutex); + auto entry = instance->databases.find(closing.key); + if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { + if (closing.closed) { + instance->databases.erase(entry); + } else { + entry->second.closeError = closing.closeError; + entry->second.closeRetrying = false; + } + } } + closing.condition->notify_all(); } - } - - // Close all descriptors without holding the lock - std::exception_ptr closeError; - for (auto& closing : descriptorsToClose) { - DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); - try { - closing.descriptor->finishClose(); - closing.closed = true; - } catch (const std::exception& error) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = error.what(); - if (!closeError) { - closeError = std::current_exception(); - } - } catch (...) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = "unknown native close failure"; - if (!closeError) { - closeError = std::current_exception(); + if (closeError) std::rethrow_exception(closeError); + + for (const auto& closing : descriptorsToWaitFor) { + std::unique_lock lock(instance->databasesMutex); + if (!closing.condition->wait_until(lock, deadline, [&]() { + auto entry = instance->databases.find(closing.key); + return entry == instance->databases.end() || + entry->second.descriptor != closing.descriptor || + (!entry->second.closeError.empty() && !entry->second.closeRetrying); + })) { + throw rocksdb_js::DBException("Timed out waiting for database close to finish during shutdown"); } } - { - std::lock_guard lock(instance->databasesMutex); - auto entry = instance->databases.find(closing.key); - if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { - if (closing.closed) { - instance->databases.erase(entry); - } else { - entry->second.closeError = closing.closeError; - entry->second.closeRetrying = false; - } + if (descriptorsToClose.empty() && descriptorsToWaitFor.empty()) { + if (!destroysInFlight) break; + std::unique_lock lock(instance->databasesMutex); + if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + return instance->destroyingPaths.empty(); + })) { + throw rocksdb_js::DBException("Timed out waiting for database destruction to finish during shutdown"); } } - closing.condition->notify_all(); - } - if (closeError) { - std::rethrow_exception(closeError); } // Purge the registry diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index cbddbbfe0..edd01b8de 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -51,6 +51,11 @@ struct DBRegistryEntry final { : descriptor(std::move(desc)), condition(std::make_shared()) {} }; +struct CloseResult final { + std::string error; + bool quarantined = false; +}; + struct DBHandleParams final { std::shared_ptr descriptor; @@ -94,7 +99,7 @@ class DBRegistry final { static std::unique_ptr instance; public: - static std::string CloseDB(const std::shared_ptr handle); + static CloseResult CloseDB(const std::shared_ptr handle); #ifdef DEBUG static void DebugLogDescriptorRefs(); #endif @@ -102,7 +107,7 @@ class DBRegistry final { static void Init(napi_env env, napi_value exports); static std::unique_ptr OpenDB(const std::string& path, const DBOptions& options); static void PurgeAll(); - static std::string PurgeIfUnreferenced(const std::string& path, bool readOnly); + static CloseResult PurgeIfUnreferenced(const std::string& path, bool readOnly); static napi_value RegistryStatus(napi_env env, napi_callback_info info); static void CloseTransactionsByEnv(napi_env env); static void RemoveListenersByEnv(napi_env env); diff --git a/src/database.ts b/src/database.ts index 2f94bd1db..21c15c83d 100644 --- a/src/database.ts +++ b/src/database.ts @@ -348,9 +348,9 @@ export class RocksDatabase extends DBI { * * Listeners are not tied to any specific database — they fire for every * matching event emitted in this process. Native lifecycle failures use - * `'database:closeFailed'` with `(path, error)` string arguments. A failure - * that prevents teardown quarantines the path until `destroy()` or - * `shutdown()` retries cleanup. + * `'database:closeFailed'` with `(path, error)` string arguments. The event + * reports both completed and incomplete teardowns; only an incomplete + * teardown quarantines the path until `destroy()` or `shutdown()` retries. * * @example * ```typescript diff --git a/src/load-binding.ts b/src/load-binding.ts index e08b2b46d..73584f189 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -511,8 +511,8 @@ export type RocksDatabaseConfig = { verificationTableEntries?: number; compactOnClose?: boolean; /** - * Maximum seconds an open or destroy call waits for another lifecycle - * operation on the same path. Defaults to 30. + * Maximum seconds an open, destroy, or shutdown call waits for another + * lifecycle operation. Defaults to 30. */ lifecycleWaitSeconds?: number; /** diff --git a/test/checkpoint.test.ts b/test/checkpoint.test.ts index 5b8ad48d8..69fac4708 100644 --- a/test/checkpoint.test.ts +++ b/test/checkpoint.test.ts @@ -220,9 +220,9 @@ describe('Checkpoints', () => { // close()) bypasses the async-work tracker. The checkpoint registers in // operationsInFlight, so finishClose() waits for the copy to finish // before resetting descriptor->db — the worker never touches a freed DB. - // The in-flight op still holds a descriptor reference, so destroy() throws - // rather than tearing the database down mid-copy. - expect(() => db.destroy()).toThrow(); + // The operation reference can outlive registry removal safely because + // finishClose waits for it and resets the native DB before physical destroy. + expect(() => db.destroy()).not.toThrow(); // The checkpoint itself completed (finishClose waited for it), so the // promise settles cleanly and nothing crashes. diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 2c558cfe5..1a1a3aa15 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,7 +1,7 @@ import { RocksDatabase } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; -import { existsSync } from 'node:fs'; +import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -10,6 +10,7 @@ const destroyFailureFixture = join(__dirname, 'fixtures', 'fork-destroy-failure. const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts'); const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure.mts'); const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); +const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mts'); function runDestroyFixture( fixture: string, @@ -100,6 +101,26 @@ describe('Destroy', () => { } )); + it.skipIf(process.platform === 'win32')( + 'quarantines a path when post-destroy cleanup fails', + () => + dbRunner(({ db, dbPath }) => { + const lockedDirectory = join(dbPath, 'transaction_logs', 'locked'); + mkdirSync(lockedDirectory, { recursive: true }); + writeFileSync(join(lockedDirectory, 'leftover'), 'data'); + chmodSync(lockedDirectory, 0o000); + try { + expect(() => db.destroy()).toThrow('Failed to remove database directory'); + expect(() => RocksDatabase.open(dbPath)).toThrow('previous destroy cleanup failed'); + } finally { + chmodSync(lockedDirectory, 0o700); + } + db.destroy(); + const reopened = RocksDatabase.open(dbPath); + reopened.close(); + }) + ); + it('waits for physical destruction before reopening the same path', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_DELAY_MS: '2000', @@ -142,4 +163,10 @@ describe('Destroy', () => { ROCKSDB_JS_CLOSE_FAILURE: '1', }); }, 15_000); + + it('waits for an in-progress shutdown retry before reopening', async () => { + await runDestroyFixture(shutdownRetryFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FAILURE: '1', + }); + }, 15_000); }); diff --git a/test/drop.test.ts b/test/drop.test.ts index 62e64aaec..d88632a47 100644 --- a/test/drop.test.ts +++ b/test/drop.test.ts @@ -224,7 +224,7 @@ describe('Drop', () => { { name: 'doomed', pessimistic: true }, ], }, - async ({ db: victim }, { db: doomed }, { db: stale }) => { + async ({ db: victim, dbPath }, { db: doomed }, { db: stale }) => { await expect( stale.transaction(async (txn: Transaction) => { await victim.put('live', 'A', { transaction: txn }); @@ -240,6 +240,8 @@ describe('Drop', () => { stale.close(); doomed.close(); expect(() => victim.close()).toThrow('Failed to flush database during close'); + const reopened = RocksDatabase.open(dbPath); + reopened.close(); } )); diff --git a/test/fixtures/fork-shutdown-retry.mts b/test/fixtures/fork-shutdown-retry.mts new file mode 100644 index 000000000..109100dbc --- /dev/null +++ b/test/fixtures/fork-shutdown-retry.mts @@ -0,0 +1,36 @@ +import { RocksDatabase } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { Worker } from 'node:worker_threads'; + +const path = process.argv[2]; +const db = RocksDatabase.open(path); +db.putSync('key', 'value'); +try { + db.close(); + throw new Error('Expected the initial close to fail'); +} catch (error) { + if (!String(error).includes('Injected database close failure')) throw error; +} + +delete process.env.ROCKSDB_JS_CLOSE_FAILURE; +process.env.ROCKSDB_JS_CLOSE_RETRY_DELAY_MS = '1000'; +const worker = new Worker(createWorkerBootstrapScript('./test/workers/shutdown-retry-worker.mts'), { + eval: true, +}); +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} +const started = await nextMessage(); +const shutdownResult = nextMessage(); +const reopened = RocksDatabase.open(path); +const elapsed = Date.now() - started; +if (elapsed < 500) throw new Error(`Open did not wait for the shutdown retry (${elapsed}ms)`); +if (reopened.getSync('key') !== 'value') throw new Error('Shutdown retry did not preserve data'); +const result = await shutdownResult; +if (!result.shutdown) throw new Error(`Shutdown retry failed: ${JSON.stringify(result)}`); +delete process.env.ROCKSDB_JS_CLOSE_RETRY_DELAY_MS; +reopened.destroy(); +await worker.terminate(); diff --git a/test/workers/shutdown-retry-worker.mts b/test/workers/shutdown-retry-worker.mts new file mode 100644 index 000000000..98bcda646 --- /dev/null +++ b/test/workers/shutdown-retry-worker.mts @@ -0,0 +1,11 @@ +import { shutdown } from '../../src/index.ts'; +import { parentPort } from 'node:worker_threads'; + +if (!parentPort) throw new Error('Shutdown retry worker requires a parent port'); +parentPort.postMessage(Date.now()); +try { + shutdown(); + parentPort.postMessage({ shutdown: true }); +} catch (error) { + parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) }); +} From bd4b571de80b6b06c1ad7e1306d0c85d9261a109 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:16:23 -0600 Subject: [PATCH 11/49] Update stream destroy lifecycle coverage --- test/backup-stream.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/backup-stream.test.ts b/test/backup-stream.test.ts index 7d3d90f2a..d09ccfcf7 100644 --- a/test/backup-stream.test.ts +++ b/test/backup-stream.test.ts @@ -188,7 +188,7 @@ describe('Streaming backups', () => { await expect(settled).resolves.toBe('settled'); })); - it('throws on destroy() during a stream and the stream still settles', () => + it('waits for an in-flight stream before destroying', () => dbRunner(async ({ db }) => { await writeAll(db, 200); @@ -203,9 +203,9 @@ describe('Streaming backups', () => { ); await new Promise((r) => setTimeout(r, 50)); - // The in-flight stream pins the descriptor, so destroy() refuses to tear - // the database down mid-stream and throws instead. - expect(() => db.destroy()).toThrow(); + // The stream pins the descriptor and registers an in-flight operation, so + // destroy waits for it before closing the native DB and removing the path. + expect(() => db.destroy()).not.toThrow(); await expect(settled).resolves.toBe('settled'); })); From b6b4856240fb5ecaa7dd55361a5819e82179ab39 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:24:46 -0600 Subject: [PATCH 12/49] Close final lifecycle race windows --- README.md | 5 +- src/binding/database/backup.cpp | 39 ++++++++- src/binding/database/db_registry.cpp | 110 ++++++++++++++++++------ src/load-binding.ts | 1 + test/destroy.test.ts | 15 +++- test/fixtures/fork-backup-destroy.mts | 20 +++++ test/fixtures/fork-destroy-failure.mts | 12 +++ test/fixtures/fork-shutdown-failure.mts | 12 +++ 8 files changed, 179 insertions(+), 35 deletions(-) create mode 100644 test/fixtures/fork-backup-destroy.mts diff --git a/README.md b/README.md index d88928f10..92be23365 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ Sets global database settings. Defaults to 32MB. Set to `0` (zero) disables block cache for future opened databases. Existing block cache for any opened databases is resized immediately. Negative values throw an error. - `compactOnClose: boolean` When `true`, compacts the database on close. Defaults to `false`. - - `lifecycleWaitSeconds: number` Maximum time a synchronous open, destroy, or shutdown waits for + - `lifecycleWaitSeconds: number` Total maximum time a synchronous open, destroy, or shutdown waits for another lifecycle operation before throwing a retryable timeout error. Defaults to `30` seconds and must be a positive integer. - `verificationTableEntries: number` The number of slots in the process-global @@ -1968,6 +1968,9 @@ console.log(currentThreadId()); Returns an array containing that status of all active RocksDB instances. - `path: string` The database path. +- `closeError?: string` The native lifecycle error retaining this registry entry. +- `destroyCleanupPending?: boolean` The native database is closed, but physical path cleanup must + finish before the next open. Opening the path retries cleanup automatically. - `refCount: number` The number of JavaScript database instances plus the registry's reference. - `columnFamiles: object` A map of column family names and their their info. - `userSharedBuffers: number` The count of active user shared buffers. diff --git a/src/binding/database/backup.cpp b/src/binding/database/backup.cpp index 295bb77e0..3b876fc4e 100644 --- a/src/binding/database/backup.cpp +++ b/src/binding/database/backup.cpp @@ -5,15 +5,18 @@ #include "database/db_handle.h" #include "database/db_registry.h" #include "core/file_lock.h" +#include "core/test_seam.h" #include "napi/async.h" #include "napi/helpers.h" #include "napi/macros.h" #include "rocksdb/env.h" #include "rocksdb/status.h" #include "rocksdb/utilities/backup_engine.h" +#include #include #include #include +#include #include namespace rocksdb_js { @@ -67,6 +70,17 @@ struct AsyncBackupState final : BaseAsyncState> { } }; +struct BackupInFlightClaim final { + DBDescriptor* descriptor; + const bool& handedOff; + + ~BackupInFlightClaim() { + if (!handedOff && --descriptor->operationsInFlight == 0 && descriptor->isClosing()) { + descriptor->operationsInFlight.notify_all(); + } + } +}; + /** * State for the `backupRestore` async work. There is no open database during a * restore, so the base handle is null. @@ -135,7 +149,8 @@ static napi_value queueBackupWork( State* state, napi_async_execute_callback execute, napi_async_complete_callback complete, - bool registerWork + bool registerWork, + bool* queued = nullptr ) { NAPI_STATUS_THROWS(::napi_create_reference(env, resolve, 1, &state->resolveRef)); NAPI_STATUS_THROWS(::napi_create_reference(env, reject, 1, &state->rejectRef)); @@ -150,6 +165,7 @@ static napi_value queueBackupWork( } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (queued) *queued = true; NAPI_RETURN_UNDEFINED(); } @@ -180,6 +196,10 @@ static rocksdb::Status runCreateBackup(AsyncBackupState* state) { if (!state->descriptor || !state->handle || state->handle->isCancelled()) { return rocksdb::Status::Aborted("Database closed during backup operation"); } + const int backupDelayMs = testDelayMs("ROCKSDB_JS_BACKUP_DELAY_MS"); + if (backupDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(backupDelayMs)); + } const std::string& backupDir = state->engineOptions.backup_dir; @@ -341,10 +361,19 @@ napi_value Database::Backup(napi_env env, napi_callback_info info) { bool checkDiskSpace = true; NAPI_STATUS_THROWS(getProperty(env, options, "checkDiskSpace", checkDiskSpace)); + auto descriptor = (*dbHandle)->descriptor; + ++descriptor->operationsInFlight; + bool handedOff = false; + BackupInFlightClaim claim{descriptor.get(), handedOff}; + if (descriptor->isClosing()) { + ::napi_throw_error(env, nullptr, "Database is closing"); + NAPI_RETURN_UNDEFINED(); + } + auto state = new AsyncBackupState( env, *dbHandle, - (*dbHandle)->descriptor, + descriptor, std::move(engineOptions), std::move(createOptions), std::move(appMetadata) @@ -361,6 +390,9 @@ napi_value Database::Backup(napi_env env, napi_callback_info info) { [](napi_env, void* data) { // execute auto state = reinterpret_cast(data); state->status = runCreateBackup(state); + if (--state->descriptor->operationsInFlight == 0 && state->descriptor->isClosing()) { + state->descriptor->operationsInFlight.notify_all(); + } state->signalExecuteCompleted(); }, [](napi_env env, napi_status status, void* data) { // complete @@ -379,7 +411,8 @@ napi_value Database::Backup(napi_env env, napi_callback_info info) { } delete state; }, - true // registerWork + true, // registerWork + &handedOff ); } diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index feefa8afe..f89b31c08 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -57,6 +57,28 @@ struct ClosingDescriptor final { ) : key(key), descriptor(std::move(descriptor)), condition(std::move(condition)) {} }; +std::string destroyPhysicalPath(const std::string& path) { + rocksdb::Status status = rocksdb::DestroyDB(path, rocksdb::Options()); + if (!status.ok()) return status.ToString(); + + std::error_code cleanupError; + std::filesystem::remove_all(path, cleanupError); + if (cleanupError) return "Failed to remove database directory: " + cleanupError.message(); + return {}; +} + +void emitCloseFailures(const std::vector& descriptors) { + if (!GlobalEvents::hasListeners()) return; + for (const auto& closing : descriptors) { + if (!closing.closeError.empty()) { + emitGlobalEvent( + "database:closeFailed", + ListenerData::fromStrings({closing.key.path, closing.closeError}) + ); + } + } +} + } // namespace // Initialize the static instance @@ -299,10 +321,9 @@ void DBRegistry::DestroyDB(const std::string& path) { for (const auto& closing : claimed) { closing.condition->notify_all(); } - if (closeError) { - std::rethrow_exception(closeError); - } + emitCloseFailures(claimed); if (alreadyClosing.empty()) { + if (closeError) std::rethrow_exception(closeError); break; } for (const auto& closing : alreadyClosing) { @@ -316,6 +337,7 @@ void DBRegistry::DestroyDB(const std::string& path) { throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": an open descriptor is still closing"); } } + if (closeError) std::rethrow_exception(closeError); } { @@ -338,22 +360,11 @@ void DBRegistry::DestroyDB(const std::string& path) { throw rocksdb_js::DBException("Injected database destruction failure"); } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); - rocksdb::Status status = rocksdb::DestroyDB(path, rocksdb::Options()); - if (!status.ok()) { - const std::string error = status.ToString(); + const std::string destroyError = destroyPhysicalPath(path); + if (!destroyError.empty()) { std::lock_guard lock(instance->databasesMutex); - instance->databases[DBKey{path, false}].closeError = error; - throw rocksdb_js::DBException(error); - } - - // remove the database directory including transaction logs - std::error_code cleanupError; - std::filesystem::remove_all(path, cleanupError); - if (cleanupError) { - const std::string error = cleanupError.message(); - std::lock_guard lock(instance->databasesMutex); - instance->databases[DBKey{path, false}].closeError = error; - throw rocksdb_js::DBException("Failed to remove database directory: " + error); + instance->databases[DBKey{path, false}].closeError = destroyError; + throw rocksdb_js::DBException(destroyError); } DEBUG_LOG("%p DBRegistry::DestroyDB Successfully destroyed database at \"%s\"\n", instance.get(), path.c_str()); @@ -412,6 +423,32 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons continue; } + auto failedDestroy = instance->databases.find(DBKey{path, false}); + if (failedDestroy != instance->databases.end() && + !failedDestroy->second.descriptor && !failedDestroy->second.closeError.empty() + ) { + instance->databases.erase(failedDestroy); + instance->destroyingPaths.insert(path); + lock.unlock(); + std::string destroyError; + { + DestroyPathGuard pathGuard( + instance->databasesMutex, + instance->lifecycleCondition, + instance->destroyingPaths, + path + ); + destroyError = destroyPhysicalPath(path); + if (!destroyError.empty()) { + std::lock_guard cleanupLock(instance->databasesMutex); + instance->databases[DBKey{path, false}].closeError = destroyError; + } + } + if (!destroyError.empty()) throw rocksdb_js::DBException(destroyError); + lock.lock(); + continue; + } + for (bool readOnly : {false, true}) { auto registeredEntry = instance->databases.find(DBKey{path, readOnly}); if (registeredEntry != instance->databases.end() && @@ -690,6 +727,7 @@ void DBRegistry::PurgeAll() { } closing.condition->notify_all(); } + emitCloseFailures(descriptorsToClose); if (closeError) { std::rethrow_exception(closeError); } @@ -713,9 +751,6 @@ napi_value DBRegistry::RegistryStatus(napi_env env, napi_callback_info info) { size_t i = 0; for (auto& [key, entry] : instance->databases) { - if (!entry.descriptor) { - continue; - } napi_value database; NAPI_STATUS_THROWS(::napi_create_object(env, &database)); napi_value pathValue; @@ -731,6 +766,21 @@ napi_value DBRegistry::RegistryStatus(napi_env env, napi_callback_info info) { )); NAPI_STATUS_THROWS(::napi_set_named_property(env, database, "closeError", closeErrorValue)); } + if (!entry.descriptor) { + napi_value pending; + NAPI_STATUS_THROWS(::napi_get_boolean(env, true, &pending)); + NAPI_STATUS_THROWS(::napi_set_named_property(env, database, "destroyCleanupPending", pending)); + napi_value zero; + NAPI_STATUS_THROWS(::napi_create_uint32(env, 0, &zero)); + for (const char* property : {"refCount", "transactions", "closables", "locks", "listenerCallbacks"}) { + NAPI_STATUS_THROWS(::napi_set_named_property(env, database, property, zero)); + } + napi_value columnFamilies; + NAPI_STATUS_THROWS(::napi_create_object(env, &columnFamilies)); + NAPI_STATUS_THROWS(::napi_set_named_property(env, database, "columnFamilies", columnFamilies)); + NAPI_STATUS_THROWS(::napi_set_element(env, result, i++, database)); + continue; + } napi_value modeValue; std::string mode = entry.descriptor->mode == DBMode::Optimistic ? "optimistic" : "pessimistic"; NAPI_STATUS_THROWS(::napi_create_string_utf8(env, mode.c_str(), mode.size(), &modeValue)); @@ -945,6 +995,7 @@ void DBRegistry::Shutdown() { while (true) { std::vector descriptorsToClose; std::vector descriptorsToWaitFor; + std::string destroyCleanupError; bool destroysInFlight; { std::unique_lock lock(instance->databasesMutex); @@ -958,11 +1009,10 @@ void DBRegistry::Shutdown() { continue; } if (!entry.descriptor) { - if (!entry.closeError.empty()) { - throw rocksdb_js::DBException( + if (!entry.closeError.empty() && destroyCleanupError.empty()) { + destroyCleanupError = "Cannot complete shutdown: database \"" + key.path + - "\" requires destroy() cleanup: " + entry.closeError - ); + "\" requires destroy cleanup: " + entry.closeError; } continue; } @@ -1008,7 +1058,7 @@ void DBRegistry::Shutdown() { } closing.condition->notify_all(); } - if (closeError) std::rethrow_exception(closeError); + emitCloseFailures(descriptorsToClose); for (const auto& closing : descriptorsToWaitFor) { std::unique_lock lock(instance->databasesMutex); @@ -1021,8 +1071,7 @@ void DBRegistry::Shutdown() { throw rocksdb_js::DBException("Timed out waiting for database close to finish during shutdown"); } } - if (descriptorsToClose.empty() && descriptorsToWaitFor.empty()) { - if (!destroysInFlight) break; + if (destroysInFlight) { std::unique_lock lock(instance->databasesMutex); if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { return instance->destroyingPaths.empty(); @@ -1030,6 +1079,11 @@ void DBRegistry::Shutdown() { throw rocksdb_js::DBException("Timed out waiting for database destruction to finish during shutdown"); } } + if (closeError) std::rethrow_exception(closeError); + if (!destroyCleanupError.empty()) { + throw rocksdb_js::DBException(destroyCleanupError); + } + if (descriptorsToClose.empty() && descriptorsToWaitFor.empty()) break; } // Purge the registry diff --git a/src/load-binding.ts b/src/load-binding.ts index 73584f189..2a3abd5e9 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -634,6 +634,7 @@ export type RegistryStatusTransaction = { export type RegistryStatusDB = { path: string; closeError?: string; + destroyCleanupPending?: boolean; refCount: number; columnFamilies: string[]; transactions: number; diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 1a1a3aa15..cb603c82a 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase } from '../src/index.ts'; +import { RocksDatabase, registryStatus } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; @@ -11,6 +11,7 @@ const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts' const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure.mts'); const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mts'); +const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); function runDestroyFixture( fixture: string, @@ -75,6 +76,12 @@ describe('Destroy', () => { expect(db.isOpen()).toBe(false); })); + it('waits for an in-flight directory backup before destroying', async () => { + await runDestroyFixture(backupDestroyFixture, generateDBPath(), { + ROCKSDB_JS_BACKUP_DELAY_MS: '500', + }); + }); + it('should destroy all related instances', () => dbRunner( { dbOptions: [{}, { name: 'test' }, { readOnly: true }] }, @@ -111,11 +118,13 @@ describe('Destroy', () => { chmodSync(lockedDirectory, 0o000); try { expect(() => db.destroy()).toThrow('Failed to remove database directory'); - expect(() => RocksDatabase.open(dbPath)).toThrow('previous destroy cleanup failed'); + expect( + registryStatus().find((entry) => entry.path === dbPath)?.destroyCleanupPending + ).toBe(true); + expect(() => RocksDatabase.open(dbPath)).toThrow('Failed to remove database directory'); } finally { chmodSync(lockedDirectory, 0o700); } - db.destroy(); const reopened = RocksDatabase.open(dbPath); reopened.close(); }) diff --git a/test/fixtures/fork-backup-destroy.mts b/test/fixtures/fork-backup-destroy.mts new file mode 100644 index 000000000..27861cb78 --- /dev/null +++ b/test/fixtures/fork-backup-destroy.mts @@ -0,0 +1,20 @@ +import { RocksDatabase } from '../../src/index.ts'; +import { rmSync } from 'node:fs'; + +const path = process.argv[2]; +const backupPath = `${path}-backup`; +const db = RocksDatabase.open(path); +db.putSync('key', 'value'); + +try { + const backup = db.backup(backupPath); + const startedAt = Date.now(); + db.destroy(); + const destroyDuration = Date.now() - startedAt; + await backup; + if (destroyDuration < 400) { + throw new Error(`Destroy did not wait for the directory backup (${destroyDuration}ms)`); + } +} finally { + rmSync(backupPath, { force: true, recursive: true }); +} diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index 9999a4d72..a090d2dcc 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -4,6 +4,9 @@ const path = process.argv[2]; const closeFailure = process.env.ROCKSDB_JS_CLOSE_FAILURE === '1'; const db = RocksDatabase.open(path); db.putSync('key', 'value'); +let resolveCloseFailure: (args: unknown[]) => void; +const closeFailureEvent = new Promise((resolve) => (resolveCloseFailure = resolve)); +RocksDatabase.on('database:closeFailed', (...args) => resolveCloseFailure(args)); let destroyError: unknown; try { @@ -18,6 +21,15 @@ if (!String(destroyError).includes(expectedError)) throw new Error(`Expected injected destroy failure, received: ${String(destroyError)}`); if (closeFailure) { + const args = await Promise.race([ + closeFailureEvent, + new Promise((_, reject) => + setTimeout(() => reject(new Error('Destroy did not emit database:closeFailed')), 1_000) + ), + ]); + if (args[0] !== path || args[1] !== expectedError) { + throw new Error(`Unexpected database:closeFailed arguments: ${JSON.stringify(args)}`); + } const startedAt = Date.now(); try { RocksDatabase.open(path); diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts index 19d868d4e..a834c5671 100644 --- a/test/fixtures/fork-shutdown-failure.mts +++ b/test/fixtures/fork-shutdown-failure.mts @@ -3,6 +3,9 @@ import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; const path = process.argv[2]; const db = RocksDatabase.open(path); db.putSync('key', 'value'); +let resolveCloseFailure: (args: unknown[]) => void; +const closeFailureEvent = new Promise((resolve) => (resolveCloseFailure = resolve)); +RocksDatabase.on('database:closeFailed', (...args) => resolveCloseFailure(args)); try { shutdown(); @@ -10,6 +13,15 @@ try { } catch (error) { if (!String(error).includes('Injected database close failure')) throw error; } +const args = await Promise.race([ + closeFailureEvent, + new Promise((_, reject) => + setTimeout(() => reject(new Error('Shutdown did not emit database:closeFailed')), 1_000) + ), +]); +if (args[0] !== path || args[1] !== 'Injected database close failure') { + throw new Error(`Unexpected database:closeFailed arguments: ${JSON.stringify(args)}`); +} const startedAt = Date.now(); try { RocksDatabase.open(path, { readOnly: true }); From 1ab06ed95ebfaefbb332085a455d3a0917f1b01f Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:29:28 -0600 Subject: [PATCH 13/49] Gate opens across process shutdown --- src/binding/database/db_descriptor.cpp | 5 ++- src/binding/database/db_registry.cpp | 46 ++++++++++++++++++++++++-- src/binding/database/db_registry.h | 1 + src/binding/database/db_settings.cpp | 9 +++++ test/destroy.test.ts | 1 + 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 99af48e07..29f3cfbbd 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -439,6 +439,9 @@ void DBDescriptor::finishClose() { // An in-flight commit pins this descriptor through its transaction and DB // handles, so reaching this release pass means only idle per-env TSFNs remain. + // Release rather than abort: queued completions are still delivered before + // finalization, while a later registration observes commitCompletionsClosed + // and falls back to the legacy libuv path. { std::lock_guard lock(this->commitMutex); for (auto& [env, completion] : this->commitCompletions) { @@ -542,8 +545,8 @@ void DBDescriptor::finishClose() { // Unregister from transaction log store registry - this will clean up stores // when the last descriptor for this path is closed if (!this->transactionLogsUnregistered) { - this->transactionLogsUnregistered = true; TransactionLogStoreRegistry::Unregister(this->path); + this->transactionLogsUnregistered = true; } this->transactions.clear(); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index f89b31c08..19f38ed62 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -43,6 +43,28 @@ class DestroyPathGuard final { const std::string& path; }; +class ShutdownGuard final { +public: + ShutdownGuard( + std::mutex& mutex, + std::condition_variable& condition, + bool& shutdownInProgress + ) : mutex(mutex), condition(condition), shutdownInProgress(shutdownInProgress) {} + + ~ShutdownGuard() { + { + std::lock_guard lock(this->mutex); + this->shutdownInProgress = false; + } + this->condition.notify_all(); + } + +private: + std::mutex& mutex; + std::condition_variable& condition; + bool& shutdownInProgress; +}; + struct ClosingDescriptor final { DBKey key; std::shared_ptr descriptor; @@ -242,9 +264,10 @@ void DBRegistry::DestroyDB(const std::string& path) { { std::unique_lock lock(instance->databasesMutex); if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { - return instance->destroyingPaths.find(path) == instance->destroyingPaths.end(); + return !instance->shutdownInProgress && + instance->destroyingPaths.find(path) == instance->destroyingPaths.end(); })) { - throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": another destroy is still in progress"); + throw rocksdb_js::DBException("Timed out waiting to destroy database \"" + path + "\": another lifecycle operation is still in progress"); } instance->destroyingPaths.insert(path); } @@ -351,7 +374,7 @@ void DBRegistry::DestroyDB(const std::string& path) { } } - // Now the database lock should be released, safe to destroy + // All in-process descriptors are closed; physical destruction can proceed. const int destroyDelayMs = testDelayMs("ROCKSDB_JS_DESTROY_DELAY_MS"); if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); @@ -411,6 +434,14 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons DBKey key{path, options.readOnly}; decltype(instance->databases)::iterator entryIterator; while (true) { + if (instance->shutdownInProgress) { + if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + return !instance->shutdownInProgress; + })) { + throw rocksdb_js::DBException("Timed out opening database \"" + path + "\": shutdown is still in progress"); + } + continue; + } if (!instance->destroyingPaths.empty() && instance->destroyingPaths.find(path) != instance->destroyingPaths.end() ) { @@ -992,6 +1023,15 @@ void DBRegistry::Shutdown() { if (!shutdownLock.try_lock_until(deadline)) { throw rocksdb_js::DBException("Timed out waiting for another database shutdown to finish"); } + { + std::lock_guard lock(instance->databasesMutex); + instance->shutdownInProgress = true; + } + ShutdownGuard shutdownGuard( + instance->databasesMutex, + instance->lifecycleCondition, + instance->shutdownInProgress + ); while (true) { std::vector descriptorsToClose; std::vector descriptorsToWaitFor; diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index edd01b8de..7091a5ee6 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -87,6 +87,7 @@ class DBRegistry final { */ std::mutex databasesMutex; std::timed_mutex shutdownMutex; + bool shutdownInProgress = false; // Destruction owns a physical path across every (path, readOnly) entry. // Waiters must re-resolve databases after every wake because the closer can // erase the node while the mutex is released. diff --git a/src/binding/database/db_settings.cpp b/src/binding/database/db_settings.cpp index 77a5ee27b..b0cfe00fa 100644 --- a/src/binding/database/db_settings.cpp +++ b/src/binding/database/db_settings.cpp @@ -219,6 +219,15 @@ napi_value DBSettings::Config(napi_env env, napi_callback_info info) { double lifecycleWaitSeconds = 0; bool lifecycleWaitProvided = false; NAPI_STATUS_THROWS(::napi_has_named_property(env, params, "lifecycleWaitSeconds", &lifecycleWaitProvided)); + if (lifecycleWaitProvided) { + napi_value lifecycleWaitValue; + NAPI_STATUS_THROWS(::napi_get_named_property(env, params, "lifecycleWaitSeconds", &lifecycleWaitValue)); + napi_valuetype lifecycleWaitType; + NAPI_STATUS_THROWS(::napi_typeof(env, lifecycleWaitValue, &lifecycleWaitType)); + if (lifecycleWaitType == napi_undefined || lifecycleWaitType == napi_null) { + lifecycleWaitProvided = false; + } + } if (lifecycleWaitProvided) { status = rocksdb_js::getProperty(env, params, "lifecycleWaitSeconds", lifecycleWaitSeconds, true); if (status != napi_ok) { diff --git a/test/destroy.test.ts b/test/destroy.test.ts index cb603c82a..4cf90974f 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -53,6 +53,7 @@ describe('Destroy', () => { expect(() => RocksDatabase.config({ lifecycleWaitSeconds: '30' as unknown as number })).toThrow( 'Lifecycle wait seconds must be a number' ); + expect(() => RocksDatabase.config({ lifecycleWaitSeconds: undefined })).not.toThrow(); expect(() => RocksDatabase.config({ lifecycleWaitSeconds: 30 })).not.toThrow(); }); From b4fa7b40815bd568705f9cc040792138da6bcb64 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:32:00 -0600 Subject: [PATCH 14/49] Rescan lifecycle state after destroy --- src/binding/database/db_registry.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 19f38ed62..e3ba02bec 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -1118,6 +1118,11 @@ void DBRegistry::Shutdown() { })) { throw rocksdb_js::DBException("Timed out waiting for database destruction to finish during shutdown"); } + if (closeError) std::rethrow_exception(closeError); + if (!destroyCleanupError.empty()) { + throw rocksdb_js::DBException(destroyCleanupError); + } + continue; } if (closeError) std::rethrow_exception(closeError); if (!destroyCleanupError.empty()) { From 08c1e52fa9d913f5f03ee53a33ac722628cff883 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:37:16 -0600 Subject: [PATCH 15/49] Keep destroy recovery explicit --- README.md | 2 +- src/binding/database/database.cpp | 15 ++++++- src/binding/database/db_registry.cpp | 60 +++++++++++++--------------- src/database.ts | 2 +- src/load-binding.ts | 2 +- test/destroy.test.ts | 19 ++++++++- 6 files changed, 61 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 92be23365..dc91e3ea0 100644 --- a/README.md +++ b/README.md @@ -1970,7 +1970,7 @@ Returns an array containing that status of all active RocksDB instances. - `path: string` The database path. - `closeError?: string` The native lifecycle error retaining this registry entry. - `destroyCleanupPending?: boolean` The native database is closed, but physical path cleanup must - finish before the next open. Opening the path retries cleanup automatically. + finish before the next open. Call `destroy()` or `shutdown()` to retry cleanup. - `refCount: number` The number of JavaScript database instances plus the registry's reference. - `columnFamiles: object` A map of column family names and their their info. - `userSharedBuffers: number` The count of active user shared buffers. diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index e3ca632b2..13384f7db 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -449,8 +449,21 @@ napi_value Database::Destroy(napi_env env, napi_callback_info info) { THROW_IF_READONLY((*dbHandle)->descriptor, "Destroy failed: "); if (*dbHandle) { + std::string path = (*dbHandle)->path; + napi_valuetype pathType; + NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &pathType)); + if (pathType == napi_string) { + NAPI_STATUS_THROWS_ERROR(rocksdb_js::getString(env, argv[0], path), "Database path must be a string"); + } else if (pathType != napi_undefined) { + ::napi_throw_type_error(env, nullptr, "Database path must be a string"); + return nullptr; + } + if (path.empty()) { + ::napi_throw_error(env, nullptr, "Database path is required for destroy"); + return nullptr; + } try { - DBRegistry::DestroyDB((*dbHandle)->path); + DBRegistry::DestroyDB(path); } catch (const std::exception& e) { DEBUG_LOG("%p Database::Destroy Error: %s\n", dbHandle->get(), e.what()); ::napi_throw_error(env, nullptr, e.what()); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index e3ba02bec..56f0a0235 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -454,32 +454,6 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons continue; } - auto failedDestroy = instance->databases.find(DBKey{path, false}); - if (failedDestroy != instance->databases.end() && - !failedDestroy->second.descriptor && !failedDestroy->second.closeError.empty() - ) { - instance->databases.erase(failedDestroy); - instance->destroyingPaths.insert(path); - lock.unlock(); - std::string destroyError; - { - DestroyPathGuard pathGuard( - instance->databasesMutex, - instance->lifecycleCondition, - instance->destroyingPaths, - path - ); - destroyError = destroyPhysicalPath(path); - if (!destroyError.empty()) { - std::lock_guard cleanupLock(instance->databasesMutex); - instance->databases[DBKey{path, false}].closeError = destroyError; - } - } - if (!destroyError.empty()) throw rocksdb_js::DBException(destroyError); - lock.lock(); - continue; - } - for (bool readOnly : {false, true}) { auto registeredEntry = instance->databases.find(DBKey{path, readOnly}); if (registeredEntry != instance->databases.end() && @@ -1035,7 +1009,7 @@ void DBRegistry::Shutdown() { while (true) { std::vector descriptorsToClose; std::vector descriptorsToWaitFor; - std::string destroyCleanupError; + std::vector destroyCleanupEntries; bool destroysInFlight; { std::unique_lock lock(instance->databasesMutex); @@ -1043,17 +1017,14 @@ void DBRegistry::Shutdown() { DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); descriptorsToClose.reserve(instance->databases.size()); descriptorsToWaitFor.reserve(instance->databases.size()); + destroyCleanupEntries.reserve(instance->databases.size()); for (auto& [key, entry] : instance->databases) { if (instance->destroyingPaths.find(key.path) != instance->destroyingPaths.end()) { continue; } if (!entry.descriptor) { - if (!entry.closeError.empty() && destroyCleanupError.empty()) { - destroyCleanupError = - "Cannot complete shutdown: database \"" + key.path + - "\" requires destroy cleanup: " + entry.closeError; - } + if (!entry.closeError.empty()) destroyCleanupEntries.push_back(key); continue; } ClosingDescriptor closing{key, entry.descriptor, entry.condition}; @@ -1111,6 +1082,27 @@ void DBRegistry::Shutdown() { throw rocksdb_js::DBException("Timed out waiting for database close to finish during shutdown"); } } + + std::string destroyCleanupError; + for (const auto& key : destroyCleanupEntries) { + const std::string cleanupError = destroyPhysicalPath(key.path); + { + std::lock_guard lock(instance->databasesMutex); + auto entry = instance->databases.find(key); + if (entry != instance->databases.end() && !entry->second.descriptor) { + if (cleanupError.empty()) { + instance->databases.erase(entry); + } else { + entry->second.closeError = cleanupError; + } + } + } + if (!cleanupError.empty() && destroyCleanupError.empty()) { + destroyCleanupError = + "Cannot complete shutdown: database \"" + key.path + + "\" requires destroy cleanup: " + cleanupError; + } + } if (destroysInFlight) { std::unique_lock lock(instance->databasesMutex); if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { @@ -1128,7 +1120,9 @@ void DBRegistry::Shutdown() { if (!destroyCleanupError.empty()) { throw rocksdb_js::DBException(destroyCleanupError); } - if (descriptorsToClose.empty() && descriptorsToWaitFor.empty()) break; + if (descriptorsToClose.empty() && descriptorsToWaitFor.empty() && + destroyCleanupEntries.empty() + ) break; } // Purge the registry diff --git a/src/database.ts b/src/database.ts index 21c15c83d..91878f4cb 100644 --- a/src/database.ts +++ b/src/database.ts @@ -408,7 +408,7 @@ export class RocksDatabase extends DBI { // committed destroy(): void { - this.store.db.destroy(); + this.store.db.destroy(this.store.path); } async drop(): Promise { diff --git a/src/load-binding.ts b/src/load-binding.ts index 2a3abd5e9..cdc8cf3da 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -428,7 +428,7 @@ export type NativeDatabase = { reject: RejectCallback, targetPath: string ): void; - destroy(): void; + destroy(path?: string): void; drop(resolve: ResolveCallback, reject: RejectCallback): void; dropSync(): void; flush(resolve: ResolveCallback, reject: RejectCallback, options?: FlushOptions): void; diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 4cf90974f..d4f4f976f 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase, registryStatus } from '../src/index.ts'; +import { RocksDatabase, registryStatus, shutdown } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; @@ -68,6 +68,13 @@ describe('Destroy', () => { expect(db.isOpen()).toBe(false); })); + it('should destroy a database from an unopened handle', () => + dbRunner(({ db, dbPath }) => { + db.close(); + new RocksDatabase(dbPath).destroy(); + expect(existsSync(dbPath)).toBe(false); + })); + it('should destroy an open database', () => dbRunner(({ db, dbPath }) => { db.putSync('key', 'value'); @@ -113,6 +120,9 @@ describe('Destroy', () => { 'quarantines a path when post-destroy cleanup fails', () => dbRunner(({ db, dbPath }) => { + const healthyPath = generateDBPath(); + const healthy = RocksDatabase.open(healthyPath); + healthy.putSync('key', 'value'); const lockedDirectory = join(dbPath, 'transaction_logs', 'locked'); mkdirSync(lockedDirectory, { recursive: true }); writeFileSync(join(lockedDirectory, 'leftover'), 'data'); @@ -122,10 +132,15 @@ describe('Destroy', () => { expect( registryStatus().find((entry) => entry.path === dbPath)?.destroyCleanupPending ).toBe(true); - expect(() => RocksDatabase.open(dbPath)).toThrow('Failed to remove database directory'); + expect(() => RocksDatabase.open(dbPath)).toThrow('previous destroy cleanup failed'); } finally { chmodSync(lockedDirectory, 0o700); } + shutdown(); + expect(registryStatus().some((entry) => entry.path === dbPath)).toBe(false); + const healthyReopened = RocksDatabase.open(healthyPath); + expect(healthyReopened.getSync('key')).toBe('value'); + healthyReopened.destroy(); const reopened = RocksDatabase.open(dbPath); reopened.close(); }) From 5faf1ef999719e8ca893c31e7685ea6d3aaf32e7 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:39:34 -0600 Subject: [PATCH 16/49] Release cancelled lifecycle operations --- src/binding/database/backup.cpp | 7 +++++++ src/binding/database/backup_stream.cpp | 5 +++++ src/binding/database/checkpoint.cpp | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/src/binding/database/backup.cpp b/src/binding/database/backup.cpp index 3b876fc4e..330657e06 100644 --- a/src/binding/database/backup.cpp +++ b/src/binding/database/backup.cpp @@ -398,6 +398,13 @@ napi_value Database::Backup(napi_env env, napi_callback_info info) { [](napi_env env, napi_status status, void* data) { // complete auto state = reinterpret_cast(data); state->deleteAsyncWork(); + // A cancelled queued work item never ran execute, so complete owns the + // in-flight decrement in that case. + if (status == napi_cancelled && + --state->descriptor->operationsInFlight == 0 && state->descriptor->isClosing() + ) { + state->descriptor->operationsInFlight.notify_all(); + } if (status != napi_cancelled) { if (state->status.ok()) { napi_value result; diff --git a/src/binding/database/backup_stream.cpp b/src/binding/database/backup_stream.cpp index 05d328ed2..5d53d98ea 100644 --- a/src/binding/database/backup_stream.cpp +++ b/src/binding/database/backup_stream.cpp @@ -571,6 +571,11 @@ void backupStreamExecute(napi_env, void* data) { void backupStreamComplete(napi_env env, napi_status status, void* data) { auto* state = static_cast(data); state->deleteAsyncWork(); + if (status == napi_cancelled && + --state->descriptor->operationsInFlight == 0 && state->descriptor->isClosing() + ) { + state->descriptor->operationsInFlight.notify_all(); + } // Release the tsfn. On a normal run its queue is already drained; on a // teardown abort a trampoline may still be queued. Either way, tsfnFinalize diff --git a/src/binding/database/checkpoint.cpp b/src/binding/database/checkpoint.cpp index 930cca065..7a500dde4 100644 --- a/src/binding/database/checkpoint.cpp +++ b/src/binding/database/checkpoint.cpp @@ -167,6 +167,11 @@ napi_value Database::CreateCheckpoint(napi_env env, napi_callback_info info) { [](napi_env env, napi_status status, void* data) { // complete auto state = reinterpret_cast(data); state->deleteAsyncWork(); + if (status == napi_cancelled && + --state->descriptor->operationsInFlight == 0 && state->descriptor->isClosing() + ) { + state->descriptor->operationsInFlight.notify_all(); + } if (status != napi_cancelled) { if (state->status.ok()) { napi_value undefined; From 2be5d39c7129ae2c70a2f3c4dbe205cace344e16 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 14 Aug 2026 23:40:58 -0600 Subject: [PATCH 17/49] Preserve read-only destroy protection --- README.md | 9 ++++++++- src/binding/database/database.cpp | 22 ++++++++++++++++++++-- src/database.ts | 2 +- src/load-binding.ts | 2 +- test/destroy.test.ts | 4 ++++ 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dc91e3ea0..565f61b0f 100644 --- a/README.md +++ b/README.md @@ -356,7 +356,14 @@ db.compactSync({ bottommost: true }); ### `db.destroy(): void` Completely removes a database based on the `db` instance's path including all data, column families, -and files on disk. +and files on disk. Destruction owns the physical path for the process: it closes every writable and +read-only handle for that path, waits for registered backups and checkpoints to stop using the +native database, and prevents another handle from reopening the path until removal finishes. Those +waits are synchronous and can outlive `lifecycleWaitSeconds` once destruction has claimed the path, +because releasing the native database beneath an active copy would be unsafe. + +The instance does not need to be open, which allows an explicit `destroy()` retry after failed +physical cleanup. A read-only instance cannot destroy the database. ```typescript db.destroy(); diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 13384f7db..4ee4d3f0e 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -444,11 +444,25 @@ napi_value Database::CompactSync(napi_env env, napi_callback_info info) { * ``` */ napi_value Database::Destroy(napi_env env, napi_callback_info info) { - NAPI_METHOD_ARGV(1); + NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE(); - THROW_IF_READONLY((*dbHandle)->descriptor, "Destroy failed: "); if (*dbHandle) { + bool readOnly = (*dbHandle)->descriptor && (*dbHandle)->descriptor->readOnly; + napi_valuetype readOnlyType; + NAPI_STATUS_THROWS(::napi_typeof(env, argv[1], &readOnlyType)); + if (readOnlyType == napi_boolean) { + bool requestedReadOnly = false; + NAPI_STATUS_THROWS_ERROR(rocksdb_js::getValue(env, argv[1], requestedReadOnly), "Read-only flag must be a boolean"); + readOnly = readOnly || requestedReadOnly; + } else if (readOnlyType != napi_undefined) { + ::napi_throw_type_error(env, nullptr, "Read-only flag must be a boolean"); + return nullptr; + } + if (readOnly) { + ::napi_throw_error(env, "ERR_DATABASE_READONLY", "Destroy failed: Unsupported operation in read-only mode"); + return nullptr; + } std::string path = (*dbHandle)->path; napi_valuetype pathType; NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &pathType)); @@ -462,6 +476,10 @@ napi_value Database::Destroy(napi_env env, napi_callback_info info) { ::napi_throw_error(env, nullptr, "Database path is required for destroy"); return nullptr; } + if (!(*dbHandle)->path.empty() && path != (*dbHandle)->path) { + ::napi_throw_error(env, nullptr, "Destroy path must match the open database"); + return nullptr; + } try { DBRegistry::DestroyDB(path); } catch (const std::exception& e) { diff --git a/src/database.ts b/src/database.ts index 91878f4cb..75160e562 100644 --- a/src/database.ts +++ b/src/database.ts @@ -408,7 +408,7 @@ export class RocksDatabase extends DBI { // committed destroy(): void { - this.store.db.destroy(this.store.path); + this.store.db.destroy(this.store.path, this.store.readOnly); } async drop(): Promise { diff --git a/src/load-binding.ts b/src/load-binding.ts index cdc8cf3da..b2b210309 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -428,7 +428,7 @@ export type NativeDatabase = { reject: RejectCallback, targetPath: string ): void; - destroy(path?: string): void; + destroy(path?: string, readOnly?: boolean): void; drop(resolve: ResolveCallback, reject: RejectCallback): void; dropSync(): void; flush(resolve: ResolveCallback, reject: RejectCallback, options?: FlushOptions): void; diff --git a/test/destroy.test.ts b/test/destroy.test.ts index d4f4f976f..51862608f 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -71,6 +71,10 @@ describe('Destroy', () => { it('should destroy a database from an unopened handle', () => dbRunner(({ db, dbPath }) => { db.close(); + expect(() => new RocksDatabase(dbPath, { readOnly: true }).destroy()).toThrow( + 'Unsupported operation in read-only mode' + ); + expect(existsSync(dbPath)).toBe(true); new RocksDatabase(dbPath).destroy(); expect(existsSync(dbPath)).toBe(false); })); From c78a015632427fcbdeae70a622f4acd3064297a8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 00:02:56 -0600 Subject: [PATCH 18/49] Document destroy during stream backup --- docs/backups.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/backups.md b/docs/backups.md index 1728bfb60..52aba9969 100644 --- a/docs/backups.md +++ b/docs/backups.md @@ -202,8 +202,9 @@ The archive contains the standard RocksDB files for the snapshot (`CURRENT`, a ` consumer therefore lets obsolete SST files accumulate (transient disk pressure) until the stream finishes. This is the cost of streaming with no scratch copy; a fast consumer is unaffected. - **The database must stay open for the whole stream.** Closing or destroying the database while a - stream is in flight aborts it: the backup promise rejects, and `destroy()` throws rather than - tearing the database down underneath the copy. + stream is in flight aborts the stream and makes the backup promise reject. `destroy()` waits for + the native producer to release its in-flight claim, then closes every handle for the path and + removes the database; it never tears the native database down underneath the copy. - **A consumer error aborts the backup.** If `stream.write()` rejects (or the stream is aborted), the backup promise rejects and the native producer stops. From 031d96fc74162b932120ef501b454e887a7a4ef4 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 00:10:07 -0600 Subject: [PATCH 19/49] Restrict destroy to known database handles --- README.md | 5 +++-- src/binding/database/database.cpp | 23 +++++------------------ src/database.ts | 2 +- src/load-binding.ts | 2 +- test/destroy.test.ts | 7 +++++-- 5 files changed, 15 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 565f61b0f..943a91695 100644 --- a/README.md +++ b/README.md @@ -362,8 +362,9 @@ native database, and prevents another handle from reopening the path until remov waits are synchronous and can outlive `lifecycleWaitSeconds` once destruction has claimed the path, because releasing the native database beneath an active copy would be unsafe. -The instance does not need to be open, which allows an explicit `destroy()` retry after failed -physical cleanup. A read-only instance cannot destroy the database. +A previously opened instance does not need to remain open, which allows an explicit `destroy()` +retry after failed physical cleanup. A never-opened or read-only instance cannot destroy the +database; `shutdown()` can recover a tombstone when its original handle is no longer available. ```typescript db.destroy(); diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 4ee4d3f0e..43bf06221 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -444,16 +444,16 @@ napi_value Database::CompactSync(napi_env env, napi_callback_info info) { * ``` */ napi_value Database::Destroy(napi_env env, napi_callback_info info) { - NAPI_METHOD_ARGV(2); + NAPI_METHOD_ARGV(1); UNWRAP_DB_HANDLE(); if (*dbHandle) { bool readOnly = (*dbHandle)->descriptor && (*dbHandle)->descriptor->readOnly; napi_valuetype readOnlyType; - NAPI_STATUS_THROWS(::napi_typeof(env, argv[1], &readOnlyType)); + NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &readOnlyType)); if (readOnlyType == napi_boolean) { bool requestedReadOnly = false; - NAPI_STATUS_THROWS_ERROR(rocksdb_js::getValue(env, argv[1], requestedReadOnly), "Read-only flag must be a boolean"); + NAPI_STATUS_THROWS_ERROR(rocksdb_js::getValue(env, argv[0], requestedReadOnly), "Read-only flag must be a boolean"); readOnly = readOnly || requestedReadOnly; } else if (readOnlyType != napi_undefined) { ::napi_throw_type_error(env, nullptr, "Read-only flag must be a boolean"); @@ -463,25 +463,12 @@ napi_value Database::Destroy(napi_env env, napi_callback_info info) { ::napi_throw_error(env, "ERR_DATABASE_READONLY", "Destroy failed: Unsupported operation in read-only mode"); return nullptr; } - std::string path = (*dbHandle)->path; - napi_valuetype pathType; - NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &pathType)); - if (pathType == napi_string) { - NAPI_STATUS_THROWS_ERROR(rocksdb_js::getString(env, argv[0], path), "Database path must be a string"); - } else if (pathType != napi_undefined) { - ::napi_throw_type_error(env, nullptr, "Database path must be a string"); - return nullptr; - } - if (path.empty()) { + if ((*dbHandle)->path.empty()) { ::napi_throw_error(env, nullptr, "Database path is required for destroy"); return nullptr; } - if (!(*dbHandle)->path.empty() && path != (*dbHandle)->path) { - ::napi_throw_error(env, nullptr, "Destroy path must match the open database"); - return nullptr; - } try { - DBRegistry::DestroyDB(path); + DBRegistry::DestroyDB((*dbHandle)->path); } catch (const std::exception& e) { DEBUG_LOG("%p Database::Destroy Error: %s\n", dbHandle->get(), e.what()); ::napi_throw_error(env, nullptr, e.what()); diff --git a/src/database.ts b/src/database.ts index 75160e562..e5c255357 100644 --- a/src/database.ts +++ b/src/database.ts @@ -408,7 +408,7 @@ export class RocksDatabase extends DBI { // committed destroy(): void { - this.store.db.destroy(this.store.path, this.store.readOnly); + this.store.db.destroy(this.store.readOnly); } async drop(): Promise { diff --git a/src/load-binding.ts b/src/load-binding.ts index b2b210309..4c05dcb95 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -428,7 +428,7 @@ export type NativeDatabase = { reject: RejectCallback, targetPath: string ): void; - destroy(path?: string, readOnly?: boolean): void; + destroy(readOnly?: boolean): void; drop(resolve: ResolveCallback, reject: RejectCallback): void; dropSync(): void; flush(resolve: ResolveCallback, reject: RejectCallback, options?: FlushOptions): void; diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 51862608f..34eea0f66 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -68,14 +68,17 @@ describe('Destroy', () => { expect(db.isOpen()).toBe(false); })); - it('should destroy a database from an unopened handle', () => + it('should reject destroy from a never-opened handle', () => dbRunner(({ db, dbPath }) => { db.close(); expect(() => new RocksDatabase(dbPath, { readOnly: true }).destroy()).toThrow( 'Unsupported operation in read-only mode' ); + expect(() => new RocksDatabase(dbPath).destroy()).toThrow( + 'Database path is required for destroy' + ); expect(existsSync(dbPath)).toBe(true); - new RocksDatabase(dbPath).destroy(); + db.destroy(); expect(existsSync(dbPath)).toBe(false); })); From b675f159ae15ab05f3df02e23e25237cc1817d66 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 00:32:33 -0600 Subject: [PATCH 20/49] Run lifecycle fault fixtures under Node --- test/destroy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 34eea0f66..862eca06d 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -19,7 +19,7 @@ function runDestroyFixture( env?: NodeJS.ProcessEnv ): Promise { return new Promise((resolve, reject) => { - const child = spawn(process.execPath, ['--expose-gc', fixture, dbPath], { + const child = spawn('node', ['--expose-gc', fixture, dbPath], { env: { ...process.env, ...env }, }); let stderr = ''; From 64849f9eb3cb73bef3ccce27e4cd2c6ff9252c9a Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 00:40:19 -0600 Subject: [PATCH 21/49] Resolve Node for lifecycle fixtures --- test/destroy.test.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 862eca06d..dffaa858a 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -12,6 +12,11 @@ const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mts'); const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); +const nodeExecutable = + process.env.NODE_BINARY ?? + (process.versions.bun || process.versions.deno + ? (process.env.npm_node_execpath ?? 'node') + : process.execPath); function runDestroyFixture( fixture: string, @@ -19,7 +24,8 @@ function runDestroyFixture( env?: NodeJS.ProcessEnv ): Promise { return new Promise((resolve, reject) => { - const child = spawn('node', ['--expose-gc', fixture, dbPath], { + // These fixtures depend on Node's type stripping, GC flag, and worker semantics. + const child = spawn(nodeExecutable, ['--expose-gc', fixture, dbPath], { env: { ...process.env, ...env }, }); let stderr = ''; @@ -30,7 +36,10 @@ function runDestroyFixture( child.kill(); reject(new Error(`Destroy fixture timed out\n${stderr}`)); }, 10_000); - child.on('error', reject); + child.on('error', (error) => { + clearTimeout(timeout); + reject(new Error(`Unable to run lifecycle fixture with Node (${nodeExecutable}): ${error}`)); + }); child.on('close', (code, signal) => { clearTimeout(timeout); if (code === 0 && signal === null) { From 44a22f76705279ccdc9e3449df73d88766560c28 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 00:59:13 -0600 Subject: [PATCH 22/49] Consume close-failure seam natively --- src/binding/core/test_seam.h | 15 +++++++++++++++ src/binding/database/db_descriptor.cpp | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index c7e3c15fb..4e0720c35 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -16,6 +16,21 @@ inline int testDelayMs(const char* envName) { return value ? ::atoi(value) : 0; } +// Consume native fault flags in the same C runtime that reads them. JavaScript +// process.env deletion does not reliably update the MSVC runtime environment. +inline bool testConsumeFlag(const char* envName) { + const char* value = ::getenv(envName); + if (!value || ::atoi(value) <= 0) { + return false; + } +#ifdef _WIN32 + ::_putenv_s(envName, ""); +#else + ::unsetenv(envName); +#endif + return true; +} + // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N // transaction commits to fail with TryAgain (the caller rolls back so no data is committed), // reproducing an ERR_TRY_AGAIN that a real memtable flush would cause but that is finicky to diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 29f3cfbbd..40d3cdf7e 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -461,7 +461,7 @@ void DBDescriptor::finishClose() { } } - if (testDelayMs("ROCKSDB_JS_CLOSE_FAILURE") > 0) { + if (testConsumeFlag("ROCKSDB_JS_CLOSE_FAILURE")) { throw rocksdb_js::DBException("Injected database close failure"); } if (!this->db) { From c49d5204fa6f458ee9c72744f71fbca40e7f83c1 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 01:23:40 -0600 Subject: [PATCH 23/49] Release copy pins before promise settlement --- src/binding/database/backup.cpp | 10 +++++++++- src/binding/database/checkpoint.cpp | 7 ++++++- test/backup.test.ts | 2 +- test/checkpoint.test.ts | 3 ++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/binding/database/backup.cpp b/src/binding/database/backup.cpp index 330657e06..a4a663010 100644 --- a/src/binding/database/backup.cpp +++ b/src/binding/database/backup.cpp @@ -60,7 +60,7 @@ struct AsyncBackupState final : BaseAsyncState> { // Our descriptor ref can be the reason a concurrent close skipped its // registry purge (use_count() > 1), so on release we must retry the purge or // the registry entry — and the open RocksDB — would linger forever. - ~AsyncBackupState() override { + void releaseDescriptor() { if (this->descriptor) { std::string path = this->descriptor->path; bool readOnly = this->descriptor->readOnly; @@ -68,6 +68,10 @@ struct AsyncBackupState final : BaseAsyncState> { DBRegistry::PurgeIfUnreferenced(path, readOnly); } } + + ~AsyncBackupState() override { + this->releaseDescriptor(); + } }; struct BackupInFlightClaim final { @@ -405,6 +409,10 @@ napi_value Database::Backup(napi_env env, napi_callback_info info) { ) { state->descriptor->operationsInFlight.notify_all(); } + // Promise settlement is the public completion boundary. Release the + // descriptor pin and retry any deferred registry purge first so an + // immediate registryStatus() / shutdown() cannot observe stale state. + state->releaseDescriptor(); if (status != napi_cancelled) { if (state->status.ok()) { napi_value result; diff --git a/src/binding/database/checkpoint.cpp b/src/binding/database/checkpoint.cpp index 7a500dde4..ddf8a1fc2 100644 --- a/src/binding/database/checkpoint.cpp +++ b/src/binding/database/checkpoint.cpp @@ -47,7 +47,7 @@ struct AsyncCheckpointState final : BaseAsyncState> { descriptor(std::move(descriptor)), targetPath(std::move(targetPath)) {} - ~AsyncCheckpointState() override { + void releaseDescriptor() { if (this->descriptor) { std::string path = this->descriptor->path; bool readOnly = this->descriptor->readOnly; @@ -55,6 +55,10 @@ struct AsyncCheckpointState final : BaseAsyncState> { DBRegistry::PurgeIfUnreferenced(path, readOnly); } } + + ~AsyncCheckpointState() override { + this->releaseDescriptor(); + } }; /** @@ -172,6 +176,7 @@ napi_value Database::CreateCheckpoint(napi_env env, napi_callback_info info) { ) { state->descriptor->operationsInFlight.notify_all(); } + state->releaseDescriptor(); if (status != napi_cancelled) { if (state->status.ok()) { napi_value undefined; diff --git a/test/backup.test.ts b/test/backup.test.ts index 18c5ff8f0..7e5eba489 100644 --- a/test/backup.test.ts +++ b/test/backup.test.ts @@ -475,7 +475,7 @@ describe('Backups', () => { // backup must retry it on release so the entry does not leak (a leaked // entry keeps the RocksDB open forever and shows up in registryStatus() // long after every handle is closed). - expect(registryStatus().length).toBe(0); + expect(registryStatus()).toEqual([]); })); it('should reject listing a non-existent backup directory', async () => { diff --git a/test/checkpoint.test.ts b/test/checkpoint.test.ts index 69fac4708..aed386c2f 100644 --- a/test/checkpoint.test.ts +++ b/test/checkpoint.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase } from '../src/index.ts'; +import { RocksDatabase, registryStatus } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { existsSync, mkdirSync, rmSync } from 'node:fs'; import { join } from 'node:path'; @@ -204,6 +204,7 @@ describe('Checkpoints', () => { () => 'settled' ) ).resolves.toBe('settled'); + expect(registryStatus()).toEqual([]); })); it('should not free the database under an in-flight checkpoint when destroy() races it', () => From 9c62cfb9b90be3e0e2eef6f7def446c198bcf950 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 01:30:14 -0600 Subject: [PATCH 24/49] Align stream backup completion cleanup --- src/binding/database/backup_stream.cpp | 9 ++++++++- test/backup-stream.test.ts | 5 +++-- test/backup.test.ts | 4 ++-- test/checkpoint.test.ts | 4 ++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/binding/database/backup_stream.cpp b/src/binding/database/backup_stream.cpp index 5d53d98ea..6eb8c6803 100644 --- a/src/binding/database/backup_stream.cpp +++ b/src/binding/database/backup_stream.cpp @@ -129,7 +129,7 @@ struct AsyncBackupStreamState final : BaseAsyncState> // Our descriptor ref can be the reason a concurrent close skipped its // registry purge (use_count() > 1), so on release we must retry the purge or // the registry entry — and the open RocksDB — would linger forever. - ~AsyncBackupStreamState() override { + void releaseDescriptor() { if (this->descriptor) { std::string path = this->descriptor->path; bool readOnly = this->descriptor->readOnly; @@ -138,6 +138,10 @@ struct AsyncBackupStreamState final : BaseAsyncState> } } + ~AsyncBackupStreamState() override { + this->releaseDescriptor(); + } + /** * Called on the JS thread (promise continuation / failure path) to wake the * blocked worker. `ok == false` requests the worker to abort the stream. @@ -587,6 +591,9 @@ void backupStreamComplete(napi_env env, napi_status status, void* data) { state->tsfn = nullptr; } + // Promise settlement is the public completion boundary. The worker is done, + // so pending JS acknowledgements no longer need the descriptor pin. + state->releaseDescriptor(); if (status != napi_cancelled) { if (state->status.ok()) { napi_value undefined; diff --git a/test/backup-stream.test.ts b/test/backup-stream.test.ts index d09ccfcf7..e655d562a 100644 --- a/test/backup-stream.test.ts +++ b/test/backup-stream.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase } from '../src/index.ts'; +import { RocksDatabase, registryStatus } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { gunzipSync } from 'node:zlib'; @@ -164,7 +164,7 @@ describe('Streaming backups', () => { })); it('settles without hanging when the database is closed mid-stream', () => - dbRunner({ skipOpen: true }, async ({ db }) => { + dbRunner({ skipOpen: true }, async ({ db, dbPath }) => { db.open(); await writeAll(db, 200); @@ -186,6 +186,7 @@ describe('Streaming backups', () => { // If the worker deadlocked against close(), this would time out. await expect(settled).resolves.toBe('settled'); + expect(registryStatus().some((entry) => entry.path === dbPath)).toBe(false); })); it('waits for an in-flight stream before destroying', () => diff --git a/test/backup.test.ts b/test/backup.test.ts index 7e5eba489..6aba2969e 100644 --- a/test/backup.test.ts +++ b/test/backup.test.ts @@ -452,7 +452,7 @@ describe('Backups', () => { ); it('should not crash when closing during a backup', () => - dbRunner({ skipOpen: true }, async ({ db }) => { + dbRunner({ skipOpen: true }, async ({ db, dbPath }) => { db.open(); await writeAll(db, 200); @@ -475,7 +475,7 @@ describe('Backups', () => { // backup must retry it on release so the entry does not leak (a leaked // entry keeps the RocksDB open forever and shows up in registryStatus() // long after every handle is closed). - expect(registryStatus()).toEqual([]); + expect(registryStatus().some((entry) => entry.path === dbPath)).toBe(false); })); it('should reject listing a non-existent backup directory', async () => { diff --git a/test/checkpoint.test.ts b/test/checkpoint.test.ts index aed386c2f..9d958bce2 100644 --- a/test/checkpoint.test.ts +++ b/test/checkpoint.test.ts @@ -187,7 +187,7 @@ describe('Checkpoints', () => { })); it('should not crash when closing during a checkpoint', () => - dbRunner(async ({ db }) => { + dbRunner(async ({ db, dbPath }) => { await writeAll(db, 200); const checkpointDir = tempDir(); @@ -204,7 +204,7 @@ describe('Checkpoints', () => { () => 'settled' ) ).resolves.toBe('settled'); - expect(registryStatus()).toEqual([]); + expect(registryStatus().some((entry) => entry.path === dbPath)).toBe(false); })); it('should not free the database under an in-flight checkpoint when destroy() races it', () => From 2302dd715230118ddb6316507409947a133151f8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 01:41:10 -0600 Subject: [PATCH 25/49] Seed retry delay before fixture startup --- test/destroy.test.ts | 1 + test/fixtures/fork-shutdown-retry.mts | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/test/destroy.test.ts b/test/destroy.test.ts index dffaa858a..0677fa47b 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -208,6 +208,7 @@ describe('Destroy', () => { it('waits for an in-progress shutdown retry before reopening', async () => { await runDestroyFixture(shutdownRetryFixture, generateDBPath(), { ROCKSDB_JS_CLOSE_FAILURE: '1', + ROCKSDB_JS_CLOSE_RETRY_DELAY_MS: '1000', }); }, 15_000); }); diff --git a/test/fixtures/fork-shutdown-retry.mts b/test/fixtures/fork-shutdown-retry.mts index 109100dbc..0bc93e407 100644 --- a/test/fixtures/fork-shutdown-retry.mts +++ b/test/fixtures/fork-shutdown-retry.mts @@ -12,8 +12,6 @@ try { if (!String(error).includes('Injected database close failure')) throw error; } -delete process.env.ROCKSDB_JS_CLOSE_FAILURE; -process.env.ROCKSDB_JS_CLOSE_RETRY_DELAY_MS = '1000'; const worker = new Worker(createWorkerBootstrapScript('./test/workers/shutdown-retry-worker.mts'), { eval: true, }); @@ -31,6 +29,5 @@ if (elapsed < 500) throw new Error(`Open did not wait for the shutdown retry (${ if (reopened.getSync('key') !== 'value') throw new Error('Shutdown retry did not preserve data'); const result = await shutdownResult; if (!result.shutdown) throw new Error(`Shutdown retry failed: ${JSON.stringify(result)}`); -delete process.env.ROCKSDB_JS_CLOSE_RETRY_DELAY_MS; reopened.destroy(); await worker.terminate(); From c6e43c13bd36e6ebab7791bc3e7258f3c9515f6f Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 11:27:00 -0600 Subject: [PATCH 26/49] Fix cross-thread destruction cleanup Co-Authored-By: GPT-5 Codex --- src/binding/binding.cpp | 18 +++++++++------ src/binding/core/test_seam.h | 31 +++++++++++++++----------- src/binding/database/db_descriptor.cpp | 2 +- src/binding/database/db_handle.cpp | 12 ++++++---- src/binding/database/db_registry.cpp | 26 +++++++++++---------- test/destroy.test.ts | 15 ++++++++++++- test/fixtures/fork-destroy-open.mts | 13 +++++++++++ 7 files changed, 79 insertions(+), 38 deletions(-) diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 19aecb435..eeae7372d 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -39,16 +39,19 @@ namespace rocksdb_js { * Shutdown function to ensure that we write in-memory data from all databases. */ napi_value Shutdown(napi_env env, napi_callback_info info) { + std::string error; try { DBRegistry::Shutdown(); - } catch (const std::exception& error) { - ::napi_throw_error(env, nullptr, error.what()); - return nullptr; + } catch (const std::exception& exception) { + error = exception.what(); } catch (...) { - ::napi_throw_error(env, nullptr, "Unknown native database shutdown failure"); - return nullptr; + error = "Unknown native database shutdown failure"; } GlobalEvents::Shutdown(); + if (!error.empty()) { + ::napi_throw_error(env, nullptr, error.c_str()); + return nullptr; + } napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; @@ -162,6 +165,7 @@ napi_value TransactionLogMapCount(napi_env env, napi_callback_info info) { static std::atomic moduleRefCount{0}; NAPI_MODULE_INIT() { + initializeTestSeams(); #ifdef DEBUG // disable buffering for stderr to ensure messages are written immediately ::setvbuf(stderr, nullptr, _IONBF, 0); @@ -233,9 +237,9 @@ NAPI_MODULE_INIT() { ::fprintf(stderr, "rocksdb-js %s cleanup failed: unknown native error\n", name); } }; - cleanup("global events", []() { rocksdb_js::GlobalEvents::Shutdown(); }); - cleanup("transaction logs", []() { rocksdb_js::TransactionLogStoreRegistry::Shutdown(); }); cleanup("database registry", []() { rocksdb_js::DBRegistry::Shutdown(); }); + cleanup("transaction logs", []() { rocksdb_js::TransactionLogStoreRegistry::Shutdown(); }); + cleanup("global events", []() { rocksdb_js::GlobalEvents::Shutdown(); }); DEBUG_LOG("Binding::Init env cleanup done\n"); } else if (newRefCount < 0) { DEBUG_LOG("Binding::Init WARNING: Module ref count went negative!\n"); diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index 4e0720c35..feaf200a1 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -3,6 +3,7 @@ #include #include +#include // Deterministic test seams that widen a race window are gated on a millisecond // delay read from an environment variable (0 = disabled). They are inert in @@ -16,19 +17,23 @@ inline int testDelayMs(const char* envName) { return value ? ::atoi(value) : 0; } -// Consume native fault flags in the same C runtime that reads them. JavaScript -// process.env deletion does not reliably update the MSVC runtime environment. -inline bool testConsumeFlag(const char* envName) { - const char* value = ::getenv(envName); - if (!value || ::atoi(value) <= 0) { - return false; - } -#ifdef _WIN32 - ::_putenv_s(envName, ""); -#else - ::unsetenv(envName); -#endif - return true; +// Snapshot native fault flags once; mutating process.env while native workers +// can read it is unsafe, and process.env deletion does not update MSVC's CRT. +inline std::atomic& closeFailureFlag() { + static std::atomic pending{false}; + return pending; +} + +inline void initializeTestSeams() { + static std::once_flag initialized; + std::call_once(initialized, []() { + const char* value = ::getenv("ROCKSDB_JS_CLOSE_FAILURE"); + closeFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); + }); +} + +inline bool testConsumeCloseFailure() { + return closeFailureFlag().exchange(false, std::memory_order_relaxed); } // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 40d3cdf7e..274ad7357 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -461,7 +461,7 @@ void DBDescriptor::finishClose() { } } - if (testConsumeFlag("ROCKSDB_JS_CLOSE_FAILURE")) { + if (testConsumeCloseFailure()) { throw rocksdb_js::DBException("Injected database close failure"); } if (!this->db) { diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index 60ab1ba29..13e520039 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -151,13 +151,17 @@ void DBHandle::close() { // DBRegistry::CloseTransactionsByEnv from the env cleanup hook // (HarperFast/rocksdb-js#741). - // release our reference to the descriptor - this->descriptor.reset(); + // A foreign thread can close a handle while its owner is copying this + // shared_ptr for an operation. Keep that member owner-thread-only; the + // descriptor itself is already closed before any foreign close returns. + if (std::this_thread::get_id() == this->ownerThreadId) { + this->descriptor.reset(); + } } // N-API references are environment-thread-affine. Destroying a shared - // descriptor can close this handle from another worker; retain the refs in - // that case so the owning environment's later close/finalizer releases them. + // descriptor can close this handle from another worker; retain the refs and + // descriptor until the owning environment's close or finalizer releases them. if (std::this_thread::get_id() == this->ownerThreadId) { for (auto& [name, ref] : this->logRefs) { DEBUG_LOG("%p DBHandle::close Releasing transaction log JS reference \"%s\"\n", this, name.c_str()); diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 56f0a0235..5a01aca83 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -89,15 +89,15 @@ std::string destroyPhysicalPath(const std::string& path) { return {}; } +void emitCloseFailure(const std::string& path, const std::string& error) { + if (!error.empty() && GlobalEvents::hasListeners()) { + emitGlobalEvent("database:closeFailed", ListenerData::fromStrings({path, error})); + } +} + void emitCloseFailures(const std::vector& descriptors) { - if (!GlobalEvents::hasListeners()) return; for (const auto& closing : descriptors) { - if (!closing.closeError.empty()) { - emitGlobalEvent( - "database:closeFailed", - ListenerData::fromStrings({closing.key.path, closing.closeError}) - ); - } + emitCloseFailure(closing.key.path, closing.closeError); } } @@ -228,9 +228,7 @@ CloseResult DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn if (condition) { condition->notify_all(); } - if (!closeError.empty() && GlobalEvents::hasListeners()) { - emitGlobalEvent("database:closeFailed", ListenerData::fromStrings({path, closeError})); - } + emitCloseFailure(path, closeError); return CloseResult{closeError, quarantined}; } @@ -385,8 +383,11 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); const std::string destroyError = destroyPhysicalPath(path); if (!destroyError.empty()) { - std::lock_guard lock(instance->databasesMutex); - instance->databases[DBKey{path, false}].closeError = destroyError; + { + std::lock_guard lock(instance->databasesMutex); + instance->databases[DBKey{path, false}].closeError = destroyError; + } + emitCloseFailure(path, destroyError); throw rocksdb_js::DBException(destroyError); } @@ -1097,6 +1098,7 @@ void DBRegistry::Shutdown() { } } } + emitCloseFailure(key.path, cleanupError); if (!cleanupError.empty() && destroyCleanupError.empty()) { destroyCleanupError = "Cannot complete shutdown: database \"" + key.path + diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 0677fa47b..17108f11d 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -135,9 +135,17 @@ describe('Destroy', () => { it.skipIf(process.platform === 'win32')( 'quarantines a path when post-destroy cleanup fails', () => - dbRunner(({ db, dbPath }) => { + dbRunner(async ({ db, dbPath }) => { const healthyPath = generateDBPath(); const healthy = RocksDatabase.open(healthyPath); + let resolveCloseFailure: (args: unknown[]) => void; + const closeFailure = new Promise((resolve) => { + resolveCloseFailure = resolve; + }); + const listener = (...args: unknown[]) => { + if (args[0] === dbPath) resolveCloseFailure(args); + }; + RocksDatabase.on('database:closeFailed', listener); healthy.putSync('key', 'value'); const lockedDirectory = join(dbPath, 'transaction_logs', 'locked'); mkdirSync(lockedDirectory, { recursive: true }); @@ -149,7 +157,12 @@ describe('Destroy', () => { registryStatus().find((entry) => entry.path === dbPath)?.destroyCleanupPending ).toBe(true); expect(() => RocksDatabase.open(dbPath)).toThrow('previous destroy cleanup failed'); + await expect(closeFailure).resolves.toMatchObject([ + dbPath, + expect.stringContaining('Failed to remove database directory'), + ]); } finally { + RocksDatabase.off('database:closeFailed', listener); chmodSync(lockedDirectory, 0o700); } shutdown(); diff --git a/test/fixtures/fork-destroy-open.mts b/test/fixtures/fork-destroy-open.mts index ee1145cfc..7e6e55574 100644 --- a/test/fixtures/fork-destroy-open.mts +++ b/test/fixtures/fork-destroy-open.mts @@ -30,6 +30,17 @@ if (!destroying.destroying) const registryDeadline = Date.now() + 5_000; while (registryStatus().some((entry) => entry.path === path)) { if (Date.now() >= registryDeadline) throw new Error('Timed out waiting for the destroy window'); + try { + const value = original.getSync('before-destroy'); + if (value !== 'present') throw new Error(`Destroy raced a read with ${String(value)}`); + } catch (error) { + if ( + !String(error).includes('Database not open') && + !String(error).includes('Database is closing') + ) { + throw error; + } + } await delay(1); } @@ -43,6 +54,7 @@ if (process.env.ROCKSDB_JS_TEST_SHUTDOWN_DURING_DESTROY === '1') { if (shutdownDuration < 500) throw new Error(`Shutdown did not wait for destroy (${shutdownDuration}ms)`); await worker.terminate(); + original.close(); process.exit(0); } const reopened = RocksDatabase.open(path); @@ -57,3 +69,4 @@ if (reopened.getSync('after-destroy') !== 'present') throw new Error('Reopened database is not usable'); reopened.close(); await worker.terminate(); +original.close(); From 5f95a54af9eda59d1e1360ecbb5278979d4a4666 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 11:42:44 -0600 Subject: [PATCH 27/49] Address lifecycle review feedback Co-Authored-By: GPT-5 Codex --- README.md | 8 +++++++- src/binding/binding.cpp | 2 +- src/binding/database/db_registry.cpp | 6 +++++- test/destroy.test.ts | 8 ++++---- test/fixtures/fork-close-failure.mts | 1 - test/fixtures/fork-destroy-failure.mts | 1 - test/fixtures/fork-gc-close-failure.mts | 1 - test/fixtures/fork-shutdown-failure.mts | 1 - 8 files changed, 17 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 943a91695..3e9fe11b7 100644 --- a/README.md +++ b/README.md @@ -2002,7 +2002,13 @@ any descriptor whose teardown did not complete: ```typescript import { shutdown } from '@harperfast/rocksdb-js'; -process.on('exit', shutdown); +process.on('exit', () => { + try { + shutdown(); + } catch (error) { + console.error('rocksdb-js shutdown failed', error); + } +}); ``` ### `versions: { 'rocksdb': string; 'rocksdb-js': string }` diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index eeae7372d..9dbc365c1 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -47,11 +47,11 @@ napi_value Shutdown(napi_env env, napi_callback_info info) { } catch (...) { error = "Unknown native database shutdown failure"; } - GlobalEvents::Shutdown(); if (!error.empty()) { ::napi_throw_error(env, nullptr, error.c_str()); return nullptr; } + GlobalEvents::Shutdown(); napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 5a01aca83..5c8eda7b3 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -361,18 +361,22 @@ void DBRegistry::DestroyDB(const std::string& path) { if (closeError) std::rethrow_exception(closeError); } + std::vector> conditions; { std::lock_guard lock(instance->databasesMutex); for (auto it = instance->databases.begin(); it != instance->databases.end();) { if (it->first.path == path) { + conditions.push_back(it->second.condition); it = instance->databases.erase(it); } else { ++it; } } } + for (const auto& condition : conditions) { + condition->notify_all(); + } - // All in-process descriptors are closed; physical destruction can proceed. const int destroyDelayMs = testDelayMs("ROCKSDB_JS_DESTROY_DELAY_MS"); if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 17108f11d..b9f9d7bff 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase, registryStatus, shutdown } from '../src/index.ts'; +import { RocksDatabase, registryStatus } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; @@ -132,7 +132,7 @@ describe('Destroy', () => { } )); - it.skipIf(process.platform === 'win32')( + it.skipIf(process.platform === 'win32' || (process.getuid?.() ?? 0) === 0)( 'quarantines a path when post-destroy cleanup fails', () => dbRunner(async ({ db, dbPath }) => { @@ -163,9 +163,9 @@ describe('Destroy', () => { ]); } finally { RocksDatabase.off('database:closeFailed', listener); - chmodSync(lockedDirectory, 0o700); + if (existsSync(lockedDirectory)) chmodSync(lockedDirectory, 0o700); } - shutdown(); + db.destroy(); expect(registryStatus().some((entry) => entry.path === dbPath)).toBe(false); const healthyReopened = RocksDatabase.open(healthyPath); expect(healthyReopened.getSync('key')).toBe('value'); diff --git a/test/fixtures/fork-close-failure.mts b/test/fixtures/fork-close-failure.mts index 1a830754d..a4eb74236 100644 --- a/test/fixtures/fork-close-failure.mts +++ b/test/fixtures/fork-close-failure.mts @@ -26,7 +26,6 @@ try { if (Date.now() - startedAt >= 1_000) throw new Error('Open waited instead of reporting the failed automatic close immediately'); -delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); if (registryStatus().some((entry) => entry.path === path)) throw new Error('Shutdown retry did not clear the quarantined automatic close'); diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index a090d2dcc..73846ce12 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -39,7 +39,6 @@ if (closeFailure) { } if (Date.now() - startedAt >= 1_000) throw new Error('Opening a quarantined descriptor waited instead of failing immediately'); - delete process.env.ROCKSDB_JS_CLOSE_FAILURE; db.destroy(); process.exit(0); } diff --git a/test/fixtures/fork-gc-close-failure.mts b/test/fixtures/fork-gc-close-failure.mts index cbc73eb36..6c1e9a759 100644 --- a/test/fixtures/fork-gc-close-failure.mts +++ b/test/fixtures/fork-gc-close-failure.mts @@ -33,7 +33,6 @@ if ( throw new Error('Automatic close failure was not quarantined'); } -delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); const reopened = RocksDatabase.open(path); if (reopened.getSync('key') !== 'value') diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts index a834c5671..592532b42 100644 --- a/test/fixtures/fork-shutdown-failure.mts +++ b/test/fixtures/fork-shutdown-failure.mts @@ -33,7 +33,6 @@ try { if (Date.now() - startedAt >= 1_000) throw new Error('Cross-mode open waited instead of reporting the quarantined path immediately'); -delete process.env.ROCKSDB_JS_CLOSE_FAILURE; shutdown(); if (registryStatus().some((entry) => entry.path === path)) throw new Error('Shutdown retry did not clear the quarantined descriptor'); From 8f29c72d7293d4672f4b4803b3804b422d792f04 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 23:09:43 -0600 Subject: [PATCH 28/49] fix(lifecycle): close teardown review gaps Co-Authored-By: GPT-5 Codex --- AGENTS.md | 8 +++++- README.md | 10 +++++--- src/binding/database/database.cpp | 33 ++++++++++++++++++++++--- src/binding/database/database.h | 6 ++++- src/binding/database/db_handle.cpp | 5 +--- src/binding/database/db_registry.cpp | 32 ++++++------------------ src/database.ts | 2 +- src/load-binding.ts | 1 + src/store.ts | 4 +-- test/destroy.test.ts | 8 +++++- test/fixtures/fork-shutdown-failure.mts | 1 + 11 files changed, 67 insertions(+), 43 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a559e9e3f..dc4677c1a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -325,7 +325,13 @@ sufficient (env teardown does not honor tsfn acquire counts); see their own `shared_ptr` for the duration of a copy (backup, backup stream, checkpoint) make a racing close skip the purge (`use_count > 1`), so their state destructors re-run `PurgeIfUnreferenced` after releasing the ref — without that retry the skipped purge is permanent - and the entry (plus the open RocksDB) leaks (HarperFast/rocksdb-js#672). + and the entry (plus the open RocksDB) leaks (HarperFast/rocksdb-js#672). Once `beginClose()` wins, + `DBHandle::opened()` must report false even while the native DB still exists. Any synchronous N-API + path that dereferences `descriptor->db` or the handle's column family must take an `OperationGuard` + immediately after `UNWRAP_DB_HANDLE_AND_OPEN()`; `finishClose()` can reset the column-family pointer + from another env after the in-flight count drains. A failed physical destroy leaves a registry + tombstone, but `shutdown()` is deliberately non-destructive: it reports the tombstone and only an + explicit `destroy()` retries path deletion. 7. **One writable BackupEngine per backup directory (kernel advisory lock)**: each backup op opens its own short-lived `rocksdb::BackupEngine`/`BackupEngineReadOnly` (`src/binding/database/backup.cpp`), and RocksDB only serializes work _within_ a single engine — it has no cross-engine lock on the directory. diff --git a/README.md b/README.md index 3e9fe11b7..069c96e6a 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Closes a database. This function can be called multiple times and will only clos database. A database instance can be reopened once it is closed. A flush or compaction failure is reported as an exception after native teardown completes. All native close errors emit `database:closeFailed`; when teardown does not complete, same-path opens also fail until -`destroy()` or `shutdown()` retries cleanup. The quarantine applies to both writable and read-only +`destroy()` retries cleanup. The quarantine applies to both writable and read-only opens because both modes share the physical path lifecycle. ```typescript @@ -364,7 +364,8 @@ because releasing the native database beneath an active copy would be unsafe. A previously opened instance does not need to remain open, which allows an explicit `destroy()` retry after failed physical cleanup. A never-opened or read-only instance cannot destroy the -database; `shutdown()` can recover a tombstone when its original handle is no longer available. +database. `shutdown()` reports a pending cleanup tombstone but never retries deletion; only an +explicit `destroy()` can remove the path. ```typescript db.destroy(); @@ -1978,7 +1979,7 @@ Returns an array containing that status of all active RocksDB instances. - `path: string` The database path. - `closeError?: string` The native lifecycle error retaining this registry entry. - `destroyCleanupPending?: boolean` The native database is closed, but physical path cleanup must - finish before the next open. Call `destroy()` or `shutdown()` to retry cleanup. + finish before the next open. Call `destroy()` to retry cleanup. - `refCount: number` The number of JavaScript database instances plus the registry's reference. - `columnFamiles: object` A map of column family names and their their info. - `userSharedBuffers: number` The count of active user shared buffers. @@ -1998,7 +1999,8 @@ The `shutdown()` will flush all in-memory data to disk and wait for any outstand finish, for all open databases. It is highly recommended to call this in a `process` `exit` event listener (on the main thread), to ensure that all data is flushed to disk before the process exits. It throws the first close failure after attempting every claimed database; call it again to retry -any descriptor whose teardown did not complete: +any descriptor whose native teardown did not complete. It reports pending destroy-cleanup +tombstones without deleting their paths; retry those with an explicit `destroy()`: ```typescript import { shutdown } from '@harperfast/rocksdb-js'; diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 43bf06221..d7c5979cc 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -389,6 +389,7 @@ napi_value Database::Compact(napi_env env, napi_callback_info info) { napi_value Database::CompactSync(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(3); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice startSlice; rocksdb::Slice* startPtr = nullptr; @@ -508,6 +509,7 @@ static bool isColumnFamilyAlreadyDropped(const rocksdb::Status& status) { napi_value Database::Drop(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); if ((*dbHandle)->getColumnFamilyName() == "default") { return doClear(env, info, "Drop failed"); @@ -568,12 +570,12 @@ napi_value Database::Drop(napi_env env, napi_callback_info info) { napi_value Database::DropSync(napi_env env, napi_callback_info info) { NAPI_METHOD(); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); if ((*dbHandle)->getColumnFamilyName() == "default") { return doClearSync(env, info, "Drop failed"); } - ACQUIRE_OPERATIONS_LOCK(); DEBUG_LOG("%p Database::DropSync dropping database: %s\n", dbHandle->get(), (*dbHandle)->path.c_str()); rocksdb::Status status = (*dbHandle)->descriptor->db->DropColumnFamily((*dbHandle)->getColumnFamilyHandle()); if (!status.ok() && !isColumnFamilyAlreadyDropped(status)) { @@ -1001,6 +1003,7 @@ napi_value Database::Resume(napi_env env, napi_callback_info info) { napi_value Database::GetCompression(napi_env env, napi_callback_info info) { NAPI_METHOD(); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); rocksdb::Options opts = (*dbHandle)->descriptor->db->GetOptions((*dbHandle)->getColumnFamilyHandle()); std::string name = compressionNameFromType(opts.compression); @@ -1075,6 +1078,7 @@ napi_value Database::GetLogOptions(napi_env env, napi_callback_info info) { napi_value Database::GetCount(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); DBIteratorOptions itOptions; itOptions.initFromNapiObject(env, argv[0]); @@ -1332,6 +1336,7 @@ napi_value Database::GetMonotonicTimestamp(napi_env env, napi_callback_info info napi_value Database::GetOldestSnapshotTimestamp(napi_env env, napi_callback_info info) { NAPI_METHOD(); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); uint64_t timestamp = 0; bool success = (*dbHandle)->descriptor->db->GetIntProperty( @@ -1362,6 +1367,7 @@ napi_value Database::GetOldestSnapshotTimestamp(napi_env env, napi_callback_info napi_value Database::GetDBProperty(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(1); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); NAPI_GET_STRING(argv[0], propertyName, "Property name is required"); @@ -1398,6 +1404,7 @@ napi_value Database::GetDBProperty(napi_env env, napi_callback_info info) { napi_value Database::GetDBIntProperty(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(1); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); NAPI_GET_STRING(argv[0], propertyName, "Property name is required"); @@ -1428,6 +1435,7 @@ napi_value Database::GetDBIntProperty(napi_env env, napi_callback_info info) { napi_value Database::GetStat(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(1); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); NAPI_GET_STRING(argv[0], statName, "Stat name is required"); return (*dbHandle)->getStat(env, statName); } @@ -1444,6 +1452,7 @@ napi_value Database::GetStat(napi_env env, napi_callback_info info) { napi_value Database::GetStats(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(1); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); bool all = false; NAPI_STATUS_THROWS(::napi_get_value_bool(env, argv[0], &all)); @@ -1793,6 +1802,7 @@ napi_value Database::GetUserSharedBuffer(napi_env env, napi_callback_info info) NAPI_METHOD_ARGV(3); NAPI_GET_BUFFER(argv[0], key, "Key is required"); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); std::string keyStr(key + keyStart, keyEnd - keyStart); // if we have a callback, add it as a listener @@ -1839,6 +1849,20 @@ napi_value Database::HasLock(napi_env env, napi_callback_info info) { return result; } +/** + * Checks if the RocksDB database is closing or quarantined. + */ +napi_value Database::IsClosing(napi_env env, napi_callback_info info) { + NAPI_METHOD(); + UNWRAP_DB_HANDLE(); + + const bool closing = dbHandle != nullptr && *dbHandle && (*dbHandle)->descriptor && + (*dbHandle)->descriptor->isClosing(); + napi_value result; + NAPI_STATUS_THROWS(::napi_get_boolean(env, closing, &result)); + return result; +} + /** * Checks if the RocksDB database is open. */ @@ -2105,10 +2129,10 @@ napi_value Database::PurgeLogs(napi_env env, napi_callback_info info) { */ napi_value Database::PutSync(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(3); - NAPI_GET_BUFFER(argv[0], key, "Key is required"); - NAPI_GET_BUFFER(argv[1], value, nullptr); UNWRAP_DB_HANDLE_AND_OPEN(); ACQUIRE_OPERATIONS_LOCK(); + NAPI_GET_BUFFER(argv[0], key, "Key is required"); + NAPI_GET_BUFFER(argv[1], value, nullptr); // THROW_IF_READONLY((*dbHandle)->descriptor, "Put failed: "); rocksdb::Status status; @@ -2191,9 +2215,9 @@ napi_value Database::PutSync(napi_env env, napi_callback_info info) { */ napi_value Database::RemoveSync(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); - NAPI_GET_BUFFER(argv[0], key, "Key is required"); UNWRAP_DB_HANDLE_AND_OPEN(); ACQUIRE_OPERATIONS_LOCK(); + NAPI_GET_BUFFER(argv[0], key, "Key is required"); // THROW_IF_READONLY((*dbHandle)->descriptor, "Remove failed: "); rocksdb::Status status; @@ -2407,6 +2431,7 @@ void Database::Init(napi_env env, napi_value exports) { { "getSync", nullptr, GetSync, nullptr, nullptr, nullptr, napi_default, nullptr }, { "getUserSharedBuffer", nullptr, GetUserSharedBuffer, nullptr, nullptr, nullptr, napi_default, nullptr }, { "hasLock", nullptr, HasLock, nullptr, nullptr, nullptr, napi_default, nullptr }, + { "closing", nullptr, nullptr, IsClosing, nullptr, nullptr, napi_default, nullptr }, { "listeners", nullptr, Listeners, nullptr, nullptr, nullptr, napi_default, nullptr }, { "listLogs", nullptr, ListLogs, nullptr, nullptr, nullptr, napi_default, nullptr }, { "notify", nullptr, Notify, nullptr, nullptr, nullptr, napi_default, nullptr }, diff --git a/src/binding/database/database.h b/src/binding/database/database.h index 7ff785fd8..11ee429c2 100644 --- a/src/binding/database/database.h +++ b/src/binding/database/database.h @@ -227,7 +227,10 @@ inline void vtPopulateIfSettled( UNWRAP_DB_HANDLE(); \ do { \ if (dbHandle == nullptr || !(*dbHandle)->opened()) { \ - ::napi_throw_error(env, nullptr, "Database not open"); \ + const char* message = dbHandle != nullptr && *dbHandle && (*dbHandle)->descriptor && (*dbHandle)->descriptor->isClosing() \ + ? "Database is closing" \ + : "Database not open"; \ + ::napi_throw_error(env, nullptr, message); \ NAPI_RETURN_UNDEFINED(); \ } \ } while (0) @@ -327,6 +330,7 @@ struct Database final { static napi_value GetSync(napi_env env, napi_callback_info info); static napi_value GetUserSharedBuffer(napi_env env, napi_callback_info info); static napi_value HasLock(napi_env env, napi_callback_info info); + static napi_value IsClosing(napi_env env, napi_callback_info info); static napi_value IsOpen(napi_env env, napi_callback_info info); static napi_value Listeners(napi_env env, napi_callback_info info); static napi_value ListLogs(napi_env env, napi_callback_info info); diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index 13e520039..a699e4e65 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -362,10 +362,7 @@ void DBHandle::open(const std::string& path, const DBOptions& options) { * Checks if the referenced database is opened. */ bool DBHandle::opened() const { - if (this->descriptor && this->descriptor->db) { - return true; - } - return false; + return this->descriptor && !this->descriptor->isClosing(); } /** diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 5c8eda7b3..5e579c015 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -1014,7 +1014,7 @@ void DBRegistry::Shutdown() { while (true) { std::vector descriptorsToClose; std::vector descriptorsToWaitFor; - std::vector destroyCleanupEntries; + std::string destroyCleanupError; bool destroysInFlight; { std::unique_lock lock(instance->databasesMutex); @@ -1022,14 +1022,17 @@ void DBRegistry::Shutdown() { DEBUG_LOG("%p DBRegistry::Shutdown Shutting down %zu databases\n", instance.get(), instance->databases.size()); descriptorsToClose.reserve(instance->databases.size()); descriptorsToWaitFor.reserve(instance->databases.size()); - destroyCleanupEntries.reserve(instance->databases.size()); for (auto& [key, entry] : instance->databases) { if (instance->destroyingPaths.find(key.path) != instance->destroyingPaths.end()) { continue; } if (!entry.descriptor) { - if (!entry.closeError.empty()) destroyCleanupEntries.push_back(key); + if (!entry.closeError.empty() && destroyCleanupError.empty()) { + destroyCleanupError = + "Cannot complete shutdown: database \"" + key.path + + "\" requires explicit destroy() cleanup: " + entry.closeError; + } continue; } ClosingDescriptor closing{key, entry.descriptor, entry.condition}; @@ -1088,27 +1091,6 @@ void DBRegistry::Shutdown() { } } - std::string destroyCleanupError; - for (const auto& key : destroyCleanupEntries) { - const std::string cleanupError = destroyPhysicalPath(key.path); - { - std::lock_guard lock(instance->databasesMutex); - auto entry = instance->databases.find(key); - if (entry != instance->databases.end() && !entry->second.descriptor) { - if (cleanupError.empty()) { - instance->databases.erase(entry); - } else { - entry->second.closeError = cleanupError; - } - } - } - emitCloseFailure(key.path, cleanupError); - if (!cleanupError.empty() && destroyCleanupError.empty()) { - destroyCleanupError = - "Cannot complete shutdown: database \"" + key.path + - "\" requires destroy cleanup: " + cleanupError; - } - } if (destroysInFlight) { std::unique_lock lock(instance->databasesMutex); if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { @@ -1127,7 +1109,7 @@ void DBRegistry::Shutdown() { throw rocksdb_js::DBException(destroyCleanupError); } if (descriptorsToClose.empty() && descriptorsToWaitFor.empty() && - destroyCleanupEntries.empty() + destroyCleanupError.empty() ) break; } diff --git a/src/database.ts b/src/database.ts index e5c255357..ab5fa9098 100644 --- a/src/database.ts +++ b/src/database.ts @@ -350,7 +350,7 @@ export class RocksDatabase extends DBI { * matching event emitted in this process. Native lifecycle failures use * `'database:closeFailed'` with `(path, error)` string arguments. The event * reports both completed and incomplete teardowns; only an incomplete - * teardown quarantines the path until `destroy()` or `shutdown()` retries. + * teardown quarantines the path until `destroy()` retries. * * @example * ```typescript diff --git a/src/load-binding.ts b/src/load-binding.ts index 4c05dcb95..210a5f961 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -469,6 +469,7 @@ export type NativeDatabase = { hasLock(key: BufferWithDataView): boolean; listeners(event: string | BufferWithDataView): number; listLogs(): string[]; + closing: boolean; opened: boolean; open(path: string, options?: NativeDatabaseOptions): void; populateVersion(keyLengthOrKeyBuffer: number | Buffer, version: number): void; diff --git a/src/store.ts b/src/store.ts index eeb6be55a..1a75613f1 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1172,7 +1172,7 @@ export class Store { putSync(context: StoreContext, key: Key, value: any, options?: StorePutOptions): void { if (!this.db.opened) { - throw new Error('Database not open'); + throw new Error(this.db.closing ? 'Database is closing' : 'Database not open'); } // IMPORTANT! @@ -1187,7 +1187,7 @@ export class Store { removeSync(context: StoreContext, key: Key, options?: StoreRemoveOptions): void { if (!this.db.opened) { - throw new Error('Database not open'); + throw new Error(this.db.closing ? 'Database is closing' : 'Database not open'); } context.removeSync(this.encodeKey(key), this.getTxnId(options)); diff --git a/test/destroy.test.ts b/test/destroy.test.ts index b9f9d7bff..e02197916 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,4 +1,4 @@ -import { RocksDatabase, registryStatus } from '../src/index.ts'; +import { RocksDatabase, registryStatus, shutdown } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; @@ -161,6 +161,12 @@ describe('Destroy', () => { dbPath, expect.stringContaining('Failed to remove database directory'), ]); + chmodSync(lockedDirectory, 0o700); + expect(() => shutdown()).toThrow('requires explicit destroy() cleanup'); + expect(existsSync(dbPath)).toBe(true); + expect( + registryStatus().find((entry) => entry.path === dbPath)?.destroyCleanupPending + ).toBe(true); } finally { RocksDatabase.off('database:closeFailed', listener); if (existsSync(lockedDirectory)) chmodSync(lockedDirectory, 0o700); diff --git a/test/fixtures/fork-shutdown-failure.mts b/test/fixtures/fork-shutdown-failure.mts index 592532b42..ef7778f12 100644 --- a/test/fixtures/fork-shutdown-failure.mts +++ b/test/fixtures/fork-shutdown-failure.mts @@ -22,6 +22,7 @@ const args = await Promise.race([ if (args[0] !== path || args[1] !== 'Injected database close failure') { throw new Error(`Unexpected database:closeFailed arguments: ${JSON.stringify(args)}`); } +if (db.isOpen()) throw new Error('A quarantined database still reports itself open'); const startedAt = Date.now(); try { RocksDatabase.open(path, { readOnly: true }); From f4b155003d54300c7b858677628bbe5001798b8c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 23:35:42 -0600 Subject: [PATCH 29/49] fix(lifecycle): quarantine unsafe close failures Co-Authored-By: GPT-5 Codex --- AGENTS.md | 5 ++++- README.md | 10 +++++----- src/binding/core/test_seam.h | 11 +++++++++++ src/binding/database/database.cpp | 4 +++- src/binding/database/db_descriptor.cpp | 9 +++++++-- src/binding/database/db_descriptor.h | 2 +- src/binding/database/db_handle.cpp | 1 + src/binding/database/db_handle.h | 2 ++ src/binding/database/db_registry.cpp | 6 ++++-- src/database.ts | 4 ++-- test/destroy.test.ts | 7 +++++++ test/drop.test.ts | 2 ++ test/fixtures/fork-flush-failure.mts | 27 ++++++++++++++++++++++++++ 13 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 test/fixtures/fork-flush-failure.mts diff --git a/AGENTS.md b/AGENTS.md index dc4677c1a..f4ed28bf9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -329,7 +329,10 @@ sufficient (env teardown does not honor tsfn acquire counts); see `DBHandle::opened()` must report false even while the native DB still exists. Any synchronous N-API path that dereferences `descriptor->db` or the handle's column family must take an `OperationGuard` immediately after `UNWRAP_DB_HANDLE_AND_OPEN()`; `finishClose()` can reset the column-family pointer - from another env after the in-flight count drains. A failed physical destroy leaves a registry + from another env after the in-flight count drains. `DBHandle::close()` itself is cross-env and must + serialize mutation of its `shared_ptr` members. A close-time flush failure keeps the native DB + quarantined so `shutdown()` can retry without losing `disableWAL` writes; an explicit destroy may + force teardown because the caller requested deletion. A failed physical destroy leaves a registry tombstone, but `shutdown()` is deliberately non-destructive: it reports the tombstone and only an explicit `destroy()` retries path deletion. 7. **One writable BackupEngine per backup directory (kernel advisory lock)**: each backup op opens its diff --git a/README.md b/README.md index 069c96e6a..45a490368 100644 --- a/README.md +++ b/README.md @@ -141,11 +141,11 @@ Creates a new database instance. ### `db.close()` Closes a database. This function can be called multiple times and will only close an opened -database. A database instance can be reopened once it is closed. A flush or compaction failure is -reported as an exception after native teardown completes. All native close errors emit -`database:closeFailed`; when teardown does not complete, same-path opens also fail until -`destroy()` retries cleanup. The quarantine applies to both writable and read-only -opens because both modes share the physical path lifecycle. +database. A database instance can be reopened once it is closed. A flush failure leaves the native +database quarantined so `shutdown()` can retry without losing unflushed data; an explicit +`destroy()` can instead delete it. A compaction failure is reported after native teardown +completes. All native close errors emit `database:closeFailed`. The quarantine applies to both +writable and read-only opens because both modes share the physical path lifecycle. ```typescript const db = RocksDatabase.open('foo'); diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index feaf200a1..effdcd7ee 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -24,11 +24,18 @@ inline std::atomic& closeFailureFlag() { return pending; } +inline std::atomic& closeFlushFailureFlag() { + static std::atomic pending{false}; + return pending; +} + inline void initializeTestSeams() { static std::once_flag initialized; std::call_once(initialized, []() { const char* value = ::getenv("ROCKSDB_JS_CLOSE_FAILURE"); closeFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); + value = ::getenv("ROCKSDB_JS_CLOSE_FLUSH_FAILURE"); + closeFlushFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); }); } @@ -36,6 +43,10 @@ inline bool testConsumeCloseFailure() { return closeFailureFlag().exchange(false, std::memory_order_relaxed); } +inline bool testConsumeCloseFlushFailure() { + return closeFlushFailureFlag().exchange(false, std::memory_order_relaxed); +} + // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N // transaction commits to fail with TryAgain (the caller rolls back so no data is committed), // reproducing an ERR_TRY_AGAIN that a real memtable flush would cause but that is finicky to diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index d7c5979cc..5e9e0f149 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -194,7 +194,7 @@ napi_value Database::Close(napi_env env, napi_callback_info info) { if (!closeResult.error.empty()) { std::string message = closeResult.error; if (closeResult.quarantined) { - message += ". Call destroy() or shutdown() to retry cleanup"; + message += ". Call shutdown() to retry close, or destroy() to delete the database"; } ::napi_throw_error(env, nullptr, message.c_str()); return nullptr; @@ -1663,6 +1663,7 @@ napi_value Database::GetSync(napi_env env, napi_callback_info info) { napi_value Database::VerifyVersion(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice keySlice; if (!rocksdb_js::getSliceFromArg(env, argv[0], keySlice, (*dbHandle)->defaultKeyBufferPtr, "Key must be a buffer")) { @@ -1699,6 +1700,7 @@ napi_value Database::VerifyVersion(napi_env env, napi_callback_info info) { napi_value Database::PopulateVersion(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice keySlice; if (!rocksdb_js::getSliceFromArg(env, argv[0], keySlice, (*dbHandle)->defaultKeyBufferPtr, "Key must be a buffer")) { diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 274ad7357..0a4305563 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -416,7 +416,7 @@ void DBDescriptor::close() { this->finishClose(); } -void DBDescriptor::finishClose() { +void DBDescriptor::finishClose(bool destroying) { DEBUG_LOG("%p DBDescriptor::close Closing \"%s\" (mode=%s read-only=%s closables=%zu columns=%zu transactions=%zu)\n", this, this->path.c_str(), this->mode == DBMode::Optimistic ? "optimistic" : "pessimistic", this->readOnly ? "true" : "false", this->closables.size(), this->columns.size(), this->transactions.size()); @@ -471,9 +471,14 @@ void DBDescriptor::finishClose() { // We want to ensure that all in-memory data is written to disk. Keep the waiting default: an // immediate flush races transaction-log-store teardown (AGENTS invariant 15). std::string closeError; - rocksdb::Status status = this->flush(); + rocksdb::Status status = testConsumeCloseFlushFailure() + ? rocksdb::Status::IOError("Injected database close flush failure") + : this->flush(); if (!status.ok()) { closeError = "Failed to flush database during close: " + status.ToString(); + if (!destroying) { + throw rocksdb_js::DBException(closeError); + } } // Trigger manual compaction on all column families to reclaim space from diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index 792ea0c60..ca2afc6e2 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -482,7 +482,7 @@ struct DBDescriptor final : public std::enable_shared_from_this { * Only valid after `beginClose()` returned true; `close()` is the all-in-one * entry point that claims and then runs this. */ - void finishClose(); + void finishClose(bool destroying = false); void attach(std::shared_ptr closable); void detach(std::shared_ptr closable); diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index a699e4e65..88a61816f 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -126,6 +126,7 @@ rocksdb::Status DBHandle::clear() { * Closes the DBHandle. */ void DBHandle::close() { + std::lock_guard closeLock(this->closeMutex); DEBUG_LOG("%p DBHandle::close dbDescriptor=%p (ref count = %ld)\n", this, this->descriptor.get(), this->descriptor.use_count()); // cancel all active async work before closing diff --git a/src/binding/database/db_handle.h b/src/binding/database/db_handle.h index 625f51e77..60ffbf067 100644 --- a/src/binding/database/db_handle.h +++ b/src/binding/database/db_handle.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "rocksdb/db.h" #include "database/db_descriptor.h" @@ -72,6 +73,7 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro * get the `TransactionLog` class. */ napi_ref exportsRef; + std::mutex closeMutex; /** * The default transaction log store. diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 5e579c015..bc13de015 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -307,7 +307,7 @@ void DBRegistry::DestroyDB(const std::string& path) { DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); try { - closing.descriptor->finishClose(); + closing.descriptor->finishClose(true); closing.closed = true; } catch (const std::exception& error) { closing.closed = closing.descriptor->isClosed(); @@ -469,7 +469,9 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons "Cannot open database \"" + path + "\": previous " + (destroyCleanupFailed ? "destroy cleanup" : "close") + " failed: " + registeredEntry->second.closeError + - (destroyCleanupFailed ? ". Call destroy() to retry cleanup" : ". Call destroy() or shutdown() to retry cleanup") + (destroyCleanupFailed + ? ". Call destroy() to retry cleanup" + : ". Call shutdown() to retry close, or destroy() to delete the database") ); } } diff --git a/src/database.ts b/src/database.ts index ab5fa9098..7eef42c64 100644 --- a/src/database.ts +++ b/src/database.ts @@ -349,8 +349,8 @@ export class RocksDatabase extends DBI { * Listeners are not tied to any specific database — they fire for every * matching event emitted in this process. Native lifecycle failures use * `'database:closeFailed'` with `(path, error)` string arguments. The event - * reports both completed and incomplete teardowns; only an incomplete - * teardown quarantines the path until `destroy()` retries. + * reports both completed and incomplete teardowns; an incomplete close can + * be retried by `shutdown()` or explicitly deleted by `destroy()`. * * @example * ```typescript diff --git a/test/destroy.test.ts b/test/destroy.test.ts index e02197916..c1211ae08 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -11,6 +11,7 @@ const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts' const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure.mts'); const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failure.mts'); const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mts'); +const flushFailureFixture = join(__dirname, 'fixtures', 'fork-flush-failure.mts'); const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); const nodeExecutable = process.env.NODE_BINARY ?? @@ -224,6 +225,12 @@ describe('Destroy', () => { }); }, 15_000); + it('quarantines a flush failure until shutdown preserves the unflushed data', async () => { + await runDestroyFixture(flushFailureFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FLUSH_FAILURE: '1', + }); + }, 15_000); + it('waits for an in-progress shutdown retry before reopening', async () => { await runDestroyFixture(shutdownRetryFixture, generateDBPath(), { ROCKSDB_JS_CLOSE_FAILURE: '1', diff --git a/test/drop.test.ts b/test/drop.test.ts index d88632a47..c19987d16 100644 --- a/test/drop.test.ts +++ b/test/drop.test.ts @@ -240,6 +240,8 @@ describe('Drop', () => { stale.close(); doomed.close(); expect(() => victim.close()).toThrow('Failed to flush database during close'); + expect(() => RocksDatabase.open(dbPath)).toThrow('previous close failed'); + victim.destroy(); const reopened = RocksDatabase.open(dbPath); reopened.close(); } diff --git a/test/fixtures/fork-flush-failure.mts b/test/fixtures/fork-flush-failure.mts new file mode 100644 index 000000000..7c1dca4de --- /dev/null +++ b/test/fixtures/fork-flush-failure.mts @@ -0,0 +1,27 @@ +import { RocksDatabase, registryStatus, shutdown } from '../../src/index.ts'; + +const path = process.argv[2]; +const db = RocksDatabase.open(path, { disableWAL: true }); +db.putSync('key', 'unflushed'); + +try { + db.close(); + throw new Error('Expected close to surface the injected flush failure'); +} catch (error) { + if (!String(error).includes('Injected database close flush failure')) throw error; +} +if (db.isOpen()) throw new Error('A flush-failed database still reports itself open'); +if (!registryStatus().some((entry) => entry.path === path && entry.closeError)) + throw new Error('Flush failure did not quarantine the descriptor'); +try { + RocksDatabase.open(path); + throw new Error('Expected the flush failure to block reopen'); +} catch (error) { + if (!String(error).includes('previous close failed')) throw error; +} + +shutdown(); +const reopened = RocksDatabase.open(path); +if (reopened.getSync('key') !== 'unflushed') + throw new Error('Shutdown retry did not preserve the unflushed write'); +reopened.destroy(); From e7909e52cc0d00dae0bd98d0024d67f27b728ee0 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 17 Aug 2026 01:09:36 -0600 Subject: [PATCH 30/49] Fix worker teardown ordering in CI gates Co-Authored-By: GPT-5 Codex --- benchmark/setup.ts | 22 ++++++++----------- stress-test/db-instances.stress.test.ts | 14 ++++++++---- .../workers/stress-db-instances-worker.mts | 1 + .../workers/stress-transaction-put-worker.mts | 4 ++++ 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/benchmark/setup.ts b/benchmark/setup.ts index 89c93e79e..820c39f46 100644 --- a/benchmark/setup.ts +++ b/benchmark/setup.ts @@ -381,6 +381,7 @@ export function workerBenchmark(type: string, options: any): void { } const workerState: WorkerState[] = []; + let dbPath: string; const workerPayload = { suites: workerCurrentSuites.map((suite) => suite.name), benchmark: benchmarkName, @@ -408,11 +409,7 @@ export function workerBenchmark(type: string, options: any): void { if (mode === 'run') { return; } - const path = join( - 'benchmark', - 'data', - `rocksdb-benchmark-${randomBytes(8).toString('hex')}` - ); + dbPath = join('benchmark', 'data', `rocksdb-benchmark-${randomBytes(8).toString('hex')}`); let teardownTimeoutId: NodeJS.Timeout; await Promise.race([ @@ -436,7 +433,7 @@ export function workerBenchmark(type: string, options: any): void { benchmarkFile: pathToFileURL(benchmarkFile).toString(), benchmarkWorkerId: i + 1, mode, - path, + path: dbPath, }); // important! these promises need to be referenced as // properties of `state` because they are reset by reference @@ -492,6 +489,11 @@ export function workerBenchmark(type: string, options: any): void { return workerState[i].exitPromise.promise; }) ); + try { + rmSync(dbPath, { force: true, recursive: true, maxRetries: 3 }); + } catch (err) { + console.warn(`Benchmark teardown failed to delete db path: ${err}`); + } resolve(); }, @@ -529,13 +531,7 @@ export async function workerInit(): Promise { await teardown(ctx); } if (ctx.db) { - // console.log('workerTeardown', workerData.benchmarkWorkerId, workerData.mode, type, path); - ctx.db.close(); - try { - rmSync(path, { force: true, recursive: true, maxRetries: 3 }); - } catch (err) { - console.warn(`Benchmark teardown failed to delete db path: ${err}`); - } + await ctx.db.close(); } parentPort!.postMessage({ teardownDone: true, benchmarkWorkerId }); process.exit(0); diff --git a/stress-test/db-instances.stress.test.ts b/stress-test/db-instances.stress.test.ts index 895946aa9..967e974a2 100644 --- a/stress-test/db-instances.stress.test.ts +++ b/stress-test/db-instances.stress.test.ts @@ -39,8 +39,6 @@ describe('Stress DB Instances', () => { worker.on('message', (event) => { if (event.done) { resolve(); - } else if (event.closed) { - resolve(); } }); }) @@ -48,14 +46,22 @@ describe('Stress DB Instances', () => { } await Promise.all(promises); - promises.length = 0; const [before] = registryStatus(); + const closePromises = workers.map( + (worker) => + new Promise((resolve, reject) => { + worker.on('error', reject); + worker.on('message', (event) => { + if (event.closed) resolve(); + }); + }) + ); for (const worker of workers) { worker.postMessage({ close: true }); } - await Promise.all(promises); + await Promise.all(closePromises); if (globalThis.gc) { globalThis.gc(); diff --git a/stress-test/workers/stress-db-instances-worker.mts b/stress-test/workers/stress-db-instances-worker.mts index 9908e23c0..857dba3ef 100644 --- a/stress-test/workers/stress-db-instances-worker.mts +++ b/stress-test/workers/stress-db-instances-worker.mts @@ -21,5 +21,6 @@ parentPort?.on('message', (event) => { db.close(); } parentPort?.postMessage({ closed: true }); + parentPort?.close(); } }); diff --git a/stress-test/workers/stress-transaction-put-worker.mts b/stress-test/workers/stress-transaction-put-worker.mts index e82439a41..e14cda5f0 100644 --- a/stress-test/workers/stress-transaction-put-worker.mts +++ b/stress-test/workers/stress-transaction-put-worker.mts @@ -27,7 +27,9 @@ async function runTransactions10k() { await last; } + db.close(); parentPort?.postMessage({ done: true }); + parentPort?.close(); } async function runTransactions10kWithLogs() { @@ -44,5 +46,7 @@ async function runTransactions10kWithLogs() { }); } + db.close(); parentPort?.postMessage({ done: true }); + parentPort?.close(); } From 1519246742de50f63d6dfc97938f67c80858b1ed Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 17 Aug 2026 01:26:02 -0600 Subject: [PATCH 31/49] Surface worker benchmark teardown failures Co-Authored-By: GPT-5 Codex --- benchmark/setup.ts | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/benchmark/setup.ts b/benchmark/setup.ts index 820c39f46..b501a28ee 100644 --- a/benchmark/setup.ts +++ b/benchmark/setup.ts @@ -235,6 +235,7 @@ interface WorkerState { benchPromise: ReturnType>; exitPromise: ReturnType>; teardownPromise: ReturnType>; + teardownError?: Error; } interface WorkerBenchmarkOptions extends BenchmarkOptions { @@ -381,7 +382,7 @@ export function workerBenchmark(type: string, options: any): void { } const workerState: WorkerState[] = []; - let dbPath: string; + const dbPath = join('benchmark', 'data', `rocksdb-benchmark-${randomBytes(8).toString('hex')}`); const workerPayload = { suites: workerCurrentSuites.map((suite) => suite.name), benchmark: benchmarkName, @@ -409,8 +410,6 @@ export function workerBenchmark(type: string, options: any): void { if (mode === 'run') { return; } - dbPath = join('benchmark', 'data', `rocksdb-benchmark-${randomBytes(8).toString('hex')}`); - let teardownTimeoutId: NodeJS.Timeout; await Promise.race([ activeBenchmark, @@ -437,7 +436,7 @@ export function workerBenchmark(type: string, options: any): void { }); // important! these promises need to be referenced as // properties of `state` because they are reset by reference - const state = { + const state: WorkerState = { worker, benchPromise: withResolvers(), exitPromise: withResolvers(), @@ -445,7 +444,12 @@ export function workerBenchmark(type: string, options: any): void { }; workerState[i] = state; worker.on('error', reject); - worker.on('exit', () => { + worker.on('exit', (code) => { + if (code !== 0 && !state.teardownError) { + state.teardownError = new Error( + `Benchmark worker ${i + 1} exited with code ${code}` + ); + } state.benchPromise.resolve(); state.teardownPromise.resolve(); state.exitPromise.resolve(); @@ -458,6 +462,9 @@ export function workerBenchmark(type: string, options: any): void { state.benchPromise.resolve(); } else if (event.teardownDone) { state.teardownPromise.resolve(); + } else if (event.teardownError) { + state.teardownError = new Error(event.teardownError); + state.teardownPromise.resolve(); } else if (event.timeout) { state.teardownPromise.resolve(); state.benchPromise.reject(new Error('Benchmark timed out')); @@ -489,13 +496,17 @@ export function workerBenchmark(type: string, options: any): void { return workerState[i].exitPromise.promise; }) ); - try { - rmSync(dbPath, { force: true, recursive: true, maxRetries: 3 }); - } catch (err) { - console.warn(`Benchmark teardown failed to delete db path: ${err}`); + const teardownError = workerState.find((state) => state.teardownError)?.teardownError; + if (!teardownError) { + try { + rmSync(dbPath, { force: true, recursive: true, maxRetries: 3 }); + } catch (err) { + console.warn(`Benchmark teardown failed to delete db path: ${err}`); + } } resolve(); + if (teardownError) throw teardownError; }, } ); @@ -531,7 +542,15 @@ export async function workerInit(): Promise { await teardown(ctx); } if (ctx.db) { - await ctx.db.close(); + try { + await ctx.db.close(); + } catch (error) { + parentPort!.postMessage({ + teardownError: error instanceof Error ? error.message : String(error), + benchmarkWorkerId, + }); + process.exit(1); + } } parentPort!.postMessage({ teardownDone: true, benchmarkWorkerId }); process.exit(0); From 7122bdd6c241b5ec5b540a42242df1aeab331718 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 17 Aug 2026 01:26:04 -0600 Subject: [PATCH 32/49] Keep VT fast paths teardown-independent Co-Authored-By: GPT-5 Codex --- AGENTS.md | 5 ++++- src/binding/database/database.cpp | 2 -- src/binding/database/database.h | 10 +++++----- src/binding/database/db_handle.cpp | 2 ++ src/binding/database/db_handle.h | 8 ++++++++ 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f4ed28bf9..c601686b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -329,7 +329,10 @@ sufficient (env teardown does not honor tsfn acquire counts); see `DBHandle::opened()` must report false even while the native DB still exists. Any synchronous N-API path that dereferences `descriptor->db` or the handle's column family must take an `OperationGuard` immediately after `UNWRAP_DB_HANDLE_AND_OPEN()`; `finishClose()` can reset the column-family pointer - from another env after the in-flight count drains. `DBHandle::close()` itself is cross-env and must + from another env after the in-flight count drains. The VT-only `verifyVersion` / `populateVersion` + fast paths are the exception: `DBHandle::open()` snapshots their immutable per-open VT epoch and + column-family ID so they do not touch teardown-owned native state or register an in-flight operation. + `DBHandle::close()` itself is cross-env and must serialize mutation of its `shared_ptr` members. A close-time flush failure keeps the native DB quarantined so `shutdown()` can retry without losing `disableWAL` writes; an explicit destroy may force teardown because the caller requested deletion. A failed physical destroy leaves a registry diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 5e9e0f149..9730d1802 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -1663,7 +1663,6 @@ napi_value Database::GetSync(napi_env env, napi_callback_info info) { napi_value Database::VerifyVersion(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); - ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice keySlice; if (!rocksdb_js::getSliceFromArg(env, argv[0], keySlice, (*dbHandle)->defaultKeyBufferPtr, "Key must be a buffer")) { @@ -1700,7 +1699,6 @@ napi_value Database::VerifyVersion(napi_env env, napi_callback_info info) { napi_value Database::PopulateVersion(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); - ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice keySlice; if (!rocksdb_js::getSliceFromArg(env, argv[0], keySlice, (*dbHandle)->defaultKeyBufferPtr, "Key must be a buffer")) { diff --git a/src/binding/database/database.h b/src/binding/database/database.h index 11ee429c2..1d4c4f34a 100644 --- a/src/binding/database/database.h +++ b/src/binding/database/database.h @@ -42,11 +42,11 @@ inline std::atomic* vtSlotFor( const rocksdb::Slice& key ) { if (!vt) return nullptr; - // Per-open epoch, not the descriptor pointer: the pointer is reused across a - // close/reopen of the same path while cfId stays stable (HarperFast/harper#1864). - uint64_t dbId = dbHandle->descriptor->vtEpoch; - uint32_t cfId = dbHandle->getColumnFamilyHandle()->GetID(); - return vt->slotFor(dbId, cfId, key); + return vt->slotFor( + dbHandle->verificationTableDbId, + dbHandle->verificationTableColumnFamilyId, + key + ); } /** diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index 88a61816f..cc124d244 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -349,6 +349,8 @@ void DBHandle::open(const std::string& path, const DBOptions& options) { auto handleParams = DBRegistry::OpenDB(path, options); this->columnDescriptor = std::move(handleParams->columnDescriptor); this->descriptor = std::move(handleParams->descriptor); + this->verificationTableDbId = this->descriptor->vtEpoch; + this->verificationTableColumnFamilyId = this->columnDescriptor->column->GetID(); this->disableWAL = options.disableWAL; this->enableVerificationTable = options.verificationTable; diff --git a/src/binding/database/db_handle.h b/src/binding/database/db_handle.h index 60ffbf067..7fa31cc91 100644 --- a/src/binding/database/db_handle.h +++ b/src/binding/database/db_handle.h @@ -62,6 +62,14 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro */ bool enableVerificationTable = false; + /** + * Immutable VerificationTable address components for this open lifecycle. + * These let VT-only fast paths avoid dereferencing teardown-owned native + * descriptors or registering as in-flight database operations. + */ + uint64_t verificationTableDbId = 0; + uint32_t verificationTableColumnFamilyId = 0; + /** * The node environment. */ From d76818f961765fddd25e5105475c8ef686e497cf Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 17 Aug 2026 01:59:51 -0600 Subject: [PATCH 33/49] Serialize iterators with forced teardown Co-Authored-By: GPT-5 Codex --- AGENTS.md | 4 ++- src/binding/database/database.cpp | 4 +++ src/binding/database/database.h | 29 ------------------- src/binding/database/db_descriptor.h | 25 ++++++++++++++++ src/binding/iterator/db_iterator.cpp | 32 +++++++++++++++++++-- src/binding/iterator/db_iterator_handle.cpp | 13 ++++++--- src/binding/iterator/db_iterator_handle.h | 3 ++ test/destroy.test.ts | 8 ++++++ test/fixtures/fork-destroy-open.mts | 19 +++++++++++- test/workers/destroy-open-worker.mts | 4 ++- 10 files changed, 102 insertions(+), 39 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c601686b0..ca421ea9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -332,7 +332,9 @@ sufficient (env teardown does not honor tsfn acquire counts); see from another env after the in-flight count drains. The VT-only `verifyVersion` / `populateVersion` fast paths are the exception: `DBHandle::open()` snapshots their immutable per-open VT epoch and column-family ID so they do not touch teardown-owned native state or register an in-flight operation. - `DBHandle::close()` itself is cross-env and must + Async N-API setup must hold the guard until it hands off to `DBHandle::registerAsyncWork()`. Iterators + take the guard through construction/descriptor attachment, then serialize each native iterator call + against foreign forced close with their per-iterator mutex. `DBHandle::close()` itself is cross-env and must serialize mutation of its `shared_ptr` members. A close-time flush failure keeps the native DB quarantined so `shutdown()` can retry without losing `disableWAL` writes; an explicit destroy may force teardown because the caller requested deletion. A failed physical destroy leaves a registry diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 9730d1802..8f02d9a88 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -67,6 +67,7 @@ napi_value Database::Constructor(napi_env env, napi_callback_info info) { static napi_value doClear(napi_env env, napi_callback_info info, const char* failureMsg) { NAPI_METHOD_ARGV(2); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); napi_value resolve = argv[0]; napi_value reject = argv[1]; @@ -257,6 +258,7 @@ napi_value Database::Columns(napi_env env, napi_callback_info info) { napi_value Database::Compact(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(5); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); napi_value resolve = argv[0]; napi_value reject = argv[1]; @@ -655,6 +657,7 @@ napi_value Database::FlushSync(napi_env env, napi_callback_info info) { napi_value Database::Flush(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(3); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); napi_value resolve = argv[0]; napi_value reject = argv[1]; @@ -758,6 +761,7 @@ napi_value Database::Get(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(5); UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); rocksdb::Slice keySlice; if (!rocksdb_js::getSliceFromArg(env, argv[0], keySlice, (*dbHandle)->defaultKeyBufferPtr, "Key must be a buffer")) { return nullptr; diff --git a/src/binding/database/database.h b/src/binding/database/database.h index 1d4c4f34a..61449c5eb 100644 --- a/src/binding/database/database.h +++ b/src/binding/database/database.h @@ -235,35 +235,6 @@ inline void vtPopulateIfSettled( } \ } while (0) -/** - * RAII guard that tracks in-flight operations on a DBDescriptor. - * Increments counter on construction, decrements on destruction. - * Notifies waiters via atomic::notify_all() when count reaches zero. - */ -struct OperationGuard { - std::shared_ptr descriptor; - - explicit OperationGuard(std::shared_ptr desc) : descriptor(std::move(desc)) { - if (descriptor) { - ++descriptor->operationsInFlight; - } - } - - ~OperationGuard() { - if (descriptor) { - if (--descriptor->operationsInFlight == 0 && descriptor->isClosing()) { - descriptor->operationsInFlight.notify_all(); - } - } - } - - // Non-copyable, non-movable - OperationGuard(const OperationGuard&) = delete; - OperationGuard& operator=(const OperationGuard&) = delete; - OperationGuard(OperationGuard&&) = delete; - OperationGuard& operator=(OperationGuard&&) = delete; -}; - /** * Registers an in-flight operation to prevent use-after-free during shutdown. * Also checks if the database is closing and throws an error if so. diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index ca2afc6e2..edd25c6f6 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -615,6 +615,31 @@ struct DBDescriptor final : public std::enable_shared_from_this { ); }; +/** + * Pins a descriptor operation across cross-environment teardown. Callers must + * check `isClosing()` after construction and before touching native DB state. + */ +struct OperationGuard final { + std::shared_ptr descriptor; + + explicit OperationGuard(std::shared_ptr desc) : descriptor(std::move(desc)) { + if (descriptor) { + ++descriptor->operationsInFlight; + } + } + + ~OperationGuard() { + if (descriptor && --descriptor->operationsInFlight == 0 && descriptor->isClosing()) { + descriptor->operationsInFlight.notify_all(); + } + } + + OperationGuard(const OperationGuard&) = delete; + OperationGuard& operator=(const OperationGuard&) = delete; + OperationGuard(OperationGuard&&) = delete; + OperationGuard& operator=(OperationGuard&&) = delete; +}; + /** * State to pass into `napi_call_threadsafe_function()` for a lock callback. */ diff --git a/src/binding/iterator/db_iterator.cpp b/src/binding/iterator/db_iterator.cpp index d2ac626e3..e6576762d 100644 --- a/src/binding/iterator/db_iterator.cpp +++ b/src/binding/iterator/db_iterator.cpp @@ -5,6 +5,7 @@ #include "napi/macros.h" #include "transaction/transaction.h" #include "core/platform.h" +#include "core/test_seam.h" #include "napi/helpers.h" #include "napi/async.h" @@ -140,6 +141,20 @@ napi_value DBIterator::Constructor(napi_env env, napi_callback_info info) { } DEBUG_LOG("DBIterator::Constructor Initializing iterator handle with Database instance (dbHandle=%p)\n", (*dbHandle).get()); } + const int setupDelayMs = testDelayMs("ROCKSDB_JS_ITERATOR_SETUP_DELAY_MS"); + if (setupDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(setupDelayMs)); + } + auto descriptor = (*dbHandle)->descriptor; + if (!descriptor) { + ::napi_throw_error(env, nullptr, "Database not open"); + return nullptr; + } + OperationGuard operationGuard(descriptor); + if (descriptor->isClosing()) { + ::napi_throw_error(env, nullptr, "Database is closing"); + return nullptr; + } // Resolve start/end key pointers from the shared default key buffer char* keyBufferPtr = (*dbHandle)->defaultKeyBufferPtr; @@ -212,7 +227,7 @@ napi_value DBIterator::Constructor(napi_env env, napi_callback_info info) { std::shared_ptr* itHandle = nullptr; \ do { \ NAPI_STATUS_THROWS(::napi_unwrap(env, jsThis, reinterpret_cast(&itHandle))); \ - if (!itHandle || (*itHandle)->iterator == nullptr) { \ + if (!itHandle || !*itHandle) { \ ::napi_throw_error(env, nullptr, fnName " failed: Iterator not initialized"); \ return nullptr; \ } \ @@ -264,6 +279,11 @@ napi_value DBIterator::Next(napi_env env, napi_callback_info info) { UNWRAP_ITERATOR_HANDLE("Next"); auto& it = *itHandle; + std::lock_guard iteratorLock(it->iteratorMutex); + if (!it->iterator) { + ::napi_throw_error(env, nullptr, "Next failed: Iterator not initialized"); + return nullptr; + } napi_value result; if (!it->iterator->Valid()) { @@ -350,7 +370,10 @@ napi_value DBIterator::Return(napi_env env, napi_callback_info info) { UNWRAP_ITERATOR_HANDLE("Return"); DEBUG_LOG("%p DBIterator::Return Closing iterator handle\n", (*itHandle).get()); - (*itHandle)->close(); + if (!(*itHandle)->closeIfOpen()) { + ::napi_throw_error(env, nullptr, "Return failed: Iterator not initialized"); + return nullptr; + } NAPI_RETURN_UNDEFINED(); } @@ -364,7 +387,10 @@ napi_value DBIterator::Throw(napi_env env, napi_callback_info info) { UNWRAP_ITERATOR_HANDLE("Throw"); DEBUG_LOG("%p DBIterator::Throw Closing iterator handle\n", (*itHandle).get()); - (*itHandle)->close(); + if (!(*itHandle)->closeIfOpen()) { + ::napi_throw_error(env, nullptr, "Throw failed: Iterator not initialized"); + return nullptr; + } NAPI_RETURN_UNDEFINED(); } diff --git a/src/binding/iterator/db_iterator_handle.cpp b/src/binding/iterator/db_iterator_handle.cpp index 9ccfc589e..98b646862 100644 --- a/src/binding/iterator/db_iterator_handle.cpp +++ b/src/binding/iterator/db_iterator_handle.cpp @@ -60,15 +60,20 @@ DBIteratorHandle::~DBIteratorHandle() { } void DBIteratorHandle::close() { + this->closeIfOpen(); +} + +bool DBIteratorHandle::closeIfOpen() { + std::lock_guard lock(this->iteratorMutex); DEBUG_LOG("%p DBIteratorHandle::close dbHandle=%p dbDescriptor=%p\n", this, this->dbHandle.get(), this->dbHandle->descriptor.get()); - if (this->iterator) { - this->iterator->Reset(); - this->iterator.reset(); - } + if (!this->iterator) return false; + this->iterator->Reset(); + this->iterator.reset(); if (this->txnHandle) { auto txnHandle = std::move(this->txnHandle); txnHandle->unregisterIterator(); } + return true; } void DBIteratorHandle::init(DBIteratorOptions& options) { diff --git a/src/binding/iterator/db_iterator_handle.h b/src/binding/iterator/db_iterator_handle.h index 06e237275..be0172d1f 100644 --- a/src/binding/iterator/db_iterator_handle.h +++ b/src/binding/iterator/db_iterator_handle.h @@ -1,6 +1,7 @@ #ifndef __DB_ITERATOR_HANDLE_H__ #define __DB_ITERATOR_HANDLE_H__ +#include #include "database/db_handle.h" #include "iterator/db_iterator.h" #include "transaction/transaction_handle.h" @@ -45,6 +46,7 @@ struct DBIteratorHandle final : Closable, public std::enable_shared_from_this { }); }, 15_000); + it('closes an iterator safely when destroy races its construction', async () => { + await runDestroyFixture(destroyOpenFixture, generateDBPath(), { + ROCKSDB_JS_DESTROY_DELAY_MS: '2000', + ROCKSDB_JS_ITERATOR_SETUP_DELAY_MS: '250', + ROCKSDB_JS_TEST_ITERATOR_DESTROY_RACE: '1', + }); + }, 15_000); + it('waits for physical destruction before shutdown completes', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_DELAY_MS: '2000', diff --git a/test/fixtures/fork-destroy-open.mts b/test/fixtures/fork-destroy-open.mts index 7e6e55574..718dcdd8e 100644 --- a/test/fixtures/fork-destroy-open.mts +++ b/test/fixtures/fork-destroy-open.mts @@ -10,7 +10,10 @@ original.useLog('cross-env-close'); const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { eval: true, - workerData: { path }, + workerData: { + path, + destroyStartDelayMs: process.env.ROCKSDB_JS_TEST_ITERATOR_DESTROY_RACE === '1' ? 50 : 0, + }, }); function nextMessage(): Promise { @@ -27,6 +30,20 @@ const destroying = await nextMessage(); if (!destroying.destroying) throw new Error(`Destroy worker did not start: ${JSON.stringify(destroying)}`); +if (process.env.ROCKSDB_JS_TEST_ITERATOR_DESTROY_RACE === '1') { + try { + const rows = original.getRange({ limit: 1 }).asArray; + if (rows.length !== 1) throw new Error(`Iterator returned ${rows.length} rows before destroy`); + } catch (error) { + if ( + !String(error).includes('Database not open') && + !String(error).includes('Database is closing') + ) { + throw error; + } + } +} + const registryDeadline = Date.now() + 5_000; while (registryStatus().some((entry) => entry.path === path)) { if (Date.now() >= registryDeadline) throw new Error('Timed out waiting for the destroy window'); diff --git a/test/workers/destroy-open-worker.mts b/test/workers/destroy-open-worker.mts index 7fe2c0585..58d331047 100644 --- a/test/workers/destroy-open-worker.mts +++ b/test/workers/destroy-open-worker.mts @@ -1,4 +1,5 @@ import { RocksDatabase } from '../../src/index.ts'; +import { setTimeout as delay } from 'node:timers/promises'; import { parentPort, workerData } from 'node:worker_threads'; const db = RocksDatabase.open(workerData.path); @@ -6,8 +7,9 @@ if (!parentPort) throw new Error('Destroy/open worker requires a parent port'); const port = parentPort; port.postMessage({ ready: true }); -port.once('message', () => { +port.once('message', async () => { port.postMessage({ destroying: true }); + if (workerData.destroyStartDelayMs > 0) await delay(workerData.destroyStartDelayMs); try { db.destroy(); port.postMessage({ destroyed: true }); From 69cd76fb35c926b4e7bcbe966704c5b544dbc9a9 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 19 Aug 2026 22:28:12 -0600 Subject: [PATCH 34/49] Address remaining lifecycle review findings - shutdown() no longer permanently throws once a destroy-cleanup tombstone exists; it stays non-destructive (per AGENTS.md) and skips the entry instead of poisoning every later call - binding.cpp always releases global listener threadsafe functions, even when DBRegistry::Shutdown() throws - compactSync() cancels its manual compaction when finishClose() is draining in-flight operations, instead of blocking the untimed drain (and cascading OpenDB timeouts) for the compaction's full duration - Iterator Return()/Throw() are idempotent again on an already-closed iterator, matching close() elsewhere, instead of throwing over a clean loop exit or the caller's real error - narrow the AGENTS.md VT fast-path claim to what's actually true - log the retained path on a benchmark teardown failure instead of leaking it silently - add a deterministic test for iteratorMutex serializing Next() against a foreign forced close, plus a return()/throw() idempotency unit test --- AGENTS.md | 10 +++- benchmark/setup.ts | 2 + src/binding/binding.cpp | 4 +- src/binding/database/db_descriptor.cpp | 10 ++++ src/binding/database/db_descriptor.h | 10 ++++ src/binding/database/db_registry.cpp | 23 ++++----- src/binding/iterator/db_iterator.cpp | 26 ++++++---- test/destroy.test.ts | 17 ++++++- test/fixtures/fork-iterator-next-race.mts | 58 +++++++++++++++++++++++ test/ranges.test.ts | 23 +++++++++ 10 files changed, 156 insertions(+), 27 deletions(-) create mode 100644 test/fixtures/fork-iterator-next-race.mts diff --git a/AGENTS.md b/AGENTS.md index ca421ea9b..02b71dfd3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -330,8 +330,14 @@ sufficient (env teardown does not honor tsfn acquire counts); see path that dereferences `descriptor->db` or the handle's column family must take an `OperationGuard` immediately after `UNWRAP_DB_HANDLE_AND_OPEN()`; `finishClose()` can reset the column-family pointer from another env after the in-flight count drains. The VT-only `verifyVersion` / `populateVersion` - fast paths are the exception: `DBHandle::open()` snapshots their immutable per-open VT epoch and - column-family ID so they do not touch teardown-owned native state or register an in-flight operation. + fast paths narrow, but do not remove, that requirement: both still start with + `UNWRAP_DB_HANDLE_AND_OPEN()`, so they still gate on `descriptor`/`isClosing()`. What + `DBHandle::open()`'s snapshotted `verificationTableDbId` / `verificationTableColumnFamilyId` actually + avoids is the `getColumnFamilyHandle()` dereference and the `OperationGuard`'s in-flight + registration — the two things that are unsafe to skip everywhere else. `PutSync`/`RemoveSync`/ + `TransactionHandle` still compute the VT address as `descriptor->vtEpoch` + + `getColumnFamilyHandle()->GetID()` rather than reading the cached fields, so there are two + spellings of the same address computation that must stay in agreement. Async N-API setup must hold the guard until it hands off to `DBHandle::registerAsyncWork()`. Iterators take the guard through construction/descriptor attachment, then serialize each native iterator call against foreign forced close with their per-iterator mutex. `DBHandle::close()` itself is cross-env and must diff --git a/benchmark/setup.ts b/benchmark/setup.ts index b501a28ee..e586ff79b 100644 --- a/benchmark/setup.ts +++ b/benchmark/setup.ts @@ -503,6 +503,8 @@ export function workerBenchmark(type: string, options: any): void { } catch (err) { console.warn(`Benchmark teardown failed to delete db path: ${err}`); } + } else { + console.warn(`Benchmark teardown failed; retaining ${dbPath} for inspection`); } resolve(); diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index 9dbc365c1..d5e39f401 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -47,11 +47,13 @@ napi_value Shutdown(napi_env env, napi_callback_info info) { } catch (...) { error = "Unknown native database shutdown failure"; } + // Release global listener threadsafe functions on every path, including a + // failed shutdown -- otherwise they outlive this N-API environment. + GlobalEvents::Shutdown(); if (!error.empty()) { ::napi_throw_error(env, nullptr, error.c_str()); return nullptr; } - GlobalEvents::Shutdown(); napi_value result; NAPI_STATUS_THROWS(::napi_get_undefined(env, &result)); return result; diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 0a4305563..7ac0de4c4 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -425,11 +425,18 @@ void DBDescriptor::finishClose(bool destroying) { // Wait for all in-flight operations to complete before cleanup. // The closing flag is already set, so new operations will fail with "Database is closing". // Existing operations will decrement operationsInFlight and notify us when done. + // A long-running compactRange() holding an OperationGuard is the one + // in-flight op that can run unboundedly, so ask it to cancel rather + // than blocking this untimed wait for its full duration. + this->compactCancelRequested.store(true); DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); uint32_t current; while ((current = this->operationsInFlight.load()) != 0) { this->operationsInFlight.wait(current); } + // Clear it before any further compaction runs below (compact-on-close), + // which must not be cancelled -- nothing external is waiting on it. + this->compactCancelRequested.store(false); DEBUG_LOG("%p DBDescriptor::close All operations complete \"%s\"\n", this, this->path.c_str()); // Drain the commit pipeline before flushing so its data is included in @@ -2253,6 +2260,9 @@ rocksdb::Status DBDescriptor::compactRange( std::lock_guard lock(this->compactMutex); DEBUG_LOG("%p DBDescriptor::compactRange Compacting range (bottommost=%d)\n", this, bottommost); rocksdb::CompactRangeOptions options; + // Let a concurrent finishClose() interrupt this compaction rather than + // wait out its full, unbounded duration; see compactCancelRequested. + options.canceled = &this->compactCancelRequested; if (bottommost) { // RocksDB defaults this to kIfHaveCompactionFilter, so with no compaction filter installed // the bottommost level is skipped — and that is where the bulk of the data sits. Rewriting diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index edd25c6f6..c0198a3a3 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -293,6 +293,16 @@ struct DBDescriptor final : public std::enable_shared_from_this { bool closeWorkersStopped = false; bool transactionLogsUnregistered = false; + /** + * Set by finishClose() only while it is draining operationsInFlight, so an + * OperationGuard-holding compactRange() in progress on another thread can + * cancel its manual compaction and release the guard promptly instead of + * blocking the untimed drain wait for the compaction's full duration. + * Cleared once the drain completes so the close-time "compact on close" + * pass below always runs to completion. + */ + std::atomic compactCancelRequested{false}; + /** * Counter tracking in-flight database operations. close() uses * atomic::wait() to block until this reaches zero. diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index bc13de015..2d284b030 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -1016,7 +1016,6 @@ void DBRegistry::Shutdown() { while (true) { std::vector descriptorsToClose; std::vector descriptorsToWaitFor; - std::string destroyCleanupError; bool destroysInFlight; { std::unique_lock lock(instance->databasesMutex); @@ -1030,11 +1029,13 @@ void DBRegistry::Shutdown() { continue; } if (!entry.descriptor) { - if (!entry.closeError.empty() && destroyCleanupError.empty()) { - destroyCleanupError = - "Cannot complete shutdown: database \"" + key.path + - "\" requires explicit destroy() cleanup: " + entry.closeError; - } + // A prior destroy() left a tombstone (descriptor cleared, + // closeError set) after its physical cleanup failed. That + // failure was already surfaced via database:closeFailed and + // stays visible in registryStatus().destroyCleanupPending. + // shutdown() is deliberately non-destructive -- only an + // explicit destroy() retries path deletion -- so skip it + // here rather than re-throwing the same error forever. continue; } ClosingDescriptor closing{key, entry.descriptor, entry.condition}; @@ -1101,18 +1102,10 @@ void DBRegistry::Shutdown() { throw rocksdb_js::DBException("Timed out waiting for database destruction to finish during shutdown"); } if (closeError) std::rethrow_exception(closeError); - if (!destroyCleanupError.empty()) { - throw rocksdb_js::DBException(destroyCleanupError); - } continue; } if (closeError) std::rethrow_exception(closeError); - if (!destroyCleanupError.empty()) { - throw rocksdb_js::DBException(destroyCleanupError); - } - if (descriptorsToClose.empty() && descriptorsToWaitFor.empty() && - destroyCleanupError.empty() - ) break; + if (descriptorsToClose.empty() && descriptorsToWaitFor.empty()) break; } // Purge the registry diff --git a/src/binding/iterator/db_iterator.cpp b/src/binding/iterator/db_iterator.cpp index e6576762d..b800dc6c5 100644 --- a/src/binding/iterator/db_iterator.cpp +++ b/src/binding/iterator/db_iterator.cpp @@ -8,6 +8,8 @@ #include "core/test_seam.h" #include "napi/helpers.h" #include "napi/async.h" +#include +#include namespace rocksdb_js { @@ -280,6 +282,14 @@ napi_value DBIterator::Next(napi_env env, napi_callback_info info) { auto& it = *itHandle; std::lock_guard iteratorLock(it->iteratorMutex); + // Test-only: widen the window where a foreign finishClose()'s closables + // sweep is blocked on iteratorMutex behind this call, so a fixture can + // reliably position a forced close mid-Next() rather than only ever + // between calls. + const int nextDelayMs = testDelayMs("ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS"); + if (nextDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(nextDelayMs)); + } if (!it->iterator) { ::napi_throw_error(env, nullptr, "Next failed: Iterator not initialized"); return nullptr; @@ -370,10 +380,11 @@ napi_value DBIterator::Return(napi_env env, napi_callback_info info) { UNWRAP_ITERATOR_HANDLE("Return"); DEBUG_LOG("%p DBIterator::Return Closing iterator handle\n", (*itHandle).get()); - if (!(*itHandle)->closeIfOpen()) { - ::napi_throw_error(env, nullptr, "Return failed: Iterator not initialized"); - return nullptr; - } + // Idempotent by design: a foreign forced close (finishClose()'s closables + // sweep) or an earlier limit-triggered return() may have already closed + // this iterator, and a clean loop exit / a second explicit return() must + // not turn into a thrown error. + (*itHandle)->close(); NAPI_RETURN_UNDEFINED(); } @@ -387,10 +398,9 @@ napi_value DBIterator::Throw(napi_env env, napi_callback_info info) { UNWRAP_ITERATOR_HANDLE("Throw"); DEBUG_LOG("%p DBIterator::Throw Closing iterator handle\n", (*itHandle).get()); - if (!(*itHandle)->closeIfOpen()) { - ::napi_throw_error(env, nullptr, "Throw failed: Iterator not initialized"); - return nullptr; - } + // Idempotent for the same reason as Return above -- must not replace the + // caller's real thrown error with a spurious native one. + (*itHandle)->close(); NAPI_RETURN_UNDEFINED(); } diff --git a/test/destroy.test.ts b/test/destroy.test.ts index aa8815bc8..4a04ab830 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -13,6 +13,7 @@ const shutdownFailureFixture = join(__dirname, 'fixtures', 'fork-shutdown-failur const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mts'); const flushFailureFixture = join(__dirname, 'fixtures', 'fork-flush-failure.mts'); const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); +const iteratorNextRaceFixture = join(__dirname, 'fixtures', 'fork-iterator-next-race.mts'); const nodeExecutable = process.env.NODE_BINARY ?? (process.versions.bun || process.versions.deno @@ -163,7 +164,11 @@ describe('Destroy', () => { expect.stringContaining('Failed to remove database directory'), ]); chmodSync(lockedDirectory, 0o700); - expect(() => shutdown()).toThrow('requires explicit destroy() cleanup'); + // shutdown() is deliberately non-destructive: it must not retry + // path deletion (only an explicit destroy() call may), so a + // pending tombstone does not make it throw, and it does not + // clear the tombstone even though the underlying cause is fixed. + shutdown(); expect(existsSync(dbPath)).toBe(true); expect( registryStatus().find((entry) => entry.path === dbPath)?.destroyCleanupPending @@ -196,6 +201,16 @@ describe('Destroy', () => { }); }, 15_000); + it('serializes an in-progress Next() against a foreign forced close', async () => { + // Unlike the constructor race above, this positions the destroy while + // a Next() call already holds iteratorMutex, so it must block on the + // mutex rather than racing it -- the actual case iteratorMutex exists + // for. See fork-iterator-next-race.mts. + await runDestroyFixture(iteratorNextRaceFixture, generateDBPath(), { + ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS: '250', + }); + }, 15_000); + it('waits for physical destruction before shutdown completes', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_DELAY_MS: '2000', diff --git a/test/fixtures/fork-iterator-next-race.mts b/test/fixtures/fork-iterator-next-race.mts new file mode 100644 index 000000000..8422e9b46 --- /dev/null +++ b/test/fixtures/fork-iterator-next-race.mts @@ -0,0 +1,58 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { Worker } from 'node:worker_threads'; + +const path = process.argv[2]; +const original = RocksDatabase.open(path); +original.putSync('a', '1'); +original.putSync('b', '2'); + +const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { + eval: true, + workerData: { path, destroyStartDelayMs: 50 }, +}); + +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} + +const ready = await nextMessage(); +if (!ready.ready) throw new Error(`Worker failed to initialize: ${JSON.stringify(ready)}`); + +const iterator = original.getRange({})[Symbol.iterator](); +const first = iterator.next(); +if (first.done) throw new Error('Expected a first row before the destroy race'); + +worker.postMessage({ destroy: true }); +const destroying = await nextMessage(); +if (!destroying.destroying) + throw new Error(`Worker did not start destroying: ${JSON.stringify(destroying)}`); + +// Holds iteratorMutex for the ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS test seam +// while the worker's destroy() -- ticking on its own 50ms delay -- reaches +// finishClose()'s closables sweep and blocks on the same mutex behind this +// call. This is the case iteratorMutex exists for: a foreign forced close +// racing an in-progress Next(), not just one racing the constructor. +const second = iterator.next(); +if (second.done) throw new Error('Expected a second row before destroy claimed the iterator'); + +const destroyResult = await nextMessage(); +if (!destroyResult.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyResult)}`); + +if (registryStatus().some((entry) => entry.path === path)) { + throw new Error('Expected destroy to fully clear the registry entry'); +} + +// The mutex handoff must leave the iterator cleanly (not torn/crashing) +// closed once finishClose() gets its turn. +try { + iterator.next(); + throw new Error('Expected Next to fail once destroy closed the iterator'); +} catch (error) { + if (!String(error).includes('Iterator not initialized')) throw error; +} + +await worker.terminate(); diff --git a/test/ranges.test.ts b/test/ranges.test.ts index 1abde0857..b3a384f5d 100644 --- a/test/ranges.test.ts +++ b/test/ranges.test.ts @@ -165,6 +165,29 @@ describe('Ranges', () => { } })); + it('is idempotent when return()/throw() are called on an already-closed iterator', () => + dbRunner(async ({ db }) => { + for (const key of ['a', 'b']) { + await db.put(key, `value ${key}`); + } + + const iter = db.getRange()[Symbol.iterator](); + iter.next(); + iter.return!(); + // A second return() (e.g. a `finally` block after an earlier + // `break`) must stay a no-op, not throw over the already-closed + // native iterator. + expect(() => iter.return!()).not.toThrow(); + + const limited = db.getRange({ limit: 1 })[Symbol.iterator](); + limited.next(); // yields the one row within the limit + limited.next(); // over the limit: auto-closes the native iterator + // The native iterator is already closed at this point; the + // caller's own thrown error must survive, not get replaced by a + // native "Iterator not initialized" error. + expect(() => limited.throw!(new Error('caller error'))).toThrow('caller error'); + })); + it('should get iterate in reverse', () => dbRunner(async ({ db }) => { for (const key of ['a', 'b', 'c', 'd', 'e']) { From ade3651734c244ea9f731d64321754df8c3f674d Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 24 Aug 2026 15:49:48 -0600 Subject: [PATCH 35/49] Address remaining lifecycle review threads - DBIterator::Next() no longer pays a getenv() scan per row for the ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS seam; it is snapshotted once in initializeTestSeams() alongside the close-failure flags. Next() returns one row per call, so this was ~a quarter of the per-row getRange cost for a seam that is unset in production. - Extract closeClaimedDescriptors() in db_registry.cpp: the finishClose() -> erase-or-quarantine -> notify -> emit tail was copied four times (PurgeIfUnreferenced, DestroyDB, PurgeAll, Shutdown), each handling closeError/closeRetrying slightly differently. Only the claim predicate genuinely differs per caller, so that is all that is left at the call sites. The completed-but-errored policy that had drifted is now one named option: fatal for shutdown()/PurgeAll() because dropping a failed close-time flush would hide possible data loss, non-fatal for destroy(), whose caller asked for the data to be deleted anyway. - getKeysCount() was the remaining unbounded OperationGuard holder that finishClose()'s untimed drain could not cancel. The scan now polls isClosing() per row and reports the abort instead of a partial count, on both the database and transaction paths, so a foreign destroy() is no longer blocked for the length of the range (and concurrent OpenDB() calls for that path no longer time out behind it). The comment claiming compaction was the only unbounded in-flight op is corrected. Co-Authored-By: Claude Opus --- AGENTS.md | 20 ++ src/binding/core/test_seam.h | 19 ++ src/binding/database/database.cpp | 11 +- src/binding/database/db_descriptor.cpp | 7 +- src/binding/database/db_registry.cpp | 219 ++++++++---------- src/binding/iterator/db_iterator.cpp | 2 +- src/binding/iterator/db_iterator_handle.cpp | 21 ++ src/binding/iterator/db_iterator_handle.h | 10 + src/binding/transaction/transaction.cpp | 5 +- .../transaction/transaction_handle.cpp | 6 +- src/binding/transaction/transaction_handle.h | 4 +- test/destroy.test.ts | 9 + test/fixtures/fork-count-destroy-race.mts | 59 +++++ 13 files changed, 251 insertions(+), 141 deletions(-) create mode 100644 test/fixtures/fork-count-destroy-race.mts diff --git a/AGENTS.md b/AGENTS.md index 02b71dfd3..d8c7dc4d4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -221,6 +221,11 @@ sufficient (env teardown does not honor tsfn acquire counts); see rising edge emits); malformed/negative falls back to the default - `ROCKSDB_JS_DESTROY_DELAY_MS` - Test-only: delay after descriptor teardown and before physical database destruction (widens same-path reopen races) +- `ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS` / `ROCKSDB_JS_COUNT_DELAY_MS` - Test-only: per-row delays in + `DBIterator::Next()` and `DBIteratorHandle::countRemaining()`. Both are read **once** in + `initializeTestSeams()` rather than per row: these are the two per-row native loops, and a + `getenv()` scan per row is a measurable share of their cost for a seam unset in production. Add + new per-row seams the same way. ## Test Structure @@ -346,6 +351,21 @@ sufficient (env teardown does not honor tsfn acquire counts); see force teardown because the caller requested deletion. A failed physical destroy leaves a registry tombstone, but `shutdown()` is deliberately non-destructive: it reports the tombstone and only an explicit `destroy()` retries path deletion. + Because `finishClose()` drains `operationsInFlight` with an **untimed** wait, any operation that can + run unboundedly while holding an `OperationGuard` must abort itself once `closing` is published, or + it blocks teardown — and, since the blocked closer holds the path gate, times out every concurrent + `OpenDB()` for that path. There are two such operations and they cancel differently: the whole-range + count scan (`DBIteratorHandle::countRemaining`, behind `getKeysCount()` on both the database and + transaction paths) polls `isClosing()` per row and reports the abort to its caller rather than a + partial count; a manual `compactRange()` cannot poll from inside RocksDB, so it gets an explicit + cancel token (`DBDescriptor::compactCancelRequested` → `CompactRangeOptions::canceled`). + Everything the four registry teardown paths do _after_ claiming a descriptor — + `finishClose()`, erase-or-quarantine, notify, emit `database:closeFailed` — is one helper, + `closeClaimedDescriptors` in `db_registry.cpp`; only the claim predicate differs per caller. Its + `failOnCompletedWithError` option is the one deliberate asymmetry: a close that finished native + teardown but reported an error (a failed close-time flush) is fatal for `shutdown()`/`PurgeAll()` + because dropping it silently would hide possible data loss, and non-fatal for `destroy()`, whose + caller asked for the data to be deleted anyway. 7. **One writable BackupEngine per backup directory (kernel advisory lock)**: each backup op opens its own short-lived `rocksdb::BackupEngine`/`BackupEngineReadOnly` (`src/binding/database/backup.cpp`), and RocksDB only serializes work _within_ a single engine — it has no cross-engine lock on the directory. diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index effdcd7ee..d2c922097 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -29,6 +29,21 @@ inline std::atomic& closeFlushFailureFlag() { return pending; } +// DBIterator::Next() returns one row per call, so its seam is snapshotted here +// rather than read per call: a getenv() scan per row is a measurable share of +// the per-row cost for a seam that is unset in production. +inline std::atomic& iteratorNextDelayMsFlag() { + static std::atomic delayMs{0}; + return delayMs; +} + +// Per-row delay for DBIteratorHandle::countRemaining(), snapshotted for the +// same reason. +inline std::atomic& countScanDelayMsFlag() { + static std::atomic delayMs{0}; + return delayMs; +} + inline void initializeTestSeams() { static std::once_flag initialized; std::call_once(initialized, []() { @@ -36,6 +51,10 @@ inline void initializeTestSeams() { closeFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); value = ::getenv("ROCKSDB_JS_CLOSE_FLUSH_FAILURE"); closeFlushFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); + iteratorNextDelayMsFlag().store( + testDelayMs("ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS"), std::memory_order_relaxed); + countScanDelayMsFlag().store( + testDelayMs("ROCKSDB_JS_COUNT_DELAY_MS"), std::memory_order_relaxed); }); } diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 8f02d9a88..19c9f7fbe 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -1103,12 +1103,15 @@ napi_value Database::GetCount(napi_env env, napi_callback_info info) { ::napi_throw_error(env, nullptr, errorMsg.c_str()); NAPI_RETURN_UNDEFINED(); } - txnHandle->getCount(itOptions, count, *dbHandle); + if (!txnHandle->getCount(itOptions, count, *dbHandle)) { + ::napi_throw_error(env, nullptr, "Get count failed: Database is closing"); + NAPI_RETURN_UNDEFINED(); + } } else { std::unique_ptr itHandle = std::make_unique(*dbHandle, itOptions); - while (itHandle->iterator->Valid()) { - ++count; - itHandle->iterator->Next(); + if (!itHandle->countRemaining(count)) { + ::napi_throw_error(env, nullptr, "Get count failed: Database is closing"); + NAPI_RETURN_UNDEFINED(); } } diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 7ac0de4c4..47fa86ca4 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -425,9 +425,10 @@ void DBDescriptor::finishClose(bool destroying) { // Wait for all in-flight operations to complete before cleanup. // The closing flag is already set, so new operations will fail with "Database is closing". // Existing operations will decrement operationsInFlight and notify us when done. - // A long-running compactRange() holding an OperationGuard is the one - // in-flight op that can run unboundedly, so ask it to cancel rather - // than blocking this untimed wait for its full duration. + // Unbounded in-flight operations must abort once `closing` is published + // rather than block this untimed wait for their full duration. A count + // scan polls isClosing() itself; a manual compactRange() cannot, so it + // gets an explicit cancel token. this->compactCancelRequested.store(true); DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); uint32_t current; diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 2d284b030..4a69d355f 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -101,6 +101,76 @@ void emitCloseFailures(const std::vector& descriptors) { } } +struct ClaimedCloseOptions final { + // destroy() is deleting the data, so it forces teardown and does not treat a + // close that finished but reported an error (a failed close-time flush) as + // fatal. Every other caller does: dropping that error silently would hide + // possible data loss. The entry is erased either way, so the failure is + // reported once rather than wedging the path. + bool destroying = false; + bool failOnCompletedWithError = true; +}; + +/** + * Runs finishClose() over descriptors already claimed by the caller, then + * erases each entry or quarantines it with its close error, notifies that + * path's waiters, and emits `database:closeFailed`. + * + * Returns the first exception the caller should rethrow, or null. Claiming + * differs per caller (one path, every path, or a single unreferenced + * descriptor); everything after the claim is this one policy. + */ +std::exception_ptr closeClaimedDescriptors( + std::vector& claimed, + const ClaimedCloseOptions& options, + std::unordered_map& databases, + std::mutex& databasesMutex +) { + std::exception_ptr closeError; + + for (auto& closing : claimed) { + DEBUG_LOG("DBRegistry::closeClaimedDescriptors Closing descriptor %p for \"%s\" (ref count = %ld)\n", + closing.descriptor.get(), closing.key.path.c_str(), closing.descriptor.use_count()); + + std::exception_ptr thrown; + try { + closing.descriptor->finishClose(options.destroying); + closing.closed = true; + } catch (const std::exception& error) { + closing.closed = closing.descriptor->isClosed(); + closing.closeError = error.what(); + thrown = std::current_exception(); + } catch (...) { + closing.closed = closing.descriptor->isClosed(); + closing.closeError = "unknown native close failure"; + thrown = std::current_exception(); + } + + if (thrown && !closeError && (!closing.closed || options.failOnCompletedWithError)) { + closeError = thrown; + } + + { + std::lock_guard lock(databasesMutex); + auto entry = databases.find(closing.key); + if (entry != databases.end() && entry->second.descriptor == closing.descriptor) { + if (closing.closed) { + databases.erase(entry); + } else { + entry->second.closeError = closing.closeError; + entry->second.closeRetrying = false; + DEBUG_LOG("DBRegistry::closeClaimedDescriptors Quarantined \"%s\": %s\n", + closing.key.path.c_str(), closing.closeError.c_str()); + } + } + } + closing.condition->notify_all(); + } + + emitCloseFailures(claimed); + return closeError; +} + } // namespace // Initialize the static instance @@ -200,35 +270,17 @@ CloseResult DBRegistry::PurgeIfUnreferenced(const std::string& path, bool readOn if (descriptor) { // We claimed the close under the lock via beginClose(); run the actual // teardown now. The local copy keeps the descriptor alive throughout. - try { - descriptor->finishClose(); - } catch (const std::exception& error) { - closeError = error.what(); - } catch (...) { - closeError = "unknown native close failure"; - } - - std::lock_guard lock(instance->databasesMutex); - auto eraseIt = instance->databases.find(key); - // Only erase the entry we claimed. A brand-new descriptor cannot appear - // because OpenDB blocks until we notify below. - if (eraseIt != instance->databases.end() && eraseIt->second.descriptor == descriptor) { - if (closeError.empty() || descriptor->isClosed()) { - instance->databases.erase(eraseIt); - } else { - eraseIt->second.closeError = closeError; - quarantined = true; - DEBUG_LOG("%p DBRegistry::PurgeIfUnreferenced Quarantined \"%s\": %s\n", - instance.get(), path.c_str(), closeError.c_str()); - } - } + // Only the entry we claimed is erased -- a brand-new descriptor cannot + // appear because OpenDB blocks until the helper notifies. + std::vector claimed; + claimed.emplace_back(key, descriptor, condition); + // The close error is reported through CloseResult, not thrown. + closeClaimedDescriptors( + claimed, ClaimedCloseOptions{}, instance->databases, instance->databasesMutex); + closeError = claimed.front().closeError; + quarantined = !closeError.empty() && !claimed.front().closed; } - // notify only waiters for this specific path - if (condition) { - condition->notify_all(); - } - emitCloseFailure(path, closeError); return CloseResult{closeError, quarantined}; } @@ -300,49 +352,15 @@ void DBRegistry::DestroyDB(const std::string& path) { } } - // Keep entries discoverable while finishClose runs: env cleanup uses the - // registry to remove callbacks owned by a worker that exits mid-close. - std::exception_ptr closeError; - for (auto& closing : claimed) { - DEBUG_LOG("%p DBRegistry::DestroyDB Closing descriptor %p for \"%s\" (ref count = %ld)\n", - instance.get(), closing.descriptor.get(), path.c_str(), closing.descriptor.use_count()); - try { - closing.descriptor->finishClose(true); - closing.closed = true; - } catch (const std::exception& error) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = error.what(); - if (!closing.closed && !closeError) { - closeError = std::current_exception(); - } - } catch (...) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = "unknown native close failure"; - if (!closing.closed && !closeError) { - closeError = std::current_exception(); - } - } - } - - { - std::lock_guard lock(instance->databasesMutex); - for (const auto& closing : claimed) { - auto entry = instance->databases.find(closing.key); - if (entry == instance->databases.end() || entry->second.descriptor != closing.descriptor) { - continue; - } - if (closing.closed) { - instance->databases.erase(entry); - } else { - entry->second.closeError = closing.closeError; - entry->second.closeRetrying = false; - } - } - } - for (const auto& closing : claimed) { - closing.condition->notify_all(); - } - emitCloseFailures(claimed); + // Each entry stays discoverable until its own close finishes: env cleanup + // uses the registry to remove callbacks owned by a worker that exits + // mid-close. + std::exception_ptr closeError = closeClaimedDescriptors( + claimed, + ClaimedCloseOptions{.destroying = true, .failOnCompletedWithError = false}, + instance->databases, + instance->databasesMutex + ); if (alreadyClosing.empty()) { if (closeError) std::rethrow_exception(closeError); break; @@ -713,33 +731,8 @@ void DBRegistry::PurgeAll() { condition->notify_all(); } - for (auto& closing : descriptorsToClose) { - try { - closing.descriptor->finishClose(); - closing.closed = true; - } catch (const std::exception& error) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = error.what(); - if (!closeError) closeError = std::current_exception(); - } catch (...) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = "unknown native close failure"; - if (!closeError) closeError = std::current_exception(); - } - { - std::lock_guard lock(instance->databasesMutex); - auto entry = instance->databases.find(closing.key); - if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { - if (closing.closed) { - instance->databases.erase(entry); - } else { - entry->second.closeError = closing.closeError; - } - } - } - closing.condition->notify_all(); - } - emitCloseFailures(descriptorsToClose); + closeError = closeClaimedDescriptors( + descriptorsToClose, ClaimedCloseOptions{}, instance->databases, instance->databasesMutex); if (closeError) { std::rethrow_exception(closeError); } @@ -1051,36 +1044,8 @@ void DBRegistry::Shutdown() { } } - std::exception_ptr closeError; - for (auto& closing : descriptorsToClose) { - DEBUG_LOG("%p DBRegistry::Shutdown Closing database: %s\n", instance.get(), closing.descriptor->path.c_str()); - try { - closing.descriptor->finishClose(); - closing.closed = true; - } catch (const std::exception& error) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = error.what(); - if (!closeError) closeError = std::current_exception(); - } catch (...) { - closing.closed = closing.descriptor->isClosed(); - closing.closeError = "unknown native close failure"; - if (!closeError) closeError = std::current_exception(); - } - { - std::lock_guard lock(instance->databasesMutex); - auto entry = instance->databases.find(closing.key); - if (entry != instance->databases.end() && entry->second.descriptor == closing.descriptor) { - if (closing.closed) { - instance->databases.erase(entry); - } else { - entry->second.closeError = closing.closeError; - entry->second.closeRetrying = false; - } - } - } - closing.condition->notify_all(); - } - emitCloseFailures(descriptorsToClose); + std::exception_ptr closeError = closeClaimedDescriptors( + descriptorsToClose, ClaimedCloseOptions{}, instance->databases, instance->databasesMutex); for (const auto& closing : descriptorsToWaitFor) { std::unique_lock lock(instance->databasesMutex); diff --git a/src/binding/iterator/db_iterator.cpp b/src/binding/iterator/db_iterator.cpp index b800dc6c5..1728298b0 100644 --- a/src/binding/iterator/db_iterator.cpp +++ b/src/binding/iterator/db_iterator.cpp @@ -286,7 +286,7 @@ napi_value DBIterator::Next(napi_env env, napi_callback_info info) { // sweep is blocked on iteratorMutex behind this call, so a fixture can // reliably position a forced close mid-Next() rather than only ever // between calls. - const int nextDelayMs = testDelayMs("ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS"); + const int nextDelayMs = iteratorNextDelayMsFlag().load(std::memory_order_relaxed); if (nextDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(nextDelayMs)); } diff --git a/src/binding/iterator/db_iterator_handle.cpp b/src/binding/iterator/db_iterator_handle.cpp index 98b646862..9daecee2b 100644 --- a/src/binding/iterator/db_iterator_handle.cpp +++ b/src/binding/iterator/db_iterator_handle.cpp @@ -1,5 +1,7 @@ #include "iterator/db_iterator_handle.h" #include "database/db_descriptor.h" +#include "core/test_seam.h" +#include #include namespace rocksdb_js { @@ -103,6 +105,25 @@ void DBIteratorHandle::init(DBIteratorOptions& options) { } } +bool DBIteratorHandle::countRemaining(uint64_t& count) { + const DBDescriptor* descriptor = this->dbHandle->descriptor.get(); + // Test-only: stretch the scan so a fixture can land a foreign destroy() + // inside it rather than only before or after. + const int rowDelayMs = countScanDelayMsFlag().load(std::memory_order_relaxed); + count = 0; + while (this->iterator->Valid()) { + if (descriptor->isClosing()) { + return false; + } + if (rowDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(rowDelayMs)); + } + ++count; + this->iterator->Next(); + } + return true; +} + void DBIteratorHandle::seek(DBIteratorOptions& options) { if (options.reverse) { this->iterator->SeekToLast(); diff --git a/src/binding/iterator/db_iterator_handle.h b/src/binding/iterator/db_iterator_handle.h index be0172d1f..45a4e8a20 100644 --- a/src/binding/iterator/db_iterator_handle.h +++ b/src/binding/iterator/db_iterator_handle.h @@ -54,6 +54,16 @@ struct DBIteratorHandle final : Closable, public std::enable_shared_from_this dbHandle; std::shared_ptr txnHandle; bool exclusiveStart; diff --git a/src/binding/transaction/transaction.cpp b/src/binding/transaction/transaction.cpp index 3b2d6e158..ef98174ab 100644 --- a/src/binding/transaction/transaction.cpp +++ b/src/binding/transaction/transaction.cpp @@ -1055,7 +1055,10 @@ napi_value Transaction::GetCount(napi_env env, napi_callback_info info) { itOptions.values = false; uint64_t count = 0; - (*txnHandle)->getCount(itOptions, count); + if (!(*txnHandle)->getCount(itOptions, count)) { + ::napi_throw_error(env, nullptr, "Get count failed: Database is closing"); + NAPI_RETURN_UNDEFINED(); + } napi_value result; NAPI_STATUS_THROWS(::napi_create_int64(env, count, &result)); diff --git a/src/binding/transaction/transaction_handle.cpp b/src/binding/transaction/transaction_handle.cpp index a0e346130..84637a739 100644 --- a/src/binding/transaction/transaction_handle.cpp +++ b/src/binding/transaction/transaction_handle.cpp @@ -590,7 +590,7 @@ napi_value TransactionHandle::get( return returnStatus; } -void TransactionHandle::getCount( +bool TransactionHandle::getCount( DBIteratorOptions& itOptions, uint64_t& count, std::shared_ptr dbHandleOverride @@ -602,9 +602,7 @@ void TransactionHandle::getCount( std::unique_ptr itHandle = std::make_unique(this->shared_from_this(), itOptions, dbHandleOverride); - for (count = 0; itHandle->iterator->Valid(); ++count) { - itHandle->iterator->Next(); - } + return itHandle->countRemaining(count); } /** diff --git a/src/binding/transaction/transaction_handle.h b/src/binding/transaction/transaction_handle.h index 85c01e33a..9ac8b0dd8 100644 --- a/src/binding/transaction/transaction_handle.h +++ b/src/binding/transaction/transaction_handle.h @@ -232,8 +232,10 @@ struct TransactionHandle final : Closable, AsyncWorkHandle, std::enable_shared_f * @param dbHandleOverride - Database handle override to use instead of the * transaction's database handle when called via the `NativeDatabase` with * the `transaction` property set. + * @returns False when the descriptor began closing mid-scan; see + * `DBIteratorHandle::countRemaining()`. */ - void getCount( + [[nodiscard]] bool getCount( DBIteratorOptions& itOptions, uint64_t& count, std::shared_ptr dbHandleOverride = nullptr diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 4a04ab830..0a22e1107 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -14,6 +14,7 @@ const shutdownRetryFixture = join(__dirname, 'fixtures', 'fork-shutdown-retry.mt const flushFailureFixture = join(__dirname, 'fixtures', 'fork-flush-failure.mts'); const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); const iteratorNextRaceFixture = join(__dirname, 'fixtures', 'fork-iterator-next-race.mts'); +const countDestroyRaceFixture = join(__dirname, 'fixtures', 'fork-count-destroy-race.mts'); const nodeExecutable = process.env.NODE_BINARY ?? (process.versions.bun || process.versions.deno @@ -211,6 +212,14 @@ describe('Destroy', () => { }); }, 15_000); + it('aborts an in-flight getCount() when a foreign destroy begins', async () => { + // getCount() scans the whole range under one OperationGuard, which + // finishClose() drains with an untimed wait. See fork-count-destroy-race.mts. + await runDestroyFixture(countDestroyRaceFixture, generateDBPath(), { + ROCKSDB_JS_COUNT_DELAY_MS: '50', + }); + }, 15_000); + it('waits for physical destruction before shutdown completes', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_DELAY_MS: '2000', diff --git a/test/fixtures/fork-count-destroy-race.mts b/test/fixtures/fork-count-destroy-race.mts new file mode 100644 index 000000000..c6c36d471 --- /dev/null +++ b/test/fixtures/fork-count-destroy-race.mts @@ -0,0 +1,59 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { Worker } from 'node:worker_threads'; + +const path = process.argv[2]; +const original = RocksDatabase.open(path); +for (let i = 0; i < 20; i++) { + original.putSync(`key-${i}`, i); +} + +const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { + eval: true, + workerData: { path, destroyStartDelayMs: 50 }, +}); + +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} + +const ready = await nextMessage(); +if (!ready.ready) throw new Error(`Worker failed to initialize: ${JSON.stringify(ready)}`); + +worker.postMessage({ destroy: true }); +const destroying = await nextMessage(); +if (!destroying.destroying) + throw new Error(`Worker did not start destroying: ${JSON.stringify(destroying)}`); + +// 20 rows x ROCKSDB_JS_COUNT_DELAY_MS holds an OperationGuard well past the +// worker's 50ms destroy delay. finishClose() drains in-flight operations with +// an untimed wait, so without the isClosing() poll in countRemaining() this +// scan would run to completion, block the destroy for its full duration, and +// report a count of a database that is being deleted. +const started = Date.now(); +let countError: unknown; +try { + original.getKeysCount({}); +} catch (error) { + countError = error; +} +const elapsed = Date.now() - started; + +if (!countError) + throw new Error(`Expected getKeysCount to abort, but it returned after ${elapsed}ms`); +if (!String(countError).includes('Database is closing')) throw countError; +if (elapsed >= 1000) { + throw new Error(`getKeysCount ran ${elapsed}ms; it should abort rather than finish the scan`); +} + +const destroyResult = await nextMessage(); +if (!destroyResult.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyResult)}`); + +if (registryStatus().some((entry) => entry.path === path)) { + throw new Error('Expected destroy to fully clear the registry entry'); +} + +await worker.terminate(); From 0d7f873ea53809121154e4cb246fa2605c9e9da5 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 25 Aug 2026 05:38:45 -0600 Subject: [PATCH 36/49] Keep close-time compaction cancellable through the full drain, guard GetCount against concurrent close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit finishClose() cleared compactCancelRequested right after the operationsInFlight drain, but an async compact() releases its OperationGuard at setup handoff and is not awaited until the closables sweep — so it can still be running after the drain returns, and clearing the token there left it able to stall teardown (and every concurrent open on the path) indefinitely. Keep the token armed for finishClose()'s whole duration instead, and have the close-time compact-on-close pass opt out via a new compactRange() `cancellable` param rather than relying on the shared flag being cleared. Transaction::GetCount now takes an OperationGuard and checks isClosing() before scanning: without it, finishClose()'s drain can return immediately and the closables sweep can roll back the transaction while the count scan is parked between rows, reading freed memory. Carries the in-progress PR #787 lifecycle repair plan describing the fuller atomic-admission fix these two changes are a first slice of. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013CvCKcGKG7vyhRts1Mv4Wh --- .pr787-lifecycle-repair-plan.md | 31 +++++++++++++++++++++++++ src/binding/database/db_descriptor.cpp | 16 +++++++------ src/binding/database/db_descriptor.h | 20 ++++++++++------ src/binding/transaction/transaction.cpp | 14 +++++++++++ 4 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 .pr787-lifecycle-repair-plan.md diff --git a/.pr787-lifecycle-repair-plan.md b/.pr787-lifecycle-repair-plan.md new file mode 100644 index 000000000..58ed001fe --- /dev/null +++ b/.pr787-lifecycle-repair-plan.md @@ -0,0 +1,31 @@ +# PR 787 lifecycle repair plan + +## Root cause and invariant + +Invariant: once descriptor close or handle cancellation begins, no new native operation may be admitted; teardown must not release native state until every operation admitted before that transition has drained. + +Current admission violates this invariant twice. `OperationGuard` increments `operationsInFlight` without serializing against `DBDescriptor::beginClose()`, so close can observe zero before a late increment. `AsyncWorkHandle::registerAsyncWork()` likewise increments without serializing against cancellation, and `waitForAsyncWorkCompletion()` continues after five seconds even if work remains. + +## Chosen repair + +1. Add a descriptor operation-admission mutex. `beginClose()` takes it while publishing `closing`; `OperationGuard` takes it while checking `closing` and incrementing `operationsInFlight`, and exposes whether admission succeeded. Existing callers stop touching native state when admission fails. +2. Make async-work admission atomic with cancellation under `AsyncWorkHandle::waitMutex`. Admission returns false after cancellation. Cancellation publishes under that mutex. Waiting becomes predicate-based and does not continue teardown while work remains. +3. Use a shared scoped async registration helper at N-API setup boundaries. It refuses admission after cancellation and transfers an admitted registration to queued state where needed. +4. Register transaction and iterator setup against their transaction handle, database handle, and descriptor before dereferencing teardown-owned state. Keep descriptor-wide operations pinned only for the duration in which native state is directly used; queued async state remains protected by the handle close drain. +5. Add deterministic setup/next race seams and subprocess tests covering database and transaction iterators, repeated idempotent iterator return, and forced destroy during `Next()`. +6. Address the remaining review feedback mechanically: retry destroy tombstone cleanup during shutdown, narrow the VT invariant text/use cached VT coordinates consistently, and diagnose retained benchmark data. + +## Approaches considered + +- Different layer — serialize only JavaScript `destroy()`/`open()` calls. Rejected: worker environments have separate JavaScript heaps, while the violated registry and native handles are process-global; a JS mutex cannot own the invariant across envs. +- Deeper cause — remove foreign forced close and require every environment to cooperatively close. Rejected: `destroy()` must invalidate all handles to the physical path, including handles in environments that may no longer service messages; cooperative teardown cannot guarantee physical deletion safety. +- Do less — retain the current per-call-site checks and increase/remove the five-second timeout. Rejected: timeout changes do not close either admission race, and a late registration can still begin after the waiter observed zero. +- Chosen — enforce atomic admission in the shared descriptor and handle primitives, then keep call sites responsible only for transferring valid claims. This is the only option that makes the close transition and every admission mutually ordered across worker threads without changing the public API. + +## Verification + +- Build the native debug binding first. +- Run focused destroy/iterator/transaction lifecycle tests repeatedly, including parallel subprocess batches. +- Run `pnpm check`, `pnpm test`, and `pnpm test:native` as the repository full gates. +- End-to-end route: the Node subprocess integration fixture uses real worker threads, a shared native descriptor, real RocksDB iterators/transactions, and physical destroy/reopen. +- Bug-proof route: the new next-race test must fail when the per-iterator mutex is removed; the admission-race tests must fail when admission/cancellation serialization is removed. diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index 47fa86ca4..af259dce0 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -428,16 +428,15 @@ void DBDescriptor::finishClose(bool destroying) { // Unbounded in-flight operations must abort once `closing` is published // rather than block this untimed wait for their full duration. A count // scan polls isClosing() itself; a manual compactRange() cannot, so it - // gets an explicit cancel token. + // gets an explicit cancel token. The token stays armed past this drain: + // an async compact() released its OperationGuard at setup handoff, so it + // is still running here and is not awaited until the closables sweep. this->compactCancelRequested.store(true); DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); uint32_t current; while ((current = this->operationsInFlight.load()) != 0) { this->operationsInFlight.wait(current); } - // Clear it before any further compaction runs below (compact-on-close), - // which must not be cancelled -- nothing external is waiting on it. - this->compactCancelRequested.store(false); DEBUG_LOG("%p DBDescriptor::close All operations complete \"%s\"\n", this, this->path.c_str()); // Drain the commit pipeline before flushing so its data is included in @@ -504,7 +503,7 @@ void DBDescriptor::finishClose(bool destroying) { } for (const auto& columnDesc : pinnedColumns) { if (columnDesc && columnDesc->column) { - this->compactRange(columnDesc->column.get(), nullptr, nullptr); + this->compactRange(columnDesc->column.get(), nullptr, nullptr, false, false); } } } @@ -2256,14 +2255,17 @@ rocksdb::Status DBDescriptor::compactRange( rocksdb::ColumnFamilyHandle* column, const rocksdb::Slice* start, const rocksdb::Slice* end, - bool bottommost + bool bottommost, + bool cancellable ) { std::lock_guard lock(this->compactMutex); DEBUG_LOG("%p DBDescriptor::compactRange Compacting range (bottommost=%d)\n", this, bottommost); rocksdb::CompactRangeOptions options; // Let a concurrent finishClose() interrupt this compaction rather than // wait out its full, unbounded duration; see compactCancelRequested. - options.canceled = &this->compactCancelRequested; + if (cancellable) { + options.canceled = &this->compactCancelRequested; + } if (bottommost) { // RocksDB defaults this to kIfHaveCompactionFilter, so with no compaction filter installed // the bottommost level is skipped — and that is where the bulk of the data sits. Rewriting diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index c0198a3a3..1a13a82e3 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -294,12 +294,15 @@ struct DBDescriptor final : public std::enable_shared_from_this { bool transactionLogsUnregistered = false; /** - * Set by finishClose() only while it is draining operationsInFlight, so an - * OperationGuard-holding compactRange() in progress on another thread can - * cancel its manual compaction and release the guard promptly instead of - * blocking the untimed drain wait for the compaction's full duration. - * Cleared once the drain completes so the close-time "compact on close" - * pass below always runs to completion. + * Armed by finishClose() for its whole duration so an in-progress manual + * compactRange() on another thread aborts instead of making teardown wait + * out its full, unbounded duration. It covers both shapes: compactSync() + * holds an OperationGuard and so blocks the drain, while an async compact() + * released its guard at setup handoff and is not awaited until the closables + * sweep -- clearing the token after the drain left that second one able to + * stall teardown (and every concurrent open on the path) indefinitely. + * Close-initiated compaction passes `cancellable = false` rather than + * clearing this, since nothing external is waiting on it. */ std::atomic compactCancelRequested{false}; @@ -621,7 +624,10 @@ struct DBDescriptor final : public std::enable_shared_from_this { rocksdb::ColumnFamilyHandle* column, const rocksdb::Slice* start, const rocksdb::Slice* end, - bool bottommost = false + bool bottommost = false, + // Close-initiated compaction opts out: nothing external is waiting on it, + // and `compactCancelRequested` stays armed for the whole of finishClose(). + bool cancellable = true ); }; diff --git a/src/binding/transaction/transaction.cpp b/src/binding/transaction/transaction.cpp index ef98174ab..af9948bb4 100644 --- a/src/binding/transaction/transaction.cpp +++ b/src/binding/transaction/transaction.cpp @@ -1050,6 +1050,20 @@ napi_value Transaction::GetCount(napi_env env, napi_callback_info info) { NAPI_METHOD_ARGV(1); UNWRAP_TRANSACTION_HANDLE("GetCount"); + // Without this claim finishClose()'s drain returns immediately and its + // closables sweep rolls back the transaction while the scan is parked + // between rows, leaving the iterator reading freed memory. + auto& txnDbHandle = (*txnHandle)->dbHandle; + if (!txnDbHandle || !txnDbHandle->descriptor) { + ::napi_throw_error(env, nullptr, "Get count failed: Database not open"); + NAPI_RETURN_UNDEFINED(); + } + OperationGuard operationGuard(txnDbHandle->descriptor); + if (txnDbHandle->descriptor->isClosing()) { + ::napi_throw_error(env, nullptr, "Get count failed: Database is closing"); + NAPI_RETURN_UNDEFINED(); + } + DBIteratorOptions itOptions; itOptions.initFromNapiObject(env, argv[0]); itOptions.values = false; From 35a99b0b15e86be737c48c29a5dcfb1b173e0e59 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 26 Aug 2026 07:35:56 -0600 Subject: [PATCH 37/49] Make async-work admission and cancellation mutually exclusive; wait unbounded for drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AsyncWorkHandle::registerAsyncWork() unconditionally incremented its counter with no serialization against cancelAllAsyncWork(), and waitForAsyncWorkCompletion() gave up after a hardcoded 5s even if work remained. Since Flush/Compact/Clear/async Get in database.cpp only hold the descriptor's operationsInFlight guard through synchronous setup (not through the queued execute callback), the 5s bound let DBDescriptor::finishClose() reach this->db.reset() while a slow flush() (legitimately waiting out a write stall per AGENTS.md invariant 16, which has no bound) was still executing against it — a genuine use-after-free, not a theoretical one. registerAsyncWork()/cancelAllAsyncWork() now share waitMutex so admission and cancellation can never interleave: a registration either fully lands before cancellation publishes, or is refused. waitForAsyncWorkCompletion() is now unbounded, matching the existing unbounded operationsInFlight wait pattern elsewhere in db_descriptor.cpp. Every registerAsyncWork() call site (database.cpp Clear/Compact/Flush/Get, backup.cpp's shared queueBackupWork, checkpoint.cpp, transaction.cpp's two Commit() paths) is wired through a new admitAsyncWorkOrReject() helper that rejects the already-constructed promise and tears down cleanly on refusal instead of proceeding into a closing handle. ScopedAsyncWorkRegistration (transaction_handle.cpp, used for cross-column-family transactional reads) now tracks admission via ok() so its destructor can't underflow the count on a refused registration, and both of its call sites in TransactionHandle::get() check ok() explicitly rather than relying on the (currently-true but unenforced) correlation with isCancelled(). backup_stream.cpp's registration is left unchecked, with an explanatory comment: its operationsInFlight claim is held through the whole async execution already, so it can't hit refusal in practice. Corrected the README's lifecycleWaitSeconds doc: it said "Total maximum time," which contradicted the existing note that destroy()/shutdown()'s wait for in-flight backups/checkpoints is intentionally unbounded — reworded to clarify it only bounds the wait for a conflicting lifecycle op on the same path. Deleted .pr787-lifecycle-repair-plan.md (superseded by this commit and AGENTS.md invariant 17, which documents the fix). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019nt6hrpR4pnotKkZ8AUVv5 --- .pr787-lifecycle-repair-plan.md | 31 ----- AGENTS.md | 21 +++ README.md | 9 +- src/binding/database/backup.cpp | 4 +- src/binding/database/backup_stream.cpp | 9 +- src/binding/database/checkpoint.cpp | 4 +- src/binding/database/database.cpp | 18 ++- src/binding/napi/async.h | 123 ++++++++++++------ src/binding/transaction/transaction.cpp | 24 +++- .../transaction/transaction_handle.cpp | 45 ++++--- 10 files changed, 183 insertions(+), 105 deletions(-) delete mode 100644 .pr787-lifecycle-repair-plan.md diff --git a/.pr787-lifecycle-repair-plan.md b/.pr787-lifecycle-repair-plan.md deleted file mode 100644 index 58ed001fe..000000000 --- a/.pr787-lifecycle-repair-plan.md +++ /dev/null @@ -1,31 +0,0 @@ -# PR 787 lifecycle repair plan - -## Root cause and invariant - -Invariant: once descriptor close or handle cancellation begins, no new native operation may be admitted; teardown must not release native state until every operation admitted before that transition has drained. - -Current admission violates this invariant twice. `OperationGuard` increments `operationsInFlight` without serializing against `DBDescriptor::beginClose()`, so close can observe zero before a late increment. `AsyncWorkHandle::registerAsyncWork()` likewise increments without serializing against cancellation, and `waitForAsyncWorkCompletion()` continues after five seconds even if work remains. - -## Chosen repair - -1. Add a descriptor operation-admission mutex. `beginClose()` takes it while publishing `closing`; `OperationGuard` takes it while checking `closing` and incrementing `operationsInFlight`, and exposes whether admission succeeded. Existing callers stop touching native state when admission fails. -2. Make async-work admission atomic with cancellation under `AsyncWorkHandle::waitMutex`. Admission returns false after cancellation. Cancellation publishes under that mutex. Waiting becomes predicate-based and does not continue teardown while work remains. -3. Use a shared scoped async registration helper at N-API setup boundaries. It refuses admission after cancellation and transfers an admitted registration to queued state where needed. -4. Register transaction and iterator setup against their transaction handle, database handle, and descriptor before dereferencing teardown-owned state. Keep descriptor-wide operations pinned only for the duration in which native state is directly used; queued async state remains protected by the handle close drain. -5. Add deterministic setup/next race seams and subprocess tests covering database and transaction iterators, repeated idempotent iterator return, and forced destroy during `Next()`. -6. Address the remaining review feedback mechanically: retry destroy tombstone cleanup during shutdown, narrow the VT invariant text/use cached VT coordinates consistently, and diagnose retained benchmark data. - -## Approaches considered - -- Different layer — serialize only JavaScript `destroy()`/`open()` calls. Rejected: worker environments have separate JavaScript heaps, while the violated registry and native handles are process-global; a JS mutex cannot own the invariant across envs. -- Deeper cause — remove foreign forced close and require every environment to cooperatively close. Rejected: `destroy()` must invalidate all handles to the physical path, including handles in environments that may no longer service messages; cooperative teardown cannot guarantee physical deletion safety. -- Do less — retain the current per-call-site checks and increase/remove the five-second timeout. Rejected: timeout changes do not close either admission race, and a late registration can still begin after the waiter observed zero. -- Chosen — enforce atomic admission in the shared descriptor and handle primitives, then keep call sites responsible only for transferring valid claims. This is the only option that makes the close transition and every admission mutually ordered across worker threads without changing the public API. - -## Verification - -- Build the native debug binding first. -- Run focused destroy/iterator/transaction lifecycle tests repeatedly, including parallel subprocess batches. -- Run `pnpm check`, `pnpm test`, and `pnpm test:native` as the repository full gates. -- End-to-end route: the Node subprocess integration fixture uses real worker threads, a shared native descriptor, real RocksDB iterators/transactions, and physical destroy/reopen. -- Bug-proof route: the new next-race test must fail when the per-iterator mutex is removed; the admission-race tests must fail when admission/cancellation serialization is removed. diff --git a/AGENTS.md b/AGENTS.md index d8c7dc4d4..5988f4a82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -646,6 +646,27 @@ sufficient (env teardown does not honor tsfn acquire counts); see is the descriptor's single `CommitWorker` thread (see "Commit execution" above), which dispatches every `Transaction.commit()` in order — so opting a flush into a stall queues up every commit behind it, including ones from callers that never touched flush. +17. **Async-work admission and cancellation share one mutex; the drain that follows must never time + out**: `AsyncWorkHandle` (`napi/async.h`) tracks in-flight async work per `DBHandle`/ + `TransactionHandle`. `registerAsyncWork()` and `cancelAllAsyncWork()` both take `waitMutex`, so a + registration that races a close either lands (and is counted) before cancellation publishes, or + is refused — there is no window where it is admitted after `waitForAsyncWorkCompletion()` has + already observed the count at zero. Refusal returns `false`; every call site (the shared + `admitAsyncWorkOrReject()` helper, or `ScopedAsyncWorkRegistration::ok()` for the RAII + cross-handle case in `transaction_handle.cpp`) must fail the operation — reject the + already-constructed promise and touch no native state — rather than proceed with work nothing is + tracking anymore. `waitForAsyncWorkCompletion()` itself has no timeout: `DBHandle::close()` / + `TransactionHandle::close()` call it immediately before releasing the `rocksdb::DB`, column + family, or transaction that admitted work may still be using, and a flush legitimately waiting + out a write stall (invariant 16) can run far longer than any fixed bound. A bounded wait that + gives up anyway — the previous 5-second default — let `finishClose()` reach `this->db.reset()` + while a flush was still executing against it, a genuine use-after-free. `database.cpp`'s + `Flush`/`Compact`/`Clear`/async `Get` rely entirely on this drain for safety: their + `OperationGuard` from `ACQUIRE_OPERATIONS_LOCK()` covers only the synchronous setup, not the + queued execute callback. Backup/checkpoint/backup-stream additionally hold the descriptor's + `operationsInFlight` claim through their whole async execution (the pinning pattern from + invariant 9), so for those this drain is defense in depth rather than the only thing preventing + a use-after-free. 17. **An env's pending transactions are reaped by its cleanup hook — never by `DBHandle::close()`**: `transactionAdd` stores a strong `shared_ptr` in the process-global diff --git a/README.md b/README.md index 45a490368..0f0a027b7 100644 --- a/README.md +++ b/README.md @@ -187,9 +187,12 @@ Sets global database settings. Defaults to 32MB. Set to `0` (zero) disables block cache for future opened databases. Existing block cache for any opened databases is resized immediately. Negative values throw an error. - `compactOnClose: boolean` When `true`, compacts the database on close. Defaults to `false`. - - `lifecycleWaitSeconds: number` Total maximum time a synchronous open, destroy, or shutdown waits for - another lifecycle operation before throwing a retryable timeout error. Defaults to `30` seconds - and must be a positive integer. + - `lifecycleWaitSeconds: number` How long a synchronous open, destroy, or shutdown waits for a + _conflicting_ lifecycle operation already in progress on the same path (e.g. another open or + close) before throwing a retryable timeout error. It does not bound the separate, intentionally + unbounded wait that `destroy()`/`shutdown()` make for in-flight backups, checkpoints, or other + async work still using the database — see [`db.destroy()`](#dbdestroy-void). Defaults to `30` + seconds and must be a positive integer. - `verificationTableEntries: number` The number of slots in the process-global [Verification Table](#verification-table). Each slot is 8 bytes, so the default of `131072` (128K) slots is 1 MB. Set to `0` to disable the verification table. This must be configured diff --git a/src/binding/database/backup.cpp b/src/binding/database/backup.cpp index a4a663010..6c7b14b05 100644 --- a/src/binding/database/backup.cpp +++ b/src/binding/database/backup.cpp @@ -165,7 +165,9 @@ static napi_value queueBackupWork( NAPI_STATUS_THROWS(::napi_create_async_work(env, nullptr, name, execute, complete, state, &state->asyncWork)); if (registerWork && state->handle) { - state->handle->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, state->handle.get(), state, "Database is closing")) { + NAPI_RETURN_UNDEFINED(); + } } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); diff --git a/src/binding/database/backup_stream.cpp b/src/binding/database/backup_stream.cpp index 6eb8c6803..054f7e46e 100644 --- a/src/binding/database/backup_stream.cpp +++ b/src/binding/database/backup_stream.cpp @@ -718,7 +718,14 @@ napi_value Database::BackupStream(napi_env env, napi_callback_info info) { &state->asyncWork )); - (*dbHandle)->registerAsyncWork(); + // The operationsInFlight claim taken above is held for the whole stream + // (released at the end of backupStreamExecute, not here), so it already + // rules out a concurrent cancelAllAsyncWork() racing this registration — + // finishClose()'s first (unbounded) wait cannot reach the closables sweep + // that would call it until this claim releases. Admission is therefore + // guaranteed to succeed; discard the result rather than threading the + // tsfn's ownership through the same reject path as the simpler async ops. + (void)(*dbHandle)->registerAsyncWork(); // On a queue failure the claim rolls the counter back (execute never runs). // The state/tsfn leak on this rare N-API failure path matches the existing diff --git a/src/binding/database/checkpoint.cpp b/src/binding/database/checkpoint.cpp index ddf8a1fc2..588fc2dc7 100644 --- a/src/binding/database/checkpoint.cpp +++ b/src/binding/database/checkpoint.cpp @@ -194,7 +194,9 @@ napi_value Database::CreateCheckpoint(napi_env env, napi_callback_info info) { &state->asyncWork )); - (*dbHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*dbHandle).get(), state, "Database is closing")) { + NAPI_RETURN_UNDEFINED(); + } // On a queue failure the claim above rolls the counter back (execute never // runs); the state leak on this rare N-API failure path matches the existing diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 19c9f7fbe..2bfd4ba26 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -125,7 +125,9 @@ static napi_value doClear(napi_env env, napi_callback_info info, const char* fai )); // Register the async work with the database handle - (*dbHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*dbHandle).get(), state, "Database is closing")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); @@ -366,7 +368,9 @@ napi_value Database::Compact(napi_env env, napi_callback_info info) { &state->asyncWork )); - (*dbHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*dbHandle).get(), state, "Database is closing")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); @@ -725,7 +729,9 @@ napi_value Database::Flush(napi_env env, napi_callback_info info) { &state->asyncWork )); - (*dbHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*dbHandle).get(), state, "Database is closing")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); @@ -868,7 +874,11 @@ napi_value Database::Get(napi_env env, napi_callback_info info) { // performs at the end of the execute handler. Without it the count goes negative, // so close() does not wait for this read and the worker dereferences a descriptor // that close() has already reset. - (*dbHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*dbHandle).get(), state, "Database is closing")) { + napi_value returnStatus; + NAPI_STATUS_THROWS(::napi_create_uint32(env, 1, &returnStatus)); + return returnStatus; + } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); diff --git a/src/binding/napi/async.h b/src/binding/napi/async.h index bae24ccf7..b949711aa 100644 --- a/src/binding/napi/async.h +++ b/src/binding/napi/async.h @@ -141,67 +141,69 @@ struct AsyncWorkHandle { std::mutex waitMutex; std::condition_variable asyncWorkComplete; - void registerAsyncWork() { + /** + * Admits one unit of async work. Returns false, without incrementing the + * count, once cancellation has been published — the caller must not queue + * the async work or otherwise touch native state in that case. Admission + * and `cancelAllAsyncWork()` share `waitMutex` so the two can never + * interleave: either this call is fully visible to the wait below before + * cancellation publishes, or it is refused. Without that, a registration + * racing a close's cancel+wait could land after the wait already observed + * zero, letting new work run concurrently with (or after) the native + * state that close() goes on to release. + */ + [[nodiscard]] bool registerAsyncWork() { + std::lock_guard lock(this->waitMutex); + if (this->cancelled.load()) { + return false; + } ++this->activeAsyncWorkCount; + return true; } void unregisterAsyncWork() { + std::lock_guard lock(this->waitMutex); auto activeAsyncWorkCount = --this->activeAsyncWorkCount; if (activeAsyncWorkCount > 0) { DEBUG_LOG("%p AsyncWorkHandle::unregisterAsyncWork Still have %u active async work tasks\n", this, activeAsyncWorkCount); } else if (activeAsyncWorkCount == 0) { DEBUG_LOG("%p AsyncWorkHandle::unregisterAsyncWork All async work has completed, notifying\n", this); - this->asyncWorkComplete.notify_one(); + this->asyncWorkComplete.notify_all(); } } void cancelAllAsyncWork() { + std::lock_guard lock(this->waitMutex); this->cancelled.store(true); } /** - * Waits for in-flight async work to finish. Returns true when the count - * actually reached zero, false when the timeout expired with work still - * running — callers that destroy state the work is using MUST check it. + * Blocks until every admitted unit of async work has completed. This must + * not time out: the caller is about to release native state (a + * `rocksdb::DB`, a column family, a transaction) that admitted work may + * still be using. A flush legitimately waiting out a write stall (see + * AGENTS.md invariant 16) can run far longer than any fixed bound, and a + * bounded wait that gives up anyway turns into a use-after-free once the + * caller proceeds to tear down that state. + * + * Supersedes the bounded, bool-returning version this replaced (a 5s + * timeout with a "leak instead of free" fallback in + * `TransactionHandle::close()`) — that was a deliberate stopgap tracked as + * HarperFast/rocksdb-js#784, this unbounded wait *is* #784. Every caller + * can now assume the drain always completes; there is no timed-out case + * left to handle. */ - bool waitForAsyncWorkCompletion( - std::chrono::milliseconds timeout = std::chrono::milliseconds(5000) - ) { - auto start = std::chrono::steady_clock::now(); - const auto pollInterval = std::chrono::milliseconds(10); + void waitForAsyncWorkCompletion() { std::unique_lock lock(this->waitMutex); - auto activeAsyncWorkCount = this->activeAsyncWorkCount.load(); - - if (activeAsyncWorkCount == 0) { + if (this->activeAsyncWorkCount.load() == 0) { DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion no async work to wait for\n", this); - return true; - } - - while (activeAsyncWorkCount > 0) { - auto elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - start); - if (elapsed >= timeout) { - DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion timeout waiting for async work completion, %u items remaining\n", this, activeAsyncWorkCount); - return false; - } - - auto remainingTime = timeout - elapsed; - auto waitTime = std::min(pollInterval, remainingTime); - - DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion waiting for %u active work items\n", this, activeAsyncWorkCount); - - bool completed = this->asyncWorkComplete.wait_for(lock, waitTime, [this] { - return this->activeAsyncWorkCount.load() == 0; - }); - - if (completed) { - DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion all async work execution completed\n", this); - return true; - } - - activeAsyncWorkCount = this->activeAsyncWorkCount.load(); + return; } - - return true; + DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion waiting for active work items\n", this); + this->asyncWorkComplete.wait(lock, [this] { + return this->activeAsyncWorkCount.load() == 0; + }); + DEBUG_LOG("%p AsyncWorkHandle::waitForAsyncWorkCompletion all async work execution completed\n", this); } bool isCancelled() const { @@ -209,10 +211,51 @@ struct AsyncWorkHandle { } void resetCancelled() { + std::lock_guard lock(this->waitMutex); this->cancelled.store(false); } }; +/** + * Admits `state`'s async work onto `handle`. On success, returns true and the + * caller proceeds to `napi_queue_async_work()` as usual. On refusal + * (cancellation already published by a concurrent close), tears down the + * async work object created for `state` (if any) and the promise references + * already captured on it, rejects the promise with `message`, deletes + * `state`, and returns false — the caller must return to JS immediately + * without dereferencing any native state `state` was set up to use. + */ +template +bool admitAsyncWorkOrReject(napi_env env, AsyncWorkHandle* handle, State* state, const char* message) { + if (handle->registerAsyncWork()) { + return true; + } + + // Nothing was incremented, so mark completed before the destructor's + // signalExecuteCompleted() runs — otherwise it would call + // unregisterAsyncWork() for a registration that never succeeded. + state->completed.store(true); + + if (state->asyncWork) { + if (::napi_delete_async_work(env, state->asyncWork) == napi_ok) { + state->asyncWork = nullptr; + } + } + + napi_value error = nullptr; + napi_value messageValue; + if (::napi_create_string_utf8(env, message, NAPI_AUTO_LENGTH, &messageValue) == napi_ok) { + ::napi_create_error(env, nullptr, messageValue, &error); + } + if (error == nullptr) { + ::napi_get_undefined(env, &error); + } + state->callReject(error); + + delete state; + return false; +} + } // namespace rocksdb_js #endif diff --git a/src/binding/transaction/transaction.cpp b/src/binding/transaction/transaction.cpp index af9948bb4..9e8dc246b 100644 --- a/src/binding/transaction/transaction.cpp +++ b/src/binding/transaction/transaction.cpp @@ -789,8 +789,24 @@ napi_value Transaction::Commit(napi_env env, napi_callback_info info) { // below rather than re-creating a tsfn the close will never release. NAPI_STATUS_THROWS(descriptor->registerCommitCompletion(env, commitCompletionCallJs, completionsClosed)); if (!completionsClosed) { - // register the commit with the transaction handle so close() can wait - (*txnHandle)->registerAsyncWork(); + // Register the commit with the transaction handle so close() can wait. + // Refusal means a concurrent close() already published cancellation (and, + // by the same happens-before edge, already forced state to Aborted) — + // undo the completion registration just above and reject rather than + // dispatching into a transaction that is being torn down. + if (!(*txnHandle)->registerAsyncWork()) { + descriptor->finishCommitCompletion(env); + // Nothing was incremented on the handle, so mark completed before + // the destructor's signalExecuteCompleted() runs — otherwise it + // would call unregisterAsyncWork() for a registration that never + // succeeded. + state->completed.store(true); + napi_value error; + rocksdb_js::createJSError(env, "ERR_TRANSACTION_CLOSING", "Transaction is closing", error); + state->callReject(error); + delete state; + NAPI_RETURN_UNDEFINED(); + } // Commit-lane stage: RocksDB commit, then marshal the completion // back to the originating env. @@ -871,7 +887,9 @@ napi_value Transaction::Commit(napi_env env, napi_callback_info info) { )); // register the async work with the transaction handle - (*txnHandle)->registerAsyncWork(); + if (!admitAsyncWorkOrReject(env, (*txnHandle).get(), state, "Transaction is closing")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); diff --git a/src/binding/transaction/transaction_handle.cpp b/src/binding/transaction/transaction_handle.cpp index 84637a739..9003b3d73 100644 --- a/src/binding/transaction/transaction_handle.cpp +++ b/src/binding/transaction/transaction_handle.cpp @@ -19,19 +19,28 @@ namespace { * handle's column descriptor. This bridges the gap between the caller's open * check and the transaction's async-work registration without making the * worker access a concurrently closing DBHandle. + * + * Admission can be refused if `handle`'s cancellation was already published + * (a concurrent close); `admitted` tracks that so the destructor only + * unregisters a claim it actually holds — unregistering an admission that + * never succeeded would underflow the target's active-work count and could + * make a later close() stop waiting while real work is still outstanding. + * Callers must check `ok()` after construction and treat a false result the + * same as the target already being closed. */ struct ScopedAsyncWorkRegistration { AsyncWorkHandle* handle; + bool admitted; explicit ScopedAsyncWorkRegistration(AsyncWorkHandle* handle) - : handle(handle) { + : handle(handle), admitted(false) { if (this->handle) { - this->handle->registerAsyncWork(); + this->admitted = this->handle->registerAsyncWork(); } } ~ScopedAsyncWorkRegistration() { - if (this->handle) { + if (this->handle && this->admitted) { this->handle->unregisterAsyncWork(); } } @@ -39,6 +48,10 @@ struct ScopedAsyncWorkRegistration { ScopedAsyncWorkRegistration(const ScopedAsyncWorkRegistration&) = delete; ScopedAsyncWorkRegistration& operator=(const ScopedAsyncWorkRegistration&) = delete; + bool ok() const { + return !this->handle || this->admitted; + } + void release() { this->handle = nullptr; } @@ -317,8 +330,12 @@ void TransactionHandle::close() { // Drain BEFORE touching anything the in-flight work owns. Nothing below is // safe while a commit is still executing: `state` feeds the commitAborted() // decision, releaseIntent() mutates VT state the commit is using, and - // `delete txn` hands RocksDB a dangling transaction. - const bool drained = this->waitForAsyncWorkCompletion(); + // `delete txn` hands RocksDB a dangling transaction. waitForAsyncWorkCompletion() + // is unbounded (HarperFast/rocksdb-js#784) and cancelAllAsyncWork() above shares + // a mutex with registerAsyncWork(), so this always drains to zero — there is no + // timed-out case left to handle here; an earlier version of this close() leaked + // the transaction on a 5s drain timeout as a stopgap for exactly this window. + this->waitForAsyncWorkCompletion(); // Test seam: widen the PATH A vs PATH B race window (see txnCloseTestDelayMs). // This window is real in production (PATH B fires after waitForAsyncWorkCompletion @@ -329,20 +346,6 @@ void TransactionHandle::close() { std::this_thread::sleep_for(std::chrono::milliseconds(closeDelayMs)); } - if (!drained) { - // The drain timed out with work still executing against `txn` (e.g. a - // worker env torn down during a slow commit). Destroying now would free - // a transaction RocksDB is still using and could mark the log aborted - // while the data commit goes on to succeed — so deliberately leak - // instead. The in-flight commit owns its own cleanup (it releases VT - // intents and resolves the log position on completion); this close must - // not steal it. A leaked transaction is recoverable; a use-after-free - // and a log/data disagreement are not. The complete admission-and-drain - // contract that removes this window is HarperFast/rocksdb-js#784. - DEBUG_LOG("%p TransactionHandle::close async work still in flight after drain timeout; leaking txn rather than freeing it\n", this); - return; - } - // Only now that no native work can be running: settle the final state. if (this->state == TransactionState::Pending || this->state == TransactionState::Committing) { this->state = TransactionState::Aborted; @@ -418,7 +421,7 @@ napi_value TransactionHandle::get( // the middle of this setup. Async fallback transfers this registration to its // state; synchronous and failed setup paths release it on return. ScopedAsyncWorkRegistration transactionRegistration(this); - if (this->isCancelled() || !this->txn) { + if (!transactionRegistration.ok() || this->isCancelled() || !this->txn) { ::napi_throw_error(env, nullptr, "Transaction is closed"); return nullptr; } @@ -441,7 +444,7 @@ napi_value TransactionHandle::get( // that handle while copying its descriptor so a concurrent close cannot reset // columnDescriptor in the gap between the caller's open check and this read. ScopedAsyncWorkRegistration targetHandleRegistration(dbHandleOverride.get()); - if (dbHandleOverride && dbHandle->isCancelled()) { + if (dbHandleOverride && (!targetHandleRegistration.ok() || dbHandle->isCancelled())) { ::napi_throw_error(env, nullptr, "Database closed during transaction get operation"); return nullptr; } From 4459c419f26eafbfc5ee218dea96f230d77a2667 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 26 Aug 2026 08:09:10 -0600 Subject: [PATCH 38/49] fix(async): release admitted async-work claims on a queue failure Gemini + Harper-domain pre-push review (round 1) found that every admitAsyncWorkOrReject() call site followed a successful admission with a bare NAPI_STATUS_THROWS(::napi_queue_async_work(...)). On the rare napi_queue_async_work() failure that macro throws and returns immediately, leaking `state` with its AsyncWorkHandle registration still counted. Since waitForAsyncWorkCompletion() is now unbounded (this branch's whole point), that stuck count blocks the handle's close forever, which blocks every later OpenDB() for its path -- worse than the leak it fixed. Added queueAsyncWorkOrReject() alongside admitAsyncWorkOrReject() in async.h: releases the admitted claim via signalExecuteCompleted(), deletes the async work object, rejects the promise, deletes state. Takes an `admitted` flag for backup.cpp's queueBackupWork(), whose registration is conditional on its registerWork/state->handle parameters -- unregistering an admission that never happened would underflow the count the same way. backup_stream.cpp's AsyncBackupStreamState is refcounted (acquire()/ release(), shared with an N-API tsfn) rather than a plain heap object, so the generic helper's `delete state` would double-free against tsfnFinalize()'s later release(). Wrote the queue-failure cleanup by hand there instead, mirroring backupStreamComplete()'s existing teardown sequence (delete async work, release the tsfn, drop the descriptor pin, reject, release the constructor's own ref). Also asserted the invariant its "(void)registerAsyncWork()" comment already claimed (registration there is guaranteed by the operationsInFlight claim held for the whole stream) -- the review's other nit, that the claim was load-bearing but unenforced. transaction_handle.cpp's async Get fallback was checked and left alone: its queue call runs before a still-in-scope PendingAsyncState RAII guard is released, so a queue failure already unwinds and deletes state correctly. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019nt6hrpR4pnotKkZ8AUVv5 --- src/binding/database/backup.cpp | 6 ++- src/binding/database/backup_stream.cpp | 58 +++++++++++++++++++++---- src/binding/database/checkpoint.cpp | 7 ++- src/binding/database/database.cpp | 18 ++++++-- src/binding/napi/async.h | 53 ++++++++++++++++++++++ src/binding/transaction/transaction.cpp | 4 +- 6 files changed, 128 insertions(+), 18 deletions(-) diff --git a/src/binding/database/backup.cpp b/src/binding/database/backup.cpp index 6c7b14b05..3229ce5df 100644 --- a/src/binding/database/backup.cpp +++ b/src/binding/database/backup.cpp @@ -164,13 +164,17 @@ static napi_value queueBackupWork( NAPI_STATUS_THROWS(::napi_create_async_work(env, nullptr, name, execute, complete, state, &state->asyncWork)); + bool admitted = false; if (registerWork && state->handle) { if (!admitAsyncWorkOrReject(env, state->handle.get(), state, "Database is closing")) { NAPI_RETURN_UNDEFINED(); } + admitted = true; } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue backup work", admitted)) { + NAPI_RETURN_UNDEFINED(); + } if (queued) *queued = true; NAPI_RETURN_UNDEFINED(); diff --git a/src/binding/database/backup_stream.cpp b/src/binding/database/backup_stream.cpp index 054f7e46e..17042e84e 100644 --- a/src/binding/database/backup_stream.cpp +++ b/src/binding/database/backup_stream.cpp @@ -718,19 +718,61 @@ napi_value Database::BackupStream(napi_env env, napi_callback_info info) { &state->asyncWork )); + // This state is refcounted (not a plain `delete`), shared with the tsfn + // above, so queueAsyncWorkOrReject()'s generic `delete state` would + // double-free against tsfnFinalize()'s later release(). Mirror + // backupStreamComplete()'s cleanup by hand instead for both failure exits + // below, since execute/complete never run to do it themselves: delete the + // async work object, release the tsfn (tsfnFinalize drops its ref), drop + // the descriptor pin, reject, and release the constructor's own ref. + // `admitted` selects how the AsyncWorkHandle claim is released: a queue + // failure must decrement a real registration (signalExecuteCompleted()), + // while a refused registration never incremented anything and must only + // be marked completed -- decrementing there would underflow the count. + auto rejectAndCleanup = [&](bool admitted, const char* message) { + if (admitted) { + state->signalExecuteCompleted(); + } else { + state->completed.store(true); + } + state->deleteAsyncWork(); + if (state->tsfn != nullptr) { + ::napi_release_threadsafe_function(state->tsfn, napi_tsfn_release); + state->tsfn = nullptr; + } + state->releaseDescriptor(); + + napi_value error = nullptr; + napi_value messageValue; + if (::napi_create_string_utf8(env, message, NAPI_AUTO_LENGTH, &messageValue) == napi_ok) { + ::napi_create_error(env, nullptr, messageValue, &error); + } + if (error == nullptr) { + ::napi_get_undefined(env, &error); + } + state->callReject(error); + state->release(); + }; + // The operationsInFlight claim taken above is held for the whole stream // (released at the end of backupStreamExecute, not here), so it already // rules out a concurrent cancelAllAsyncWork() racing this registration — // finishClose()'s first (unbounded) wait cannot reach the closables sweep // that would call it until this claim releases. Admission is therefore - // guaranteed to succeed; discard the result rather than threading the - // tsfn's ownership through the same reject path as the simpler async ops. - (void)(*dbHandle)->registerAsyncWork(); - - // On a queue failure the claim rolls the counter back (execute never runs). - // The state/tsfn leak on this rare N-API failure path matches the existing - // async methods (e.g. Database::Backup). - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + // expected to always succeed, but unlike the simpler async ops this state + // can't route refusal through admitAsyncWorkOrReject() (its `delete state` + // would double-free here) -- so on the vanishingly unlikely chance a future + // change adds another cancelAllAsyncWork() path for this handle, reject + // properly instead of silently proceeding with an unregistered stream. + if (!(*dbHandle)->registerAsyncWork()) { + rejectAndCleanup(false, "Database is closing"); + NAPI_RETURN_UNDEFINED(); + } + + if (::napi_queue_async_work(env, state->asyncWork) != napi_ok) { + rejectAndCleanup(true, "Failed to queue backup stream work"); + NAPI_RETURN_UNDEFINED(); + } // The worker now owns the in-flight decrement (end of execute). handedOff = true; diff --git a/src/binding/database/checkpoint.cpp b/src/binding/database/checkpoint.cpp index 588fc2dc7..a30d2699a 100644 --- a/src/binding/database/checkpoint.cpp +++ b/src/binding/database/checkpoint.cpp @@ -198,10 +198,9 @@ napi_value Database::CreateCheckpoint(napi_env env, napi_callback_info info) { NAPI_RETURN_UNDEFINED(); } - // On a queue failure the claim above rolls the counter back (execute never - // runs); the state leak on this rare N-API failure path matches the existing - // async methods (e.g. Database::Backup). - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue checkpoint work")) { + NAPI_RETURN_UNDEFINED(); + } // The worker now owns the in-flight decrement (end of execute); stop the // claim from releasing it here. diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 2bfd4ba26..f26c531d7 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -129,7 +129,9 @@ static napi_value doClear(napi_env env, napi_callback_info info, const char* fai NAPI_RETURN_UNDEFINED(); } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue clear work")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_RETURN_UNDEFINED(); } @@ -372,7 +374,9 @@ napi_value Database::Compact(napi_env env, napi_callback_info info) { NAPI_RETURN_UNDEFINED(); } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue compact work")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_RETURN_UNDEFINED(); } @@ -733,7 +737,9 @@ napi_value Database::Flush(napi_env env, napi_callback_info info) { NAPI_RETURN_UNDEFINED(); } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue flush work")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_RETURN_UNDEFINED(); } @@ -880,7 +886,11 @@ napi_value Database::Get(napi_env env, napi_callback_info info) { return returnStatus; } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue get work")) { + napi_value returnStatus; + NAPI_STATUS_THROWS(::napi_create_uint32(env, 1, &returnStatus)); + return returnStatus; + } napi_value returnStatus; NAPI_STATUS_THROWS(::napi_create_uint32(env, 1, &returnStatus)); diff --git a/src/binding/napi/async.h b/src/binding/napi/async.h index b949711aa..83be13e26 100644 --- a/src/binding/napi/async.h +++ b/src/binding/napi/async.h @@ -256,6 +256,59 @@ bool admitAsyncWorkOrReject(napi_env env, AsyncWorkHandle* handle, State* state, return false; } +/** + * Queues `state`'s async work after a successful `admitAsyncWorkOrReject()`. + * On success, returns true and the caller returns its normal pending value. + * On the rare `napi_queue_async_work()` failure, the admission above already + * incremented the target's active-work count with nothing left to run it + * down — a bare `NAPI_STATUS_THROWS` here would return without releasing + * that claim, leaking `state` and leaving the count permanently off by one. + * Since a bounded `waitForAsyncWorkCompletion()` no longer exists to paper + * over a stuck count (see the note on that method), a single stranded claim + * blocks that handle's close forever, which in turn blocks every later + * `OpenDB()` for its path. This releases the claim via + * `signalExecuteCompleted()` (unlike `admitAsyncWorkOrReject()`'s refusal + * path, this one actually decrements — the claim was real), deletes the + * async work object and `state`, rejects the promise with `message`, and + * returns false. + * + * `admitted` must be false when the caller skipped `admitAsyncWorkOrReject()` + * for this `state` (e.g. backup.cpp's `queueBackupWork(..., registerWork)` + * with `registerWork` false) — otherwise `signalExecuteCompleted()` would + * unregister a claim that was never taken, underflowing the count. + */ +template +bool queueAsyncWorkOrReject(napi_env env, State* state, const char* message, bool admitted = true) { + if (::napi_queue_async_work(env, state->asyncWork) == napi_ok) { + return true; + } + + if (admitted) { + state->signalExecuteCompleted(); + } else { + state->completed.store(true); + } + + if (state->asyncWork) { + if (::napi_delete_async_work(env, state->asyncWork) == napi_ok) { + state->asyncWork = nullptr; + } + } + + napi_value error = nullptr; + napi_value messageValue; + if (::napi_create_string_utf8(env, message, NAPI_AUTO_LENGTH, &messageValue) == napi_ok) { + ::napi_create_error(env, nullptr, messageValue, &error); + } + if (error == nullptr) { + ::napi_get_undefined(env, &error); + } + state->callReject(error); + + delete state; + return false; +} + } // namespace rocksdb_js #endif diff --git a/src/binding/transaction/transaction.cpp b/src/binding/transaction/transaction.cpp index 9e8dc246b..797244b7f 100644 --- a/src/binding/transaction/transaction.cpp +++ b/src/binding/transaction/transaction.cpp @@ -891,7 +891,9 @@ napi_value Transaction::Commit(napi_env env, napi_callback_info info) { NAPI_RETURN_UNDEFINED(); } - NAPI_STATUS_THROWS(::napi_queue_async_work(env, state->asyncWork)); + if (!queueAsyncWorkOrReject(env, state, "Failed to queue commit work")) { + NAPI_RETURN_UNDEFINED(); + } NAPI_RETURN_UNDEFINED(); } From 91046f37c4b8673a0bf09c7a09c79a410b103c8b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 26 Aug 2026 09:36:54 -0600 Subject: [PATCH 39/49] fix(close): cancel compaction before async drain Co-Authored-By: GPT-5 Codex --- src/binding/database/db_descriptor.cpp | 7 ++----- src/binding/database/db_descriptor.h | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index af259dce0..d81711141 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -427,11 +427,8 @@ void DBDescriptor::finishClose(bool destroying) { // Existing operations will decrement operationsInFlight and notify us when done. // Unbounded in-flight operations must abort once `closing` is published // rather than block this untimed wait for their full duration. A count - // scan polls isClosing() itself; a manual compactRange() cannot, so it - // gets an explicit cancel token. The token stays armed past this drain: - // an async compact() released its OperationGuard at setup handoff, so it - // is still running here and is not awaited until the closables sweep. - this->compactCancelRequested.store(true); + // scan polls isClosing() itself; a manual compactRange() uses the cancel + // token armed by beginClose(), before DBHandle::close() starts its drain. DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); uint32_t current; while ((current = this->operationsInFlight.load()) != 0) { diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index 1a13a82e3..76cc16e40 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -294,13 +294,11 @@ struct DBDescriptor final : public std::enable_shared_from_this { bool transactionLogsUnregistered = false; /** - * Armed by finishClose() for its whole duration so an in-progress manual - * compactRange() on another thread aborts instead of making teardown wait - * out its full, unbounded duration. It covers both shapes: compactSync() - * holds an OperationGuard and so blocks the drain, while an async compact() - * released its guard at setup handoff and is not awaited until the closables - * sweep -- clearing the token after the drain left that second one able to - * stall teardown (and every concurrent open on the path) indefinitely. + * Armed when close is claimed so an in-progress manual + * compactRange() on another thread aborts before DBHandle::close() waits for + * its async work to drain. It covers both shapes: compactSync() holds an + * OperationGuard and so blocks the drain, while an async compact() released + * its guard at setup handoff and is not awaited until the closables sweep. * Close-initiated compaction passes `cancellable = false` rather than * clearing this, since nothing external is waiting on it. */ @@ -488,7 +486,13 @@ struct DBDescriptor final : public std::enable_shared_from_this { * under the same lock) waits instead of handing the descriptor to a new * handle that would then be closed out from under it. */ - bool beginClose() { return !this->closing.exchange(true); } + bool beginClose() { + if (this->closing.exchange(true)) { + return false; + } + this->compactCancelRequested.store(true); + return true; + } /** * Performs the actual close work (flush, close handles, release resources). From e5e55bc21547a931628842a14e390f540b4f40b9 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 1 Sep 2026 22:36:17 -0600 Subject: [PATCH 40/49] docs: fix lifecycle invariant numbering Co-Authored-By: GPT-5 Codex --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 5988f4a82..3be618a56 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -668,7 +668,7 @@ sufficient (env teardown does not honor tsfn acquire counts); see invariant 9), so for those this drain is defense in depth rather than the only thing preventing a use-after-free. -17. **An env's pending transactions are reaped by its cleanup hook — never by `DBHandle::close()`**: +18. **An env's pending transactions are reaped by its cleanup hook — never by `DBHandle::close()`**: `transactionAdd` stores a strong `shared_ptr` in the process-global `DBDescriptor`, and only commit/abort call `transactionRemove` (the JS wrap finalizer drops the JS-side ref and, per invariant 13, `onWrapperCollected()` reaps a transaction whose From ea93ce7ba8d3ac0cbf030b7c52c54420d65cab74 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 2 Sep 2026 08:27:21 -0600 Subject: [PATCH 41/49] fix(lifecycle): release the registry before static destruction A close-time flush failure quarantines the descriptor so shutdown()/destroy() can retry it. At process exit there is no later retry, and the registry singleton is a namespace-scope static: whatever it still holds is destroyed from an atexit handler. Closing a RocksDB database there runs CancelAllBackgroundWork() -> PeriodicTaskScheduler::Unregister() after RocksDB's own function-local statics are gone, so port::Mutex::Lock() gets EINVAL and RocksDB's PthreadCall aborts: rocksdb-js database registry cleanup failed: Failed to flush database during close: IO error: ... 000010.log: Permission denied pthread lock: Invalid argument That is the full-suite exit failure reported on this PR. It reproduces from test/background-error.test.ts alone, whose fixtures leave a database with a sticky RocksDB background error, so close() keeps failing its flush. DBRegistry::Teardown() now releases the remaining entries from the module's env-cleanup hook, while RocksDB is still usable; the failure is still reported, the process just exits cleanly. Those fixtures also tear down with destroy() so they stop leaving a broken database (and a leaked temp dir) behind. Also in this commit: - Regression coverage for the manual-compaction cancellation contract, per the review thread on compactCancelRequested. A new startup-snapshotted seam (ROCKSDB_JS_COMPACT_DELAY_MS) parks a cancellable compactRange() until the close claim arms the token, so the sync and async fixtures assert prompt destroy + a cancelled compaction without depending on host I/O timing. Verified against three mutations: unwiring options.canceled fails both fixtures, moving the arm past the in-flight drain fails the sync one, and removing Teardown() aborts/hangs the quarantined-exit fixture. - The contract itself is now written down on the member declaration and in AGENTS.md: one arming site, never cleared, never aliased onto `closing` (RocksDB writes through that pointer). - DestroyDB()/Shutdown() give each wait its own lifecycleWaitSeconds budget instead of sharing one deadline with the gate/lock acquisition, which could make a post-claim wait time out immediately and leave the database on disk. - A close/open cycle on the same RocksDatabase instance is now covered: open() clears the handle's async-work cancellation, so async work is admitted again. Refs #787 Co-Authored-By: Claude Opus --- AGENTS.md | 37 ++++++++- src/binding/binding.cpp | 8 ++ src/binding/core/test_seam.h | 31 +++++++- src/binding/database/db_descriptor.cpp | 16 ++++ src/binding/database/db_descriptor.h | 40 ++++++++-- src/binding/database/db_registry.cpp | 47 +++++++++-- src/binding/database/db_registry.h | 16 ++++ test/background-error.test.ts | 14 +++- test/destroy.test.ts | 21 +++++ test/fixtures/fork-compact-cancel-async.mts | 88 +++++++++++++++++++++ test/fixtures/fork-compact-cancel-sync.mts | 70 ++++++++++++++++ test/fixtures/fork-quarantined-exit.mts | 35 ++++++++ test/lifecycle.test.ts | 20 +++++ 13 files changed, 423 insertions(+), 20 deletions(-) create mode 100644 test/fixtures/fork-compact-cancel-async.mts create mode 100644 test/fixtures/fork-compact-cancel-sync.mts create mode 100644 test/fixtures/fork-quarantined-exit.mts diff --git a/AGENTS.md b/AGENTS.md index 3be618a56..f5ac95b97 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -226,6 +226,16 @@ sufficient (env teardown does not honor tsfn acquire counts); see `initializeTestSeams()` rather than per row: these are the two per-row native loops, and a `getenv()` scan per row is a measurable share of their cost for a seam unset in production. Add new per-row seams the same way. +- `ROCKSDB_JS_COMPACT_DELAY_MS` - Test-only: upper bound (ms) that a **cancellable** manual + `compactRange()` parks before handing the range to RocksDB, returning as soon as the + descriptor's cancel token is armed. Lets a fixture hold a compaction across a foreign close + claim without depending on how long a real compaction runs; snapshotted in + `initializeTestSeams()` like the per-row seams above, so the production path costs one relaxed + load per manual compaction. Used by `test/fixtures/fork-compact-cancel-{sync,async}.mts` +- `ROCKSDB_JS_CLOSE_FLUSH_FAILURE` - Test-only: number of close-time flushes to fail with an + injected `IOError` (a **count**, not a flag; `1` is one failure). More than one is what leaves a + descriptor still quarantined at process exit, since the exit-time `DBRegistry::Shutdown()` + consumes a failure of its own on the retry — see invariant 19 ## Test Structure @@ -358,7 +368,14 @@ sufficient (env teardown does not honor tsfn acquire counts); see count scan (`DBIteratorHandle::countRemaining`, behind `getKeysCount()` on both the database and transaction paths) polls `isClosing()` per row and reports the abort to its caller rather than a partial count; a manual `compactRange()` cannot poll from inside RocksDB, so it gets an explicit - cancel token (`DBDescriptor::compactCancelRequested` → `CompactRangeOptions::canceled`). + cancel token (`DBDescriptor::compactCancelRequested` → `CompactRangeOptions::canceled`). That + token is armed in exactly one place — `beginClose()`, in the same transition that publishes + `closing` — and is never cleared and never aliased onto `closing` itself: RocksDB writes through + the pointer it is given (`DisableManualCompaction()` sets the caller's atomic), and `closing` + means the registry has an owner committed to running `finishClose()`, which RocksDB must not be + able to publish. The full contract, and what each of + `test/fixtures/fork-compact-cancel-{sync,async}.mts` does and does not pin down, is on the + member declaration. Everything the four registry teardown paths do _after_ claiming a descriptor — `finishClose()`, erase-or-quarantine, notify, emit `database:closeFailed` — is one helper, `closeClaimedDescriptors` in `db_registry.cpp`; only the claim predicate differs per caller. Its @@ -709,6 +726,24 @@ sufficient (env teardown does not honor tsfn acquire counts); see never reproduces natively or on glibc, so the repro test is `skipIf(darwin)` (and, like the repo's other teardown repros, gated to Node). +19. **No `rocksdb::DB` may outlive the module's env-cleanup hook**: `DBRegistry::instance` is a + namespace-scope `static`, so anything still in `instance->databases` when the hook returns is + destroyed from an `atexit` handler. Closing a RocksDB database there runs + `DBImpl::CancelAllBackgroundWork()` → `PeriodicTaskScheduler::Unregister()` **after** RocksDB's + own function-local statics have been destroyed, and RocksDB's `PthreadCall` wrapper reacts to + the resulting `EINVAL` by printing `pthread lock: Invalid argument` and calling `std::abort()` — + a process that ran every test successfully then dies with SIGABRT (or, on the injected-flush + path, hangs forever in `WaitForFlushMemTables` with no background thread left to service it). + `DBRegistry::Shutdown()` normally empties the map, but a descriptor whose close-time flush + failed is deliberately **quarantined** (invariant 6) so `shutdown()`/`destroy()` can retry — + and at process exit there is no later retry. `DBRegistry::Teardown()`, called from the hook + right after `Shutdown()`, releases whatever is left while RocksDB is still usable. The general + trap: anything the registry retains for a caller to retry needs a defined terminal owner, + because "the static destructor" is not one. Covered by + `test/fixtures/fork-quarantined-exit.mts`; the same shape reproduces from JS by leaving a + database with a sticky RocksDB background error (`test/background-error.test.ts` used to, + which is why its fixtures now tear down with `destroy()` rather than `close()`). + ## Debugging native heap corruption AddressSanitizer is the first choice (`ROCKSDB_ASAN=1 node-gyp rebuild` toggles `-fsanitize=address` diff --git a/src/binding/binding.cpp b/src/binding/binding.cpp index d5e39f401..5530daca7 100644 --- a/src/binding/binding.cpp +++ b/src/binding/binding.cpp @@ -240,6 +240,14 @@ NAPI_MODULE_INIT() { } }; cleanup("database registry", []() { rocksdb_js::DBRegistry::Shutdown(); }); + // Shutdown() leaves a descriptor whose close-time flush failed + // quarantined in the registry so shutdown()/destroy() can retry it. + // The process is exiting, so there is no later retry -- and a + // descriptor that survives to the registry singleton's static + // destructor closes its rocksdb::DB from an atexit handler, after + // RocksDB's own statics are gone, which aborts. Release whatever is + // left here, while that is still safe. + cleanup("database registry teardown", []() { rocksdb_js::DBRegistry::Teardown(); }); cleanup("transaction logs", []() { rocksdb_js::TransactionLogStoreRegistry::Shutdown(); }); cleanup("global events", []() { rocksdb_js::GlobalEvents::Shutdown(); }); DEBUG_LOG("Binding::Init env cleanup done\n"); diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index d2c922097..d26536110 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -24,8 +24,12 @@ inline std::atomic& closeFailureFlag() { return pending; } -inline std::atomic& closeFlushFailureFlag() { - static std::atomic pending{false}; +// A count, not a flag: a quarantined descriptor is only reached at process exit +// when the shutdown() retry fails too, so reproducing that state needs the +// close-time flush to fail more than once. `=1` behaves exactly as the previous +// boolean did. +inline std::atomic& closeFlushFailureFlag() { + static std::atomic pending{0}; return pending; } @@ -44,17 +48,30 @@ inline std::atomic& countScanDelayMsFlag() { return delayMs; } +// Upper bound, in milliseconds, that a cancellable manual compactRange() parks +// before handing the range to RocksDB. It returns as soon as the descriptor's +// cancel token is armed, so a fixture can hold a compaction across a foreign +// close claim without depending on how long a real compaction happens to take. +// Snapshotted here rather than read per call so the production path (unset) +// costs one relaxed load. +inline std::atomic& compactCancelDelayMsFlag() { + static std::atomic delayMs{0}; + return delayMs; +} + inline void initializeTestSeams() { static std::once_flag initialized; std::call_once(initialized, []() { const char* value = ::getenv("ROCKSDB_JS_CLOSE_FAILURE"); closeFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); value = ::getenv("ROCKSDB_JS_CLOSE_FLUSH_FAILURE"); - closeFlushFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); + closeFlushFailureFlag().store(value ? ::atoi(value) : 0, std::memory_order_relaxed); iteratorNextDelayMsFlag().store( testDelayMs("ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS"), std::memory_order_relaxed); countScanDelayMsFlag().store( testDelayMs("ROCKSDB_JS_COUNT_DELAY_MS"), std::memory_order_relaxed); + compactCancelDelayMsFlag().store( + testDelayMs("ROCKSDB_JS_COMPACT_DELAY_MS"), std::memory_order_relaxed); }); } @@ -63,7 +80,13 @@ inline bool testConsumeCloseFailure() { } inline bool testConsumeCloseFlushFailure() { - return closeFlushFailureFlag().exchange(false, std::memory_order_relaxed); + int cur = closeFlushFailureFlag().load(std::memory_order_relaxed); + while (cur > 0) { + if (closeFlushFailureFlag().compare_exchange_weak(cur, cur - 1, std::memory_order_relaxed)) { + return true; + } + } + return false; } // Deterministic one-shot(-per-N) seam for the stranded-snapshot retry path: forces the next N diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index d81711141..d9466c600 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include namespace rocksdb_js { @@ -2262,6 +2263,21 @@ rocksdb::Status DBDescriptor::compactRange( // wait out its full, unbounded duration; see compactCancelRequested. if (cancellable) { options.canceled = &this->compactCancelRequested; + // Test seam (inert unless ROCKSDB_JS_COMPACT_DELAY_MS is set): park here, + // still inside the caller's OperationGuard / async-work registration, + // until a foreign close arms the token. That makes the ordering + // observable from JS without depending on how long a real compaction + // happens to run. Bounded so a fixture that never closes still finishes. + const int cancelWaitMs = compactCancelDelayMsFlag().load(std::memory_order_relaxed); + if (cancelWaitMs > 0) { + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::milliseconds(cancelWaitMs); + while (!this->compactCancelRequested.load() && + std::chrono::steady_clock::now() < deadline + ) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } } if (bottommost) { // RocksDB defaults this to kIfHaveCompactionFilter, so with no compaction filter installed diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index 76cc16e40..a36f4e910 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -294,13 +294,37 @@ struct DBDescriptor final : public std::enable_shared_from_this { bool transactionLogsUnregistered = false; /** - * Armed when close is claimed so an in-progress manual - * compactRange() on another thread aborts before DBHandle::close() waits for - * its async work to drain. It covers both shapes: compactSync() holds an - * OperationGuard and so blocks the drain, while an async compact() released - * its guard at setup handoff and is not awaited until the closables sweep. - * Close-initiated compaction passes `cancellable = false` rather than - * clearing this, since nothing external is waiting on it. + * Cancellation token handed to `rocksdb::CompactRangeOptions::canceled` by + * every cancellable `compactRange()`. Contract, in three parts: + * + * 1. It is armed in exactly one place -- `beginClose()`, under the same + * transition that publishes `closing` -- so a close claim and manual + * compaction cancellation are one step. Arming it any later reopens the + * window this exists to close: `finishClose()` drains + * `operationsInFlight` with an untimed wait and its closables sweep + * drains each handle's async work, and a manual compaction blocks both + * (compactSync() holds an OperationGuard for its whole duration; an + * async compact() released its guard at setup handoff and is awaited by + * `DBHandle::close()` during the sweep). `DestroyDB`/`Shutdown` also + * claim every entry for a path under one lock and then close them + * sequentially, so arming at claim time cancels compactions on + * descriptors whose own `finishClose()` has not started yet. + * 2. It is never cleared. Close-initiated compaction opts out with + * `cancellable = false` instead, since nothing external is waiting on + * it, and a descriptor never leaves the closing state. + * 3. It is private to this descriptor and never aliased onto `closing`. + * RocksDB writes through this pointer -- `DisableManualCompaction()` + * sets the caller's atomic -- and `closing == true` means the registry + * has an owner committed to running `finishClose()`. Letting RocksDB + * publish that state would leave a half-closed descriptor with no + * closer, wedging every later open of the path. + * + * Covered by `test/fixtures/fork-compact-cancel-{sync,async}.mts`: both fail + * if the token stops reaching RocksDB, and the sync one additionally fails if + * arming moves past the in-flight drain. Neither separates arming here from + * arming at the top of `finishClose()` -- for a single descriptor those are + * equivalent, and the batch-claim case in (1) is what makes this the right + * home. */ std::atomic compactCancelRequested{false}; @@ -490,6 +514,8 @@ struct DBDescriptor final : public std::enable_shared_from_this { if (this->closing.exchange(true)) { return false; } + // The only site that arms compaction cancellation; see the + // compactCancelRequested contract above before moving it. this->compactCancelRequested.store(true); return true; } diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 4a69d355f..511dcddb7 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -309,11 +309,17 @@ void DBRegistry::DestroyDB(const std::string& path) { } DEBUG_LOG("%p DBRegistry::DestroyDB Destroying \"%s\"\n", instance.get(), path.c_str()); - const auto deadline = std::chrono::steady_clock::now() + + // `lifecycleWaitSeconds` bounds each wait separately rather than the whole + // operation. Sharing one deadline across acquiring the path gate and the + // later waits for a foreign closer lets a slow acquisition spend the entire + // budget, so destroy() could claim the path and then immediately report a + // timeout with the database still on disk. + const auto waitBudget = std::chrono::seconds(DBSettings::getInstance().getLifecycleWaitSeconds()); { std::unique_lock lock(instance->databasesMutex); - if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + const auto gateDeadline = std::chrono::steady_clock::now() + waitBudget; + if (!instance->lifecycleCondition.wait_until(lock, gateDeadline, [&]() { return !instance->shutdownInProgress && instance->destroyingPaths.find(path) == instance->destroyingPaths.end(); })) { @@ -367,7 +373,8 @@ void DBRegistry::DestroyDB(const std::string& path) { } for (const auto& closing : alreadyClosing) { std::unique_lock lock(instance->databasesMutex); - if (!closing.condition->wait_until(lock, deadline, [&]() { + const auto drainDeadline = std::chrono::steady_clock::now() + waitBudget; + if (!closing.condition->wait_until(lock, drainDeadline, [&]() { auto entry = instance->databases.find(closing.key); return entry == instance->databases.end() || entry->second.descriptor != closing.descriptor || @@ -991,10 +998,11 @@ void DBRegistry::ReleaseParkTimeoutsByEnv(napi_env env) { */ void DBRegistry::Shutdown() { if (instance) { - const auto deadline = std::chrono::steady_clock::now() + + // One budget per wait, not one for the whole shutdown; see DestroyDB. + const auto waitBudget = std::chrono::seconds(DBSettings::getInstance().getLifecycleWaitSeconds()); std::unique_lock shutdownLock(instance->shutdownMutex, std::defer_lock); - if (!shutdownLock.try_lock_until(deadline)) { + if (!shutdownLock.try_lock_until(std::chrono::steady_clock::now() + waitBudget)) { throw rocksdb_js::DBException("Timed out waiting for another database shutdown to finish"); } { @@ -1049,7 +1057,8 @@ void DBRegistry::Shutdown() { for (const auto& closing : descriptorsToWaitFor) { std::unique_lock lock(instance->databasesMutex); - if (!closing.condition->wait_until(lock, deadline, [&]() { + const auto drainDeadline = std::chrono::steady_clock::now() + waitBudget; + if (!closing.condition->wait_until(lock, drainDeadline, [&]() { auto entry = instance->databases.find(closing.key); return entry == instance->databases.end() || entry->second.descriptor != closing.descriptor || @@ -1061,7 +1070,8 @@ void DBRegistry::Shutdown() { if (destroysInFlight) { std::unique_lock lock(instance->databasesMutex); - if (!instance->lifecycleCondition.wait_until(lock, deadline, [&]() { + const auto destroyDeadline = std::chrono::steady_clock::now() + waitBudget; + if (!instance->lifecycleCondition.wait_until(lock, destroyDeadline, [&]() { return instance->destroyingPaths.empty(); })) { throw rocksdb_js::DBException("Timed out waiting for database destruction to finish during shutdown"); @@ -1080,6 +1090,29 @@ void DBRegistry::Shutdown() { } } +/** + * Release every remaining registry entry (see the header for why this cannot be + * left to the singleton's static destructor). + */ +void DBRegistry::Teardown() { + if (!instance) { + return; + } + + std::unordered_map entries; + { + std::lock_guard lock(instance->databasesMutex); + entries.swap(instance->databases); + } + + // Destroy outside the lock: ~DBDescriptor closes the RocksDB database and + // joins its worker threads. A descriptor already claimed by a failed close + // short-circuits its own close() and is simply released here. + DEBUG_LOG("%p DBRegistry::Teardown Releasing %zu remaining descriptor(s)\n", + instance.get(), entries.size()); + entries.clear(); +} + /** * Get the number of databases in the registry. */ diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index 7091a5ee6..bcfe781b4 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -115,6 +115,22 @@ class DBRegistry final { static void ReleaseCommitCompletionsByEnv(napi_env env); static void ReleaseParkTimeoutsByEnv(napi_env env); static void Shutdown(); + /** + * Releases every remaining registry entry. Called from the module env + * cleanup hook after `Shutdown()`, i.e. while the process is still running + * normally. Nothing may keep a `rocksdb::DB` alive past that point: the + * registry singleton is a namespace-scope static, so anything still in the + * map is destroyed from an `atexit` handler, and closing a RocksDB database + * there runs `DBImpl::CancelAllBackgroundWork()` after RocksDB's own + * function-local statics (the `PeriodicTaskScheduler` timer and its + * `port::Mutex`) have already been destroyed -- which aborts the process in + * `port::Mutex::Lock()` with `pthread lock: Invalid argument`. + * + * `Shutdown()` normally empties the map on its own; a descriptor whose + * close-time flush failed is deliberately quarantined instead, and at + * process exit there is no later `shutdown()`/`destroy()` to retry it. + */ + static void Teardown(); static size_t Size(); }; diff --git a/test/background-error.test.ts b/test/background-error.test.ts index b7395b196..8287ea594 100644 --- a/test/background-error.test.ts +++ b/test/background-error.test.ts @@ -35,7 +35,19 @@ describe('background error', () => { } catch { /* best effort */ } - db.close(); + try { + // destroy(), not close(): a recorded background error is sticky, so + // the close-time flush keeps failing and would leave the descriptor + // quarantined in the registry for the rest of the process (the + // registry then reports it again at exit). destroy() forces teardown. + if (process.env.KEEP_FILES) { + db.close(); + } else { + db.destroy(); + } + } catch { + /* best effort */ + } if (!process.env.KEEP_FILES) { rmSync(path, { force: true, recursive: true, maxRetries: 3 }); } diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 0a22e1107..2377ef2ef 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -15,6 +15,9 @@ const flushFailureFixture = join(__dirname, 'fixtures', 'fork-flush-failure.mts' const backupDestroyFixture = join(__dirname, 'fixtures', 'fork-backup-destroy.mts'); const iteratorNextRaceFixture = join(__dirname, 'fixtures', 'fork-iterator-next-race.mts'); const countDestroyRaceFixture = join(__dirname, 'fixtures', 'fork-count-destroy-race.mts'); +const compactCancelSyncFixture = join(__dirname, 'fixtures', 'fork-compact-cancel-sync.mts'); +const compactCancelAsyncFixture = join(__dirname, 'fixtures', 'fork-compact-cancel-async.mts'); +const quarantinedExitFixture = join(__dirname, 'fixtures', 'fork-quarantined-exit.mts'); const nodeExecutable = process.env.NODE_BINARY ?? (process.versions.bun || process.versions.deno @@ -269,4 +272,22 @@ describe('Destroy', () => { ROCKSDB_JS_CLOSE_RETRY_DELAY_MS: '1000', }); }, 15_000); + + it('cancels an in-flight synchronous compaction when destroy claims the descriptor', async () => { + await runDestroyFixture(compactCancelSyncFixture, generateDBPath(), { + ROCKSDB_JS_COMPACT_DELAY_MS: '10000', + }); + }, 20_000); + + it('cancels an in-flight asynchronous compaction when destroy claims the descriptor', async () => { + await runDestroyFixture(compactCancelAsyncFixture, generateDBPath(), { + ROCKSDB_JS_COMPACT_DELAY_MS: '10000', + }); + }, 20_000); + + it('exits cleanly with a descriptor still quarantined at process exit', async () => { + await runDestroyFixture(quarantinedExitFixture, generateDBPath(), { + ROCKSDB_JS_CLOSE_FLUSH_FAILURE: '2', + }); + }, 15_000); }); diff --git a/test/fixtures/fork-compact-cancel-async.mts b/test/fixtures/fork-compact-cancel-async.mts new file mode 100644 index 000000000..b67af4516 --- /dev/null +++ b/test/fixtures/fork-compact-cancel-async.mts @@ -0,0 +1,88 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { setTimeout as delay } from 'node:timers/promises'; +import { Worker } from 'node:worker_threads'; + +// The async shape of fork-compact-cancel-sync.mts. An async compact() releases +// its OperationGuard at setup handoff, so it is not covered by finishClose()'s +// in-flight drain -- it is awaited later, by DBHandle::close() during the +// closables sweep. Both shapes therefore depend on the token being armed by the +// close claim rather than at any point inside finishClose(). +// +// Mutation coverage differs between the two shapes, deliberately: unwiring +// options.canceled fails both, while moving the arm past the in-flight drain +// only fails the sync fixture (an async compaction is not counted by that +// drain, so a late arm still beats the closables sweep). +const path = process.argv[2]; +const db = RocksDatabase.open(path); +for (let i = 0; i < 20; i++) { + db.putSync(`key-${i}`, i); +} +db.flushSync(); + +const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { + eval: true, + workerData: { path, destroyStartDelayMs: 0 }, +}); + +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} + +const ready = await nextMessage(); +if (!ready.ready) throw new Error(`Worker failed to initialize: ${JSON.stringify(ready)}`); + +// Start the compaction first and let the libuv worker pick it up, so it is +// parked in the seam *before* the close claim. Otherwise the execute callback's +// own opened()/isCancelled() check would reject it and the token would never be +// exercised -- which the assertion below distinguishes. +const compacting = db.compact(); +let compactSettled = false; +const outcome = compacting.then( + () => { + compactSettled = true; + return null; + }, + (error: unknown) => { + compactSettled = true; + return error; + } +); +await delay(250); +if (compactSettled) throw new Error('Compaction settled before the close claim; seam not active'); + +const started = Date.now(); +worker.postMessage({ destroy: true }); +const destroying = await nextMessage(); +if (!destroying.destroying) + throw new Error(`Worker did not start destroying: ${JSON.stringify(destroying)}`); + +const compactError = await outcome; +const elapsed = Date.now() - started; + +if (!compactError) + throw new Error( + `Expected compact() to be cancelled, but it resolved after ${elapsed}ms -- ` + + 'the cancel token is no longer reaching rocksdb::CompactRangeOptions::canceled' + ); +if (String(compactError).includes('Database closed during compact operation')) + throw new Error( + 'Compaction was rejected at entry rather than cancelled mid-flight; the fixture ' + + 'no longer exercises the cancel token' + ); +if (!/cancel|paused|incomplete/i.test(String(compactError))) throw compactError; +if (elapsed >= 2000) + throw new Error( + `compact() ran ${elapsed}ms after the close claim; it should have been cancelled immediately` + ); + +const destroyResult = await nextMessage(); +if (!destroyResult.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyResult)}`); + +if (registryStatus().some((entry) => entry.path === path)) + throw new Error('Expected destroy to fully clear the registry entry'); + +await worker.terminate(); diff --git a/test/fixtures/fork-compact-cancel-sync.mts b/test/fixtures/fork-compact-cancel-sync.mts new file mode 100644 index 000000000..8da98b1ec --- /dev/null +++ b/test/fixtures/fork-compact-cancel-sync.mts @@ -0,0 +1,70 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { Worker } from 'node:worker_threads'; + +// Regression coverage for the manual-compaction cancellation contract (see +// DBDescriptor::compactCancelRequested). compactSync() holds an OperationGuard +// for the whole compaction, and finishClose() drains those guards with an +// untimed wait -- so a compaction that is not told to cancel blocks the close, +// and with it every concurrent open of the path. +// +// ROCKSDB_JS_COMPACT_DELAY_MS parks the compaction inside that guard until the +// token is armed (or the bound expires), which removes the dependency on how +// long a real compaction happens to run. The two assertions below fail for the +// two ways this can regress: a slow close means arming moved after the drain, +// and a successful compaction means the token stopped reaching RocksDB. +const path = process.argv[2]; +const db = RocksDatabase.open(path); +for (let i = 0; i < 20; i++) { + db.putSync(`key-${i}`, i); +} +db.flushSync(); + +const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { + eval: true, + workerData: { path, destroyStartDelayMs: 50 }, +}); + +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} + +const ready = await nextMessage(); +if (!ready.ready) throw new Error(`Worker failed to initialize: ${JSON.stringify(ready)}`); + +worker.postMessage({ destroy: true }); +const destroying = await nextMessage(); +if (!destroying.destroying) + throw new Error(`Worker did not start destroying: ${JSON.stringify(destroying)}`); + +const started = Date.now(); +let compactError: unknown; +try { + db.compactSync(); +} catch (error) { + compactError = error; +} +const elapsed = Date.now() - started; + +if (!compactError) + throw new Error( + `Expected compactSync to be cancelled, but it succeeded after ${elapsed}ms -- ` + + 'either the token is armed after finishClose() drains in-flight operations, ' + + 'or it no longer reaches rocksdb::CompactRangeOptions::canceled' + ); +if (!/cancel|paused|incomplete/i.test(String(compactError))) throw compactError; +if (elapsed >= 2000) + throw new Error( + `compactSync ran ${elapsed}ms; the close claim should have cancelled it immediately` + ); + +const destroyResult = await nextMessage(); +if (!destroyResult.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyResult)}`); + +if (registryStatus().some((entry) => entry.path === path)) + throw new Error('Expected destroy to fully clear the registry entry'); + +await worker.terminate(); diff --git a/test/fixtures/fork-quarantined-exit.mts b/test/fixtures/fork-quarantined-exit.mts new file mode 100644 index 000000000..250728fd8 --- /dev/null +++ b/test/fixtures/fork-quarantined-exit.mts @@ -0,0 +1,35 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; + +// A close-time flush failure quarantines the descriptor: it stays in the +// process-global registry, still holding an open rocksdb::DB, so an explicit +// shutdown()/destroy() can retry the close. This fixture never issues that +// retry -- the quarantine survives to process exit, which is the one case the +// registry cannot defer. +// +// The registry singleton is a namespace-scope static, so anything still in its +// map is destroyed from an atexit handler. Closing a RocksDB database there +// runs DBImpl::CancelAllBackgroundWork() after RocksDB's own function-local +// statics (the PeriodicTaskScheduler timer and its port::Mutex) have already +// been destroyed, and RocksDB's PthreadCall aborts the process: +// +// pthread lock: Invalid argument +// +// The module's env-cleanup hook therefore releases whatever Shutdown() left +// behind (DBRegistry::Teardown), while RocksDB is still usable. The harness +// asserts a zero exit code and no signal, which is what fails without it. +// +// ROCKSDB_JS_CLOSE_FLUSH_FAILURE is set to 2: one failure for the explicit +// close below, one for the exit-time shutdown retry. +const path = process.argv[2]; +const db = RocksDatabase.open(path, { disableWAL: true }); +db.putSync('key', 'unflushed'); + +try { + db.close(); + throw new Error('Expected close to surface the injected flush failure'); +} catch (error) { + if (!String(error).includes('Injected database close flush failure')) throw error; +} + +if (!registryStatus().some((entry) => entry.path === path && entry.closeError)) + throw new Error('Flush failure did not quarantine the descriptor'); diff --git a/test/lifecycle.test.ts b/test/lifecycle.test.ts index 1e8bda96a..0e916641c 100644 --- a/test/lifecycle.test.ts +++ b/test/lifecycle.test.ts @@ -67,4 +67,24 @@ describe('Lifecycle', () => { await expect(db3.get('foo')).rejects.toThrow('Database not open'); } )); + + // close() publishes async-work cancellation on the handle, and every async + // admission (get, flush, compact, clear, backup, checkpoint) is refused while + // it stands. Reopening the same instance has to clear it, or a documented + // reopen would come back with every async operation permanently rejected as + // "Database is closing". + it('accepts async work again after a close/open cycle on the same instance', () => + dbRunner(async ({ db }) => { + db.putSync('before', 'close'); + await db.flush(); + db.close(); + + db.open(); + expect(db.isOpen()).toBe(true); + db.putSync('after', 'reopen'); + await expect(db.flush()).resolves.toBeUndefined(); + await expect(db.compact()).resolves.toBeUndefined(); + expect(await db.get('before')).toBe('close'); + expect(await db.get('after')).toBe('reopen'); + })); }); From 9fdb15c181a011d8ccfdb27bef95d27fa7645377 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 2 Sep 2026 11:31:15 -0600 Subject: [PATCH 42/49] fix(close): cancel a handle's own async compaction before draining it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit db.close() cancels async work and then waits for it with no timeout, but the only cancellation RocksDB honours for a manual compaction is CompactRangeOptions::canceled — and that token was armed exclusively by DBDescriptor::beginClose(), which DBRegistry::CloseDB does not reach until after DBHandle::close() has already returned from its drain. An async compact() therefore ran to completion with canceled == false, parking the JS thread (and the path gate, so every concurrent open of that path times out) for the compaction's full duration: minutes to hours on a large column family, where the pre-change bounded drain gave up after 5s. Arming the descriptor token from CloseDB is not the fix — it is never cleared, so one handle closing would permanently kill manual compaction for every other handle sharing the process-global descriptor. Instead the token to hand RocksDB is now chosen by the drain that awaits the caller: - synchronous compactSync()/clearSync() hold an OperationGuard for the whole compaction and are awaited by finishClose()'s operationsInFlight wait, so they keep the descriptor token that beginClose() arms ahead of it; - async compact()/clear() released their guard at setup handoff and are awaited by DBHandle::close()'s async-work drain, so they get a new per-handle token that close() arms immediately before that drain. compactRange() takes the token as a parameter rather than deciding for itself; close-initiated compaction passes nullptr as before. The foreign-destroy path reaches the per-handle token through finishClose()'s closables sweep, so fork-compact-cancel-async.mts still holds. fork-compact-cancel-close.mts is the new self-close case: with the async compaction parked in the ROCKSDB_JS_COMPACT_DELAY_MS seam, close() returns in ~6ms with the fix and blocks 9760ms without it. Also from the same review round: - DBHandle::open() clears cancellation only after adopting the new descriptor. OpenDB() blocks while a foreign destroy owns the old path, and that destroy's closables sweep force-closes this still-attached handle mid-wait, re-arming cancellation over the newly opened descriptor — leaving a handle whose sync methods work while every async admission rejects as "Database is closing" for the rest of its life. Written up as AGENTS.md invariant 20. - ROCKSDB_JS_DESTROY_FAILURE is snapshotted in initializeTestSeams() like the other fault flags instead of re-reading getenv on a teardown thread, and the five previously undocumented seams are listed in AGENTS.md. - Destroy on a never-opened handle said "Database path is required for destroy" to a caller who did pass a path; it now names the real condition. - README no longer claims close reports a compactOnClose failure — only the WaitForCompact status is captured, and a skipped compaction loses no data, so failing the close (and quarantining the path) for it would be wrong. - Two comments cited AGENTS invariant 15 for the flush-stall rule, which is 16. - runDestroyFixture removes the database directory it generated, unless the fixture failed or KEEP_FILES is set. Verification: pnpm build, pnpm check, pnpm test:native (159/159), and pnpm test (855 passed / 3 skipped, exit 0) all pass. Co-Authored-By: Claude Opus --- AGENTS.md | 62 ++++++++++++++---- README.md | 5 +- src/binding/core/test_seam.h | 9 +++ src/binding/database/database.cpp | 24 +++++-- src/binding/database/db_descriptor.cpp | 32 ++++++---- src/binding/database/db_descriptor.h | 45 +++++++------ src/binding/database/db_handle.cpp | 27 ++++++-- src/binding/database/db_handle.h | 41 +++++++++++- src/binding/database/db_registry.cpp | 2 +- test/destroy.test.ts | 20 +++++- test/fixtures/fork-compact-cancel-close.mts | 71 +++++++++++++++++++++ 11 files changed, 275 insertions(+), 63 deletions(-) create mode 100644 test/fixtures/fork-compact-cancel-close.mts diff --git a/AGENTS.md b/AGENTS.md index f5ac95b97..aa5f2048d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -227,15 +227,25 @@ sufficient (env teardown does not honor tsfn acquire counts); see `getenv()` scan per row is a measurable share of their cost for a seam unset in production. Add new per-row seams the same way. - `ROCKSDB_JS_COMPACT_DELAY_MS` - Test-only: upper bound (ms) that a **cancellable** manual - `compactRange()` parks before handing the range to RocksDB, returning as soon as the - descriptor's cancel token is armed. Lets a fixture hold a compaction across a foreign close - claim without depending on how long a real compaction runs; snapshotted in - `initializeTestSeams()` like the per-row seams above, so the production path costs one relaxed - load per manual compaction. Used by `test/fixtures/fork-compact-cancel-{sync,async}.mts` + `compactRange()` parks before handing the range to RocksDB, returning as soon as the cancel + token that caller was given is armed. Lets a fixture hold a compaction across a close claim + without depending on how long a real compaction runs; snapshotted in `initializeTestSeams()` + like the per-row seams above, so the production path costs one relaxed load per manual + compaction. Used by `test/fixtures/fork-compact-cancel-{sync,async,close}.mts` - `ROCKSDB_JS_CLOSE_FLUSH_FAILURE` - Test-only: number of close-time flushes to fail with an injected `IOError` (a **count**, not a flag; `1` is one failure). More than one is what leaves a descriptor still quarantined at process exit, since the exit-time `DBRegistry::Shutdown()` consumes a failure of its own on the retry — see invariant 19 +- `ROCKSDB_JS_CLOSE_FAILURE` / `ROCKSDB_JS_DESTROY_FAILURE` - Test-only: inject a one-shot native + close failure, or fail every physical `DestroyDB` for the life of the process. Both are fault + **flags**, so both are snapshotted in `initializeTestSeams()` rather than re-read: `::getenv` + races a `process.env` write from another thread, and these are read from teardown paths that can + run on any thread +- `ROCKSDB_JS_CLOSE_RETRY_DELAY_MS` - Test-only: delay inside a _resumed_ `finishClose()` (the + `shutdown()`/`destroy()` retry of a quarantined descriptor), widening the window in which a + concurrent open must wait for the retry rather than reopen the path +- `ROCKSDB_JS_BACKUP_DELAY_MS` / `ROCKSDB_JS_ITERATOR_SETUP_DELAY_MS` - Test-only: delays inside a + native backup copy and iterator construction, holding each across a concurrent destroy claim ## Test Structure @@ -368,14 +378,27 @@ sufficient (env teardown does not honor tsfn acquire counts); see count scan (`DBIteratorHandle::countRemaining`, behind `getKeysCount()` on both the database and transaction paths) polls `isClosing()` per row and reports the abort to its caller rather than a partial count; a manual `compactRange()` cannot poll from inside RocksDB, so it gets an explicit - cancel token (`DBDescriptor::compactCancelRequested` → `CompactRangeOptions::canceled`). That - token is armed in exactly one place — `beginClose()`, in the same transition that publishes - `closing` — and is never cleared and never aliased onto `closing` itself: RocksDB writes through - the pointer it is given (`DisableManualCompaction()` sets the caller's atomic), and `closing` - means the registry has an owner committed to running `finishClose()`, which RocksDB must not be - able to publish. The full contract, and what each of - `test/fixtures/fork-compact-cancel-{sync,async}.mts` does and does not pin down, is on the - member declaration. + cancel token → `CompactRangeOptions::canceled`. There are **two** such tokens, and which one a + compaction is handed is decided by the drain that awaits it, not by where the compaction runs. + `DBDescriptor::compactCancelRequested` covers a **synchronous** `compactSync()`/`clearSync()`, + which holds an `OperationGuard` for its whole duration and so is awaited by `finishClose()`'s + untimed `operationsInFlight` wait; it is armed in exactly one place — `beginClose()`, in the same + transition that publishes `closing`. `DBHandle::compactCancelRequested` covers an **async** + `compact()`/`clear()`, which released its guard at setup handoff and is instead awaited by + `DBHandle::close()`'s untimed async-work drain; it is armed by that same `close()`, immediately + before the drain. The split is not redundancy: a self-close (`db.close()` → + `DBRegistry::CloseDB`) reaches `DBHandle::close()` **before** `PurgeIfUnreferenced`/`beginClose()`, + so the descriptor token is not yet armed when the async drain starts — a `db.close()` racing its + own `db.compact()` would park the JS thread for the compaction's full duration. Arming the + descriptor token from `CloseDB` is not the alternative: it is never cleared, so one handle + closing would permanently kill manual compaction for every other handle sharing the + process-global descriptor. The per-handle token _is_ cleared, by `DBHandle::open()`, and only + after the new descriptor is adopted (see invariant 20). Neither token is ever aliased onto + `closing` itself: RocksDB writes through the pointer it is given + (`DisableManualCompaction()` sets the caller's atomic), and `closing` means the registry has an + owner committed to running `finishClose()`, which RocksDB must not be able to publish. The full + contract, and what each of `test/fixtures/fork-compact-cancel-{sync,async,close}.mts` does and + does not pin down, is on the two member declarations. Everything the four registry teardown paths do _after_ claiming a descriptor — `finishClose()`, erase-or-quarantine, notify, emit `database:closeFailed` — is one helper, `closeClaimedDescriptors` in `db_registry.cpp`; only the claim predicate differs per caller. Its @@ -744,6 +767,19 @@ sufficient (env teardown does not honor tsfn acquire counts); see database with a sticky RocksDB background error (`test/background-error.test.ts` used to, which is why its fixtures now tear down with `destroy()` rather than `close()`). +20. **A reopened handle clears its cancellation only after it has adopted the new descriptor**: + `DBHandle::close()` publishes `cancelled` (and, per invariant 6, the per-handle compaction + token), and every async admission refuses while either stands — so `DBHandle::open()` has to + clear them for a documented close/reopen cycle to work. It must do so **after** + `DBRegistry::OpenDB()` returns, not before the call. `OpenDB()` blocks while a foreign + `destroy()`/`shutdown()` owns the old path, and that teardown's closables sweep force-closes + this still-attached handle mid-wait, re-arming both flags. Clearing first therefore leaves the + re-arm standing over the _newly_ opened descriptor: `opened()` reports true and the sync + methods keep working, while every `get`/`flush`/`compact`/`clear`/`commit` on that instance + rejects with "Database is closing" for the rest of its life. The general shape — publish-then- + clear across a blocking call another thread can publish into — is the same one invariant 6's + cancel token and invariant 17's admission mutex exist to keep out of the teardown paths. + ## Debugging native heap corruption AddressSanitizer is the first choice (`ROCKSDB_ASAN=1 node-gyp rebuild` toggles `-fsanitize=address` diff --git a/README.md b/README.md index 0f0a027b7..cd1cea278 100644 --- a/README.md +++ b/README.md @@ -143,8 +143,9 @@ Creates a new database instance. Closes a database. This function can be called multiple times and will only close an opened database. A database instance can be reopened once it is closed. A flush failure leaves the native database quarantined so `shutdown()` can retry without losing unflushed data; an explicit -`destroy()` can instead delete it. A compaction failure is reported after native teardown -completes. All native close errors emit `database:closeFailed`. The quarantine applies to both +`destroy()` can instead delete it. A failure while waiting for compaction to settle is reported +after native teardown completes; the optional `compactOnClose` pass itself is best-effort and its +errors do not fail the close, since a skipped compaction loses no data. All native close errors emit `database:closeFailed`. The quarantine applies to both writable and read-only opens because both modes share the physical path lifecycle. ```typescript diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index d26536110..2f50db73e 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -24,6 +24,13 @@ inline std::atomic& closeFailureFlag() { return pending; } +// Snapshotted for the same reason as closeFailureFlag(): a fault flag must not +// be re-read from the environment on a native thread. +inline std::atomic& destroyFailureFlag() { + static std::atomic pending{false}; + return pending; +} + // A count, not a flag: a quarantined descriptor is only reached at process exit // when the shutdown() retry fails too, so reproducing that state needs the // close-time flush to fail more than once. `=1` behaves exactly as the previous @@ -64,6 +71,8 @@ inline void initializeTestSeams() { std::call_once(initialized, []() { const char* value = ::getenv("ROCKSDB_JS_CLOSE_FAILURE"); closeFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); + value = ::getenv("ROCKSDB_JS_DESTROY_FAILURE"); + destroyFailureFlag().store(value && ::atoi(value) > 0, std::memory_order_relaxed); value = ::getenv("ROCKSDB_JS_CLOSE_FLUSH_FAILURE"); closeFlushFailureFlag().store(value ? ::atoi(value) : 0, std::memory_order_relaxed); iteratorNextDelayMsFlag().store( diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index f26c531d7..c95942979 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -94,7 +94,9 @@ static napi_value doClear(napi_env env, napi_callback_info info, const char* fai if (!state->handle || !state->handle->opened() || state->handle->isCancelled()) { state->status = rocksdb::Status::Aborted("Database closed during clear operation"); } else { - state->status = state->handle->clear(); + // awaited by DBHandle::close()'s async-work drain, so the token + // that drain arms is the one that can cancel it + state->status = state->handle->clear(&state->handle->compactCancelRequested); } // signal that execute handler is complete state->signalExecuteCompleted(); @@ -155,7 +157,9 @@ static napi_value doClearSync(napi_env env, napi_callback_info info, const char* UNWRAP_DB_HANDLE_AND_OPEN(); ACQUIRE_OPERATIONS_LOCK(); - rocksdb::Status status = (*dbHandle)->clear(); + // synchronous: counted by operationsInFlight, which beginClose() arms the + // descriptor token ahead of + rocksdb::Status status = (*dbHandle)->clear(&(*dbHandle)->descriptor->compactCancelRequested); if (!status.ok()) { ROCKSDB_STATUS_CREATE_NAPI_ERROR(status, failureMsg); ::napi_throw(env, error); @@ -342,7 +346,11 @@ napi_value Database::Compact(napi_env env, napi_callback_info info) { state->handle->columnDescriptor->column.get(), startPtr, endPtr, - state->bottommost + state->bottommost, + // awaited by DBHandle::close()'s async-work drain; the + // descriptor token is not armed until beginClose(), which a + // self-close does not reach until after that drain returns + &state->handle->compactCancelRequested ); } // signal that execute handler is complete @@ -437,7 +445,10 @@ napi_value Database::CompactSync(napi_env env, napi_callback_info info) { (*dbHandle)->columnDescriptor->column.get(), startPtr, endPtr, - bottommost + bottommost, + // synchronous: counted by operationsInFlight, which beginClose() + // arms the descriptor token ahead of + &(*dbHandle)->descriptor->compactCancelRequested ), "Compact failed" ); @@ -475,7 +486,10 @@ napi_value Database::Destroy(napi_env env, napi_callback_info info) { return nullptr; } if ((*dbHandle)->path.empty()) { - ::napi_throw_error(env, nullptr, "Database path is required for destroy"); + // `path` is populated by open(), so the only way to reach this is a + // handle that was never opened -- say so, rather than sending the + // caller (who did pass a path) to look at path configuration. + ::napi_throw_error(env, nullptr, "Database must be opened before it can be destroyed"); return nullptr; } try { diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index d9466c600..ea18fdc27 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -428,8 +428,10 @@ void DBDescriptor::finishClose(bool destroying) { // Existing operations will decrement operationsInFlight and notify us when done. // Unbounded in-flight operations must abort once `closing` is published // rather than block this untimed wait for their full duration. A count - // scan polls isClosing() itself; a manual compactRange() uses the cancel - // token armed by beginClose(), before DBHandle::close() starts its drain. + // scan polls isClosing() itself; a synchronous manual compactRange() + // uses the cancel token armed by beginClose(). An async one is not + // counted here at all -- it is awaited by the closables sweep below and + // cancelled by the per-handle token DBHandle::close() arms. DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); uint32_t current; while ((current = this->operationsInFlight.load()) != 0) { @@ -474,7 +476,7 @@ void DBDescriptor::finishClose(bool destroying) { } // We want to ensure that all in-memory data is written to disk. Keep the waiting default: an - // immediate flush races transaction-log-store teardown (AGENTS invariant 15). + // immediate flush races transaction-log-store teardown (AGENTS invariant 16). std::string closeError; rocksdb::Status status = testConsumeCloseFlushFailure() ? rocksdb::Status::IOError("Injected database close flush failure") @@ -501,7 +503,11 @@ void DBDescriptor::finishClose(bool destroying) { } for (const auto& columnDesc : pinnedColumns) { if (columnDesc && columnDesc->column) { - this->compactRange(columnDesc->column.get(), nullptr, nullptr, false, false); + // Best-effort: a skipped compaction loses no data, so its status + // deliberately does not become a close failure (which would + // quarantine the path). Only the WaitForCompact below is + // reported -- see README "db.close()". + this->compactRange(columnDesc->column.get(), nullptr, nullptr, false, nullptr); } } } @@ -2254,25 +2260,25 @@ rocksdb::Status DBDescriptor::compactRange( const rocksdb::Slice* start, const rocksdb::Slice* end, bool bottommost, - bool cancellable + std::atomic* canceled ) { std::lock_guard lock(this->compactMutex); DEBUG_LOG("%p DBDescriptor::compactRange Compacting range (bottommost=%d)\n", this, bottommost); rocksdb::CompactRangeOptions options; - // Let a concurrent finishClose() interrupt this compaction rather than - // wait out its full, unbounded duration; see compactCancelRequested. - if (cancellable) { - options.canceled = &this->compactCancelRequested; + // Let a concurrent close interrupt this compaction rather than wait out its + // full, unbounded duration; see compactCancelRequested. + if (canceled) { + options.canceled = canceled; // Test seam (inert unless ROCKSDB_JS_COMPACT_DELAY_MS is set): park here, // still inside the caller's OperationGuard / async-work registration, - // until a foreign close arms the token. That makes the ordering - // observable from JS without depending on how long a real compaction - // happens to run. Bounded so a fixture that never closes still finishes. + // until a close arms the token. That makes the ordering observable from + // JS without depending on how long a real compaction happens to run. + // Bounded so a fixture that never closes still finishes. const int cancelWaitMs = compactCancelDelayMsFlag().load(std::memory_order_relaxed); if (cancelWaitMs > 0) { const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(cancelWaitMs); - while (!this->compactCancelRequested.load() && + while (!canceled->load() && std::chrono::steady_clock::now() < deadline ) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index a36f4e910..63b5887b7 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -301,30 +301,32 @@ struct DBDescriptor final : public std::enable_shared_from_this { * transition that publishes `closing` -- so a close claim and manual * compaction cancellation are one step. Arming it any later reopens the * window this exists to close: `finishClose()` drains - * `operationsInFlight` with an untimed wait and its closables sweep - * drains each handle's async work, and a manual compaction blocks both - * (compactSync() holds an OperationGuard for its whole duration; an - * async compact() released its guard at setup handoff and is awaited by - * `DBHandle::close()` during the sweep). `DestroyDB`/`Shutdown` also - * claim every entry for a path under one lock and then close them - * sequentially, so arming at claim time cancels compactions on - * descriptors whose own `finishClose()` has not started yet. + * `operationsInFlight` with an untimed wait, and a synchronous + * `compactSync()`/`clearSync()` blocks it for the compaction's whole + * duration. `DestroyDB`/`Shutdown` also claim every entry for a path + * under one lock and then close them sequentially, so arming at claim + * time cancels compactions on descriptors whose own `finishClose()` has + * not started yet. * 2. It is never cleared. Close-initiated compaction opts out with * `cancellable = false` instead, since nothing external is waiting on * it, and a descriptor never leaves the closing state. - * 3. It is private to this descriptor and never aliased onto `closing`. + * 3. It covers synchronous compaction only. Async `compact()`/`clear()` + * released their OperationGuard at setup handoff, so this drain does not + * await them; they are awaited by `DBHandle::close()` and cancelled by + * the per-handle token, which a self-close arms *before* it ever reaches + * `beginClose()`. See `DBHandle::compactCancelRequested`. + * 4. It is private to this descriptor and never aliased onto `closing`. * RocksDB writes through this pointer -- `DisableManualCompaction()` * sets the caller's atomic -- and `closing == true` means the registry * has an owner committed to running `finishClose()`. Letting RocksDB * publish that state would leave a half-closed descriptor with no * closer, wedging every later open of the path. * - * Covered by `test/fixtures/fork-compact-cancel-{sync,async}.mts`: both fail - * if the token stops reaching RocksDB, and the sync one additionally fails if - * arming moves past the in-flight drain. Neither separates arming here from - * arming at the top of `finishClose()` -- for a single descriptor those are - * equivalent, and the batch-claim case in (1) is what makes this the right - * home. + * Covered by `test/fixtures/fork-compact-cancel-sync.mts`, which fails both + * if the token stops reaching RocksDB and if arming moves past the in-flight + * drain. It does not separate arming here from arming at the top of + * `finishClose()` -- for a single descriptor those are equivalent, and the + * batch-claim case in (1) is what makes this the right home. */ std::atomic compactCancelRequested{false}; @@ -637,7 +639,7 @@ struct DBDescriptor final : public std::enable_shared_from_this { /** * Flushes every column family's memtable. `allowWriteStall = false` (the RocksDB default) * makes this WAIT, unbounded, on the calling thread — see the `FlushOptions` JSDoc in - * `src/load-binding.ts` and AGENTS invariant 15. + * `src/load-binding.ts` and AGENTS invariant 16. */ rocksdb::Status flush(bool allowWriteStall = false); @@ -655,9 +657,14 @@ struct DBDescriptor final : public std::enable_shared_from_this { const rocksdb::Slice* start, const rocksdb::Slice* end, bool bottommost = false, - // Close-initiated compaction opts out: nothing external is waiting on it, - // and `compactCancelRequested` stays armed for the whole of finishClose(). - bool cancellable = true + // The token RocksDB polls to abandon this compaction. Which one to pass + // is decided by the drain that awaits the caller, not by this class: + // `&DBDescriptor::compactCancelRequested` for a synchronous caller + // holding an OperationGuard, `&DBHandle::compactCancelRequested` for one + // running as admitted async work. See DBHandle::compactCancelRequested. + // Close-initiated compaction passes nullptr and opts out: nothing + // external is waiting on it. + std::atomic* canceled = nullptr ); }; diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index cc124d244..668be5bd9 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -80,7 +80,7 @@ DBHandle::~DBHandle() { /** * Clears all data in the database's column family. */ -rocksdb::Status DBHandle::clear() { +rocksdb::Status DBHandle::clear(std::atomic* compactCanceled) { if (!this->opened() || this->isCancelled()) { DEBUG_LOG("%p Database closed during clear operation\n", this); return rocksdb::Status::Aborted("Database closed during clear operation"); @@ -90,7 +90,9 @@ rocksdb::Status DBHandle::clear() { rocksdb::Status status = this->descriptor->compactRange( this->columnDescriptor->column.get(), nullptr, - nullptr + nullptr, + false, + compactCanceled ); if (!status.ok()) { // A dropped column family is effectively already empty — clear is a no-op. @@ -130,6 +132,12 @@ void DBHandle::close() { DEBUG_LOG("%p DBHandle::close dbDescriptor=%p (ref count = %ld)\n", this, this->descriptor.get(), this->descriptor.use_count()); // cancel all active async work before closing + // + // An async compact() cannot see that flag: its execute callback checks it + // once at entry and then blocks inside RocksDB, which only reads + // CompactRangeOptions::canceled. Arm that token here too, before the drain + // below waits it out -- see DBHandle::compactCancelRequested. + this->compactCancelRequested.store(true); this->cancelAllAsyncWork(); // wait for all async work to complete before closing @@ -340,10 +348,6 @@ void DBHandle::collectTransactionLogSummary(TransactionLogStoreStats& total, uin * @param options - The options for the database. */ void DBHandle::open(const std::string& path, const DBOptions& options) { - // Reset the cancelled state in case this handle was previously closed - // and is being re-opened - this->resetCancelled(); - this->path = path; auto handleParams = DBRegistry::OpenDB(path, options); @@ -354,6 +358,17 @@ void DBHandle::open(const std::string& path, const DBOptions& options) { this->disableWAL = options.disableWAL; this->enableVerificationTable = options.verificationTable; + // Clear the cancellation this handle may carry from a previous close, but + // only now that the new descriptor is adopted. OpenDB() blocks while a + // foreign destroy owns the old path, and that destroy's closables sweep + // closes this still-attached handle -- which re-arms cancellation. Resetting + // before the open would leave that re-arm standing over the newly opened + // descriptor, and since async admission refuses on it, every later get, + // flush, compact and commit through this handle would reject as + // "Database is closing" while the sync methods kept working. + this->resetCancelled(); + this->compactCancelRequested.store(false); + // Note: We cannot attach this handle to the descriptor because we don't // have the smart pointer to the dbHandle instance, so the caller needs to // do it. diff --git a/src/binding/database/db_handle.h b/src/binding/database/db_handle.h index 7fa31cc91..1dca9f47c 100644 --- a/src/binding/database/db_handle.h +++ b/src/binding/database/db_handle.h @@ -1,6 +1,7 @@ #ifndef __DB_HANDLE_H__ #define __DB_HANDLE_H__ +#include #include #include #include @@ -70,6 +71,38 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro uint64_t verificationTableDbId = 0; uint32_t verificationTableColumnFamilyId = 0; + /** + * Cancellation token handed to `rocksdb::CompactRangeOptions::canceled` by + * an async `compact()` issued through this handle. It is the per-handle twin + * of `DBDescriptor::compactCancelRequested`, and the split is load-bearing: + * the two cancellations are armed by different owners at different points of + * teardown. + * + * - The descriptor token is armed by `beginClose()`, which runs before + * `finishClose()` drains `operationsInFlight` -- the wait that a + * `compactSync()` blocks, since it holds an OperationGuard for its whole + * duration. + * - This token is armed by `close()`, which runs before + * `waitForAsyncWorkCompletion()` -- the wait that an async `compact()` + * blocks, since it released its OperationGuard at setup handoff and is + * only awaited by this handle's async-work drain. + * + * A self-close (`db.close()` -> `DBRegistry::CloseDB`) reaches that drain + * *before* it reaches `beginClose()`, so the descriptor token cannot cover + * the async case: the JS thread would park for the compaction's full + * duration. Arming the descriptor token from `CloseDB` instead is not an + * option -- it is never cleared, so one handle closing would permanently + * kill manual compaction for every other handle sharing the descriptor. + * + * Unlike the descriptor token this one IS cleared, by `open()`, because a + * handle outlives its close and may be reopened. + * + * Covered by `test/fixtures/fork-compact-cancel-close.mts` (self-close) and + * `test/fixtures/fork-compact-cancel-async.mts` (foreign destroy, which + * reaches it through `finishClose()`'s closables sweep). + */ + std::atomic compactCancelRequested{false}; + /** * The node environment. */ @@ -117,7 +150,13 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro DBHandle(napi_env env, napi_ref exportsRef); ~DBHandle(); - rocksdb::Status clear(); + /** + * Clears the handle's column family, compacting afterwards to reclaim space. + * `compactCanceled` is the token that compaction hands RocksDB; pass the + * descriptor's for a synchronous caller and this handle's for an async one + * (see `compactCancelRequested`). + */ + rocksdb::Status clear(std::atomic* compactCanceled); void close() override; napi_value get( napi_env env, diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 511dcddb7..9b7dc659b 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -406,7 +406,7 @@ void DBRegistry::DestroyDB(const std::string& path) { if (destroyDelayMs > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(destroyDelayMs)); } - if (testDelayMs("ROCKSDB_JS_DESTROY_FAILURE") > 0) { + if (destroyFailureFlag().load(std::memory_order_relaxed)) { throw rocksdb_js::DBException("Injected database destruction failure"); } DEBUG_LOG("%p DBRegistry::DestroyDB Calling rocksdb::DestroyDB for \"%s\"\n", instance.get(), path.c_str()); diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 2377ef2ef..0df2c9dfd 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -1,7 +1,7 @@ import { RocksDatabase, registryStatus, shutdown } from '../src/index.ts'; import { dbRunner, generateDBPath } from './lib/util.ts'; import { spawn } from 'node:child_process'; -import { chmodSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { chmodSync, existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -17,6 +17,7 @@ const iteratorNextRaceFixture = join(__dirname, 'fixtures', 'fork-iterator-next- const countDestroyRaceFixture = join(__dirname, 'fixtures', 'fork-count-destroy-race.mts'); const compactCancelSyncFixture = join(__dirname, 'fixtures', 'fork-compact-cancel-sync.mts'); const compactCancelAsyncFixture = join(__dirname, 'fixtures', 'fork-compact-cancel-async.mts'); +const compactCancelCloseFixture = join(__dirname, 'fixtures', 'fork-compact-cancel-close.mts'); const quarantinedExitFixture = join(__dirname, 'fixtures', 'fork-quarantined-exit.mts'); const nodeExecutable = process.env.NODE_BINARY ?? @@ -48,7 +49,14 @@ function runDestroyFixture( }); child.on('close', (code, signal) => { clearTimeout(timeout); - if (code === 0 && signal === null) { + const passed = code === 0 && signal === null; + // The child owns dbPath, and several fixtures deliberately end with a + // live or quarantined database, so nothing else removes it. Keep it + // on failure (and under KEEP_FILES) so the state is inspectable. + if (passed && !process.env.KEEP_FILES) { + rmSync(dbPath, { force: true, recursive: true, maxRetries: 3, retryDelay: 500 }); + } + if (passed) { resolve(); } else { reject(new Error(`Destroy fixture failed (code=${code}, signal=${signal})\n${stderr}`)); @@ -90,7 +98,7 @@ describe('Destroy', () => { 'Unsupported operation in read-only mode' ); expect(() => new RocksDatabase(dbPath).destroy()).toThrow( - 'Database path is required for destroy' + 'Database must be opened before it can be destroyed' ); expect(existsSync(dbPath)).toBe(true); db.destroy(); @@ -285,6 +293,12 @@ describe('Destroy', () => { }); }, 20_000); + it('cancels an in-flight asynchronous compaction when the owning handle closes', async () => { + await runDestroyFixture(compactCancelCloseFixture, generateDBPath(), { + ROCKSDB_JS_COMPACT_DELAY_MS: '10000', + }); + }, 20_000); + it('exits cleanly with a descriptor still quarantined at process exit', async () => { await runDestroyFixture(quarantinedExitFixture, generateDBPath(), { ROCKSDB_JS_CLOSE_FLUSH_FAILURE: '2', diff --git a/test/fixtures/fork-compact-cancel-close.mts b/test/fixtures/fork-compact-cancel-close.mts new file mode 100644 index 000000000..2b7bbbbbf --- /dev/null +++ b/test/fixtures/fork-compact-cancel-close.mts @@ -0,0 +1,71 @@ +import { RocksDatabase } from '../../src/index.ts'; +import { setTimeout as delay } from 'node:timers/promises'; + +// The self-close half of the manual-compaction cancellation contract (see +// DBHandle::compactCancelRequested). Unlike the two fork-compact-cancel-* +// fixtures, nothing foreign is involved: this process owns the database and +// closes it while its own async compact() is running. +// +// That ordering is what the descriptor-wide token cannot cover. +// DBRegistry::CloseDB reaches DBHandle::close() -- which cancels async work and +// then waits for it *without a timeout* -- before it reaches +// PurgeIfUnreferenced/beginClose(), the only site that arms the descriptor +// token. An async compact() checks the handle's cancelled flag once at entry +// and then blocks inside RocksDB, which reads CompactRangeOptions::canceled and +// nothing else, so a close that only sets the flag parks the JS thread for the +// compaction's full duration (minutes to hours on a large column family). +// +// The seam parks the compaction until its token is armed, so "close returned +// promptly" is a direct assertion that close armed a token the compaction was +// actually given. Fails if the async compact goes back to the descriptor token, +// or if arming moves after the async-work drain. +const path = process.argv[2]; +const db = RocksDatabase.open(path); +for (let i = 0; i < 20; i++) { + db.putSync(`key-${i}`, i); +} +db.flushSync(); + +// Start the compaction and let the libuv worker park it in the seam before +// closing; otherwise the execute callback's own opened()/isCancelled() check +// rejects it at entry and the token is never exercised. +const compacting = db.compact(); +let compactSettled = false; +const outcome = compacting.then( + () => { + compactSettled = true; + return null; + }, + (error: unknown) => { + compactSettled = true; + return error; + } +); +await delay(250); +if (compactSettled) throw new Error('Compaction settled before close; seam not active'); + +const started = Date.now(); +db.close(); +const closeElapsed = Date.now() - started; + +const compactError = await outcome; + +if (closeElapsed >= 2000) + throw new Error( + `db.close() blocked ${closeElapsed}ms on its own in-flight compact() -- ` + + 'the async compaction is not being handed a cancel token that close() arms' + ); +if (!compactError) + throw new Error('Expected the in-flight compact() to be cancelled by close(), but it resolved'); +if (String(compactError).includes('Database closed during compact operation')) + throw new Error( + 'Compaction was rejected at entry rather than cancelled mid-flight; the fixture ' + + 'no longer exercises the cancel token' + ); +if (!/cancel|paused|incomplete/i.test(String(compactError))) throw compactError; + +// Clearing the token on reopen is covered by test/lifecycle.test.ts's +// close/open cycle, which awaits a compact() on the reopened instance; a token +// left armed cancels it. Re-asserting it here would mean parking a second +// compaction in the seam for its full bound. +db.destroy(); From 1926048236a13bf0e1f1455bcf49b6676e0de288 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 2 Sep 2026 12:00:40 -0600 Subject: [PATCH 43/49] fix(close): cancel attached handles' compactions before finishClose blocks DBHandle::compactCancelRequested was armed only by that handle's own close(), which on a foreign teardown (destroy()/shutdown(), or the last-handle purge from another env) is reached by finishClose()'s closables sweep -- the LAST step of teardown. Three steps before it can each block on the very compaction the sweep would cancel: - the optional compactOnClose pass takes DBDescriptor::compactMutex, which the running manual compaction holds for its whole duration; - WaitForCompact() does not return while a manual compaction is running; - the sweep's own untimed async-work drain then waits it out. The closer holds the path gate throughout, so every concurrent open of that path times out after lifecycleWaitSeconds while it waits -- minutes to hours on a large column family. finishClose() therefore now publishes cancellation on every attached closable up front, before its first blocking step, through a new Closable::cancelBlockingWork() hook (default no-op; only DBHandle overrides it). Arming another thread's handle is safe precisely because this token is per-handle and IS cleared, by DBHandle::open() after it adopts the new descriptor (invariant 20) -- the descriptor-wide token, which is never cleared, could not be used this way. fork-compact-cancel-destroy.mts pins the ordering: with compactOnClose enabled and an async compact() parked in the ROCKSDB_JS_COMPACT_DELAY_MS seam, a foreign destroy completes in ~490ms with the fix and takes 7770ms with the arming loop removed. fork-compact-cancel-async.mts cannot see this -- with compactOnClose off, nothing between the in-flight wait and the sweep touches the compaction. Also: - clear() reports a cancelled leading compaction as "Database closed during clear operation" rather than RocksDB's "Manual compaction paused", which left the caller unable to tell whether the clear had partially applied. Nothing is deleted when the compaction is cancelled: DeleteFilesInRange runs after it. - runDestroyFixture's cleanup rmSync runs after the Promise executor returned, so a Windows EBUSY past maxRetries would have killed the worker instead of failing the test; a leftover directory is not worth that. Verification: pnpm build, pnpm check, pnpm test:native, pnpm test. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0114M7CN7LeYsih4iXsnX7Pa --- AGENTS.md | 21 +++-- README.md | 5 +- src/binding/core/closable.h | 11 +++ src/binding/database/db_descriptor.cpp | 22 ++++- src/binding/database/db_handle.cpp | 6 ++ src/binding/database/db_handle.h | 46 +++++----- test/destroy.test.ts | 16 +++- test/fixtures/fork-compact-cancel-destroy.mts | 90 +++++++++++++++++++ 8 files changed, 182 insertions(+), 35 deletions(-) create mode 100644 test/fixtures/fork-compact-cancel-destroy.mts diff --git a/AGENTS.md b/AGENTS.md index aa5f2048d..e4513407a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -231,7 +231,7 @@ sufficient (env teardown does not honor tsfn acquire counts); see token that caller was given is armed. Lets a fixture hold a compaction across a close claim without depending on how long a real compaction runs; snapshotted in `initializeTestSeams()` like the per-row seams above, so the production path costs one relaxed load per manual - compaction. Used by `test/fixtures/fork-compact-cancel-{sync,async,close}.mts` + compaction. Used by `test/fixtures/fork-compact-cancel-{sync,async,close,destroy}.mts` - `ROCKSDB_JS_CLOSE_FLUSH_FAILURE` - Test-only: number of close-time flushes to fail with an injected `IOError` (a **count**, not a flag; `1` is one failure). More than one is what leaves a descriptor still quarantined at process exit, since the exit-time `DBRegistry::Shutdown()` @@ -385,20 +385,29 @@ sufficient (env teardown does not honor tsfn acquire counts); see untimed `operationsInFlight` wait; it is armed in exactly one place — `beginClose()`, in the same transition that publishes `closing`. `DBHandle::compactCancelRequested` covers an **async** `compact()`/`clear()`, which released its guard at setup handoff and is instead awaited by - `DBHandle::close()`'s untimed async-work drain; it is armed by that same `close()`, immediately - before the drain. The split is not redundancy: a self-close (`db.close()` → + `DBHandle::close()`'s untimed async-work drain. It has **two** arming sites, one per closer: + `DBHandle::close()` arms its own immediately before that drain, and `finishClose()` arms every + still-attached handle's through `Closable::cancelBlockingWork()` before its *first* blocking + step. The second is not belt-and-braces. A foreign `destroy()`/`shutdown()` reaches the handle + only through the closables sweep, which is the last step of teardown — and three earlier steps + can each block on that compaction: the optional `compactOnClose` pass takes `compactMutex`, + which the running compaction holds; `WaitForCompact()` does not return while a manual + compaction runs; then the sweep's own drain waits it out. The closer holds the path gate + throughout, so a late arm times out every concurrent `OpenDB()` for that path. + The split between the two tokens is not redundancy either: a self-close (`db.close()` → `DBRegistry::CloseDB`) reaches `DBHandle::close()` **before** `PurgeIfUnreferenced`/`beginClose()`, so the descriptor token is not yet armed when the async drain starts — a `db.close()` racing its own `db.compact()` would park the JS thread for the compaction's full duration. Arming the descriptor token from `CloseDB` is not the alternative: it is never cleared, so one handle closing would permanently kill manual compaction for every other handle sharing the - process-global descriptor. The per-handle token _is_ cleared, by `DBHandle::open()`, and only + process-global descriptor. That is also what makes it safe for `finishClose()` to arm handles + this thread does not own: the per-handle token _is_ cleared, by `DBHandle::open()`, and only after the new descriptor is adopted (see invariant 20). Neither token is ever aliased onto `closing` itself: RocksDB writes through the pointer it is given (`DisableManualCompaction()` sets the caller's atomic), and `closing` means the registry has an owner committed to running `finishClose()`, which RocksDB must not be able to publish. The full - contract, and what each of `test/fixtures/fork-compact-cancel-{sync,async,close}.mts` does and - does not pin down, is on the two member declarations. + contract, and what each of `test/fixtures/fork-compact-cancel-{sync,async,close,destroy}.mts` + does and does not pin down, is on the two member declarations. Everything the four registry teardown paths do _after_ claiming a descriptor — `finishClose()`, erase-or-quarantine, notify, emit `database:closeFailed` — is one helper, `closeClaimedDescriptors` in `db_registry.cpp`; only the claim predicate differs per caller. Its diff --git a/README.md b/README.md index cd1cea278..1bbe9f0be 100644 --- a/README.md +++ b/README.md @@ -145,8 +145,9 @@ database. A database instance can be reopened once it is closed. A flush failure database quarantined so `shutdown()` can retry without losing unflushed data; an explicit `destroy()` can instead delete it. A failure while waiting for compaction to settle is reported after native teardown completes; the optional `compactOnClose` pass itself is best-effort and its -errors do not fail the close, since a skipped compaction loses no data. All native close errors emit `database:closeFailed`. The quarantine applies to both -writable and read-only opens because both modes share the physical path lifecycle. +errors do not fail the close, since a skipped compaction loses no data. All native close errors +emit `database:closeFailed`. The quarantine applies to both writable and read-only opens because +both modes share the physical path lifecycle. ```typescript const db = RocksDatabase.open('foo'); diff --git a/src/binding/core/closable.h b/src/binding/core/closable.h index a837fb08a..71ea74691 100644 --- a/src/binding/core/closable.h +++ b/src/binding/core/closable.h @@ -6,6 +6,17 @@ namespace rocksdb_js { struct Closable { virtual ~Closable() = default; virtual void close() = 0; + + /** + * Publishes cancellation for work that `close()` will later wait on but + * that cannot poll a flag from where it runs -- today only a manual + * RocksDB compaction, which abandons its range solely through the + * `CompactRangeOptions::canceled` pointer it was handed. Teardown calls + * this on every attached closable before its first step that can block on + * such work, which is earlier than the closables sweep that closes them. + * Default no-op: most closables have nothing that outlives a flag check. + */ + virtual void cancelBlockingWork() {} }; } // namespace rocksdb_js diff --git a/src/binding/database/db_descriptor.cpp b/src/binding/database/db_descriptor.cpp index ea18fdc27..c497548c9 100644 --- a/src/binding/database/db_descriptor.cpp +++ b/src/binding/database/db_descriptor.cpp @@ -421,6 +421,24 @@ void DBDescriptor::finishClose(bool destroying) { DEBUG_LOG("%p DBDescriptor::close Closing \"%s\" (mode=%s read-only=%s closables=%zu columns=%zu transactions=%zu)\n", this, this->path.c_str(), this->mode == DBMode::Optimistic ? "optimistic" : "pessimistic", this->readOnly ? "true" : "false", this->closables.size(), this->columns.size(), this->transactions.size()); + // Publish cancellation for work that later steps here will block on but + // that cannot poll `closing` from where it runs. A manual compaction + // admitted through one of these handles blocks the optional close-time + // compaction (compactMutex), then WaitForCompact(), then the closables + // sweep's own async drain -- and RocksDB abandons it only through the token + // that handle gave it. Arming at the sweep would be three blocking steps too + // late, which is why this runs first. Safe to do to handles this thread does + // not own: the per-handle token is cleared by DBHandle::open(), unlike the + // descriptor-wide one. + { + std::lock_guard closablesLock(this->txnsMutex); + for (auto& [key, weakClosable] : this->closables) { + if (auto closable = weakClosable.lock()) { + closable->cancelBlockingWork(); + } + } + } + const bool retryingClose = this->closeWorkersStopped; if (!this->closeWorkersStopped) { // Wait for all in-flight operations to complete before cleanup. @@ -430,8 +448,8 @@ void DBDescriptor::finishClose(bool destroying) { // rather than block this untimed wait for their full duration. A count // scan polls isClosing() itself; a synchronous manual compactRange() // uses the cancel token armed by beginClose(). An async one is not - // counted here at all -- it is awaited by the closables sweep below and - // cancelled by the per-handle token DBHandle::close() arms. + // counted here at all -- it is awaited by the closables sweep below, + // and by then the per-handle token armed above has already cancelled it. DEBUG_LOG("%p DBDescriptor::close Waiting for %u in-flight operations \"%s\"\n", this, this->operationsInFlight.load(), this->path.c_str()); uint32_t current; while ((current = this->operationsInFlight.load()) != 0) { diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index 668be5bd9..1a1def8cc 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -101,6 +101,12 @@ rocksdb::Status DBHandle::clear(std::atomic* compactCanceled) { if (status.IsColumnFamilyDropped()) { return rocksdb::Status::OK(); } + if (status.IsIncomplete() && (this->isCancelled() || this->descriptor->isClosing())) { + // A close cancelled the compaction this clear starts with, so nothing + // was deleted. Reporting RocksDB's "Manual compaction paused" would + // leave the caller unable to tell whether the clear partially applied. + return rocksdb::Status::Aborted("Database closed during clear operation"); + } return status; } // it appears we do not need to call WaitForCompact for this to work diff --git a/src/binding/database/db_handle.h b/src/binding/database/db_handle.h index 1dca9f47c..83b1b038f 100644 --- a/src/binding/database/db_handle.h +++ b/src/binding/database/db_handle.h @@ -73,36 +73,34 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro /** * Cancellation token handed to `rocksdb::CompactRangeOptions::canceled` by - * an async `compact()` issued through this handle. It is the per-handle twin - * of `DBDescriptor::compactCancelRequested`, and the split is load-bearing: - * the two cancellations are armed by different owners at different points of - * teardown. + * an async `compact()`/`clear()` issued through this handle. Its twin, + * `DBDescriptor::compactCancelRequested`, serves the synchronous callers; + * the split exists because the two are awaited by different, differently + * ordered drains: * - * - The descriptor token is armed by `beginClose()`, which runs before - * `finishClose()` drains `operationsInFlight` -- the wait that a - * `compactSync()` blocks, since it holds an OperationGuard for its whole - * duration. - * - This token is armed by `close()`, which runs before - * `waitForAsyncWorkCompletion()` -- the wait that an async `compact()` - * blocks, since it released its OperationGuard at setup handoff and is - * only awaited by this handle's async-work drain. + * - a sync caller holds an `OperationGuard`, so it is awaited by + * `finishClose()`'s `operationsInFlight` wait, which `beginClose()` + * precedes; + * - an async caller released its guard at setup handoff, so it is awaited + * by `close()`'s async-work drain -- and `DBRegistry::CloseDB` reaches + * that drain *before* `beginClose()`, so the descriptor token is still + * unarmed there. * - * A self-close (`db.close()` -> `DBRegistry::CloseDB`) reaches that drain - * *before* it reaches `beginClose()`, so the descriptor token cannot cover - * the async case: the JS thread would park for the compaction's full - * duration. Arming the descriptor token from `CloseDB` instead is not an - * option -- it is never cleared, so one handle closing would permanently - * kill manual compaction for every other handle sharing the descriptor. + * Arming the descriptor token earlier is not the alternative: it is never + * cleared, so one handle closing would kill manual compaction for every + * other handle on the shared descriptor. This token is per-handle and IS + * cleared, by `open()`, which is also what makes it safe for + * `finishClose()` to arm it on handles it does not own (see + * `cancelBlockingWork()`). * - * Unlike the descriptor token this one IS cleared, by `open()`, because a - * handle outlives its close and may be reopened. - * - * Covered by `test/fixtures/fork-compact-cancel-close.mts` (self-close) and - * `test/fixtures/fork-compact-cancel-async.mts` (foreign destroy, which - * reaches it through `finishClose()`'s closables sweep). + * Armed by `close()` (self-close) and by `finishClose()` before its first + * blocking step (foreign close); covered by + * `test/fixtures/fork-compact-cancel-{close,async}.mts`. */ std::atomic compactCancelRequested{false}; + void cancelBlockingWork() override { this->compactCancelRequested.store(true); } + /** * The node environment. */ diff --git a/test/destroy.test.ts b/test/destroy.test.ts index 0df2c9dfd..b6d688b1b 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -18,6 +18,7 @@ const countDestroyRaceFixture = join(__dirname, 'fixtures', 'fork-count-destroy- const compactCancelSyncFixture = join(__dirname, 'fixtures', 'fork-compact-cancel-sync.mts'); const compactCancelAsyncFixture = join(__dirname, 'fixtures', 'fork-compact-cancel-async.mts'); const compactCancelCloseFixture = join(__dirname, 'fixtures', 'fork-compact-cancel-close.mts'); +const compactCancelDestroyFixture = join(__dirname, 'fixtures', 'fork-compact-cancel-destroy.mts'); const quarantinedExitFixture = join(__dirname, 'fixtures', 'fork-quarantined-exit.mts'); const nodeExecutable = process.env.NODE_BINARY ?? @@ -54,7 +55,14 @@ function runDestroyFixture( // live or quarantined database, so nothing else removes it. Keep it // on failure (and under KEEP_FILES) so the state is inspectable. if (passed && !process.env.KEEP_FILES) { - rmSync(dbPath, { force: true, recursive: true, maxRetries: 3, retryDelay: 500 }); + // This runs after the Promise executor returned, so a throw here + // (Windows EBUSY past maxRetries) would be an uncaught exception + // that kills the worker rather than a failed test. + try { + rmSync(dbPath, { force: true, recursive: true, maxRetries: 3, retryDelay: 500 }); + } catch { + // leftover directory only; the fixture itself passed + } } if (passed) { resolve(); @@ -299,6 +307,12 @@ describe('Destroy', () => { }); }, 20_000); + it('cancels an in-flight asynchronous compaction before close-time compaction blocks on it', async () => { + await runDestroyFixture(compactCancelDestroyFixture, generateDBPath(), { + ROCKSDB_JS_COMPACT_DELAY_MS: '8000', + }); + }, 20_000); + it('exits cleanly with a descriptor still quarantined at process exit', async () => { await runDestroyFixture(quarantinedExitFixture, generateDBPath(), { ROCKSDB_JS_CLOSE_FLUSH_FAILURE: '2', diff --git a/test/fixtures/fork-compact-cancel-destroy.mts b/test/fixtures/fork-compact-cancel-destroy.mts new file mode 100644 index 000000000..3f547c820 --- /dev/null +++ b/test/fixtures/fork-compact-cancel-destroy.mts @@ -0,0 +1,90 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { setTimeout as delay } from 'node:timers/promises'; +import { Worker } from 'node:worker_threads'; + +// A foreign destroy must cancel an async compaction *before* the first step of +// finishClose() that blocks on it -- not when the closables sweep eventually +// closes the owning handle. +// +// finishClose() runs: in-flight wait (an async compact holds no OperationGuard, +// so this returns immediately) -> flush -> the optional compactOnClose pass, +// which takes DBDescriptor::compactMutex -> WaitForCompact(), which does not +// return while a manual compaction is running -> only then the closables sweep. +// The running compaction holds compactMutex, so with cancellation deferred to +// the sweep the destroy parks for the compaction's full remaining duration +// while holding destroyingPaths -- and every concurrent open of the path then +// times out after lifecycleWaitSeconds. +// +// compactOnClose is enabled here to make that ordering observable with the +// existing seam: the parked compaction holds compactMutex, so a late arm shows +// up as a slow destroy. fork-compact-cancel-async.mts cannot see it -- the seam +// parks before db->CompactRange, so with compactOnClose off nothing between the +// in-flight wait and the sweep touches the compaction. +const path = process.argv[2]; +RocksDatabase.config({ compactOnClose: true }); +const db = RocksDatabase.open(path); +for (let i = 0; i < 20; i++) { + db.putSync(`key-${i}`, i); +} +db.flushSync(); + +const worker = new Worker(createWorkerBootstrapScript('./test/workers/destroy-open-worker.mts'), { + eval: true, + workerData: { path, destroyStartDelayMs: 0 }, +}); + +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} + +const ready = await nextMessage(); +if (!ready.ready) throw new Error(`Worker failed to initialize: ${JSON.stringify(ready)}`); + +const compacting = db.compact(); +let compactSettled = false; +const outcome = compacting.then( + () => { + compactSettled = true; + return null; + }, + (error: unknown) => { + compactSettled = true; + return error; + } +); +await delay(250); +if (compactSettled) throw new Error('Compaction settled before the destroy claim; seam not active'); + +const started = Date.now(); +worker.postMessage({ destroy: true }); +const destroying = await nextMessage(); +if (!destroying.destroying) + throw new Error(`Worker did not start destroying: ${JSON.stringify(destroying)}`); + +const destroyResult = await nextMessage(); +const elapsed = Date.now() - started; +if (!destroyResult.destroyed) throw new Error(`Destroy failed: ${JSON.stringify(destroyResult)}`); +if (elapsed >= 2500) + throw new Error( + `destroy() took ${elapsed}ms while an async compact() was in flight -- ` + + 'finishClose() is not cancelling attached handles before it blocks on them' + ); + +const compactError = await outcome; +if (!compactError) + throw new Error('Expected the in-flight compact() to be cancelled by the foreign destroy'); +if (String(compactError).includes('Database closed during compact operation')) + throw new Error( + 'Compaction was rejected at entry rather than cancelled mid-flight; the fixture ' + + 'no longer exercises the cancel token' + ); +if (!/cancel|paused|incomplete/i.test(String(compactError))) throw compactError; + +if (registryStatus().some((entry) => entry.path === path)) + throw new Error('Expected destroy to fully clear the registry entry'); + +await worker.terminate(); From 41bd966b9b06af1718f8e20098815f35e828d27f Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 2 Sep 2026 12:24:14 -0600 Subject: [PATCH 44/49] docs(close): correct the compaction-token contract after the second arm Review nits on 19260482, all of them staleness the previous commit introduced: - DBHandle::compactCancelRequested still cited fork-compact-cancel-{close,async} as its coverage, when fork-compact-cancel-destroy is what actually proves the foreign-close arm happens before finishClose() can block. Its 26-line restatement of the two-token split is now one pointer to AGENTS.md invariant 6, which carries that contract. - DBDescriptor::compactCancelRequested announced "three parts" over four, and its part 3 described the per-handle token as armed only by a self-close. - Invariant 6 pointed at the member declarations for the fixture contract while they pointed back at it; it now states what each of the four fixtures does and does not pin down, and the members point at it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0114M7CN7LeYsih4iXsnX7Pa --- AGENTS.md | 13 +++++++---- src/binding/database/db_descriptor.h | 10 +++++---- src/binding/database/db_handle.h | 32 +++++++++------------------- 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e4513407a..1f52fa1cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -387,7 +387,7 @@ sufficient (env teardown does not honor tsfn acquire counts); see `compact()`/`clear()`, which released its guard at setup handoff and is instead awaited by `DBHandle::close()`'s untimed async-work drain. It has **two** arming sites, one per closer: `DBHandle::close()` arms its own immediately before that drain, and `finishClose()` arms every - still-attached handle's through `Closable::cancelBlockingWork()` before its *first* blocking + still-attached handle's through `Closable::cancelBlockingWork()` before its _first_ blocking step. The second is not belt-and-braces. A foreign `destroy()`/`shutdown()` reaches the handle only through the closables sweep, which is the last step of teardown — and three earlier steps can each block on that compaction: the optional `compactOnClose` pass takes `compactMutex`, @@ -405,9 +405,14 @@ sufficient (env teardown does not honor tsfn acquire counts); see after the new descriptor is adopted (see invariant 20). Neither token is ever aliased onto `closing` itself: RocksDB writes through the pointer it is given (`DisableManualCompaction()` sets the caller's atomic), and `closing` means the registry has an - owner committed to running `finishClose()`, which RocksDB must not be able to publish. The full - contract, and what each of `test/fixtures/fork-compact-cancel-{sync,async,close,destroy}.mts` - does and does not pin down, is on the two member declarations. + owner committed to running `finishClose()`, which RocksDB must not be able to publish. + `test/fixtures/fork-compact-cancel-{sync,async,close,destroy}.mts` cover the four close paths in + the same order as above; each fails if `options.canceled` stops reaching RocksDB, `sync` also + fails if the descriptor arm moves past the in-flight drain, and `destroy` also fails if the + foreign arm moves back to the closables sweep. None of them separates arming in `beginClose()` + from arming at the top of `finishClose()` — for a single descriptor those are equivalent, and + what makes `beginClose()` the right home is that `DestroyDB`/`Shutdown` claim every entry for a + path under one lock and then close them sequentially. Everything the four registry teardown paths do _after_ claiming a descriptor — `finishClose()`, erase-or-quarantine, notify, emit `database:closeFailed` — is one helper, `closeClaimedDescriptors` in `db_registry.cpp`; only the claim predicate differs per caller. Its diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index 63b5887b7..b751420ee 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -295,7 +295,7 @@ struct DBDescriptor final : public std::enable_shared_from_this { /** * Cancellation token handed to `rocksdb::CompactRangeOptions::canceled` by - * every cancellable `compactRange()`. Contract, in three parts: + * every cancellable *synchronous* `compactRange()`. Contract, in four parts: * * 1. It is armed in exactly one place -- `beginClose()`, under the same * transition that publishes `closing` -- so a close claim and manual @@ -312,9 +312,11 @@ struct DBDescriptor final : public std::enable_shared_from_this { * it, and a descriptor never leaves the closing state. * 3. It covers synchronous compaction only. Async `compact()`/`clear()` * released their OperationGuard at setup handoff, so this drain does not - * await them; they are awaited by `DBHandle::close()` and cancelled by - * the per-handle token, which a self-close arms *before* it ever reaches - * `beginClose()`. See `DBHandle::compactCancelRequested`. + * await them; they are awaited by `DBHandle::close()`'s async-work drain + * and cancelled by the per-handle token, which a self-close arms before + * it ever reaches `beginClose()` and a foreign close arms from + * `finishClose()` ahead of its first blocking step. See + * `DBHandle::compactCancelRequested` and AGENTS.md invariant 6. * 4. It is private to this descriptor and never aliased onto `closing`. * RocksDB writes through this pointer -- `DisableManualCompaction()` * sets the caller's atomic -- and `closing == true` means the registry diff --git a/src/binding/database/db_handle.h b/src/binding/database/db_handle.h index 83b1b038f..b8bd74a14 100644 --- a/src/binding/database/db_handle.h +++ b/src/binding/database/db_handle.h @@ -73,29 +73,17 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro /** * Cancellation token handed to `rocksdb::CompactRangeOptions::canceled` by - * an async `compact()`/`clear()` issued through this handle. Its twin, - * `DBDescriptor::compactCancelRequested`, serves the synchronous callers; - * the split exists because the two are awaited by different, differently - * ordered drains: + * an async `compact()`/`clear()` issued through this handle -- the twin of + * `DBDescriptor::compactCancelRequested`, which serves the synchronous + * callers. Armed by `close()` (self-close) and, for a foreign close, by + * `finishClose()` before its first blocking step, via + * `cancelBlockingWork()`. Safe to arm from a thread that does not own this + * handle because this token is cleared by `open()`; the descriptor's is + * never cleared. * - * - a sync caller holds an `OperationGuard`, so it is awaited by - * `finishClose()`'s `operationsInFlight` wait, which `beginClose()` - * precedes; - * - an async caller released its guard at setup handoff, so it is awaited - * by `close()`'s async-work drain -- and `DBRegistry::CloseDB` reaches - * that drain *before* `beginClose()`, so the descriptor token is still - * unarmed there. - * - * Arming the descriptor token earlier is not the alternative: it is never - * cleared, so one handle closing would kill manual compaction for every - * other handle on the shared descriptor. This token is per-handle and IS - * cleared, by `open()`, which is also what makes it safe for - * `finishClose()` to arm it on handles it does not own (see - * `cancelBlockingWork()`). - * - * Armed by `close()` (self-close) and by `finishClose()` before its first - * blocking step (foreign close); covered by - * `test/fixtures/fork-compact-cancel-{close,async}.mts`. + * Why there are two tokens, why each is armed where it is, and what each of + * `test/fixtures/fork-compact-cancel-{sync,async,close,destroy}.mts` does + * and does not pin down: AGENTS.md invariant 6. */ std::atomic compactCancelRequested{false}; From 9fdc7c3d0d4aee27342bc1153c62b019a1d3fe03 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 2 Sep 2026 12:28:18 -0600 Subject: [PATCH 45/49] docs(close): say what the compaction fixtures do not prove The invariant-6 rewrite claimed the four fork-compact-cancel fixtures fail if `options.canceled` stops reaching RocksDB, which is true but reads as if they exercise RocksDB abandoning a live compaction. They do not, on purpose: ROCKSDB_JS_COMPACT_DELAY_MS parks *before* CompactRange so a fixture never depends on how long a real compaction runs, and unwiring the token is caught because the compaction then succeeds rather than returning Incomplete. What the fixtures pin down is our half of the contract -- which token is armed, how early, and that it is handed over. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0114M7CN7LeYsih4iXsnX7Pa --- AGENTS.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1f52fa1cc..079a38f3b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -407,12 +407,16 @@ sufficient (env teardown does not honor tsfn acquire counts); see (`DisableManualCompaction()` sets the caller's atomic), and `closing` means the registry has an owner committed to running `finishClose()`, which RocksDB must not be able to publish. `test/fixtures/fork-compact-cancel-{sync,async,close,destroy}.mts` cover the four close paths in - the same order as above; each fails if `options.canceled` stops reaching RocksDB, `sync` also - fails if the descriptor arm moves past the in-flight drain, and `destroy` also fails if the - foreign arm moves back to the closables sweep. None of them separates arming in `beginClose()` - from arming at the top of `finishClose()` — for a single descriptor those are equivalent, and - what makes `beginClose()` the right home is that `DestroyDB`/`Shutdown` claim every entry for a - path under one lock and then close them sequentially. + the same order as above. What they pin down is **our** half of the contract — that the right + token is armed, early enough, and handed to RocksDB: each fails if `options.canceled` stops + being passed (the compaction then succeeds instead of returning `Incomplete`), `sync` also fails + if the descriptor arm moves past the in-flight drain, and `destroy` also fails if the foreign + arm moves back to the closables sweep. They do **not** exercise RocksDB aborting a compaction + already in progress: `ROCKSDB_JS_COMPACT_DELAY_MS` parks before `CompactRange`, deliberately, so + the fixtures do not depend on how long a real compaction runs. Nor does any of them separate + arming in `beginClose()` from arming at the top of `finishClose()` — for a single descriptor + those are equivalent, and what makes `beginClose()` the right home is that `DestroyDB`/`Shutdown` + claim every entry for a path under one lock and then close them sequentially. Everything the four registry teardown paths do _after_ claiming a descriptor — `finishClose()`, erase-or-quarantine, notify, emit `database:closeFailed` — is one helper, `closeClaimedDescriptors` in `db_registry.cpp`; only the claim predicate differs per caller. Its From e8eb0d9b89e5bc4ee65159a57add639d131e6c87 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 2 Sep 2026 13:25:18 -0600 Subject: [PATCH 46/49] fix(lifecycle): attach handles before releasing open gate Co-Authored-By: GPT-5 Codex --- AGENTS.md | 37 +++++++++++---- src/binding/core/test_seam.h | 9 ++++ src/binding/database/database.cpp | 5 --- src/binding/database/db_handle.cpp | 33 +++----------- src/binding/database/db_registry.cpp | 52 ++++++++++++++++++---- src/binding/database/db_registry.h | 14 +++--- test/destroy.test.ts | 7 +++ test/fixtures/fork-open-attach-destroy.mts | 49 ++++++++++++++++++++ test/workers/open-attach-worker.mts | 25 +++++++++++ 9 files changed, 174 insertions(+), 57 deletions(-) create mode 100644 test/fixtures/fork-open-attach-destroy.mts create mode 100644 test/workers/open-attach-worker.mts diff --git a/AGENTS.md b/AGENTS.md index 079a38f3b..4223dc0cb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -221,6 +221,10 @@ sufficient (env teardown does not honor tsfn acquire counts); see rising edge emits); malformed/negative falls back to the default - `ROCKSDB_JS_DESTROY_DELAY_MS` - Test-only: delay after descriptor teardown and before physical database destruction (widens same-path reopen races) +- `ROCKSDB_JS_OPEN_ATTACH_DELAY_MS` - Test-only: delay after `DBRegistry::OpenDB()` has atomically + adopted and attached a handle, but before the native open returns to JavaScript (proves a forced + destroy cannot claim the descriptor during the former return/adopt/attach gap). Snapshotted in + `initializeTestSeams()` so every open avoids a `getenv()` call - `ROCKSDB_JS_ITERATOR_NEXT_DELAY_MS` / `ROCKSDB_JS_COUNT_DELAY_MS` - Test-only: per-row delays in `DBIterator::Next()` and `DBIteratorHandle::countRemaining()`. Both are read **once** in `initializeTestSeams()` rather than per row: these are the two per-row native loops, and a @@ -785,18 +789,33 @@ sufficient (env teardown does not honor tsfn acquire counts); see database with a sticky RocksDB background error (`test/background-error.test.ts` used to, which is why its fixtures now tear down with `destroy()` rather than `close()`). -20. **A reopened handle clears its cancellation only after it has adopted the new descriptor**: +20. **A reopened handle clears its cancellation only after `DBRegistry::OpenDB()` finishes its + lifecycle waits**: `DBHandle::close()` publishes `cancelled` (and, per invariant 6, the per-handle compaction token), and every async admission refuses while either stands — so `DBHandle::open()` has to - clear them for a documented close/reopen cycle to work. It must do so **after** - `DBRegistry::OpenDB()` returns, not before the call. `OpenDB()` blocks while a foreign + clear them for a documented close/reopen cycle to work. `OpenDB()` blocks while a foreign `destroy()`/`shutdown()` owns the old path, and that teardown's closables sweep force-closes - this still-attached handle mid-wait, re-arming both flags. Clearing first therefore leaves the - re-arm standing over the _newly_ opened descriptor: `opened()` reports true and the sync - methods keep working, while every `get`/`flush`/`compact`/`clear`/`commit` on that instance - rejects with "Database is closing" for the rest of its life. The general shape — publish-then- - clear across a blocking call another thread can publish into — is the same one invariant 6's - cancel token and invariant 17's admission mutex exist to keep out of the teardown paths. + this still-attached handle mid-wait, re-arming both flags. Clearing before those waits therefore + leaves the re-arm standing over the _newly_ opened descriptor. The reset now runs after every + wait and while `databasesMutex` still excludes a new closer, immediately before the handle is + attached to the selected descriptor. Because the reset takes the handle's async-work + `waitMutex` and attachment takes the descriptor's `txnsMutex`, the established order is + `databasesMutex` → `waitMutex` → `txnsMutex`; code holding `waitMutex` must release it before + calling back into the registry. Async-state destructors that retry `PurgeIfUnreferenced()` + release their descriptor before the base destructor unregisters async work, preserving that + order. + +21. **Handle adoption and descriptor attachment are one registry-locked publication**: + `DBRegistry::OpenDB()` selects the descriptor and column family, clears stale close cancellation, + publishes every per-open handle field, and inserts the handle into `DBDescriptor::closables` + before releasing `databasesMutex`. A forced `destroy()`/`shutdown()` claims under the same mutex, + so it either precedes the open or sees the fully adopted handle in the closables sweep. Returning + `DBHandleParams` and attaching in `Database::Open()` left a gap where teardown could reset + `descriptor->db` while the invisible handle retained a `ColumnFamilyHandle`; destroying that + column handle after its DB is a native use-after-free. No handle shared-pointer field may be read + after the registry lock drops, because a foreign sweep may immediately reset it. Covered by + `test/fixtures/fork-open-attach-destroy.mts`; `ROCKSDB_JS_OPEN_ATTACH_DELAY_MS` widens the point + after atomic publication so the fixture can assert the exact closables count before destroy. ## Debugging native heap corruption diff --git a/src/binding/core/test_seam.h b/src/binding/core/test_seam.h index 2f50db73e..1860070f7 100644 --- a/src/binding/core/test_seam.h +++ b/src/binding/core/test_seam.h @@ -66,6 +66,13 @@ inline std::atomic& compactCancelDelayMsFlag() { return delayMs; } +// Delays DBHandle::open() after the registry has returned. A handle must +// already be adopted and attached while this delay is observable. +inline std::atomic& openAttachDelayMsFlag() { + static std::atomic delayMs{0}; + return delayMs; +} + inline void initializeTestSeams() { static std::once_flag initialized; std::call_once(initialized, []() { @@ -81,6 +88,8 @@ inline void initializeTestSeams() { testDelayMs("ROCKSDB_JS_COUNT_DELAY_MS"), std::memory_order_relaxed); compactCancelDelayMsFlag().store( testDelayMs("ROCKSDB_JS_COMPACT_DELAY_MS"), std::memory_order_relaxed); + openAttachDelayMsFlag().store( + testDelayMs("ROCKSDB_JS_OPEN_ATTACH_DELAY_MS"), std::memory_order_relaxed); }); } diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index c95942979..cf42566a1 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -2140,11 +2140,6 @@ napi_value Database::Open(napi_env env, napi_callback_info info) { try { (*dbHandle)->open(path, dbHandleOptions); - - // now that the database is open and the dbHandle has a reference to - // the descriptor, we can attach the database instance's smart_ptr to - // the descriptor so it gets cleaned up when the descriptor is closed - (*dbHandle)->descriptor->attach(*dbHandle); } catch (const std::exception& e) { DEBUG_LOG("%p Database::Open Error: %s\n", dbHandle->get(), e.what()); ::napi_throw_error(env, nullptr, e.what()); diff --git a/src/binding/database/db_handle.cpp b/src/binding/database/db_handle.cpp index 1a1def8cc..2200147e9 100644 --- a/src/binding/database/db_handle.cpp +++ b/src/binding/database/db_handle.cpp @@ -4,6 +4,7 @@ #include "database/db_registry.h" #include "database/db_settings.h" #include "transaction_log/transaction_log_store_registry.h" +#include "core/test_seam.h" #include "core/verification_table.h" namespace rocksdb_js { @@ -354,32 +355,12 @@ void DBHandle::collectTransactionLogSummary(TransactionLogStoreStats& total, uin * @param options - The options for the database. */ void DBHandle::open(const std::string& path, const DBOptions& options) { - this->path = path; - - auto handleParams = DBRegistry::OpenDB(path, options); - this->columnDescriptor = std::move(handleParams->columnDescriptor); - this->descriptor = std::move(handleParams->descriptor); - this->verificationTableDbId = this->descriptor->vtEpoch; - this->verificationTableColumnFamilyId = this->columnDescriptor->column->GetID(); - this->disableWAL = options.disableWAL; - this->enableVerificationTable = options.verificationTable; - - // Clear the cancellation this handle may carry from a previous close, but - // only now that the new descriptor is adopted. OpenDB() blocks while a - // foreign destroy owns the old path, and that destroy's closables sweep - // closes this still-attached handle -- which re-arms cancellation. Resetting - // before the open would leave that re-arm standing over the newly opened - // descriptor, and since async admission refuses on it, every later get, - // flush, compact and commit through this handle would reject as - // "Database is closing" while the sync methods kept working. - this->resetCancelled(); - this->compactCancelRequested.store(false); - - // Note: We cannot attach this handle to the descriptor because we don't - // have the smart pointer to the dbHandle instance, so the caller needs to - // do it. - - // at this point, the DBDescriptor has at least 2 refs: the registry and this handle + DBRegistry::OpenDB(this->shared_from_this(), path, options); + + const int openAttachDelayMs = openAttachDelayMsFlag().load(std::memory_order_relaxed); + if (openAttachDelayMs > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(openAttachDelayMs)); + } } /** diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 9b7dc659b..1fa6c5179 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -439,25 +439,41 @@ void DBRegistry::Init(napi_env env, napi_value exports) { /** * Open a RocksDB database with column family, caches it in the registry, and - * return a handle to it. + * attach the provided handle to it. * + * @param handle - The handle that will own the selected database and column family. * @param path - The filesystem path to the database. * @param options - The options for the database. - * @return A handle to the RocksDB database including the transaction db and - * column family handle. */ -std::unique_ptr DBRegistry::OpenDB(const std::string& path, const DBOptions& options) { +void DBRegistry::OpenDB( + const std::shared_ptr& handle, + const std::string& path, + const DBOptions& options +) { // ensure the registry has already been initialized if (!instance) { DEBUG_LOG("DBRegistry::OpenDB Registry not initialized!\n"); throw rocksdb_js::DBException("DBRegistry not initialized!"); } + if (!handle) { + throw rocksdb_js::DBException("Cannot open a database with an invalid handle"); + } + if (handle->descriptor && + (!handle->descriptor->isClosing() || handle->descriptor->path != path || + handle->descriptor->readOnly != options.readOnly) + ) { + throw rocksdb_js::DBException( + "Cannot open database \"" + path + "\": handle is still attached to database \"" + + handle->descriptor->path + "\"" + ); + } DEBUG_LOG("%p DBRegistry::OpenDB Opening database \"%s\" (mode=%s read-only=%s column family=\"%s\")\n", instance.get(), path.c_str(), options.mode == DBMode::Optimistic ? "optimistic" : "pessimistic", options.readOnly ? "true" : "false", options.name.empty() ? "default" : options.name.c_str()); std::unordered_map> columns; std::string name = options.name.empty() ? "default" : options.name; - std::shared_ptr descriptor; + std::shared_ptr previousDescriptor; + std::shared_ptr previousColumnDescriptor; std::unique_lock lock(instance->databasesMutex); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(DBSettings::getInstance().getLifecycleWaitSeconds()); @@ -687,9 +703,29 @@ std::unique_ptr DBRegistry::OpenDB(const std::string& path, cons columnDescriptor = columns[rocksdb::kDefaultColumnFamilyName]; } - std::unique_ptr handle = std::make_unique(entry.descriptor, columnDescriptor); - DEBUG_LOG("%p DBRegistry::OpenDB Created DBHandleParams %p for \"%s\" (ref count = %ld)\n", instance.get(), handle.get(), path.c_str(), entry.descriptor.use_count()); - return handle; + std::string openedPath(path); + const uint64_t verificationTableDbId = entry.descriptor->vtEpoch; + const uint32_t verificationTableColumnFamilyId = columnDescriptor->column->GetID(); + + // The registry lock is the lifecycle linearization point: reset is ordered + // after every wait above, and teardown cannot claim this descriptor between + // publishing the handle's native state and making it visible in closables. + handle->resetCancelled(); + handle->compactCancelRequested.store(false); + entry.descriptor->attach(handle); + + previousColumnDescriptor = std::move(handle->columnDescriptor); + previousDescriptor = std::move(handle->descriptor); + handle->path = std::move(openedPath); + handle->columnDescriptor = std::move(columnDescriptor); + handle->descriptor = entry.descriptor; + handle->verificationTableDbId = verificationTableDbId; + handle->verificationTableColumnFamilyId = verificationTableColumnFamilyId; + handle->disableWAL = options.disableWAL; + handle->enableVerificationTable = options.verificationTable; + + DEBUG_LOG("%p DBRegistry::OpenDB Attached DBHandle %p for \"%s\" (ref count = %ld)\n", + instance.get(), handle.get(), path.c_str(), entry.descriptor.use_count()); } /** diff --git a/src/binding/database/db_registry.h b/src/binding/database/db_registry.h index bcfe781b4..0260d74b2 100644 --- a/src/binding/database/db_registry.h +++ b/src/binding/database/db_registry.h @@ -57,14 +57,6 @@ struct CloseResult final { }; -struct DBHandleParams final { - std::shared_ptr descriptor; - std::shared_ptr columnDescriptor; - - DBHandleParams(std::shared_ptr descriptor, std::shared_ptr columnDescriptor) - : descriptor(std::move(descriptor)), columnDescriptor(std::move(columnDescriptor)) {} -}; - /** * Tracks all RocksDB databases instances using a RocksDBDescriptor that * contains a weak reference to the database and column families. @@ -106,7 +98,11 @@ class DBRegistry final { #endif static void DestroyDB(const std::string& path); static void Init(napi_env env, napi_value exports); - static std::unique_ptr OpenDB(const std::string& path, const DBOptions& options); + static void OpenDB( + const std::shared_ptr& handle, + const std::string& path, + const DBOptions& options + ); static void PurgeAll(); static CloseResult PurgeIfUnreferenced(const std::string& path, bool readOnly); static napi_value RegistryStatus(napi_env env, napi_callback_info info); diff --git a/test/destroy.test.ts b/test/destroy.test.ts index b6d688b1b..2b58be0e6 100644 --- a/test/destroy.test.ts +++ b/test/destroy.test.ts @@ -6,6 +6,7 @@ import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; const destroyOpenFixture = join(__dirname, 'fixtures', 'fork-destroy-open.mts'); +const openAttachDestroyFixture = join(__dirname, 'fixtures', 'fork-open-attach-destroy.mts'); const destroyFailureFixture = join(__dirname, 'fixtures', 'fork-destroy-failure.mts'); const closeFailureFixture = join(__dirname, 'fixtures', 'fork-close-failure.mts'); const gcCloseFailureFixture = join(__dirname, 'fixtures', 'fork-gc-close-failure.mts'); @@ -213,6 +214,12 @@ describe('Destroy', () => { }); }, 15_000); + it('attaches a racing open before destroy can claim its descriptor', async () => { + await runDestroyFixture(openAttachDestroyFixture, generateDBPath(), { + ROCKSDB_JS_OPEN_ATTACH_DELAY_MS: '2000', + }); + }, 15_000); + it('closes an iterator safely when destroy races its construction', async () => { await runDestroyFixture(destroyOpenFixture, generateDBPath(), { ROCKSDB_JS_DESTROY_DELAY_MS: '2000', diff --git a/test/fixtures/fork-open-attach-destroy.mts b/test/fixtures/fork-open-attach-destroy.mts new file mode 100644 index 000000000..94230363a --- /dev/null +++ b/test/fixtures/fork-open-attach-destroy.mts @@ -0,0 +1,49 @@ +import { RocksDatabase, registryStatus } from '../../src/index.ts'; +import { createWorkerBootstrapScript } from '../lib/worker-bootstrap.ts'; +import { setTimeout as delay } from 'node:timers/promises'; +import { Worker } from 'node:worker_threads'; + +const path = process.argv[2]; +const owner = RocksDatabase.open(path); +owner.putSync('before-destroy', 'present'); + +const worker = new Worker(createWorkerBootstrapScript('./test/workers/open-attach-worker.mts'), { + eval: true, + workerData: { path }, +}); + +function nextMessage(): Promise { + return new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); +} + +const ready = await nextMessage(); +if (!ready.ready) throw new Error(`Open worker failed to initialize: ${JSON.stringify(ready)}`); +worker.postMessage({ open: true }); +const opening = await nextMessage(); +if (!opening.opening) throw new Error(`Open worker did not start: ${JSON.stringify(opening)}`); + +await delay(100); +const registryEntry = registryStatus().find((entry) => entry.path === path); +const closables = registryEntry?.closables; + +owner.destroy(); +const result = await nextMessage(); +await worker.terminate(); + +if (closables !== 2) { + throw new Error( + `Expected both database handles to be attached before open returned, got ${closables}` + ); +} +if (result.error) { + throw new Error(`Racing open failed: ${result.error}`); +} +if (result.openedBeforeReopen !== false) { + throw new Error(`Destroy did not close the racing open: ${JSON.stringify(result)}`); +} +if (result.reopened !== true || result.value !== 'present') { + throw new Error(`Foreign-closed handle did not reopen cleanly: ${JSON.stringify(result)}`); +} diff --git a/test/workers/open-attach-worker.mts b/test/workers/open-attach-worker.mts new file mode 100644 index 000000000..1cd4a3677 --- /dev/null +++ b/test/workers/open-attach-worker.mts @@ -0,0 +1,25 @@ +import { RocksDatabase } from '../../src/index.ts'; +import { parentPort, workerData } from 'node:worker_threads'; + +if (!parentPort) throw new Error('Open/attach worker requires a parent port'); +const port = parentPort; +port.postMessage({ ready: true }); + +port.once('message', async () => { + port.postMessage({ opening: true }); + try { + const db = RocksDatabase.open(workerData.path); + const openedBeforeReopen = db.isOpen(); + db.open(); + db.putSync('after-destroy', 'present'); + await db.flush(); + port.postMessage({ + openedBeforeReopen, + reopened: db.isOpen(), + value: await db.get('after-destroy'), + }); + db.close(); + } catch (error) { + port.postMessage({ error: error instanceof Error ? error.message : String(error) }); + } +}); From d3944d9aa4f9b60e74c9f20547a579955bcc824a Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 2 Sep 2026 13:42:45 -0600 Subject: [PATCH 47/49] fix(lifecycle): retain failed open path for recovery Co-Authored-By: GPT-5 Codex --- AGENTS.md | 7 +++++-- src/binding/database/db_registry.cpp | 3 +-- test/fixtures/fork-destroy-failure.mts | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4223dc0cb..ae6000a09 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -807,8 +807,11 @@ sufficient (env teardown does not honor tsfn acquire counts); see 21. **Handle adoption and descriptor attachment are one registry-locked publication**: `DBRegistry::OpenDB()` selects the descriptor and column family, clears stale close cancellation, - publishes every per-open handle field, and inserts the handle into `DBDescriptor::closables` - before releasing `databasesMutex`. A forced `destroy()`/`shutdown()` claims under the same mutex, + publishes every descriptor-backed handle field, and inserts the handle into + `DBDescriptor::closables` before releasing `databasesMutex`. The owner-thread-only `path` is set + before registry work so that a failed open can still follow a quarantine error's `destroy()` + recovery, but teardown never reads that field from `closables`. A forced + `destroy()`/`shutdown()` claims under the same mutex, so it either precedes the open or sees the fully adopted handle in the closables sweep. Returning `DBHandleParams` and attaching in `Database::Open()` left a gap where teardown could reset `descriptor->db` while the invisible handle retained a `ColumnFamilyHandle`; destroying that diff --git a/src/binding/database/db_registry.cpp b/src/binding/database/db_registry.cpp index 1fa6c5179..273528035 100644 --- a/src/binding/database/db_registry.cpp +++ b/src/binding/database/db_registry.cpp @@ -467,6 +467,7 @@ void DBRegistry::OpenDB( handle->descriptor->path + "\"" ); } + handle->path = path; DEBUG_LOG("%p DBRegistry::OpenDB Opening database \"%s\" (mode=%s read-only=%s column family=\"%s\")\n", instance.get(), path.c_str(), options.mode == DBMode::Optimistic ? "optimistic" : "pessimistic", options.readOnly ? "true" : "false", options.name.empty() ? "default" : options.name.c_str()); @@ -703,7 +704,6 @@ void DBRegistry::OpenDB( columnDescriptor = columns[rocksdb::kDefaultColumnFamilyName]; } - std::string openedPath(path); const uint64_t verificationTableDbId = entry.descriptor->vtEpoch; const uint32_t verificationTableColumnFamilyId = columnDescriptor->column->GetID(); @@ -716,7 +716,6 @@ void DBRegistry::OpenDB( previousColumnDescriptor = std::move(handle->columnDescriptor); previousDescriptor = std::move(handle->descriptor); - handle->path = std::move(openedPath); handle->columnDescriptor = std::move(columnDescriptor); handle->descriptor = entry.descriptor; handle->verificationTableDbId = verificationTableDbId; diff --git a/test/fixtures/fork-destroy-failure.mts b/test/fixtures/fork-destroy-failure.mts index 73846ce12..9c10c7e95 100644 --- a/test/fixtures/fork-destroy-failure.mts +++ b/test/fixtures/fork-destroy-failure.mts @@ -31,15 +31,16 @@ if (closeFailure) { throw new Error(`Unexpected database:closeFailed arguments: ${JSON.stringify(args)}`); } const startedAt = Date.now(); + const recovery = new RocksDatabase(path); try { - RocksDatabase.open(path); + recovery.open(); throw new Error('Expected the failed descriptor to remain quarantined'); } catch (error) { if (!String(error).includes(`previous close failed: ${expectedError}`)) throw error; } if (Date.now() - startedAt >= 1_000) throw new Error('Opening a quarantined descriptor waited instead of failing immediately'); - db.destroy(); + recovery.destroy(); process.exit(0); } From 85caaac7a5bf6768e294ab52285579eb9832e0e7 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 2 Sep 2026 13:58:31 -0600 Subject: [PATCH 48/49] docs(lifecycle): correct cancellation ownership Co-Authored-By: GPT-5 Codex --- AGENTS.md | 2 +- src/binding/database/db_descriptor.h | 6 +++--- src/binding/database/db_handle.h | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ae6000a09..f510d0a91 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -405,7 +405,7 @@ sufficient (env teardown does not honor tsfn acquire counts); see descriptor token from `CloseDB` is not the alternative: it is never cleared, so one handle closing would permanently kill manual compaction for every other handle sharing the process-global descriptor. That is also what makes it safe for `finishClose()` to arm handles - this thread does not own: the per-handle token _is_ cleared, by `DBHandle::open()`, and only + this thread does not own: the per-handle token _is_ cleared, by `DBRegistry::OpenDB()`, and only after the new descriptor is adopted (see invariant 20). Neither token is ever aliased onto `closing` itself: RocksDB writes through the pointer it is given (`DisableManualCompaction()` sets the caller's atomic), and `closing` means the registry has an diff --git a/src/binding/database/db_descriptor.h b/src/binding/database/db_descriptor.h index b751420ee..329e4b816 100644 --- a/src/binding/database/db_descriptor.h +++ b/src/binding/database/db_descriptor.h @@ -307,9 +307,9 @@ struct DBDescriptor final : public std::enable_shared_from_this { * under one lock and then close them sequentially, so arming at claim * time cancels compactions on descriptors whose own `finishClose()` has * not started yet. - * 2. It is never cleared. Close-initiated compaction opts out with - * `cancellable = false` instead, since nothing external is waiting on - * it, and a descriptor never leaves the closing state. + * 2. It is never cleared. Close-initiated compaction passes a null cancel + * token instead, since nothing external is waiting on it, and a + * descriptor never leaves the closing state. * 3. It covers synchronous compaction only. Async `compact()`/`clear()` * released their OperationGuard at setup handoff, so this drain does not * await them; they are awaited by `DBHandle::close()`'s async-work drain diff --git a/src/binding/database/db_handle.h b/src/binding/database/db_handle.h index b8bd74a14..90a4e15f9 100644 --- a/src/binding/database/db_handle.h +++ b/src/binding/database/db_handle.h @@ -78,8 +78,8 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro * callers. Armed by `close()` (self-close) and, for a foreign close, by * `finishClose()` before its first blocking step, via * `cancelBlockingWork()`. Safe to arm from a thread that does not own this - * handle because this token is cleared by `open()`; the descriptor's is - * never cleared. + * handle because this token is cleared by `DBRegistry::OpenDB()`; the + * descriptor's is never cleared. * * Why there are two tokens, why each is armed where it is, and what each of * `test/fixtures/fork-compact-cancel-{sync,async,close,destroy}.mts` does From d082b42586af726c40f14887a8383acb23a59415 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 2 Sep 2026 14:20:51 -0600 Subject: [PATCH 49/49] Make open/attach race fixture deterministic Co-Authored-By: GPT-5 Codex --- test/fixtures/fork-open-attach-destroy.mts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/fixtures/fork-open-attach-destroy.mts b/test/fixtures/fork-open-attach-destroy.mts index 94230363a..9d0f5160e 100644 --- a/test/fixtures/fork-open-attach-destroy.mts +++ b/test/fixtures/fork-open-attach-destroy.mts @@ -25,9 +25,12 @@ worker.postMessage({ open: true }); const opening = await nextMessage(); if (!opening.opening) throw new Error(`Open worker did not start: ${JSON.stringify(opening)}`); -await delay(100); -const registryEntry = registryStatus().find((entry) => entry.path === path); -const closables = registryEntry?.closables; +const registryDeadline = Date.now() + 5_000; +let closables: number | undefined; +while (closables !== 2 && Date.now() < registryDeadline) { + closables = registryStatus().find((entry) => entry.path === path)?.closables; + if (closables !== 2) await delay(1); +} owner.destroy(); const result = await nextMessage();