Pin the load-bearing native and encoder dependencies

.github/workflows/update-rocksdb-js.yml

@@ -34,7 +34,7 @@ jobs:
           package-manager-cache: false
 
       - name: Install latest rocksdb-js
-        run: npm install --save @harperfast/rocksdb-js@latest
+        run: npm install --save-exact @harperfast/rocksdb-js@latest
 
       - name: Stage changes
         id: stage-changes

AGENTS.md

@@ -114,7 +114,7 @@ Use this to land in the right folder before grepping. Every top-level folder is
 
 - **`bin/`** — covered above (it's source).
 - **`benchmarks/`** — HNSW vector-search benchmark only (`hnsw-search.js`). Stand-alone; not part of CI.
-- **`build-tools/`** — shell scripts for the build pipeline (`build.sh`, `build-studio.sh`, `download-prebuilds.js`). No tests.
+- **`build-tools/`** — build-pipeline scripts. Tests: `unitTests/build-tools/`; run `npm run test:unit:main` after changes.
 - **`dev/`** — single dev utility (`sync-commits.js`) for cross-repo commit syncing. Not runtime.
 - **`integrationTests/`** — end-to-end tests against a built distribution. Run with `npm run test:integration` / `npm run test:integration:all`. Subdirs mirror source. See `integrationTests/README.md`.
 - **`unitTests/`** — Mocha unit tests; subdir per source layer. Run with `npm run test:unit:<layer>`.

build-tools/check-shrinkwrap-pins.mjs

@@ -24,13 +24,18 @@
 // "latest" dist-tag: if the range excludes a newer major, a broken install could never
 // reach it either, so comparing to absolute latest would flag a canary as fine when it
 // has actually gone vacuous within the range that matters.
+// Exact manifest specs remain in the installed-version check, but cannot discriminate a
+// shrinkwrap install from a fresh resolution, so the discrimination check skips them.
 //
 // Usage: node check-shrinkwrap-pins.mjs <package-root>
 
 import { execFileSync } from 'node:child_process';
-import { readFileSync } from 'node:fs';
+import { existsSync, readFileSync } from 'node:fs';
 
-const CHECKED_DEPS = ['@harperfast/rocksdb-js', 'fastify'];
+const CHECKED_DEPS = ['@harperfast/rocksdb-js', 'fastify', '@aws-sdk/client-s3'];
+const ROCKSDB_SINGLE_INSTANCE_DEPS = ['@harperfast/extended-iterable', 'msgpackr'];
+const REGISTRY_QUERY_ATTEMPTS = 3;
+const retryWait = new Int32Array(new SharedArrayBuffer(4));
 
 const pkgRoot = process.argv[2];
 if (!pkgRoot) {
@@ -85,54 +90,164 @@ for (const dep of CHECKED_DEPS) {
 		continue;
 	}
 
-	console.log(`shrinkwrap honored: ${dep}@${installed} matches the packed pin`);
+	const status = isExactVersion(range) ? 'shrinkwrap pin matches (exact manifest spec)' : 'shrinkwrap honored';
+	console.log(`${status}: ${dep}@${installed} matches the packed pin`);
 }
 
+verifyRocksDbDependencyAlignment();
+
 // A canary only proves the check works while its pin lags the registry -- if a lock bump
-// ever lands both canaries on registry-latest, the pin-match loop above would pass on a
+// ever lands every ranged canary on registry-latest, the pin-match loop above would pass on a
 // reverted, broken Dockerfile just as easily as on this one. Fail loudly rather than let
 // that happen silently.
 verifyCanariesDiscriminate(pins);
 
 process.exit(failed ? 1 : 0);
 
 function verifyCanariesDiscriminate(pins) {
+	const rangedPins = Object.fromEntries(Object.entries(pins).filter(([, { range }]) => !isExactVersion(range)));
+	if (Object.keys(pins).length === 0) return;
+	if (Object.keys(rangedPins).length === 0) {
+		console.error(
+			'::error::every checked canary has an exact declared version -- exact dependencies cannot distinguish a shrinkwrap install from a fresh resolution. Add a canary with a ranged manifest spec.'
+		);
+		failed = true;
+		return;
+	}
+
 	// What a broken/reverted install would actually resolve to: the max version satisfying
 	// the declared range, not the registry's bare "latest" dist-tag (which could be a newer
 	// major the range excludes, and a broken install could never reach that either).
 	const rangeLatest = {};
-	for (const [dep, { range }] of Object.entries(pins)) {
-		try {
-			// `npm view <dep>@<range> version --json` returns every matching version, not
-			// just the max, and the order isn't a documented contract (it can track
-			// publish/insertion order rather than semver order, e.g. a backported patch
-			// published after a newer minor) -- compare them ourselves rather than trust
-			// the last array entry.
-			const out = execFileSync('npm', ['view', `${dep}@${range}`, 'version', '--json'], { encoding: 'utf8' });
-			const versions = JSON.parse(out);
-			rangeLatest[dep] = Array.isArray(versions)
-				? versions.reduce((max, v) => (compareVersions(v, max) > 0 ? v : max))
-				: versions;
-		} catch (e) {
-			console.log(
-				`::warning::could not check registry-latest-in-range for ${dep}@${range} (${e.message}) -- skipping discrimination check for it`
-			);
+	const failedQueries = [];
+	for (const [dep, { range }] of Object.entries(rangedPins)) {
+		for (let attempt = 1; attempt <= REGISTRY_QUERY_ATTEMPTS; attempt++) {
+			try {
+				// `npm view <dep>@<range> version --json` returns every matching version, not
+				// just the max, and the order isn't a documented contract (it can track
+				// publish/insertion order rather than semver order, e.g. a backported patch
+				// published after a newer minor) -- compare them ourselves rather than trust
+				// the last array entry.
+				const out = execFileSync('npm', ['view', `${dep}@${range}`, 'version', '--json'], { encoding: 'utf8' });
+				const versions = JSON.parse(out);
+				if (Array.isArray(versions) && versions.length === 0) {
+					reportMissingRange(dep, range);
+					return;
+				}
+				if (
+					(Array.isArray(versions) && versions.some((version) => !isExactVersion(version))) ||
+					(!Array.isArray(versions) && !isExactVersion(versions))
+				) {
+					throw new Error('npm returned an invalid version payload');
+				}
+				rangeLatest[dep] = Array.isArray(versions)
+					? versions.reduce((max, v) => (compareVersions(v, max) > 0 ? v : max))
+					: versions;
+				break;
+			} catch (e) {
+				if (isNpmNotFoundError(e)) {
+					reportMissingRange(dep, range);
+					return;
+				}
+				console.log(
+					`::warning::registry query attempt ${attempt}/${REGISTRY_QUERY_ATTEMPTS} failed for ${dep}@${range} (${e.message})`
+				);
+				if (attempt === REGISTRY_QUERY_ATTEMPTS) failedQueries.push(`${dep}@${range}`);
+				else Atomics.wait(retryWait, 0, 0, 1000 * attempt);
+			}
 		}
 	}
 	const checkable = Object.keys(rangeLatest);
-	if (checkable.length === 0) {
-		console.log('::warning::registry unreachable -- could not verify the canary set still discriminates');
-		return;
-	}
 	const stillDiscriminates = checkable.some((dep) => rangeLatest[dep] !== pins[dep].pinned);
-	if (!stillDiscriminates) {
+	if (stillDiscriminates) return;
+	if (failedQueries.length > 0) {
 		console.error(
-			`::error::every checked canary (${checkable.join(', ')}) is now pinned at the latest version its declared range allows -- this check would pass even on a reverted, unpinned install. Pick a new canary whose shrinkwrap pin lags what its range allows.`
+			`::error title=Retry dependency canary check::Could not verify that the shrinkwrap canaries still discriminate after ${REGISTRY_QUERY_ATTEMPTS} registry query attempts for ${failedQueries.join(', ')}. Retry this job; if the error persists, check npm/registry/runner configuration and confirm the listed package ranges match published versions.`
 		);
 		failed = true;
+		return;
+	}
+	console.error(
+		`::error::every checked canary (${checkable.join(', ')}) is now pinned at the latest version its declared range allows -- this check would pass even on a reverted, unpinned install. Pick a new canary whose shrinkwrap pin lags what its range allows.`
+	);
+	failed = true;
+}
+
+function verifyRocksDbDependencyAlignment() {
+	let rocksdbManifest;
+	try {
+		rocksdbManifest = JSON.parse(readFileSync(`${pkgRoot}/node_modules/@harperfast/rocksdb-js/package.json`, 'utf8'));
+	} catch (e) {
+		console.error(`::error::could not inspect rocksdb-js dependency alignment: ${e.message}`);
+		failed = true;
+		return;
+	}
+
+	for (const dep of ROCKSDB_SINGLE_INSTANCE_DEPS) {
+		const rootSpec = manifest.dependencies?.[dep];
+		const rocksdbSpec = rocksdbManifest.dependencies?.[dep];
+		if (!isExactVersion(rootSpec)) {
+			console.error(
+				`::error::the root ${dep} spec must be exact, received ${rootSpec ?? 'missing'} -- update it with rocksdb-js to preserve one module instance`
+			);
+			failed = true;
+			continue;
+		}
+		if (rootSpec !== rocksdbSpec) {
+			console.error(
+				`::error::the root ${dep} pin ${rootSpec} does not match rocksdb-js ${rocksdbSpec ?? 'missing'} -- update these pins together to preserve one module instance`
+			);
+			failed = true;
+			continue;
+		}
+
+		try {
+			const installed = JSON.parse(readFileSync(`${pkgRoot}/node_modules/${dep}/package.json`, 'utf8')).version;
+			if (installed !== rootSpec) {
+				console.error(`::error::${dep} resolved to ${installed}, expected the aligned exact pin ${rootSpec}`);
+				failed = true;
+			}
+		} catch (e) {
+			console.error(`::error::could not inspect the root ${dep} instance: ${e.message}`);
+			failed = true;
+		}
+
+		const nestedManifest = `${pkgRoot}/node_modules/@harperfast/rocksdb-js/node_modules/${dep}/package.json`;
+		if (existsSync(nestedManifest)) {
+			let nestedVersion = 'unknown';
+			try {
+				nestedVersion = JSON.parse(readFileSync(nestedManifest, 'utf8')).version;
+			} catch {}
+			console.error(
+				`::error::rocksdb-js loaded a nested ${dep}@${nestedVersion} -- root and rocksdb-js must share one module instance`
+			);
+			failed = true;
+		}
 	}
 }
 
+function isExactVersion(range) {
+	return (
+		typeof range === 'string' &&
+		/^\d+\.\d+\.\d+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/.test(range)
+	);
+}
+
+function isNpmNotFoundError(error) {
+	try {
+		return JSON.parse(error.stdout).error?.code === 'E404';
+	} catch {
+		return false;
+	}
+}
+
+function reportMissingRange(dep, range) {
+	console.error(
+		`::error::${dep}@${range} matches no published version in the configured registry -- correct the declared range in package.json or confirm the configured registry carries this package`
+	);
+	failed = true;
+}
+
 // Numeric major.minor.patch comparison, ignoring any prerelease/build suffix -- sufficient
 // for the stable releases this check compares (avoids depending on a semver-parsing
 // package that may not be resolvable from this script's own location).

package.json

@@ -178,8 +178,8 @@
 		"@fastify/compress": "^8.3.1",
 		"@fastify/cors": "^11.2.0",
 		"@fastify/static": "^9.1.3",
-		"@harperfast/extended-iterable": "^1.0.1",
-		"@harperfast/rocksdb-js": "^2.7.1",
+		"@harperfast/extended-iterable": "1.0.3",
+		"@harperfast/rocksdb-js": "2.7.1",
 		"@harperfast/skills": "^1.10.8",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",
@@ -223,7 +223,7 @@
 		"minimist": "1.2.8",
 		"moment": "2.30.1",
 		"mqtt-packet": "~9.0.1",
-		"msgpackr": "^2.0.5",
+		"msgpackr": "2.0.5",
 		"needle": "3.5.0",
 		"node-forge": "^1.3.1",
 		"node-stream-zip": "1.16.0",
@@ -244,7 +244,7 @@
 		"ses": "^1.15.0",
 		"stream-chain": "2.2.5",
 		"stream-json": "1.9.1",
-		"structon": "^1.0.7",
+		"structon": "1.0.7",
 		"systeminformation": "^5.31.4",
 		"tar-fs": "^3.1.2",
 		"tar-stream": "^3.1.8",
@@ -261,9 +261,9 @@
 		}
 	},
 	"optionalDependencies": {
-		"bufferutil": "^4.0.9",
-		"segfault-handler": "^1.3.0",
-		"utf-8-validate": "^5.0.10"
+		"bufferutil": "4.1.0",
+		"segfault-handler": "1.3.0",
+		"utf-8-validate": "5.0.10"
 	},
 	"peerDependencies": {
 		"@aws-sdk/client-bedrock-runtime": "^3.0.0",