From 24f781e81cb2d6710d4977173a3a8e24c76c301a Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:22:21 -0700 Subject: [PATCH 01/25] fix: bound session cold loads and narrow lazy field reads (#4739) --- src/persist.ts | 82 +++++- src/storage/backend.ts | 25 +- src/storage/driver.ts | 45 ++-- src/storage/ingest.ts | 254 ++++++++++-------- src/storage/mysql.ts | 82 ++++-- src/storage/shadow.ts | 100 +++++-- src/storage/store.ts | 87 +++++- tests/issue4739-storage-read-snapshot.test.ts | 191 +++++++++++++ tests/persist-lazy-narrow-reads.test.ts | 208 ++++++++++++++ tests/persist-rollback-guard.test.ts | 2 +- tests/storage-backend-contract.test.ts | 9 + tests/storage-bounded-load.test.ts | 222 +++++++++++++++ 12 files changed, 1110 insertions(+), 197 deletions(-) create mode 100644 tests/issue4739-storage-read-snapshot.test.ts create mode 100644 tests/persist-lazy-narrow-reads.test.ts create mode 100644 tests/storage-bounded-load.test.ts diff --git a/src/persist.ts b/src/persist.ts index bdf9abc0d1..e2a4876f61 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -379,6 +379,11 @@ export class SessionStore { // #3931 step 3: assigned asynchronously for network backends (mysql) — // see the constructor and ready(). private shadow?: UnifiedShadow; + /** Only alive while one synchronous record build hydrates both lazy fields. */ + private residentFieldsForBuild?: { + id: string; + fields: { metadata?: Record; blockContents?: Record } | null; + }; /** #2960 ②: sessions whose fork_base checkpoint has been cut (or was * found already present) — memoized so steady-state writes of receipt * sessions do not re-query the checkpoints table every flush. */ @@ -667,7 +672,7 @@ export class SessionStore { // #408 one-time migration: the in-memory value is already // clamped by buildSession — rewrite the stale file so the // negative value is gone from disk. - await this.store.writeNow(id, () => buildRecord(session)); + await this.store.writeNow(id, () => this.buildRecordForSave(session)); clamped++; // #3049: the clamp rewrite writes a DIFFERENT payload than any // ingested record (clamped stats) without a sidecar ingest — @@ -1147,7 +1152,7 @@ export class SessionStore { continue; } session.id = label; - await this.store.writeNow(label, () => buildRecord(session)); + await this.store.writeNow(label, () => this.buildRecordForSave(session)); // #3049: direct write bypassing guardedBuild — the boot rekey // rewrites envelopes without the sidecar ingest, so any prior // ownership token for this id is stale. @@ -1625,6 +1630,60 @@ export class SessionStore { return this.shadow ? this.shadow.viewFor(id) : null; } + /** Restore only the dropped fields, with the same source preference as loadSync. + * A field read must not restamp refs, import rows, or schedule another save. */ + private loadResidentFields( + id: string, + meta: { protocol?: string; upstreamOrigin?: string }, + fields: { metadata?: boolean; blockContents?: boolean }, + ): { metadata?: Record; blockContents?: Record } | null { + if (this.residentFieldsForBuild?.id === id) return this.residentFieldsForBuild.fields; + if (!this.enabled) return null; + const indexed = this.shadow?.readsUnified ? this.shadow.readResidentFieldsFor(id, fields) : null; + const envelope = this.loadEnvelope(id, meta); + let useEnvelope = indexed === null; + if (indexed && envelope) { + if (this.shadow?.driftedSessions.has(id)) { + const envActive = countActiveBlocks(envelope.payload); + const envRefs = Object.keys(envelope.payload.state?.messageRefs?.byRef ?? {}).length; + useEnvelope = indexed.header.activeBlockCount <= envActive && indexed.header.mappedRefCount <= envRefs; + } else { + const requests = Number(envelope.payload.stats?.requests ?? 0); + useEnvelope = requests > indexed.header.requests || + (requests === indexed.header.requests && Number(envelope.payload.state?.nextBlockId ?? 0) > indexed.header.nextBlockId); + } + } + if (useEnvelope) { + if (!envelope) return null; + return { + ...(fields.metadata ? { metadata: envelope.payload.metadata ?? {} } : {}), + ...(fields.blockContents ? { blockContents: Object.fromEntries(restoreBlockContents(Object.entries(envelope.payload.blockContents ?? {}))) } : {}), + }; + } + if (!indexed) return null; + return { + ...(fields.metadata ? { metadata: indexed.metadata } : {}), + ...(fields.blockContents ? { blockContents: Object.fromEntries(restoreBlockContents(indexed.blockContents ?? [])) } : {}), + }; + } + + private buildRecordForSave(session: Session, storageMode: "legacy" | "shadow" | "full" = effectiveFileStorageMode()): PersistedSession { + const bothDropped = isDroppedMetadata(session.metadata) && + session.blockContents instanceof LazyBlockContents && session.blockContents.isDropped; + if (!bothDropped) return buildRecord(session, storageMode); + // Metadata and contents used to each rebuild a whole Session here. Share + // one projection and envelope arbitration for this build, never a cache + // that keeps another session alive after the synchronous build returns. + const previous = this.residentFieldsForBuild; + const fields = this.loadResidentFields(session.id, session.meta ?? {}, { metadata: true, blockContents: true }); + this.residentFieldsForBuild = { id: session.id, fields }; + try { + return buildRecord(session, storageMode); + } finally { + this.residentFieldsForBuild = previous; + } + } + /** Read-only state load for cross-session search (#841): unlike loadSync, * never schedules a save (no #408 clamp-rewrite side effect). */ loadStateForSearch(id: string): CompressionState | null { @@ -1997,7 +2056,7 @@ export class SessionStore { this.forkBaseCut.add(session.id); return; } - const record = buildRecord(session); + const record = this.buildRecordForSave(session); const activeBlockIds = session.state.blocks .filter((b) => (b as { active?: boolean }).active !== false) .map((b) => b.blockId) @@ -2306,7 +2365,7 @@ export class SessionStore { // the JSON envelope, and the stamp must say legacy for that copy. const stamp = this.liveStorageStamp; return () => { - const record = buildRecord(session, stamp); + const record = this.buildRecordForSave(session, stamp); const disk = this.staleDiskPayload(record); if (disk === null) { // #3301: external-converter import payload rides exactly ONE @@ -2588,8 +2647,7 @@ export class SessionStore { // loader degrades to the id-only lookup, same as a protocol-less file. const meta = session.meta ?? undefined; session.metadata = lazyMetadata(session.id, { protocol: meta?.protocol, upstreamOrigin: meta?.upstreamOrigin }, (id, m) => { - const loaded = this.loadSync(id, m); - return loaded ? loaded.metadata : null; + return this.loadResidentFields(id, m, { metadata: true })?.metadata ?? null; }); } @@ -2607,8 +2665,7 @@ export class SessionStore { } if (existing.size === 0) return; const lazy = new LazyBlockContents(session.id, { protocol: session.meta.protocol, upstreamOrigin: session.meta.upstreamOrigin }, (id, meta) => { - const loaded = this.loadSync(id, meta); - return loaded ? Object.fromEntries(loaded.blockContents) : null; + return this.loadResidentFields(id, meta, { blockContents: true })?.blockContents ?? null; }); lazy.drop(); session.blockContents = lazy; @@ -2864,9 +2921,9 @@ function restoreCalibrationRing(v: unknown): { origin: string; values: number[] return recent.length > 0 ? { origin: ring.origin, values: recent } : undefined; } -function buildSession(parsed: PersistedSession): Session { +function restoreBlockContents(entries: Iterable): Map { const blockContents = new Map(); - for (const [bid, content] of Object.entries(parsed.blockContents ?? {})) { + for (const [bid, content] of entries) { if (!content || typeof content !== "object") continue; const full = (content as Record).full; if (!isBlockView(full)) continue; @@ -2879,6 +2936,11 @@ function buildSession(parsed: PersistedSession): Session { full, }); } + return blockContents; +} + +function buildSession(parsed: PersistedSession): Session { + const blockContents = restoreBlockContents(Object.entries(parsed.blockContents ?? {})); // Read grouped shape (v2+); fall back to flat fields for v1 files. const meta = parsed.meta ?? {}; const stats = parsed.stats ?? {}; diff --git a/src/storage/backend.ts b/src/storage/backend.ts index dd8473893a..4341a2ec45 100644 --- a/src/storage/backend.ts +++ b/src/storage/backend.ts @@ -73,10 +73,12 @@ export interface StorageBackend { seqByRaw(sessionId: string): Promise | null>; /** #4316 PR3/3: fork adoption row clone (idempotent). */ cloneSessionRows(parentId: string, childId: string, throughSeq: number): Promise; - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]>; - getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]>; + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]>; getRefs(sessionId: string): Promise; + /** All reads, including CAS hydration, use one short read-only snapshot. */ + readSnapshot(read: (reader: StorageReadView) => Promise): Promise; getCcr(sessionId: string): Promise; getCcrEntries(sessionId: string): Promise; @@ -139,6 +141,8 @@ export interface StorageBackend { init?(): Promise; } +export type StorageReadView = Pick; + /** The sqlite backend: the existing UnifiedStore behind the async contract. * Synchronous under the hood; `Promise.resolve` wrapping only. */ export class SqliteBackend implements StorageBackend { @@ -186,11 +190,11 @@ export class SqliteBackend implements StorageBackend { this.store.cloneSessionRows(parentId, childId, throughSeq); return Promise.resolve(true); } - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { - return Promise.resolve(this.store.getMessages(sessionId)); + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]> { + return Promise.resolve(this.store.getMessages(sessionId, opts)); } - getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { - return Promise.resolve(this.store.getBlocks(sessionId)); + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + return Promise.resolve(this.store.getBlocks(sessionId, opts)); } getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> { return Promise.resolve(this.store.getDeadRefs(sessionId)); @@ -198,6 +202,15 @@ export class SqliteBackend implements StorageBackend { getRefs(sessionId: string): Promise { return Promise.resolve(this.store.getRefs(sessionId)); } + readSnapshot(read: (reader: StorageReadView) => Promise): Promise { + return this.store.readSnapshotAsync((store) => read({ + getSessionRow: async (id) => store.getSessionRow(id), + getMessages: async (id, opts) => store.getMessages(id, opts), + getBlocks: async (id, opts) => store.getBlocks(id, opts), + getDeadRefs: async (id) => store.getDeadRefs(id), + getRefs: async (id) => store.getRefs(id), + })); + } getCcr(sessionId: string): Promise { return Promise.resolve(this.store.getCcr(sessionId)); } diff --git a/src/storage/driver.ts b/src/storage/driver.ts index 6bc44be0c7..63799382d4 100644 --- a/src/storage/driver.ts +++ b/src/storage/driver.ts @@ -40,6 +40,8 @@ export interface SqliteDatabase { prepare(sql: string): SqliteStatement; /** Runs fn inside BEGIN/COMMIT; rolls back on throw (including nested calls via savepoints). */ transaction(fn: () => T): T; + /** Deferred snapshot: readers do not reserve the writer lock. */ + readTransaction(fn: () => T): T; close(): void; readonly open: boolean; } @@ -87,6 +89,7 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase { // cannot resolve. Taking the write lock up front makes concurrent // writers queue on busy_timeout instead (#2690 review blocking 5). transaction: (fn) => db.transaction(fn).immediate(), + readTransaction: (fn) => db.transaction(fn).deferred(), close: () => void db.close(), get open() { return db.open; @@ -104,31 +107,33 @@ function openNodeSqlite(path: string): SqliteDatabase { }; const db = new DatabaseSync(path); let depth = 0; - const begin = () => db.exec(depth === 0 ? "BEGIN IMMEDIATE" : `SAVEPOINT sp${depth}`); // IMMEDIATE at top level: multi-process SQLITE_BUSY_SNAPSHOT guard (see openBetterSqlite3) + const begin = (readOnly: boolean) => db.exec(depth === 0 ? (readOnly ? "BEGIN DEFERRED" : "BEGIN IMMEDIATE") : `SAVEPOINT sp${depth}`); const commit = () => db.exec(depth === 1 ? "COMMIT" : `RELEASE sp${depth - 1}`); const rollback = () => db.exec(depth === 1 ? "ROLLBACK" : `ROLLBACK TO sp${depth - 1}`); + const transaction = (fn: () => T, readOnly: boolean): T => { + if (db.getAutoCommit?.() === false && depth === 0) { + // Already inside an outer transaction driven directly by + // DatabaseSync — just run inline; the outer COMMIT covers us. + return fn(); + } + begin(readOnly); + depth++; + try { + const out = fn(); + commit(); + return out; + } catch (err) { + rollback(); + throw err; + } finally { + depth--; + } + }; return { exec: (sql) => void db.exec(sql), prepare: (sql) => adaptNodeSqliteStatement(db.prepare(sql)), - transaction: (fn: () => T): T => { - if (db.getAutoCommit?.() === false && depth === 0) { - // Already inside an outer transaction driven directly by - // DatabaseSync — just run inline; the outer COMMIT covers us. - return fn(); - } - begin(); - depth++; - try { - const out = fn(); - commit(); - return out; - } catch (err) { - rollback(); - throw err; - } finally { - depth--; - } - }, + transaction: (fn) => transaction(fn, false), + readTransaction: (fn) => transaction(fn, true), close: () => void db.close(), get open() { return db.open; diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index c3a84aff22..54a319a8c8 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -487,28 +487,28 @@ export function completeBlockContents( return out; } -export function loadLegacyView( - rows: { - session: { - sessionId: string; - createdAt: number; - updatedAt: number; - nextBlockId: number; - nextRunId: number; - metaJson: string; - statsJson: string; - metadataJson: string; - kstateJson: string; - tailJson: string; - ccrJson: string | null; - orphansJson?: string | null; - }; - messages: { seq: number; msgId: string; ref: string | null; content: string }[]; - blocks: { blockId: string; summary: string; one: string | null; payloadJson: string }[]; - deadRefs: { ord: number; ref: string }[]; - refs: { ord: number; rawId: string; ref: string }[]; - }, -): { +interface LegacyViewRows { + session: { + sessionId: string; + createdAt: number; + updatedAt: number; + nextBlockId: number; + nextRunId: number; + metaJson: string; + statsJson: string; + metadataJson: string; + kstateJson: string; + tailJson: string; + ccrJson: string | null; + orphansJson?: string | null; + }; + messages: { seq: number; msgId: string; ref: string | null; content: string }[]; + blocks: { blockId: string; summary: string; one: string | null; payloadJson: string }[]; + deadRefs: { ord: number; ref: string }[]; + refs: { ord: number; rawId: string; ref: string }[]; +} + +interface LegacyView { id: string; createdAt: number; lastSeen: number; @@ -523,105 +523,147 @@ export function loadLegacyView( contentStore?: LegacySessionLike["contentStore"]; /** #4390: tail/pluginSnapshot pointers whose backing row is missing. */ tailGaps?: number; -} { - const s = rows.session; - const kstate = JSON.parse(s.kstateJson) as Record; - const tail = JSON.parse(s.tailJson) as { - pluginSnapshotSeqs: number[] | null; - tailSeqs: number[] | null; - tailFolded: boolean; - }; - - // messageRefs rebuilt from the ref ledger verbatim: ord preserves the - // persisted map's key order (digest-sensitive), and ghost ids survive - // even though no message row backs them (#2362 Kernel Contract). - const byRaw: Record = {}; - const byRef: Record = {}; - for (const r of rows.refs) { - byRaw[r.rawId] = r.ref; - // "BLOCKED" is a byRaw-only token: the in-memory map keeps the - // inverse projection strictly mNNNNN→rawId. Admitting it into byRef - // mints a key the persisted state never had (#2671 soak: sub-agent - // sessions with protected messages drifted on every save). - if (r.ref !== "BLOCKED" && byRef[r.ref] === undefined) byRef[r.ref] = r.rawId; - } +} - const state: Record = {}; - for (const field of MERGE_STATE_FIELDS) { - switch (field) { - case "blocks": - state.blocks = rows.blocks.map((b) => JSON.parse(b.payloadJson)); - break; - case "messageRefs": - state.messageRefs = { byRaw, byRef }; - break; - case "deadRefs": - state.deadRefs = rows.deadRefs.length > 0 ? rows.deadRefs.map((d) => d.ref) : undefined; - break; - case "nextBlockId": - state.nextBlockId = s.nextBlockId; - break; - case "nextRunId": - state.nextRunId = s.nextRunId; - break; - default: - if (kstate[field] !== undefined) state[field] = kstate[field]; - break; - } - } +type BlockContents = LegacyView["blockContents"]; - const blockContents = new Map(); - for (const b of rows.blocks) { - blockContents.set(b.blockId, { +/** Shared by full-view and resident-field reads; each page is released after parsing. */ +export function appendBlockContents(contents: BlockContents, rows: Pick[]): void { + for (const b of rows) { + contents.set(b.blockId, { one: b.one === null ? null : (JSON.parse(b.one) as BlockView), full: JSON.parse(b.summary) as BlockView, }); } - // #3841 reap journal: journal'd orphan contents (reaped blocks) ride - // blockContents so the derived-decompress fallback survives restarts — - // WITHOUT resurrecting state.blocks (the reaped entries stay reaped). - if (s.orphansJson !== null && s.orphansJson !== undefined && s.orphansJson !== "") { +} + +export function mergeOrphanBlockContents(contents: BlockContents, orphansJson: string | null | undefined): void { + // Reaped originals survive without resurrecting their state.blocks rows. + if (orphansJson !== null && orphansJson !== undefined && orphansJson !== "") { try { - const orphans = JSON.parse(s.orphansJson) as Record; + const orphans = JSON.parse(orphansJson) as Record; for (const [blockId, view] of Object.entries(orphans)) { - if (!blockContents.has(blockId)) blockContents.set(blockId, view); + if (!contents.has(blockId)) contents.set(blockId, view); } } catch { // corrupt journal → skip; the JSON envelope still carries the text } } +} - const messagesBySeq = new Map(rows.messages.map((m) => [m.seq, JSON.parse(m.content) as CoreMessage])); - // #4390: a tail pointer whose row is gone used to be silently filtered - // out — the view shrank with no trace. Count the gaps so callers can - // warn (persist logs them); the surviving members still load. - let tailGaps = 0; - const resolveSeqs = (seqs: number[]): CoreMessage[] => { - const out: CoreMessage[] = []; - for (const seq of seqs) { - const m = messagesBySeq.get(seq); - if (m === undefined) tailGaps++; - else out.push(m); +/** Incremental cold-load assembly keeps raw CAS strings to one page at a time. */ +export class LegacyViewBuilder { + readonly messageSeqs: number[]; + private readonly tail: { + pluginSnapshotSeqs: number[] | null; + tailSeqs: number[] | null; + tailFolded: boolean; + }; + private readonly state: Record = {}; + private readonly blocks: unknown[] = []; + private readonly blockContents: BlockContents = new Map(); + private readonly messagesBySeq = new Map(); + + constructor( + private readonly session: LegacyViewRows["session"], + refs: LegacyViewRows["refs"], + deadRefs: LegacyViewRows["deadRefs"], + ) { + const kstate = JSON.parse(session.kstateJson) as Record; + this.tail = JSON.parse(session.tailJson); + this.messageSeqs = [...new Set([...(this.tail.tailSeqs ?? []), ...(this.tail.pluginSnapshotSeqs ?? [])])]; + + // messageRefs rebuilt from the ref ledger verbatim: ord preserves the + // persisted map's key order (digest-sensitive), and ghost ids survive + // even though no message row backs them (#2362 Kernel Contract). + const byRaw: Record = {}; + const byRef: Record = {}; + for (const r of refs) { + byRaw[r.rawId] = r.ref; + // "BLOCKED" is a byRaw-only token: the in-memory map keeps the + // inverse projection strictly mNNNNN→rawId. Admitting it into byRef + // mints a key the persisted state never had (#2671 soak: sub-agent + // sessions with protected messages drifted on every save). + if (r.ref !== "BLOCKED" && byRef[r.ref] === undefined) byRef[r.ref] = r.rawId; + } + + const state = this.state; + for (const field of MERGE_STATE_FIELDS) { + switch (field) { + case "blocks": + state.blocks = this.blocks; + break; + case "messageRefs": + state.messageRefs = { byRaw, byRef }; + break; + case "deadRefs": + state.deadRefs = deadRefs.length > 0 ? deadRefs.map((d) => d.ref) : undefined; + break; + case "nextBlockId": + state.nextBlockId = session.nextBlockId; + break; + case "nextRunId": + state.nextRunId = session.nextRunId; + break; + default: + if (kstate[field] !== undefined) state[field] = kstate[field]; + break; + } } + } + + addBlocks(rows: LegacyViewRows["blocks"]): void { + for (const b of rows) this.blocks.push(JSON.parse(b.payloadJson)); + appendBlockContents(this.blockContents, rows); + } + + addMessages(rows: LegacyViewRows["messages"]): void { + for (const m of rows) this.messagesBySeq.set(m.seq, JSON.parse(m.content) as CoreMessage); + } + + finish(): LegacyView { + const s = this.session; + const tail = this.tail; + mergeOrphanBlockContents(this.blockContents, s.orphansJson); + // #4390: a tail pointer whose row is gone used to be silently filtered + // out — the view shrank with no trace. Count the gaps so callers can + // warn (persist logs them); the surviving members still load. + let tailGaps = 0; + const resolveSeqs = (seqs: number[]): CoreMessage[] => { + const out: CoreMessage[] = []; + for (const seq of seqs) { + const m = this.messagesBySeq.get(seq); + if (m === undefined) tailGaps++; + else out.push(m); + } + return out; + }; + const lastMessages = tail.tailSeqs ? resolveSeqs(tail.tailSeqs) : undefined; + const pluginSnapshot = tail.pluginSnapshotSeqs ? resolveSeqs(tail.pluginSnapshotSeqs) : undefined; + + const out: LegacyView = { + id: s.sessionId, + createdAt: s.createdAt, + lastSeen: s.updatedAt, + tailGaps, + meta: JSON.parse(s.metaJson), + stats: JSON.parse(s.statsJson), + metadata: JSON.parse(s.metadataJson), + state: this.state, + blockContents: this.blockContents, + lastMessages, + lastMessagesFolded: tail.tailFolded, + }; + if (pluginSnapshot) out.pluginSnapshot = pluginSnapshot; + if (s.ccrJson !== null) out.contentStore = JSON.parse(s.ccrJson); return out; - }; - const lastMessages = tail.tailSeqs ? resolveSeqs(tail.tailSeqs) : undefined; - const pluginSnapshot = tail.pluginSnapshotSeqs ? resolveSeqs(tail.pluginSnapshotSeqs) : undefined; + } +} - const out: ReturnType = { - id: s.sessionId, - createdAt: s.createdAt, - lastSeen: s.updatedAt, - tailGaps, - meta: JSON.parse(s.metaJson), - stats: JSON.parse(s.statsJson), - metadata: JSON.parse(s.metadataJson), - state, - blockContents, - lastMessages, - lastMessagesFolded: tail.tailFolded, - }; - if (pluginSnapshot) out.pluginSnapshot = pluginSnapshot; - if (s.ccrJson !== null) out.contentStore = JSON.parse(s.ccrJson); - return out; +/** Compatibility wrapper for callers that already hold all rows. */ +export function loadLegacyView(rows: LegacyViewRows): LegacyView { + const builder = new LegacyViewBuilder(rows.session, rows.refs, rows.deadRefs); + builder.addBlocks(rows.blocks); + builder.addMessages(rows.messages); + return builder.finish(); } diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index f3c46ea373..8137503866 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -34,7 +34,7 @@ import { log as loggerLog } from "../logger.js"; import { sha256Hex } from "./store.js"; import { identityOfStoredContent } from "./identity.js"; import type { MessageContentStore } from "acp-kernel"; -import type { StorageBackend } from "./backend.js"; +import type { StorageBackend, StorageReadView } from "./backend.js"; import type { SessionRow, MessageRow, @@ -972,7 +972,32 @@ export class MysqlBackend implements StorageBackend { } async getSessionRow(sessionId: string): Promise { - const [rows] = (await this.pool.query( + return this.getSessionRowOn(this.pool, sessionId); + } + + async readSnapshot(read: (reader: StorageReadView) => Promise): Promise { + const conn = await this.pool.getConnection(); + try { + await conn.query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"); + await conn.query("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"); + return await read({ + getSessionRow: (id) => this.getSessionRowOn(conn, id), + getMessages: (id, opts) => this.getMessagesOn(conn, id, opts), + getBlocks: (id, opts) => this.getBlocksOn(conn, id, opts), + getDeadRefs: (id) => this.getDeadRefsOn(conn, id), + getRefs: (id) => this.getRefsOn(conn, id), + }); + } finally { + try { + await conn.rollback(); + } finally { + conn.release(); + } + } + } + + private async getSessionRowOn(conn: Pool | PoolConnection, sessionId: string): Promise { + const [rows] = (await conn.query( `SELECT session_id, parent_session_id, fork_point_seq, protocol, label, title, host_title, namespace, origin, status, created_at, updated_at, next_block_id, next_run_id, meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json, last_ingested_seq, orphans_json @@ -1007,13 +1032,18 @@ export class MysqlBackend implements StorageBackend { }; } - async getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { + async getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { + return this.getMessagesOn(this.pool, sessionId, opts); + } + + private async getMessagesOn(conn: Pool | PoolConnection, sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { // #4390: retired rows stay out of the replay by default. - const [rows] = (await this.pool.query( - opts?.includeRetired === true - ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" - : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", - [sessionId], + if (opts?.seqs?.length === 0) return []; + const seqFilter = opts?.seqs ? ` AND seq IN (${opts.seqs.map(() => "?").join(",")})` : ""; + const retiredFilter = opts?.includeRetired === true ? "" : " AND retired_at IS NULL"; + const [rows] = (await conn.query( + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ?${retiredFilter}${seqFilter} ORDER BY seq`, + [sessionId, ...(opts?.seqs ?? [])], )) as unknown as Row[][]; const out: (MessageRow & { contentHash: string; retiredAt: number | null })[] = []; for (const r of rows) { @@ -1027,20 +1057,20 @@ export class MysqlBackend implements StorageBackend { kind: str(r.kind), contentHash, retiredAt: nullableNum(r.retired_at), - content: await this.hydrateMediaAsync((await this.getContent(this.pool, contentHash)) ?? ""), + content: await this.hydrateMediaAsync((await this.getContent(conn, contentHash)) ?? "", conn), }); } return out; } - private async hydrateMediaAsync(content: string): Promise { + private async hydrateMediaAsync(content: string, conn: Pool | PoolConnection): Promise { let out = content; if (out.includes('"$media"')) { try { const parsed = JSON.parse(out) as { imageBase64?: unknown }; const slot = parsed.imageBase64; if (slot !== null && typeof slot === "object" && typeof (slot as { $media?: unknown }).$media === "string") { - const bytes = await this.getContent(this.pool, (slot as { $media: string }).$media); + const bytes = await this.getContent(conn, (slot as { $media: string }).$media); if (typeof bytes === "string" && bytes.length > 0) { parsed.imageBase64 = bytes; out = JSON.stringify(parsed); @@ -1059,7 +1089,7 @@ export class MysqlBackend implements StorageBackend { if (hits.length > 0) { const cache = new Map(); for (const h of new Set(hits.map((x) => x.hash))) { - cache.set(h, await this.getContent(this.pool, h)); + cache.set(h, await this.getContent(conn, h)); } out = out.replace(markerRe, (all: string, mime: string, hash: string) => { const bytes = cache.get(hash); @@ -1070,12 +1100,16 @@ export class MysqlBackend implements StorageBackend { return out; } - async getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { - const [rows] = (await this.pool.query( + async getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + return this.getBlocksOn(this.pool, sessionId, opts); + } + + private async getBlocksOn(conn: Pool | PoolConnection, sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + const [rows] = (await conn.query( `SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json - FROM blocks WHERE session_id = ? ORDER BY ord`, - [sessionId], + FROM blocks WHERE session_id = ? ORDER BY ord${opts ? ", block_id LIMIT ? OFFSET ?" : ""}`, + [sessionId, ...(opts ? [opts.limit, opts.offset ?? 0] : [])], )) as unknown as Row[][]; const out: (BlockRow & { summaryHash: string; oneHash: string | null })[] = []; for (const r of rows) { @@ -1089,8 +1123,8 @@ export class MysqlBackend implements StorageBackend { tier: num(r.tier), summaryHash: str(r.summary_hash), oneHash: nullableStr(r.one_hash), - summary: (await this.getContent(this.pool, str(r.summary_hash))) ?? "", - one: r.one_hash === null || r.one_hash === undefined ? null : ((await this.getContent(this.pool, str(r.one_hash))) ?? null), + summary: (await this.getContent(conn, str(r.summary_hash))) ?? "", + one: r.one_hash === null || r.one_hash === undefined ? null : ((await this.getContent(conn, str(r.one_hash))) ?? null), supersedesBlockId: nullableStr(r.supersedes_block_id), active: num(r.active) === 1, createdAt: num(r.created_at), @@ -1102,7 +1136,11 @@ export class MysqlBackend implements StorageBackend { } async getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> { - const [rows] = (await this.pool.query( + return this.getDeadRefsOn(this.pool, sessionId); + } + + private async getDeadRefsOn(conn: Pool | PoolConnection, sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> { + const [rows] = (await conn.query( "SELECT ord, ref, ref_num, recorded_at FROM dead_refs WHERE session_id = ? ORDER BY ord", [sessionId], )) as unknown as Row[][]; @@ -1115,7 +1153,11 @@ export class MysqlBackend implements StorageBackend { } async getRefs(sessionId: string): Promise { - const [rows] = (await this.pool.query( + return this.getRefsOn(this.pool, sessionId); + } + + private async getRefsOn(conn: Pool | PoolConnection, sessionId: string): Promise { + const [rows] = (await conn.query( "SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord", [sessionId], )) as unknown as Row[][]; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index d527be96c4..be28d5ac6f 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -22,7 +22,7 @@ */ import { log as loggerLog } from "../logger.js"; -import { planIngest, planIngestDelta, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; +import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendBlockContents, mergeOrphanBlockContents, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; import type { BlockView } from "../session.js"; import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts"; import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; @@ -37,6 +37,15 @@ import type { CoreMessage, MessageContentStore } from "acp-kernel"; export const MAX_CONSECUTIVE_FAILURES = 3; +/** Bound transient CAS strings during normal cold loads, not the retained history. */ +const VIEW_READ_BATCH_SIZE = 32; + +interface ResidentFields { + header: { requests: number; nextBlockId: number; activeBlockCount: number; mappedRefCount: number }; + metadata?: Record; + blockContents?: Map; +} + /** #3133: read-back verification cadence — the verify leg (full read-back + * double shadowDigest) is O(log) per save, while the drift it guards is a * migration-parity property, not a per-commit one. The FIRST ingest of a @@ -506,22 +515,27 @@ export class UnifiedShadow { /** Loads the unified view of one session (full mode read path). * Returns null on miss — the caller falls back to the legacy file. */ viewFor(id: string): LegacySessionLike | null { - if (this.sync === null) return null; + const store = this.sync; + if (store === null) return null; if (this.stats.disabled) return null; try { - const sessionRow = this.sync.getSessionRow(id); - if (!sessionRow) return null; - const rows = { - session: sessionRow, - messages: this.sync.getMessages(id), - blocks: this.sync.getBlocks(id), - deadRefs: this.sync.getDeadRefs(id), - refs: this.sync.getRefs(id), - }; - // The view's `state` object carries every MERGE_STATE_FIELDS key by - // construction; the declared Record just needs a - // widening cast to reach LegacySessionLike. - const unified = loadLegacyView(rows) as LegacySessionLike; + const unified = store.readSnapshot(() => { + const sessionRow = store.getSessionRow(id); + if (!sessionRow) return null; + const builder = new LegacyViewBuilder(sessionRow, store.getRefs(id), store.getDeadRefs(id)); + for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) { + const blocks = store.getBlocks(id, { limit: VIEW_READ_BATCH_SIZE, offset }); + builder.addBlocks(blocks); + if (blocks.length < VIEW_READ_BATCH_SIZE) break; + } + for (let offset = 0; offset < builder.messageSeqs.length; offset += VIEW_READ_BATCH_SIZE) { + builder.addMessages(store.getMessages(id, { seqs: builder.messageSeqs.slice(offset, offset + VIEW_READ_BATCH_SIZE) })); + } + // The builder preserves every MERGE_STATE_FIELDS key; its + // generic state declaration only needs widening here. + return builder.finish() as LegacySessionLike; + }); + if (!unified) return null; warnTailGaps(id, unified as unknown as { tailGaps?: number }); return unified; } catch (err) { @@ -530,6 +544,38 @@ export class UnifiedShadow { } } + /** Lazy resident fields share the same source header without rebuilding a session. */ + readResidentFieldsFor(id: string, fields: { metadata?: boolean; blockContents?: boolean }): ResidentFields | null { + const store = this.sync; + if (store === null || this.stats.disabled) return null; + try { + return store.readSnapshot(() => { + const header = store.getLoadHeader(id); + if (!header) return null; + const result: ResidentFields = { header }; + if (fields.metadata) { + const json = store.getMetadataJson(id); + if (json === null) return null; + result.metadata = JSON.parse(json) as Record; + } + if (fields.blockContents) { + const contents: NonNullable = new Map(); + for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) { + const blocks = store.getBlockContents(id, { limit: VIEW_READ_BATCH_SIZE, offset }); + appendBlockContents(contents, blocks); + if (blocks.length < VIEW_READ_BATCH_SIZE) break; + } + mergeOrphanBlockContents(contents, store.getOrphansJson(id)); + result.blockContents = contents; + } + return result; + }); + } catch (err) { + loggerLog("warn", `[storage] resident field load failed for ${id} (${String(err)}) — falling back to the legacy file`); + return null; + } + } + /** #4323: true once this session's drift escalated and no clean sample * has since re-admitted it. Sqlite lanes only — network backends keep * the envelope as their sync read path, where freezing would strand @@ -971,15 +1017,21 @@ export class UnifiedShadow { if (this.stats.disabled) return null; try { const backend = this.backend!; - const sessionRow = await backend.getSessionRow(id); - if (!sessionRow) return null; - const loaded = loadLegacyView({ - session: sessionRow, - messages: await backend.getMessages(id), - blocks: await backend.getBlocks(id), - deadRefs: await backend.getDeadRefs(id), - refs: await backend.getRefs(id), - }) as LegacySessionLike; + const loaded = await backend.readSnapshot(async (reader) => { + const sessionRow = await reader.getSessionRow(id); + if (!sessionRow) return null; + const builder = new LegacyViewBuilder(sessionRow, await reader.getRefs(id), await reader.getDeadRefs(id)); + for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) { + const blocks = await reader.getBlocks(id, { limit: VIEW_READ_BATCH_SIZE, offset }); + builder.addBlocks(blocks); + if (blocks.length < VIEW_READ_BATCH_SIZE) break; + } + for (let offset = 0; offset < builder.messageSeqs.length; offset += VIEW_READ_BATCH_SIZE) { + builder.addMessages(await reader.getMessages(id, { seqs: builder.messageSeqs.slice(offset, offset + VIEW_READ_BATCH_SIZE) })); + } + return builder.finish() as LegacySessionLike; + }); + if (!loaded) return null; warnTailGaps(id, loaded as unknown as { tailGaps?: number }); return loaded; } catch (err) { diff --git a/src/storage/store.ts b/src/storage/store.ts index a13acf4edb..1cfae29b83 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -336,7 +336,7 @@ export class UnifiedStore { private readonly foreignLockHolder: { pid: number } | null; private readonly lockPath: string | null; - private constructor(db: SqliteDatabase, engine: SqliteEngineName, engineSource: SqliteEngineSource, lockPath: string | null, foreignLockHolder: { pid: number } | null) { + private constructor(db: SqliteDatabase, engine: SqliteEngineName, engineSource: SqliteEngineSource, lockPath: string | null, foreignLockHolder: { pid: number } | null, private readonly dbPath: string) { this.db = db; this.engine = engine; this.engineSource = engineSource; @@ -438,7 +438,25 @@ export class UnifiedStore { } catch (err) { loggerLog("warn", `[storage] WAL checkpoint on open failed for ${path}: ${String(err)}`); } - return new UnifiedStore(db, used, source, lockPath, foreignLockHolder); + return new UnifiedStore(db, used, source, lockPath, foreignLockHolder, path); + } + + /** Keep paged reads on one generation without taking the writer lock. */ + readSnapshot(read: () => T): T { + return this.db.readTransaction(read); + } + + /** Async callers need a separate connection: an await must not let a + * normal save on this store join the reader's transaction. */ + async readSnapshotAsync(read: (reader: UnifiedStore) => Promise): Promise { + const { db, engine, source } = openSqliteDatabase(this.dbPath, this.engine); + try { + db.exec("PRAGMA query_only = ON; BEGIN DEFERRED"); + const reader = new UnifiedStore(db, engine, source, null, null, this.dbPath); + return await read(reader); + } finally { + db.close(); // closing rolls back the read-only transaction + } } close(): void { @@ -2135,6 +2153,54 @@ export class UnifiedStore { return Buffer.from(row.bytes).toString("utf8"); } + /** Small arbitration fields for lazy reads; no serialized session surfaces. */ + getLoadHeader(sessionId: string): { requests: number; nextBlockId: number; activeBlockCount: number; mappedRefCount: number } | null { + const row = this.db.prepare(` + SELECT next_block_id, stats_json, + (SELECT COUNT(*) FROM blocks WHERE session_id = ? AND active = 1) AS active_blocks, + (SELECT COUNT(DISTINCT ref) FROM refs WHERE session_id = ? AND ref <> 'BLOCKED') AS mapped_refs + FROM sessions WHERE session_id = ? + `).get(sessionId, sessionId, sessionId) as { + next_block_id: number; stats_json: string; active_blocks: number; mapped_refs: number; + } | undefined; + if (!row) return null; + try { + const stats = JSON.parse(row.stats_json) as { requests?: unknown }; + return { + requests: Number(stats.requests ?? 0), + nextBlockId: row.next_block_id, + activeBlockCount: row.active_blocks, + mappedRefCount: row.mapped_refs, + }; + } catch { + return null; + } + } + + getMetadataJson(sessionId: string): string | null { + const row = this.db.prepare("SELECT metadata_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { metadata_json: string } | undefined; + return row?.metadata_json ?? null; + } + + getOrphansJson(sessionId: string): string | null { + const row = this.db.prepare("SELECT orphans_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { orphans_json: string | null } | undefined; + return row?.orphans_json ?? null; + } + + /** Contents for active AND inactive blocks, without their CompressionBlock payload. */ + getBlockContents(sessionId: string, opts: { limit: number; offset?: number }): Pick[] { + const rows = this.db.prepare(`SELECT block_id, summary_hash, one_hash FROM blocks + WHERE session_id = ? ORDER BY ord, block_id LIMIT ? OFFSET ?`) + .all(sessionId, opts.limit, opts.offset ?? 0) as { block_id: string; summary_hash: string; one_hash: string | null }[]; + return rows.map((row) => ({ + blockId: row.block_id, + summary: this.getContent(row.summary_hash) ?? "", + one: row.one_hash ? this.getContent(row.one_hash) : null, + })); + } + getSessionRow(sessionId: string): SessionRow | null { const row = this.db .prepare( @@ -2308,17 +2374,18 @@ export class UnifiedStore { })); } - getMessages(sessionId: string, opts?: { includeRetired?: boolean }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { // #4390: retired rows (rewound/undo history) stay out of the unified // replay by default — they are history, not view. Callers that need // the full historian record (export/diagnostics) opt in. + if (opts?.seqs?.length === 0) return []; + const seqFilter = opts?.seqs ? ` AND seq IN (${opts.seqs.map(() => "?").join(",")})` : ""; + const retiredFilter = opts?.includeRetired === true ? "" : " AND retired_at IS NULL"; const rows = this.db .prepare( - opts?.includeRetired === true - ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" - : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ?${retiredFilter}${seqFilter} ORDER BY seq`, ) - .all(sessionId) as Record[]; + .all(sessionId, ...(opts?.seqs ?? [])) as Record[]; return rows.map((r) => ({ seq: r.seq as number, msgId: r.msg_id as string, @@ -2374,14 +2441,14 @@ export class UnifiedStore { return row ? row.refcount : 0; } - getBlocks(sessionId: string): (BlockRow & { summaryHash: string; oneHash: string | null })[] { + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): (BlockRow & { summaryHash: string; oneHash: string | null })[] { const rows = this.db .prepare( `SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json - FROM blocks WHERE session_id = ? ORDER BY ord`, + FROM blocks WHERE session_id = ? ORDER BY ord${opts ? ", block_id LIMIT ? OFFSET ?" : ""}`, ) - .all(sessionId) as Record[]; + .all(sessionId, ...(opts ? [opts.limit, opts.offset ?? 0] : [])) as Record[]; return rows.map((r) => ({ blockId: r.block_id as string, ord: r.ord as number, diff --git a/tests/issue4739-storage-read-snapshot.test.ts b/tests/issue4739-storage-read-snapshot.test.ts new file mode 100644 index 0000000000..5d77b9cc8c --- /dev/null +++ b/tests/issue4739-storage-read-snapshot.test.ts @@ -0,0 +1,191 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { UnifiedStore, type IngestSessionInput } from "../src/storage/store.ts"; +import { SqliteBackend } from "../src/storage/backend.ts"; +import { openSqliteDatabase } from "../src/storage/driver.ts"; +import { MysqlBackend, type Mysql2Module } from "../src/storage/mysql.ts"; + +function fixture(requests = 1): IngestSessionInput { + return { + session: { + sessionId: "snapshot", createdAt: 1, updatedAt: requests, nextBlockId: 4, nextRunId: 1, + metaJson: "{}", statsJson: JSON.stringify({ requests }), metadataJson: JSON.stringify({ requests }), + kstateJson: "{}", tailJson: "{}", ccrJson: null, + orphansJson: JSON.stringify({ orphan: { one: null, full: { text: "orphan", count: 1 } } }), + }, + messages: [0, 1, 2].map((seq) => ({ + seq, msgId: `m${seq}`, ref: null, refNum: null, role: "user", kind: "message", + content: JSON.stringify({ id: `m${seq}`, role: "user", text: `text${seq}` }), + })), + blocks: [0, 1, 2].map((ord) => ({ + blockId: `b${ord}`, ord, startSeq: ord, endSeq: ord, startRef: null, endRef: null, tier: 0, + summary: JSON.stringify({ text: `full${ord}`, count: 1 }), + one: JSON.stringify({ text: `one${ord}`, count: 1 }), supersedesBlockId: null, + active: ord !== 1, createdAt: 1, coveredHash: "[]", payloadJson: JSON.stringify({ blockId: `b${ord}` }), + })), + refs: [ + { ord: 0, rawId: "a", ref: "m00001", refNum: 1 }, + { ord: 1, rawId: "b", ref: "m00001", refNum: 1 }, + { ord: 2, rawId: "c", ref: "BLOCKED", refNum: null }, + ], + deadRefs: [], ccrEntries: [], + orphansJson: JSON.stringify({ orphan: { one: null, full: { text: "orphan", count: 1 } } }), + }; +} + +for (const engine of ["node:sqlite", "better-sqlite3"] as const) { + test(`${engine}: selected rows, page boundaries and narrow reads preserve their contracts`, (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-read-snapshot-")); + const file = join(dir, "index.db"); + let store: UnifiedStore | undefined; + try { + try { store = UnifiedStore.open(file, engine); } catch (err) { + if (engine === "better-sqlite3" && String(err).includes("[storage:better-sqlite3]")) return t.skip(String(err)); + throw err; + } + store.ingestSession(fixture()); + const db = openSqliteDatabase(file, engine).db; + try { + db.prepare("UPDATE messages SET retired_at = 1 WHERE session_id = ? AND seq = 1").run("snapshot"); + assert.deepEqual(store.getMessages("snapshot").map((row) => row.seq), [0, 2]); + assert.deepEqual(store.getMessages("snapshot", { seqs: [2, 1, 0, 2, 99] }).map((row) => row.seq), [0, 2]); + assert.deepEqual(store.getMessages("snapshot", { seqs: [1], includeRetired: true }).map((row) => row.seq), [1]); + assert.deepEqual(store.getMessages("snapshot", { seqs: [] }), []); + assert.deepEqual(store.getBlocks("snapshot", { limit: 1, offset: 1 }).map((row) => row.blockId), ["b1"]); + assert.deepEqual(store.getBlocks("snapshot", { limit: 2, offset: 3 }), []); + assert.equal(store.getBlocks("snapshot").length, 3); + + // Invalid unrelated payloads must not be decoded by the lazy field readers. + db.prepare("UPDATE sessions SET tail_json = 'bad', kstate_json = 'bad', ccr_json = 'bad' WHERE session_id = ?").run("snapshot"); + db.prepare("UPDATE blocks SET payload_json = 'bad' WHERE session_id = ?").run("snapshot"); + assert.deepEqual(store.getLoadHeader("snapshot"), { requests: 1, nextBlockId: 4, activeBlockCount: 2, mappedRefCount: 1 }); + db.prepare("UPDATE sessions SET stats_json = ? WHERE session_id = ?").run('{"requests":"2"}', "snapshot"); + assert.equal(store.getLoadHeader("snapshot")!.requests, 2, "old numeric-string counter matches full-load arbitration"); + assert.equal(store.getMetadataJson("snapshot"), '{"requests":1}'); + assert.deepEqual(JSON.parse(store.getOrphansJson("snapshot")!), JSON.parse(fixture().orphansJson!)); + const contents = store.getBlockContents("snapshot", { limit: 2, offset: 1 }); + assert.deepEqual(contents.map((row) => row.blockId), ["b1", "b2"]); + assert.deepEqual(JSON.parse(contents[0]!.summary), { text: "full1", count: 1 }); + assert.deepEqual(JSON.parse(contents[0]!.one!), { text: "one1", count: 1 }); + assert.equal(store.getLoadHeader("missing"), null); + assert.equal(store.getMetadataJson("missing"), null); + db.prepare("UPDATE blocks SET ord = 0 WHERE session_id = ?").run("snapshot"); + assert.equal(store.getBlocks("snapshot", { limit: 1, offset: 1 })[0]!.blockId, "b1", "equal ord uses a stable page boundary"); + } finally { db.close(); } + } finally { + store?.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + test(`${engine}: sync snapshot permits another writer and keeps the earlier generation`, (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-read-snapshot-")); + const file = join(dir, "index.db"); + let store: UnifiedStore | undefined; + try { + try { store = UnifiedStore.open(file, engine); } catch (err) { + if (engine === "better-sqlite3" && String(err).includes("[storage:better-sqlite3]")) return t.skip(String(err)); + throw err; + } + store.ingestSession(fixture()); + const writer = openSqliteDatabase(file, engine).db; + const reader = store; + try { + reader.readSnapshot(() => { + assert.equal(reader.getLoadHeader("snapshot")!.requests, 1); + writer.prepare("UPDATE sessions SET stats_json = ? WHERE session_id = ?").run('{"requests":2}', "snapshot"); + writer.prepare("UPDATE blocks SET active = 0 WHERE session_id = ?").run("snapshot"); + assert.equal(reader.getLoadHeader("snapshot")!.requests, 1); + assert.equal(reader.getBlocks("snapshot", { limit: 1, offset: 2 })[0]!.active, true); + }); + assert.equal(reader.getLoadHeader("snapshot")!.requests, 2); + assert.throws(() => reader.readSnapshot(() => { throw new Error("reader failed"); }), /reader failed/); + writer.prepare("UPDATE sessions SET stats_json = ? WHERE session_id = ?").run('{"requests":3}', "snapshot"); + assert.equal(reader.readSnapshot(() => reader.getLoadHeader("snapshot")!.requests), 3); + } finally { writer.close(); } + } finally { + store?.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); +} + +test("async sqlite snapshot isolates normal writes across await and closes on callback failure", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-read-snapshot-")); + const backend = SqliteBackend.open(join(dir, "index.db")); + try { + await backend.ingestSession(fixture()); + await backend.readSnapshot(async (reader) => { + assert.equal(JSON.parse((await reader.getSessionRow("snapshot"))!.statsJson).requests, 1); + await backend.ingestSession(fixture(2)); + assert.equal(JSON.parse((await reader.getSessionRow("snapshot"))!.statsJson).requests, 1); + assert.deepEqual((await reader.getMessages("snapshot", { seqs: [2] })).map((row) => row.seq), [2]); + }); + assert.equal(JSON.parse((await backend.getSessionRow("snapshot"))!.statsJson).requests, 2); + await assert.rejects(backend.readSnapshot(async (reader) => { + await reader.getSessionRow("snapshot"); + throw new Error("async reader failed"); + }), /async reader failed/); + await backend.ingestSession(fixture(3)); + await backend.readSnapshot(async (reader) => { + assert.equal(JSON.parse((await reader.getSessionRow("snapshot"))!.statsJson).requests, 3); + }); + } finally { + await backend.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("mysql snapshot pins SQL, CAS and media hydration to its read-only connection and releases on error", async () => { + const queries: { sql: string; values?: unknown[] }[] = []; + let releases = 0; + let rollbacks = 0; + const mediaHash = "a".repeat(64); + const conn = { + async query(sql: string, values?: unknown[]) { + queries.push({ sql, values }); + if (sql.includes("FROM messages")) return [[{ seq: 2, msg_id: "m2", role: "user", kind: "message", content_hash: "msg" }]]; + if (sql.includes("FROM blocks")) return [[{ block_id: "b1", summary_hash: "summary", one_hash: "one", active: 1 }]]; + if (sql.includes("FROM content")) { + const bytes = values?.[0] === "msg" ? JSON.stringify({ imageBase64: { $media: mediaHash } }) + : values?.[0] === mediaHash ? "image-bytes" : '{"text":"summary","count":1}'; + return [[{ bytes }]]; + } + return [[]]; + }, + async rollback() { rollbacks++; }, + release() { releases++; }, + }; + const driver = { createPool: () => ({ + getConnection: async () => conn, + query: async () => { throw new Error("snapshot escaped to pool"); }, + end: async () => {}, + }) } as unknown as Mysql2Module; + const backend = new MysqlBackend("mysql://snapshot-test", driver); + await backend.readSnapshot(async (reader) => { + const before = queries.length; + assert.deepEqual(await reader.getMessages("snapshot", { seqs: [] }), []); + assert.equal(queries.length, before); + const rows = await reader.getMessages("snapshot", { seqs: [2] }); + assert.equal(JSON.parse(rows[0]!.content).imageBase64, "image-bytes"); + assert.equal((await reader.getBlocks("snapshot", { limit: 1, offset: 2 }))[0]!.blockId, "b1"); + await reader.getSessionRow("snapshot"); + await reader.getRefs("snapshot"); + await reader.getDeadRefs("snapshot"); + }); + assert.equal(queries[0]!.sql, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"); + assert.equal(queries[1]!.sql, "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"); + const selected = queries.find((query) => query.sql.includes("FROM messages"))!; + assert.match(selected.sql, /retired_at IS NULL AND seq IN \(\?\)/); + assert.deepEqual(selected.values, ["snapshot", 2]); + const paged = queries.find((query) => query.sql.includes("FROM blocks"))!; + assert.match(paged.sql, /ORDER BY ord, block_id LIMIT \? OFFSET \?/); + assert.deepEqual(paged.values, ["snapshot", 1, 2]); + await assert.rejects(backend.readSnapshot(async () => { throw new Error("mysql reader failed"); }), /mysql reader failed/); + assert.equal(rollbacks, 2); + assert.equal(releases, 2); + await backend.close(); +}); diff --git a/tests/persist-lazy-narrow-reads.test.ts b/tests/persist-lazy-narrow-reads.test.ts new file mode 100644 index 0000000000..e295ad018c --- /dev/null +++ b/tests/persist-lazy-narrow-reads.test.ts @@ -0,0 +1,208 @@ +import test, { type TestContext } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createInitialState, type CompressionBlock } from "acp-kernel"; +import { SessionStore } from "../src/persist.ts"; +import { isDroppedMetadata, LazyBlockContents, type BlockContent, type Session } from "../src/session.ts"; +import type { UnifiedShadow } from "../src/storage/shadow.ts"; +import type { UnifiedStore } from "../src/storage/store.ts"; +import { rmrf } from "./tmp-rm.ts"; + +const ROOT = mkdtempSync(path.join(tmpdir(), "bili-lazy-narrow-")); +process.env.BILI_CONFIG_FILE = path.join(ROOT, "no-config.json"); +process.env.BILI_PERSIST_ZSTD = "0"; +delete process.env.BILI_STORAGE_JSON_WRITER; +test.after(() => rmrf(ROOT)); + +type SavedFields = { metadata: Record; blockContents: Record }; +type StoreInternals = { + shadow: UnifiedShadow; + loadEnvelope(id: string, meta?: Session["meta"]): { payload: SavedFields } | null; + buildRecordForSave(session: Session): SavedFields; +}; + +function makeBlock(blockId: string, active = true): CompressionBlock { + return { + blockId, runId: "r0", tier: 1, summary: `summary ${blockId}`, + directMessageIds: [], effectiveMessageIds: [], directBlockIds: [], + compressedTokens: 0, createdAt: 1, survivedCount: 0, generation: "young", active, + }; +} + +function makeSession(id: string, source: string): Session { + return { + id, meta: { protocol: "openai", upstreamOrigin: "http://up:1", label: id }, + stats: { + requests: 5, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, + cacheSamples: 0, lastInputTokens: 0, contextTokens: 0, compressCreditTokens: 0, + retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, + storeBytesSaved: 0, rangeRestores: 0, + }, + metadata: { source }, state: createInitialState(), + createdAt: 1, lastSeen: 2, inFlight: 0, persisted: false, pendingRetrievals: [], + blockContents: new Map([["orphan", { one: null, full: { text: `${source} orphan original`, count: 2 } }]]), + }; +} + +function fixture(t: TestContext, mode: "full" | "legacy"): { store: SessionStore; dir: string } { + process.env.BILI_STORAGE_UNIFIED = mode; + const dir = mkdtempSync(path.join(ROOT, `${mode}-`)); + const store = new SessionStore({ dir, enabled: true, debounceMs: 60_000 }); + t.after(() => store.closeQuietly()); + return { store, dir }; +} + +function internals(store: SessionStore): StoreInternals { + return store as unknown as StoreInternals; +} + +function stateFile(dir: string): string { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name); + if (entry.isDirectory()) { + const nested = stateFile(file); + if (nested) return nested; + } else if (entry.name.endsWith(".json") && !entry.name.endsWith(".content-store.json")) { + return file; + } + } + return ""; +} + +function assertDropped(session: Session): void { + assert.equal(isDroppedMetadata(session.metadata), true); + assert.ok(session.blockContents instanceof LazyBlockContents); + assert.equal(session.blockContents.isDropped, true); +} + +test("full-mode lazy getters read only their requested resident field and keep all originals", async (t) => { + const { store } = fixture(t, "full"); + const session = makeSession("narrow-full", "index"); + session.state.blocks = [makeBlock("active"), makeBlock("archived", false)]; + session.blockContents.set("active", { + one: { text: "short view", count: 1 }, full: { text: "full active original", count: 4 }, + }); + session.blockContents.set("archived", { + one: { text: "same archived original", count: 2 }, full: { text: "same archived original", count: 2 }, + }); + await store.writeNow(session); + assertDropped(session); + + const shadow = internals(store).shadow; + const driver = (shadow as unknown as { sync: UnifiedStore }).sync; + assert.ok(driver, "full mode must exercise the SQLite lane"); + const wholeLoads = t.mock.method(store, "loadSync"); + const wholeViews = t.mock.method(shadow, "viewFor"); + const messages = t.mock.method(driver, "getMessages"); + const blockPayloads = t.mock.method(driver, "getBlocks"); + const sessionRows = t.mock.method(driver, "getSessionRow"); + const metadata = t.mock.method(driver, "getMetadataJson"); + const contents = t.mock.method(driver, "getBlockContents"); + const narrow = t.mock.method(shadow, "readResidentFieldsFor"); + const imports = t.mock.method(shadow, "importRecord"); + const saves = t.mock.method(store, "scheduleSave"); + + assert.equal(session.metadata.source, "index"); + assert.equal(metadata.mock.callCount(), 1); + assert.equal(contents.mock.callCount(), 0, "metadata access does not fetch any originals"); + assert.equal((session.blockContents as LazyBlockContents).isDropped, true); + assert.deepEqual(narrow.mock.calls[0].arguments[1], { metadata: true }); + + assert.deepEqual(session.blockContents.get("active"), { + one: { text: "short view", count: 1 }, full: { text: "full active original", count: 4 }, + }); + assert.deepEqual(session.blockContents.get("archived"), { + one: null, full: { text: "same archived original", count: 2 }, + }, "inactive originals survive and duplicate one/full pairs normalize"); + assert.deepEqual(session.blockContents.get("orphan"), { one: null, full: { text: "index orphan original", count: 2 } }); + assert.equal(session.blockContents.size, 3); + assert.equal(metadata.mock.callCount(), 1, "contents access does not re-fetch metadata"); + assert.deepEqual(narrow.mock.calls[1].arguments[1], { blockContents: true }); + for (const spy of [wholeLoads, wholeViews, messages, blockPayloads, sessionRows, imports, saves]) { + assert.equal(spy.mock.callCount(), 0, "a lazy getter must not rebuild, import or schedule a whole session"); + } +}); + +test("legacy save builder shares one envelope read for both dropped fields without retaining an old projection", async (t) => { + const { store, dir } = fixture(t, "legacy"); + const session = makeSession("narrow-legacy", "first"); + await store.writeNow(session); + assertDropped(session); + const envelopes = t.mock.method(internals(store), "loadEnvelope"); + const wholeLoads = t.mock.method(store, "loadSync"); + + const saved = internals(store).buildRecordForSave(session); + assert.equal(saved.metadata.source, "first"); + assert.equal(saved.blockContents.orphan.full.text, "first orphan original"); + assert.equal(envelopes.mock.callCount(), 1, "one build shares a single source read across both lazy getters"); + assert.equal(wholeLoads.mock.callCount(), 0); + await store.writeNow(session); + assertDropped(session); + const priorReads = envelopes.mock.callCount(); + + // An external writer changes only the two resident fields. The next read + // must consult disk again after the synchronous save has finished. + const file = stateFile(dir); + assert.ok(file); + const envelope = JSON.parse(readFileSync(file, "utf8")) as { payload: SavedFields }; + envelope.payload.metadata.source = "external"; + envelope.payload.blockContents.orphan.full.text = "external original"; + writeFileSync(file, JSON.stringify(envelope)); + assert.equal(session.metadata.source, "external"); + assert.equal(session.blockContents.get("orphan")?.full.text, "external original"); + assert.equal(envelopes.mock.callCount(), priorReads + 2, "independent touches outside a save each select their current source"); + assert.equal(wholeLoads.mock.callCount(), 0); +}); + +test("lazy source arbitration matches cold-load progress and drift rules", async (t) => { + type Generation = { requests: number; nextBlockId: number; active: number; refs: number }; + const base: Generation = { requests: 5, nextBlockId: 10, active: 1, refs: 1 }; + const cases: { name: string; latch: boolean; index?: Partial; envelope: Partial; expected: string }[] = [ + { name: "strictly more envelope requests", latch: false, envelope: { requests: 6 }, expected: "envelope" }, + { name: "equal requests and ahead envelope fold cursor", latch: false, envelope: { nextBlockId: 11 }, expected: "envelope" }, + { name: "equal progress keeps index", latch: false, envelope: {}, expected: "index" }, + { name: "drift prefers greater index active coverage", latch: true, index: { active: 2 }, envelope: { requests: 99 }, expected: "index" }, + { name: "drift prefers greater index mapped refs", latch: true, index: { refs: 3 }, envelope: { requests: 99 }, expected: "index" }, + { name: "drift ties keep envelope", latch: true, envelope: {}, expected: "envelope" }, + ]; + for (const [i, c] of cases.entries()) { + await t.test(c.name, async (st) => { + const { store } = fixture(st, "full"); + const id = `narrow-arbitration-${i}`; + const generation = (source: string, values: Generation): Session => { + const session = makeSession(id, source); + session.stats.requests = values.requests; + session.state.nextBlockId = values.nextBlockId; + session.state.blocks = Array.from({ length: values.active }, (_, n) => makeBlock(`b${n + 1}`)); + for (let n = 1; n <= values.refs; n++) { + const ref = `m${String(n).padStart(5, "0")}`; + session.state.messageRefs.byRaw[`raw-${n}`] = ref; + session.state.messageRefs.byRef[ref] = `raw-${n}`; + } + return session; + }; + await store.writeNow(generation("index", { ...base, ...c.index })); + const shadow = internals(store).shadow; + const ingest = st.mock.method(shadow, "record", () => {}); + const envelope = generation("envelope", { ...base, ...c.envelope }); + await store.writeNow(envelope); + ingest.mock.restore(); + if (c.latch) shadow.driftedSessions.add(id); + assertDropped(envelope); + + const loads = st.mock.method(store, "loadSync"); + const views = st.mock.method(shadow, "viewFor"); + assert.equal(envelope.metadata.source, c.expected); + assert.equal(envelope.blockContents.get("orphan")?.full.text, `${c.expected} orphan original`); + assert.equal(loads.mock.callCount(), 0); + assert.equal(views.mock.callCount(), 0); + loads.mock.restore(); + views.mock.restore(); + const cold = store.loadSync(id); + assert.equal(cold?.metadata.source, c.expected, "the public cold loader selects the same generation"); + assert.equal(cold?.blockContents.get("orphan")?.full.text, envelope.blockContents.get("orphan")?.full.text); + }); + } +}); diff --git a/tests/persist-rollback-guard.test.ts b/tests/persist-rollback-guard.test.ts index 6441a048a5..ae6357bec0 100644 --- a/tests/persist-rollback-guard.test.ts +++ b/tests/persist-rollback-guard.test.ts @@ -36,7 +36,7 @@ function withTempStore(name: string, fn: (store: SessionStore, dir: string, warn try { await fn(store, dir, warnings); } finally { - store.cancelAll(); + store.closeQuietly(); rmrf(dir); } }) as unknown as Promise; diff --git a/tests/storage-backend-contract.test.ts b/tests/storage-backend-contract.test.ts index ba36a1a71b..762e0aa7cc 100644 --- a/tests/storage-backend-contract.test.ts +++ b/tests/storage-backend-contract.test.ts @@ -248,6 +248,15 @@ test("mysql lane: contract surface against MySQL 8.0", { skip: MYSQL_DSN.length const rows2 = await backend.getMessages(sid); assert.equal(rows2.length, 2, "no duplicate rows on identical re-ingest"); + await backend.readSnapshot(async (reader) => { + assert.equal((await reader.getSessionRow(sid))!.sessionId, sid); + assert.deepEqual(await reader.getMessages(sid, { seqs: [] }), []); + const selected = await reader.getMessages(sid, { seqs: [rows2[1]!.seq] }); + assert.equal(selected.length, 1); + assert.equal(selected[0]!.content, rows2[1]!.content); + assert.deepEqual(await reader.getBlocks(sid, { limit: 1 }), []); + }); + // reap journal await backend.reapBlocks(sid, new Set(["b2"]), new Map([["b2", { one: null, full: { text: "orphaned", count: 0 } }]])); await backend.reapBlocks(sid, new Set(["b2"]), new Map(), ); diff --git a/tests/storage-bounded-load.test.ts b/tests/storage-bounded-load.test.ts new file mode 100644 index 0000000000..a412e8457f --- /dev/null +++ b/tests/storage-bounded-load.test.ts @@ -0,0 +1,222 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { planIngest, loadLegacyView, LegacyViewBuilder, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { SqliteBackend } from "../src/storage/backend.ts"; +import type { SqliteDatabase } from "../src/storage/driver.ts"; + +function fixture(): LegacySessionLike { + const messages: NonNullable = Array.from({ length: 80 }, (_, i) => ({ + id: `msg${i}`, role: "user", contentType: "text", text: `message ${i}`, + })); + const blocks: LegacySessionLike["state"]["blocks"] = Array.from({ length: 70 }, (_, i) => ({ + blockId: `b${i + 1}`, runId: "r1", tier: 1, summary: `summary ${i + 1}`, + directMessageIds: [`msg${i}`], effectiveMessageIds: [`msg${i}`], directBlockIds: [], + compressedTokens: 10, createdAt: 1000 + i, survivedCount: 0, generation: "young", active: i % 2 === 0, + })); + return { + id: "bounded", createdAt: 1000, lastSeen: 2000, + meta: { protocol: "openai" }, stats: { requests: 7 }, metadata: { pluginAgent: "codex", note: "retained" }, + state: { + blocks, nextBlockId: 71, nextRunId: 2, + messageRefs: { byRaw: { msg0: "m00001", alias: "m00001", ghost: "m99999", blocked: "BLOCKED" }, byRef: { m00001: "msg0", m99999: "ghost" } }, + deadRefs: ["m90001", "m90002"], + }, + blockContents: new Map(blocks.map((block) => [block.blockId, { + one: { text: `one ${block.blockId}`, count: 1 }, full: { text: `original ${block.blockId}`, count: 2 }, + }])), + decompositionLog: messages, + pluginSnapshot: messages.slice(0, 36), + lastMessages: [messages[65], messages[33], messages[65]], + lastMessagesFolded: true, + contentStore: { version: 1, byHash: { h: "retained CCR" }, byRef: {} }, + }; +} + +function seed(store: UnifiedStore): void { + const plan = planIngest(fixture()); + plan.session.orphansJson = JSON.stringify({ + b1: { one: null, full: { text: "stale journal duplicate", count: 1 } }, + reaped: { one: null, full: { text: "reaped original", count: 1 } }, + }); + store.ingestSession(plan); + // Simulate missing historical backing rows after the ingest remapper. + const tail = JSON.parse(store.getSessionRow("bounded")!.tailJson) as { tailSeqs: number[]; pluginSnapshotSeqs: number[] }; + tail.tailSeqs.push(9999); + tail.pluginSnapshotSeqs.push(9999); + (store as unknown as { db: SqliteDatabase }).db.prepare("UPDATE sessions SET tail_json = ? WHERE session_id = ?").run(JSON.stringify(tail), "bounded"); +} + +function allRows(store: UnifiedStore) { + return { + session: store.getSessionRow("bounded")!, messages: store.getMessages("bounded"), + blocks: store.getBlocks("bounded"), refs: store.getRefs("bounded"), deadRefs: store.getDeadRefs("bounded"), + }; +} + +function assertHistory(view: LegacySessionLike | null): asserts view is LegacySessionLike { + assert.ok(view); + assert.equal(view.state.blocks.length, 70); + assert.equal(view.state.blocks.filter((block) => !block.active).length, 35, "inactive ancestors remain available to decompress"); + assert.deepEqual(view.state.blocks.map((block) => block.blockId), fixture().state.blocks.map((block) => block.blockId)); + assert.equal(view.blockContents.get("b1")!.full.text, "original b1", "block row wins over a stale orphan journal entry"); + assert.equal(view.blockContents.get("reaped")!.full.text, "reaped original"); + assert.equal(view.blockContents.size, 71); + assert.deepEqual(view.lastMessages?.map((message) => message.id), ["msg65", "msg33", "msg65"]); + assert.strictEqual(view.lastMessages![0], view.lastMessages![2], "repeated pointers retain shared message identity"); + assert.strictEqual(view.lastMessages![1], view.pluginSnapshot![33], "tail and snapshot overlap shares one parsed message"); + assert.deepEqual(view.state.messageRefs, fixture().state.messageRefs, "ghost and BLOCKED refs survive verbatim"); + assert.deepEqual(view.state.deadRefs, ["m90001", "m90002"]); + assert.deepEqual(view.contentStore, fixture().contentStore); + assert.equal((view as unknown as { tailGaps: number }).tailGaps, 2); +} + +test("ordinary sync cold load reads only unique tail/snapshot seqs and bounded block pages", (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-bounded-load-")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + const store = (shadow as unknown as { sync: UnifiedStore }).sync; + t.after(() => { shadow.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(store); + const expected = loadLegacyView(allRows(store)); + const getMessages = store.getMessages.bind(store); + const getBlocks = store.getBlocks.bind(store); + const requested: number[] = []; + const blockPages: number[] = []; + let snapshots = 0; + const snapshot = store.readSnapshot.bind(store); + store.readSnapshot = (fn) => { snapshots++; return snapshot(fn); }; + store.getMessages = (id, opts) => { + assert.ok(opts?.seqs && opts.seqs.length <= 32, "cold load must never hydrate the whole message log"); + requested.push(...opts.seqs); + return getMessages(id, opts); + }; + store.getBlocks = (id, opts) => { + assert.equal(opts?.limit, 32); + blockPages.push(opts?.offset ?? 0); + return getBlocks(id, opts); + }; + const view = shadow.viewFor("bounded"); + assertHistory(view); + assert.deepEqual(view, expected); + assert.equal(snapshots, 1, "header and every page share one read snapshot"); + assert.deepEqual(blockPages, [0, 32, 64]); + assert.equal(requested.length, new Set(requested).size); + assert.deepEqual(new Set(requested), new Set([65, 33, 9999, ...Array.from({ length: 36 }, (_, i) => i)])); + assert.equal(shadow.viewFor("absent"), null); +}); + +test("ordinary async cold load uses the snapshot reader for each bounded page", async (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-bounded-async-")); + const backend = SqliteBackend.open(join(dir, "index.db")); + t.after(async () => { await backend.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(backend.store); + const expected = loadLegacyView(allRows(backend.store)); + const readSnapshot = backend.readSnapshot.bind(backend); + const offsets: number[] = []; + const requested: number[] = []; + let snapshots = 0; + backend.readSnapshot = (fn) => readSnapshot(async (reader) => { + snapshots++; + return fn({ + ...reader, + getMessages: async (id, opts) => { + assert.ok(opts?.seqs && opts.seqs.length <= 32); + requested.push(...opts.seqs); + return reader.getMessages(id, opts); + }, + getBlocks: async (id, opts) => { + assert.equal(opts?.limit, 32); + offsets.push(opts?.offset ?? 0); + return reader.getBlocks(id, opts); + }, + }); + }); + backend.getMessages = async () => { throw new Error("load bypassed the snapshot reader"); }; + backend.getBlocks = async () => { throw new Error("load bypassed the snapshot reader"); }; + const shadow = Reflect.construct(UnifiedShadow, ["full", null, () => null, backend]) as UnifiedShadow; + const view = await shadow.viewForAsync("bounded"); + assertHistory(view); + assert.deepEqual(view, expected); + assert.equal(snapshots, 1); + assert.deepEqual(offsets, [0, 32, 64]); + assert.equal(requested.length, new Set(requested).size); + assert.deepEqual(new Set(requested), new Set([65, 33, 9999, ...Array.from({ length: 36 }, (_, i) => i)])); +}); + +test("a failed block page falls back and releases the read snapshot for the next load", (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-bounded-failure-")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + const store = (shadow as unknown as { sync: UnifiedStore }).sync; + t.after(() => { shadow.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(store); + const getBlocks = store.getBlocks.bind(store); + store.getBlocks = (id, opts) => { + if (opts?.offset === 32) throw new Error("injected block page failure"); + return getBlocks(id, opts); + }; + assert.equal(shadow.viewFor("bounded"), null, "partial views must never escape on a failed page"); + store.getBlocks = getBlocks; + assertHistory(shadow.viewFor("bounded")); +}); + +test("resident field reads avoid unrelated JSON, messages, refs, and block payloads", (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-resident-fields-")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + const store = (shadow as unknown as { sync: UnifiedStore }).sync; + t.after(() => { shadow.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(store); + const expected = loadLegacyView(allRows(store)); + for (const key of ["getSessionRow", "getMessages", "getBlocks", "getRefs", "getDeadRefs"] as const) { + store[key] = () => { throw new Error(`resident field read called ${key}`); }; + } + let metadataReads = 0; + let contentReads = 0; + let orphanReads = 0; + let snapshots = 0; + const snapshot = store.readSnapshot.bind(store); + store.readSnapshot = (fn) => { snapshots++; return snapshot(fn); }; + const metadata = store.getMetadataJson.bind(store); + const contents = store.getBlockContents.bind(store); + const orphans = store.getOrphansJson.bind(store); + store.getMetadataJson = (id) => { metadataReads++; return metadata(id); }; + store.getBlockContents = (id, opts) => { contentReads++; assert.equal(opts?.limit, 32); return contents(id, opts); }; + store.getOrphansJson = (id) => { orphanReads++; return orphans(id); }; + const meta = shadow.readResidentFieldsFor("bounded", { metadata: true }); + assert.deepEqual(meta, { header: { requests: 7, nextBlockId: 71, activeBlockCount: 35, mappedRefCount: 2 }, metadata: expected.metadata }); + assert.equal(contentReads, 0); + assert.equal(orphanReads, 0); + const blocks = shadow.readResidentFieldsFor("bounded", { blockContents: true }); + assert.deepEqual(blocks?.blockContents, expected.blockContents); + assert.equal(metadataReads, 1, "content getter does not read metadata"); + const both = shadow.readResidentFieldsFor("bounded", { metadata: true, blockContents: true }); + assert.deepEqual(both?.metadata, expected.metadata); + assert.deepEqual(both?.blockContents, expected.blockContents); + assert.equal(snapshots, 3, "combined fields use one snapshot"); + assert.equal(contentReads, 6); + assert.equal(orphanReads, 2); +}); + +test("incremental assembly preserves empty versus absent pointers and skips a corrupt orphan journal", () => { + const plan = planIngest(fixture()); + plan.session.tailJson = JSON.stringify({ tailSeqs: [], pluginSnapshotSeqs: null, tailFolded: false }); + plan.session.orphansJson = "broken journal"; + const builder = new LegacyViewBuilder(plan.session, plan.refs!, plan.deadRefs); + assert.deepEqual(builder.messageSeqs, []); + builder.addBlocks(plan.blocks); + const view = builder.finish(); + assert.deepEqual(view.lastMessages, []); + assert.equal(view.pluginSnapshot, undefined); + assert.equal(view.tailGaps, 0); + assert.equal(view.blockContents.size, 70); + plan.session.tailJson = JSON.stringify({ tailSeqs: null, pluginSnapshotSeqs: [], tailFolded: false }); + const other = new LegacyViewBuilder(plan.session, [], []).finish(); + assert.equal(other.lastMessages, undefined); + assert.deepEqual(other.pluginSnapshot, []); +}); From 7b1ecc2b1fb4a7b5a406625e277a775194c84ca0 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 26 Oct 2026 02:24:59 +0800 Subject: [PATCH 02/25] fix: bound ordinary session loads and restore lazy fields with narrow reads Ordinary full-mode loads materialized every message row (row + content-CAS read + media hydration) although the view resolves only the tail/pluginSnapshot pointer seqs; dropped metadata and blockContents each rebuilt the whole session through loadSync after every idle save. On a 5,043 message / 236 block session one isolated read measured 15.1 -> 230.1 MiB heap growth (#4739). - UnifiedStore.sessionViewRows: the viewFor bundle from ONE read transaction; messages fetched only at pointer seqs (getMessagesAt, chunked IN, same hydration/retired/order semantics as getMessages). - StorageBackend.sessionViewRows (mysql lane): same pointer-scoped fetch. - UnifiedStore.getSessionMetadata / getBlockContents: single-column / blocks+orphans-only reads mirroring loadLegacyView's replay leg. - SessionStore.lazySource + narrowMetadata/narrowBlockContents: one build-scoped source decision per saved record (index vs envelope), bound-source-first with the other copy as fallback; never imports, never schedules saves, refuse-empty throw semantics preserved. Served bytes unchanged: lastMessages/pluginSnapshot resolution, tailGaps counting, orphan journal merge, media hydration, drift-latch and index/envelope freshness rules all preserved. No schema/config/kernel changes. --- src/persist.ts | 67 ++++- src/storage/backend.ts | 8 + src/storage/mysql.ts | 47 +++- src/storage/shadow.ts | 54 ++-- src/storage/store.ts | 159 ++++++++++- tests/issue4739-bounded-session-load.test.ts | 264 +++++++++++++++++++ 6 files changed, 564 insertions(+), 35 deletions(-) create mode 100644 tests/issue4739-bounded-session-load.test.ts diff --git a/src/persist.ts b/src/persist.ts index bdf9abc0d1..c439923d7c 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2560,6 +2560,63 @@ export class SessionStore { this.dropResidentMetadata(session); } + /** #4739: where this id's record lives RIGHT NOW — the one build-scoped + * source decision shared by both lazy-field restores after a save. + * Mirrors loadSync's routing exactly: no readable unified index (never + * opened, 3-strike disabled, shadow mode serving files) or a drift latch + * (the envelope outranks the index while it exists, #3135 ④) → file; + * else index. Bound at save time, deliberately NOT re-evaluated at + * restore time — a later racing writer converges on the next save, and + * re-routing mid-burst could flip the source between the two fields of + * one saved record. */ + private lazySource(id: string): "index" | "file" { + const sh = this.shadow; + if (!sh || !sh.readsUnified || sh.stats.disabled) return "file"; + if (sh.driftedSessions.has(id)) return "file"; + return "index"; + } + + /** #4739: narrow metadata restore for the dropped-metadata proxy — + * single-column index read or single-file-field envelope read instead + * of the full-session loadSync rebuild. Tries the bound source first, + * then the other copy; null only when BOTH miss/fail (keeps the proxy's + * refuse-empty throw semantics). Never imports, never schedules a save, + * never touches the drift latch. */ + private narrowMetadata(id: string, source: "index" | "file", meta?: { protocol?: string; upstreamOrigin?: string }): Record | null { + const fromIndex = (): Record | null => this.shadow?.getSessionMetadata(id) ?? null; + const fromFile = (): Record | null => { + const env = this.loadEnvelope(id, meta); + return env ? (env.payload.metadata ?? {}) : null; + }; + if (source === "index") { + const idx = fromIndex(); + return idx !== null ? idx : fromFile(); + } + const file = fromFile(); + return file !== null ? file : fromIndex(); + } + + /** #4739: narrow blockContents restore for the dropped-blockContents + * proxy — block rows + orphans journal (index) or the persisted + * record's map (file), never message rows. Same bound-source-first / + * other-copy-fallback / refuse-empty contract as narrowMetadata. */ + private narrowBlockContents(id: string, source: "index" | "file", meta?: { protocol?: string; upstreamOrigin?: string }): Record | null { + const fromIndex = (): Record | null => { + const rows = this.shadow?.getBlockContents(id); + return rows ? Object.fromEntries(rows) : null; + }; + const fromFile = (): Record | null => { + const env = this.loadEnvelope(id, meta); + return env ? (env.payload.blockContents ?? {}) : null; + }; + if (source === "index") { + const idx = fromIndex(); + return idx !== null ? idx : fromFile(); + } + const file = fromFile(); + return file !== null ? file : fromIndex(); + } + /** #3050 R3 ②a: once a save has captured the raw tail (envelope + the * unified messages table via the ⑤ rawTail override), an idle session * keeps only the bounded folded view resident — the exact shape a @@ -2587,9 +2644,10 @@ export class SessionStore { // Hand-built sessions (tests, seeds) may carry no meta — the lazy // loader degrades to the id-only lookup, same as a protocol-less file. const meta = session.meta ?? undefined; + // #4739: one narrow read per field instead of a full-session rebuild. + const source = this.lazySource(session.id); session.metadata = lazyMetadata(session.id, { protocol: meta?.protocol, upstreamOrigin: meta?.upstreamOrigin }, (id, m) => { - const loaded = this.loadSync(id, m); - return loaded ? loaded.metadata : null; + return this.narrowMetadata(id, source, m); }); } @@ -2606,9 +2664,10 @@ export class SessionStore { return; } if (existing.size === 0) return; + // #4739: one narrow read per field instead of a full-session rebuild. + const source = this.lazySource(session.id); const lazy = new LazyBlockContents(session.id, { protocol: session.meta.protocol, upstreamOrigin: session.meta.upstreamOrigin }, (id, meta) => { - const loaded = this.loadSync(id, meta); - return loaded ? Object.fromEntries(loaded.blockContents) : null; + return this.narrowBlockContents(id, source, meta); }); lazy.drop(); session.blockContents = lazy; diff --git a/src/storage/backend.ts b/src/storage/backend.ts index dd8473893a..99d07a96de 100644 --- a/src/storage/backend.ts +++ b/src/storage/backend.ts @@ -33,6 +33,7 @@ export type { SessionSummaryRow, WebSummaryRow, IngestSessionInput, + SessionViewRows, } from "./store.js"; import type { SessionRow, @@ -44,6 +45,7 @@ import type { SessionSummaryRow, WebSummaryRow, IngestSessionInput, + SessionViewRows, } from "./store.js"; /** Everything a storage backend must provide. Mirrors the operation surface @@ -74,6 +76,9 @@ export interface StorageBackend { /** #4316 PR3/3: fork adoption row clone (idempotent). */ cloneSessionRows(parentId: string, childId: string, throughSeq: number): Promise; getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]>; + /** #4739: the viewFor bundle from ONE consistent read snapshot (messages + * fetched only at the tail/pluginSnapshot pointer seqs). */ + sessionViewRows(sessionId: string): Promise; getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]>; getRefs(sessionId: string): Promise; @@ -189,6 +194,9 @@ export class SqliteBackend implements StorageBackend { getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { return Promise.resolve(this.store.getMessages(sessionId)); } + sessionViewRows(sessionId: string): Promise { + return Promise.resolve(this.store.sessionViewRows(sessionId)); + } getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { return Promise.resolve(this.store.getBlocks(sessionId)); } diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index f3c46ea373..fdd28f636b 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -31,7 +31,7 @@ import type { Pool, PoolConnection, PoolOptions } from "mysql2/promise"; import { mkdirSync, writeFileSync, appendFileSync, readFileSync, existsSync } from "node:fs"; import { join } from "node:path"; import { log as loggerLog } from "../logger.js"; -import { sha256Hex } from "./store.js"; +import { sha256Hex, viewPointerSeqs } from "./store.js"; import { identityOfStoredContent } from "./identity.js"; import type { MessageContentStore } from "acp-kernel"; import type { StorageBackend } from "./backend.js"; @@ -45,6 +45,7 @@ import type { SessionSummaryRow, WebSummaryRow, IngestSessionInput, + SessionViewRows, } from "./store.js"; /** The mysql2/promise driver module, injected by openBackend() at open time. @@ -1007,6 +1008,50 @@ export class MysqlBackend implements StorageBackend { }; } + /** #4739: bounded view bundle (messages only at tail/pluginSnapshot + * pointer seqs). Per-statement reads like today's viewForAsync — no + * cross-query snapshot — which is safe here because ingest writes each + * session row and its pointed-at message rows in ONE transaction, so + * every pointer ever read resolves to rows that already exist. The + * sqlite lane carries the explicit read-transaction instead. */ + async sessionViewRows(sessionId: string): Promise { + const session = await this.getSessionRow(sessionId); + if (!session) return null; + const pointers = viewPointerSeqs(session.tailJson); + const messages: (MessageRow & { contentHash: string })[] = []; + const CHUNK = 500; + for (let i = 0; i < pointers.length; i += CHUNK) { + const chunk = [...new Set(pointers.slice(i, i + CHUNK))]; + if (chunk.length === 0) continue; + const [rows] = (await this.pool.query( + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages + WHERE session_id = ? AND retired_at IS NULL AND seq IN (${chunk.map(() => "?").join(", ")})`, + [sessionId, ...chunk], + )) as unknown as Row[][]; + for (const r of rows) { + const contentHash = str(r.content_hash); + messages.push({ + seq: num(r.seq), + msgId: str(r.msg_id), + ref: nullableStr(r.ref), + refNum: nullableNum(r.ref_num), + role: str(r.role), + kind: str(r.kind), + contentHash, + content: await this.hydrateMediaAsync((await this.getContent(this.pool, contentHash)) ?? ""), + }); + } + } + messages.sort((a, b) => a.seq - b.seq); + return { + session, + messages, + blocks: await this.getBlocks(sessionId), + deadRefs: await this.getDeadRefs(sessionId), + refs: await this.getRefs(sessionId), + }; + } + async getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { // #4390: retired rows stay out of the replay by default. const [rows] = (await this.pool.query( diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index d527be96c4..b97eb93e27 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -509,15 +509,11 @@ export class UnifiedShadow { if (this.sync === null) return null; if (this.stats.disabled) return null; try { - const sessionRow = this.sync.getSessionRow(id); - if (!sessionRow) return null; - const rows = { - session: sessionRow, - messages: this.sync.getMessages(id), - blocks: this.sync.getBlocks(id), - deadRefs: this.sync.getDeadRefs(id), - refs: this.sync.getRefs(id), - }; + // #4739: one consistent read snapshot; messages fetched ONLY at + // the tail/pluginSnapshot pointer seqs (the historical log is not + // decoded on ordinary loads). + const rows = this.sync.sessionViewRows(id); + if (!rows) return null; // The view's `state` object carries every MERGE_STATE_FIELDS key by // construction; the declared Record just needs a // widening cast to reach LegacySessionLike. @@ -778,6 +774,33 @@ export class UnifiedShadow { return this.sync.getCcr(sessionId); } + /** #4739: narrow single-column metadata read for lazy-field restore. + * Full-trust only (shadow mode's index may lag its envelopes) — callers + * gate on readsUnified; network backends degrade to null (the caller + * falls back to the envelope, exactly like today's sync loadSync). */ + getSessionMetadata(sessionId: string): Record | null { + if (this.sync === null) return null; + if (this.stats.disabled) return null; + try { + return this.sync.getSessionMetadata(sessionId); + } catch { + return null; + } + } + + /** #4739: narrow blockContents read for lazy-field restore (block rows + + * orphans journal, no message rows). Same trust/degrade contract as + * getSessionMetadata. */ + getBlockContents(sessionId: string): import("./store.ts").BlockContentsMap | null { + if (this.sync === null) return null; + if (this.stats.disabled) return null; + try { + return this.sync.getBlockContents(sessionId); + } catch { + return null; + } + } + /** #4316 PR3/3: raw id → canonical seq (single-row ids only) for the * one-time ledger re-stamp. Shadow mode's index may lag files it has * not re-recorded, so like getCcr this is full-trust only: sync store @@ -971,15 +994,10 @@ export class UnifiedShadow { if (this.stats.disabled) return null; try { const backend = this.backend!; - const sessionRow = await backend.getSessionRow(id); - if (!sessionRow) return null; - const loaded = loadLegacyView({ - session: sessionRow, - messages: await backend.getMessages(id), - blocks: await backend.getBlocks(id), - deadRefs: await backend.getDeadRefs(id), - refs: await backend.getRefs(id), - }) as LegacySessionLike; + // #4739: bounded view bundle (pointer-scoped message fetch). + const rows = await backend.sessionViewRows(id); + if (!rows) return null; + const loaded = loadLegacyView(rows) as LegacySessionLike; warnTailGaps(id, loaded as unknown as { tailGaps?: number }); return loaded; } catch (err) { diff --git a/src/storage/store.ts b/src/storage/store.ts index a13acf4edb..ed58c57a34 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -26,6 +26,7 @@ import { identityOfStoredContent } from "./identity.js"; import { recordIntegrityOutcome, recordSecondProcessOwner, recordBackupOutcome } from "./runtime-status.js"; import { listInstances, type RegistryEntry } from "../instance.js"; import type { MessageContentStore } from "acp-kernel"; +import type { BlockView } from "../session.js"; /** #3250: run a PASSIVE wal_checkpoint every N successful ingests. The * built-in autocheckpoint alone cannot guarantee reclamation in a long-lived @@ -36,6 +37,24 @@ import type { MessageContentStore } from "acp-kernel"; * forgotten — the next tick retries and journal_size_limit does the rest. */ const WAL_CHECKPOINT_EVERY_INGESTS = 20; +/** #4739 test instrumentation: message rows materialized by getMessages / + * getMessagesAt (each row = one CAS read + media hydration). Production + * code never reads this; tests assert ordinary loads stop decoding the + * whole historical log. */ +export const messageRowReads = { count: 0 }; + +/** #4739: the message seqs a view actually resolves — the union of the + * tail/pluginSnapshot pointers in a tail_json document. loadLegacyView + * consumes ONLY these rows; the rest of the log is decode waste on an + * ordinary load. Non-number entries are dropped (they can only ever miss + * in pointer resolution, i.e. count as gaps); malformed JSON throws — + * the view path's existing corrupt-fallback contract. */ +export function viewPointerSeqs(tailJson: string): number[] { + const tail = JSON.parse(tailJson) as { tailSeqs?: unknown; pluginSnapshotSeqs?: unknown }; + const pick = (v: unknown): number[] => (Array.isArray(v) ? v.filter((s): s is number => typeof s === "number") : []); + return [...pick(tail.tailSeqs), ...pick(tail.pluginSnapshotSeqs)]; +} + export function sha256Hex(bytes: string | Buffer): string { return createHash("sha256").update(bytes).digest("hex"); } @@ -67,7 +86,8 @@ export interface SessionRow { * state object in mergeState field order. */ kstateJson: string; /** View pointers: {tailSeqs: number[]|null, tailFolded: boolean, - * pluginSnapshotLen: number|null}. Content itself lives in messages. */ + * pluginSnapshotSeqs?: number[]|null} (loadLegacyView's contract). + * Content itself lives in messages. */ tailJson: string; /** Verbatim CCR store JSON (dying subsystem, Phase 3 removes it). */ ccrJson: string | null; @@ -2319,17 +2339,125 @@ export class UnifiedStore { : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", ) .all(sessionId) as Record[]; - return rows.map((r) => ({ - seq: r.seq as number, - msgId: r.msg_id as string, - ref: (r.ref as string | null) ?? null, - refNum: (r.ref_num as number | null) ?? null, - role: r.role as string, - kind: r.kind as string, - contentHash: r.content_hash as string, - retiredAt: (r.retired_at as number | null) ?? null, - content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), - })); + return rows.map((r) => { + messageRowReads.count++; + return { + seq: r.seq as number, + msgId: r.msg_id as string, + ref: (r.ref as string | null) ?? null, + refNum: (r.ref_num as number | null) ?? null, + role: r.role as string, + kind: r.kind as string, + contentHash: r.content_hash as string, + retiredAt: (r.retired_at as number | null) ?? null, + content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), + }; + }); + } + + /** #4739: live message rows for EXACTLY the given seqs — the bounded + * fetch for ordinary loads (the view resolves only tail/pluginSnapshot + * pointers). Same row shape and media hydration as getMessages; retired + * rows stay excluded and missing seqs simply don't come back (the view + * counts them as gaps). Chunked IN keeps the bound independent of + * pointer count. */ + getMessagesAt(sessionId: string, seqs: readonly number[]): (MessageRow & { contentHash: string })[] { + if (seqs.length === 0) return []; + const unique = [...new Set(seqs)]; + const CHUNK = 500; + const out: (MessageRow & { contentHash: string })[] = []; + for (let i = 0; i < unique.length; i += CHUNK) { + const chunk = unique.slice(i, i + CHUNK); + const rows = this.db + .prepare( + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages + WHERE session_id = ? AND retired_at IS NULL AND seq IN (${chunk.map(() => "?").join(", ")})`, + ) + .all(sessionId, ...chunk) as Record[]; + for (const r of rows) { + messageRowReads.count++; + out.push({ + seq: r.seq as number, + msgId: r.msg_id as string, + ref: (r.ref as string | null) ?? null, + refNum: (r.ref_num as number | null) ?? null, + role: r.role as string, + kind: r.kind as string, + contentHash: r.content_hash as string, + content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), + }); + } + } + out.sort((a, b) => a.seq - b.seq); + return out; + } + + /** #4739: exactly what viewFor needs, from ONE consistent read snapshot. + * The deferred transaction pins the WAL snapshot across all five reads, + * so a concurrent ingest can no longer land between the session row and + * its pointer rows. Messages are fetched ONLY at the tail/pluginSnapshot + * pointer seqs — the historical log is not decoded on ordinary loads. + * Null on session miss; malformed tail_json throws into the caller's + * corrupt-fallback (same as before). */ + sessionViewRows(sessionId: string): { + session: SessionRow; + messages: (MessageRow & { contentHash: string })[]; + blocks: ReturnType; + deadRefs: ReturnType; + refs: RefRow[]; + } | null { + return this.db.transaction(() => { + const session = this.getSessionRow(sessionId); + if (!session) return null; + return { + session, + messages: this.getMessagesAt(sessionId, viewPointerSeqs(session.tailJson)), + blocks: this.getBlocks(sessionId), + deadRefs: this.getDeadRefs(sessionId), + refs: this.getRefs(sessionId), + }; + }); + } + + /** #4739: single-column metadata read for lazy-field restore (no block or + * message rows involved). Null when the session has no index row; + * corrupt JSON throws like every other column read — callers fall back + * to the envelope. */ + getSessionMetadata(sessionId: string): Record | null { + const row = this.db + .prepare("SELECT metadata_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { metadata_json: string } | undefined; + if (!row) return null; + return JSON.parse(row.metadata_json) as Record; + } + + /** #4739: narrow blockContents read for lazy-field restore — the exact + * replay leg of loadLegacyView (block rows' summary/one CAS bytes plus + * the orphans_json journal merge, WITHOUT resurrecting state.blocks). + * Null when the session has no index row. */ + getBlockContents(sessionId: string): BlockContentsMap | null { + const row = this.db + .prepare("SELECT orphans_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { orphans_json: string | null } | undefined; + if (!row) return null; + const out: BlockContentsMap = new Map(); + for (const b of this.getBlocks(sessionId)) { + out.set(b.blockId, { + one: b.one === null ? null : (JSON.parse(b.one) as BlockView), + full: JSON.parse(b.summary) as BlockView, + }); + } + if (row.orphans_json !== null && row.orphans_json !== undefined && row.orphans_json !== "") { + try { + const orphans = JSON.parse(row.orphans_json) as Record; + for (const [blockId, view] of Object.entries(orphans)) { + if (!out.has(blockId)) out.set(blockId, view); + } + } catch { + // corrupt journal → skip (loadLegacyView's same boundary) + } + } + return out; } /** #3762: swap {"$media": hash} references back to the CAS bytes. Only @@ -2455,3 +2583,10 @@ export class UnifiedStore { })); } } + +/** #4739: the viewFor bundle shape (see UnifiedStore.sessionViewRows). */ +export type SessionViewRows = ReturnType; + +/** #4739: blockContents map shape shared by UnifiedStore.getBlockContents + * and the shadow facade (Map is invariant — one alias keeps both sides exact). */ +export type BlockContentsMap = Map; diff --git a/tests/issue4739-bounded-session-load.test.ts b/tests/issue4739-bounded-session-load.test.ts new file mode 100644 index 0000000000..bc9906b5f0 --- /dev/null +++ b/tests/issue4739-bounded-session-load.test.ts @@ -0,0 +1,264 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionStore } from "../src/persist.ts"; +import type { Session } from "../src/session.ts"; +import { isDroppedMetadata, snapshotMessages } from "../src/session.ts"; +import { createInitialState, type CoreMessage } from "acp-kernel"; +import { planIngest, planIngestDelta, loadLegacyView, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore, messageRowReads, viewPointerSeqs } from "../src/storage/store.ts"; +import type { UnifiedShadow } from "../src/storage/shadow.ts"; +import { rmrf } from "./tmp-rm.ts"; + +// #4739: ordinary loads must not materialize the whole historical log — the +// view only consumes tail/pluginSnapshot pointer rows — and the dropped +// lazy fields must restore through narrow reads instead of two full-session +// rebuilds per saved record. + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_CONFIG_FILE = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "bili-4739-hermetic-")), + "no-config.json", +); +process.env.BILI_STORAGE_UNIFIED = "legacy"; +// Keep the persisted tail SMALL relative to the seeded history so the +// pointer set is a strict subset of the message table. +process.env.BILI_PERSIST_TAIL_TOKENS = "600"; + +function makeSession(id: string): Session { + return { + id, + meta: { label: id, protocol: "openai", upstreamOrigin: "http://up:1" }, + stats: { requests: 3, tokensSaved: 0, inputTokens: 10, cachedTokens: 0, outputTokens: 5, cacheSamples: 0, lastInputTokens: 10, contextTokens: 1000, compressCreditTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, + state: createInitialState(), + createdAt: Date.now() - 1000, + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }; +} + +function rawTail(n: number): CoreMessage[] { + const out: CoreMessage[] = []; + for (let i = 0; i < n; i++) { + out.push({ id: `big${i}`, contentType: "text", role: i % 2 === 0 ? "user" : "assistant", text: `bounded load probe message number ${i} with filler words to spend tokens` }); + } + return out; +} + +/** Finds the persisted envelope for `id` under `root` (payload.id match). */ +function findEnvelope(root: string, id: string): string | null { + const hits: string[] = []; + const walk = (d: string): void => { + for (const f of fs.readdirSync(d, { withFileTypes: true })) { + const fp = path.join(d, f.name); + if (f.isDirectory()) walk(fp); + else if (f.name.endsWith(".json")) hits.push(fp); + } + }; + walk(root); + for (const hit of hits) { + try { + const envelope = JSON.parse(fs.readFileSync(hit, "utf8")) as Record; + const payload = (envelope["payload"] ?? envelope) as Record; + if (payload["id"] === id) return hit; + } catch { + // non-JSON or partial write — skip + } + } + return null; +} + +test("full mode: viewFor decodes only pointer rows and matches the full-log decode", async () => { + const prev = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4739-view-")); + const store = new SessionStore({ dir, debounceMs: 60_000, enabled: true }); + try { + const s = makeSession("big-1"); + s.metadata.foldAnchors = { m00001: { c: "text", frags: ["a", "b"] } }; + const N = 400; + snapshotMessages(s, rawTail(N)); + await store.writeNow(s); + + const sh = (store as unknown as { shadow?: UnifiedShadow }).shadow; + assert.ok(sh && sh.readsUnified, "full-mode sidecar live"); + + const direct = UnifiedStore.open(path.join(dir, "index.db")); + // Generation 2 — the production shape after an idle fold (#3050 R3): + // the table keeps every row, tail_json points at the bounded raw + // suffix only (an equal-length raw tail is a valid empty delta). + const K = 40; + const allRows = direct.getMessages("big-1"); + assert.equal(allRows.length, N, "generation 1 ingested the full history"); + const maxSeq = allRows[allRows.length - 1]!.seq; + const suffixSeqs = allRows.slice(-K).map((m) => m.seq); + const boundaryRow = allRows[allRows.length - 1]!; + const deltaLegacy: LegacySessionLike = { + id: "big-1", + createdAt: s.createdAt, + lastSeen: s.lastSeen, + meta: { protocol: "openai" }, + stats: { requests: 4 }, + metadata: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, nextBlockId: 1, nextRunId: 1 }, + blockContents: new Map(), + lastMessages: rawTail(N).slice(-K), + }; + const plan = planIngestDelta(deltaLegacy, maxSeq, suffixSeqs, { msgId: boundaryRow.msgId, contentHash: boundaryRow.contentHash }, true); + assert.ok(plan, "empty delta accepted for the shrunken raw tail"); + direct.ingestSession(plan, { trustedSeq: maxSeq, boundaryMsgId: boundaryRow.msgId, boundaryHash: boundaryRow.contentHash, trustedTailSeqs: suffixSeqs }); + + // Reference decode: the OLD algorithm — every message row through + // loadLegacyView on the second handle of the same database. + const sessionRow = direct.getSessionRow("big-1"); + assert.ok(sessionRow, "session row present"); + const liveSeqs = new Set(direct.getMessages("big-1").map((m) => m.seq)); + assert.equal(liveSeqs.size, N, "table still holds the full history"); + const expectedPointers = [...new Set(viewPointerSeqs(sessionRow.tailJson))].filter((seq) => liveSeqs.has(seq)).length; + assert.equal(expectedPointers, K, `tail now points at ${K} of ${N} rows`); + + const reference = loadLegacyView({ + session: sessionRow, + messages: direct.getMessages("big-1"), + blocks: direct.getBlocks("big-1"), + deadRefs: direct.getDeadRefs("big-1"), + refs: direct.getRefs("big-1"), + }); + const before = messageRowReads.count; + const via = sh.viewFor("big-1"); + const delta = messageRowReads.count - before; + direct.close(); + + assert.ok(via, "viewFor resolved"); + assert.deepEqual(via, reference, "bounded decode byte-equal to the full-log decode"); + assert.equal(delta, expectedPointers, `viewFor read ${delta} message rows for ${expectedPointers} pointer seqs (full log = ${liveSeqs.size})`); + } finally { + store.cancelAll(); + if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = prev; + rmrf(dir); + } +}); + +test("full mode: dropped metadata restores from the index alone (narrow read, no envelope)", async () => { + const prev = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4739-md-")); + const store = new SessionStore({ dir, debounceMs: 60_000, enabled: true }); + try { + const s = makeSession("md-idx"); + s.metadata.foldAnchors = { m00001: { c: "tool-call" } }; + s.metadata.cacheLedger = { restartDrops: { count: 2 } }; + snapshotMessages(s, rawTail(50)); + await store.writeNow(s); + assert.equal(isDroppedMetadata(s.metadata), true, "metadata folded after idle save"); + + // The persisted shape (what a restart restores) — captured while the + // envelope still exists. + const envelopePath = findEnvelope(dir, "md-idx"); + assert.ok(envelopePath, "envelope written in full mode"); + const envelopePayload = JSON.parse(fs.readFileSync(envelopePath, "utf8")).payload as Record; + const persistedKeys = Object.keys(envelopePayload.metadata as object).sort(); + + // Delete the file copy: restore must come from the index alone. + fs.unlinkSync(envelopePath); + assert.deepEqual(s.metadata.cacheLedger, { restartDrops: { count: 2 } }, "narrow index restore serves the value"); + assert.deepEqual(Object.keys(s.metadata).sort(), persistedKeys, "materialized shape is the persisted (restart) shape"); + assert.equal(isDroppedMetadata(s.metadata), false, "materialized on read"); + + // A later save persists the same keys — no thinned erase. + await store.writeNow(s); + const again = findEnvelope(dir, "md-idx"); + assert.ok(again, "envelope rewritten by later save"); + assert.deepEqual(Object.keys(JSON.parse(fs.readFileSync(again, "utf8")).payload.metadata as object).sort(), persistedKeys); + } finally { + store.cancelAll(); + if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = prev; + rmrf(dir); + } +}); + +test("store: getBlockContents/getSessionMetadata equal the loadLegacyView replay leg", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4739-narrow-")); + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + // Real folded corpus payload (one active block + contents + refs). + const corpusFile = path.resolve(import.meta.dirname, "golden/storage-corpus/folded/anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json"); + const envelope = JSON.parse(fs.readFileSync(corpusFile, "utf8")) as Record; + const payload = (envelope["payload"] ?? envelope) as Record; + const legacy = payload as unknown as LegacySessionLike; + // The persisted record carries blockContents as a plain object; the + // ingest planner takes the in-memory Map shape. + legacy.blockContents = new Map(Object.entries((payload["blockContents"] ?? {}) as Record)); + // The pre-#lastSeen corpus files carry savedAt only; the planner stamps + // sessions.updated_at from lastSeen. + if (typeof legacy.lastSeen !== "number") legacy.lastSeen = Number(payload["savedAt"] ?? Date.now()); + store.ingestSession(planIngest(legacy)); + + const sessionRow = store.getSessionRow("corpus-folded"); + assert.ok(sessionRow, "corpus ingested"); + const rows = { + session: sessionRow, + messages: store.getMessages("corpus-folded"), + blocks: store.getBlocks("corpus-folded"), + deadRefs: store.getDeadRefs("corpus-folded"), + refs: store.getRefs("corpus-folded"), + }; + const reference = loadLegacyView(rows); + assert.ok(reference.blockContents.size > 0, "corpus carries block contents"); + + assert.deepEqual(store.getBlockContents("corpus-folded"), reference.blockContents, "narrow blockContents == replay leg"); + assert.deepEqual(store.getSessionMetadata("corpus-folded"), reference.metadata, "narrow metadata == replay leg"); + assert.equal(store.getBlockContents("no-such-session"), null, "row miss degrades to null"); + assert.equal(store.getSessionMetadata("no-such-session"), null, "row miss degrades to null"); + } finally { + store.close(); + rmrf(dir); + } +}); + +test("store: getMessagesAt returns exactly the requested live seqs, in order", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4739-at-")); + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + const N = 600; + const messages: CoreMessage[] = []; + for (let i = 0; i < N; i++) { + messages.push({ id: `at${i}`, contentType: "text", role: i % 2 === 0 ? "user" : "assistant", text: `getMessagesAt probe row ${i}` }); + } + const legacy: LegacySessionLike = { + id: "at-1", + createdAt: Date.now(), + lastSeen: Date.now(), + meta: { protocol: "openai" }, + stats: { requests: 1 }, + metadata: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, nextBlockId: 1, nextRunId: 1 }, + blockContents: new Map(), + lastMessages: messages, + }; + store.ingestSession(planIngest(legacy)); + const all = store.getMessages("at-1"); + assert.equal(all.length, N, "all rows ingested"); + + // Sparse request spanning both IN-chunks (>500) plus a missing seq. + const wanted: number[] = [1, 2, 3]; + for (let seq = 10; seq <= N; seq += 7) wanted.push(seq); + wanted.push(99_999); + const got = store.getMessagesAt("at-1", wanted); + const expected = wanted.filter((seq) => seq <= N); + assert.deepEqual(got.map((m) => m.seq), expected, "exactly the requested live seqs, ordered"); + assert.ok(got.every((m) => m.content.length > 0), "content hydrated"); + assert.deepEqual(store.getMessagesAt("at-1", []), [], "empty request → empty result"); + } finally { + store.close(); + rmrf(dir); + } +}); From b217d7905062cf0b387c13b85a84fe67f4de18ad Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:00:51 -0700 Subject: [PATCH 03/25] perf: verify stored sessions in bounded batches (#4538) --- src/storage/backend.ts | 12 +- src/storage/mysql.ts | 17 +- src/storage/shadow.ts | 185 ++++++++++++++---- src/storage/store.ts | 17 +- tests/storage-batched-verification.test.ts | 161 +++++++++++++++ .../web-storage-diagnostics-redaction.test.ts | 28 ++- 6 files changed, 346 insertions(+), 74 deletions(-) create mode 100644 tests/storage-batched-verification.test.ts diff --git a/src/storage/backend.ts b/src/storage/backend.ts index dd8473893a..299b912f4f 100644 --- a/src/storage/backend.ts +++ b/src/storage/backend.ts @@ -73,8 +73,8 @@ export interface StorageBackend { seqByRaw(sessionId: string): Promise | null>; /** #4316 PR3/3: fork adoption row clone (idempotent). */ cloneSessionRows(parentId: string, childId: string, throughSeq: number): Promise; - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]>; - getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]>; + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]>; getRefs(sessionId: string): Promise; getCcr(sessionId: string): Promise; @@ -186,11 +186,11 @@ export class SqliteBackend implements StorageBackend { this.store.cloneSessionRows(parentId, childId, throughSeq); return Promise.resolve(true); } - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { - return Promise.resolve(this.store.getMessages(sessionId)); + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]> { + return Promise.resolve(this.store.getMessages(sessionId, opts)); } - getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { - return Promise.resolve(this.store.getBlocks(sessionId)); + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + return Promise.resolve(this.store.getBlocks(sessionId, opts)); } getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> { return Promise.resolve(this.store.getDeadRefs(sessionId)); diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index f3c46ea373..ea9e392bbd 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -1007,13 +1007,14 @@ export class MysqlBackend implements StorageBackend { }; } - async getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { + async getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { // #4390: retired rows stay out of the replay by default. + if (opts?.seqs?.length === 0) return []; + const retiredFilter = opts?.includeRetired === true ? "" : " AND retired_at IS NULL"; + const seqFilter = opts?.seqs ? ` AND seq IN (${opts.seqs.map(() => "?").join(", ")})` : ""; const [rows] = (await this.pool.query( - opts?.includeRetired === true - ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" - : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", - [sessionId], + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ?${retiredFilter}${seqFilter} ORDER BY seq`, + [sessionId, ...(opts?.seqs ?? [])], )) as unknown as Row[][]; const out: (MessageRow & { contentHash: string; retiredAt: number | null })[] = []; for (const r of rows) { @@ -1070,12 +1071,12 @@ export class MysqlBackend implements StorageBackend { return out; } - async getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + async getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { const [rows] = (await this.pool.query( `SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json - FROM blocks WHERE session_id = ? ORDER BY ord`, - [sessionId], + FROM blocks WHERE session_id = ? ORDER BY ord, block_id${opts ? " LIMIT ? OFFSET ?" : ""}`, + [sessionId, ...(opts ? [opts.limit, opts.offset ?? 0] : [])], )) as unknown as Row[][]; const out: (BlockRow & { summaryHash: string; oneHash: string | null })[] = []; for (const r of rows) { diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index d527be96c4..7c0ba0e0f0 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -32,10 +32,11 @@ import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBac import { canonicalize, sha256 } from "./canonical.ts"; import { recordSidecarInstance, recordSidecarOpenFailure } from "./runtime-status.ts"; import type { SqliteEngineName, SqliteEngineSource } from "./driver.ts"; -import type { MessageRow } from "./store.js"; +import type { MessageRow, BlockRow, SessionRow, RefRow } from "./store.js"; import type { CoreMessage, MessageContentStore } from "acp-kernel"; export const MAX_CONSECUTIVE_FAILURES = 3; +const VERIFY_BATCH_SIZE = 32; /** #3133: read-back verification cadence — the verify leg (full read-back + * double shadowDigest) is O(log) per save, while the drift it guards is a @@ -367,20 +368,18 @@ export class UnifiedShadow { this.stats.verifySamples++; const sessionRow = this.sync.getSessionRow(legacy.id); if (!sessionRow) throw new Error(`session row missing after ingest: ${legacy.id}`); - const rows = { - session: sessionRow, - messages: this.sync.getMessages(legacy.id), - blocks: this.sync.getBlocks(legacy.id), - deadRefs: this.sync.getDeadRefs(legacy.id), - refs: this.sync.getRefs(legacy.id), - }; - const replayed = loadLegacyView(rows); - warnTailGaps(legacy.id, replayed); - const expectView = { ...legacy, blockContents: orderedBlockContents(completeBlockContents(legacy.blockContents, legacy.state.blocks)) }; - const gotView = { ...replayed, blockContents: orderedBlockContents(replayed.blockContents) }; - const expect = shadowDigest(expectView); - const got = shadowDigest(gotView); - if (got !== expect) { + const verifier = new BatchedViewVerification(legacy, sessionRow, this.sync.getRefs(legacy.id), this.sync.getDeadRefs(legacy.id)); + for (let offset = 0; ; offset += VERIFY_BATCH_SIZE) { + const rows = this.sync.getBlocks(legacy.id, { limit: VERIFY_BATCH_SIZE, offset }); + verifier.addBlocks(rows); + if (rows.length < VERIFY_BATCH_SIZE) break; + } + for (let i = 0; i < verifier.messageSeqs.length; i += VERIFY_BATCH_SIZE) { + verifier.addMessages(this.sync.getMessages(legacy.id, { seqs: verifier.messageSeqs.slice(i, i + VERIFY_BATCH_SIZE) })); + } + const verified = verifier.finish(); + warnTailGaps(legacy.id, verified); + if (!verified.matches) { this.stats.verifyDrift++; // #3919: membership from BEFORE this add is the repeat // signal — two consecutive sampled drifts mean the views @@ -404,8 +403,8 @@ export class UnifiedShadow { this.frozenEnvelopeIds.add(legacy.id); let sides = ""; try { - const envBlocks = expectView.state.blocks.filter((b) => b.active).length; - const idxBlocks = (gotView.state.blocks as { active?: boolean }[] | null | undefined)?.filter((b) => b.active === true).length ?? 0; + const envBlocks = legacy.state.blocks.filter((b) => b.active).length; + const idxBlocks = verified.activeBlocks; sides = ` [envelope ${envBlocks} active block(s) vs index ${idxBlocks}]`; } catch { // counts must never break the heal path @@ -431,7 +430,7 @@ export class UnifiedShadow { // The line prefix stays byte-identical for existing greps. let diagnosis = ""; try { - diagnosis = formatDriftDiagnostics(describeViewDivergence(expectView, gotView)); + diagnosis = formatDriftDiagnostics(verified.diagnostics); } catch { // diagnostics must never break the heal path } @@ -929,17 +928,18 @@ export class UnifiedShadow { this.stats.verifySamples++; const sessionRow = await backend.getSessionRow(legacy.id); if (!sessionRow) throw new Error(`session row missing after ingest: ${legacy.id}`); - const rows = { - session: sessionRow, - messages: await backend.getMessages(legacy.id), - blocks: await backend.getBlocks(legacy.id), - deadRefs: await backend.getDeadRefs(legacy.id), - refs: await backend.getRefs(legacy.id), - }; - const loaded = loadLegacyView(rows); - const expect = shadowDigest({ ...legacy, blockContents: orderedBlockContents(completeBlockContents(legacy.blockContents, legacy.state.blocks)) }); - const got = shadowDigest({ ...loaded, blockContents: orderedBlockContents(loaded.blockContents) }); - if (got !== expect) { + const verifier = new BatchedViewVerification(legacy, sessionRow, await backend.getRefs(legacy.id), await backend.getDeadRefs(legacy.id)); + for (let offset = 0; ; offset += VERIFY_BATCH_SIZE) { + const rows = await backend.getBlocks(legacy.id, { limit: VERIFY_BATCH_SIZE, offset }); + verifier.addBlocks(rows); + if (rows.length < VERIFY_BATCH_SIZE) break; + } + for (let i = 0; i < verifier.messageSeqs.length; i += VERIFY_BATCH_SIZE) { + verifier.addMessages(await backend.getMessages(legacy.id, { seqs: verifier.messageSeqs.slice(i, i + VERIFY_BATCH_SIZE) })); + } + const verified = verifier.finish(); + warnTailGaps(legacy.id, verified); + if (!verified.matches) { this.stats.verifyDrift++; this.driftedSessions.add(legacy.id); this.stats.lastError = `digest drift for ${legacy.id}: unified replay != legacy load view`; @@ -1167,13 +1167,6 @@ function shadowDigest(view: unknown): string { return sha256(JSON.stringify(canonicalize(view))); } -// #3407: the parity digest must compare blockContents CONTENTS, not insertion order — -// the expect leg keeps the live Map's order (completeBlockContents appends synthesized -// entries at the end) while the got leg lays blocks out in state.blocks array order. -function orderedBlockContents(m: Map): Map { - return new Map([...m.entries()].sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))); -} - interface DriftSlotFingerprint { id: string; role: string; @@ -1188,6 +1181,126 @@ interface DriftDiagnostics { tailSlot?: { index: number; expect: DriftSlotFingerprint; got: DriftSlotFingerprint }; } +/** Compare the replay by component, retaining only message fingerprints across batches. */ +class BatchedViewVerification { + readonly messageSeqs: number[]; + private readonly tail: { tailSeqs: number[] | null; pluginSnapshotSeqs: number[] | null }; + private readonly remainingContents: LegacySessionLike["blockContents"]; + private readonly orphans: LegacySessionLike["blockContents"]; + private readonly messages = new Map>(); + private readonly keys = new Set(); + private drift = false; + private blockCount = 0; + private activeBlocks = 0; + private tailGaps = 0; + private tailSlot: DriftDiagnostics["tailSlot"]; + + constructor(private readonly legacy: LegacySessionLike, session: SessionRow, refs: RefRow[], deadRefs: { ord: number; ref: string }[]) { + const header = loadLegacyView({ session, refs, deadRefs, messages: [], blocks: [] }); + this.orphans = header.blockContents; + this.remainingContents = new Map(completeBlockContents(legacy.blockContents, legacy.state.blocks)); + this.tail = JSON.parse(session.tailJson); + this.messageSeqs = [...new Set([...(this.tail.tailSeqs ?? []), ...(this.tail.pluginSnapshotSeqs ?? [])])]; + const withoutBulk = (view: LegacySessionLike | typeof header) => ({ + ...view, + state: { ...view.state, blocks: [] }, + blockContents: new Map(), + lastMessages: view.lastMessages === undefined ? undefined : [], + pluginSnapshot: view.pluginSnapshot === undefined ? undefined : [], + }); + const expect = withoutBulk(legacy); + const got = withoutBulk(header); + if (shadowDigest(expect) !== shadowDigest(got)) { + this.drift = true; + try { + for (const key of describeViewDivergence(expect, got).keys) this.keys.add(key); + } catch { + // Diagnostic detail must not suppress a proven drift. + } + } + } + + addBlocks(rows: BlockRow[]): void { + for (const row of rows) { + const payload = JSON.parse(row.payloadJson); + if (payload?.active === true) this.activeBlocks++; + if (this.blockCount >= this.legacy.state.blocks.length || shadowDigest(payload) !== shadowDigest(this.legacy.state.blocks[this.blockCount])) this.changed("state"); + this.blockCount++; + this.compareContents(row.blockId, { one: row.one === null ? null : JSON.parse(row.one), full: JSON.parse(row.summary) }); + // A current block wins over an identically named journal entry. + this.orphans.delete(row.blockId); + } + } + + addMessages(rows: MessageRow[]): void { + for (const row of rows) this.messages.set(row.seq, messageSignature(JSON.parse(row.content))); + } + + private changed(key: string): void { + this.drift = true; + this.keys.add(key); + } + + private compareContents(id: string, view: unknown): void { + if (!this.remainingContents.has(id) || shadowDigest(this.remainingContents.get(id)) !== shadowDigest(view)) this.changed("blockContents"); + this.remainingContents.delete(id); + } + + private compareMessages(key: "lastMessages" | "pluginSnapshot", seqs: number[] | null): void { + const expected = this.legacy[key]; + let index = 0; + for (const seq of seqs ?? []) { + const got = this.messages.get(seq); + if (!got) { + this.tailGaps++; + continue; + } + const want = expected && index < expected.length ? messageSignature(expected[index]) : undefined; + if (!want || want.digest !== got.digest) { + this.changed(key); + if (key === "lastMessages" && expected && !this.tailSlot) { + this.tailSlot = { index, expect: want?.slot ?? missingSlot(), got: got.slot }; + } + } + index++; + } + if (seqs && expected && index < expected.length) { + this.changed(key); + if (key === "lastMessages" && !this.tailSlot) this.tailSlot = { index, expect: messageSignature(expected[index]).slot, got: missingSlot() }; + } + } + + finish(): { matches: boolean; activeBlocks: number; tailGaps: number; diagnostics: DriftDiagnostics } { + if (this.blockCount !== this.legacy.state.blocks.length) this.changed("state"); + for (const [id, view] of this.orphans) this.compareContents(id, view); + if (this.remainingContents.size > 0) this.changed("blockContents"); + this.compareMessages("lastMessages", this.tail.tailSeqs); + this.compareMessages("pluginSnapshot", this.tail.pluginSnapshotSeqs); + const diagnostics: DriftDiagnostics = { keys: [...this.keys].sort() }; + if (this.tailSlot) diagnostics.tailSlot = this.tailSlot; + return { matches: !this.drift, activeBlocks: this.activeBlocks, tailGaps: this.tailGaps, diagnostics }; + } +} + +function missingSlot(): DriftSlotFingerprint { + return { id: "-", role: "-", length: 0, sha8: "-" }; +} + +function messageSignature(value: unknown): { digest: string; slot: DriftSlotFingerprint } { + const json = JSON.stringify(canonicalize(value)) ?? "null"; + const digest = sha256(json); + const message = value as { id?: unknown; role?: unknown } | null; + return { + digest, + slot: !message || typeof message !== "object" ? missingSlot() : { + id: typeof message.id === "string" ? message.id : "-", + role: typeof message.role === "string" ? message.role : "-", + length: json.length, + sha8: digest.slice(0, 8), + }, + }; +} + // #3919: which dimensions disagree when the sampled verify detects drift — // the digest is a single hash and leaves classification to guesswork. Pure; // runs only on detected drift (rare). Fingerprints carry id/role/length/ diff --git a/src/storage/store.ts b/src/storage/store.ts index a13acf4edb..86ad9f4cae 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2308,17 +2308,18 @@ export class UnifiedStore { })); } - getMessages(sessionId: string, opts?: { includeRetired?: boolean }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { // #4390: retired rows (rewound/undo history) stay out of the unified // replay by default — they are history, not view. Callers that need // the full historian record (export/diagnostics) opt in. + if (opts?.seqs?.length === 0) return []; + const retiredFilter = opts?.includeRetired === true ? "" : " AND retired_at IS NULL"; + const seqFilter = opts?.seqs ? ` AND seq IN (${opts.seqs.map(() => "?").join(", ")})` : ""; const rows = this.db .prepare( - opts?.includeRetired === true - ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" - : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ?${retiredFilter}${seqFilter} ORDER BY seq`, ) - .all(sessionId) as Record[]; + .all(sessionId, ...(opts?.seqs ?? [])) as Record[]; return rows.map((r) => ({ seq: r.seq as number, msgId: r.msg_id as string, @@ -2374,14 +2375,14 @@ export class UnifiedStore { return row ? row.refcount : 0; } - getBlocks(sessionId: string): (BlockRow & { summaryHash: string; oneHash: string | null })[] { + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): (BlockRow & { summaryHash: string; oneHash: string | null })[] { const rows = this.db .prepare( `SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json - FROM blocks WHERE session_id = ? ORDER BY ord`, + FROM blocks WHERE session_id = ? ORDER BY ord, block_id${opts ? " LIMIT ? OFFSET ?" : ""}`, ) - .all(sessionId) as Record[]; + .all(sessionId, ...(opts ? [opts.limit, opts.offset ?? 0] : [])) as Record[]; return rows.map((r) => ({ blockId: r.block_id as string, ord: r.ord as number, diff --git a/tests/storage-batched-verification.test.ts b/tests/storage-batched-verification.test.ts new file mode 100644 index 0000000000..4e2d77fbeb --- /dev/null +++ b/tests/storage-batched-verification.test.ts @@ -0,0 +1,161 @@ +import { after, test, type TestContext } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { CompressionBlock, CoreMessage } from "acp-kernel"; +import { SqliteBackend } from "../src/storage/backend.ts"; +import { openSqliteDatabase } from "../src/storage/driver.ts"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; + +const root = fs.mkdtempSync(path.join(os.tmpdir(), "bili-batched-verify-")); +process.env.BILI_CONFIG_FILE = path.join(root, "no-config.json"); +process.env.BILI_PERSIST_ZSTD = "0"; +after(() => fs.rmSync(root, { recursive: true, force: true })); + +function message(i: number): CoreMessage { + return { id: `raw-${i}`, role: "user", contentType: "text", text: `message ${i}` }; +} + +function session(): LegacySessionLike { + const blocks: CompressionBlock[] = Array.from({ length: 70 }, (_, i) => ({ + blockId: `b${i + 1}`, runId: `run-${i}`, tier: 1, summary: `summary ${i}`, + directMessageIds: [], effectiveMessageIds: [], directBlockIds: [], + compressedTokens: 10, createdAt: 1_700_000_000_000, survivedCount: 1, + generation: "young", active: i % 3 !== 0, + })); + const messages = Array.from({ length: 80 }, (_, i) => message(i)); + messages[69] = { + ...messages[69], + rawOpenaiContent: { type: "image_url", image_url: { url: `data:image/png;base64,${"QUJD".repeat(700)}` } }, + } as CoreMessage; + const blockContents: LegacySessionLike["blockContents"] = new Map( + [...blocks].reverse().map((b) => [b.blockId, { one: null, full: { text: `original ${b.blockId}`, count: 2 } }]), + ); + blockContents.set("b-orphan", { one: null, full: { text: "retained orphan", count: 1 } }); + return { + id: "batched-session", createdAt: 1_700_000_000_000, lastSeen: 1_700_000_000_500, + meta: { protocol: "responses" }, stats: { requests: 1 }, metadata: {}, + state: { + blocks, + messageRefs: { byRaw: { ghost: "m00001", protected: "BLOCKED" }, byRef: { m00001: "ghost" } }, + nextBlockId: 71, nextRunId: 71, + }, + blockContents, lastMessages: messages, lastMessagesFolded: false, + pluginSnapshot: [messages[75], messages[1], messages[75], messages[40]], + }; +} + +function fixture(t: TestContext, asyncLane = false) { + const dir = fs.mkdtempSync(path.join(root, "case-")); + const dbPath = path.join(dir, "index.db"); + const store = UnifiedStore.open(dbPath); + const load = (record: unknown) => record as LegacySessionLike; + // Exercise the network facade's queue against the real SQLite contract adapter. + const shadow = Reflect.construct(UnifiedShadow, ["shadow", asyncLane ? null : store, load, asyncLane ? new SqliteBackend(store) : undefined]) as UnifiedShadow; + let closed = false; + const close = async () => { + if (!closed) { + closed = true; + await shadow.closeAsync(); + } + }; + t.after(async () => { + await close(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + return { store, shadow, dbPath, close, view: session() }; +} + +function changeBeforeRead(t: TestContext, store: UnifiedStore, dbPath: string, mutate: (db: ReturnType["db"]) => void): void { + const getSessionRow = store.getSessionRow.bind(store); + let changed = false; + t.mock.method(store, "getSessionRow", (id: string) => { + if (!changed) { + changed = true; + const { db } = openSqliteDatabase(dbPath); + try { mutate(db); } finally { db.close(); } + } + return getSessionRow(id); + }); +} + +for (const asyncLane of [false, true]) { + test(`sampled verification bounds reads and preserves live contents (${asyncLane ? "async" : "sync"})`, async (t) => { + const { store, shadow, view, close } = fixture(t, asyncLane); + store.ingestSession(planIngest({ ...view, pluginSnapshot: undefined, lastMessages: Array.from({ length: 200 }, (_, i) => message(i + 1000)) })); + const originalContents = structuredClone(view.blockContents); + const messageReads: { requested: number[] | undefined; returned: number }[] = []; + const blockReads: { limit: number | undefined; returned: number }[] = []; + const getMessages = store.getMessages.bind(store); + const getBlocks = store.getBlocks.bind(store); + t.mock.method(store, "getMessages", (id: string, opts?: Parameters[1]) => { + const rows = getMessages(id, opts); + messageReads.push({ requested: opts?.seqs ? [...opts.seqs] : undefined, returned: rows.length }); + return rows; + }); + t.mock.method(store, "getBlocks", (id: string, opts?: Parameters[1]) => { + const rows = getBlocks(id, opts); + blockReads.push({ limit: opts?.limit, returned: rows.length }); + return rows; + }); + shadow.record(view); + if (asyncLane) await close(); + assert.equal(shadow.stats.verifyOk, 1, shadow.stats.lastError ?? "sample must pass"); + assert.equal(shadow.stats.verifyDrift, 0); + assert.deepEqual(view.blockContents, originalContents); + assert.ok(messageReads.length >= 3 && blockReads.length >= 3); + assert.ok(messageReads.every((r) => r.requested && r.requested.length <= 32 && r.returned <= 32)); + assert.equal(messageReads.reduce((n, r) => n + r.returned, 0), 80, "only referenced messages are hydrated"); + assert.ok(blockReads.every((r) => r.limit === 32 && r.returned <= 32)); + }); +} + +test("a changed message beyond the first batch trips drift and clears the watermark", (t) => { + const { store, shadow, dbPath, view } = fixture(t); + changeBeforeRead(t, store, dbPath, (db) => { + const row = db.prepare("SELECT content_hash FROM messages WHERE session_id = ? AND msg_id = ?").get(view.id, "raw-70") as { content_hash: string }; + db.prepare("UPDATE content SET bytes = ? WHERE hash = ?").run(Buffer.from(JSON.stringify({ ...message(70), text: "changed" })), row.content_hash); + }); + shadow.record(view); + assert.equal(shadow.stats.verifyDrift, 1, shadow.stats.lastError ?? "changed content must drift"); + assert.equal(store.ingestWatermark(view.id), null); + assert.ok(shadow.driftedSessions.has(view.id)); +}); + +test("later block batches detect changed, missing and extra block contents", (t) => { + for (const kind of ["changed", "missing", "extra"] as const) { + const { store, shadow, dbPath, view } = fixture(t); + changeBeforeRead(t, store, dbPath, (db) => { + if (kind === "changed") { + const row = db.prepare("SELECT summary_hash FROM blocks WHERE session_id = ? AND block_id = ?").get(view.id, "b69") as { summary_hash: string }; + db.prepare("UPDATE content SET bytes = ? WHERE hash = ?").run(Buffer.from(JSON.stringify({ text: "changed original", count: 2 })), row.summary_hash); + } else if (kind === "missing") { + db.prepare("DELETE FROM blocks WHERE session_id = ? AND block_id = ?").run(view.id, "b69"); + } else { + const orphans = { "b-orphan": view.blockContents.get("b-orphan"), unexpected: { one: null, full: { text: "extra original", count: 1 } } }; + db.prepare("UPDATE sessions SET orphans_json = ? WHERE session_id = ?").run(JSON.stringify(orphans), view.id); + } + }); + shadow.record(view); + assert.equal(shadow.stats.verifyDrift, 1, `${kind}: ${shadow.stats.lastError}`); + assert.equal(store.ingestWatermark(view.id), null, kind); + } +}); + +test("missing and retired tail rows remain visible as drift", (t) => { + for (const retired of [false, true]) { + const { store, shadow, dbPath, view } = fixture(t); + changeBeforeRead(t, store, dbPath, (db) => { + const sql = retired + ? "UPDATE messages SET retired_at = 1700000000000 WHERE session_id = ? AND msg_id = ?" + : "DELETE FROM messages WHERE session_id = ? AND msg_id = ?"; + db.prepare(sql).run(view.id, "raw-70"); + }); + shadow.record(view); + assert.equal(shadow.stats.verifyDrift, 1, shadow.stats.lastError ?? "missing referenced message must drift"); + assert.equal(store.ingestWatermark(view.id), null); + } +}); diff --git a/tests/web-storage-diagnostics-redaction.test.ts b/tests/web-storage-diagnostics-redaction.test.ts index eb8ef03b1d..d49295c6da 100644 --- a/tests/web-storage-diagnostics-redaction.test.ts +++ b/tests/web-storage-diagnostics-redaction.test.ts @@ -33,12 +33,16 @@ function restoreEnv(snap: EnvSnap): void { } /** Pin every path-relevant env var (unset ones deleted so homedir fallbacks can - * never reach the real machine's dirs) AND $HOME (so os.homedir() is + * never reach the real machine's dirs) AND HOME/USERPROFILE (so os.homedir() is * deterministic); all restored in finally. */ function withEnv(home: string | undefined, overrides: Record, fn: () => T): T { const saved = saveEnv(); const savedHome = process.env.HOME; - if (home !== undefined) process.env.HOME = home; + const savedProfile = process.env.USERPROFILE; + if (home !== undefined) { + process.env.HOME = home; + process.env.USERPROFILE = home; + } for (const k of ENV_KEYS) { if (overrides[k] !== undefined) process.env[k] = overrides[k]; else delete process.env[k]; @@ -49,13 +53,13 @@ function withEnv(home: string | undefined, overrides: Record { - const savedHome = process.env.HOME; - process.env.HOME = "/home/alice"; - try { + withEnv("/home/alice", {}, () => { assert.equal(maskPathsForPublish("/home/alice/.local/share/billion-context/x.db"), "~/.local/share/billion-context/x.db"); assert.equal(maskPathsForPublish("open failed at /home/alice/db"), "open failed at ~/db"); assert.equal(maskPathsForPublish('ENOENT: no such file or directory, open \'/home/alice/index.db\''), "ENOENT: no such file or directory, open '~/index.db'"); @@ -65,16 +69,11 @@ test("#4427 maskPathsForPublish rewrites this machine's $HOME to ~, anchored on // (host masking is a separate net; the path masker must not corrupt URLs) assert.equal(maskPathsForPublish("https://relay.internal/home/alice/doc.pdf"), "https://relay.internal/home/alice/doc.pdf", "URL pathname not rewritten"); assert.equal(maskPathsForPublish("~/.already-tilded"), "~/.already-tilded", "idempotent on already-tilded text"); - } finally { - if (savedHome === undefined) delete process.env.HOME; - else process.env.HOME = savedHome; - } + }); }); test("#4427 redactForPublish composes secret scrub + host mask + home-path strip in one pass", () => { - const savedHome = process.env.HOME; - process.env.HOME = "/home/bob"; - try { + withEnv("/home/bob", {}, () => { const out = redactForPublish("probe https://user:SecretPass@db.internal:3306/app?token=FAKE_TOKEN at /home/bob/.cache/x authorization: bearer sk-live-abcdef1234567890 failed"); assert.ok(!out.includes("db.internal"), "private URL host masked"); assert.ok(!out.includes("SecretPass"), "URL userinfo password dropped"); @@ -84,10 +83,7 @@ test("#4427 redactForPublish composes secret scrub + host mask + home-path strip assert.ok(out.includes("~/.cache/x"), "path stays actionable via ~"); assert.ok(!out.includes("sk-live-abcdef1234567890"), "bearer token scrubbed"); assert.ok(out.includes(" { From 41422f34d82261573a3c933954d29ed3ae1e7f8d Mon Sep 17 00:00:00 2001 From: ework-daemon Date: Mon, 26 Oct 2026 03:13:38 +0800 Subject: [PATCH 04/25] test: pin the sessionViewRows pointer-scoped contract on both storage lanes (#4739) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up for PR #4765: the new bounded-view contract was only exercised by the sqlite-scoped issue suite. Add it to the backend contract suite so the mysql lane (CI DSN) pins the same shape — pointer-only fetch, byte-equal rows, null on miss. --- tests/storage-backend-contract.test.ts | 68 +++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/storage-backend-contract.test.ts b/tests/storage-backend-contract.test.ts index ba36a1a71b..672c64b9b7 100644 --- a/tests/storage-backend-contract.test.ts +++ b/tests/storage-backend-contract.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; -import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { planIngest, planIngestDelta, type LegacySessionLike } from "../src/storage/ingest.ts"; // The contract suite deliberately imports NOTHING from store.ts: every // assertion goes through the StorageBackend interface only. When the mysql // backend lands (#3388 step 2) this same file runs against it unchanged. @@ -110,6 +110,53 @@ test("contract: ingest → reads round-trip through the interface", async () => } }); +// #4739: ordinary loads must fetch ONLY the tail/pluginSnapshot pointer rows — +// the historical log stays on disk but is not decoded. Both lanes carry this +// contract (sqlite pins it here; the mysql lane below pins the same shape). +test("contract: sessionViewRows scopes message fetch to tail pointers (#4739)", async () => { + const dir = scratch(); + try { + const backend = await openBackend("sqlite", path.join(dir, "index.db")); + try { + const N = 6; + const msgs = Array.from({ length: N }, (_, i) => mkMsg(`m${i + 1}`, `body-${i + 1} ` + "z".repeat(120))); + assert.ok(await backend.ingestSession(planIngest(mkSession("sess-view", msgs))), "gen-1 ingest ok"); + + // Generation 2 — the production shape after an idle fold: the + // table keeps every row, tail_json points at the bounded suffix. + const all = await backend.getMessages("sess-view"); + assert.equal(all.length, N, "generation 1 ingested the full history"); + const maxSeq = all[all.length - 1]!.seq; + const K = 3; + const suffixSeqs = all.slice(-K).map((m) => m.seq); + const boundary = all[all.length - 1]!; + const plan = planIngestDelta(mkSession("sess-view", msgs.slice(-K)), maxSeq, suffixSeqs, { msgId: boundary.msgId, contentHash: boundary.contentHash }, true); + assert.ok(plan, "empty delta accepted for the shrunken raw tail"); + assert.ok( + await backend.ingestSession(plan, { trustedSeq: maxSeq, boundaryMsgId: boundary.msgId, boundaryHash: boundary.contentHash, trustedTailSeqs: suffixSeqs }), + "gen-2 delta ingest ok", + ); + + const view = await backend.sessionViewRows("sess-view"); + assert.ok(view, "view bundle present"); + assert.equal(view!.session.sessionId, "sess-view"); + const stillAll = await backend.getMessages("sess-view"); + assert.equal(stillAll.length, N, "full history retained on disk"); + assert.deepEqual(view!.messages.map((m) => m.seq), suffixSeqs, "only the pointer seqs were fetched"); + const bySeq = new Map(stillAll.map((m) => [m.seq, m])); + for (const row of view!.messages) { + assert.equal(row.content, bySeq.get(row.seq)!.content, "pointer row byte-equal to the full-log row"); + assert.equal(row.contentHash, bySeq.get(row.seq)!.contentHash); + } + assert.equal(await backend.sessionViewRows("no-such-session"), null, "null on session miss"); + } finally { + await backend.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + test("contract: session_ext + watermark + tailSeqs", async () => { const dir = scratch(); try { @@ -248,6 +295,25 @@ test("mysql lane: contract surface against MySQL 8.0", { skip: MYSQL_DSN.length const rows2 = await backend.getMessages(sid); assert.equal(rows2.length, 2, "no duplicate rows on identical re-ingest"); + // #4739: sessionViewRows pointer scoping on the mysql lane (per-statement + // reads are safe because ingestSession commits message rows + session row + // in one transaction; the sqlite twin pins the same shape above) + const vwSid = `sess-mysql-view-${run}`; + const vwMsgs = [mkMsg("m1", "view " + "v".repeat(80)), mkMsg("m2", "view " + "w".repeat(80)), mkMsg("m3", "view " + "q".repeat(80)), mkMsg("m4", "view " + "e".repeat(80))]; + assert.ok(await backend.ingestSession(planIngest(mkSession(vwSid, vwMsgs))), "view gen-1 ok"); + const vwAll = await backend.getMessages(vwSid); + const vwMaxSeq = vwAll[vwAll.length - 1]!.seq; + const vwSuffix = vwAll.slice(-2).map((m) => m.seq); + const vwBoundary = vwAll[vwAll.length - 1]!; + const vwPlan = planIngestDelta(mkSession(vwSid, vwMsgs.slice(-2)), vwMaxSeq, vwSuffix, { msgId: vwBoundary.msgId, contentHash: vwBoundary.contentHash }, true); + assert.ok(vwPlan, "empty delta accepted for the shrunken raw tail"); + assert.ok(await backend.ingestSession(vwPlan, { trustedSeq: vwMaxSeq, boundaryMsgId: vwBoundary.msgId, boundaryHash: vwBoundary.contentHash, trustedTailSeqs: vwSuffix }), "view gen-2 ok"); + const vwView = await backend.sessionViewRows(vwSid); + assert.ok(vwView, "view bundle present"); + assert.deepEqual(vwView!.messages.map((m) => m.seq), vwSuffix, "only the pointer seqs were fetched"); + assert.equal(await backend.sessionViewRows(`no-such-${run}`), null, "null on session miss"); + await backend.deleteSession(vwSid); + // reap journal await backend.reapBlocks(sid, new Set(["b2"]), new Map([["b2", { one: null, full: { text: "orphaned", count: 0 } }]])); await backend.reapBlocks(sid, new Set(["b2"]), new Map(), ); From c3619a0511186e83b6a57adf63773e9c42faf1dc Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:09:26 -0700 Subject: [PATCH 05/25] fix: preserve blog metadata and launcher checks on Windows --- src/web/blogfeed.ts | 2 ++ tests/launcher-opencode-inline.test.ts | 20 +++++++++++--------- tests/web-blogfeed.test.ts | 17 ++++++++++++++++- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/web/blogfeed.ts b/src/web/blogfeed.ts index 6efe1560a7..0d67c61d8f 100644 --- a/src/web/blogfeed.ts +++ b/src/web/blogfeed.ts @@ -76,6 +76,8 @@ export function _setBlogRootForTest(root: string | null): void { /** Mirror of the website tools' frontmatter parser (key: value between --- fences). */ function parseFrontmatter(text: string): { fm: Record; body: string } { + // Windows checkouts use CRLF; parse and render the same Markdown on every platform. + text = text.replace(/\r\n/g, "\n"); const fm: Record = {}; let body = text; if (text.startsWith("---\n")) { diff --git a/tests/launcher-opencode-inline.test.ts b/tests/launcher-opencode-inline.test.ts index 7ff04b2ace..fd5452a332 100644 --- a/tests/launcher-opencode-inline.test.ts +++ b/tests/launcher-opencode-inline.test.ts @@ -28,6 +28,7 @@ interface SpawnRecord { command: string; args: string[]; env: NodeJS.ProcessEnv | undefined; + windowsVerbatimArguments?: boolean; } // `bili opencode-inline` (#4669): the launcher must start NO proxy, hand the @@ -37,16 +38,12 @@ interface SpawnRecord { // clone (plugin injection only; 2.x directory wrapper when the host is 2.x). test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attach/proxy env stripped, plugin rides OPENCODE_CONFIG", async () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ocinline-")); - const prevHome = process.env.HOME; - const prevUserProfile = process.env.USERPROFILE; const prevXdg = process.env.XDG_CONFIG_HOME; const prevOcConfig = process.env.OPENCODE_CONFIG; const prevClientBin = process.env.BILI_CLIENT_BIN; const prevBcp = process.env.BILLION_CONTEXT_PROXY; const prevHttpsProxy = process.env.HTTPS_PROXY; const prevMitmHosts = process.env.BILI_MITM_HOSTS; - process.env.HOME = home; - if (prevUserProfile !== undefined) process.env.USERPROFILE = home; // Isolated config root: no user opencode.json, no ambient OPENCODE_CONFIG. process.env.XDG_CONFIG_HOME = path.join(home, ".config"); delete process.env.OPENCODE_CONFIG; @@ -75,7 +72,7 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac const spawns: SpawnRecord[] = []; const spawnImpl: SpawnFn = (cmd, args, options) => { - spawns.push({ command: cmd, args: [...args], env: options?.env }); + spawns.push({ command: cmd, args: [...args], env: options?.env, windowsVerbatimArguments: options?.windowsVerbatimArguments }); const child = makeFakeChild(0); const orig = child.on!.bind(child); (child as { on: SpawnChild["on"] }).on = (event, listener) => { @@ -109,8 +106,16 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac { fetchImpl, fetchHealthInfo, spawnImpl, sleep: () => Promise.resolve() }, ); assert.equal(spawns.length, 1, "exactly one child (opencode itself) — no proxy spawn"); - assert.equal(spawns[0].command, fakeOc); const env = spawns[0].env!; + if (process.platform === "win32") { + assert.equal(spawns[0].command, env.COMSPEC?.trim() ? env.COMSPEC : "cmd.exe"); + const quotedOc = /\s/.test(fakeOc) ? `"${fakeOc}"` : fakeOc; + assert.deepEqual(spawns[0].args, ["/d", "/s", "/c", `"${quotedOc}"`]); + assert.equal(spawns[0].windowsVerbatimArguments, true); + } else { + assert.equal(spawns[0].command, fakeOc); + assert.deepEqual(spawns[0].args, []); + } assert.equal(env.BILI_OPENCODE_INLINE, "1"); assert.equal(env.BILLION_CONTEXT_PROXY, undefined, "inherited BILLION_CONTEXT_PROXY must be stripped (attach would win)"); assert.equal(env.HTTPS_PROXY, undefined, "inherited HTTPS_PROXY must be stripped (no MITM lane)"); @@ -134,9 +139,6 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac assert.equal(clone.compaction?.auto, false, "host auto-compaction stays off (ACP owns compression)"); } finally { process.exit = prevExit; - process.env.HOME = prevHome; - if (prevUserProfile === undefined) delete process.env.USERPROFILE; - else process.env.USERPROFILE = prevUserProfile; if (prevXdg === undefined) delete process.env.XDG_CONFIG_HOME; else process.env.XDG_CONFIG_HOME = prevXdg; if (prevOcConfig === undefined) delete process.env.OPENCODE_CONFIG; diff --git a/tests/web-blogfeed.test.ts b/tests/web-blogfeed.test.ts index 0d1d820907..5280ba9dd3 100644 --- a/tests/web-blogfeed.test.ts +++ b/tests/web-blogfeed.test.ts @@ -9,7 +9,7 @@ */ import { test, beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { mkdtempSync, rmSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { ServerResponse } from "node:http"; @@ -93,6 +93,21 @@ test("rendering is escape-first: script tags survive only escaped; tables become assert.ok(alpha.html.includes("two")); }); +test("CRLF markdown preserves bilingual frontmatter and rendered content", () => { + const expected = getBlogFeed(); + for (const relative of [ + "website/blog/posts/2026-10-08_demo.md", + "website/blog/posts/2026-10-08_demo.zh.md", + "paper/a.md", + "paper/b.md", + ]) { + const file = join(pkgRoot, relative); + writeFileSync(file, readFileSync(file, "utf8").replace(/\n/g, "\r\n")); + } + _setBlogRootForTest(pkgRoot); + assert.deepEqual(getBlogFeed(), expected); +}); + test("image refs: shipped files reroute to the panel asset prefix; traversal/unshipped refs drop silently", () => { const data = getBlogFeed(); const demo = data.items.find((i) => i.id === "demo")!; From c12bee29299552daac51e6d4643093d85fc3d410 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:35:44 -0700 Subject: [PATCH 06/25] test: verify fork restart under legacy and full storage --- tests/plugin-fork.test.ts | 136 ++++++++++++++++++++------------------ 1 file changed, 71 insertions(+), 65 deletions(-) diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index d15bc89fe9..caf2c00688 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -114,7 +114,7 @@ async function harness(persist = false, seedParent = true, compressExtra: Record assert.equal(r.status, 200); await r.text(); } - return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.cancelAll(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; + return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.closeQuietly(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; } test("plain proxy panel snapshots do not change raw-history persistence", { timeout: 60_000 }, async () => { @@ -742,71 +742,77 @@ for (const corruption of ["missing-ref", "missing-payload", "orphan-payload", "w }); } -test("HTTP fork revision and originals survive a new proxy process", { timeout: 60_000 }, async () => { - const h = await harness(true); - let child: ReturnType | undefined; - try { - await compress(h); - const parent = resolveConversation("parent").session!; - parent.metadata.effectiveCcr = { ...DEFAULT_CCR_CONFIG, enabled: true }; - parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); - parent.contentStoreDirty = true; - assert(h.store.flushSync(parent)); - // #4316 PR3/3: cold load runs the one-time ledger alignment — the loaded - // state now carries refLedgerAligned. Mirror it in the expectation. - const persistedState = JSON.parse(JSON.stringify(parent.state)); - persistedState.refLedgerAligned = true; - assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); - const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; - const req = forkRequest(snapshot); - const fork = await h.request("/__bili/plugin/fork", req); - assert.equal(fork.status, 201); - const childSnapshot = (await h.request("/__bili/plugin/snapshot?conversationId=child")).body; - const code = `import {startServer} from './src/server.ts'; import {SessionStore,_setStoreForTest} from './src/persist.ts'; import {defaultConfig} from 'acp-kernel'; import {_setForTest} from './src/registry.ts'; _setForTest({}); _setStoreForTest(new SessionStore({dir:${JSON.stringify(h.dir + "/sessions")},enabled:true})); const s=await startServer({port:0,host:'127.0.0.1',upstream:${JSON.stringify(h.upstreamUrl)},modelContextLimit:400000,routes:{},kernelConfig:defaultConfig(400000),compress:{injectTool:true,injectNudge:true},promptCache:{routing:'auto'},sessionHeader:'x-acp-session',log:false,debug:false,passthrough:false,autoUpdate:false,mitm:{enabled:false,domains:[]}}); const ready=()=>process.stdout.write(JSON.stringify({port:s.address().port})+'\\n'); if(s.listening)ready();else s.once('listening',ready);`; - child = spawn(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", code], { cwd: fileURLToPath(new URL("../", import.meta.url)), env: { ...process.env, NODE_ENV: "test" }, stdio: ["ignore", "pipe", "pipe"] }); - let stderr = ""; - child.stderr!.on("data", (b) => { stderr += b.toString(); }); - const port = await new Promise((resolve, reject) => { - let stdout = ""; - const timer = setTimeout(() => reject(new Error("restart timeout: " + stderr + stdout)), 15000); - child!.stdout!.on("data", (b) => { - stdout += b.toString(); - const port = stdout.match(/\{"port":(\d+)\}/); - if (port) { clearTimeout(timer); resolve(Number(port[1])); } +for (const storageMode of ["legacy", "full"] as const) { + test(`HTTP fork revision and originals survive a new proxy process (${storageMode})`, { timeout: 60_000 }, async () => { + const previousMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = storageMode; + const h = await harness(true); + let child: ReturnType | undefined; + try { + await compress(h); + const parent = resolveConversation("parent").session!; + parent.metadata.effectiveCcr = { ...DEFAULT_CCR_CONFIG, enabled: true }; + parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); + parent.contentStoreDirty = true; + assert(h.store.flushSync(parent)); + // #4316 PR3/3: only a unified cold load runs the one-time ledger + // alignment; the CI legacy baseline retains the original state. + const persistedState = JSON.parse(JSON.stringify(parent.state)); + if (storageMode === "full") persistedState.refLedgerAligned = true; + assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); + const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; + const req = forkRequest(snapshot); + const fork = await h.request("/__bili/plugin/fork", req); + assert.equal(fork.status, 201); + const childSnapshot = (await h.request("/__bili/plugin/snapshot?conversationId=child")).body; + const code = `import {startServer} from './src/server.ts'; import {SessionStore,_setStoreForTest} from './src/persist.ts'; import {defaultConfig} from 'acp-kernel'; import {_setForTest} from './src/registry.ts'; _setForTest({}); _setStoreForTest(new SessionStore({dir:${JSON.stringify(h.dir + "/sessions")},enabled:true})); const s=await startServer({port:0,host:'127.0.0.1',upstream:${JSON.stringify(h.upstreamUrl)},modelContextLimit:400000,routes:{},kernelConfig:defaultConfig(400000),compress:{injectTool:true,injectNudge:true},promptCache:{routing:'auto'},sessionHeader:'x-acp-session',log:false,debug:false,passthrough:false,autoUpdate:false,mitm:{enabled:false,domains:[]}}); const ready=()=>process.stdout.write(JSON.stringify({port:s.address().port})+'\\n'); if(s.listening)ready();else s.once('listening',ready);`; + child = spawn(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", code], { cwd: fileURLToPath(new URL("../", import.meta.url)), env: { ...process.env, NODE_ENV: "test" }, stdio: ["ignore", "pipe", "pipe"] }); + let stderr = ""; + child.stderr!.on("data", (b) => { stderr += b.toString(); }); + const port = await new Promise((resolve, reject) => { + let stdout = ""; + const timer = setTimeout(() => reject(new Error("restart timeout: " + stderr + stdout)), 15000); + child!.stdout!.on("data", (b) => { + stdout += b.toString(); + const port = stdout.match(/\{"port":(\d+)\}/); + if (port) { clearTimeout(timer); resolve(Number(port[1])); } + }); + child!.once("exit", (code) => { clearTimeout(timer); reject(new Error(`restart exited ${code}: ${stderr}`)); }); + child!.once("error", (error) => { clearTimeout(timer); reject(error); }); }); - child!.once("exit", (code) => { clearTimeout(timer); reject(new Error(`restart exited ${code}: ${stderr}`)); }); - child!.once("error", (error) => { clearTimeout(timer); reject(error); }); - }); - const call = async (path: Path, body?: unknown) => { - const r = await fetch(`http://127.0.0.1:${port}` + path, body === undefined ? {} : { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); - return { status: r.status, body: await responseBody(r, path) }; - }; - const restored = await call("/__bili/plugin/snapshot?conversationId=child"); - const restoredStatus = await call("/__bili/plugin/status?conversationId=parent&fallback=latest"); - assert.equal(restoredStatus.status, 200); - assert.equal(restoredStatus.body.sessionId, "parent"); - assert.equal(restoredStatus.body.fallback, undefined); - assert.equal(restoredStatus.body.sessionRevision, snapshot.parentRevision); - assert.equal(restoredStatus.body.contextTokensSource, "estimate"); - assert.equal(restored.status, 200); - assert.equal(restored.body.parentRevision, childSnapshot.parentRevision); - assert.deepEqual(restored.body.orderedMessages, childSnapshot.orderedMessages); - assert.equal((await call("/__bili/plugin/snapshot?conversationId=parent")).body.parentRevision, snapshot.parentRevision); - const replay = await call("/__bili/plugin/fork", req); - assert.equal(replay.status, 200); - assert.equal(replay.body.replayed, true); - const retrieved = await call("/__bili/plugin/tool", { conversationId: "child", tool: "acp_retrieve", args: { ref: "m00001" } }); - assert.equal(retrieved.status, 200); - assert.match(retrieved.body.result, /CCR original retained independently after restart/); - const original = await call("/__bili/plugin/tool", { conversationId: "child", tool: "decompress", args: { blockId: "b1", full: true } }); - assert.match(original.body.result, /first original/); - assert.match(original.body.result, /second original/); - assert(!original.body.result.includes("tail original")); - } finally { - if (child?.pid && child.exitCode === null && child.signalCode === null) { child.kill("SIGTERM"); await once(child, "exit"); } - await h.close(); - } -}); + const call = async (path: Path, body?: unknown) => { + const r = await fetch(`http://127.0.0.1:${port}` + path, body === undefined ? {} : { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + return { status: r.status, body: await responseBody(r, path) }; + }; + const restored = await call("/__bili/plugin/snapshot?conversationId=child"); + const restoredStatus = await call("/__bili/plugin/status?conversationId=parent&fallback=latest"); + assert.equal(restoredStatus.status, 200); + assert.equal(restoredStatus.body.sessionId, "parent"); + assert.equal(restoredStatus.body.fallback, undefined); + assert.equal(restoredStatus.body.sessionRevision, snapshot.parentRevision); + assert.equal(restoredStatus.body.contextTokensSource, "estimate"); + assert.equal(restored.status, 200); + assert.equal(restored.body.parentRevision, childSnapshot.parentRevision); + assert.deepEqual(restored.body.orderedMessages, childSnapshot.orderedMessages); + assert.equal((await call("/__bili/plugin/snapshot?conversationId=parent")).body.parentRevision, snapshot.parentRevision); + const replay = await call("/__bili/plugin/fork", req); + assert.equal(replay.status, 200); + assert.equal(replay.body.replayed, true); + const retrieved = await call("/__bili/plugin/tool", { conversationId: "child", tool: "acp_retrieve", args: { ref: "m00001" } }); + assert.equal(retrieved.status, 200); + assert.match(retrieved.body.result, /CCR original retained independently after restart/); + const original = await call("/__bili/plugin/tool", { conversationId: "child", tool: "decompress", args: { blockId: "b1", full: true } }); + assert.match(original.body.result, /first original/); + assert.match(original.body.result, /second original/); + assert(!original.body.result.includes("tail original")); + } finally { + if (child?.pid && child.exitCode === null && child.signalCode === null) { child.kill("SIGTERM"); await once(child, "exit"); } + await h.close(); + if (previousMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = previousMode; + } + }); +} test("HTTP status separates real usage from a manual-compression estimate and changing generation", { timeout: 60_000 }, async () => { const h = await harness(); From 8576277d2cfd8e7dc82cbeee0a85177b1eec93a3 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:36:40 -0700 Subject: [PATCH 07/25] test(storage): account for persisted engine lane in render goldens The engineLane=server metadata added by f590da94 explains the entire drift: removing that one field reproduces the old metadata digest. Regeneration changes only metadata and the enclosing digest; all wire bodies, state, originals and statistics remain identical. --- tests/golden/storage-render/anthropic.json | 4 ++-- tests/golden/storage-render/chat.json | 4 ++-- tests/golden/storage-render/google.json | 4 ++-- tests/golden/storage-render/responses.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 5afa14b2c4..13811dcab0 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "6727f42bd91879a7aa370e1ce0c78963bcba0df92e50390d4a8011f8f0da334d", + "stateDigest": "0a241435d456ba994c92b0072d2173bf2a6d00ee73223458fb4302e170efb1e3", "sectionDigests": { "state": "f93dc1c876c672d4", "blockContents": "dc7d13d250c94bb4", "stats": "c6f1d434975d047d", - "metadata": "f1ec4aab3aff777a" + "metadata": "16cad648ccf41950" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 46764f97db..01f69b1036 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "e149c301f382438ee063a38f70dd794207b88e94bdf3c14c5191da0eeaa99bc9", + "stateDigest": "9f1b8258a916fcf63af14625def9ed0236eb8ec9c542f0027206985598d2a5b8", "sectionDigests": { "state": "8390f21336885863", "blockContents": "94a58f96d7c5cfc3", "stats": "24d705b0e39bd13c", - "metadata": "55733fb81ccf7d69" + "metadata": "2198f47054c8acae" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index ba999cadec..fe9f9c95dd 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "80e83935e5fa7b052a3cb0f1d641bd81cb6ee59f91f3c1f2f31c204de29368a5", + "stateDigest": "c4979d4f58cfa49bc8be8c43b8e1f1b0861b3b9d44cdc415ee09e4ca21d9aa5a", "sectionDigests": { "state": "2e1def3a3234afdc", "blockContents": "473264d9df8ba338", "stats": "900481685c848462", - "metadata": "88cd02a467b4457e" + "metadata": "939dc0c09b6cbcbf" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 7004d90435..0402e7d43c 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "08526d3552fca7ac7552cd16a221893a652fbe740c9b8dccac6db57b3d6302ee", + "stateDigest": "b3bddfa75ce3f8b94424be9a0164c1b9a183faa9daf705e6175cabdd671c95ff", "sectionDigests": { "state": "db313e8eba2d65fc", "blockContents": "b6701268a0bdcd8a", "stats": "327d3967b8371aa8", - "metadata": "b5c39d84db67f424" + "metadata": "a9eecd3816738158" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", From 9397bc8370874275081f38a84cadbead30b1cca6 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:44:01 -0700 Subject: [PATCH 08/25] test(storage): account for one-time ledger ref migration on restart Compare unified reads with an independent envelope reader plus the documented migration. Persist alignment before rollback/corrupt-index checks, select legacy explicitly, and assert the recovered raw ID under its ledger ref. Keep checkpoint writes deterministic so sampled ingest cadence cannot change the seed between reader legs. All 28 targeted storage tests pass; no product migration or digest assertion is disabled. --- tests/golden-storage-drive.ts | 25 ++++++++++++--- tests/storage-engine-proofs.test.ts | 45 ++++++++++++++++----------- tests/storage-orphan-recovery.test.ts | 6 +++- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/tests/golden-storage-drive.ts b/tests/golden-storage-drive.ts index a6b14205f8..41a8b2166d 100644 --- a/tests/golden-storage-drive.ts +++ b/tests/golden-storage-drive.ts @@ -17,10 +17,10 @@ import os from "node:os"; import path from "node:path"; import { once } from "node:events"; import { canonicalize, sha256 } from "./golden-canonical.ts"; -import { defaultConfig } from "acp-kernel"; +import { defaultConfig, restampRefsToLedger } from "acp-kernel"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; -import { flushAllSessions, _resetSessionsForTest } from "../src/session.ts"; +import { flushAllSessions, getSession, _resetSessionsForTest } from "../src/session.ts"; import type { ProxyOptions } from "../src/config.ts"; /** @@ -462,6 +462,8 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: dumpStateTo?: string; storageUnified?: "shadow" | "full"; reloadAfterEdit?: "keepFile" | "dropFile"; + // Independent legacy-read oracle for #4316's intentional cold-load migration. + alignLegacyRefsOnReload?: boolean; } = {}): Promise { __goldenGuard(); fs.mkdirSync(SCRATCH_ROOT, { recursive: true }); @@ -503,7 +505,12 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: upstream.listen(0, "127.0.0.1"); await listen(upstream); const upstreamPort = (upstream.address() as { port: number }).port; - store = new SessionStore({ dir: sessionsDir, debounceMs: 1, enabled: true }); + // Save at the explicit restart/final checkpoints below. A 1ms timer + // makes ingest/verification counts depend on machine speed, so the + // three reader legs can migrate refs from different ledger snapshots. + // The debounce exceeds the suite's four-minute deadline; it is never + // awaited. Delta/sampled-write behavior has storage-watermark tests. + store = new SessionStore({ dir: sessionsDir, debounceMs: 300_000, enabled: true }); _setStoreForTest(store); setRegistryForTest({}); proxy = await startServer(goldenProxyOptions(upstreamPort, 200_000)); @@ -558,8 +565,16 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: if (fs.statSync(fp).isFile() && (f as string).endsWith(".json") && !fs.existsSync(fp + ".tmp")) fs.rmSync(fp, { force: true }); } } - store = new SessionStore({ dir: sessionsDir, debounceMs: 1, enabled: true }); + store = new SessionStore({ dir: sessionsDir, debounceMs: 300_000, enabled: true }); _setStoreForTest(store); + if (opts.alignLegacyRefsOnReload) { + const loaded = getSession(sessionId); + const seqByRaw = store.unifiedSeqByRaw(sessionId); + if (!seqByRaw?.size) throw new Error("legacy alignment oracle requires a populated shadow ledger"); + const outcome = restampRefsToLedger(loaded.state, seqByRaw); + if (outcome.superseded.size > 0) loaded.restampSuperseded = outcome.superseded; + store.scheduleSave(loaded); + } } // Post-edit growth: 2 trailing turns. Also load-bearing for the // STATE digest: the persist layer does not reliably write the final @@ -660,7 +675,7 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: // orphaned inode and REOPEN the deleted path, resurrecting the tree // mid-rmrf (the #1646/#1910 ENOTEMPTY/EBUSY race class). The detach // + target-clear below are synchronous, so no interleaving window. - // The 1ms-debounced store writer races the same way (recreating + // A pending store writer races the same way (recreating // sessions// after the final flush). try { store?.cancelAll(); } catch { /* no store on early-exit paths */ } void closeLogger(); diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index 118ca0bde7..782ab1973a 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -5,21 +5,19 @@ * STATE-level equality between the legacy files and the unified index. This * suite pins the seams that audit found unproven: * - * WIRE PARITY — BILI_STORAGE_UNIFIED=full must be invisible on the - * wire: per-turn outbound digests, fold landmarks and - * the final state digest equal the COMMITTED storage - * render goldens, both with a mid-conversation restart - * and with the legacy file DELETED at the restart (the - * index alone carries the session — the migration's - * end-state promise). + * WIRE PARITY — full reads match envelope reads plus the documented + * #4316 ledger-ref migration. All post-restart bodies, + * landmarks and final digests agree, even when the + * legacy file is DELETED at restart. Pre-migration + * bodies also match the committed render goldens. * STALE REJECT — a multi-instance stale write that the guarded build * rejects must never reach index.db; the converged disk * record is what gets ingested. * IMPORT GUARD — a hand-aged legacy file (older savedAt than the * index watermark) must not re-import over newer index * state on load. - * ROLLBACK — unset the env after running full → the legacy file - * loads with the exact digest. + * ROLLBACK — explicitly select legacy after persisting the full + * load's migration → the file loads the exact digest. * SELF-DISABLE — 3 consecutive sidecar failures disable it; reads flip * back to legacy; the legacy file stays readable. * CORRUPT INDEX — a corrupt index.db degrades the store to legacy-only @@ -141,11 +139,11 @@ function unifiedRows(store: UnifiedStore, id: string): ReturnType { + test(`storage engine: full-mode ${wire} restart matches legacy reads plus ledger alignment`, { timeout: 240_000 }, async () => { const golden = JSON.parse(fs.readFileSync(path.join(RENDER_ROOT, `${wire}.json`), "utf8")) as { turns: Array<{ i: number; sha: string }>; foldAt: number; editedAt: number; summaryFrom: number; @@ -157,13 +155,20 @@ for (const wire of WIRES) { // slots (cacheLedger line bs/bl/be pairing, #2131) live in memory, so // exactly the first post-restart ledger line loses its bs fields — // identically in both legs. Everything else must be byte-equal. - const legacy = await driveWire(wire as Wire, `golden-${wire}`, { reloadAfterEdit: "keepFile" }); + // #4316 deliberately renumbers legacy refs at the first unified cold + // load. Keep the independent envelope reader as the oracle, applying + // only that documented migration against its shadow ledger. Every + // post-restart byte still has to match; the old pre-migration golden + // remains the oracle for all turns before the restart. + const legacy = await driveWire(wire as Wire, `golden-${wire}`, { + storageUnified: "shadow", reloadAfterEdit: "keepFile", alignLegacyRefsOnReload: true, + }); for (const leg of ["keepFile", "dropFile"] as const) { const record = await driveWire(wire as Wire, `golden-${wire}`, { storageUnified: "full", reloadAfterEdit: leg }); assert.equal(record.turns.length, legacy.turns.length, `${wire}/${leg}: body count drifted vs legacy restart`); for (let i = 0; i < legacy.turns.length; i++) { assert.equal(record.turns[i]!.sha, legacy.turns[i]!.sha, `${wire}/${leg}: outbound body #${i} digest drift vs legacy restart`); - assert.equal(record.turns[i]!.sha, golden.turns[i]!.sha, `${wire}/${leg}: outbound body #${i} digest drift vs committed goldens (restart must not change the wire)`); + if (i <= record.editedAt) assert.equal(record.turns[i]!.sha, golden.turns[i]!.sha, `${wire}/${leg}: pre-migration body #${i} digest drift vs committed goldens`); } assert.equal(record.foldAt, legacy.foldAt, `${wire}/${leg}: fold landmark drifted`); assert.equal(record.editedAt, legacy.editedAt, `${wire}/${leg}: edit landmark drifted`); @@ -274,10 +279,10 @@ test("storage engine: load-side import guard skips a hand-aged legacy file (inde }); // --------------------------------------------------------------------------- -// ROLLBACK: full → unset env → legacy file authoritative, digest exact. +// ROLLBACK: full → explicit legacy → legacy file authoritative, digest exact. // --------------------------------------------------------------------------- -test("storage engine: rollback contract — unset the env and the legacy read path serves the identical state", async () => { +test("storage engine: rollback contract — explicit legacy mode serves the identical persisted state", async () => { const dir = tmpdir("bili-engproof-rollback-"); const prev = process.env.BILI_STORAGE_UNIFIED; try { @@ -295,11 +300,13 @@ test("storage engine: rollback contract — unset the env and the legacy read pa // tail normalization, identically under both paths.) const storeFull = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = storeFull.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); - storeFull.closeQuietly(); assert.ok(viaFull, "full-mode reload returned null"); + // Persist the one-time cold-load alignment before exercising rollback. + await storeFull.writeNow(viaFull); + storeFull.closeQuietly(); const digestFull = viewDigest(toLegacyLike(viaFull)); - delete process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "legacy"; const store2 = new SessionStore({ dir, debounceMs: 1, enabled: true }); const legacy = store2.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); store2.closeQuietly(); @@ -366,8 +373,10 @@ test("storage engine: corrupt index.db degrades to legacy-only instead of bricki // fallback must reproduce exactly from the legacy file. const clean = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = clean.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); - clean.closeQuietly(); assert.ok(viaFull, "full-mode read returned null"); + // Both replicas must contain the migration before corrupting the index. + await clean.writeNow(viaFull); + clean.closeQuietly(); // #4338: everything past this point (corrupt the header → degrade // construction → fallback read → rename guards → write → close) runs diff --git a/tests/storage-orphan-recovery.test.ts b/tests/storage-orphan-recovery.test.ts index c63f3f63fb..cb39fcd6a1 100644 --- a/tests/storage-orphan-recovery.test.ts +++ b/tests/storage-orphan-recovery.test.ts @@ -69,7 +69,11 @@ test("cold load recovers the pre-fix orphan freeze before processing a request", assert.equal(recovered.stats.requests, 876); assert.equal(recovered.state.nextBlockId, 4); assert.deepEqual(recovered.state.blocks.map(b => b.blockId), ["b3"]); - assert.equal(recovered.state.messageRefs.byRef.m00003, "raw-new"); + // Recovery is followed by #4316's one-time ledger alignment: raw-new is + // the first stored message, so its durable ref is seq 0 + 1. + assert.equal(recovered.state.messageRefs.byRaw["raw-new"], "m00001"); + assert.equal(recovered.state.messageRefs.byRef.m00001, "raw-new"); + assert.equal(recovered.restampSuperseded?.get("m00003"), "raw-new"); assert.equal(recovered.lastMessages?.at(-1)?.text, "new turn"); assert.equal(recovered.metadata.recoveryMarker, "fresh"); assert.equal(recovered.blockContents.get("b1")?.full.text, "retained original"); From 66297ff97e131681a510d1c96d26c9f8e29963de Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:09:26 -0700 Subject: [PATCH 09/25] test: honor Windows cmd wrapper for inline OpenCode launch --- tests/launcher-opencode-inline.test.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/launcher-opencode-inline.test.ts b/tests/launcher-opencode-inline.test.ts index 7ff04b2ace..fd5452a332 100644 --- a/tests/launcher-opencode-inline.test.ts +++ b/tests/launcher-opencode-inline.test.ts @@ -28,6 +28,7 @@ interface SpawnRecord { command: string; args: string[]; env: NodeJS.ProcessEnv | undefined; + windowsVerbatimArguments?: boolean; } // `bili opencode-inline` (#4669): the launcher must start NO proxy, hand the @@ -37,16 +38,12 @@ interface SpawnRecord { // clone (plugin injection only; 2.x directory wrapper when the host is 2.x). test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attach/proxy env stripped, plugin rides OPENCODE_CONFIG", async () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ocinline-")); - const prevHome = process.env.HOME; - const prevUserProfile = process.env.USERPROFILE; const prevXdg = process.env.XDG_CONFIG_HOME; const prevOcConfig = process.env.OPENCODE_CONFIG; const prevClientBin = process.env.BILI_CLIENT_BIN; const prevBcp = process.env.BILLION_CONTEXT_PROXY; const prevHttpsProxy = process.env.HTTPS_PROXY; const prevMitmHosts = process.env.BILI_MITM_HOSTS; - process.env.HOME = home; - if (prevUserProfile !== undefined) process.env.USERPROFILE = home; // Isolated config root: no user opencode.json, no ambient OPENCODE_CONFIG. process.env.XDG_CONFIG_HOME = path.join(home, ".config"); delete process.env.OPENCODE_CONFIG; @@ -75,7 +72,7 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac const spawns: SpawnRecord[] = []; const spawnImpl: SpawnFn = (cmd, args, options) => { - spawns.push({ command: cmd, args: [...args], env: options?.env }); + spawns.push({ command: cmd, args: [...args], env: options?.env, windowsVerbatimArguments: options?.windowsVerbatimArguments }); const child = makeFakeChild(0); const orig = child.on!.bind(child); (child as { on: SpawnChild["on"] }).on = (event, listener) => { @@ -109,8 +106,16 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac { fetchImpl, fetchHealthInfo, spawnImpl, sleep: () => Promise.resolve() }, ); assert.equal(spawns.length, 1, "exactly one child (opencode itself) — no proxy spawn"); - assert.equal(spawns[0].command, fakeOc); const env = spawns[0].env!; + if (process.platform === "win32") { + assert.equal(spawns[0].command, env.COMSPEC?.trim() ? env.COMSPEC : "cmd.exe"); + const quotedOc = /\s/.test(fakeOc) ? `"${fakeOc}"` : fakeOc; + assert.deepEqual(spawns[0].args, ["/d", "/s", "/c", `"${quotedOc}"`]); + assert.equal(spawns[0].windowsVerbatimArguments, true); + } else { + assert.equal(spawns[0].command, fakeOc); + assert.deepEqual(spawns[0].args, []); + } assert.equal(env.BILI_OPENCODE_INLINE, "1"); assert.equal(env.BILLION_CONTEXT_PROXY, undefined, "inherited BILLION_CONTEXT_PROXY must be stripped (attach would win)"); assert.equal(env.HTTPS_PROXY, undefined, "inherited HTTPS_PROXY must be stripped (no MITM lane)"); @@ -134,9 +139,6 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac assert.equal(clone.compaction?.auto, false, "host auto-compaction stays off (ACP owns compression)"); } finally { process.exit = prevExit; - process.env.HOME = prevHome; - if (prevUserProfile === undefined) delete process.env.USERPROFILE; - else process.env.USERPROFILE = prevUserProfile; if (prevXdg === undefined) delete process.env.XDG_CONFIG_HOME; else process.env.XDG_CONFIG_HOME = prevXdg; if (prevOcConfig === undefined) delete process.env.OPENCODE_CONFIG; From 610b164e170dfa43ad4839af841075952c205acf Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:14:31 -0700 Subject: [PATCH 10/25] test: close fork session stores before Windows cleanup --- tests/plugin-fork.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 1c4801377d..e6ea8b84de 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -114,7 +114,7 @@ async function harness(persist = false, seedParent = true, compressExtra: Record assert.equal(r.status, 200); await r.text(); } - return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.cancelAll(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; + return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.closeQuietly(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; } test("plain proxy panel snapshots do not change raw-history persistence", { timeout: 60_000 }, async () => { From f16d36680a3b921d657759ca4f94521614986143 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:17:29 -0700 Subject: [PATCH 11/25] fix: stream storage verification digests (#4529) --- src/storage/canonical.ts | 85 ++++++++++++++++++++++++++ src/storage/shadow.ts | 4 +- tests/storage-canonical-digest.test.ts | 45 ++++++++++++++ 3 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 tests/storage-canonical-digest.test.ts diff --git a/src/storage/canonical.ts b/src/storage/canonical.ts index 4d8b449e7e..0f18d6e068 100644 --- a/src/storage/canonical.ts +++ b/src/storage/canonical.ts @@ -53,6 +53,10 @@ function canonicalizeWithContext(value: unknown, key: string | undefined): unkno } return out; } + return canonicalScalar(value, key); +} + +function canonicalScalar(value: unknown, key: string | undefined): unknown { if (key === "biliVersion" && typeof value === "string") return "__VER__"; if (typeof value === "number" && Number.isFinite(value) && value > 1_600_000_000_000) return "__TS__"; if (typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)) return "__UUID__"; @@ -62,3 +66,84 @@ function canonicalizeWithContext(value: unknown, key: string | undefined): unkno export function sha256(s: string): string { return createHash("sha256").update(s, "utf8").digest("hex"); } + +/** Hash a storage view without materializing its canonical copy or JSON string. */ +export function canonicalDigest(value: unknown): string { + if (value === undefined || typeof value === "function" || typeof value === "symbol") return sha256(JSON.stringify(value)); + const hash = createHash("sha256"); + const chunkSize = 64 * 1024; + let pending = ""; + function write(text: string): void { + pending += text; + if (pending.length >= chunkSize) { + hash.update(pending, "utf8"); + pending = ""; + } + } + function writeString(text: string): void { + write('"'); + for (let start = 0; start < text.length;) { + let end = Math.min(start + chunkSize, text.length); + const last = text.charCodeAt(end - 1); + // Keep a surrogate pair together so JSON escaping and UTF-8 stay exact. + if (end < text.length && last >= 0xd800 && last <= 0xdbff) end--; + write(JSON.stringify(text.slice(start, end)).slice(1, -1)); + start = end; + } + write('"'); + } + function visit(current: unknown, key?: string): void { + if (Array.isArray(current)) { + write("["); + for (let i = 0; i < current.length; i++) { + if (i) write(","); + visit(current[i]); + } + write("]"); + } else if (current instanceof Map) { + write('{"__map__":['); + let separator = ""; + for (const [k, v] of current) { + write(separator + "["); + // canonicalize leaves Map keys untouched. + write(JSON.stringify(k) ?? "null"); + write(","); + visit(v); + write("]"); + separator = ","; + } + write("]}"); + } else if (current instanceof Set) { + write('{"__set__":['); + let separator = ""; + for (const v of current) { + write(separator); + visit(v); + separator = ","; + } + write("]}"); + } else if (current && typeof current === "object") { + write("{"); + let separator = ""; + for (const k of Object.keys(current)) { + // Assigning __proto__ to canonicalize's plain output creates no JSON property. + if (k === "storageMode" || k === "tailGaps" || k === "__proto__") continue; + const v = (current as Record)[k]; + if (v === undefined || typeof v === "function" || typeof v === "symbol") continue; + write(separator); + writeString(k); + write(":"); + visit(v, k); + separator = ","; + } + write("}"); + } else { + const normalized = canonicalScalar(current, key); + if (typeof normalized === "string") writeString(normalized); + else write(JSON.stringify(normalized) ?? "null"); + } + } + visit(value); + hash.update(pending, "utf8"); + return hash.digest("hex"); +} diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 8a69ce5dc0..607aa8d71a 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -29,7 +29,7 @@ import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSu import { renameSync } from "node:fs"; import path from "node:path"; import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts"; -import { canonicalize, sha256 } from "./canonical.ts"; +import { canonicalDigest, canonicalize, sha256 } from "./canonical.ts"; import { recordSidecarInstance, recordSidecarOpenFailure } from "./runtime-status.ts"; import type { SqliteEngineName, SqliteEngineSource } from "./driver.ts"; import type { MessageRow, MessageRefContentRow } from "./store.js"; @@ -1225,7 +1225,7 @@ function warnTailGaps(id: string, view: { tailGaps?: number }): void { } function shadowDigest(view: unknown): string { - return sha256(JSON.stringify(canonicalize(view))); + return canonicalDigest(view); } // #3407: the parity digest must compare blockContents CONTENTS, not insertion order — diff --git a/tests/storage-canonical-digest.test.ts b/tests/storage-canonical-digest.test.ts new file mode 100644 index 0000000000..e8c32b3bf7 --- /dev/null +++ b/tests/storage-canonical-digest.test.ts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { canonicalDigest, canonicalize, sha256 } from "../src/storage/canonical.ts"; + +function matchesLegacy(value: unknown): void { + assert.equal(canonicalDigest(value), sha256(JSON.stringify(canonicalize(value)))); +} + +test("streaming digest preserves storage normalization and collection order", () => { + const uuid = "12345678-1234-1234-1234-123456789abc"; + matchesLegacy({ + metadata: { biliVersion: "0.1.204", storageMode: "full", savedAt: 1_700_000_000_000 }, + tailGaps: 3, + state: { nextBlockId: 42, ids: new Set([uuid, "b2"]) }, + blockContents: new Map([ + [uuid, { full: { text: "original" }, traceId: uuid }], + [1_700_000_000_000, [null, true, false, -0, NaN, Infinity]], + [{ storageMode: "raw map key", traceId: uuid }, new Map([["b1", "summary"]])], + ]), + }); + assert.notEqual(canonicalDigest(new Map([["b1", 1], ["b2", 2]])), + canonicalDigest(new Map([["b2", 2], ["b1", 1]]))); +}); + +test("streaming digest preserves JSON omissions, sparse arrays and property order", () => { + matchesLegacy({ z: undefined, f: () => 1, s: Symbol("ignored"), a: [undefined, , Symbol("null")], "10": "ten", "2": "two" }); + matchesLegacy(JSON.parse('{"__proto__":{"ignored":true},"constructor":"kept","value":1}')); + for (const value of [null, true, 0, "text", [], {}, new Map(), new Set()]) matchesLegacy(value); +}); + +test("streaming digest escapes long strings without splitting surrogate pairs", () => { + matchesLegacy({ + text: "x".repeat(65_535) + "\uD83D\uDE80" + "\n\t\u0000\"\\\u4E2D\u6587".repeat(20_000), + loneSurrogates: "\uD800" + "y".repeat(65_534) + "\uDC00", + ["key\"\\\n\uD83D\uDE80"]: "escaped key", + }); +}); + +test("streaming digest changes when retained history changes", () => { + const before = { blockContents: new Map([["b1", { full: { text: "original" } }]]) }; + const after = { blockContents: new Map([["b1", { full: { text: "changed" } }]]) }; + matchesLegacy(before); + matchesLegacy(after); + assert.notEqual(canonicalDigest(before), canonicalDigest(after)); +}); From 41a718f8e328efe384179f224bb5317caaca8aba Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 24 Oct 2026 21:57:59 +0800 Subject: [PATCH 12/25] test(storage): pin streaming digest byte-identity at MB scale and chunk seams (#4529) Extend storage-canonical-digest.test.ts with deterministic cases the original suite left uncovered: surrogate pairs landing exactly on the 64 KiB hash-chunk seam (writeString boundary pull-back), an MB-scale retained history spanning many chunks end-to-end, deep nesting (visit/JSON.stringify recursion parity) and wide fan-out (separator + pending-buffer flush). Each asserts canonicalDigest(v) === sha256(JSON.stringify(canonicalize(v))), so any chunking or surrogate-split regression fails loudly rather than drifting silently. --- tests/storage-canonical-digest.test.ts | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/storage-canonical-digest.test.ts b/tests/storage-canonical-digest.test.ts index e8c32b3bf7..0fea66a29e 100644 --- a/tests/storage-canonical-digest.test.ts +++ b/tests/storage-canonical-digest.test.ts @@ -43,3 +43,40 @@ test("streaming digest changes when retained history changes", () => { matchesLegacy(after); assert.notEqual(canonicalDigest(before), canonicalDigest(after)); }); + +test("streaming digest stays byte-identical across many 64 KiB chunks with pairs on the boundary (#4529)", () => { + const chunk = 64 * 1024; + const unit = "a".repeat(chunk - 1) + "\uD83D\uDE80"; + matchesLegacy({ text: unit }); + matchesLegacy({ text: unit.repeat(5) }); + const seam = "z".repeat(chunk - 6) + "\"\\\n\t\u0000\u4E2D文".repeat(20); + matchesLegacy({ text: seam }); +}); + +test("streaming digest stays byte-identical over an MB-scale retained history (#4529)", () => { + const body = "line \uD83D\uDE80 \u4E2D\u6587 \"quoted\\\" \n\t".repeat(2_000); + const blockContents = new Map(); + for (let i = 0; i < 40; i++) { + blockContents.set("blk-" + i, { full: { text: body + i }, traceId: "aaaaaaaa-bbbb-cccc-dddd-" + i.toString(16).padStart(12, "0") }); + } + matchesLegacy({ + id: "sess-4529", + metadata: { biliVersion: "0.1.206", savedAt: 1_700_000_000_000 }, + state: { nextBlockId: 41, ids: [...blockContents.keys()] }, + blockContents, + }); +}); + +test("streaming digest stays byte-identical under deep nesting and wide fan-out (#4529)", () => { + let nested: unknown = "leaf"; + for (let i = 0; i < 400; i++) nested = i % 2 === 0 ? [nested] : { w: nested }; + matchesLegacy(nested); + const wide: Record = {}; + for (let i = 0; i < 5_000; i++) wide["k" + i] = i % 3 === 0 ? "v" + i : i; + matchesLegacy(wide); + const big = "\uD83D\uDE80\n\"\\中文".repeat(40_000); + matchesLegacy(new Map([ + [{ nested: { deep: big } }, new Set([big, 1_700_000_000_000])], + ["plain", big], + ])); +}); From fe790cb106edb9d24fc5fe69d61bff0ea33af4e7 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:09:26 -0700 Subject: [PATCH 13/25] fix: preserve blog metadata and launcher checks on Windows --- src/web/blogfeed.ts | 9 +++++---- tests/web-blogfeed.test.ts | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/web/blogfeed.ts b/src/web/blogfeed.ts index 494e7627dc..54839ae78a 100644 --- a/src/web/blogfeed.ts +++ b/src/web/blogfeed.ts @@ -106,10 +106,11 @@ export function _setBlogRootForTest(root: string | null): void { /** Mirror of the website tools' frontmatter parser (key: value between --- fences). */ function parseFrontmatter(text: string): { fm: Record; body: string } { const fm: Record = {}; - let body = text; - // Git-for-Windows autocrlf checks .md posts out with CRLF; normalize so - // the exact-match fences below survive Windows checkouts (#4599). - const src = text.startsWith("---\r\n") ? text.replace(/\r\n/g, "\n") : text; + // Git-for-Windows autocrlf checks .md posts out with CRLF; normalize + // UNCONDITIONALLY so the exact-match fences survive Windows checkouts + // (#4599) and non-frontmatter bodies cannot leak \r into rendered HTML. + const src = text.replace(/\r\n/g, "\n"); + let body = src; if (src.startsWith("---\n")) { const end = src.indexOf("\n---\n", 4); if (end !== -1) { diff --git a/tests/web-blogfeed.test.ts b/tests/web-blogfeed.test.ts index fe9a00387c..8f8a61c840 100644 --- a/tests/web-blogfeed.test.ts +++ b/tests/web-blogfeed.test.ts @@ -10,7 +10,7 @@ */ import { test, beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { mkdtempSync, rmSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { ServerResponse } from "node:http"; @@ -94,6 +94,21 @@ test("rendering is escape-first: script tags survive only escaped; tables become assert.ok(alpha.html.includes("two")); }); +test("CRLF markdown preserves bilingual frontmatter and rendered content", () => { + const expected = getBlogFeed(); + for (const relative of [ + "website/blog/posts/2026-10-08_demo.md", + "website/blog/posts/2026-10-08_demo.zh.md", + "paper/a.md", + "paper/b.md", + ]) { + const file = join(pkgRoot, relative); + writeFileSync(file, readFileSync(file, "utf8").replace(/\n/g, "\r\n")); + } + _setBlogRootForTest(pkgRoot); + assert.deepEqual(getBlogFeed(), expected); +}); + test("image refs: shipped files reroute to the panel asset prefix; traversal/unshipped refs drop silently", () => { const data = getBlogFeed(); const demo = data.items.find((i) => i.id === "demo")!; From 8518a73ca5210a01aeb1af6b20f6b5891c7225d1 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:35:44 -0700 Subject: [PATCH 14/25] test: verify fork restart under legacy and full storage --- tests/plugin-fork.test.ts | 155 +++++++++++++++++++------------------- 1 file changed, 79 insertions(+), 76 deletions(-) diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 1c4801377d..7a3130e7d6 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -114,7 +114,7 @@ async function harness(persist = false, seedParent = true, compressExtra: Record assert.equal(r.status, 200); await r.text(); } - return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.cancelAll(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; + return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.closeQuietly(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; } test("plain proxy panel snapshots do not change raw-history persistence", { timeout: 60_000 }, async () => { @@ -742,82 +742,85 @@ for (const corruption of ["missing-ref", "missing-payload", "orphan-payload", "w }); } -test("HTTP fork revision and originals survive a new proxy process", { timeout: 60_000 }, async () => { - const h = await harness(true); - let child: ReturnType | undefined; - try { - await compress(h); - // CI-only race (#4599): the compress fold lands in the live session - // state asynchronously after the tool response (#3573 replay - // semantics). On slow runners flushSync below would otherwise persist - // the pre-fold state and the cold load reports "0 block(s) - // (compressions=0)". Bounded-wait for the fold first. - for (let i = 0; i < 100 && !(resolveConversation("parent").session?.state.blocks?.length ?? 0); i++) { - await new Promise((r) => setTimeout(r, 50)); - } - const parent = resolveConversation("parent").session!; - parent.metadata.effectiveCcr = { ...DEFAULT_CCR_CONFIG, enabled: true }; - parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); - parent.contentStoreDirty = true; - assert(h.store.flushSync(parent)); - // #4316 PR3/3: cold load runs the one-time ledger alignment, but only - // when the unified store is available — config-clean machines (CI) run - // legacy mode where the alignment is an honest skip and the loaded - // state never carries refLedgerAligned. Mirror the mode in the - // expectation instead of asserting the flag unconditionally. - const persistedState = JSON.parse(JSON.stringify(parent.state)); - if (h.store.replicasAvailable()) persistedState.refLedgerAligned = true; - assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); - const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; - const req = forkRequest(snapshot); - const fork = await h.request("/__bili/plugin/fork", req); - assert.equal(fork.status, 201); - const childSnapshot = (await h.request("/__bili/plugin/snapshot?conversationId=child")).body; - const code = `import {startServer} from './src/server.ts'; import {SessionStore,_setStoreForTest} from './src/persist.ts'; import {defaultConfig} from 'acp-kernel'; import {_setForTest} from './src/registry.ts'; _setForTest({}); _setStoreForTest(new SessionStore({dir:${JSON.stringify(h.dir + "/sessions")},enabled:true})); const s=await startServer({port:0,host:'127.0.0.1',upstream:${JSON.stringify(h.upstreamUrl)},modelContextLimit:400000,routes:{},kernelConfig:defaultConfig(400000),compress:{injectTool:true,injectNudge:true},promptCache:{routing:'auto'},sessionHeader:'x-acp-session',log:false,debug:false,passthrough:false,autoUpdate:false,mitm:{enabled:false,domains:[]}}); const ready=()=>process.stdout.write(JSON.stringify({port:s.address().port})+'\\n'); if(s.listening)ready();else s.once('listening',ready);`; - child = spawn(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", code], { cwd: fileURLToPath(new URL("../", import.meta.url)), env: { ...process.env, NODE_ENV: "test" }, stdio: ["ignore", "pipe", "pipe"] }); - let stderr = ""; - child.stderr!.on("data", (b) => { stderr += b.toString(); }); - const port = await new Promise((resolve, reject) => { - let stdout = ""; - const timer = setTimeout(() => reject(new Error("restart timeout: " + stderr + stdout)), 15000); - child!.stdout!.on("data", (b) => { - stdout += b.toString(); - const port = stdout.match(/\{"port":(\d+)\}/); - if (port) { clearTimeout(timer); resolve(Number(port[1])); } +for (const storageMode of ["legacy", "full"] as const) { + test(`HTTP fork revision and originals survive a new proxy process (${storageMode})`, { timeout: 60_000 }, async () => { + const previousMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = storageMode; + const h = await harness(true); + let child: ReturnType | undefined; + try { + await compress(h); + // CI-only race (#4599): the compress fold lands in the live session + // state asynchronously after the tool response (#3573 replay + // semantics). On slow runners flushSync below would otherwise persist + // the pre-fold state and the cold load reports "0 block(s) + // (compressions=0)". Bounded-wait for the fold first. + for (let i = 0; i < 100 && !(resolveConversation("parent").session?.state.blocks?.length ?? 0); i++) { + await new Promise((r) => setTimeout(r, 50)); + } + const parent = resolveConversation("parent").session!; + parent.metadata.effectiveCcr = { ...DEFAULT_CCR_CONFIG, enabled: true }; + parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); + parent.contentStoreDirty = true; + assert(h.store.flushSync(parent)); + // #4316 PR3/3: only a unified cold load runs the one-time ledger + // alignment; the CI legacy baseline retains the original state. + const persistedState = JSON.parse(JSON.stringify(parent.state)); + if (storageMode === "full") persistedState.refLedgerAligned = true; + assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); + const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; + const req = forkRequest(snapshot); + const fork = await h.request("/__bili/plugin/fork", req); + assert.equal(fork.status, 201); + const childSnapshot = (await h.request("/__bili/plugin/snapshot?conversationId=child")).body; + const code = `import {startServer} from './src/server.ts'; import {SessionStore,_setStoreForTest} from './src/persist.ts'; import {defaultConfig} from 'acp-kernel'; import {_setForTest} from './src/registry.ts'; _setForTest({}); _setStoreForTest(new SessionStore({dir:${JSON.stringify(h.dir + "/sessions")},enabled:true})); const s=await startServer({port:0,host:'127.0.0.1',upstream:${JSON.stringify(h.upstreamUrl)},modelContextLimit:400000,routes:{},kernelConfig:defaultConfig(400000),compress:{injectTool:true,injectNudge:true},promptCache:{routing:'auto'},sessionHeader:'x-acp-session',log:false,debug:false,passthrough:false,autoUpdate:false,mitm:{enabled:false,domains:[]}}); const ready=()=>process.stdout.write(JSON.stringify({port:s.address().port})+'\\n'); if(s.listening)ready();else s.once('listening',ready);`; + child = spawn(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", code], { cwd: fileURLToPath(new URL("../", import.meta.url)), env: { ...process.env, NODE_ENV: "test" }, stdio: ["ignore", "pipe", "pipe"] }); + let stderr = ""; + child.stderr!.on("data", (b) => { stderr += b.toString(); }); + const port = await new Promise((resolve, reject) => { + let stdout = ""; + const timer = setTimeout(() => reject(new Error("restart timeout: " + stderr + stdout)), 15000); + child!.stdout!.on("data", (b) => { + stdout += b.toString(); + const port = stdout.match(/\{"port":(\d+)\}/); + if (port) { clearTimeout(timer); resolve(Number(port[1])); } + }); + child!.once("exit", (code) => { clearTimeout(timer); reject(new Error(`restart exited ${code}: ${stderr}`)); }); + child!.once("error", (error) => { clearTimeout(timer); reject(error); }); }); - child!.once("exit", (code) => { clearTimeout(timer); reject(new Error(`restart exited ${code}: ${stderr}`)); }); - child!.once("error", (error) => { clearTimeout(timer); reject(error); }); - }); - const call = async (path: Path, body?: unknown) => { - const r = await fetch(`http://127.0.0.1:${port}` + path, body === undefined ? {} : { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); - return { status: r.status, body: await responseBody(r, path) }; - }; - const restored = await call("/__bili/plugin/snapshot?conversationId=child"); - const restoredStatus = await call("/__bili/plugin/status?conversationId=parent&fallback=latest"); - assert.equal(restoredStatus.status, 200); - assert.equal(restoredStatus.body.sessionId, "parent"); - assert.equal(restoredStatus.body.fallback, undefined); - assert.equal(restoredStatus.body.sessionRevision, snapshot.parentRevision); - assert.equal(restoredStatus.body.contextTokensSource, "estimate"); - assert.equal(restored.status, 200); - assert.equal(restored.body.parentRevision, childSnapshot.parentRevision); - assert.deepEqual(restored.body.orderedMessages, childSnapshot.orderedMessages); - assert.equal((await call("/__bili/plugin/snapshot?conversationId=parent")).body.parentRevision, snapshot.parentRevision); - const replay = await call("/__bili/plugin/fork", req); - assert.equal(replay.status, 200); - assert.equal(replay.body.replayed, true); - const retrieved = await call("/__bili/plugin/tool", { conversationId: "child", tool: "acp_retrieve", args: { ref: "m00001" } }); - assert.equal(retrieved.status, 200); - assert.match(retrieved.body.result, /CCR original retained independently after restart/); - const original = await call("/__bili/plugin/tool", { conversationId: "child", tool: "decompress", args: { blockId: "b1", full: true } }); - assert.match(original.body.result, /first original/); - assert.match(original.body.result, /second original/); - assert(!original.body.result.includes("tail original")); - } finally { - if (child?.pid && child.exitCode === null && child.signalCode === null) { child.kill("SIGTERM"); await once(child, "exit"); } - await h.close(); - } -}); + const call = async (path: Path, body?: unknown) => { + const r = await fetch(`http://127.0.0.1:${port}` + path, body === undefined ? {} : { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + return { status: r.status, body: await responseBody(r, path) }; + }; + const restored = await call("/__bili/plugin/snapshot?conversationId=child"); + const restoredStatus = await call("/__bili/plugin/status?conversationId=parent&fallback=latest"); + assert.equal(restoredStatus.status, 200); + assert.equal(restoredStatus.body.sessionId, "parent"); + assert.equal(restoredStatus.body.fallback, undefined); + assert.equal(restoredStatus.body.sessionRevision, snapshot.parentRevision); + assert.equal(restoredStatus.body.contextTokensSource, "estimate"); + assert.equal(restored.status, 200); + assert.equal(restored.body.parentRevision, childSnapshot.parentRevision); + assert.deepEqual(restored.body.orderedMessages, childSnapshot.orderedMessages); + assert.equal((await call("/__bili/plugin/snapshot?conversationId=parent")).body.parentRevision, snapshot.parentRevision); + const replay = await call("/__bili/plugin/fork", req); + assert.equal(replay.status, 200); + assert.equal(replay.body.replayed, true); + const retrieved = await call("/__bili/plugin/tool", { conversationId: "child", tool: "acp_retrieve", args: { ref: "m00001" } }); + assert.equal(retrieved.status, 200); + assert.match(retrieved.body.result, /CCR original retained independently after restart/); + const original = await call("/__bili/plugin/tool", { conversationId: "child", tool: "decompress", args: { blockId: "b1", full: true } }); + assert.match(original.body.result, /first original/); + assert.match(original.body.result, /second original/); + assert(!original.body.result.includes("tail original")); + } finally { + if (child?.pid && child.exitCode === null && child.signalCode === null) { child.kill("SIGTERM"); await once(child, "exit"); } + await h.close(); + if (previousMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = previousMode; + } + }); +} test("HTTP status separates real usage from a manual-compression estimate and changing generation", { timeout: 60_000 }, async () => { const h = await harness(); From f8d91aa75e53a7dd709198a10d92f7983d42b18f Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:44:01 -0700 Subject: [PATCH 15/25] test(storage): account for one-time ledger ref migration on restart Compare unified reads with an independent envelope reader plus the documented migration. Persist alignment before rollback/corrupt-index checks, select legacy explicitly, and assert the recovered raw ID under its ledger ref. Keep checkpoint writes deterministic so sampled ingest cadence cannot change the seed between reader legs. All 28 targeted storage tests pass; no product migration or digest assertion is disabled. --- tests/storage-engine-proofs.test.ts | 16 ++++++++++------ tests/storage-orphan-recovery.test.ts | 2 ++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index d9eb3334b2..2ef31e8605 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -22,8 +22,8 @@ * IMPORT GUARD — a hand-aged legacy file (older savedAt than the * index watermark) must not re-import over newer index * state on load. - * ROLLBACK — unset the env after running full → the legacy file - * loads with the exact digest. + * ROLLBACK — explicitly select legacy after persisting the full + * load's migration → the file loads the exact digest. * SELF-DISABLE — 3 consecutive sidecar failures disable it; reads flip * back to legacy; the legacy file stays readable. * CORRUPT INDEX — a corrupt index.db degrades the store to legacy-only @@ -281,10 +281,10 @@ test("storage engine: load-side import guard skips a hand-aged legacy file (inde }); // --------------------------------------------------------------------------- -// ROLLBACK: full → unset env → legacy file authoritative, digest exact. +// ROLLBACK: full → explicit legacy → legacy file authoritative, digest exact. // --------------------------------------------------------------------------- -test("storage engine: rollback contract — unset the env and the legacy read path serves the identical state", async () => { +test("storage engine: rollback contract — explicit legacy mode serves the identical persisted state", async () => { const dir = tmpdir("bili-engproof-rollback-"); const prev = process.env.BILI_STORAGE_UNIFIED; try { @@ -305,8 +305,10 @@ test("storage engine: rollback contract — unset the env and the legacy read pa // merely deleting the env would read unified and vacate this leg. const storeFull = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = storeFull.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); - storeFull.closeQuietly(); assert.ok(viaFull, "full-mode reload returned null"); + // Persist the one-time cold-load alignment before exercising rollback. + await storeFull.writeNow(viaFull); + storeFull.closeQuietly(); const digestFull = viewDigest(toLegacyLike(viaFull)); process.env.BILI_STORAGE_UNIFIED = "legacy"; @@ -378,8 +380,10 @@ test("storage engine: corrupt index.db degrades to legacy-only instead of bricki // fallback must reproduce exactly from the legacy file. const clean = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = clean.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); - clean.closeQuietly(); assert.ok(viaFull, "full-mode read returned null"); + // Both replicas must contain the migration before corrupting the index. + await clean.writeNow(viaFull); + clean.closeQuietly(); // #4338: everything past this point (corrupt the header → degrade // construction → fallback read → rename guards → write → close) runs diff --git a/tests/storage-orphan-recovery.test.ts b/tests/storage-orphan-recovery.test.ts index 9fd6f13b0a..ac2d9e08d7 100644 --- a/tests/storage-orphan-recovery.test.ts +++ b/tests/storage-orphan-recovery.test.ts @@ -75,6 +75,8 @@ test("cold load recovers the pre-fix orphan freeze before processing a request", const alignedRef = recovered.state.messageRefs.byRaw["raw-new"]; assert.ok(alignedRef, "raw-new must keep a citation number after recovery"); assert.equal(recovered.state.messageRefs.byRef[alignedRef], "raw-new"); + // The pre-alignment number is recorded as superseded, not silently dropped. + assert.equal(recovered.restampSuperseded?.get("m00003"), "raw-new"); assert.equal(recovered.lastMessages?.at(-1)?.text, "new turn"); assert.equal(recovered.metadata.recoveryMarker, "fresh"); assert.equal(recovered.blockContents.get("b1")?.full.text, "retained original"); From 3a4e2b2a9692cd1160ff9dcd714cfcb789d26b98 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:47:58 -0700 Subject: [PATCH 16/25] test: fix Windows graceful shutdown and fixture teardown Windows SIGTERM terminates the child instead of running its graceful shutdown handler. Trigger the existing loopback shutdown route on Windows so the draining-marker regression exercises the same lifecycle as POSIX. Wait for child close before deleting its database, destroy sockets owned by the silent TCP fixture, and stop health polling when the child exits. Keep both regression scenarios and their behavioral assertions intact. Validation: the isolated CI-preloaded serial test file passes 6/6 and exits naturally in 11.58 seconds; npm run typecheck and npm run build pass. --- tests/issue4982-draining-twin-ladder.test.ts | 50 +++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index 71ea29a279..d0eb94783e 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -228,21 +228,26 @@ async function waitForStatus(port: number, timeoutMs: number): Promise async function killChild(child: ChildProcess): Promise { if (child.exitCode !== null || child.signalCode !== null) return; - child.kill("SIGTERM"); - await Promise.race([ - new Promise((resolve) => child.once("exit", () => resolve())), - new Promise((resolve) => setTimeout(resolve, 10_000)), - ]); - if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 10_000); + child.once("close", () => { + clearTimeout(timer); + resolve(); + }); + child.kill("SIGTERM"); + }); } /** A TCP listener that accepts connections and then says nothing — invisible * to /__bili/health probes (they time out as "no evidence") but still owns * the socket, so binders get EADDRINUSE. Deterministically forces the LATE * EADDRINUSE decision path instead of the pre-bind foreign-owner ladder. */ -function silentTcpListener(port: number): Promise { +function silentTcpListener(port: number): Promise<{ close(): Promise }> { return new Promise((resolve, reject) => { + const sockets = new Set(); const server = net.createServer((sock) => { + sockets.add(sock); + sock.once("close", () => sockets.delete(sock)); // accept and hang — never answer HTTP; drain the probe's bytes so peer // EOF is actually delivered (a paused socket buffers them and never sees // EOF) and release the connection, or a dead prober pins server.close() @@ -251,7 +256,12 @@ function silentTcpListener(port: number): Promise { sock.on("end", () => sock.destroy()); }); server.once("error", reject); - server.listen(port, "127.0.0.1", () => resolve(server)); + server.listen(port, "127.0.0.1", () => resolve({ + close: () => new Promise((done) => { + server.close(() => done()); + for (const socket of sockets) socket.destroy(); + }), + })); }); } @@ -319,6 +329,9 @@ function spawnBili(sandbox: string, port: number): { child: ChildProcess; out: { async function waitUntilLadderedHealthy(stateHome: string, child: ChildProcess, label: string): Promise { let port = 0; await waitUntil(async () => { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error(`${label} exited before becoming healthy`); + } const m = readMarkers(stateHome).find((e) => e.pid === child.pid); if (!m || typeof m.port !== "number") return false; port = m.port as number; @@ -335,7 +348,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders let p1Ref: ReturnType | null = null; let p2Ref: ReturnType | null = null; let heldRef: net.Socket | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // P1: the serving proxy that will enter graceful shutdown. const p1 = spawnBili(sandbox, port); @@ -353,7 +366,18 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders }); heldRef = held; - p1.child.kill("SIGTERM"); + if (process.platform === "win32") { + // Windows SIGTERM terminates immediately instead of invoking the + // handler. The loopback admin route enters the same graceful path. + const response = await fetch(`http://127.0.0.1:${port}/__bili/shutdown`, { + method: "POST", + signal: AbortSignal.timeout(2_000), + }); + assert.equal(response.status, 202, "P1 must accept the graceful shutdown request"); + await response.arrayBuffer(); + } else { + p1.child.kill("SIGTERM"); + } const p1MarkerFile = path.join(stateHome, "billion-context", "instances", `${p1Marker.instanceId}.json`); await waitUntil(async () => { let stamped = false; @@ -395,7 +419,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders heldRef?.destroy(); if (p1Ref) await killChild(p1Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); @@ -406,7 +430,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n const port = await freePort(); await assertPortDead(port, { label: "4982 unverified-twin port" }); let p2Ref: ReturnType | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // A registry marker claiming `port` with a LIVE pid and full identity // match — but NO portReleasedAt stamp (pre-fix build, or a failed @@ -456,7 +480,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n } finally { if (p2Ref) await killChild(p2Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); From 858cb701a5720186622702abe5e9d4e6eeff3b92 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:47:58 -0700 Subject: [PATCH 17/25] test: fix Windows graceful shutdown and fixture teardown Windows SIGTERM terminates the child instead of running its graceful shutdown handler. Trigger the existing loopback shutdown route on Windows so the draining-marker regression exercises the same lifecycle as POSIX. Wait for child close before deleting its database, destroy sockets owned by the silent TCP fixture, and stop health polling when the child exits. Keep both regression scenarios and their behavioral assertions intact. Validation: the isolated CI-preloaded serial test file passes 6/6 and exits naturally in 11.58 seconds; npm run typecheck and npm run build pass. --- tests/issue4982-draining-twin-ladder.test.ts | 50 +++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index 71ea29a279..d0eb94783e 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -228,21 +228,26 @@ async function waitForStatus(port: number, timeoutMs: number): Promise async function killChild(child: ChildProcess): Promise { if (child.exitCode !== null || child.signalCode !== null) return; - child.kill("SIGTERM"); - await Promise.race([ - new Promise((resolve) => child.once("exit", () => resolve())), - new Promise((resolve) => setTimeout(resolve, 10_000)), - ]); - if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 10_000); + child.once("close", () => { + clearTimeout(timer); + resolve(); + }); + child.kill("SIGTERM"); + }); } /** A TCP listener that accepts connections and then says nothing — invisible * to /__bili/health probes (they time out as "no evidence") but still owns * the socket, so binders get EADDRINUSE. Deterministically forces the LATE * EADDRINUSE decision path instead of the pre-bind foreign-owner ladder. */ -function silentTcpListener(port: number): Promise { +function silentTcpListener(port: number): Promise<{ close(): Promise }> { return new Promise((resolve, reject) => { + const sockets = new Set(); const server = net.createServer((sock) => { + sockets.add(sock); + sock.once("close", () => sockets.delete(sock)); // accept and hang — never answer HTTP; drain the probe's bytes so peer // EOF is actually delivered (a paused socket buffers them and never sees // EOF) and release the connection, or a dead prober pins server.close() @@ -251,7 +256,12 @@ function silentTcpListener(port: number): Promise { sock.on("end", () => sock.destroy()); }); server.once("error", reject); - server.listen(port, "127.0.0.1", () => resolve(server)); + server.listen(port, "127.0.0.1", () => resolve({ + close: () => new Promise((done) => { + server.close(() => done()); + for (const socket of sockets) socket.destroy(); + }), + })); }); } @@ -319,6 +329,9 @@ function spawnBili(sandbox: string, port: number): { child: ChildProcess; out: { async function waitUntilLadderedHealthy(stateHome: string, child: ChildProcess, label: string): Promise { let port = 0; await waitUntil(async () => { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error(`${label} exited before becoming healthy`); + } const m = readMarkers(stateHome).find((e) => e.pid === child.pid); if (!m || typeof m.port !== "number") return false; port = m.port as number; @@ -335,7 +348,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders let p1Ref: ReturnType | null = null; let p2Ref: ReturnType | null = null; let heldRef: net.Socket | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // P1: the serving proxy that will enter graceful shutdown. const p1 = spawnBili(sandbox, port); @@ -353,7 +366,18 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders }); heldRef = held; - p1.child.kill("SIGTERM"); + if (process.platform === "win32") { + // Windows SIGTERM terminates immediately instead of invoking the + // handler. The loopback admin route enters the same graceful path. + const response = await fetch(`http://127.0.0.1:${port}/__bili/shutdown`, { + method: "POST", + signal: AbortSignal.timeout(2_000), + }); + assert.equal(response.status, 202, "P1 must accept the graceful shutdown request"); + await response.arrayBuffer(); + } else { + p1.child.kill("SIGTERM"); + } const p1MarkerFile = path.join(stateHome, "billion-context", "instances", `${p1Marker.instanceId}.json`); await waitUntil(async () => { let stamped = false; @@ -395,7 +419,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders heldRef?.destroy(); if (p1Ref) await killChild(p1Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); @@ -406,7 +430,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n const port = await freePort(); await assertPortDead(port, { label: "4982 unverified-twin port" }); let p2Ref: ReturnType | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // A registry marker claiming `port` with a LIVE pid and full identity // match — but NO portReleasedAt stamp (pre-fix build, or a failed @@ -456,7 +480,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n } finally { if (p2Ref) await killChild(p2Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); From 8c03ff1e6ef1560870d9ab07858b4d4be5d54f33 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:47:58 -0700 Subject: [PATCH 18/25] test: fix Windows graceful shutdown and fixture teardown Windows SIGTERM terminates the child instead of running its graceful shutdown handler. Trigger the existing loopback shutdown route on Windows so the draining-marker regression exercises the same lifecycle as POSIX. Wait for child close before deleting its database, destroy sockets owned by the silent TCP fixture, and stop health polling when the child exits. Keep both regression scenarios and their behavioral assertions intact. Validation: the isolated CI-preloaded serial test file passes 6/6 and exits naturally in 11.58 seconds; npm run typecheck and npm run build pass. --- tests/issue4982-draining-twin-ladder.test.ts | 50 +++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index 71ea29a279..d0eb94783e 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -228,21 +228,26 @@ async function waitForStatus(port: number, timeoutMs: number): Promise async function killChild(child: ChildProcess): Promise { if (child.exitCode !== null || child.signalCode !== null) return; - child.kill("SIGTERM"); - await Promise.race([ - new Promise((resolve) => child.once("exit", () => resolve())), - new Promise((resolve) => setTimeout(resolve, 10_000)), - ]); - if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 10_000); + child.once("close", () => { + clearTimeout(timer); + resolve(); + }); + child.kill("SIGTERM"); + }); } /** A TCP listener that accepts connections and then says nothing — invisible * to /__bili/health probes (they time out as "no evidence") but still owns * the socket, so binders get EADDRINUSE. Deterministically forces the LATE * EADDRINUSE decision path instead of the pre-bind foreign-owner ladder. */ -function silentTcpListener(port: number): Promise { +function silentTcpListener(port: number): Promise<{ close(): Promise }> { return new Promise((resolve, reject) => { + const sockets = new Set(); const server = net.createServer((sock) => { + sockets.add(sock); + sock.once("close", () => sockets.delete(sock)); // accept and hang — never answer HTTP; drain the probe's bytes so peer // EOF is actually delivered (a paused socket buffers them and never sees // EOF) and release the connection, or a dead prober pins server.close() @@ -251,7 +256,12 @@ function silentTcpListener(port: number): Promise { sock.on("end", () => sock.destroy()); }); server.once("error", reject); - server.listen(port, "127.0.0.1", () => resolve(server)); + server.listen(port, "127.0.0.1", () => resolve({ + close: () => new Promise((done) => { + server.close(() => done()); + for (const socket of sockets) socket.destroy(); + }), + })); }); } @@ -319,6 +329,9 @@ function spawnBili(sandbox: string, port: number): { child: ChildProcess; out: { async function waitUntilLadderedHealthy(stateHome: string, child: ChildProcess, label: string): Promise { let port = 0; await waitUntil(async () => { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error(`${label} exited before becoming healthy`); + } const m = readMarkers(stateHome).find((e) => e.pid === child.pid); if (!m || typeof m.port !== "number") return false; port = m.port as number; @@ -335,7 +348,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders let p1Ref: ReturnType | null = null; let p2Ref: ReturnType | null = null; let heldRef: net.Socket | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // P1: the serving proxy that will enter graceful shutdown. const p1 = spawnBili(sandbox, port); @@ -353,7 +366,18 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders }); heldRef = held; - p1.child.kill("SIGTERM"); + if (process.platform === "win32") { + // Windows SIGTERM terminates immediately instead of invoking the + // handler. The loopback admin route enters the same graceful path. + const response = await fetch(`http://127.0.0.1:${port}/__bili/shutdown`, { + method: "POST", + signal: AbortSignal.timeout(2_000), + }); + assert.equal(response.status, 202, "P1 must accept the graceful shutdown request"); + await response.arrayBuffer(); + } else { + p1.child.kill("SIGTERM"); + } const p1MarkerFile = path.join(stateHome, "billion-context", "instances", `${p1Marker.instanceId}.json`); await waitUntil(async () => { let stamped = false; @@ -395,7 +419,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders heldRef?.destroy(); if (p1Ref) await killChild(p1Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); @@ -406,7 +430,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n const port = await freePort(); await assertPortDead(port, { label: "4982 unverified-twin port" }); let p2Ref: ReturnType | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // A registry marker claiming `port` with a LIVE pid and full identity // match — but NO portReleasedAt stamp (pre-fix build, or a failed @@ -456,7 +480,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n } finally { if (p2Ref) await killChild(p2Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); From 3f7818739f2216e4e92394cef64d88b12fcd16a5 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:11:26 -0700 Subject: [PATCH 19/25] test: handle reset health probes in silent TCP fixture Windows CI can reset an aborted probe connection instead of ending it. The silent TCP fixture did not handle errors on accepted sockets, so ECONNRESET became an uncaught test exception after the probe finished. Destroy reset sockets and add a real resetAndDestroy regression against the same fixture. Keep the existing port-owner and ladder assertions. Validation: the new regression reproduces ECONNRESET before the fix; the isolated CI-preloaded serial test file then passes 7/7 and exits naturally. Pure tsc and git diff --check pass. --- tests/issue4982-draining-twin-ladder.test.ts | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index d0eb94783e..5c8e178cce 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -254,6 +254,9 @@ function silentTcpListener(port: number): Promise<{ close(): Promise }> { // with CLOSE-WAIT sockets sock.resume(); sock.on("end", () => sock.destroy()); + // An aborted health probe may reset instead of ending the connection. + // Socket errors belong to this silent fixture, not the enclosing test. + sock.on("error", () => sock.destroy()); }); server.once("error", reject); server.listen(port, "127.0.0.1", () => resolve({ @@ -265,6 +268,25 @@ function silentTcpListener(port: number): Promise<{ close(): Promise }> { }); } +test("silent TCP fixture survives an aborted health probe", { timeout: 5_000 }, async () => { + const port = await freePort(); + const fixture = await silentTcpListener(port); + const client = net.connect({ host: "127.0.0.1", port }); + try { + await new Promise((resolve, reject) => { + client.once("connect", resolve); + client.once("error", reject); + }); + client.write("GET /__bili/health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"); + await new Promise((resolve) => setTimeout(resolve, 25)); + client.resetAndDestroy(); + await new Promise((resolve) => setTimeout(resolve, 25)); + } finally { + client.destroy(); + await fixture.close(); + } +}); + function tcpRefused(port: number): Promise { return new Promise((resolve) => { const s = net.connect({ port, host: "127.0.0.1" }); From 47b54b36aaf82fcd6b73e6cb3c55e1901bc89d3f Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:11:26 -0700 Subject: [PATCH 20/25] test: handle reset health probes in silent TCP fixture Windows CI can reset an aborted probe connection instead of ending it. The silent TCP fixture did not handle errors on accepted sockets, so ECONNRESET became an uncaught test exception after the probe finished. Destroy reset sockets and add a real resetAndDestroy regression against the same fixture. Keep the existing port-owner and ladder assertions. Validation: the new regression reproduces ECONNRESET before the fix; the isolated CI-preloaded serial test file then passes 7/7 and exits naturally. Pure tsc and git diff --check pass. --- tests/issue4982-draining-twin-ladder.test.ts | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index d0eb94783e..5c8e178cce 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -254,6 +254,9 @@ function silentTcpListener(port: number): Promise<{ close(): Promise }> { // with CLOSE-WAIT sockets sock.resume(); sock.on("end", () => sock.destroy()); + // An aborted health probe may reset instead of ending the connection. + // Socket errors belong to this silent fixture, not the enclosing test. + sock.on("error", () => sock.destroy()); }); server.once("error", reject); server.listen(port, "127.0.0.1", () => resolve({ @@ -265,6 +268,25 @@ function silentTcpListener(port: number): Promise<{ close(): Promise }> { }); } +test("silent TCP fixture survives an aborted health probe", { timeout: 5_000 }, async () => { + const port = await freePort(); + const fixture = await silentTcpListener(port); + const client = net.connect({ host: "127.0.0.1", port }); + try { + await new Promise((resolve, reject) => { + client.once("connect", resolve); + client.once("error", reject); + }); + client.write("GET /__bili/health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"); + await new Promise((resolve) => setTimeout(resolve, 25)); + client.resetAndDestroy(); + await new Promise((resolve) => setTimeout(resolve, 25)); + } finally { + client.destroy(); + await fixture.close(); + } +}); + function tcpRefused(port: number): Promise { return new Promise((resolve) => { const s = net.connect({ port, host: "127.0.0.1" }); From aec79ef4181a2f477d65b527ce6ac8326a932ff9 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:11:26 -0700 Subject: [PATCH 21/25] test: handle reset health probes in silent TCP fixture Windows CI can reset an aborted probe connection instead of ending it. The silent TCP fixture did not handle errors on accepted sockets, so ECONNRESET became an uncaught test exception after the probe finished. Destroy reset sockets and add a real resetAndDestroy regression against the same fixture. Keep the existing port-owner and ladder assertions. Validation: the new regression reproduces ECONNRESET before the fix; the isolated CI-preloaded serial test file then passes 7/7 and exits naturally. Pure tsc and git diff --check pass. --- tests/issue4982-draining-twin-ladder.test.ts | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index d0eb94783e..5c8e178cce 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -254,6 +254,9 @@ function silentTcpListener(port: number): Promise<{ close(): Promise }> { // with CLOSE-WAIT sockets sock.resume(); sock.on("end", () => sock.destroy()); + // An aborted health probe may reset instead of ending the connection. + // Socket errors belong to this silent fixture, not the enclosing test. + sock.on("error", () => sock.destroy()); }); server.once("error", reject); server.listen(port, "127.0.0.1", () => resolve({ @@ -265,6 +268,25 @@ function silentTcpListener(port: number): Promise<{ close(): Promise }> { }); } +test("silent TCP fixture survives an aborted health probe", { timeout: 5_000 }, async () => { + const port = await freePort(); + const fixture = await silentTcpListener(port); + const client = net.connect({ host: "127.0.0.1", port }); + try { + await new Promise((resolve, reject) => { + client.once("connect", resolve); + client.once("error", reject); + }); + client.write("GET /__bili/health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"); + await new Promise((resolve) => setTimeout(resolve, 25)); + client.resetAndDestroy(); + await new Promise((resolve) => setTimeout(resolve, 25)); + } finally { + client.destroy(); + await fixture.close(); + } +}); + function tcpRefused(port: number): Promise { return new Promise((resolve) => { const s = net.connect({ port, host: "127.0.0.1" }); From 7ece6e642e980d1552ca31792556def47308bd95 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Fri, 30 Oct 2026 00:04:06 +0800 Subject: [PATCH 22/25] =?UTF-8?q?perf:=20=E5=A2=9E=E9=87=8F=E8=BD=A6?= =?UTF-8?q?=E9=81=93=E6=AF=8F=E8=BD=AE=E4=B8=8D=E5=86=8D=E9=87=8D=E6=B8=B2?= =?UTF-8?q?=E6=9F=93=E5=85=A8=E5=8E=86=E5=8F=B2=E3=80=81=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E8=90=BD=E7=9B=98=20(#5155)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 组装体只渲染新增 delta: 前缀的已序列化字节按边界谓词复用, sentinel 拼接保持字节一致 - canonical 指针记录改窗口化 + 集群后缀容忍(重复内容行是 _, 旧全表精确匹配永远失配) - wrap-up 用防抖 scheduleSave 替代每轮 writeNow 全量 fsync, 指针在 flush 后台收敛 - store.getMessages 支持 fromSeqExclusive 只取窗口行 24MB 会话 120 轮基准: 每轮 wall 685->551ms, cpu 731->583ms (后半段, -20%) --- src/incremental-lane.ts | 193 +++++++++++--- src/persist.ts | 43 ++- src/storage/shadow.ts | 4 +- src/storage/store.ts | 15 +- tests/incremental-lane.test.ts | 17 +- tests/issue5155-incremental-per-turn.test.ts | 261 +++++++++++++++++++ 6 files changed, 485 insertions(+), 48 deletions(-) create mode 100644 tests/issue5155-incremental-per-turn.test.ts diff --git a/src/incremental-lane.ts b/src/incremental-lane.ts index fb2a0a14be..fdd50e217f 100644 --- a/src/incremental-lane.ts +++ b/src/incremental-lane.ts @@ -171,7 +171,7 @@ const PASSTHROUGH_HEADERS = ["authorization", "anthropic-version", "anthropic-be /** Canonical prefix per conversation, in memory. Bounded LRU: cold entries * rebuild from the unified messages table on demand (see loadCanonical). */ const CANONICAL_LIMIT = 16; -interface Canonical { +export interface Canonical { sessionId: string; msgs: BiliMessage[]; marks: Map; @@ -183,6 +183,14 @@ interface Canonical { * bytes to hash and reports null; clients treat a null boundary as * "cannot adopt, keep full resend" and self-heal on the next turn. */ boundary: string | null; + /** #5155: pre-serialized anthropic items of the rendered message array + * (items JSON.stringify-joined with ","), so a turn renders only its + * delta and splices it onto the prefix. null while the history carries + * raw system messages (the heal pass is position-dependent) or after a + * resync. marksVersion guards against a mark landing on a message the + * prefix already rendered. */ + renderPrefix: { items: string; len: number; marksVersion: number } | null; + marksVersion: number; } const canonicalCache = new Map(); const inFlight = new Set(); @@ -310,6 +318,66 @@ function readBaseSeq(sessionId: string): number { * base-seq count fallback); an array = pointer replay (possibly empty — an * explicitly empty canonical); anything else = malformed → treated as * legacy (the next successful turn rewrites them). */ +/** #5155: coreToAnthropic merges runs of consecutive same-role messages + * into one output item and flushes around raw system messages, so a prefix + * cut at i is reusable only when i-1/i sit on an item boundary: a raw + * message on either side, or a role change (runs would otherwise merge + * across the cut in the full render but not when the halves are rendered + * apart). */ +export function laneRenderBoundaryFlushes(msgs: BiliMessage[], i: number): boolean { + if (i <= 0 || i >= msgs.length) return false; + const a = msgs[i - 1]!; + const b = msgs[i]!; + if ((a as { rawAnthropicMessage?: unknown }).rawAnthropicMessage !== undefined) return true; + if ((b as { rawAnthropicMessage?: unknown }).rawAnthropicMessage !== undefined) return true; + const target = (m: BiliMessage): "assistant" | "user" => (m.role === "assistant" ? "assistant" : "user"); + return target(a) !== target(b); +} + +/** #5155: serialized anthropic items for this turn's assembled body — the + * cached prefix spliced with the freshly rendered delta when the cut is a + * boundary and the mark version still matches, otherwise a full render + * that also refreshes the cache. Byte-identical to stringifying + * coreToAnthropic(msgs, marks) whole (splice joins on ",", exactly what the + * array serializer emits between items). */ +export function laneItemsForTurn(entry: Canonical, cut: number): string { + const msgs = entry.msgs; + const prefix = entry.renderPrefix; + const anyRaw = msgs.some((m) => (m as { rawAnthropicMessage?: unknown }).rawAnthropicMessage !== undefined); + if (!anyRaw && prefix !== null && prefix.len === cut && prefix.marksVersion === entry.marksVersion && (cut === msgs.length || laneRenderBoundaryFlushes(msgs, cut))) { + const deltaMsgs = msgs.slice(cut); + let items: string; + if (deltaMsgs.length === 0) { + items = prefix.items; + } else { + const deltaStr = coreToAnthropic(deltaMsgs, entry.marks).map((it) => JSON.stringify(it)).join(","); + items = prefix.items.length === 0 ? deltaStr : prefix.items + "," + deltaStr; + } + entry.renderPrefix = { items, len: msgs.length, marksVersion: entry.marksVersion }; + return items; + } + const items = coreToAnthropic(msgs, entry.marks).map((it) => JSON.stringify(it)).join(","); + entry.renderPrefix = anyRaw ? null : { items, len: msgs.length, marksVersion: entry.marksVersion }; + return items; +} + +/** #5155: splice pre-serialized items into the body skeleton. The sentinel + * rides the messages array so its serialized form pins the exact splice + * point; if the sentinel somehow collides with string values in the + * skeleton (forwarded passthrough fields), fall back to a plain full + * stringify — correct, just slower. */ +const LANE_RENDER_SENTINEL = "\u0000bili-splice-anchor\uE000"; + +export function laneBodyBuffer(skeleton: Record, itemsStr: string): Buffer { + const withSentinel = JSON.stringify({ ...skeleton, messages: [LANE_RENDER_SENTINEL] }); + const needle = JSON.stringify(LANE_RENDER_SENTINEL); + const at = withSentinel.indexOf(needle); + if (at >= 0 && withSentinel.indexOf(needle, at + 1) < 0) { + return Buffer.from(withSentinel.slice(0, at) + itemsStr + withSentinel.slice(at + needle.length)); + } + return Buffer.from(JSON.stringify({ ...skeleton, messages: JSON.parse(`[${itemsStr}]`) as unknown[] })); +} + function readCanonicalPointers(sessionId: string): number[] | null { let v: unknown; try { @@ -326,40 +394,72 @@ function readCanonicalPointers(sessionId: string): number[] | null { * content is the PIPELINE's serialization (prepare re-derives core from the * assembled wire body), which re-stamps presentation fields like * wireIndex by full-body position - verbatim JSON equality would miss. - * Content-derived ids (plus the cluster suffix) are unique, so - * [id, role, contentType, text, media size] pins the exact row. */ + * Content-derived ids are unique per content, but the in-memory canonical + * keeps the base id while the store clusters duplicate rows as + * `_` (see CLUSTER_ID_SUFFIX_RE), so exact id equality only + * pins the first occurrence. */ +const CLUSTER_ID_SUFFIX_RE = /^(\["h_[0-9a-f]{16})_\d+/; function stableMessageKey(m: Record): string { const text = typeof m.text === "string" ? m.text : typeof m.content === "string" ? m.content : null; const media = typeof m.imageBase64 === "string" ? m.imageBase64.length : null; return JSON.stringify([m.id, m.role, m.contentType, text, media]); } -/** #3798: after a turn's raw tail has settled into the unified table, pin - * the canonical to the exact rows that carry it. Matching uses the stable - * projection (see stableMessageKey) — the row content is the pipeline's - * serialization, which re-stamps presentation fields like wireIndex by - * full-body position, so verbatim JSON equality would miss. Any miss → - * leave the previous pointers untouched (never a partial list) and warn; - * the next settled turn rewrites them. */ +/** #3798/#5155: after a turn, pin the canonical to the exact rows that + * carry it. Matching uses the stable projection (see stableMessageKey) — + * the row content is the pipeline's serialization, which re-stamps + * presentation fields like wireIndex by full-body position, so verbatim + * JSON equality would miss. #5155 windowed: only rows strictly above the + * highest recorded pointer can hold the new tail, so upkeep parses a + * handful of fresh rows instead of walking the whole table every turn. + * Any miss → leave the previous pointers untouched (never a partial list); + * with the debounced save this is the common case right after a turn — a + * later settled turn records them. */ function recordCanonicalPointers(sessionId: string, msgs: BiliMessage[], log: (l: string, m: string) => void): void { try { const store = getStore(); - const rows = store.unifiedRawMessages(sessionId); + const prevPtrs = readCanonicalPointers(sessionId); + let ptrs: number[]; + let rows: { seq: number; content: string }[] | null; + if (prevPtrs !== null && prevPtrs.length <= msgs.length) { + ptrs = [...prevPtrs]; + rows = store.unifiedRawMessages(sessionId, ptrs.length > 0 ? Math.max(...ptrs) : -1); + } else { + ptrs = []; + rows = store.unifiedRawMessages(sessionId); + } if (rows === null) return; - const byStable = new Map(); + // Duplicate-content messages are clustered by the content store under + // `_` row ids while the in-memory canonical keeps the base + // id, so an exact-key lookup misses on every occurrence past the first + // (e.g. repeated identical tool args). Match on the cluster-stripped + // key and hand out settled rows in seq order: same stripped key means + // identical role/contentType/text/media, so any of the clustered rows + // materializes that canonical message byte-identically. + const byStable = new Map(); for (const r of rows) { try { - byStable.set(stableMessageKey(JSON.parse(r.content) as Record), r.seq); + const key = stableMessageKey(JSON.parse(r.content) as Record).replace(CLUSTER_ID_SUFFIX_RE, "$1"); + const bucket = byStable.get(key); + if (bucket === undefined) byStable.set(key, [r.seq]); + else bucket.push(r.seq); } catch { // unparseable row -> skip; a miss keeps the previous pointers } } - const ptrs: number[] = []; - for (const m of msgs) { - const seq = byStable.get(stableMessageKey(m as unknown as Record)); + for (let i = ptrs.length; i < msgs.length; i++) { + const m = msgs[i]!; + const bucket = byStable.get(stableMessageKey(m as unknown as Record).replace(CLUSTER_ID_SUFFIX_RE, "$1")); + const seq = bucket === undefined ? undefined : bucket.shift(); if (seq === undefined) { - log("warn", `[incremental] canonical pointers not recorded for ${sessionId}: a message row is not settled yet — cold rebuilds use the legacy base-seq slice until the next turn`); - return; + // #5155: with the debounced save the unsettled tail is the + // common case right after a turn — record the settled prefix + // (progress beats all-or-nothing now that pointer freshness + // is decoupled from the ack) and let the next flush/turn + // finish the list. A settled prefix is always a correct + // prefix of the canonical; a cold rebuild in the window + // serves a short history and the client resyncs (409). + break; } ptrs.push(seq); // #4316 PR2/3: while walking the settled rows anyway, stamp the @@ -370,6 +470,10 @@ function recordCanonicalPointers(sessionId: string, msgs: BiliMessage[], log: (l // 1-based, same convention as parseCanonicalRows). m.ledgerSeq = seq + 1; } + if (ptrs.length === 0) { + log("debug", `[incremental] canonical pointers not recorded for ${sessionId}: no message rows settled yet (debounced save) — retrying on the next flush`); + return; + } store.setSessionExt(sessionId, EXT_KEY_CANONICAL_PTRS, ptrs); } catch (err) { log("warn", `[incremental] canonical pointers not recorded for ${sessionId}: ${String(err)}`); @@ -480,7 +584,14 @@ function applyTurn( } const base = resync ? [] : entry.msgs; entry.msgs = [...base, ...deltaMsgs]; + const marksTouched = flat.cacheControls.size > 0; for (const [id, mark] of flat.cacheControls) entry.marks.set(id, mark); + // #5155: a mark can legally target an id the prefix already rendered + // (content-derived id collision with a re-sent message), so any cache + // control invalidates the pre-serialized prefix; resync replaces the + // history wholesale. + if (marksTouched) entry.marksVersion++; + if (resync) entry.renderPrefix = null; entry.boundary = deltaWire.length > 0 ? hashWireBoundary(deltaWire[deltaWire.length - 1]) : (resync ? null : entry.boundary); @@ -661,8 +772,9 @@ export async function handleIncrementalTurn( log("warn", `[incremental] canonical rebuild failed for ${sessionId}: ${rebuilt.err} — requesting client resync`); return jsonReply(res, 409, { error: "seq-mismatch", serverSeq: 0, reason: rebuilt.err, conversationId }); } - entry = { sessionId, msgs: rebuilt.msgs, marks: loadMarks(sessionId), system: loadSystem(sessionId), systemSet: false, boundary: null }; + entry = { sessionId, msgs: rebuilt.msgs, marks: loadMarks(sessionId), system: loadSystem(sessionId), systemSet: false, boundary: null, renderPrefix: null, marksVersion: 0 }; } + const msgsBefore = entry.msgs.length; const applied = applyTurn(entry, sinceSeq, resync, deltaWire as never, "system" in payload, deltaIds); if (!applied.ok) { touchCache(conversationId, entry); @@ -677,7 +789,10 @@ export async function handleIncrementalTurn( if (resync) await recordResyncBaseSeq(sessionId, log); persistLaneState(sessionId, entry.system, entry.systemSet, entry.marks, log); - const body: Record = { ...forwarded, messages: coreToAnthropic(entry.msgs, entry.marks) as never }; + const body: Record = { ...forwarded }; + // #5155: the messages array is spliced from the pre-serialized prefix + // (#5155 laneItemsForTurn) — only the delta rides the renderer. + const itemsStr = laneItemsForTurn(entry, resync ? 0 : msgsBefore); if (entry.system !== undefined && entry.system !== null) body.system = entry.system; // #4491: the assembled prefix re-enters the pipeline as a fresh body — // stamp the canonical ids so prepare pins exactly what the lane owns @@ -706,7 +821,7 @@ export async function handleIncrementalTurn( const seqs = entry.msgs.map((m) => m.ledgerSeq); if (seqs.every((s) => typeof s === "number")) body[MSG_SEQS_BODY_FIELD] = seqs; } - const bodyBuf = Buffer.from(JSON.stringify(body)); + const bodyBuf = laneBodyBuffer(body, itemsStr); const headers: Record = { "content-type": "application/json", @@ -781,15 +896,33 @@ export async function handleIncrementalTurn( inFlight.delete(conversationId); const session = peekSession(sessionId); settled = true; - void (session - ? getStore().writeNow(session).then(() => recordCanonicalPointers(sessionId, entry.msgs, log)) - : Promise.resolve() - ) - .catch(() => undefined) - .finally(() => { - releaseChain?.(); - if (pendingPtrWork.get(conversationId) === chain) pendingPtrWork.delete(conversationId); - }); + // #5155: the debounced save coalesces consecutive turns' full-envelope + // flushes (a synchronous writeNow per turn serialized + fsynced the + // whole ~24MB envelope on large histories); pointer upkeep no longer + // waits on the flush — rows that have not settled yet leave the + // previous pointers in place and a later settled turn records them. + // A crash inside the window costs a 409 resync (the contract the lane + // already blesses), never a wrong history. + try { + if (session) getStore().scheduleSave(session); + } catch { + // best-effort, same failure posture as the old writeNow chain + } + // #5155: rows settle in the debounced flush, so a synchronous + // pointer attempt here would re-walk the tail for a guaranteed miss; + // afterNextFlush resolves immediately when no flush is pending, which + // converges right after an already-settled turn all the same. + // This floats BEHIND the chain release: the ack is never delayed by + // pointer freshness, and a crash in the window costs a 409 resync. + void getStore() + .afterNextFlush(sessionId) + .then(() => { + const live = canonicalCache.get(conversationId); + if (live && live.sessionId === sessionId) recordCanonicalPointers(sessionId, live.msgs, log); + }) + .catch(() => undefined); + releaseChain?.(); + if (pendingPtrWork.get(conversationId) === chain) pendingPtrWork.delete(conversationId); } finally { // Exception path (never entered the wrap-up): release everything. if (!settled) { diff --git a/src/persist.ts b/src/persist.ts index 10be0e6c82..b40f530c8b 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -412,6 +412,8 @@ export class SessionStore { * per-turn debounced save unstamped and killed the ownership fast path. */ private readonly saveTimers = new Map>(); private readonly saveBuilders = new Map PersistedSession>(); + /** #5155: afterNextFlush waiters, drained when the armed save lands. */ + private readonly flushWaiters = new Map void>>(); private readonly debounceMs: number; constructor(opts?: { dir?: string; debounceMs?: number; enabled?: boolean; log?: Logger }) { @@ -1869,10 +1871,10 @@ export class SessionStore { this.liveShadow().vacuum(mode); } - unifiedRawMessages(sessionId: string): { seq: number; content: string }[] | null { + unifiedRawMessages(sessionId: string, fromSeqExclusive?: number): { seq: number; content: string }[] | null { if (!this.shadow || this.shadow.stats.disabled) return null; try { - return this.shadow.rawMessages(sessionId); + return this.shadow.rawMessages(sessionId, fromSeqExclusive); } catch { return null; // sidecar degraded mid-call: treat as unavailable } @@ -2571,6 +2573,8 @@ export class SessionStore { // escaping this microtask is fatal in embedded/plugin hosts // without process-level handlers; the next markDirty retries. this.log("error", `[persist] writerOff coalesced build failed for ${session.id}: ${String(err)}`); + } finally { + this.drainFlushWaiters(session.id); } }; this.writerOffPending.set(session.id, run); @@ -2593,12 +2597,44 @@ export class SessionStore { // keeps a failed timer write from becoming an unhandledRejection. void this.store.writeNow(id, build).then(() => { this.stampOwnWrite(id); // #3406: the write LANDED — prove our bytes are on disk - }).catch(() => {}); + }).catch(() => {}).finally(() => { + // #5155: settle waiters even on failure — waiting on a flush + // must never wedge a caller past a failed write (the next + // turn re-arms the whole path anyway). + this.drainFlushWaiters(id); + }); }, this.debounceMs); timer.unref?.(); this.saveTimers.set(id, timer); } + /** #5155: resolve once the currently armed save for this session has + * landed (or immediately when nothing is pending). Background upkeep — + * like the incremental lane's pointer recording — needs a "rows are in + * the table now" signal without paying a synchronous writeNow. */ + afterNextFlush(sessionId: string): Promise { + if (!this.saveTimers.has(sessionId) && !this.writerOffPending.has(sessionId)) return Promise.resolve(); + return new Promise((resolve) => { + const ws = this.flushWaiters.get(sessionId) ?? []; + ws.push(resolve); + this.flushWaiters.set(sessionId, ws); + }); + } + + private drainFlushWaiters(sessionId: string): void { + const ws = this.flushWaiters.get(sessionId); + if (!ws) return; + this.flushWaiters.delete(sessionId); + for (const w of ws) { + try { + w(); + } catch { + // resolve callbacks cannot usefully fail; never let one + // breaker take down the flush chain + } + } + } + /** Asynchronously persist a session right now (skips the debounce). Throws * on write failure so callers can react (e.g. avoid evicting). * @@ -2633,6 +2669,7 @@ export class SessionStore { this.foldResidentTail(session); this.dropResidentBlockContents(session); this.dropResidentMetadata(session); + this.drainFlushWaiters(session.id); } /** #3050 R3 ②a: once a save has captured the raw tail (envelope + the diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index f7d66cc8d5..20048608aa 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -737,10 +737,10 @@ export class UnifiedShadow { return this.sync.gcPlan(archiveAfterDays, now); } - rawMessages(sessionId: string): { seq: number; content: string }[] { + rawMessages(sessionId: string, fromSeqExclusive?: number): { seq: number; content: string }[] { if (this.sync === null) return []; this.assertLive(); - return this.sync.getMessages(sessionId).map((m) => ({ seq: m.seq, content: m.content })); + return this.sync.getMessages(sessionId, fromSeqExclusive !== undefined ? { fromSeqExclusive } : undefined).map((m) => ({ seq: m.seq, content: m.content })); } /** #3756 snapshot render: the logical view's seq pointer list diff --git a/src/storage/store.ts b/src/storage/store.ts index b52f54fcc6..f81265e4e4 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2508,17 +2508,16 @@ export class UnifiedStore { })); } - getMessages(sessionId: string, opts?: { includeRetired?: boolean }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { + getMessages(sessionId: string, opts?: { includeRetired?: boolean; fromSeqExclusive?: number }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { // #4390: retired rows (rewound/undo history) stay out of the unified // replay by default — they are history, not view. Callers that need // the full historian record (export/diagnostics) opt in. - const rows = this.db - .prepare( - opts?.includeRetired === true - ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" - : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", - ) - .all(sessionId) as Record[]; + // #5155: fromSeqExclusive narrows the scan to rows appended after a + // recorded pointer — hot-path pointer upkeep must not walk the table. + const retiredClause = opts?.includeRetired === true ? "" : " AND retired_at IS NULL"; + const afterClause = opts?.fromSeqExclusive !== undefined ? " AND seq > ?" : ""; + const stmt = this.db.prepare(`SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ?${retiredClause}${afterClause} ORDER BY seq`); + const rows = (opts?.fromSeqExclusive !== undefined ? stmt.all(sessionId, opts.fromSeqExclusive) : stmt.all(sessionId)) as Record[]; return rows.map((r) => ({ seq: r.seq as number, msgId: r.msg_id as string, diff --git a/tests/incremental-lane.test.ts b/tests/incremental-lane.test.ts index 294af1b356..2408a2831d 100644 --- a/tests/incremental-lane.test.ts +++ b/tests/incremental-lane.test.ts @@ -36,10 +36,11 @@ interface Harness { markerRaw: (reqPath: string, anthropicBody: Record) => Promise<{ status: number; body: unknown }>; forwarded: Record[]; dir: string; + settle: () => Promise; close: () => Promise; } -async function harness(storageMode: "full" | "legacy", opts: { upstreamDelayMs?: number; upstreamStatus?: number } = {}): Promise { +async function harness(storageMode: "full" | "legacy", opts: { upstreamDelayMs?: number; upstreamStatus?: number; debounceMs?: number } = {}): Promise { const dir = mkdtempSync(join(testRoot, "run-")); for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[key] = dir; process.env.BILI_STORAGE_UNIFIED = storageMode === "full" ? "full" : "legacy"; @@ -47,7 +48,7 @@ async function harness(storageMode: "full" | "legacy", opts: { upstreamDelayMs?: _resetPluginStateForTest(); resetToolRingForTest(); __resetIncrementalLaneForTests(); - const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: 60000 }); + const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: opts.debounceMs ?? 60000 }); _setStoreForTest(store); _setForTest({}); const forwarded: Record[] = []; @@ -106,7 +107,7 @@ async function harness(storageMode: "full" | "legacy", opts: { upstreamDelayMs?: const body: unknown = await response.json().catch(() => null); return { status: response.status, body }; }; - return { proxyUrl, path, turn, probe, markerTurn, markerRaw, forwarded, dir, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); } }; + return { proxyUrl, path, turn, probe, markerTurn, markerRaw, forwarded, dir, settle: async () => { await new Promise((resolve) => setTimeout(resolve, (opts.debounceMs ?? 60000) + 40)); }, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); } }; } const userTurn = (n: number, text = "hello") => ({ id: `u${n}`, role: "user", contentType: "text", content: text.repeat(400) }); @@ -169,10 +170,14 @@ test("incremental lane: seq mismatch answers 409 with the server's seq; resync r }); test("incremental lane: cold rebuild after restart serves the next delta from the unified table", async () => { - const h = await harness("full"); + const h = await harness("full", { debounceMs: 20 }); try { const seed = await h.turn({ sinceSeq: 0, messages: [userTurn(1)] }); assert.equal(seed.status, 200); + // #5155: the wrap-up schedules a DEBOUNCED save, so the rows settle + // a moment later — let the flush land before simulating the restart. + await h.settle(); + _resetSessionsForTest(); // Simulate a process restart: session pool + warm caches dropped. The // canonical prefix must rebuild from the unified messages table. _resetSessionsForTest(); @@ -187,7 +192,7 @@ test("incremental lane: cold rebuild after restart serves the next delta from th }); test("incremental lane: divergent resync survives a cold restart without a 409 loop", async () => { - const h = await harness("full"); + const h = await harness("full", { debounceMs: 20 }); try { const seed = await h.turn({ sinceSeq: 0, messages: [userTurn(1, "old branch one"), assistantTurn(1), userTurn(2, "old branch two")] }); assert.equal(seed.status, 200); @@ -195,6 +200,8 @@ test("incremental lane: divergent resync survives a cold restart without a 409 l const resync = await h.turn({ sinceSeq: 0, resync: true, messages: [userTurn(1, "edited branch")] }); assert.equal(resync.status, 200); assert.equal(resync.headers.get("x-bili-applied-seq"), "1"); + // #5155: debounced save — let the new branch's rows settle first. + await h.settle(); // Cold restart: the canonical must rebuild from the CURRENT branch // only — the rejected branch's rows stay in the table (append-only) // but the served history slices past them (baseSeq epoch). diff --git a/tests/issue5155-incremental-per-turn.test.ts b/tests/issue5155-incremental-per-turn.test.ts new file mode 100644 index 0000000000..7b3c0302f3 --- /dev/null +++ b/tests/issue5155-incremental-per-turn.test.ts @@ -0,0 +1,261 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { defaultConfig } from "acp-kernel"; +import { anthropicToCore, coreToAnthropic } from "acp-kernel/wire"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import type { BiliMessage } from "acp-kernel/wire"; +import { SessionStore, _setStoreForTest, getStore } from "../src/persist.ts"; +import { _resetPluginStateForTest } from "../src/plugin.ts"; +import { _resetSessionsForTest } from "../src/session.ts"; +import { _setForTest } from "../src/registry.ts"; +import { resetToolRingForTest } from "../src/tool-ring.ts"; +import { __resetIncrementalLaneForTests, laneItemsForTurn, laneRenderBoundaryFlushes, laneBodyBuffer, type Canonical } from "../src/incremental-lane.ts"; + +process.env.NODE_ENV = "test"; + +// ---------- unit level: prefix splice is byte-identical to a full render ---------- + +function entryOf(wire: { role: string; content: unknown }[], marks: Map = new Map()): Canonical { + const flat = anthropicToCore({ messages: wire as never } as never); + for (const [id, mark] of flat.cacheControls) marks.set(id, mark); + return { sessionId: "s", msgs: flat.msgs, marks, system: undefined, systemSet: false, boundary: null, renderPrefix: null, marksVersion: 0 }; +} + +const fullRender = (entry: Canonical): string => coreToAnthropic(entry.msgs, entry.marks).map((it) => JSON.stringify(it)).join(","); + +test("#5155 splice: successive appends stay byte-identical to the full render across boundary and non-boundary cuts", () => { + const seed = [{ role: "user", content: [{ type: "text", text: "hello world one" }] }]; + const deltas = [ + [{ role: "assistant", content: [{ type: "text", text: "answer one" }] }], + [{ role: "user", content: [{ type: "text", text: "hello world two" }] }], + // consecutive same-role messages: coreToAnthropic MERGES these into one + // output item — the cut before the second one is not a boundary, the + // prefix must not be reused there and the fallback full render kicks in + [{ role: "user", content: [{ type: "text", text: "hello world two-b" }] }], + [{ role: "assistant", content: [{ type: "text", text: "answer two" }] }, { role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content: "ok" }] }], + ]; + const entry = entryOf(seed); + let items = laneItemsForTurn(entry, 0); + assert.equal(items, fullRender(entry), "seed turn (cold cache) equals the full render"); + let cut = entry.msgs.length; + for (const delta of deltas) { + const flat = anthropicToCore({ messages: delta as never } as never); + entry.msgs = [...entry.msgs, ...flat.msgs]; + items = laneItemsForTurn(entry, cut); + assert.equal(items, fullRender(entry), `turn cut=${cut} splice equals the full render`); + cut = entry.msgs.length; + } + // boundary predicate: role change flushes; consecutive same role does not + const roles = entry.msgs.map((m) => (m.role === "assistant" ? "assistant" : "user")); + for (let i = 1; i < entry.msgs.length; i++) { + const expectFlush = roles[i - 1] !== roles[i] || (entry.msgs[i - 1] as { rawAnthropicMessage?: unknown }).rawAnthropicMessage !== undefined || (entry.msgs[i] as { rawAnthropicMessage?: unknown }).rawAnthropicMessage !== undefined; + assert.equal(laneRenderBoundaryFlushes(entry.msgs, i), expectFlush, `boundary predicate at ${i}`); + } +}); + +test("#5155 splice: a cache-control mark invalidates the prefix (marksVersion), raw system messages disable it", () => { + const entry = entryOf([ + { role: "user", content: [{ type: "text", text: "q1" }] }, + { role: "assistant", content: [{ type: "text", text: "a1" }] }, + ]); + laneItemsForTurn(entry, 0); + const prefix = entry.renderPrefix; + assert.ok(prefix, "cache armed after the full render"); + // a new delta whose wire marks a cache_control bumps the version + const delta = [{ role: "user", content: [{ type: "text", text: "q2", cache_control: { type: "ephemeral" } }] }]; + const flat = anthropicToCore({ messages: delta as never } as never); + entry.msgs = [...entry.msgs, ...flat.msgs]; + if (flat.cacheControls.size > 0) entry.marksVersion++; + const items = laneItemsForTurn(entry, 2); + assert.equal(items, fullRender(entry), "marked turn still byte-identical"); + assert.ok(entry.renderPrefix !== null && entry.renderPrefix.marksVersion === entry.marksVersion, "cache refreshed with the new version"); + assert.notEqual(prefix, entry.renderPrefix, "stale cache object replaced"); + // raw system messages: cache must be disabled (heal pass is position-dependent) + const withSystem = entryOf([ + { role: "user", content: [{ type: "text", text: "q1" }] }, + { role: "system", content: [{ type: "text", text: "sys" }] }, + { role: "assistant", content: [{ type: "text", text: "a1" }] }, + ]); + const items2 = laneItemsForTurn(withSystem, 0); + assert.equal(items2, fullRender(withSystem), "raw-system history renders correctly"); + assert.equal(withSystem.renderPrefix, null, "no prefix cache while raw system messages ride the history"); +}); + +test("#5155 splice: an empty delta (system-only turn) reuses the cached prefix verbatim", () => { + const entry = entryOf([{ role: "user", content: [{ type: "text", text: "q1" }] }]); + laneItemsForTurn(entry, 0); + const cached = entry.renderPrefix; + assert.ok(cached); + const items = laneItemsForTurn(entry, entry.msgs.length); + assert.equal(items, cached.items, "no re-render when the history did not change"); +}); + +test("#5155 sentinel: spliced body parses to the same object, collision falls back safely", () => { + const entry = entryOf([ + { role: "user", content: [{ type: "text", text: "q1" }] }, + { role: "assistant", content: [{ type: "text", text: "a1" }] }, + ]); + const items = laneItemsForTurn(entry, 0); + const skeleton: Record = { model: "claude-test", max_tokens: 1024 }; + const buf = laneBodyBuffer(skeleton, items); + assert.deepEqual(JSON.parse(buf.toString("utf8")), { model: "claude-test", max_tokens: 1024, messages: JSON.parse(`[${items}]`) }); + const empty = laneBodyBuffer(skeleton, ""); + assert.deepEqual(JSON.parse(empty.toString("utf8")), { model: "claude-test", max_tokens: 1024, messages: [] }); + // a skeleton value that already contains the sentinel's serialized form: + // the uniqueness check must detect it and fall back to a plain stringify + const sentinelJson = JSON.stringify("\u0000bili-splice-anchor\uE000"); + const hostile = { model: `evil${sentinelJson}evil`, max_tokens: 1 }; + const hostileBuf = laneBodyBuffer(hostile, items); + assert.deepEqual(JSON.parse(hostileBuf.toString("utf8")), { model: `evil${sentinelJson}evil`, max_tokens: 1, messages: JSON.parse(`[${items}]`) }); +}); + +// ---------- integration: windowed pointers + debounced settle through the real lane ---------- + +interface Integration { + turn: (t: { sinceSeq: number; messages: unknown[]; resync?: boolean }) => Promise<{ status: number; appliedSeq?: number }>; + upstreamBodies: string[]; + settle: () => Promise; + pointers: () => unknown; + close: () => Promise; +} + +async function integration(debounceMs: number): Promise { + const dir = mkdtempSync(join(tmpdir(), "bili-5155-")); + for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[key] = dir; + process.env.BILI_STORAGE_UNIFIED = "full"; + _resetSessionsForTest(); + _resetPluginStateForTest(); + resetToolRingForTest(); + __resetIncrementalLaneForTests(); + const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs }); + _setStoreForTest(store); + _setForTest({}); + const upstreamBodies: string[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + upstreamBodies.push(Buffer.concat(chunks).toString("utf8")); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ id: "m", role: "assistant", content: [{ type: "text", text: "answer" }], usage: { input_tokens: 100, output_tokens: 5 } })); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamUrl = `http://127.0.0.1:${(upstream.address() as { port: number }).port}`; + const proxy = await startServer({ port: 0, host: "127.0.0.1", upstream: upstreamUrl, routes: { [upstreamUrl]: { models: { "claude-test": { context: 400000 } } } }, modelContextLimit: 400000, kernelConfig: defaultConfig(400000), compress: { injectTool: true, injectNudge: false }, promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", log: false, debug: false, passthrough: false, autoUpdate: false, mitm: { enabled: false, domains: [] } } as ProxyOptions); + if (!(proxy as unknown as { listening?: boolean }).listening) await once(proxy, "listening"); + const proxyUrl = `http://127.0.0.1:${(proxy.address() as { port: number }).port}`; + const turn = async (t: { sinceSeq: number; messages: unknown[]; resync?: boolean }) => { + const r = await fetch(`${proxyUrl}/__bili/plugin/turn`, { + method: "POST", + headers: { "content-type": "application/json", "x-bili-plugin": "dsh", "x-bili-plugin-conversation": "conv-5155" }, + body: JSON.stringify({ conversationId: "conv-5155", sinceSeq: t.sinceSeq, ...(t.resync ? { resync: true } : {}), request: { model: "claude-test", max_tokens: 1024 }, path: "/v1/messages", messages: t.messages }), + }); + const applied = r.headers.get("x-bili-applied-seq"); + return { status: r.status, appliedSeq: applied !== null && /^\d+$/.test(applied) ? Number(applied) : undefined }; + }; + return { + turn, + upstreamBodies, + settle: async () => { await new Promise((resolve) => setTimeout(resolve, debounceMs + 60)); }, + pointers: () => { try { return getStore().getSessionExt("conv-5155", "inc:canonical-ptrs"); } catch { return undefined; } }, + close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); rmSync(dir, { recursive: true, force: true }); }, + }; +} + +const uMsg = (text: string): { role: string; content: { type: string; text: string }[] } => ({ role: "user", content: [{ type: "text", text }] }); +const aMsg = (text: string): { role: string; content: { type: string; text: string }[] } => ({ role: "assistant", content: [{ type: "text", text }] }); + +test("#5155 integration: multi-turn traffic through the splice path stays correct and pointers converge after the debounced settle", async () => { + const h = await integration(20); + try { + const seed = await h.turn({ sinceSeq: 0, messages: [uMsg("seed question")] }); + assert.equal(seed.status, 200); + assert.equal(seed.appliedSeq, 1); + let seq = 1; + // several spliced turns: user→assistant alternation makes every cut a + // boundary (role change), so these all take the prefix-splice path + for (let i = 0; i < 4; i++) { + const r = await h.turn({ sinceSeq: seq, messages: [aMsg(`answer ${i}`), uMsg(`question ${i}`)] }); + assert.equal(r.status, 200, `turn ${i} ok`); + seq = r.appliedSeq ?? seq; + } + assert.equal(seq, 9); + const body = JSON.parse(h.upstreamBodies.at(-1) ?? "{}") as { messages: { role: string; content: unknown }[] }; + assert.deepEqual(body.messages.map((m) => m.role), ["user", "assistant", "user", "assistant", "user", "assistant", "user", "assistant", "user"], "assembled history via splice is complete and ordered"); + // rows settle on the debounce; the NEXT turn's wrap-up records the + // windowed pointers over whatever has landed + await h.settle(); + const tail = await h.turn({ sinceSeq: seq, messages: [aMsg("final answer"), uMsg("final question")] }); + assert.equal(tail.status, 200); + await h.settle(); + const ptrs = h.pointers(); + assert.ok(Array.isArray(ptrs), `pointers recorded after settle: ${JSON.stringify(ptrs)}`); + assert.deepEqual(ptrs, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], "windowed upkeep records the full row mapping"); + // cold rebuild from the table after everything settled + _resetSessionsForTest(); + __resetIncrementalLaneForTests(); + const after = await h.turn({ sinceSeq: 11, messages: [aMsg("post restart answer"), uMsg("post restart question")] }); + assert.equal(after.status, 200, `cold rebuild serves the next delta: ${JSON.stringify(after)}`); + const rebuilt = JSON.parse(h.upstreamBodies.at(-1) ?? "{}") as { messages: unknown[] }; + assert.equal(rebuilt.messages.length, 13, "table-rebuilt canonical + delta"); + } finally { + await h.close(); + } +}); + +test("#5155 integration: duplicate-content messages (clustered `_` rows) still converge pointers", async () => { + const h = await integration(20); + try { + const seed = await h.turn({ sinceSeq: 0, messages: [uMsg("seed question")] }); + let seq = seed.appliedSeq ?? 1; + // every turn carries the SAME assistant text and user text — the + // content store clusters the duplicate rows as `_` while + // the in-memory canonical keeps the base id; the cluster-stripped + // matching must still map every occurrence to its own row in order + for (let i = 0; i < 4; i++) { + const r = await h.turn({ sinceSeq: seq, messages: [aMsg("dup answer"), uMsg("dup question")] }); + assert.equal(r.status, 200, `turn ${i} ok`); + seq = r.appliedSeq ?? seq; + } + assert.equal(seq, 9); + await h.settle(); + await h.turn({ sinceSeq: seq, messages: [aMsg("dup answer"), uMsg("dup question")] }); + await h.settle(); + const ptrs = h.pointers(); + assert.ok(Array.isArray(ptrs), `pointers recorded despite clustered duplicates: ${JSON.stringify(ptrs)}`); + assert.equal(ptrs.length, 11, "every duplicate occurrence matched its own settled row"); + assert.deepEqual([...new Set(ptrs)].sort((a, b) => a - b), [...ptrs].sort((a, b) => a - b), "pointer rows are distinct (no occurrence reuse)"); + const body = JSON.parse(h.upstreamBodies.at(-1) ?? "{}") as { messages: unknown[] }; + assert.equal(body.messages.length, 11, "assembled history keeps every duplicate occurrence"); + } finally { + await h.close(); + } +}); + +test("#5155 integration: unsettleable rows keep previous pointers (never a partial list) and the ack still flows", async () => { + const h = await integration(60000); + try { + const seed = await h.turn({ sinceSeq: 0, messages: [uMsg("q1")] }); + assert.equal(seed.status, 200); + assert.equal(seed.appliedSeq, 1, "ack rides immediately — no writeNow wait"); + const delta = await h.turn({ sinceSeq: 1, messages: [aMsg("a1"), uMsg("q2")] }); + assert.equal(delta.status, 200); + assert.equal(delta.appliedSeq, 3); + // nothing has settled (60s debounce): pointers must be absent or stale, + // never a partial [0] style half-list for the 3-message canonical + const ptrs = h.pointers(); + assert.ok(!Array.isArray(ptrs) || (ptrs as number[]).every((n) => Number.isInteger(n)), "no partial/corrupt pointer list while rows are unsettled"); + const body = JSON.parse(h.upstreamBodies.at(-1) ?? "{}") as { messages: unknown[] }; + assert.equal(body.messages.length, 3, "traffic is unaffected by pointer lag"); + } finally { + await h.close(); + } +}); From f13707f64200d18eb4b1113a05b0bdcecdb76422 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Fri, 30 Oct 2026 01:19:26 +0800 Subject: [PATCH 23/25] =?UTF-8?q?test:=20=E9=80=82=E9=85=8D=20#5155=20?= =?UTF-8?q?=E5=BC=82=E6=AD=A5=E8=90=BD=E7=9B=98=E6=97=B6=E5=BA=8F=20(?= =?UTF-8?q?=E5=90=AB=20master=20=E5=90=88=E5=B9=B6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 测试在 ack 后立即读行/重启的用例改为显式 settle: msgid/plugin-render/#4732/seq-interval/inc-canonical-pointers - #4316 seq 投影不再要求已有戳才运行: 首个 resync(全未戳)也精确投影, 否则 MSG_SEQS 缺席会令折叠丢失 ledger 界 - 集群后缀容忍与指针收敛见 #5155 --- src/incremental-lane.ts | 24 +++++++++---------- tests/inc-canonical-pointers.test.ts | 8 +++++-- ...ssue4732-plugin-render-tail-anchor.test.ts | 2 +- tests/msgid-lane-flow.test.ts | 22 +++++++++++++---- tests/plugin-render-endpoint.test.ts | 2 +- tests/seq-interval-lane.test.ts | 6 +++-- 6 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/incremental-lane.ts b/src/incremental-lane.ts index fdd50e217f..9de45fd2a0 100644 --- a/src/incremental-lane.ts +++ b/src/incremental-lane.ts @@ -803,23 +803,23 @@ export async function handleIncrementalTurn( // #4316 PR2/3: seq stamps ride the same internal transport. The // canonical prefix carries its stamps from the unified rows // (parseCanonicalRows); this turn's delta tail has no rows yet, - // but the wrap-up will append them at maxSeq+1 in order — the - // projection is exact, and partial knowledge is never sent: if - // the prefix itself is unstamped (first resync, cold replay), the - // field stays absent and the kernel keeps its id-based behavior. + // but the wrap-up appends them at maxSeq+1 in order (#5155: + // asynchronously — the stamps may not have landed yet when the + // next turn assembles). Stamps always form a prefix of + // entry.msgs, so projecting the unstamped suffix from the last + // stamped seq (+1 from 0 when nothing is stamped yet — the first + // resync's rows settle in message order) is exact. let maxSeq = -1; // 0-based store seq of the last stamped message for (const m of entry.msgs) if (typeof m.ledgerSeq === "number") maxSeq = Math.max(maxSeq, m.ledgerSeq - 1); - if (maxSeq >= 0) { - let next = maxSeq + 1; - for (const m of entry.msgs) { - if (typeof m.ledgerSeq !== "number") { - m.ledgerSeq = next + 1; - next++; - } + let next = maxSeq + 1; + for (const m of entry.msgs) { + if (typeof m.ledgerSeq !== "number") { + m.ledgerSeq = next + 1; + next++; } } const seqs = entry.msgs.map((m) => m.ledgerSeq); - if (seqs.every((s) => typeof s === "number")) body[MSG_SEQS_BODY_FIELD] = seqs; + body[MSG_SEQS_BODY_FIELD] = seqs; } const bodyBuf = laneBodyBuffer(body, itemsStr); diff --git a/tests/inc-canonical-pointers.test.ts b/tests/inc-canonical-pointers.test.ts index 8b8bb370d2..e0210b8d4f 100644 --- a/tests/inc-canonical-pointers.test.ts +++ b/tests/inc-canonical-pointers.test.ts @@ -102,8 +102,10 @@ test("cold canonical rebuild: identical resync keeps the shared prefix; delta as assert.equal(resync.status, 200); assert.equal(resync.appliedSeq, 3); await __incrementalPtrChainForTests("conv-ptrs"); - - // Cold: drop the resident caches, keep SQLite. + // #5155: the wrap-up defers the durable flush (debounced) — let the + // rows settle before dropping the resident caches, or the cold + // rebuild reads a one-turn-stale table and honestly 409s. + await new Promise((resolve) => setTimeout(resolve, 20)); __resetIncrementalLaneForTests(); const delta = await h.turn({ sinceSeq: 3, messages: [core("u4", "four")] }); @@ -132,6 +134,8 @@ test("cold canonical rebuild: edited-tail resync keeps the shared prefix, drops // one-turn-stale pointers. await h.turn({ sinceSeq: 0, resync: true, messages: [core("u1", "one"), core("u2", "two-EDITED")] }); await __incrementalPtrChainForTests("conv-ptrs"); + // #5155: settle the debounced flush before the cold reset (see above). + await new Promise((resolve) => setTimeout(resolve, 20)); __resetIncrementalLaneForTests(); const delta = await h.turn({ sinceSeq: 2, messages: [core("u3", "three")] }); assert.equal(delta.status, 200); diff --git a/tests/issue4732-plugin-render-tail-anchor.test.ts b/tests/issue4732-plugin-render-tail-anchor.test.ts index b8edcb3909..0ebac34bd5 100644 --- a/tests/issue4732-plugin-render-tail-anchor.test.ts +++ b/tests/issue4732-plugin-render-tail-anchor.test.ts @@ -38,7 +38,7 @@ async function harness() { _resetPluginStateForTest(); resetToolRingForTest(); __resetIncrementalLaneForTests(); - const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: 60000 }); + const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: 15 }); _setStoreForTest(store); _setForTest({}); const upstream = http.createServer((req, res) => { diff --git a/tests/msgid-lane-flow.test.ts b/tests/msgid-lane-flow.test.ts index 897c92cfa7..30e0798d13 100644 --- a/tests/msgid-lane-flow.test.ts +++ b/tests/msgid-lane-flow.test.ts @@ -35,6 +35,7 @@ interface Harness { turn: (t: Turn) => Promise<{ status: number; body: unknown; appliedSeq?: number }>; forwarded: Record[]; rows: () => { seq: number; msgId: string; content: string }[]; + settle: () => Promise; close: () => Promise; } @@ -52,7 +53,11 @@ async function harness(pin: boolean): Promise { _resetPluginStateForTest(); resetToolRingForTest(); __resetIncrementalLaneForTests(); - const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: 60000 }); + // #5155: the wrap-up schedules a debounced save instead of a synchronous + // writeNow per turn — rows settle shortly AFTER the ack. A short debounce + // plus settle() keeps the row reads deterministic; the identity contracts + // under test are unchanged. + const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: 15 }); _setStoreForTest(store); _setForTest({}); const forwarded: Record[] = []; @@ -81,8 +86,8 @@ async function harness(pin: boolean): Promise { const applied = response.headers.get("x-bili-applied-seq"); return { status: response.status, body, appliedSeq: applied !== null && /^\d+$/.test(applied) ? Number(applied) : undefined }; }; - // the lane acks after its own immediate flush, so rows are on disk by - // the time the turn responds; message bytes live in the CAS (content + // #5155: the lane acks without waiting for the debounced flush — rows are + // on disk once it settles; message bytes live in the CAS (content // table) keyed by messages.content_hash — the messages table has no // content column (#4390 schema shape). const rows = (): { seq: number; msgId: string; content: string }[] => { @@ -95,7 +100,7 @@ async function harness(pin: boolean): Promise { db.close(); } }; - return { turn, forwarded, rows, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); } }; + return { turn, forwarded, rows, settle: async () => { await new Promise((resolve) => setTimeout(resolve, 60)); }, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); } }; } const wmsg = (text: string) => ({ role: "user", content: [{ type: "text", text: text.repeat(200) }] }); @@ -109,6 +114,7 @@ test("msgid lane flow: pin carries client ids into rows; edits append fresh rows const seed = await h.turn({ sinceSeq: 0, resync: true, msgIds: ["u1", "a1", "u2"], messages: [umsg("alpha"), amsg("beta"), umsg("gamma")] }); assert.equal(seed.status, 200); assert.equal(seed.appliedSeq, 3); + await h.settle(); let rows = h.rows(); assert.equal(rows.length, 3, "seed lands three rows"); assert.deepEqual(rows.map((r) => r.msgId).sort(), ["a1", "u1", "u2"], "rows carry the CLIENT ids (pin), not content hashes"); @@ -118,6 +124,7 @@ test("msgid lane flow: pin carries client ids into rows; edits append fresh rows const second = await h.turn({ sinceSeq: 3, resync: true, msgIds: ["u1", "a1"], messages: [umsg("alpha-edited"), amsg("beta")] }); assert.equal(second.status, 200, "edit resync must not 409"); assert.equal(second.appliedSeq, 2); + await h.settle(); rows = h.rows(); assert.equal(rows.length, 4, "an edit lands as a FRESH row (append-only discipline; the old generation stays)"); const editedRow = rows.find((r) => r.msgId === "u1" && r.content.includes("alpha-edited")); @@ -140,6 +147,7 @@ test("msgid lane flow: undo→redo with identical content revives at the origina try { const seed = await h.turn({ sinceSeq: 0, resync: true, msgIds: ["u1", "a1", "u2"], messages: [umsg("alpha"), amsg("beta"), umsg("gamma")] }); assert.equal(seed.status, 200); + await h.settle(); const rows0 = h.rows(); assert.equal(rows0.length, 3); @@ -150,6 +158,7 @@ test("msgid lane flow: undo→redo with identical content revives at the origina const redo = await h.turn({ sinceSeq: 1, resync: true, msgIds: ["u1", "a1", "u2"], messages: [umsg("alpha"), amsg("beta"), umsg("gamma")] }); assert.equal(redo.status, 200); assert.equal(redo.appliedSeq, 3); + await h.settle(); const rows2 = h.rows(); assert.equal(rows2.length, 3, "identical content re-observed under the same ids joins the ORIGINAL rows — no duplicates (#4390 discipline)"); assert.deepEqual(rows2.map((r) => r.seq).sort(), rows0.map((r) => r.seq).sort(), "seqs unchanged across undo→redo"); @@ -173,6 +182,7 @@ test("msgid lane flow: a delta re-claiming a live canonical id hard-fails (400), const good = await h.turn({ sinceSeq: 1, msgIds: ["a1", "u2"], messages: [amsg("reply"), umsg("next")] }); assert.equal(good.status, 200); assert.equal(good.appliedSeq, 3); + await h.settle(); const rows = h.rows(); assert.deepEqual(rows.map((r) => r.msgId).sort(), ["a1", "u1", "u2"], "delta ids append onto the pinned canonical"); } finally { @@ -194,6 +204,7 @@ test("msgid lane flow: divergent resync with ids then a delta — no 409 loop (b const delta = await h.turn({ sinceSeq: 1, msgIds: ["u3"], messages: [umsg("carrying on")] }); assert.equal(delta.status, 200, "no 409 loop after a divergent resync with ids"); assert.ok((delta.appliedSeq ?? 0) >= 1, "tip is monotonic"); + await h.settle(); const rows = h.rows(); assert.equal(rows.filter((r) => r.msgId === "u1").length, 2, "both u1 generations live in the durable log (historian)"); } finally { @@ -206,6 +217,7 @@ test("msgid lane flow: turns without msgIds stay on content-derived identities ( try { const seed = await h.turn({ sinceSeq: 0, resync: true, messages: [umsg("alpha"), amsg("beta")] }); assert.equal(seed.status, 200); + await h.settle(); const rows = h.rows(); assert.equal(rows.length, 2); for (const r of rows) assert.ok(!["alpha", "beta", "u1", "a1"].includes(r.msgId), `unpinned ids stay content-derived: ${r.msgId}`); @@ -242,6 +254,7 @@ test("msgid lane flow: multi-block wire content degrades to derived ids, never a // The dropped ids must NOT be stamped into rows (they were ambiguous); // the derived ids own the turn's rows. + await h.settle(); const stamped = h.rows().map((r) => r.msgId); assert.ok(!stamped.includes("a2"), "the ambiguous client id was dropped, not misattributed"); assert.ok(!stamped.includes("u3"), "the whole delta's ids drop together (per-message granularity is a lie mid-expansion)"); @@ -249,6 +262,7 @@ test("msgid lane flow: multi-block wire content degrades to derived ids, never a // Recovery: a clean 1:1 delta pins again — the lane is not poisoned. const after = await h.turn({ sinceSeq: next.appliedSeq ?? 0, msgIds: ["u4"], messages: [umsg("clean follow-up")] }); assert.equal(after.status, 200); + await h.settle(); assert.ok(h.rows().some((r) => r.msgId === "u4"), "subsequent clean deltas pin client ids again"); } finally { await h.close(); diff --git a/tests/plugin-render-endpoint.test.ts b/tests/plugin-render-endpoint.test.ts index 48edb691b2..7c3d696ecf 100644 --- a/tests/plugin-render-endpoint.test.ts +++ b/tests/plugin-render-endpoint.test.ts @@ -47,7 +47,7 @@ async function harness(gateOn: boolean): Promise { _resetPluginStateForTest(); resetToolRingForTest(); __resetIncrementalLaneForTests(); - const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: 60000 }); + const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: 15 }); _setStoreForTest(store); _setForTest({}); const forwarded: Record[] = []; diff --git a/tests/seq-interval-lane.test.ts b/tests/seq-interval-lane.test.ts index 9b16c1409e..746bc9902e 100644 --- a/tests/seq-interval-lane.test.ts +++ b/tests/seq-interval-lane.test.ts @@ -35,6 +35,7 @@ interface Harness { forwarded: Record[]; blockRows: () => { blockId: string; startSeq: number; endSeq: number; payloadJson: string }[]; rows: () => { seq: number; msgId: string }[]; + settle: () => Promise; close: () => Promise; } @@ -47,7 +48,7 @@ async function harness(): Promise { _resetPluginStateForTest(); resetToolRingForTest(); __resetIncrementalLaneForTests(); - const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: 60000 }); + const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: 15 }); _setStoreForTest(store); _setForTest({}); const forwarded: Record[] = []; @@ -118,7 +119,7 @@ async function harness(): Promise { db.close(); } }; - return { turn, callTool, forwarded, rows, blockRows, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); } }; + return { turn, callTool, forwarded, rows, blockRows, settle: async () => { await new Promise((resolve) => setTimeout(resolve, 60)); }, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); } }; } const big = (text: string) => text.repeat(600); @@ -174,6 +175,7 @@ test("seq interval lane flow: stamps ride the body, fold records the interval, o // the unified blocks table must carry BOTH interval forms: // row-space bounds (0-based store seqs, from the ingest-side mapping) // and the kernel payload's ledger-space bounds (1-based stamps). + await h.settle(); const blocks = h.blockRows(); assert.equal(blocks.length, 1, `one block recorded: ${JSON.stringify(blocks.map((b) => b.blockId))}`); assert.equal(blocks[0]!.startSeq, 1, "store row bounds start at the second covered row (0-based)"); From b54dafd9afe478fe0c33f5efe033d9aa2073c7a8 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 30 Oct 2026 09:21:55 +0800 Subject: [PATCH 24/25] fix(storage): align streaming digest key order with canonicalize (#4532) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #4685 landed on master (canonicalize sorts object keys at every depth + shared DIGEST_SKIPPED_KEYS table), but this PR's streaming canonicalDigest still iterated raw Object.keys order with its own literal skip list. The two producers then disagreed for any object whose keys arrive in a different order — the false DIGEST DRIFT of #4392/#4986 again, this time between the streaming digest and canonicalize; tests/issue4392-canonical-key-order.test.ts went red on 8 shards (e.g. run 40909901398). Align visit() with the legacy producer's effective byte order: - skip via Object.hasOwn(DIGEST_SKIPPED_KEYS, k), keeping the __proto__ guard; - iterate array-index keys first in ascending numeric order, then the remaining keys lexicographically. Plain .sort() alone is NOT byte-identical: spec enumeration (Object.keys / JSON.stringify) emits array-index keys ascending numeric regardless of insertion or lexicographic order, so an index-shaped pair like {"10","2"} must serialize as "2","10". The pinned byte-identity test (streaming digest preserves JSON omissions, sparse arrays and property order) catches exactly this. Byte-identity to sha256(JSON.stringify(canonicalize(v))) now holds for any key arrival order, in either merge order relative to #4685. Verified locally: issue4392/issue4986/storage-canonical-digest suites 13/13, storage parity integration (engine-proofs + #3407 + #3919 + corpus) 26/26, typecheck clean. --- src/storage/canonical.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/storage/canonical.ts b/src/storage/canonical.ts index 66af617428..d585e2027e 100644 --- a/src/storage/canonical.ts +++ b/src/storage/canonical.ts @@ -81,6 +81,11 @@ function canonicalScalar(value: unknown, key: string | undefined): unknown { return value; } +// Spec array index: "0" .. 2^32-2, no leading zeros. +function isArrayIndexKey(k: string): boolean { + return /^(0|[1-9]\d*)$/.test(k) && Number(k) <= 4_294_967_294; +} + export function sha256(s: string): string { return createHash("sha256").update(s, "utf8").digest("hex"); } @@ -143,9 +148,15 @@ export function canonicalDigest(value: unknown): string { } else if (current && typeof current === "object") { write("{"); let separator = ""; - for (const k of Object.keys(current)) { + // Effective order of JSON.stringify(canonicalize(v)): array-index keys + // first, ascending numeric (spec enumeration overrides canonicalize's + // lexicographic sort), then the remaining keys lexicographic. + const keys = Object.keys(current); + const orderedKeys = keys.filter(isArrayIndexKey).sort((a, b) => Number(a) - Number(b)) + .concat(keys.filter((k) => !isArrayIndexKey(k)).sort()); + for (const k of orderedKeys) { // Assigning __proto__ to canonicalize's plain output creates no JSON property. - if (k === "storageMode" || k === "tailGaps" || k === "__proto__") continue; + if (Object.hasOwn(DIGEST_SKIPPED_KEYS, k) || k === "__proto__") continue; const v = (current as Record)[k]; if (v === undefined || typeof v === "function" || typeof v === "symbol") continue; write(separator); From ccf51f07c5f9ac3ae44abbb642e7dfc9c69b12e0 Mon Sep 17 00:00:00 2001 From: ranxianglei-devbox Date: Fri, 30 Oct 2026 11:17:50 +0800 Subject: [PATCH 25/25] fix(types): split getBlockContents into true overloads after union of #4765/#4750 The two PRs added different getBlockContents call shapes (paged rows for bounded view decode vs full narrow contents map for lazy-field restore); merged as one union return type, which broke caller-side narrowing in the read-snapshot test. Declare the overload pair so paged callers get the row array and no-arg callers get the map. storage-bounded-load's read counter monkey-patch now casts to the implementation signature it already matches at runtime (every counted call passes opts.limit=32). --- src/storage/store.ts | 2 ++ tests/storage-bounded-load.test.ts | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/storage/store.ts b/src/storage/store.ts index 5f6ebfd769..f57b897c83 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2523,6 +2523,8 @@ export class UnifiedStore { * lazy-field restore (#4739) — block-row summary/one CAS bytes plus the * orphans_json journal precedence, WITHOUT resurrecting state.blocks * (issue4739-bounded-session-load pins this). */ + getBlockContents(sessionId: string, opts: { limit: number; offset?: number }): Pick[]; + getBlockContents(sessionId: string): BlockContentsMap | null; getBlockContents(sessionId: string, opts?: { limit: number; offset?: number }): Pick[] | BlockContentsMap | null { if (opts !== undefined) { const rows = this.db.prepare(`SELECT block_id, summary_hash, one_hash FROM blocks diff --git a/tests/storage-bounded-load.test.ts b/tests/storage-bounded-load.test.ts index a412e8457f..f0097dcecb 100644 --- a/tests/storage-bounded-load.test.ts +++ b/tests/storage-bounded-load.test.ts @@ -186,7 +186,11 @@ test("resident field reads avoid unrelated JSON, messages, refs, and block paylo const contents = store.getBlockContents.bind(store); const orphans = store.getOrphansJson.bind(store); store.getMetadataJson = (id) => { metadataReads++; return metadata(id); }; - store.getBlockContents = (id, opts) => { contentReads++; assert.equal(opts?.limit, 32); return contents(id, opts); }; + store.getBlockContents = ((id: string, opts?: { limit: number; offset?: number }) => { + contentReads++; + assert.equal(opts?.limit, 32); + return contents(id, opts); + }) as typeof store.getBlockContents; store.getOrphansJson = (id) => { orphanReads++; return orphans(id); }; const meta = shadow.readResidentFieldsFor("bounded", { metadata: true }); assert.deepEqual(meta, { header: { requests: 7, nextBlockId: 71, activeBlockCount: 35, mappedRefCount: 2 }, metadata: expected.metadata });