Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions src/decompress-shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import { safePrefix } from "./text-safe.js";
import { decompressTmpCap as knobDecompressTmpCap } from "./knobs.js";
import { toolFail, toolOk, type ProxyToolResult } from "./proxy-tool-result.js";
import { describeRestorable } from "./image-restore.js";
import { supersededRefRejection } from "./superseded-refs.js";

/** Bounded retention for large-decompress temp files. Each decompress with
* body > 10000 writes one file under tmpdir(); the reaper unlinks oldest past
Expand Down Expand Up @@ -397,11 +398,25 @@ function resolveDecompressRange(args: Record<string, unknown>, ctx: ProxyToolCtx
if (!sb || sb.kind !== "message" || !eb || eb.kind !== "message") {
return toolFail(`[decompress FAILED: startId/endId must be mNNNNN message refs (got "${startRaw}", "${endRaw}") — block ids (bN) are not valid here]`);
}
if (sb.numericId > eb.numericId) return toolFail(`[decompress FAILED: startId ${startRaw} is after endId ${endRaw} — swap them]`);
// #5049 item 4: run the endpoints through the SAME resolution ladder as
// compress (live-first, then superseded translation, refuse only dead
// cites) so a pre-restamp citation restores the same messages instead of
// falling into a generic FAILED or silently picking a shifted window.
const sup = supersededRefRejection(
[{ startRef: `m${String(sb.numericId).padStart(5, "0")}`, endRef: `m${String(eb.numericId).padStart(5, "0")}` }],
ctx.session,
);
if (sup.rejection) {
ctx.log(`[acp-decompress-range] ${block.blockId} ${startRaw}–${endRaw}: superseded refusal — ${sup.rejection}`);
return toolFail(`[decompress FAILED: ${sup.rejection}]`);
}
const sNum = refNum(sup.translatedRanges[0].startRef) ?? sb.numericId;
const eNum = refNum(sup.translatedRanges[0].endRef) ?? eb.numericId;
if (sNum > eNum) return toolFail(`[decompress FAILED: startId ${startRaw} is after endId ${endRaw} — swap them]`);
const state = ctx.session.state;
const cov = coveredMessages(state, block);
if (!cov) return toolFail(`[decompress FAILED: ${block.blockId} has no per-message coverage recorded (older block) — use plain decompress {blockId} for the whole block]`);
const pickedSet = new Set(cov.raws.filter(({ num }) => num >= sb.numericId && num <= eb.numericId).map(({ raw }) => raw));
const pickedSet = new Set(cov.raws.filter(({ num }) => num >= sNum && num <= eNum).map(({ raw }) => raw));
if (pickedSet.size === 0) return toolFail(`[decompress FAILED: ${block.blockId} covers no messages in ${startRaw}–${endRaw} (its coverage is ${cov.text})]`);
const parts: string[] = [];
let restoredFromStore = 0;
Expand Down
80 changes: 5 additions & 75 deletions src/stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import { safePrefix, safeSuffix, scrubLoneSurrogates } from "./text-safe.js";
import { applyConfiguredCompression } from "./external-summary-compress.js";
import { NUDGE_CREDIT_ROUNDS, unpromptedGateOf } from "./unprompted-gate.js";
import { describeUncoveredRuns, shortTokenCount, uncoveredHistoryRuns } from "./uncovered.js";
import { supersededRefRejection } from "./superseded-refs.js";

export type RewriteCtx = {
core: CompressionCore;
Expand Down Expand Up @@ -889,81 +890,10 @@ export function findVisibleStubResubmissions(ctx: RewriteCtx, ranges: ParsedRang
return hits;
}

/** #4316 layer 3 PR3/3, #4797 rework: resolve a compress call's refs
* against the CURRENT live map first, the superseded map second, and refuse
* only genuinely-dead cites.
*
* The original span-scan refused any range with a superseded KEY anywhere
* between its endpoints — but the restamp's vacated numbers can be handed to
* other messages as seq+1 targets in the same sweep, so in a dense space a
* superseded key is routinely ALSO the live ref of a different message
* (exactly the numbers acp_status advertises). That made the guard reject
* every advertised range and dead-loop the session: "call acp_status" →
* status lists the same live numbers → refused again — the proxy's own
* preflight rescue included (#4797).
*
* Resolution ladder, per ENDPOINT (span folding is endpoint-defined; the
* interior number scan only manufactured false hits):
* 1. live in the current map → keep as-is (collision-band artifact, the
* common case for numbers straight out of acp_status / the nudge);
* 2. superseded key (genuinely vacated) → identity-preserving translation
* oldRef → rawId → the same message's current ref — a model citing a
* number it saw BEFORE the restamp folds the SAME messages, never the
* new owner of that number;
* 3. neither → leave to the kernel's boundary-not-found handling.
* Refusal only when an endpoint is a superseded key whose translation
* dead-ends (the message lost its mapping entirely — an honest stale cite).
*
* Transitional honesty: sessions whose restamp predates kernel #5196's
* slot-retirement may still carry an ambiguity — a number restamped over a
* live occupant, so case 1 can in principle swallow a resent pre-restart
* call's intent. #4807 (#4797) traded that seconds-long replay window
* against a permanent deadlock; live-first keeps that trade. Kernels from
* #5196 onward retire every seat a mover vacates into its own seq+1, so no
* freshly stamped number ever names two messages and case 1 can never
* mis-engage on such data. */
interface SupersededResolution<T extends { startRef: string; endRef: string } = { startRef: string; endRef: string; summary: string }> {
rejection: string | null;
translatedRanges: T[];
translated: number;
}

export function supersededRefRejection<T extends { startRef: string; endRef: string }>(
ranges: T[],
session: Session,
): SupersededResolution<T> {
const superseded = session.restampSuperseded;
const out: SupersededResolution<T> = { rejection: null, translatedRanges: [], translated: 0 };
if (!superseded || superseded.size === 0) {
out.translatedRanges = ranges.map((r) => ({ ...r }));
return out;
}
const byRef = session.state.messageRefs?.byRef ?? {};
const byRaw = session.state.messageRefs?.byRaw ?? {};
const dead: string[] = [];
const resolveEndpoint = (ref: string): string => {
if (byRef[ref] !== undefined) return ref;
const rawId = superseded.get(ref);
if (rawId === undefined) return ref;
const next = byRaw[rawId];
if (typeof next === "string" && next !== ref && byRef[next] === rawId) {
out.translated++;
return next;
}
dead.push(ref);
return ref;
};
out.translatedRanges = ranges.map((range) => ({
...range,
startRef: resolveEndpoint(range.startRef),
endRef: resolveEndpoint(range.endRef),
}));
if (dead.length === 0) return out;
const listed = dead.slice(0, 5).join(", ");
const more = dead.length > 5 ? ` (+${dead.length - 5} more)` : "";
out.rejection = `ref(s) ${listed}${more} were re-numbered by the ledger alignment and no longer resolve to a live message. Re-read the current labels with acp_status and re-issue.`;
return out;
}
// #5049 item 4: supersededRefRejection (imported above) lives in
// ./superseded-refs.ts so the decompress range path can share it without an
// import cycle; re-exported for the existing import sites.
export { supersededRefRejection };

// #5683: out-of-bounds gate helpers. The recommendation set a compress call may
// cover is judged by COVERAGE over message-ref indices: every index of a
Expand Down
79 changes: 79 additions & 0 deletions src/superseded-refs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { type Session } from "./session.js";

/** #4316 layer 3 PR3/3, #4797 rework: resolve a tool call's refs
* against the CURRENT live map first, the superseded map second, and refuse
* only genuinely-dead cites. Shared by compress (ranges) and decompress
* (range endpoints, #5049 item 4) so both lanes translate pre-restamp
* citations identically.
*
* The original span-scan refused any range with a superseded KEY anywhere
* between its endpoints — but the restamp's vacated numbers can be handed to
* other messages as seq+1 targets in the same sweep, so in a dense space a
* superseded key is routinely ALSO the live ref of a different message
* (exactly the numbers acp_status advertises). That made the guard reject
* every advertised range and dead-loop the session: "call acp_status" →
* status lists the same live numbers → refused again — the proxy's own
* preflight rescue included (#4797).
*
* Resolution ladder, per ENDPOINT (span folding is endpoint-defined; the
* interior number scan only manufactured false hits):
* 1. live in the current map → keep as-is (collision-band artifact, the
* common case for numbers straight out of acp_status / the nudge);
* 2. superseded key (genuinely vacated) → identity-preserving translation
* oldRef → rawId → the same message's current ref — a model citing a
* number it saw BEFORE the restamp folds the SAME messages, never the
* new owner of that number;
* 3. neither → leave to the kernel's boundary-not-found handling.
* Refusal only when an endpoint is a superseded key whose translation
* dead-ends (the message lost its mapping entirely — an honest stale cite).
*
* Transitional honesty: sessions whose restamp predates kernel #5196's
* slot-retirement may still carry an ambiguity — a number restamped over a
* live occupant, so case 1 can in principle swallow a resent pre-restart
* call's intent. #4807 (#4797) traded that seconds-long replay window
* against a permanent deadlock; live-first keeps that trade. Kernels from
* #5196 onward retire every seat a mover vacates into its own seq+1, so no
* freshly stamped number ever names two messages and case 1 can never
* mis-engage on such data. */
interface SupersededResolution<T extends { startRef: string; endRef: string } = { startRef: string; endRef: string; summary: string }> {
rejection: string | null;
translatedRanges: T[];
translated: number;
}

export function supersededRefRejection<T extends { startRef: string; endRef: string }>(
ranges: T[],
session: Session,
): SupersededResolution<T> {
const superseded = session.restampSuperseded;
const out: SupersededResolution<T> = { rejection: null, translatedRanges: [], translated: 0 };
if (!superseded || superseded.size === 0) {
out.translatedRanges = ranges.map((r) => ({ ...r }));
return out;
}
const byRef = session.state.messageRefs?.byRef ?? {};
const byRaw = session.state.messageRefs?.byRaw ?? {};
const dead: string[] = [];
const resolveEndpoint = (ref: string): string => {
if (byRef[ref] !== undefined) return ref;
const rawId = superseded.get(ref);
if (rawId === undefined) return ref;
const next = byRaw[rawId];
if (typeof next === "string" && next !== ref && byRef[next] === rawId) {
out.translated++;
return next;
}
dead.push(ref);
return ref;
};
out.translatedRanges = ranges.map((range) => ({
...range,
startRef: resolveEndpoint(range.startRef),
endRef: resolveEndpoint(range.endRef),
}));
if (dead.length === 0) return out;
const listed = dead.slice(0, 5).join(", ");
const more = dead.length > 5 ? ` (+${dead.length - 5} more)` : "";
out.rejection = `ref(s) ${listed}${more} were re-numbered by the ledger alignment and no longer resolve to a live message. Re-read the current labels with acp_status and re-issue.`;
return out;
}
153 changes: 153 additions & 0 deletions tests/issue5049-decompress-superseded.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
// #5049 item 4: decompress range endpoints go through the SAME resolution
// ladder as compress (supersededRefRejection) — a pre-restamp citation
// translates to the live refs of the SAME messages instead of falling into a
// generic "[decompress FAILED]" or silently picking a shifted window; only a
// genuinely dead cite refuses, with the compress-side actionable wording.
import test from "node:test";
import assert from "node:assert/strict";
import {
createCore,
defaultConfig,
type Config,
type CoreMessage,
} from "acp-kernel";
import { applyCompressSettings } from "../src/compress-settings.ts";
import { storeEffectiveCcr, drainPendingRetrievals } from "../src/store.ts";
import { getSession } from "../src/session.ts";
import { applyRanges } from "../src/stream.ts";
import { parseCompressInput } from "../src/compress-tool.ts";
import { resolveDecompress } from "../src/decompress-shared.ts";

process.env.BILI_PERSIST = "0";

const pad = (n: number): string => String(n).padStart(5, "0");

// Same corpus discipline as tests/ccr-v2.test.ts: 20 messages so the
// m00001–m00007 fold range sits outside the protected recent zone.
function makeMsgs(): CoreMessage[] {
const msgs: CoreMessage[] = [];
for (let i = 0; i < 20; i++) {
msgs.push({
id: `h_${i}`,
role: i % 2 === 0 ? "user" : "assistant",
contentType: "text",
text: `\x3cacp tokens="2K" type="text"\x3em${pad(i + 1)}\x3c/acp\x3e\nHistorical detail ${i}. ${"x".repeat(2000)}`,
});
}
return msgs;
}

type FoldResult = {
core: ReturnType<typeof createCore>;
config: Config;
session: ReturnType<typeof getSession>;
msgs: CoreMessage[];
blockId: string;
};

function fold(): FoldResult {
const core = createCore();
const config = applyCompressSettings(
defaultConfig(200_000),
200_000,
{ absorb: { enabled: true, minToolTokens: 50 }, ccr: { enabled: true, minToolTokens: 50 } },
) as Config;
const session = getSession(`issue5049-${Math.random().toString(36).slice(2)}`);
storeEffectiveCcr(session, { enabled: true, minToolTokens: 50 });
const raw = makeMsgs();
const turn = core.processTurn({ messages: raw, state: session.state, config, tokenCount: 9999, renderTags: "text-only" });
session.state = turn.state;
const ctx = { core, config, messages: turn.messages, session, log: () => {} };
applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00007", summary: "Early history: messages 1-7 covered initial setup.", topic: "Early history" }] }), ctx);
const block = [...session.state.blocks].find((b) => b.active);
assert.ok(block, "a block was created");
return { core, config, session, msgs: turn.messages, blockId: block.blockId };
}

function ctxOf(f: FoldResult) {
return { core: f.core, config: f.config, messages: f.msgs, session: f.session, log: () => {} };
}

/** Simulate a ledger restamp: every live ref shifts by `delta`, the vacated
* numbers land in session.restampSuperseded (oldRef → rawId), exactly the
* shape persist.ts produces on restart reconciliation. */
function restampShift(f: FoldResult, delta: number): void {
const st = f.session.state;
const superseded = new Map<string, string>();
const byRef: Record<string, string> = {};
const byRaw: Record<string, string> = {};
// byRaw is keyed by raw id (raw → ref); the vacated KEYS of byRef are the old refs.
for (const [raw, ref] of Object.entries(st.messageRefs.byRaw)) {
const num = Number.parseInt(ref.slice(1), 10);
const next = `m${String(num + delta).padStart(5, "0")}`;
superseded.set(ref, raw);
byRef[next] = raw;
byRaw[raw] = next;
}
st.messageRefs.byRef = byRef;
st.messageRefs.byRaw = byRaw;
f.session.restampSuperseded = superseded;
}

test("pre-restamp citation translates to the same messages instead of failing (#5049 item 4)", () => {
const f = fold();
restampShift(f, 100); // m00001–m00007 → m00101–m00107
// The model cites the numbers it saw BEFORE the restamp. Pre-fix this hit
// "covers no messages in m00002–m00004 (its coverage is m00101–m00107)".
const ack = resolveDecompress({ blockId: f.blockId, startId: "m00002", endId: "m00004" }, ctxOf(f)).text;
assert.match(ack, /\[decompress .* m00002\u2013m00004: restored 3 item\(s\)/, `expected translated restore, got: ${ack.slice(0, 200)}`);
const injs = drainPendingRetrievals(f.session);
assert.equal(injs.length, 1, "one ephemeral injection queued");
const text = injs[0]!.text!;
// Same MESSAGES as the pre-restamp citation means: details 1–3 (h_1..h_3).
assert.match(text, /Historical detail 1\./);
assert.match(text, /Historical detail 2\./);
assert.match(text, /Historical detail 3\./);
assert.doesNotMatch(text, /Historical detail 0\./);
assert.doesNotMatch(text, /Historical detail 4\./);
});

test("live citations pass through untouched when a superseded map exists", () => {
const f = fold();
restampShift(f, 100);
// Numbers straight out of the POST-restamp acp_status surface are live —
// the ladder must keep them as-is (case 1), not translate or refuse.
const ack = resolveDecompress({ blockId: f.blockId, startId: "m00102", endId: "m00103" }, ctxOf(f)).text;
assert.match(ack, /restored 2 item\(s\)/, `got: ${ack.slice(0, 200)}`);
const text = drainPendingRetrievals(f.session)[0]!.text!;
assert.match(text, /Historical detail 1\./);
assert.match(text, /Historical detail 2\./);
assert.doesNotMatch(text, /Historical detail 3\./);
});

test("a dead superseded cite refuses with the compress-side actionable wording", () => {
const f = fold();
restampShift(f, 100);
// Kill the mapping of h_5 entirely (the message left the ledger map) while
// keeping its superseded key: old m00006 → h_5 has nowhere to translate to.
const st = f.session.state;
delete st.messageRefs.byRaw["h_5"];
delete st.messageRefs.byRef["m00106"];
const out = resolveDecompress({ blockId: f.blockId, startId: "m00006", endId: "m00006" }, ctxOf(f));
assert.equal(out.outcome, "failure");
assert.match(out.text, /\[decompress FAILED:/);
assert.match(out.text, /re-numbered by the ledger alignment and no longer resolve to a live message/);
assert.match(out.text, /acp_status/);
assert.equal(drainPendingRetrievals(f.session).length, 0, "a refused range queues nothing");
assert.equal(f.session.stats.rangeRestores, 0);
});

test("order check runs on the TRANSLATED endpoints (inverted old cites still swap-refused)", () => {
const f = fold();
restampShift(f, 100);
const out = resolveDecompress({ blockId: f.blockId, startId: "m00004", endId: "m00002" }, ctxOf(f));
assert.match(out.text, /swap them/);
assert.equal(drainPendingRetrievals(f.session).length, 0);
});

test("without a superseded map the old behavior is byte-identical (out-of-coverage still 'covers no messages')", () => {
const f = fold();
const out = resolveDecompress({ blockId: f.blockId, startId: "m00015", endId: "m00016" }, ctxOf(f));
assert.match(out.text, /covers no messages in m00015\u2013m00016/);
assert.equal(drainPendingRetrievals(f.session).length, 0);
});
Loading