Keep the system prompt stable across turns so prompt caching survives

src/core/orchestrator.ts

@@ -1052,11 +1052,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
       const sharedFilesBlock = sharedFilesSystemSection(resolution.grantedHandles);
       if (sharedFilesBlock) systemPrompt += `\n\n${sharedFilesBlock}`;
 
-      const stableSystemBytes = systemPrompt.length;
-      if (turnTimezone) {
-        const timeBlock = currentTimeBlock(turnTimezone, Date.now());
-        if (timeBlock) systemPrompt += `\n\n${timeBlock}`;
-      }
+      const timeBlock = turnTimezone ? currentTimeBlock(turnTimezone, Date.now()) : "";
       let memoryContext = "a channel";
       if (conversation.kind === "dm") memoryContext = "a direct message";
       else if (conversation.channelName) memoryContext = `#${conversation.channelName}`;
@@ -1861,8 +1857,12 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
           const connectionsUrl = deps.publicWebUrl ? `${deps.publicWebUrl.replace(/\/$/, "")}/keychain` : undefined;
           systemPrompt += `\n\n${renderConnectedAppsBlock(status, configuredProviders, connectionsUrl)}`;
         }
+        const stableSystemBytes = systemPrompt.length;
+        if (timeBlock) systemPrompt += `\n\n${timeBlock}`;
         systemPrompt += memoryBlock;
         if (onboardingBlock) systemPrompt += `\n\n${onboardingBlock}`;
+        const volatileContext = systemPrompt.slice(stableSystemBytes).trim();
+        systemPrompt = systemPrompt.slice(0, stableSystemBytes);
 
         if (
           ambientTurn &&
@@ -2500,7 +2500,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
         const sender = !automatedTurn && input.text.trim() ? senderNote(actor.displayName) : "";
         const unscreenedNote = inputUnscreened || inbound.unscreened.length ? unscreenedNotice("inbound content") : "";
         const turnEnv = environmentNote(
-          [manifest, principalDelivered, sender, unscreenedNote, input.conversationHeader?.trim()]
+          [manifest, principalDelivered, sender, unscreenedNote, input.conversationHeader?.trim(), volatileContext]
             .filter((s) => s && s.trim())
             .join("\n\n"),
         );
@@ -2810,7 +2810,6 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
               : {}),
             ...(securityPolicy.toolApprovals === "all" ? { toolApprovalGate: authorizeToolCall } : {}),
             systemPrompt,
-            systemCacheBoundary: stableSystemBytes,
             history: continuation?.history ?? history,
             tools,
             ...(tools.credentialExecServices ? { credentialExecServices: tools.credentialExecServices } : {}),

src/harness/context-compaction.ts

@@ -78,8 +78,11 @@ const entryTokenCache = new Map<string, number>();
 const ENTRY_TOKEN_CACHE_MAX = 50_000;
 
 export function estimateEntryTokens(entry: SessionEntry): number {
-  const payload = entry.payload as { text?: string } | null;
-  const text = typeof payload?.text === "string" ? payload.text : JSON.stringify(entry.payload ?? {});
+  const payload = entry.payload as { text?: string; environment?: string } | null;
+  const text =
+    typeof payload?.text === "string"
+      ? [payload.text, payload.environment].filter((s) => typeof s === "string" && s).join("\n\n")
+      : JSON.stringify(entry.payload ?? {});
   const key = `${entry.sessionId}:${entry.seq}:${text.length}`;
   const hit = entryTokenCache.get(key);
   if (hit !== undefined) return hit;

src/harness/harness.ts

@@ -88,7 +88,6 @@ export interface HarnessTurnInput {
   pollFire?: boolean;
   turnWallClockMs?: number;
   systemPrompt: string;
-  systemCacheBoundary?: number;
   history: SessionEntry[];
   tools: ToolContext;
   credentialExecServices?: readonly { service: string; binary: string }[];

src/harness/mock-harness.ts

@@ -325,7 +325,7 @@ export function createMockHarness(): Harness {
         } else if (command0 === "!histcount") {
           reply = `history:${turn.history.length}`;
         } else if (command0 === "!sysprompt") {
-          reply = turn.systemPrompt;
+          reply = [turn.systemPrompt, turn.environment].filter((s) => s && s.trim()).join("\n\n");
         } else if (command0 === "!wallclock") {
           reply = `wallclock:${turn.turnWallClockMs ?? 0}`;
         } else if (command0 === "!surfacename") {

src/harness/pi-harness.ts

@@ -1115,6 +1115,7 @@ function customModelsPath(): string | null {
 async function buildModelRuntime(
   keys: ProviderKeys | string,
   modelGateway?: ModelGatewayTransportConfig,
+  cacheRetention?: "long",
 ): Promise<ModelRuntime> {
   const k: ProviderKeys = typeof keys === "string" ? { anthropic: keys } : keys;
   // Custom providers must exist in the runtime's own registry — a runtime
@@ -1128,16 +1129,18 @@ async function buildModelRuntime(
   for (const [provider, apiKey] of Object.entries(k)) {
     if (apiKey) await runtime.setRuntimeApiKey(provider, apiKey, { allowNetwork: false });
   }
+  const retained = <T extends object | undefined>(options: T): T =>
+    cacheRetention ? ({ ...options, cacheRetention } as T) : options;
   const stream = runtime.stream.bind(runtime);
   runtime.stream = (<TApi extends Api>(
     model: Model<TApi>,
     context: Context,
     options?: ModelsApiStreamOptions<TApi>,
   ) => {
     const request = modelGatewayRequest(modelGateway, model);
-    if (!request) return stream(model, context, options);
+    if (!request) return stream(model, context, retained(options));
     const routedOptions = {
-      ...options,
+      ...retained(options),
       apiKey: request.apiKey,
       transformHeaders: async (headers: ProviderHeaders) => ({
         ...(options?.transformHeaders ? await options.transformHeaders(headers) : headers),
@@ -1157,9 +1160,9 @@ async function buildModelRuntime(
   const streamSimple = runtime.streamSimple.bind(runtime);
   runtime.streamSimple = ((model: Model<Api>, context: Context, options?: ModelsSimpleStreamOptions) => {
     const request = modelGatewayRequest(modelGateway, model);
-    if (!request) return streamSimple(model, context, options);
+    if (!request) return streamSimple(model, context, retained(options));
     const routedOptions = {
-      ...options,
+      ...retained(options),
       apiKey: request.apiKey,
       transformHeaders: async (headers: ProviderHeaders) => ({
         ...(options?.transformHeaders ? await options.transformHeaders(headers) : headers),
@@ -1300,32 +1303,6 @@ export function modelHasFastMode(model: unknown): boolean {
   return Boolean(m?.fastMode) || Boolean(m?.headers?.["anthropic-beta"]?.includes(FAST_MODE_BETA));
 }
 
-const ONE_HOUR_CACHE_CONTROL = { type: "ephemeral", ttl: "1h" } as const;
-
-export function applySystemPromptCacheSplit(payload: unknown, boundary: number | undefined): void {
-  if (typeof boundary !== "number" || !Number.isFinite(boundary) || boundary <= 0) return;
-  if (!payload || typeof payload !== "object") return;
-  const p = payload as { system?: unknown; tools?: unknown };
-  if (!Array.isArray(p.system) || p.system.length !== 1) return;
-  const block = p.system[0] as { type?: unknown; text?: unknown } | undefined;
-  if (!block || block.type !== "text" || typeof block.text !== "string") return;
-  const text = block.text;
-  if (boundary >= text.length) return;
-  const stable = text.slice(0, boundary);
-  const rest = text.slice(boundary);
-  if (!stable.trim() || !rest.trim()) return;
-  p.system = [
-    { type: "text", text: stable, cache_control: { ...ONE_HOUR_CACHE_CONTROL } },
-    { type: "text", text: rest },
-  ];
-  if (Array.isArray(p.tools)) {
-    for (const t of p.tools) {
-      const tool = t as { cache_control?: unknown } | undefined;
-      if (tool && tool.cache_control) tool.cache_control = { ...ONE_HOUR_CACHE_CONTROL };
-    }
-  }
-}
-
 export const OUTPUT_BUDGET_FLOOR_TOKENS = 1_024;
 export const OUTPUT_GUARD_SAFETY_TOKENS = 4_096;
 const OUTPUT_GUARD_CHARS_PER_TOKEN = 4;
@@ -1474,7 +1451,6 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
     systemPrompt: string,
     history: SessionEntry[],
     priorTurns?: ConversationTurn[],
-    systemCacheBoundary?: number,
     readOnly?: boolean,
     surfaceTools?: boolean,
     surfaceName?: string,
@@ -1488,12 +1464,6 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
     turnProviderKeys?: ProviderKeys,
   ): Promise<{ entry: TurnSession; compileMs: number }> {
     const compileStart = Date.now();
-    const cacheBoundary =
-      systemCacheSplit &&
-      typeof systemCacheBoundary === "number" &&
-      systemPrompt.slice(0, systemCacheBoundary).isWellFormed()
-        ? systemCacheBoundary
-        : undefined;
     let reconstructed: PiReplayMessage[] | null;
     try {
       reconstructed = reconstructMessagesFromHistory(history);
@@ -1527,6 +1497,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
     const modelRuntime = await buildModelRuntime(
       turnProviderKeys ?? (await resolveProviderKeys()),
       turnProviderKeys ? undefined : modelGateway,
+      systemCacheSplit ? "long" : undefined,
     );
     const ref: ToolContextRef = { current: null };
     const { resourceLoader, cwd, agentDir } = await createIsolatedResources(tempDirPrefix, composedPrompt);
@@ -1608,13 +1579,6 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
           ref.pendingPrepareNextTurn = undefined;
           ref.pendingTransformContext = undefined;
           applyFastSpeed(payload, ref.fast, (model as { api?: string } | undefined)?.api);
-          if (cacheBoundary !== undefined) {
-            try {
-              applySystemPromptCacheSplit(payload, cacheBoundary);
-            } catch (e) {
-              swallow("pi: system prompt cache split", e);
-            }
-          }
           const guarded = guardOutputBudget(payload, model);
           if (guarded.kind === "raised") {
             console.error(
@@ -1709,7 +1673,6 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
           turn.systemPrompt,
           turn.history,
           turn.priorTurns,
-          turn.systemCacheBoundary,
           turn.readOnly,
           turn.surfaceTools,
           turn.surfaceName,
@@ -1759,6 +1722,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
             type: "user",
             payload: {
               text: turn.input,
+              ...(turn.environment ? { environment: turn.environment } : {}),
               ...((turn.triggerTs ?? turn.entryTs) ? { ts: turn.triggerTs ?? turn.entryTs } : {}),
               ...(turn.attachments?.length ? { attachments: turn.attachments } : {}),
             },

src/harness/replay.ts

@@ -162,7 +162,10 @@ export function reconstructMessagesFromHistory(history: readonly SessionEntry[])
       if (ov) {
         if (ov.text.trim() || ov.files?.length) raw.push(userMsg(renderOverheard(ov), e.createdAt));
       } else {
-        const t = entryText(e);
+        const environment = (e.payload as { environment?: unknown } | null)?.environment;
+        const t = [entryText(e), typeof environment === "string" ? environment.trim() : ""]
+          .filter(Boolean)
+          .join("\n\n");
         if (t) raw.push(userMsg(t, e.createdAt));
       }
     } else if (e.type === "assistant") {

src/model/pi-models.ts

@@ -429,14 +429,15 @@ export function auxiliaryModelFor(baseModelId: string): string {
 }
 
 const CONTEXT_BUDGET_FRACTION = 0.5;
+const CONTEXT_BUDGET_CAP_TOKENS = 150_000;
 
 export function contextTokenBudgetForModel(id: string): number | undefined {
   const model = resolveModel(id);
   const window = model?.contextWindow;
   const output = model?.maxTokens;
   if (typeof window !== "number" || window <= 0 || typeof output !== "number" || output <= 0 || output >= window)
     return undefined;
-  return Math.floor((window - output) * CONTEXT_BUDGET_FRACTION);
+  return Math.min(CONTEXT_BUDGET_CAP_TOKENS, Math.floor((window - output) * CONTEXT_BUDGET_FRACTION));
 }
 
 export function modelSupportedByHarness(id: string | undefined, harness: string): boolean {

test/context-compaction.test.ts

@@ -1078,3 +1078,19 @@ test("a summary that landed while the pass was summarizing makes it drop its own
   assert.equal(contextSummaryPayload(summaries[0]!)?.text, "a rival fold");
   assert.equal(resetCalls.length, 0, "the dropped pass never reached its write");
 });
+
+test("the token estimate counts the environment note persisted on a user entry", () => {
+  const environment = `<environment>\n${"## What you remember\nlikes terse replies. ".repeat(40)}\n</environment>`;
+  const bare = {
+    sessionId: "s",
+    seq: 1,
+    parentSeq: null,
+    type: "user",
+    payload: { text: "hi" },
+    scopeLabel: "org:o",
+    createdAt: 1,
+  } as SessionEntry;
+  const withEnv = { ...bare, seq: 2, payload: { text: "hi", environment } } as SessionEntry;
+  const delta = estimateHistoryTokens([withEnv]) - estimateHistoryTokens([bare]);
+  assert.ok(delta >= countTokens(environment) * 0.9, `environment tokens must be counted (delta ${delta})`);
+});

test/model-overlay.test.ts

@@ -124,7 +124,7 @@ test("store reload and independent readers observe edits and soft deletion witho
   assert.equal(resolveModel(MODEL_ID), undefined);
   await reader.refresh();
   const inFlight = getRequiredModel(MODEL_ID);
-  assert.equal(contextTokenBudgetForModel(MODEL_ID), 190_000);
+  assert.equal(contextTokenBudgetForModel(MODEL_ID), 150_000);
   assert.deepEqual(inFlight.cost, spec.cost);
   assert.ok(defaultWebuiModelIds().includes(MODEL_ID));
   assert.equal(validateWebTurnModelOptions({ model: MODEL_ID }, null), null);
@@ -137,7 +137,7 @@ test("store reload and independent readers observe edits and soft deletion witho
     "editor",
   );
   await reader.refresh();
-  assert.equal(contextTokenBudgetForModel(MODEL_ID), 290_000);
+  assert.equal(getRequiredModel(MODEL_ID).contextWindow, 600_000);
   assert.equal(getRequiredModel(MODEL_ID).cost.input, 1);
   assert.equal(getRequiredModel(MODEL_ID).cost.tiers, undefined);
   assert.equal(inFlight.contextWindow, 400_000);