From 4db3733988d1cbc2b0fd5b875b47adbb22701e0b Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 18 Aug 2026 11:50:15 -0400 Subject: [PATCH] feat(replication): pace banked-gap reconnects to min(15s, blobGapReconnectMs/2) (#699) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The barrier-drained immediate reconnect is banking-gated but not rate-bounded: under a MOVING transient fault supply every cycle banks a little, so cycles run back-to-back (measured: 27 copy starts in 76s where 5.2.2 paced 4 at the watchdog interval), and each reconnect aborts in-flight receives, minting PENDING stubs at that rate. Pace banked reconnects to half the watchdog interval, capped at 15s; a floored cycle leaves the watchdog as pacer, exactly like a zero-bank cycle. Fixed damaged sets (the kohls case) bank nothing per cycle and are unaffected. Measured trade (sustained-fault harness): copy starts 27 -> 7, watchdog fires 1 -> 3, but blobs re-received in a 17s quiet window 1115 -> 2994 — a floored cycle keeps walking the unbankable tail that the immediate reconnect deliberately truncates. Reconnect/abort churn down, re-streamed bytes up. copyGapCursorBanking passes either way (floor=1.5s at its 3s knob). Co-Authored-By: Claude Fable 5 --- replication/replicationConnection.ts | 32 ++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/replication/replicationConnection.ts b/replication/replicationConnection.ts index a771fbe23..959a0ac15 100644 --- a/replication/replicationConnection.ts +++ b/replication/replicationConnection.ts @@ -2612,12 +2612,32 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: any) // the next pass re-delivers — this is what makes gap-heal cadence track walk time rather // than blobGapReconnectMs. Banking-gated by construction (this persist IS banked // progress), so a link that banks nothing still paces at the watchdog interval. (#699) - if (immediate && copyWatermark.barrierDrained) { - logger.warn?.( - `Banked copy cursor at held blob gap for ${remoteNodeName}; reconnecting immediately to re-stream from it (harper-pro#699)` - ); - if (options.connection) options.connection.forceReconnect(); - else ws.terminate(); + // FLOOR on the immediate reconnect: banking-gated is not rate-bounded. Under a MOVING + // transient fault supply (live write contention — distinct from a fixed damaged set, + // which banks nothing and stays watchdog-paced) every cycle banks a little, so + // back-to-back reconnects re-walk the tail continuously (measured: 27 copy starts in + // 76s) — and each reconnect aborts in-flight receives, minting PENDING stubs (#481) at + // that rate. Pace banked reconnects to half the watchdog interval, capped at 15s: heal + // cadence still tracks walk time for any realistic knob value, while a fault-dense link + // cannot churn faster than ~4 reconnects/min. The stamp lives on the shared connection + // object (each cycle is a fresh closure); a floored cycle leaves the watchdog as pacer, + // exactly like a zero-bank cycle. Inbound sessions (no connection object) have nowhere + // to carry the stamp, so they stay watchdog-paced entirely. + if (immediate && copyWatermark.barrierDrained && options.connection) { + const floorMs = Math.min(15000, Math.floor(blobGapReconnectMs / 2)); + const sinceLast = Date.now() - ((options.connection as any).lastBankedReconnectAt ?? 0); + if (sinceLast >= floorMs) { + (options.connection as any).lastBankedReconnectAt = Date.now(); + logger.warn?.( + `Banked copy cursor at held blob gap for ${remoteNodeName}; reconnecting immediately to re-stream from it (harper-pro#699)` + ); + options.connection.forceReconnect(); + } else { + logger.debug?.( + connectionId, + `banked copy cursor persisted; deferring the immediate reconnect (${sinceLast}ms since the last one, floor ${floorMs}ms) — the blob-gap watchdog paces this cycle (harper-pro#699)` + ); + } } }) .catch(onPersistFailure)