From 93eb5d7ef6f631d0e1664b40b33c45304f0b0c3c Mon Sep 17 00:00:00 2001
From: Jakub Hrozek <jakub@stacklok.com>
Date: Fri, 17 Jul 2026 20:35:00 +0100
Subject: [PATCH 01/10] Probe: drop --debug from conformance proxy (test flake
 masking)

DRAFT probe, not for merge. The tools-call-sampling conformance flake stopped
appearing on runs that carry `thv run --debug` (4/4 debug runs passed vs a >50%
base fail rate), suggesting the debug log I/O perturbs the timing race and masks
the failure.

Drop `--debug` while keeping the proxy-log capture (at default INFO level) to
test that hypothesis: if this reliably fails tools-call-sampling again, --debug
was masking it, and the captured INFO-level proxy logs (which still record a
session/-32001 event) give a non-perturbing root-cause path.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 test/conformance/run-conformance.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/conformance/run-conformance.sh b/test/conformance/run-conformance.sh
index 9b679f34ae..e0d14376e7 100755
--- a/test/conformance/run-conformance.sh
+++ b/test/conformance/run-conformance.sh
@@ -66,7 +66,7 @@ docker build -t "${IMAGE}" "${SERVER_DIR}"
 echo "==> Starting server through ToolHive"
 "${THV_BINARY}" rm -f "${SERVER_NAME}" >/dev/null 2>&1 || true
 "${THV_BINARY}" run "${IMAGE}" \
-  --transport streamable-http --target-port 3000 --name "${SERVER_NAME}" --debug
+  --transport streamable-http --target-port 3000 --name "${SERVER_NAME}"
 
 echo "==> Resolving proxy URL"
 URL=""

From a855a656da667c98c1920961ba254dd55db624ed Mon Sep 17 00:00:00 2001
From: Jakub Hrozek <jakub@stacklok.com>
Date: Mon, 20 Jul 2026 09:38:19 +0100
Subject: [PATCH 02/10] Capture backend server logs in conformance CI

Debug logs from PR #5839's failing run showed --debug does NOT mask the
sampling flake: the run failed with --debug on, and the trace localized
the stall to a missing server->client sampling/createMessage delivery
(the client never sends its sampling-response POST). So restore --debug
(it is our best signal, not a mask) and add the one artifact still
missing: the Node backend container's own logs, which reveal whether the
backend emitted the sampling request (-> proxy relay race) or not
(-> backend-side). SSE bodies are not in the proxy log, so this is the
only way to split those two legs.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 test/conformance/run-conformance.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/test/conformance/run-conformance.sh b/test/conformance/run-conformance.sh
index e0d14376e7..14157626fc 100755
--- a/test/conformance/run-conformance.sh
+++ b/test/conformance/run-conformance.sh
@@ -35,6 +35,14 @@ cleanup() {
   # results artifact, not just the client-side checks.json.
   mkdir -p "${RESULTS_DIR}"
   "${THV_BINARY}" logs --proxy "${SERVER_NAME}" > "${RESULTS_DIR}/proxy-logs.txt" 2>&1 || true
+  # Backend (Node everything-server) logs: the proxy log cannot show whether the
+  # backend emitted the server->client sampling/createMessage request (SSE bodies
+  # are not logged), so capture the container's own stdout/stderr. This is the
+  # missing leg for diagnosing the tools-call-sampling round-trip flake. Must run
+  # before `thv rm`, which destroys the container.
+  "${THV_BINARY}" logs "${SERVER_NAME}" > "${RESULTS_DIR}/backend-logs.txt" 2>&1 || true
+  backend_ctr="$(docker ps -a --filter "ancestor=${IMAGE}" --format '{{.Names}}' 2>/dev/null | head -1)"
+  [ -n "${backend_ctr}" ] && docker logs "${backend_ctr}" > "${RESULTS_DIR}/backend-docker-logs.txt" 2>&1 || true
   "${THV_BINARY}" rm "${SERVER_NAME}" >/dev/null 2>&1 || true
   [ -n "${CLONE_DIR}" ] && rm -rf "${CLONE_DIR}" || true
 }
@@ -66,7 +74,7 @@ docker build -t "${IMAGE}" "${SERVER_DIR}"
 echo "==> Starting server through ToolHive"
 "${THV_BINARY}" rm -f "${SERVER_NAME}" >/dev/null 2>&1 || true
 "${THV_BINARY}" run "${IMAGE}" \
-  --transport streamable-http --target-port 3000 --name "${SERVER_NAME}"
+  --transport streamable-http --target-port 3000 --name "${SERVER_NAME}" --debug
 
 echo "==> Resolving proxy URL"
 URL=""

From 257c99813cef5804630ebd3e6a6fa72487bc2c76 Mon Sep 17 00:00:00 2001
From: Jakub Hrozek <jakub@stacklok.com>
Date: Mon, 20 Jul 2026 10:02:40 +0100
Subject: [PATCH 03/10] Capture a pcap in conformance CI for the sampling flake

The backend/proxy logs cannot show whether the backend actually emitted the
server->client sampling/createMessage on the SSE stream (SSE bodies are not
logged, and the Node reference server may not log its own writes). A packet
capture shows the bytes on both legs and definitively splits the remaining
fork: sampling/createMessage present on backend->proxy but absent on
proxy->client => ToolHive relay bug; absent on both => backend never emitted.

Start tcpdump before the suite and stop it in cleanup(). `-i any` covers the
loopback and docker-bridge legs (the backend is a docker-published port via
docker-proxy); full snaplen is required to see SSE payloads; -U flushes per
packet. Best-effort and non-fatal when tcpdump/passwordless sudo are absent
(local macOS). The pcap lands in the results dir already uploaded as an
artifact, so no workflow change is needed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 test/conformance/run-conformance.sh | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/test/conformance/run-conformance.sh b/test/conformance/run-conformance.sh
index 14157626fc..993d04860e 100755
--- a/test/conformance/run-conformance.sh
+++ b/test/conformance/run-conformance.sh
@@ -30,6 +30,11 @@ export TOOLHIVE_SKIP_DESKTOP_CHECK=1
 
 CLONE_DIR=""
 cleanup() {
+  # Stop the packet capture first so tcpdump flushes the pcap to disk before
+  # the container/proxy are torn down. Matches on the output file so we don't
+  # kill unrelated tcpdumps. No-op when no capture was started.
+  sudo pkill -TERM -f 'tcpdump.*cf\.pcap' 2>/dev/null || true
+  sleep 1
   # Diagnostics: capture the ToolHive proxy logs before teardown so a CI failure
   # (e.g. the sampling round-trip timing out) is debuggable from the uploaded
   # results artifact, not just the client-side checks.json.
@@ -103,6 +108,23 @@ done
 echo "==> Running conformance suite (${CONFORMANCE_SUITE})"
 rm -rf "${RESULTS_DIR}"
 mkdir -p "${RESULTS_DIR}"
+
+# Packet capture (best-effort; CI/Linux only). Captures client<->proxy and
+# proxy<->backend TCP so a sampling-flake failure can be localized to the
+# server->client SSE delivery leg: does `sampling/createMessage` reach the proxy
+# from the backend, and does the proxy relay it to the client? `-i any` covers
+# both the loopback and docker-bridge legs; full snaplen (-s 0) is required to
+# see SSE payloads; -U flushes per packet so the pcap survives an abrupt stop.
+# Non-fatal when tcpdump or passwordless sudo are unavailable (e.g. local macOS).
+# Stopped in cleanup().
+if command -v tcpdump >/dev/null 2>&1 && sudo -n true 2>/dev/null; then
+  sudo tcpdump -i any -U -s 0 -w "${RESULTS_DIR}/cf.pcap" tcp >/dev/null 2>&1 &
+  sleep 1
+  echo "==> packet capture started -> ${RESULTS_DIR}/cf.pcap"
+else
+  echo "==> tcpdump/passwordless sudo unavailable; skipping packet capture"
+fi
+
 npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
   --url "${URL}" \
   --suite "${CONFORMANCE_SUITE}" \

From 581318cf9e29d3b86b5ceb187d13fbe097a0923e Mon Sep 17 00:00:00 2001
From: Jakub Hrozek <jakub@stacklok.com>
Date: Mon, 20 Jul 2026 11:32:24 +0100
Subject: [PATCH 04/10] Add direct-vs-stack sampling comparison to conformance
 CI

The pcap showed the backend never emits sampling/createMessage and that
ToolHive fronts the workload with a Squid ingress proxy (createIngressSquidContainer)
plus the Go transparent proxy. It is unresolved whether that proxy stack causes
or merely aggravates the client-ordering race that starves the server->client
sampling stream.

Add compare-sampling.sh: run the conformance suite N times against the
reference container DIRECTLY (no thv) vs THROUGH the thv stack (Go proxy ->
Squid ingress), and tally the tools-call-sampling failure rate. If it only
flakes through the stack, ToolHive owns it (likely the Squid ingress). Wire it
as a diagnostic step in the conformance job (draft-only) and bump the job
timeout to 30m to accommodate the extra runs.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 .github/workflows/e2e-tests.yml      |  8 ++-
 test/conformance/compare-sampling.sh | 79 ++++++++++++++++++++++++++++
 2 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100755 test/conformance/compare-sampling.sh

diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
index 6c909ae611..0900bd9bf2 100644
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@@ -159,7 +159,7 @@ jobs:
     name: MCP Conformance
     runs-on: ubuntu-8cores-32gb
     needs: build-binary
-    timeout-minutes: 20
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
 
@@ -187,6 +187,12 @@ jobs:
           THV_BINARY: ${{ github.workspace }}/bin/thv
         run: ./test/conformance/run-conformance.sh
 
+      - name: Compare sampling direct vs ToolHive stack (diagnostic)
+        if: always()
+        env:
+          THV_BINARY: ${{ github.workspace }}/bin/thv
+        run: ./test/conformance/compare-sampling.sh
+
       - name: Upload conformance results
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
diff --git a/test/conformance/compare-sampling.sh b/test/conformance/compare-sampling.sh
new file mode 100755
index 0000000000..6ac48263e6
--- /dev/null
+++ b/test/conformance/compare-sampling.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+# Diagnostic: does tools-call-sampling flake THROUGH the ToolHive stack
+# (Go transparent proxy -> Squid ingress -> container) but not DIRECT
+# (client -> container)? Runs the conformance suite N times each way and
+# tallies the tools-call-sampling result. Reuses the mcp-conformance-server:ci
+# image built by run-conformance.sh (run this step after it).
+#
+# Env: CONFORMANCE_VERSION (default 0.1.16), THV_BINARY (default thv),
+#      SAMPLING_COMPARE_N (default 4).
+set -uo pipefail
+
+CONFORMANCE_VERSION="${CONFORMANCE_VERSION:-0.1.16}"
+THV_BINARY="${THV_BINARY:-thv}"
+N="${SAMPLING_COMPARE_N:-4}"
+IMAGE="mcp-conformance-server:ci"
+DIRECT_NAME="conf-cmp-direct"
+PROXY_NAME="conf-cmp-proxy"
+DIRECT_PORT=3009
+
+export TOOLHIVE_DEV=true
+export TOOLHIVE_SKIP_DESKTOP_CHECK=1
+
+cleanup() {
+  docker rm -f "${DIRECT_NAME}" >/dev/null 2>&1 || true
+  "${THV_BINARY}" rm -f "${PROXY_NAME}" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+if ! docker image inspect "${IMAGE}" >/dev/null 2>&1; then
+  echo "ERROR: ${IMAGE} not present; run run-conformance.sh first (it builds it)." >&2
+  exit 1
+fi
+
+wait_ready() { # $1=url
+  for _ in $(seq 1 30); do
+    code="$(curl -s -o /dev/null -w '%{http_code}' "$1" -X POST \
+      -H 'content-type: application/json' -H 'accept: application/json, text/event-stream' \
+      -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"probe","version":"1"}}}' 2>/dev/null || true)"
+    [ "${code}" = "200" ] && return 0
+    sleep 2
+  done
+  return 1
+}
+
+run_suite() { # $1=url -> echoes PASS or FAIL for tools-call-sampling
+  out="$(npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
+    --url "$1" --suite active -o /tmp/cmp-results 2>&1 || true)"
+  if echo "$out" | grep -qE "✓ tools-call-sampling"; then echo PASS
+  elif echo "$out" | grep -qE "✗ tools-call-sampling"; then echo FAIL
+  else echo "UNKNOWN"; fi
+}
+
+echo "==> DIRECT: client -> container (no ToolHive)"
+docker rm -f "${DIRECT_NAME}" >/dev/null 2>&1 || true
+docker run -d --rm -p "${DIRECT_PORT}:3000" --name "${DIRECT_NAME}" "${IMAGE}" >/dev/null
+wait_ready "http://127.0.0.1:${DIRECT_PORT}/mcp" || { echo "direct server not ready"; }
+dfail=0
+for i in $(seq 1 "$N"); do r="$(run_suite "http://127.0.0.1:${DIRECT_PORT}/mcp")"; echo "  direct $i: $r"; [ "$r" = FAIL ] && dfail=$((dfail+1)); done
+docker rm -f "${DIRECT_NAME}" >/dev/null 2>&1 || true
+
+echo "==> STACK: client -> thv (Go proxy -> Squid ingress) -> container"
+"${THV_BINARY}" rm -f "${PROXY_NAME}" >/dev/null 2>&1 || true
+"${THV_BINARY}" run "${IMAGE}" --transport streamable-http --target-port 3000 --name "${PROXY_NAME}"
+URL=""
+for _ in $(seq 1 30); do
+  URL="$("${THV_BINARY}" list --format json 2>/dev/null | python3 -c "import sys,json;d=json.load(sys.stdin);print(next((w['url'] for w in d if w.get('name')=='${PROXY_NAME}' and w.get('status')=='running' and w.get('url')),''))" 2>/dev/null || true)"
+  [ -n "${URL}" ] && break
+  sleep 2
+done
+[ -z "${URL}" ] && { echo "ERROR: could not resolve ${PROXY_NAME} URL" >&2; exit 1; }
+wait_ready "${URL}" || { echo "stack server not ready"; }
+pfail=0
+for i in $(seq 1 "$N"); do r="$(run_suite "${URL}")"; echo "  stack  $i: $r"; [ "$r" = FAIL ] && pfail=$((pfail+1)); done
+"${THV_BINARY}" rm -f "${PROXY_NAME}" >/dev/null 2>&1 || true
+
+echo ""
+echo "===== tools-call-sampling failure rate ====="
+echo "DIRECT (no ToolHive):        ${dfail} / ${N} failed"
+echo "STACK  (thv Go+Squid proxy): ${pfail} / ${N} failed"

From 5e7f935e018099204dd898979590e889004d2fdb Mon Sep 17 00:00:00 2001
From: Jakub Hrozek <jakub@stacklok.com>
Date: Mon, 20 Jul 2026 12:03:56 +0100
Subject: [PATCH 05/10] Add squid-ingress tier to sampling comparison

The direct-vs-stack comparison proves the ToolHive stack triggers the
sampling flake but can't say whether the Squid ingress or the Go
transparent proxy widens the tools/call-vs-GET-SSE race. Add a middle
tier that hits the ingress Squid's published host port directly, dropping
the Go proxy from the path, so a failing CI run localizes the culprit:
DIRECT green + SQUID flaky => Squid; SQUID green + STACK bad => Go proxy.
Switch the tally to the single tools-call-sampling scenario so three
tiers stay affordable in CI.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 test/conformance/compare-sampling.sh | 85 ++++++++++++++++++++--------
 1 file changed, 62 insertions(+), 23 deletions(-)

diff --git a/test/conformance/compare-sampling.sh b/test/conformance/compare-sampling.sh
index 6ac48263e6..a28f74560e 100755
--- a/test/conformance/compare-sampling.sh
+++ b/test/conformance/compare-sampling.sh
@@ -1,17 +1,28 @@
 #!/bin/bash
-# Diagnostic: does tools-call-sampling flake THROUGH the ToolHive stack
-# (Go transparent proxy -> Squid ingress -> container) but not DIRECT
-# (client -> container)? Runs the conformance suite N times each way and
-# tallies the tools-call-sampling result. Reuses the mcp-conformance-server:ci
-# image built by run-conformance.sh (run this step after it).
+# Diagnostic: WHERE does tools-call-sampling flake? The reference server sends
+# its server->client sampling/createMessage on the client's standalone GET SSE
+# stream, which races the tools/call POST. Direct it never flakes; through the
+# ToolHive stack it does. This tallies the sampling result across three tiers to
+# localize which stack element widens the race:
+#
+#   DIRECT : client -> container                              (no ToolHive)
+#   SQUID  : client -> Squid ingress -> container             (no Go proxy)
+#   STACK  : client -> Go transparent proxy -> Squid -> container (full)
+#
+# DIRECT green + SQUID flaky            -> the Squid ingress is the trigger.
+# DIRECT green + SQUID green + STACK bad -> the Go transparent proxy is the trigger.
+# all flaky                            -> the container/network setup itself.
+#
+# Reuses the mcp-conformance-server:ci image built by run-conformance.sh (run
+# this step after it).
 #
 # Env: CONFORMANCE_VERSION (default 0.1.16), THV_BINARY (default thv),
-#      SAMPLING_COMPARE_N (default 4).
+#      SAMPLING_COMPARE_N (default 6).
 set -uo pipefail
 
 CONFORMANCE_VERSION="${CONFORMANCE_VERSION:-0.1.16}"
 THV_BINARY="${THV_BINARY:-thv}"
-N="${SAMPLING_COMPARE_N:-4}"
+N="${SAMPLING_COMPARE_N:-6}"
 IMAGE="mcp-conformance-server:ci"
 DIRECT_NAME="conf-cmp-direct"
 PROXY_NAME="conf-cmp-proxy"
@@ -42,23 +53,31 @@ wait_ready() { # $1=url
   return 1
 }
 
-run_suite() { # $1=url -> echoes PASS or FAIL for tools-call-sampling
-  out="$(npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
-    --url "$1" --suite active -o /tmp/cmp-results 2>&1 || true)"
-  if echo "$out" | grep -qE "✓ tools-call-sampling"; then echo PASS
-  elif echo "$out" | grep -qE "✗ tools-call-sampling"; then echo FAIL
-  else echo "UNKNOWN"; fi
+# Runs the sampling scenario $N times against $2, tallies failures into the
+# variable named by $1. Echoes per-iteration results.
+tally() { # $1=label $2=url -> echoes "<label>: <fail> / <N> failed"; sets _fail
+  local label="$1" url="$2" fail=0 r
+  for i in $(seq 1 "$N"); do
+    out="$(npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
+      --url "${url}" --scenario tools-call-sampling -o /tmp/cmp-results 2>&1 || true)"
+    if echo "$out" | grep -qE "✓ tools-call-sampling"; then r=PASS
+    elif echo "$out" | grep -qE "✗ tools-call-sampling"; then r=FAIL; fail=$((fail+1))
+    else r=UNKNOWN; fi
+    echo "  ${label} ${i}: ${r}"
+  done
+  _fail="${fail}"
 }
 
+# --- DIRECT: plain container, no ToolHive ------------------------------------
 echo "==> DIRECT: client -> container (no ToolHive)"
 docker rm -f "${DIRECT_NAME}" >/dev/null 2>&1 || true
 docker run -d --rm -p "${DIRECT_PORT}:3000" --name "${DIRECT_NAME}" "${IMAGE}" >/dev/null
-wait_ready "http://127.0.0.1:${DIRECT_PORT}/mcp" || { echo "direct server not ready"; }
-dfail=0
-for i in $(seq 1 "$N"); do r="$(run_suite "http://127.0.0.1:${DIRECT_PORT}/mcp")"; echo "  direct $i: $r"; [ "$r" = FAIL ] && dfail=$((dfail+1)); done
+wait_ready "http://127.0.0.1:${DIRECT_PORT}/mcp" || echo "  (direct server not ready)"
+tally direct "http://127.0.0.1:${DIRECT_PORT}/mcp"; dfail="${_fail}"
 docker rm -f "${DIRECT_NAME}" >/dev/null 2>&1 || true
 
-echo "==> STACK: client -> thv (Go proxy -> Squid ingress) -> container"
+# --- Bring up ONE ToolHive stack; it gives both the SQUID and STACK tiers -----
+echo "==> Starting ToolHive stack (${PROXY_NAME})"
 "${THV_BINARY}" rm -f "${PROXY_NAME}" >/dev/null 2>&1 || true
 "${THV_BINARY}" run "${IMAGE}" --transport streamable-http --target-port 3000 --name "${PROXY_NAME}"
 URL=""
@@ -68,12 +87,32 @@ for _ in $(seq 1 30); do
   sleep 2
 done
 [ -z "${URL}" ] && { echo "ERROR: could not resolve ${PROXY_NAME} URL" >&2; exit 1; }
-wait_ready "${URL}" || { echo "stack server not ready"; }
-pfail=0
-for i in $(seq 1 "$N"); do r="$(run_suite "${URL}")"; echo "  stack  $i: $r"; [ "$r" = FAIL ] && pfail=$((pfail+1)); done
+
+# --- SQUID: hit the ingress Squid directly, bypassing the Go transparent proxy.
+# thv publishes the ingress squid (container "<name>-ingress") to a host port;
+# the Go proxy forwards to it. Reach it directly to drop the Go proxy from the path.
+squid_hp="$(docker port "${PROXY_NAME}-ingress" 2>/dev/null | awk -F: 'NR==1{print $NF}')"
+sfail="n/a"
+if [ -n "${squid_hp}" ]; then
+  SQUID_URL="http://127.0.0.1:${squid_hp}/mcp"
+  echo "==> SQUID: client -> Squid ingress (${SQUID_URL}) -> container (no Go proxy)"
+  if wait_ready "${SQUID_URL}"; then
+    tally squid "${SQUID_URL}"; sfail="${_fail}"
+  else
+    echo "  (squid ingress not reachable at ${SQUID_URL}; skipping tier)"
+  fi
+else
+  echo "==> SQUID: could not resolve ${PROXY_NAME}-ingress host port; skipping tier"
+fi
+
+# --- STACK: full path through the Go transparent proxy ------------------------
+echo "==> STACK: client -> thv (Go proxy -> Squid ingress) -> container"
+wait_ready "${URL}" || echo "  (stack server not ready)"
+tally stack "${URL}"; pfail="${_fail}"
 "${THV_BINARY}" rm -f "${PROXY_NAME}" >/dev/null 2>&1 || true
 
 echo ""
-echo "===== tools-call-sampling failure rate ====="
-echo "DIRECT (no ToolHive):        ${dfail} / ${N} failed"
-echo "STACK  (thv Go+Squid proxy): ${pfail} / ${N} failed"
+echo "===== tools-call-sampling failure rate (n=${N} per tier) ====="
+echo "DIRECT (container only):        ${dfail} / ${N} failed"
+echo "SQUID  (Squid ingress only):    ${sfail} / ${N} failed"
+echo "STACK  (Go proxy + Squid):      ${pfail} / ${N} failed"

From 1275d2d1cadc97a5965603bad570d6bdd2430b04 Mon Sep 17 00:00:00 2001
From: Jakub Hrozek <jakub@stacklok.com>
Date: Mon, 20 Jul 2026 12:27:03 +0100
Subject: [PATCH 06/10] Fix sampling comparison: use full suite, not --scenario

The 3-tier comparison ran isolated `--scenario tools-call-sampling`, which
completes in ~1s and never reproduces the 60s hang (the flake only manifests
in the full-suite context), and its result grep matched the suite-summary
format, so every tier reported UNKNOWN (0/N). Run the full `active` suite per
iteration (which does reproduce), keep the suite-summary grep, and drop N to 4
so 3 tiers fit the job timeout.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 test/conformance/compare-sampling.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/test/conformance/compare-sampling.sh b/test/conformance/compare-sampling.sh
index a28f74560e..ec03f067f0 100755
--- a/test/conformance/compare-sampling.sh
+++ b/test/conformance/compare-sampling.sh
@@ -22,7 +22,7 @@ set -uo pipefail
 
 CONFORMANCE_VERSION="${CONFORMANCE_VERSION:-0.1.16}"
 THV_BINARY="${THV_BINARY:-thv}"
-N="${SAMPLING_COMPARE_N:-6}"
+N="${SAMPLING_COMPARE_N:-4}"
 IMAGE="mcp-conformance-server:ci"
 DIRECT_NAME="conf-cmp-direct"
 PROXY_NAME="conf-cmp-proxy"
@@ -58,8 +58,11 @@ wait_ready() { # $1=url
 tally() { # $1=label $2=url -> echoes "<label>: <fail> / <N> failed"; sets _fail
   local label="$1" url="$2" fail=0 r
   for i in $(seq 1 "$N"); do
+    # Run the FULL active suite, not --scenario: the sampling flake only
+    # reproduces in the full-suite context (isolated --scenario completes in
+    # ~1s and never hits the 60s hang). Grep the suite summary line.
     out="$(npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
-      --url "${url}" --scenario tools-call-sampling -o /tmp/cmp-results 2>&1 || true)"
+      --url "${url}" --suite active -o /tmp/cmp-results 2>&1 || true)"
     if echo "$out" | grep -qE "✓ tools-call-sampling"; then r=PASS
     elif echo "$out" | grep -qE "✗ tools-call-sampling"; then r=FAIL; fail=$((fail+1))
     else r=UNKNOWN; fi

From 01604731c52c3f123e9a80951937bfd21815480c Mon Sep 17 00:00:00 2001
From: Jakub Hrozek <jakub@stacklok.com>
Date: Mon, 20 Jul 2026 14:44:37 +0100
Subject: [PATCH 07/10] Make sampling comparison cold-start per iteration

The flake only reproduces on the FIRST suite run against a freshly created
stack; the previous comparison reused one warm stack across iterations and
so came back all-green (inconclusive). Give every iteration its own fresh
stack, and measure SQUID and STACK from SEPARATE fresh stacks (sharing one
would leave the second tier warm and hide the flake). This is what lets CI
localize whether the Squid ingress or the Go transparent proxy widens the
tools/call-vs-GET-SSE race. Costs 2N thv runs; N defaults to 3 to fit the
30m job budget.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 test/conformance/compare-sampling.sh | 141 +++++++++++++++------------
 1 file changed, 80 insertions(+), 61 deletions(-)

diff --git a/test/conformance/compare-sampling.sh b/test/conformance/compare-sampling.sh
index ec03f067f0..13c51a1a8d 100755
--- a/test/conformance/compare-sampling.sh
+++ b/test/conformance/compare-sampling.sh
@@ -5,24 +5,30 @@
 # ToolHive stack it does. This tallies the sampling result across three tiers to
 # localize which stack element widens the race:
 #
-#   DIRECT : client -> container                              (no ToolHive)
-#   SQUID  : client -> Squid ingress -> container             (no Go proxy)
+#   DIRECT : client -> container                                  (no ToolHive)
+#   SQUID  : client -> Squid ingress -> container                 (no Go proxy)
 #   STACK  : client -> Go transparent proxy -> Squid -> container (full)
 #
-# DIRECT green + SQUID flaky            -> the Squid ingress is the trigger.
+# DIRECT green + SQUID flaky             -> the Squid ingress is the trigger.
 # DIRECT green + SQUID green + STACK bad -> the Go transparent proxy is the trigger.
-# all flaky                            -> the container/network setup itself.
+# all flaky                             -> the container/network setup itself.
+#
+# COLD-START is the crux: the flake only manifests on the FIRST suite run
+# against a freshly-created stack; reusing one warm stack across iterations
+# never reproduces it. So every iteration gets its OWN fresh stack/container,
+# and SQUID and STACK use SEPARATE fresh stacks (measuring both off one stack
+# would leave the second tier warm and hide the flake). This costs 2N `thv run`s.
 #
 # Reuses the mcp-conformance-server:ci image built by run-conformance.sh (run
 # this step after it).
 #
 # Env: CONFORMANCE_VERSION (default 0.1.16), THV_BINARY (default thv),
-#      SAMPLING_COMPARE_N (default 6).
+#      SAMPLING_COMPARE_N (default 4) = cold stacks per tier.
 set -uo pipefail
 
 CONFORMANCE_VERSION="${CONFORMANCE_VERSION:-0.1.16}"
 THV_BINARY="${THV_BINARY:-thv}"
-N="${SAMPLING_COMPARE_N:-4}"
+N="${SAMPLING_COMPARE_N:-3}"  # cold stacks per tier; 2N thv runs must fit the 30m job budget
 IMAGE="mcp-conformance-server:ci"
 DIRECT_NAME="conf-cmp-direct"
 PROXY_NAME="conf-cmp-proxy"
@@ -53,69 +59,82 @@ wait_ready() { # $1=url
   return 1
 }
 
-# Runs the sampling scenario $N times against $2, tallies failures into the
-# variable named by $1. Echoes per-iteration results.
-tally() { # $1=label $2=url -> echoes "<label>: <fail> / <N> failed"; sets _fail
-  local label="$1" url="$2" fail=0 r
-  for i in $(seq 1 "$N"); do
-    # Run the FULL active suite, not --scenario: the sampling flake only
-    # reproduces in the full-suite context (isolated --scenario completes in
-    # ~1s and never hits the 60s hang). Grep the suite summary line.
-    out="$(npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
-      --url "${url}" --suite active -o /tmp/cmp-results 2>&1 || true)"
-    if echo "$out" | grep -qE "✓ tools-call-sampling"; then r=PASS
-    elif echo "$out" | grep -qE "✗ tools-call-sampling"; then r=FAIL; fail=$((fail+1))
-    else r=UNKNOWN; fi
-    echo "  ${label} ${i}: ${r}"
+# One FULL active-suite run against $1; echoes PASS/FAIL/UNKNOWN for the sampling
+# scenario. --scenario in isolation never reproduces the hang; the full suite does.
+run_suite_once() { # $1=url
+  local out
+  out="$(npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
+    --url "$1" --suite active -o /tmp/cmp-results 2>&1 || true)"
+  if echo "$out" | grep -qE "✓ tools-call-sampling"; then echo PASS
+  elif echo "$out" | grep -qE "✗ tools-call-sampling"; then echo FAIL
+  else echo UNKNOWN; fi
+}
+
+# Start a fresh (cold) ToolHive stack; sets STACK_URL and SQUID_URL (SQUID_URL
+# empty if the ingress host port can't be resolved).
+start_stack() {
+  "${THV_BINARY}" rm -f "${PROXY_NAME}" >/dev/null 2>&1 || true
+  "${THV_BINARY}" run "${IMAGE}" --transport streamable-http --target-port 3000 --name "${PROXY_NAME}"
+  STACK_URL=""
+  for _ in $(seq 1 30); do
+    STACK_URL="$("${THV_BINARY}" list --format json 2>/dev/null | python3 -c "import sys,json;d=json.load(sys.stdin);print(next((w['url'] for w in d if w.get('name')=='${PROXY_NAME}' and w.get('status')=='running' and w.get('url')),''))" 2>/dev/null || true)"
+    [ -n "${STACK_URL}" ] && break
+    sleep 2
   done
-  _fail="${fail}"
+  local hp; hp="$(docker port "${PROXY_NAME}-ingress" 2>/dev/null | awk -F: 'NR==1{print $NF}')"
+  SQUID_URL=""; [ -n "${hp}" ] && SQUID_URL="http://127.0.0.1:${hp}/mcp"
 }
+stop_stack() { "${THV_BINARY}" rm -f "${PROXY_NAME}" >/dev/null 2>&1 || true; }
 
-# --- DIRECT: plain container, no ToolHive ------------------------------------
-echo "==> DIRECT: client -> container (no ToolHive)"
-docker rm -f "${DIRECT_NAME}" >/dev/null 2>&1 || true
-docker run -d --rm -p "${DIRECT_PORT}:3000" --name "${DIRECT_NAME}" "${IMAGE}" >/dev/null
-wait_ready "http://127.0.0.1:${DIRECT_PORT}/mcp" || echo "  (direct server not ready)"
-tally direct "http://127.0.0.1:${DIRECT_PORT}/mcp"; dfail="${_fail}"
-docker rm -f "${DIRECT_NAME}" >/dev/null 2>&1 || true
+# --- DIRECT: fresh plain container per iteration (cold control, no ToolHive) --
+echo "==> DIRECT: client -> container (no ToolHive), cold per iteration"
+dfail=0
+for i in $(seq 1 "$N"); do
+  docker rm -f "${DIRECT_NAME}" >/dev/null 2>&1 || true
+  docker run -d --rm -p "${DIRECT_PORT}:3000" --name "${DIRECT_NAME}" "${IMAGE}" >/dev/null
+  wait_ready "http://127.0.0.1:${DIRECT_PORT}/mcp" || echo "  (direct ${i} not ready)"
+  r="$(run_suite_once "http://127.0.0.1:${DIRECT_PORT}/mcp")"; echo "  direct ${i}: ${r}"
+  [ "${r}" = FAIL ] && dfail=$((dfail+1))
+  docker rm -f "${DIRECT_NAME}" >/dev/null 2>&1 || true
+done
 
-# --- Bring up ONE ToolHive stack; it gives both the SQUID and STACK tiers -----
-echo "==> Starting ToolHive stack (${PROXY_NAME})"
-"${THV_BINARY}" rm -f "${PROXY_NAME}" >/dev/null 2>&1 || true
-"${THV_BINARY}" run "${IMAGE}" --transport streamable-http --target-port 3000 --name "${PROXY_NAME}"
-URL=""
-for _ in $(seq 1 30); do
-  URL="$("${THV_BINARY}" list --format json 2>/dev/null | python3 -c "import sys,json;d=json.load(sys.stdin);print(next((w['url'] for w in d if w.get('name')=='${PROXY_NAME}' and w.get('status')=='running' and w.get('url')),''))" 2>/dev/null || true)"
-  [ -n "${URL}" ] && break
-  sleep 2
+# --- SQUID: fresh stack per iteration, hit the ingress Squid directly ---------
+echo "==> SQUID: client -> Squid ingress -> container (no Go proxy), cold per iteration"
+sfail=0; sruns=0
+for i in $(seq 1 "$N"); do
+  start_stack
+  if [ -n "${SQUID_URL}" ] && wait_ready "${SQUID_URL}"; then
+    r="$(run_suite_once "${SQUID_URL}")"; echo "  squid ${i}: ${r} (${SQUID_URL})"
+    sruns=$((sruns+1)); [ "${r}" = FAIL ] && sfail=$((sfail+1))
+  else
+    echo "  squid ${i}: SKIP (ingress host port unresolved/unreachable)"
+  fi
+  stop_stack
 done
-[ -z "${URL}" ] && { echo "ERROR: could not resolve ${PROXY_NAME} URL" >&2; exit 1; }
 
-# --- SQUID: hit the ingress Squid directly, bypassing the Go transparent proxy.
-# thv publishes the ingress squid (container "<name>-ingress") to a host port;
-# the Go proxy forwards to it. Reach it directly to drop the Go proxy from the path.
-squid_hp="$(docker port "${PROXY_NAME}-ingress" 2>/dev/null | awk -F: 'NR==1{print $NF}')"
-sfail="n/a"
-if [ -n "${squid_hp}" ]; then
-  SQUID_URL="http://127.0.0.1:${squid_hp}/mcp"
-  echo "==> SQUID: client -> Squid ingress (${SQUID_URL}) -> container (no Go proxy)"
-  if wait_ready "${SQUID_URL}"; then
-    tally squid "${SQUID_URL}"; sfail="${_fail}"
+# --- STACK: fresh stack per iteration, full path via the Go transparent proxy -
+echo "==> STACK: client -> thv (Go proxy -> Squid ingress) -> container, cold per iteration"
+pfail=0
+for i in $(seq 1 "$N"); do
+  start_stack
+  if wait_ready "${STACK_URL}"; then
+    r="$(run_suite_once "${STACK_URL}")"; echo "  stack ${i}: ${r}"
+    [ "${r}" = FAIL ] && pfail=$((pfail+1))
   else
-    echo "  (squid ingress not reachable at ${SQUID_URL}; skipping tier)"
+    echo "  stack ${i}: not ready"
   fi
+  stop_stack
+done
+
+echo ""
+echo "===== tools-call-sampling failure rate (cold, n=${N} per tier) ====="
+echo "DIRECT (container only):     ${dfail} / ${N} failed"
+if [ "${sruns:-0}" -gt 0 ]; then
+  echo "SQUID  (Squid ingress only): ${sfail} / ${sruns} failed"
 else
-  echo "==> SQUID: could not resolve ${PROXY_NAME}-ingress host port; skipping tier"
+  echo "SQUID  (Squid ingress only): n/a (tier unreachable)"
 fi
-
-# --- STACK: full path through the Go transparent proxy ------------------------
-echo "==> STACK: client -> thv (Go proxy -> Squid ingress) -> container"
-wait_ready "${URL}" || echo "  (stack server not ready)"
-tally stack "${URL}"; pfail="${_fail}"
-"${THV_BINARY}" rm -f "${PROXY_NAME}" >/dev/null 2>&1 || true
-
+echo "STACK  (Go proxy + Squid):   ${pfail} / ${N} failed"
 echo ""
-echo "===== tools-call-sampling failure rate (n=${N} per tier) ====="
-echo "DIRECT (container only):        ${dfail} / ${N} failed"
-echo "SQUID  (Squid ingress only):    ${sfail} / ${N} failed"
-echo "STACK  (Go proxy + Squid):      ${pfail} / ${N} failed"
+echo "Read: DIRECT+SQUID green & STACK bad => Go transparent proxy is the trigger."
+echo "      DIRECT green & SQUID bad       => Squid ingress is the trigger."

From d57b24ee9c115d46e16322bc02ad45145ca05083 Mon Sep 17 00:00:00 2001
From: Jakub Hrozek <jakub@stacklok.com>
Date: Mon, 20 Jul 2026 15:41:34 +0100
Subject: [PATCH 08/10] Warm ingress Squid peer connections with standby=2

The tools-call-sampling conformance flake was localized to the ingress
Squid: on a cold stack the first GET /mcp SSE stream pays inline DNS + TCP
connect latency to the upstream while a later tools/call POST reuses a
warmed path, so the POST reaches the server and runs its handler before
the standalone GET SSE stream is established. A server-initiated request
(sampling) then has no stream to ride and the client times out at 60s.

Add standby=2 to the ingress cache_peer so Squid keeps warm idle upstream
connections; the first GET is forwarded without inline connect latency,
removing the GET-vs-POST reorder. Config-only, no ACL change. Bump the
diagnostic compare to N=8 to validate the SQUID tier goes green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 .github/workflows/e2e-tests.yml | 1 +
 pkg/container/docker/squid.go   | 7 ++++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
index 0900bd9bf2..d00a58606b 100644
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@@ -191,6 +191,7 @@ jobs:
         if: always()
         env:
           THV_BINARY: ${{ github.workspace }}/bin/thv
+          SAMPLING_COMPARE_N: "8"
         run: ./test/conformance/compare-sampling.sh
 
       - name: Upload conformance results
diff --git a/pkg/container/docker/squid.go b/pkg/container/docker/squid.go
index 24a4f08a81..cff6766cfb 100644
--- a/pkg/container/docker/squid.go
+++ b/pkg/container/docker/squid.go
@@ -398,11 +398,16 @@ func writeIngressProxyConfig(
 ) {
 	portNum := strconv.Itoa(upstreamPort)
 	squidPortNum := strconv.Itoa(squidPort)
+	// standby=2 keeps warm idle connections open to the upstream so the first
+	// request after a cold start (notably a long-lived GET SSE stream that a
+	// server-initiated request rides on) is forwarded without paying inline DNS
+	// + TCP connect latency. Without it, the cold first GET races behind a
+	// later POST that reuses a warmed path, reordering server->client streams.
 	sb.WriteString(
 		"\n# Reverse proxy setup for port " + portNum + "\n" +
 			"http_port 0.0.0.0:" + squidPortNum + " accel defaultsite=" + serverHostname + "\n" +
 			"cache_peer " + serverHostname + " parent " + portNum + " 0 no-query originserver name=origin_" +
-			portNum + " connect-timeout=5 connect-fail-limit=5\n")
+			portNum + " connect-timeout=5 connect-fail-limit=5 standby=2\n")
 
 	// Check if inbound network permissions are configured
 	if networkPermissions != nil && networkPermissions.Inbound != nil && len(networkPermissions.Inbound.AllowHost) > 0 {

From 5fa0fa0285573bc025fc24f89605e5ac228c15e0 Mon Sep 17 00:00:00 2001
From: Jakub Hrozek <jakub@stacklok.com>
Date: Mon, 20 Jul 2026 20:12:33 +0100
Subject: [PATCH 09/10] Warm transparent proxy's upstream connection pool

On a cold start, the transparent proxy forwarded requests over a bare
shared http.DefaultTransport whose per-host idle pool starts empty, so
the first upstream connection paid inline DNS+TCP dial latency. The
client's standalone GET (SSE stream) and tools/call POST are forwarded
on separate connections, so a cold dial on the GET could delay stream
establishment past the point where the backend's tool handler sends a
server-to-client request on it, causing a client timeout.

Clone http.DefaultTransport (never mutate the shared global) with a
small idle pool, and pre-warm it at startup with a burst of
side-effect-free GET requests to the upstream. This is the Go-proxy
analog of the Squid ingress cache_peer standby=2 fix for the
squid->backend hop.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 .../proxy/transparent/transparent_proxy.go    | 68 ++++++++++++++++++-
 .../proxy/transparent/warmup_test.go          | 38 +++++++++++
 2 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100644 pkg/transport/proxy/transparent/warmup_test.go

diff --git a/pkg/transport/proxy/transparent/transparent_proxy.go b/pkg/transport/proxy/transparent/transparent_proxy.go
index 47e1266393..134451016a 100644
--- a/pkg/transport/proxy/transparent/transparent_proxy.go
+++ b/pkg/transport/proxy/transparent/transparent_proxy.go
@@ -172,6 +172,21 @@ const (
 	// Matches the value used by the vMCP server.
 	defaultReadTimeout = 30 * time.Second
 
+	// upstreamMaxIdleConnsPerHost bounds the transparent proxy's upstream
+	// connection pool and doubles as the number of connections pre-warmed at
+	// startup (see warmUpstreamConnections). On a cold start, the client's
+	// standalone GET (SSE stream) and tools/call POST are forwarded to the
+	// upstream on separate connections; without a warm pool, the GET's
+	// connection pays inline dial latency and can lose the race to establish
+	// the SSE stream before the backend's tool handler sends a server-to-client
+	// request on it. This is the Go-proxy analog of the Squid ingress
+	// cache_peer standby=2 setting.
+	upstreamMaxIdleConnsPerHost = 4
+
+	// upstreamIdleConnTimeout bounds how long the upstream transport keeps
+	// warmed idle connections before closing them.
+	upstreamIdleConnTimeout = 90 * time.Second
+
 	// HealthCheckIntervalEnvVar is the environment variable name for configuring health check interval.
 	HealthCheckIntervalEnvVar = "TOOLHIVE_HEALTH_CHECK_INTERVAL"
 
@@ -1171,7 +1186,15 @@ func (p *TransparentProxy) Start(ctx context.Context) error {
 		},
 	}
 
-	proxy.Transport = newTracingTransport(http.DefaultTransport, p)
+	// Clone rather than mutate http.DefaultTransport: it's shared process-wide,
+	// and setting a small MaxIdleConnsPerHost here would starve every other
+	// consumer of the global default.
+	upstreamTransport := http.DefaultTransport.(*http.Transport).Clone()
+	upstreamTransport.MaxIdleConns = upstreamMaxIdleConnsPerHost
+	upstreamTransport.MaxIdleConnsPerHost = upstreamMaxIdleConnsPerHost
+	upstreamTransport.IdleConnTimeout = upstreamIdleConnTimeout
+
+	proxy.Transport = newTracingTransport(upstreamTransport, p)
 	proxy.ModifyResponse = func(resp *http.Response) error {
 		return p.modifyResponse(resp)
 	}
@@ -1267,9 +1290,52 @@ func (p *TransparentProxy) Start(ctx context.Context) error {
 		go p.monitorHealth(ctx)
 	}
 
+	// Pre-warm the upstream connection pool now that the listener is up.
+	// Runs in the background so a slow or unreachable upstream never delays
+	// Start() or blocks real client traffic.
+	go warmUpstreamConnections(upstreamTransport, p.targetURI)
+
 	return nil
 }
 
+// warmUpstreamConnections seeds the upstream transport's idle connection pool
+// with a burst of concurrent, side-effect-free requests before real client
+// traffic arrives, so the first standalone GET and the first POST forwarded
+// to the upstream can both reuse an already-established connection instead
+// of racing a cold TCP+dial.
+//
+// Each warmup request is a bare GET to targetURI with no Mcp-Session-Id
+// header. A streamable-HTTP backend rejects this (400 "invalid session")
+// without creating a session, but the underlying connection is still
+// established and returned to transport's idle pool. Failures are logged and
+// otherwise ignored: the proxy must still start even if the upstream isn't
+// reachable yet.
+func warmUpstreamConnections(transport http.RoundTripper, targetURI string) {
+	client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
+
+	var wg sync.WaitGroup
+	for i := 0; i < upstreamMaxIdleConnsPerHost; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, targetURI, nil)
+			if err != nil {
+				slog.Debug("failed to build upstream connection warmup request", "error", err)
+				return
+			}
+			//nolint:gosec // G704: targetURI is the configured backend MCP server
+			resp, err := client.Do(req)
+			if err != nil {
+				slog.Debug("upstream connection warmup request failed", "target", targetURI, "error", err)
+				return
+			}
+			_, _ = io.Copy(io.Discard, resp.Body)
+			_ = resp.Body.Close()
+		}()
+	}
+	wg.Wait()
+}
+
 // ListenerAddr returns the network address the proxy is listening on.
 // Returns an empty string if the proxy has not been started.
 func (p *TransparentProxy) ListenerAddr() string {
diff --git a/pkg/transport/proxy/transparent/warmup_test.go b/pkg/transport/proxy/transparent/warmup_test.go
new file mode 100644
index 0000000000..1dd04bb67a
--- /dev/null
+++ b/pkg/transport/proxy/transparent/warmup_test.go
@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package transparent
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestWarmUpstreamConnections verifies the warmup burst fires the expected
+// number of side-effect-free GET requests and tolerates a backend that
+// rejects them (the real-world case: no Mcp-Session-Id means the streamable-
+// HTTP server returns 400, but the connection is still established).
+func TestWarmUpstreamConnections(t *testing.T) {
+	t.Parallel()
+
+	var requests atomic.Int32
+	var sawSessionHeader atomic.Bool
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests.Add(1)
+		if r.Header.Get("Mcp-Session-Id") != "" {
+			sawSessionHeader.Store(true)
+		}
+		w.WriteHeader(http.StatusBadRequest)
+	}))
+	defer server.Close()
+
+	warmUpstreamConnections(http.DefaultTransport, server.URL)
+
+	assert.EqualValues(t, upstreamMaxIdleConnsPerHost, requests.Load(),
+		"expected one warmup request per configured idle connection")
+	assert.False(t, sawSessionHeader.Load(), "warmup requests must not carry a session ID")
+}

From 6c116339d67e3180cc1bb5f3484453a85c8b7eec Mon Sep 17 00:00:00 2001
From: Jakub Hrozek <jakub@stacklok.com>
Date: Mon, 20 Jul 2026 21:58:09 +0100
Subject: [PATCH 10/10] Revert "Warm transparent proxy's upstream connection
 pool"

This reverts commit 5fa0fa0285573bc025fc24f89605e5ac228c15e0.
---
 .../proxy/transparent/transparent_proxy.go    | 68 +------------------
 .../proxy/transparent/warmup_test.go          | 38 -----------
 2 files changed, 1 insertion(+), 105 deletions(-)
 delete mode 100644 pkg/transport/proxy/transparent/warmup_test.go

diff --git a/pkg/transport/proxy/transparent/transparent_proxy.go b/pkg/transport/proxy/transparent/transparent_proxy.go
index 134451016a..47e1266393 100644
--- a/pkg/transport/proxy/transparent/transparent_proxy.go
+++ b/pkg/transport/proxy/transparent/transparent_proxy.go
@@ -172,21 +172,6 @@ const (
 	// Matches the value used by the vMCP server.
 	defaultReadTimeout = 30 * time.Second
 
-	// upstreamMaxIdleConnsPerHost bounds the transparent proxy's upstream
-	// connection pool and doubles as the number of connections pre-warmed at
-	// startup (see warmUpstreamConnections). On a cold start, the client's
-	// standalone GET (SSE stream) and tools/call POST are forwarded to the
-	// upstream on separate connections; without a warm pool, the GET's
-	// connection pays inline dial latency and can lose the race to establish
-	// the SSE stream before the backend's tool handler sends a server-to-client
-	// request on it. This is the Go-proxy analog of the Squid ingress
-	// cache_peer standby=2 setting.
-	upstreamMaxIdleConnsPerHost = 4
-
-	// upstreamIdleConnTimeout bounds how long the upstream transport keeps
-	// warmed idle connections before closing them.
-	upstreamIdleConnTimeout = 90 * time.Second
-
 	// HealthCheckIntervalEnvVar is the environment variable name for configuring health check interval.
 	HealthCheckIntervalEnvVar = "TOOLHIVE_HEALTH_CHECK_INTERVAL"
 
@@ -1186,15 +1171,7 @@ func (p *TransparentProxy) Start(ctx context.Context) error {
 		},
 	}
 
-	// Clone rather than mutate http.DefaultTransport: it's shared process-wide,
-	// and setting a small MaxIdleConnsPerHost here would starve every other
-	// consumer of the global default.
-	upstreamTransport := http.DefaultTransport.(*http.Transport).Clone()
-	upstreamTransport.MaxIdleConns = upstreamMaxIdleConnsPerHost
-	upstreamTransport.MaxIdleConnsPerHost = upstreamMaxIdleConnsPerHost
-	upstreamTransport.IdleConnTimeout = upstreamIdleConnTimeout
-
-	proxy.Transport = newTracingTransport(upstreamTransport, p)
+	proxy.Transport = newTracingTransport(http.DefaultTransport, p)
 	proxy.ModifyResponse = func(resp *http.Response) error {
 		return p.modifyResponse(resp)
 	}
@@ -1290,52 +1267,9 @@ func (p *TransparentProxy) Start(ctx context.Context) error {
 		go p.monitorHealth(ctx)
 	}
 
-	// Pre-warm the upstream connection pool now that the listener is up.
-	// Runs in the background so a slow or unreachable upstream never delays
-	// Start() or blocks real client traffic.
-	go warmUpstreamConnections(upstreamTransport, p.targetURI)
-
 	return nil
 }
 
-// warmUpstreamConnections seeds the upstream transport's idle connection pool
-// with a burst of concurrent, side-effect-free requests before real client
-// traffic arrives, so the first standalone GET and the first POST forwarded
-// to the upstream can both reuse an already-established connection instead
-// of racing a cold TCP+dial.
-//
-// Each warmup request is a bare GET to targetURI with no Mcp-Session-Id
-// header. A streamable-HTTP backend rejects this (400 "invalid session")
-// without creating a session, but the underlying connection is still
-// established and returned to transport's idle pool. Failures are logged and
-// otherwise ignored: the proxy must still start even if the upstream isn't
-// reachable yet.
-func warmUpstreamConnections(transport http.RoundTripper, targetURI string) {
-	client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
-
-	var wg sync.WaitGroup
-	for i := 0; i < upstreamMaxIdleConnsPerHost; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, targetURI, nil)
-			if err != nil {
-				slog.Debug("failed to build upstream connection warmup request", "error", err)
-				return
-			}
-			//nolint:gosec // G704: targetURI is the configured backend MCP server
-			resp, err := client.Do(req)
-			if err != nil {
-				slog.Debug("upstream connection warmup request failed", "target", targetURI, "error", err)
-				return
-			}
-			_, _ = io.Copy(io.Discard, resp.Body)
-			_ = resp.Body.Close()
-		}()
-	}
-	wg.Wait()
-}
-
 // ListenerAddr returns the network address the proxy is listening on.
 // Returns an empty string if the proxy has not been started.
 func (p *TransparentProxy) ListenerAddr() string {
diff --git a/pkg/transport/proxy/transparent/warmup_test.go b/pkg/transport/proxy/transparent/warmup_test.go
deleted file mode 100644
index 1dd04bb67a..0000000000
--- a/pkg/transport/proxy/transparent/warmup_test.go
+++ /dev/null
@@ -1,38 +0,0 @@
-// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
-// SPDX-License-Identifier: Apache-2.0
-
-package transparent
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"sync/atomic"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-// TestWarmUpstreamConnections verifies the warmup burst fires the expected
-// number of side-effect-free GET requests and tolerates a backend that
-// rejects them (the real-world case: no Mcp-Session-Id means the streamable-
-// HTTP server returns 400, but the connection is still established).
-func TestWarmUpstreamConnections(t *testing.T) {
-	t.Parallel()
-
-	var requests atomic.Int32
-	var sawSessionHeader atomic.Bool
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		requests.Add(1)
-		if r.Header.Get("Mcp-Session-Id") != "" {
-			sawSessionHeader.Store(true)
-		}
-		w.WriteHeader(http.StatusBadRequest)
-	}))
-	defer server.Close()
-
-	warmUpstreamConnections(http.DefaultTransport, server.URL)
-
-	assert.EqualValues(t, upstreamMaxIdleConnsPerHost, requests.Load(),
-		"expected one warmup request per configured idle connection")
-	assert.False(t, sawSessionHeader.Load(), "warmup requests must not carry a session ID")
-}