Add untrusted mode: MCP servers that never possess credentials

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list = NotIn,notin,AfterAll,ND,aks,deriver,te,clientA,AtMost,atmost,convertIn
+ignore-words-list = NotIn,notin,AfterAll,ND,aks,deriver,te,clientA,AtMost,atmost,convertIn,userA
 skip = *.svg,*.mod,*.sum

.github/workflows/operator-ci.yml

@@ -222,3 +222,5 @@ jobs:
         chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/setup --config .chainsaw.yaml
         chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/test-scenarios --parallel 10 --config .chainsaw.yaml
         chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/cleanup --config .chainsaw.yaml
+        chainsaw test --test-dir test/e2e/chainsaw/operator/validation/mcpserver-untrusted --config .chainsaw.yaml
+        chainsaw test --test-dir test/e2e/chainsaw/operator/untrusted-egress --config .chainsaw.yaml

Taskfile.yml

@@ -332,6 +332,13 @@ tasks:
     cmds:
       - docker build --load -t ghcr.io/stacklok/toolhive/egress-proxy:local containers/egress-proxy/
 
+  build-egressbroker-image:
+    desc: Build the egress broker sidecar image with ko
+    env:
+      KO_DOCKER_REPO: ghcr.io/stacklok/toolhive/egressbroker
+    cmds:
+      - ko build --local --bare ./cmd/thv-egressbroker
+
   build-all-images:
-    desc: Build all container images (main app, vmcp, and egress proxy)
-    deps: [build-image, build-vmcp-image, build-egress-proxy]
+    desc: Build all container images (main app, vmcp, egress proxy, and egress broker)
+    deps: [build-image, build-vmcp-image, build-egress-proxy, build-egressbroker-image]

cmd/thv-egressbroker/health_test.go

@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/stacklok/toolhive/pkg/egressbroker"
+)
+
+// healthFixture builds a health handler around a freshly generated CA and a
+// configurable ping. The CA is valid (not rotation-due) at test time.
+func healthFixture(t *testing.T, ping redisPinger) (*healthServer, *atomic.Bool) {
+	t.Helper()
+	ca, err := egressbroker.GenerateBumpCA("test", time.Now())
+	require.NoError(t, err)
+	var loaded atomic.Bool
+	loaded.Store(true)
+	if ping == nil {
+		ping = func(context.Context) error { return nil }
+	}
+	return &healthServer{ca: ca, policyLoaded: &loaded, ping: ping}, &loaded
+}
+
+func TestHealthz(t *testing.T) {
+	t.Parallel()
+
+	t.Run("healthy when CA valid, policy loaded, Redis reachable", func(t *testing.T) {
+		t.Parallel()
+		h, _ := healthFixture(t, nil)
+		rec := httptest.NewRecorder()
+		h.handle(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+		assert.Equal(t, http.StatusOK, rec.Code)
+	})
+
+	t.Run("503 when Redis unreachable", func(t *testing.T) {
+		t.Parallel()
+		h, _ := healthFixture(t, func(context.Context) error { return errors.New("dial refused") })
+		rec := httptest.NewRecorder()
+		h.handle(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+		assert.Equal(t, http.StatusServiceUnavailable, rec.Code)
+		assert.Contains(t, rec.Body.String(), "redis")
+	})
+
+	t.Run("503 when policy not loaded", func(t *testing.T) {
+		t.Parallel()
+		h, loaded := healthFixture(t, nil)
+		loaded.Store(false)
+		rec := httptest.NewRecorder()
+		h.handle(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+		assert.Equal(t, http.StatusServiceUnavailable, rec.Code)
+		assert.Contains(t, rec.Body.String(), "policy")
+	})
+
+	t.Run("503 when bump CA is past rotation-due", func(t *testing.T) {
+		t.Parallel()
+		h, _ := healthFixture(t, nil)
+		// NeedsRotation fires at 50% of validity, so now+half is always past due.
+		ca, err := egressbroker.GenerateBumpCA("test", time.Now().Add(-egressbroker.CAValidity))
+		require.NoError(t, err)
+		h.ca = ca
+		rec := httptest.NewRecorder()
+		h.handle(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+		assert.Equal(t, http.StatusServiceUnavailable, rec.Code)
+		assert.Contains(t, rec.Body.String(), "rotation")
+	})
+
+	t.Run("non-GET is rejected", func(t *testing.T) {
+		t.Parallel()
+		h, _ := healthFixture(t, nil)
+		rec := httptest.NewRecorder()
+		h.handle(rec, httptest.NewRequest(http.MethodPost, "/healthz", nil))
+		assert.Equal(t, http.StatusMethodNotAllowed, rec.Code)
+	})
+
+	t.Run("health listener is loopback-only on the pinned port", func(t *testing.T) {
+		t.Parallel()
+		h, _ := healthFixture(t, nil)
+		var loadedFlag atomic.Bool
+		srv := newHealthServerHandler(&healthServer{ca: h.ca, policyLoaded: &loadedFlag, ping: h.ping})
+		assert.Equal(t, ":15083", srv.Addr)
+	})
+}