feat(harness): caller routes on the Claude harness forwarder, git through them

.github/workflows/ci.yaml

@@ -1,18 +1,15 @@
 name: CI Build
 
+# Fork (giantswarm/kagent-upstream): the consumed branch and the sync
+# candidates run this suite on every push and pull request, on GitHub-hosted
+# runners (the org has no Blacksmith runners), and report one `ci-ok` status
+# for the ruleset. No paths-ignore: a docs-only pull request must still report
+# the required checks to be mergeable. See FORK.md.
 on:
   push:
-    branches: [main, "release/**"]
-    paths:
-      - "**"
-      - "!**/*.md"
-      - "docs/env.md"
+    branches: [main, "release/**", giantswarm, "sync/**"]
   pull_request:
-    branches: [main, "release/**"]
-    paths:
-      - "**"
-      - "!**/*.md"
-      - "docs/env.md"
+    branches: [main, "release/**", giantswarm]
   merge_group:
   workflow_dispatch:
     inputs:
@@ -53,18 +50,31 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        sandbox_class: [gvisor, microvm]
+        # The Giant Swarm line of Substrate publishes the gVisor worker only
+        # (its FORK.md, "Publishing"); the platform runs no micro-VM pool.
+        sandbox_class: [gvisor]
     env:
       VERSION: v0.0.1-test
-      SUBSTRATE_VERSION: 0.3.0-alpha3
       KIND_SANDBOX_CLASS: ${{ matrix.sandbox_class }}
       # Keep the four-vCPU runner bounded; manual runs can compare higher concurrency.
       E2E_PARALLEL: ${{ inputs.e2e_parallel || '4' }}
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+      # The upstream release whose kubectl-ate performs the CA/JWT bootstrap
+      # below; the Giant Swarm line of Substrate publishes no CLI. Same pin as
+      # the Makefile's SUBSTRATE_VERSION is built from.
+      KUBECTL_ATE_VERSION: 0.0.29
+    runs-on: ubuntu-latest
     timeout-minutes: 40
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
+      - name: Resolve the Substrate pin
+        # One source of truth: the Makefile's SUBSTRATE_VERSION / SUBSTRATE_REPO
+        # (the Giant Swarm line of Substrate, gsoci.azurecr.io/giantswarm/substrate) drive
+        # the charts' dependencies and the Substrate this cluster runs.
+        run: |
+          make -s substrate-pin | tee -a "$GITHUB_ENV"
+          registry=$(make -s substrate-pin | sed -n 's|^SUBSTRATE_REPO=oci://||p')
+          echo "SUBSTRATE_IMAGE_REGISTRY=${registry%/helm}" | tee -a "$GITHUB_ENV"
       - name: Initialize Environment
         uses: ./.github/actions/initialize-environment
       - name: Set up Go
@@ -88,12 +98,13 @@ jobs:
         # See https://github.com/openai/codex/issues/14919
         run: |
           sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true
-      - name: Setup Blacksmith Builder
-        uses: useblacksmith/setup-docker-builder@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
         with:
-          cache-key: kagent-e2e
-          platforms: linux/amd64
-          nofallback: true
+          name: ${{ env.BUILDX_BUILDER_NAME }}
+          version: ${{ env.BUILDX_VERSION }}
+          driver-opts: network=host
+          use: "true"
 
       - name: Set up Helm
         uses: azure/setup-helm@v5.0.1
@@ -112,14 +123,24 @@ jobs:
 
       - name: Install Agent Substrate
         run: |
-          curl -fsSL -o kubectl-ate "https://github.com/kagent-dev/substrate/releases/download/v${SUBSTRATE_VERSION}/kubectl-ate-linux-amd64"
+          curl -fsSL -o kubectl-ate "https://github.com/kagent-dev/substrate/releases/download/v${KUBECTL_ATE_VERSION}/kubectl-ate-linux-amd64"
           chmod +x kubectl-ate
-          helm upgrade --install substrate-crds oci://ghcr.io/kagent-dev/substrate/helm/substrate-crds --version "${SUBSTRATE_VERSION}" --namespace ate-system --create-namespace
-          helm upgrade --install substrate oci://ghcr.io/kagent-dev/substrate/helm/substrate --version "${SUBSTRATE_VERSION}" --namespace ate-system \
+          # The egress gateway dials upstream HTTPS itself and trusts this CA
+          # there; the egress credential tests sign their HTTPS upstreams with it.
+          openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -days 1 -subj "/CN=kagent e2e egress upstream" \
+            -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign" \
+            -keyout "$RUNNER_TEMP/egress-ca.key" -out "$RUNNER_TEMP/egress-ca.crt"
+          helm upgrade --install substrate-crds "${SUBSTRATE_REPO}/substrate-crds" --version "${SUBSTRATE_VERSION}" --namespace ate-system --create-namespace
+          # Golden boots fetch git skill and plugin sources in atespace
+          # ate-golden, so it resolves kagent Secrets as the session atespace does.
+          helm upgrade --install substrate "${SUBSTRATE_REPO}/substrate" --version "${SUBSTRATE_VERSION}" --namespace ate-system \
             --set-string 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \
             --set-string 'ateApi.extraArgs[0]=--template-resync-interval=250ms' \
             --set 'credentialProvider.namespacePolicies[0].atespace=kagent' \
-            --set 'credentialProvider.namespacePolicies[0].allowedNamespaces[0]=kagent'
+            --set 'credentialProvider.namespacePolicies[0].allowedNamespaces[0]=kagent' \
+            --set 'credentialProvider.namespacePolicies[1].atespace=ate-golden' \
+            --set 'credentialProvider.namespacePolicies[1].allowedNamespaces[0]=kagent' \
+            --set-file "atenetEgress.upstreamTrust.caBundle=$RUNNER_TEMP/egress-ca.crt"
           ./kubectl-ate --context kind-kagent admin make-ca-pool --ca-id=1 --name=service-dns-ca-pool --secret-namespace=podcertificate-controller-system
           ./kubectl-ate --context kind-kagent admin make-ca-pool --ca-id=1 --name=pod-identity-ca-pool --secret-namespace=podcertificate-controller-system
           ./kubectl-ate --context kind-kagent admin make-jwt-pool --key-id=1 --name=actor-id-jwt-pool --secret-namespace=ate-system
@@ -128,7 +149,7 @@ jobs:
           actor_id_ca_root="$(kubectl get secret actor-id-ca-pool -n ate-system -o jsonpath='{.data.pool}' | base64 --decode | jq -r '.CAs[0].RootCertificateDER' | base64 --decode | openssl x509 -inform der -outform pem)"
           kubectl create secret generic actor-id-ca-certs -n ate-system --from-literal=ca.crt="${actor_id_ca_root}"
           kubectl create configmap ate-api-authentication -n ate-system --from-literal=authentication.yaml=$'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n  issuer: https://kubernetes.default.svc\n  audiences: [api.ate-system.svc]\n  certificateAuthorityFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n  discoveryTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token\n'
-          helm upgrade substrate oci://ghcr.io/kagent-dev/substrate/helm/substrate --version "${SUBSTRATE_VERSION}" --namespace ate-system --reuse-values --wait --timeout 5m
+          helm upgrade substrate "${SUBSTRATE_REPO}/substrate" --version "${SUBSTRATE_VERSION}" --namespace ate-system --reuse-values --wait --timeout 5m
 
       - name: Cache Cloud Hypervisor assets
         if: matrix.sandbox_class == 'microvm'
@@ -157,7 +178,7 @@ jobs:
             --set substrateWorkerPool.create=true
             --set substrateWorkerPool.replicas=4
             --set-string substrateWorkerPool.sandboxClass=${{ matrix.sandbox_class }}
-            --set-string substrateWorkerPool.workerImage=ghcr.io/kagent-dev/substrate/ateom-${{ matrix.sandbox_class }}:v${{ env.SUBSTRATE_VERSION }}
+            --set-string substrateWorkerPool.workerImage=${{ env.SUBSTRATE_IMAGE_REGISTRY }}/ateom-${{ matrix.sandbox_class }}:${{ env.SUBSTRATE_VERSION }}
             --set otel.traces.enabled=true
             --set-string otel.exporter.otlp.endpoint=http://e2e-otlp.kagent.svc.cluster.local:14317
         run: |
@@ -213,6 +234,7 @@ jobs:
           export KAGENT_E2E_CLI="$PWD/core/bin/kagent-local"
           export KAGENT_E2E_API_URL="$KAGENT_API_URL"
           export KAGENT_E2E_LOCAL_HOST=172.17.0.1
+          export KAGENT_E2E_EGRESS_CA_FILE="$RUNNER_TEMP/egress-ca.crt" KAGENT_E2E_EGRESS_CA_KEY_FILE="$RUNNER_TEMP/egress-ca.key"
           envsubst < core/test/e2e/manifests/tracing.yaml.tmpl | kubectl apply -f -
           # TestMain keeps the receiver alive for every runtime's response flush.
           export KAGENT_E2E_OTLP_LISTEN_ADDRESS=:14317
@@ -222,15 +244,27 @@ jobs:
           worker_logs_pid=$!
           kubectl logs -n ate-system -l app --all-containers --prefix --timestamps --tail=-1 --follow --max-log-requests=20 > "$RUNNER_TEMP/e2e-logs/substrate.log" 2>&1 &
           substrate_logs_pid=$!
-          trap 'kill "$worker_logs_pid" "$substrate_logs_pid" 2>/dev/null || true; wait "$worker_logs_pid" "$substrate_logs_pid" 2>/dev/null || true' EXIT
+          # Tests roll the controller (withControllerEnv, the sandbox restart), and a
+          # log read at the end holds only the final pod: every controller pod is
+          # followed from the moment it runs, into a file of its own.
+          controller_log_pids="$RUNNER_TEMP/controller-log-pids"
+          : > "$controller_log_pids"
+          (
+            followed=" "
+            while true; do
+              for pod in $(kubectl get pods -n kagent -l app.kubernetes.io/component=controller -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}'); do
+                case "$followed" in *" $pod "*) continue ;; esac
+                followed="$followed$pod "
+                kubectl logs -n kagent "$pod" --all-containers --prefix --timestamps --tail=-1 --follow > "$RUNNER_TEMP/e2e-logs/controller-$pod.log" 2>&1 &
+                echo $! >> "$controller_log_pids"
+              done
+              sleep 5
+            done
+          ) &
+          controller_logs_pid=$!
+          trap 'kill "$worker_logs_pid" "$substrate_logs_pid" "$controller_logs_pid" $(cat "$controller_log_pids") 2>/dev/null || true; wait "$worker_logs_pid" "$substrate_logs_pid" "$controller_logs_pid" $(cat "$controller_log_pids") 2>/dev/null || true' EXIT
           go test -v github.com/kagent-dev/kagent/go/core/test/e2e -shuffle=on -parallel "$E2E_PARALLEL" -count=1 -timeout=30m 2>&1 | tee "$RUNNER_TEMP/e2e-logs/tests.log"
 
-      - name: Collect E2E controller logs
-        if: always()
-        run: |
-          mkdir -p "$RUNNER_TEMP/e2e-logs"
-          kubectl logs -n kagent deployment/kagent-controller --timestamps > "$RUNNER_TEMP/e2e-logs/controller.log" 2>&1 || true
-
       - name: Upload E2E logs
         if: always()
         uses: actions/upload-artifact@v7
@@ -288,6 +322,11 @@ jobs:
         with:
           go-version-file: go/go.mod
           cache: false
+      - name: Cache the contract's upstream sources at their pins
+        uses: actions/cache@v6
+        with:
+          path: telemetry/deps
+          key: telemetry-deps-${{ hashFiles('telemetry/versions.env') }}
       - name: Check the registry and generated conventions
         run: make semconv-verify
 
@@ -296,7 +335,7 @@ jobs:
       - setup
     env:
       VERSION: v0.0.1-test
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
@@ -344,7 +383,7 @@ jobs:
       - setup
     env:
       VERSION: v0.0.1-test
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
@@ -412,6 +451,17 @@ jobs:
           export KUBEBUILDER_ASSETS="$(make -s envtest-path)"
           go test -race -skip 'TestE2E.*' -v ./...
 
+      # Fork: the claude harness splits Claude Code off only when it runs as root,
+      # so its root-only tests run again as root, with no capability beyond the
+      # CHOWN, SETUID and SETGID the claude translator asks Substrate for.
+      - name: Run the claude harness tests as root
+        working-directory: go
+        run: |
+          go test -c -o "$RUNNER_TEMP/claude-adapter.test" ./harness/claude/internal/adapter/
+          caps="--bounding-set=-all,+chown,+setuid,+setgid --inh-caps=-all --ambient-caps=-all"
+          sudo setpriv $caps -- grep -E '^Cap(Prm|Eff)' /proc/self/status
+          sudo setpriv $caps -- "$RUNNER_TEMP/claude-adapter.test" -test.count=1 -test.v
+
   helm-unit-tests:
     env:
       VERSION: v0.0.1-test
@@ -439,7 +489,7 @@ jobs:
           helm unittest helm/tools/grafana-mcp
 
   ui-tests:
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: write
@@ -491,22 +541,38 @@ jobs:
         working-directory: ./ui
         run: yarn test:pw
 
+      # What a failed attempt leaves behind: its screenshot and video, and the trace of
+      # its retry. The log names the step that timed out; only the trace shows what was
+      # on the page under the pointer while it did, and the runner discards it otherwise.
+      # Kept whether the job failed or a retry passed: a flaky test leaves the job green
+      # and is the case that most needs its trace. A run whose every attempt passed
+      # writes nothing here (Playwright's `.last-run.json` is hidden, and hidden files
+      # are not uploaded), so it keeps no artifact.
+      - name: Keep the browser suite's evidence of a failed attempt
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: ui-test-results
+          path: ui/test-results
+          if-no-files-found: ignore
+          retention-days: 7
+
   # This job builds the Docker images for the controller, UI, ADKs, and CLI on arm64.
   build:
     env:
       VERSION: v0.0.1-test
     strategy:
       matrix:
+        # Fork: the images the platform publishes (FORK.md) — the Python ADK,
+        # the Codex harness and the CLI image are not offered.
         image:
           - controller
           - ui
-          - kagent-adk
-          - cli
           - golang-adk
           - claude-harness
           - codex-harness
           - sandbox-guest
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     services:
       registry:
         image: registry:2
@@ -519,12 +585,13 @@ jobs:
         uses: docker/setup-qemu-action@v4
         with:
           platforms: linux/amd64,linux/arm64
-      - name: Setup Blacksmith Builder
-        uses: useblacksmith/setup-docker-builder@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
         with:
-          cache-key: kagent-build-${{ matrix.image }}
-          platforms: linux/amd64,linux/arm64
-          nofallback: true
+          name: ${{ env.BUILDX_BUILDER_NAME }}
+          version: ${{ env.BUILDX_VERSION }}
+          driver-opts: network=host
+          use: "true"
       - name: Run make build
         env:
           DOCKER_BUILD_ARGS: >-
@@ -649,3 +716,31 @@ jobs:
             exit 1
           fi
           echo "✓ Controller manifests are up to date"
+
+  # Fork: the one status the ruleset requires. Every job above must succeed; a
+  # leg a job skips internally (the upgrade tests on a non-release base) still
+  # ends in success, a skipped or failed job does not.
+  ci-ok:
+    if: always()
+    needs:
+      - setup
+      - test-e2e
+      - proto-check
+      - upgrade-tests
+      - rolling-upgrade-tests
+      - go-unit-tests
+      - helm-unit-tests
+      - ui-tests
+      - build
+      - go-lint
+      - python-test
+      - python-lint
+      - manifests-check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Every job succeeded
+        env:
+          RESULTS: ${{ toJSON(needs) }}
+        run: |
+          jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$RESULTS"
+          jq -e 'all(.[]; .result == "success")' <<< "$RESULTS" > /dev/null