test: live e2e suite against the real OpenRouter API + CI job (#21) #14
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # This repo had no CI. Without it the port verifier's pytest/ruff/mypy calls only | |
| # ever ran inside the sync job, so a generated PR reached review with no | |
| # independent signal. This runs the same checks on every PR and push. | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| - run: uv sync --all-extras | |
| - name: Lint | |
| run: uv run ruff check . | |
| - name: Format | |
| run: uv run ruff format --check . | |
| - name: Type check | |
| run: uv run mypy src | |
| # Deterministic tests only. tests/e2e needs OPENROUTER_API_KEY and skips | |
| # cleanly without it. | |
| - name: Tests | |
| run: uv run pytest tests/unit -q | |
| # Live end-to-end tests against the real OpenRouter API: streaming, a real | |
| # tool round, approval pause/resume, lifecycle hooks, state serialization | |
| # round-trip. Costs a few cents per run (small model, short prompts). | |
| # | |
| # Warns and exits 0 when the secret is missing (e.g. PRs from forks, where | |
| # GitHub withholds secrets) instead of failing — same pattern as upstream | |
| # typescript-agent's e2e job. | |
| e2e: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| - run: uv sync --all-extras | |
| - name: Live e2e tests | |
| env: | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| run: | | |
| if [ -z "$OPENROUTER_API_KEY" ]; then | |
| echo "::warning::OPENROUTER_API_KEY is not set; skipping live e2e tests." | |
| exit 0 | |
| fi | |
| uv run pytest tests/e2e -q | |
| # Reports the port's own mechanical gate. Advisory here, BLOCKING inside the | |
| # sync job (scripts/upstream) where it gates whether state.yaml advances. | |
| # | |
| # Advisory on purpose: the port is currently a minor version behind upstream, so | |
| # the required-API check fails by design until the first sync lands. Making that | |
| # a red required check on every unrelated PR just teaches people to ignore CI. | |
| # The signal still shows up in the job summary. | |
| verify-port: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/port-toolchain | |
| - name: Port verifier (advisory) | |
| id: verify | |
| continue-on-error: true | |
| run: | | |
| set -o pipefail | |
| ./.upstreamer/scripts/verify.sh 2>&1 | tee /tmp/verify.log | |
| - name: Summarize | |
| if: always() | |
| run: | | |
| { | |
| echo "## Port verifier" | |
| echo | |
| if [ "${{ steps.verify.outcome }}" = "success" ]; then | |
| echo "Port is in sync with its parity floor." | |
| else | |
| echo "Parity gaps below. Expected until the port catches up to upstream —" | |
| echo "advisory here, blocking inside the sync job." | |
| fi | |
| echo | |
| echo '```' | |
| cat /tmp/verify.log 2>/dev/null || echo "(no output)" | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" |