Skip to content

Commit 55da715

Browse files
karthiknadigCopilot
andcommitted
test: compare fixture revisions on one Windows host (Refs #557)
Temporary manual-only diagnostic branch; not for merge. Preserve exact base/head provenance and counterbalanced inventory-checked measurements. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 6772e5c commit 55da715

1 file changed

Lines changed: 149 additions & 192 deletions

File tree

‎.github/workflows/perf-tests.yml‎

Lines changed: 149 additions & 192 deletions
Original file line numberDiff line numberDiff line change
@@ -1,196 +1,153 @@
1-
name: Performance Tests
2-
3-
on:
4-
pull_request:
5-
branches:
6-
- main
7-
- release*
8-
- release/*
9-
- release-*
10-
workflow_dispatch:
11-
1+
name: Fixture Windows Matched Control
2+
'on':
3+
workflow_dispatch: {}
124
permissions:
13-
actions: read
145
contents: read
15-
pull-requests: write
16-
176
jobs:
18-
performance:
19-
name: E2E Performance Tests
20-
runs-on: ${{ matrix.os }}
21-
strategy:
22-
fail-fast: false
23-
matrix:
24-
include:
25-
- os: windows-latest
26-
target: x86_64-pc-windows-msvc
27-
platform: Windows
28-
comment_header: perf-windows
29-
- os: ubuntu-latest
30-
target: x86_64-unknown-linux-musl
31-
platform: Linux
32-
comment_header: perf-linux
33-
- os: macos-latest
34-
target: x86_64-apple-darwin
35-
platform: macOS
36-
comment_header: perf-macos
7+
matched-control:
8+
runs-on: windows-latest
9+
timeout-minutes: 35
3710
steps:
38-
- name: Checkout
39-
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
40-
41-
- name: Post In-Progress Comment
42-
if: >-
43-
github.event_name == 'pull_request' &&
44-
github.event.pull_request.head.repo.full_name == github.repository
45-
continue-on-error: true
46-
uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5
47-
with:
48-
header: ${{ matrix.comment_header }}
49-
message: |
50-
## Performance Report (${{ matrix.platform }}) :hourglass_flowing_sand:
51-
52-
Running performance tests against baseline `${{ github.event.pull_request.base.sha }}`.
53-
54-
- name: Set Python to PATH
55-
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
56-
with:
57-
python-version: "3.12"
58-
59-
- name: Validate Snapshot Comparator
60-
run: python -m unittest discover -s scripts/tests -p 'test_*.py' -v
61-
shell: bash
62-
63-
- name: Add Conda to PATH (Windows)
64-
if: startsWith(matrix.os, 'windows')
65-
run: |
66-
$path = $env:PATH + ";" + $env:CONDA + "\condabin"
67-
echo "PATH=$path" >> $env:GITHUB_ENV
68-
69-
- name: Add Conda to PATH (Ubuntu)
70-
if: startsWith(matrix.os, 'ubuntu')
71-
run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV
72-
shell: bash
73-
74-
- name: Install Conda + add to PATH (macOS)
75-
if: startsWith(matrix.os, 'macos')
76-
run: |
77-
curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
78-
bash ~/miniconda.sh -b -p ~/miniconda
79-
echo "PATH=$PATH:$HOME/miniconda/bin" >> $GITHUB_ENV
80-
echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV
81-
shell: bash
82-
83-
- name: Create test Conda environment
84-
run: conda create -n perf-test-env python=3.12 -y
85-
86-
- name: Create test venv
87-
run: python -m venv .venv
88-
shell: bash
89-
90-
- name: Rust Tool Chain setup
91-
uses: dtolnay/rust-toolchain@stable
92-
with:
93-
toolchain: stable
94-
targets: ${{ matrix.target }}
95-
96-
- name: Cargo Fetch
97-
run: cargo fetch
98-
shell: bash
99-
100-
- name: Build Release
101-
run: cargo build --release --target ${{ matrix.target }}
102-
shell: bash
103-
104-
- name: Run Performance Tests
105-
id: benchmark
106-
run: |
107-
set -o pipefail
108-
cargo test --release --features ci-perf --target ${{ matrix.target }} --test e2e_performance test_performance_summary -- --nocapture 2>&1 | tee perf-output.txt
109-
env:
110-
RUST_BACKTRACE: 1
111-
RUST_LOG: warn
112-
shell: bash
113-
114-
- name: Extract Performance Metrics
115-
if: steps.benchmark.outcome == 'success'
116-
run: |
117-
if ! grep -q "JSON metrics:" perf-output.txt; then
118-
echo "Performance test produced no JSON metrics" >&2
119-
exit 1
120-
fi
121-
sed -n '/JSON metrics:/,/^}/p' perf-output.txt | tail -n +2 > metrics.json
122-
python -m json.tool metrics.json > /dev/null
123-
cat metrics.json
124-
shell: bash
125-
126-
- name: Wait for Exact PR Base Performance
127-
id: wait_for_base_performance
128-
if: always() && github.event_name == 'pull_request'
129-
run: >-
130-
python scripts/wait_for_baseline.py
131-
--repository "${{ github.repository }}"
132-
--workflow perf-baseline.yml
133-
--commit "${{ github.event.pull_request.base.sha }}"
134-
--artifact "perf-baseline-${{ matrix.os }}"
135-
--timeout-seconds 1200
136-
--poll-seconds 20
137-
env:
138-
GITHUB_TOKEN: ${{ github.token }}
139-
shell: bash
140-
141-
- name: Download Exact PR Base Performance
142-
if: >-
143-
always() && github.event_name == 'pull_request' &&
144-
steps.wait_for_base_performance.outcome == 'success'
145-
uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21
146-
with:
147-
workflow: perf-baseline.yml
148-
commit: ${{ github.event.pull_request.base.sha }}
149-
workflow_conclusion: success
150-
name: perf-baseline-${{ matrix.os }}
151-
path: baseline-perf
152-
check_artifacts: true
153-
search_artifacts: true
154-
155-
- name: Download Main Performance for Manual Run
156-
if: always() && github.event_name == 'workflow_dispatch'
157-
uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21
158-
with:
159-
workflow: perf-baseline.yml
160-
branch: main
161-
workflow_conclusion: success
162-
name: perf-baseline-${{ matrix.os }}
163-
path: baseline-perf
164-
check_artifacts: true
165-
search_artifacts: true
166-
167-
- name: Compare Performance Snapshot
168-
if: always()
169-
run: >-
170-
python scripts/quality_snapshot.py performance
171-
--current metrics.json
172-
--baseline baseline-perf/metrics.json
173-
--platform "${{ matrix.platform }}"
174-
--report performance-report.md
175-
--summary "$GITHUB_STEP_SUMMARY"
176-
shell: bash
177-
178-
- name: Upload PR Performance Results
179-
if: always()
180-
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
181-
with:
182-
name: perf-pr-${{ matrix.os }}
183-
path: |
184-
metrics.json
185-
performance-report.md
186-
if-no-files-found: ignore
187-
188-
- name: Post Performance Comment
189-
if: >-
190-
always() && github.event_name == 'pull_request' &&
191-
github.event.pull_request.head.repo.full_name == github.repository
192-
continue-on-error: true
193-
uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5
194-
with:
195-
header: ${{ matrix.comment_header }}
196-
path: performance-report.md
11+
- name: Checkout exact PR head
12+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
13+
with:
14+
ref: 6772e5c3631116386de3953dec083a2d08b4fcfa
15+
- name: Initialize evidence
16+
shell: pwsh
17+
run: |
18+
New-Item -ItemType Directory -Path "$env:RUNNER_TEMP\fixture-matched-control" -ErrorAction Stop | Out-Null
19+
- name: Set Python to PATH
20+
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
21+
with:
22+
python-version: '3.12'
23+
- name: Create the standard Windows workload
24+
shell: pwsh
25+
run: |
26+
$env:PATH += ";$env:CONDA\condabin"
27+
"PATH=$env:PATH" >> $env:GITHUB_ENV
28+
conda create -n perf-test-env python=3.12 -y
29+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
30+
python -m venv .venv
31+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
32+
- name: Rust toolchain
33+
uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87
34+
with:
35+
toolchain: 1.98.1
36+
targets: x86_64-pc-windows-msvc
37+
- name: Instrument only the shared test harness after its measured passes
38+
shell: pwsh
39+
run: |
40+
@'
41+
from pathlib import Path
42+
path = Path("crates/pet/tests/e2e_performance.rs")
43+
source = path.read_text()
44+
for name in ["EnvironmentIdentity", "ManagerIdentity", "Inventory"]:
45+
marker = f"struct {name} {{"
46+
assert source.count(marker) == 1
47+
source = source.replace(marker, "#[derive(serde::Serialize)]\n" + marker)
48+
marker = 'expected_inventory.expect("Performance summary must run at least one iteration");'
49+
assert source.count(marker) == 1
50+
source = source.replace(marker, marker + '\n'
51+
' if let Some(path) = std::env::var_os("PET_DIAGNOSTIC_INVENTORY") {\n'
52+
' std::fs::write(path, serde_json::to_vec(&expected_inventory).unwrap()).unwrap();\n'
53+
' }\n')
54+
path.write_text(source)
55+
'@ | python -
56+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
57+
- name: Build exact base and head
58+
shell: pwsh
59+
run: |
60+
git fetch --no-tags --depth=1 origin 3bd7eea22298a0a8e9a08079dd2ed44212368214
61+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
62+
git worktree add --detach "$env:RUNNER_TEMP\pet-base" 3bd7eea22298a0a8e9a08079dd2ed44212368214
63+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
64+
cargo build --locked --release --target x86_64-pc-windows-msvc --bin pet --manifest-path "$env:RUNNER_TEMP\pet-base\Cargo.toml" --target-dir "$env:RUNNER_TEMP\pet-base-target"
65+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
66+
Copy-Item "$env:RUNNER_TEMP\pet-base-target\x86_64-pc-windows-msvc\release\pet.exe" "$env:RUNNER_TEMP\pet-base-exe" -ErrorAction Stop
67+
cargo build --locked --release --target x86_64-pc-windows-msvc --bin pet
68+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
69+
Copy-Item "target\x86_64-pc-windows-msvc\release\pet.exe" "$env:RUNNER_TEMP\pet-head-exe" -ErrorAction Stop
70+
cargo test --locked --release --features ci-perf --target x86_64-pc-windows-msvc --test e2e_performance --no-run --message-format=json > "$env:RUNNER_TEMP\pet-test-build.jsonl"
71+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
72+
if ((git rev-parse HEAD) -ne '6772e5c3631116386de3953dec083a2d08b4fcfa') { throw 'Unexpected head revision' }
73+
if ((git -C "$env:RUNNER_TEMP\pet-base" rev-parse HEAD) -ne '3bd7eea22298a0a8e9a08079dd2ed44212368214') { throw 'Unexpected base revision' }
74+
@{
75+
base = '3bd7eea22298a0a8e9a08079dd2ed44212368214'
76+
head = '6772e5c3631116386de3953dec083a2d08b4fcfa'
77+
baseBinarySha256 = (Get-FileHash "$env:RUNNER_TEMP\pet-base-exe" -Algorithm SHA256).Hash
78+
headBinarySha256 = (Get-FileHash "$env:RUNNER_TEMP\pet-head-exe" -Algorithm SHA256).Hash
79+
harnessSha256 = (Get-FileHash 'crates\pet\tests\e2e_performance.rs' -Algorithm SHA256).Hash
80+
rustc = (rustc -Vv) -join "`n"
81+
runnerImage = $env:ImageOS
82+
runnerImageVersion = $env:ImageVersion
83+
processor = $env:PROCESSOR_IDENTIFIER
84+
} | ConvertTo-Json | Set-Content "$env:RUNNER_TEMP\fixture-matched-control\provenance.json" -Encoding utf8
85+
- name: Counterbalanced same-host measurements
86+
shell: pwsh
87+
env:
88+
RUST_BACKTRACE: '1'
89+
RUST_LOG: warn
90+
run: |
91+
@'
92+
import hashlib
93+
import json
94+
import os
95+
from pathlib import Path
96+
import shutil
97+
import subprocess
98+
99+
temp = Path(os.environ["RUNNER_TEMP"])
100+
workspace = Path(os.environ["GITHUB_WORKSPACE"])
101+
artifacts = temp / "fixture-matched-control"
102+
artifacts.mkdir(exist_ok=True)
103+
messages = [json.loads(line) for line in (temp / "pet-test-build.jsonl").read_text(encoding="utf-8-sig").splitlines()]
104+
tests = [m["executable"] for m in messages if m.get("reason") == "compiler-artifact"
105+
and m["target"]["name"] == "e2e_performance" and m.get("executable")]
106+
if len(tests) != 1:
107+
raise RuntimeError(f"Expected one benchmark executable, got {tests!r}")
108+
executable = workspace / "target" / "x86_64-pc-windows-msvc" / "release" / "pet.exe"
109+
revisions = {"base": "3bd7eea22298a0a8e9a08079dd2ed44212368214",
110+
"head": "6772e5c3631116386de3953dec083a2d08b4fcfa"}
111+
order = ["base", "head", "head", "base", "head", "base", "base", "head"]
112+
summary = []
113+
for index, label in enumerate(order, 1):
114+
source = temp / f"pet-{label}-exe"
115+
shutil.copy2(source, executable)
116+
digest = hashlib.sha256(executable.read_bytes()).hexdigest()
117+
if digest != hashlib.sha256(source.read_bytes()).hexdigest():
118+
raise RuntimeError("Benchmark executable copy did not match source")
119+
inventory = temp / f"pet-inventory-{index}.json"
120+
environment = dict(os.environ, PET_DIAGNOSTIC_INVENTORY=str(inventory))
121+
log = artifacts / f"{index}-{label}.log"
122+
try:
123+
with log.open("w") as output:
124+
subprocess.run([tests[0], "test_performance_summary", "--exact", "--nocapture"],
125+
cwd=workspace, stdout=output, stderr=subprocess.STDOUT,
126+
check=True, timeout=180, env=environment)
127+
inventory_digest = hashlib.sha256(inventory.read_bytes()).hexdigest()
128+
finally:
129+
inventory.unlink(missing_ok=True)
130+
text = log.read_text()
131+
metrics, _ = json.JSONDecoder().raw_decode(text.split("JSON metrics:\n", 1)[1])
132+
(artifacts / f"{index}-{label}.json").write_text(json.dumps(metrics, indent=2) + "\n")
133+
result = {"iteration": index, "label": label, "revision": revisions[label], "sha256": digest,
134+
"inventory_sha256": inventory_digest,
135+
"environments_count": metrics["environments_count"],
136+
"managers_count": metrics["managers_count"],
137+
"stats": metrics["stats"], "locators": metrics["locators"], "phases": metrics["phases"]}
138+
summary.append(result)
139+
(artifacts / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")
140+
print(json.dumps({"iteration": index, "label": label, "revision": revisions[label],
141+
"sha256": digest, "refresh": metrics["stats"]["refresh_round_trip"],
142+
"startup": metrics["stats"]["server_startup"]}), flush=True)
143+
if len({v["inventory_sha256"] for v in summary}) != 1:
144+
raise RuntimeError("Exact inventory identities differ between base and head workloads")
145+
'@ | python -
146+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
147+
- name: Preserve matched control evidence
148+
if: always()
149+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
150+
with:
151+
name: fixture-557-matched-windows
152+
path: ${{ runner.temp }}\fixture-matched-control
153+
if-no-files-found: error

0 commit comments

Comments
 (0)