Skip to content

Commit 433e89c

Browse files
karthiknadigCopilot
karthiknadig
and
Copilot
committed
test: compare protocol revisions on one Windows host (Refs #532)
Run exact merged-main and protocol binaries on one host with counterbalanced passes, verified inventories, and durable failure evidence. Diagnostic-only branch; never merge this workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 55da715 commit 433e89c

1 file changed

Lines changed: 89 additions & 70 deletions

File tree

‎.github/workflows/perf-tests.yml‎

Lines changed: 89 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,22 @@
1-
name: Fixture Windows Matched Control
1+
name: Protocol Windows Matched Control
22
'on':
33
workflow_dispatch: {}
44
permissions:
55
contents: read
66
jobs:
77
matched-control:
88
runs-on: windows-latest
9-
timeout-minutes: 35
9+
timeout-minutes: 45
1010
steps:
1111
- name: Checkout exact PR head
1212
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
1313
with:
14-
ref: 6772e5c3631116386de3953dec083a2d08b4fcfa
14+
ref: 6209cbcf6eebd83d4ebfe941ed6cee2501a2bc61
15+
persist-credentials: false
1516
- name: Initialize evidence
1617
shell: pwsh
1718
run: |
18-
New-Item -ItemType Directory -Path "$env:RUNNER_TEMP\fixture-matched-control" -ErrorAction Stop | Out-Null
19+
New-Item -ItemType Directory -Path "$env:RUNNER_TEMP\protocol-matched-control" -ErrorAction Stop | Out-Null
1920
- name: Set Python to PATH
2021
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
2122
with:
@@ -57,9 +58,9 @@ jobs:
5758
- name: Build exact base and head
5859
shell: pwsh
5960
run: |
60-
git fetch --no-tags --depth=1 origin 3bd7eea22298a0a8e9a08079dd2ed44212368214
61+
git fetch --no-tags --depth=1 origin de7611fc74d96cb5de31a296178f46cc70254964
6162
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
62-
git worktree add --detach "$env:RUNNER_TEMP\pet-base" 3bd7eea22298a0a8e9a08079dd2ed44212368214
63+
git worktree add --detach "$env:RUNNER_TEMP\pet-base" de7611fc74d96cb5de31a296178f46cc70254964
6364
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
6465
cargo build --locked --release --target x86_64-pc-windows-msvc --bin pet --manifest-path "$env:RUNNER_TEMP\pet-base\Cargo.toml" --target-dir "$env:RUNNER_TEMP\pet-base-target"
6566
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
@@ -69,85 +70,103 @@ jobs:
6970
Copy-Item "target\x86_64-pc-windows-msvc\release\pet.exe" "$env:RUNNER_TEMP\pet-head-exe" -ErrorAction Stop
7071
cargo test --locked --release --features ci-perf --target x86_64-pc-windows-msvc --test e2e_performance --no-run --message-format=json > "$env:RUNNER_TEMP\pet-test-build.jsonl"
7172
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
72-
if ((git rev-parse HEAD) -ne '6772e5c3631116386de3953dec083a2d08b4fcfa') { throw 'Unexpected head revision' }
73-
if ((git -C "$env:RUNNER_TEMP\pet-base" rev-parse HEAD) -ne '3bd7eea22298a0a8e9a08079dd2ed44212368214') { throw 'Unexpected base revision' }
73+
if ((git rev-parse HEAD) -ne '6209cbcf6eebd83d4ebfe941ed6cee2501a2bc61') { throw 'Unexpected head revision' }
74+
if ((git -C "$env:RUNNER_TEMP\pet-base" rev-parse HEAD) -ne 'de7611fc74d96cb5de31a296178f46cc70254964') { throw 'Unexpected base revision' }
7475
@{
75-
base = '3bd7eea22298a0a8e9a08079dd2ed44212368214'
76-
head = '6772e5c3631116386de3953dec083a2d08b4fcfa'
76+
base = 'de7611fc74d96cb5de31a296178f46cc70254964'
77+
head = '6209cbcf6eebd83d4ebfe941ed6cee2501a2bc61'
7778
baseBinarySha256 = (Get-FileHash "$env:RUNNER_TEMP\pet-base-exe" -Algorithm SHA256).Hash
7879
headBinarySha256 = (Get-FileHash "$env:RUNNER_TEMP\pet-head-exe" -Algorithm SHA256).Hash
7980
harnessSha256 = (Get-FileHash 'crates\pet\tests\e2e_performance.rs' -Algorithm SHA256).Hash
8081
rustc = (rustc -Vv) -join "`n"
8182
runnerImage = $env:ImageOS
8283
runnerImageVersion = $env:ImageVersion
8384
processor = $env:PROCESSOR_IDENTIFIER
84-
} | ConvertTo-Json | Set-Content "$env:RUNNER_TEMP\fixture-matched-control\provenance.json" -Encoding utf8
85+
} | ConvertTo-Json | Set-Content "$env:RUNNER_TEMP\protocol-matched-control\provenance.json" -Encoding utf8
8586
- name: Counterbalanced same-host measurements
8687
shell: pwsh
8788
env:
8889
RUST_BACKTRACE: '1'
8990
RUST_LOG: warn
90-
run: |
91-
@'
92-
import hashlib
93-
import json
94-
import os
95-
from pathlib import Path
96-
import shutil
97-
import subprocess
98-
99-
temp = Path(os.environ["RUNNER_TEMP"])
100-
workspace = Path(os.environ["GITHUB_WORKSPACE"])
101-
artifacts = temp / "fixture-matched-control"
102-
artifacts.mkdir(exist_ok=True)
103-
messages = [json.loads(line) for line in (temp / "pet-test-build.jsonl").read_text(encoding="utf-8-sig").splitlines()]
104-
tests = [m["executable"] for m in messages if m.get("reason") == "compiler-artifact"
105-
and m["target"]["name"] == "e2e_performance" and m.get("executable")]
106-
if len(tests) != 1:
107-
raise RuntimeError(f"Expected one benchmark executable, got {tests!r}")
108-
executable = workspace / "target" / "x86_64-pc-windows-msvc" / "release" / "pet.exe"
109-
revisions = {"base": "3bd7eea22298a0a8e9a08079dd2ed44212368214",
110-
"head": "6772e5c3631116386de3953dec083a2d08b4fcfa"}
111-
order = ["base", "head", "head", "base", "head", "base", "base", "head"]
112-
summary = []
113-
for index, label in enumerate(order, 1):
114-
source = temp / f"pet-{label}-exe"
115-
shutil.copy2(source, executable)
116-
digest = hashlib.sha256(executable.read_bytes()).hexdigest()
117-
if digest != hashlib.sha256(source.read_bytes()).hexdigest():
118-
raise RuntimeError("Benchmark executable copy did not match source")
119-
inventory = temp / f"pet-inventory-{index}.json"
120-
environment = dict(os.environ, PET_DIAGNOSTIC_INVENTORY=str(inventory))
121-
log = artifacts / f"{index}-{label}.log"
122-
try:
123-
with log.open("w") as output:
124-
subprocess.run([tests[0], "test_performance_summary", "--exact", "--nocapture"],
125-
cwd=workspace, stdout=output, stderr=subprocess.STDOUT,
126-
check=True, timeout=180, env=environment)
127-
inventory_digest = hashlib.sha256(inventory.read_bytes()).hexdigest()
128-
finally:
129-
inventory.unlink(missing_ok=True)
130-
text = log.read_text()
131-
metrics, _ = json.JSONDecoder().raw_decode(text.split("JSON metrics:\n", 1)[1])
132-
(artifacts / f"{index}-{label}.json").write_text(json.dumps(metrics, indent=2) + "\n")
133-
result = {"iteration": index, "label": label, "revision": revisions[label], "sha256": digest,
134-
"inventory_sha256": inventory_digest,
135-
"environments_count": metrics["environments_count"],
136-
"managers_count": metrics["managers_count"],
137-
"stats": metrics["stats"], "locators": metrics["locators"], "phases": metrics["phases"]}
138-
summary.append(result)
139-
(artifacts / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")
140-
print(json.dumps({"iteration": index, "label": label, "revision": revisions[label],
141-
"sha256": digest, "refresh": metrics["stats"]["refresh_round_trip"],
142-
"startup": metrics["stats"]["server_startup"]}), flush=True)
143-
if len({v["inventory_sha256"] for v in summary}) != 1:
144-
raise RuntimeError("Exact inventory identities differ between base and head workloads")
145-
'@ | python -
146-
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
91+
run: "@'\nimport hashlib\nimport json\nimport os\nfrom pathlib import Path\nimport shutil\nimport subprocess\n\
92+
\ntemp = Path(os.environ[\"RUNNER_TEMP\"])\nworkspace = Path(os.environ[\"GITHUB_WORKSPACE\"])\nartifacts\
93+
\ = temp / \"protocol-matched-control\"\nartifacts.mkdir(exist_ok=True)\ncontrol_path = artifacts / \"control-status.json\"\
94+
\ncontrol = {\"status\": \"running\"}\ncontrol_path.write_text(json.dumps(control, indent=2) + \"\\n\",\
95+
\ encoding=\"utf-8\")\ntry:\n messages = [\n json.loads(line)\n for line in (temp / \"\
96+
pet-test-build.jsonl\").read_text(encoding=\"utf-8-sig\").splitlines()\n ]\n tests = [\n message[\"\
97+
executable\"]\n for message in messages\n if message.get(\"reason\") == \"compiler-artifact\"\
98+
\n and message[\"target\"][\"name\"] == \"e2e_performance\"\n and message.get(\"executable\"\
99+
)\n ]\n if len(tests) != 1:\n raise RuntimeError(f\"Expected one benchmark executable, got\
100+
\ {tests!r}\")\n \n executable = (\n workspace / \"target\" / \"x86_64-pc-windows-msvc\" /\
101+
\ \"release\" / \"pet.exe\"\n )\n revisions = {\n \"base\": \"de7611fc74d96cb5de31a296178f46cc70254964\"\
102+
,\n \"head\": \"6209cbcf6eebd83d4ebfe941ed6cee2501a2bc61\",\n }\n order = [\"base\", \"head\"\
103+
, \"head\", \"base\", \"head\", \"base\", \"base\", \"head\"]\n summary = []\n provenance = json.loads((artifacts\
104+
\ / \"provenance.json\").read_text(encoding=\"utf-8-sig\"))\n for index, label in enumerate(order, 1):\n\
105+
\ source = temp / f\"pet-{label}-exe\"\n shutil.copy2(source, executable)\n digest\
106+
\ = hashlib.sha256(executable.read_bytes()).hexdigest()\n if digest != provenance[f\"{label}BinarySha256\"\
107+
].lower():\n raise RuntimeError(\"Benchmark executable did not match recorded provenance\")\n\
108+
\ \n inventory = artifacts / f\"{index}-{label}.inventory.json\"\n environment = dict(\n\
109+
\ os.environ, PET_DIAGNOSTIC_INVENTORY=str(inventory)\n )\n log = artifacts / f\"\
110+
{index}-{label}.log\"\n result = {\n \"iteration\": index,\n \"label\": label,\n\
111+
\ \"revision\": revisions[label],\n \"sha256\": digest,\n \"status\": \"\
112+
running\",\n }\n summary.append(result)\n summary_path = artifacts / \"summary.json\"\
113+
\n summary_path.write_text(json.dumps(summary, indent=2) + \"\\n\")\n failure = None\n \
114+
\ try:\n with log.open(\"w\", encoding=\"utf-8\") as output:\n subprocess.run(\n\
115+
\ [\n tests[0],\n \"test_performance_summary\"\
116+
,\n \"--exact\",\n \"--nocapture\",\n ],\n\
117+
\ cwd=workspace,\n stdout=output,\n stderr=subprocess.STDOUT,\n\
118+
\ check=True,\n timeout=180,\n env=environment,\n\
119+
\ )\n except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:\n\
120+
\ failure = error\n result[\"error\"] = str(error)\n try:\n if inventory.is_file():\n\
121+
\ inventory_digest = hashlib.sha256(inventory.read_bytes()).hexdigest()\n \
122+
\ result[\"inventory_sha256\"] = inventory_digest\n (artifacts / f\"{index}-{label}.inventory.sha256\"\
123+
).write_text(\n inventory_digest + \"\\n\"\n )\n text = log.read_text(encoding=\"\
124+
utf-8\")\n marker = \"JSON metrics:\\n\"\n if marker not in text or \"inventory_sha256\"\
125+
\ not in result:\n raise ValueError(f\"Pass {index} is missing metrics or full inventory\"\
126+
)\n metrics, _ = json.JSONDecoder().raw_decode(text.rsplit(marker, 1)[1])\n if not\
127+
\ isinstance(metrics, dict):\n raise ValueError(f\"Pass {index} metrics are not an object\"\
128+
)\n (artifacts / f\"{index}-{label}.json\").write_text(\n json.dumps(metrics,\
129+
\ indent=2) + \"\\n\"\n )\n for key in [\"environments_count\", \"managers_count\"\
130+
, \"stats\", \"locators\", \"phases\"]:\n result[key] = metrics[key]\n for key\
131+
\ in [\"environments_count\", \"managers_count\"]:\n if type(metrics[key]) is not int or\
132+
\ metrics[key] < 0:\n raise ValueError(f\"Pass {index} has invalid {key}\")\n \
133+
\ if not isinstance(metrics[\"stats\"], dict):\n raise ValueError(f\"Pass {index} stats\
134+
\ are not an object\")\n for key in [\"refresh_round_trip\", \"server_startup\", \"discovery_duration\"\
135+
,\n \"request_time_to_first_env\", \"startup_time_to_first_env\",\n \
136+
\ \"cold_refresh_round_trip\", \"cold_discovery_duration\",\n \
137+
\ \"cold_request_time_to_first_env\", \"cold_startup_time_to_first_env\"]:\n metric =\
138+
\ metrics[\"stats\"][key]\n if not isinstance(metric, dict):\n raise ValueError(f\"\
139+
Pass {index} {key} is not an object\")\n for percentile in [\"p50\", \"p95\"]:\n \
140+
\ if type(metric.get(percentile)) is not int or metric[percentile] < 0:\n \
141+
\ raise ValueError(f\"Pass {index} has invalid {key}.{percentile}\")\n except (OSError, ValueError,\
142+
\ KeyError) as error:\n result[\"evidence_error\"] = str(error)\n if failure is None:\n\
143+
\ failure = error\n result[\"error\"] = str(error)\n result[\"status\"\
144+
] = \"failed\" if failure else \"completed\"\n try:\n summary_path.write_text(json.dumps(summary,\
145+
\ indent=2) + \"\\n\")\n except OSError as error:\n if failure:\n raise\
146+
\ failure from error\n raise\n if failure:\n raise failure\n print(\n\
147+
\ json.dumps(\n {\n \"iteration\": index,\n \
148+
\ \"label\": label,\n \"revision\": revisions[label],\n \"sha256\"\
149+
: digest,\n \"inventory_sha256\": inventory_digest,\n \"environments_count\"\
150+
: metrics[\"environments_count\"],\n \"managers_count\": metrics[\"managers_count\"],\n\
151+
\ \"refresh\": metrics[\"stats\"][\"refresh_round_trip\"],\n \"startup\"\
152+
: metrics[\"stats\"][\"server_startup\"],\n }\n ),\n flush=True,\n\
153+
\ )\n \n if len(summary) != 8:\n raise RuntimeError(f\"Expected eight completed passes,\
154+
\ got {len(summary)}\")\n if len({result[\"inventory_sha256\"] for result in summary}) != 1:\n \
155+
\ raise RuntimeError(\n \"Exact inventory identities differ between base and head workloads\"\
156+
\n )\n if len({result[\"environments_count\"] for result in summary}) != 1:\n raise RuntimeError(\n\
157+
\ \"Environment counts differ between base and head workloads\"\n )\n if len({result[\"\
158+
managers_count\"] for result in summary}) != 1:\n raise RuntimeError(\n \"Manager counts\
159+
\ differ between base and head workloads\"\n )\nexcept Exception as error:\n control.update(status=\"\
160+
failed\", error=f\"{type(error).__name__}: {error}\")\n try:\n control_path.write_text(json.dumps(control,\
161+
\ indent=2) + \"\\n\", encoding=\"utf-8\")\n except OSError as evidence_error:\n raise error from\
162+
\ evidence_error\n raise\nelse:\n control[\"status\"] = \"completed\"\n control_path.write_text(json.dumps(control,\
163+
\ indent=2) + \"\\n\", encoding=\"utf-8\")\n'@ | python -\nif ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE\
164+
\ }\n"
165+
timeout-minutes: 30
147166
- name: Preserve matched control evidence
148167
if: always()
149168
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
150169
with:
151-
name: fixture-557-matched-windows
152-
path: ${{ runner.temp }}\fixture-matched-control
170+
name: protocol-532-matched-windows
171+
path: ${{ runner.temp }}/protocol-matched-control
153172
if-no-files-found: error

0 commit comments

Comments
 (0)