From 02e307a9ef62be55d4ba498c159fbf56de087dc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 12:17:07 +0300 Subject: [PATCH 01/15] Add hermetic PatchEval runner --- docs/patcheval.md | 79 + pyproject.toml | 1 + src/benchkit/benchmarks/__init__.py | 3 + src/benchkit/benchmarks/patcheval.py | 592 +++++++ src/benchkit/engine.py | 26 +- src/benchkit/pi_agent.py | 8 +- src/benchkit/pi_package/package-lock.json | 1827 +++++++++++++++++++++ src/benchkit/pi_package/package.json | 8 + src/benchkit/sandbox.py | 56 +- tests/test_patcheval.py | 235 +++ tests/test_pi_harness.py | 15 +- tests/test_repair.py | 5 +- 12 files changed, 2834 insertions(+), 21 deletions(-) create mode 100644 docs/patcheval.md create mode 100644 src/benchkit/benchmarks/patcheval.py create mode 100644 src/benchkit/pi_package/package-lock.json create mode 100644 src/benchkit/pi_package/package.json create mode 100644 tests/test_patcheval.py diff --git a/docs/patcheval.md b/docs/patcheval.md new file mode 100644 index 0000000..ea96675 --- /dev/null +++ b/docs/patcheval.md @@ -0,0 +1,79 @@ +# PatchEval runner contract + +PatchEval is split deliberately. The one-shot miner and frozen corpus live in a +separate Hugging Face dataset repository. BenchKit contains only the runner, +prompt, sandbox boundary, and deterministic grader. + +## Threat model + +The Pi agent receives a parent-commit source archive in a fresh `/workspace`. +It has no host mount, Docker socket, GitHub network access, real Git history, +gold patch, Hugging Face cache, or grader assets. Its only network route is the +restricted inference proxy already used by BenchKit's Pi harness. + +After each attempt, BenchKit copies the workspace to the trusted host and diffs +it against a fresh extraction of the checksummed source archive. Agent Git +metadata is ignored. Changes matching `protected_globs` or `ignored_globs` are +left out of the submitted patch. Protected globs must cover every test tree so +an agent may write tests for itself without making those tests part of scoring. + +BenchKit then starts two new containers with `--network none`: + +1. The fail-to-pass grader applies the submitted patch and the hidden test + patch, then runs `fail_to_pass_command`. +2. The regression grader applies only the submitted patch, then runs + `regression_command` against the parent commit's previously passing tests. + +The task passes only when both exit codes are zero. A grader setup failure is a +harness error, not an incorrect answer. Raw output remains in the report for +benchmark maintainers but is never sent to the model. Repair turns receive only +the generic instruction to re-examine the issue and continue. + +## Frozen dataset layout + +```text +dataset.json +tasks.jsonl +artifacts/ + /source.tar.gz + /hidden-tests.patch +``` + +`dataset.json` uses schema version 1 and contains `release` and `task_count`. +Each line of `tasks.jsonl` contains: + +- `id`, `repository`, `issue_title`, and reviewed `issue_body` +- `runtime_image`, pinned as an OCI `name@sha256:...` reference +- relative `source_archive` and `hidden_test_patch` paths plus SHA-256 hashes +- argv arrays for `setup_command`, `fail_to_pass_command`, and + `regression_command` +- `protected_globs`, `ignored_globs`, `timeout_s`, and `validated: true` + +Source archives must contain repository contents at their root, exclude `.git`, +and dereference symlinks. Runtime images are immutable, Debian-compatible +Python environments that provide Node 22.19 or newer, npm, Git, GNU tar, +coreutils `timeout`, ripgrep, a non-root `node` user, and all task dependencies. +BenchKit wraps each runtime image with the locked Pi package and verifies that +`pi --version` is exactly 0.84.2. + +The miner must set `validated: true` only after checking, in clean containers, +that the hidden test fails on the parent, passes on the original fix, and the +full regression command passes on both. A frozen release never changes in +place; later date windows receive a new release name. + +## Model prompt + +The runner sends the reviewed issue title and full reviewed body, without issue +comments, pull-request text, labels, URLs, commit identifiers, or benchmark +instructions: + +```text +Fix the following issue in the current repository. Inspect the code, make the necessary changes, and verify your solution. + +# {issue_title} + +{issue_body} +``` + +Set `BENCHKIT_PATCHEVAL_DATASET` to an immutable local snapshot of the published +dataset before running `patcheval`. diff --git a/pyproject.toml b/pyproject.toml index 292734c..d08a0a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ benchkit = [ "datasets/*.txt", "git_surgery/**/*.sh", "git_surgery/**/*.md", + "pi_package/*.json", "templates/*.html", "tui/*.tcss", ] diff --git a/src/benchkit/benchmarks/__init__.py b/src/benchkit/benchmarks/__init__.py index e672bba..b8dd88c 100644 --- a/src/benchkit/benchmarks/__init__.py +++ b/src/benchkit/benchmarks/__init__.py @@ -14,6 +14,7 @@ from benchkit.benchmarks.mmlu import MMLU from benchkit.benchmarks.mmlu_pro import MMLUPro from benchkit.benchmarks.openbookqa import OpenBookQA +from benchkit.benchmarks.patcheval import PatchEval from benchkit.benchmarks.piqa import PIQA from benchkit.benchmarks.ruler import RULER, RULERFull from benchkit.benchmarks.sanity import Sanity @@ -25,6 +26,7 @@ # Generative suites first, then the multiple-choice ones. "aider-polyglot": AiderPolyglot, "git-surgery": GitSurgery, + "patcheval": PatchEval, "sanity": Sanity, "humaneval": HumanEval, "humaneval-plus": HumanEvalPlus, @@ -51,6 +53,7 @@ DESCRIPTIONS: dict[str, str] = { "aider-polyglot": "repository editing across six languages with the Pi agent", "git-surgery": "stateful Git operations in isolated repositories with Pi", + "patcheval": "real Python bug fixes with externally isolated hidden tests", "sanity": "25 curated checks across code, math, instructions, science, and commonsense", "humaneval": "Python function completions with the original unit tests", "humaneval-plus": "HumanEval with 122k+ tougher EvalPlus test inputs", diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py new file mode 100644 index 0000000..d3de6af --- /dev/null +++ b/src/benchkit/benchmarks/patcheval.py @@ -0,0 +1,592 @@ +"""Hermetic bug-fix tasks scored by hidden tests outside the agent sandbox.""" + +from __future__ import annotations + +import fnmatch +import hashlib +import json +import os +import shutil +import stat +import subprocess +import tarfile +import tempfile +import uuid +from dataclasses import dataclass +from pathlib import Path, PurePosixPath + +from benchkit.benchmarks.base import Task +from benchkit.evaluation import EvaluationResult +from benchkit.sandbox import ( + DockerTaskEnvironment, + LatestPiImage, + SandboxError, + _run, + patcheval_pi_image, +) + +_SCHEMA_VERSION = 1 +_TASKS_FILE = "tasks.jsonl" +_RELEASE_FILE = "dataset.json" +_GENERIC_REPAIR = ( + "The issue is not fully resolved yet. Re-examine your changes and continue " + "working on the fix. Use the repository and any tests you can run to validate " + "your next attempt." +) +_MAX_PATCH_BYTES = 5 * 1024 * 1024 +_MAX_CANDIDATE_FILES = 50_000 +_MAX_CANDIDATE_BYTES = 512 * 1024 * 1024 + + +@dataclass(frozen=True) +class PatchEvalSpec: + """One frozen PatchEval task and its host-only grading assets.""" + + id: str + repository: str + issue_title: str + issue_body: str + runtime_image: str + source_archive: Path + hidden_test_patch: Path + source_sha256: str + hidden_test_sha256: str + fail_to_pass_command: tuple[str, ...] + regression_command: tuple[str, ...] + setup_command: tuple[str, ...] + protected_globs: tuple[str, ...] + ignored_globs: tuple[str, ...] + timeout_s: int + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _safe_relative(root: Path, value: object, field: str) -> Path: + if not isinstance(value, str) or not value: + raise ValueError(f"{field} must be a non-empty relative path") + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts: + raise ValueError(f"{field} must stay inside the dataset root") + return root.joinpath(*path.parts) + + +def _strings(record: dict, field: str, *, required: bool = True) -> tuple[str, ...]: + value = record.get(field) + if value is None and not required: + return () + if ( + not isinstance(value, list) + or not value + or not all(isinstance(item, str) and item for item in value) + ): + raise ValueError(f"{field} must be a non-empty list of strings") + return tuple(value) + + +def _load_spec(root: Path, record: object) -> PatchEvalSpec: + if not isinstance(record, dict): + raise ValueError("each PatchEval task must be a JSON object") + required_strings = ( + "id", + "repository", + "issue_title", + "issue_body", + "runtime_image", + "source_sha256", + "hidden_test_sha256", + ) + for field in required_strings: + if not isinstance(record.get(field), str) or not record[field].strip(): + raise ValueError(f"{field} must be a non-empty string") + if record.get("validated") is not True: + raise ValueError(f"PatchEval task {record['id']!r} is not miner-validated") + + source = _safe_relative(root, record.get("source_archive"), "source_archive") + hidden = _safe_relative(root, record.get("hidden_test_patch"), "hidden_test_patch") + for path, field, expected in ( + (source, "source_archive", record["source_sha256"]), + (hidden, "hidden_test_patch", record["hidden_test_sha256"]), + ): + if not path.is_file(): + raise ValueError(f"{field} does not exist for task {record['id']!r}") + actual = _sha256(path) + if actual != expected: + raise ValueError( + f"{field} checksum mismatch for task {record['id']!r}: " + f"expected {expected}, got {actual}" + ) + + timeout_s = record.get("timeout_s", 300) + if isinstance(timeout_s, bool) or not isinstance(timeout_s, int): + raise ValueError("timeout_s must be an integer") + if not 10 <= timeout_s <= 1800: + raise ValueError("timeout_s must be between 10 and 1800 seconds") + + return PatchEvalSpec( + id=record["id"], + repository=record["repository"], + issue_title=record["issue_title"].strip(), + issue_body=record["issue_body"].strip(), + runtime_image=record["runtime_image"], + source_archive=source, + hidden_test_patch=hidden, + source_sha256=record["source_sha256"], + hidden_test_sha256=record["hidden_test_sha256"], + fail_to_pass_command=_strings(record, "fail_to_pass_command"), + regression_command=_strings(record, "regression_command"), + setup_command=_strings(record, "setup_command", required=False), + protected_globs=_strings(record, "protected_globs"), + ignored_globs=_strings(record, "ignored_globs", required=False), + timeout_s=timeout_s, + ) + + +def _safe_extract(archive: Path, destination: Path) -> None: + """Extract a miner-produced source archive without following archive links.""" + with tarfile.open(archive, "r:*") as handle: + members = handle.getmembers() + for member in members: + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts: + raise ValueError(f"unsafe path in source archive: {member.name!r}") + if member.issym() or member.islnk(): + raise ValueError( + f"source archives must dereference links: {member.name!r}" + ) + if member.isdev() or member.isfifo(): + raise ValueError(f"unsupported archive entry: {member.name!r}") + handle.extractall(destination, members=members, filter="fully_trusted") + + +def _git( + worktree: Path, args: list[str], *, check: bool = True +) -> subprocess.CompletedProcess[str]: + environment = os.environ.copy() + environment.update( + GIT_CONFIG_NOSYSTEM="1", + GIT_CONFIG_GLOBAL=os.devnull, + ) + completed = subprocess.run( + ["git", "-C", str(worktree), *args], + text=True, + capture_output=True, + env=environment, + timeout=60, + ) + if check and completed.returncode: + detail = (completed.stderr or completed.stdout).strip()[-2000:] + raise RuntimeError(f"trusted patch extraction failed: {detail}") + return completed + + +def _remove_entry(path: Path) -> None: + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.exists(): + shutil.rmtree(path) + + +def _copy_candidate(source: Path, destination: Path) -> None: + """Copy only ordinary files, directories, and symlinks from the sandbox.""" + file_count = 0 + byte_count = 0 + + def copy_entry(current: Path, target: Path) -> None: + nonlocal file_count, byte_count + if current.name == ".git": + return + info = current.lstat() + file_count += 1 + if file_count > _MAX_CANDIDATE_FILES: + raise ValueError("candidate workspace contains too many files") + if stat.S_ISLNK(info.st_mode): + target.symlink_to(os.readlink(current)) + elif stat.S_ISDIR(info.st_mode): + target.mkdir(mode=info.st_mode & 0o777, exist_ok=True) + for child in current.iterdir(): + copy_entry(child, target / child.name) + elif stat.S_ISREG(info.st_mode): + byte_count += info.st_size + if byte_count > _MAX_CANDIDATE_BYTES: + raise ValueError("candidate workspace is too large") + shutil.copyfile(current, target, follow_symlinks=False) + target.chmod(info.st_mode & 0o777) + else: + raise ValueError(f"candidate contains unsupported file: {current.name!r}") + + for child in source.iterdir(): + copy_entry(child, destination / child.name) + + +def _matches(path: str, patterns: tuple[str, ...]) -> bool: + return any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns) + + +def _trusted_patch( + spec: PatchEvalSpec, environment: DockerTaskEnvironment +) -> tuple[str, list[str], list[str]]: + """Diff a downloaded workspace against the host's pristine source archive.""" + with tempfile.TemporaryDirectory(prefix="benchkit-patcheval-diff-") as directory: + root = Path(directory) + worktree = root / "trusted" + candidate = root / "candidate" + worktree.mkdir() + candidate.mkdir() + _safe_extract(spec.source_archive, worktree) + environment.download(f"{environment.workdir}/.", candidate) + + _git(worktree, ["init", "-q"]) + _git(worktree, ["add", "-A"]) + _git( + worktree, + [ + "-c", + "user.name=BenchKit", + "-c", + "user.email=benchkit@localhost", + "commit", + "-q", + "--no-gpg-sign", + "-m", + "pristine", + ], + ) + for child in worktree.iterdir(): + if child.name != ".git": + _remove_entry(child) + _copy_candidate(candidate, worktree) + _git(worktree, ["add", "--intent-to-add", "--", "."]) + changed = _git( + worktree, ["diff", "--name-only", "--no-renames", "HEAD", "--"] + ).stdout.splitlines() + excluded = [ + path + for path in changed + if _matches(path, spec.protected_globs + spec.ignored_globs) + ] + included = [path for path in changed if path not in excluded] + if not included: + return "", changed, excluded + patch = _git( + worktree, + ["diff", "--binary", "--no-renames", "HEAD", "--", *included], + ).stdout + if len(patch.encode()) > _MAX_PATCH_BYTES: + raise ValueError("candidate patch exceeds the 5 MiB limit") + return patch, changed, excluded + + +def _docker_exec( + docker: str, + container: str, + command: list[str], + *, + workdir: str | None = None, + timeout: float = 60, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + return _run( + [ + docker, + "exec", + *(["--workdir", workdir] if workdir else []), + container, + *command, + ], + timeout=timeout, + check=check, + ) + + +def _grade_once( + spec: PatchEvalSpec, + image: LatestPiImage, + patch: str, + command: tuple[str, ...], + *, + hidden_tests: bool, + owner_id: str, +) -> dict: + docker = image.docker + label = "f2p" if hidden_tests else "regression" + container = f"benchkit-patcheval-{label}-{uuid.uuid4().hex[:12]}" + resource_labels = [ + "--label", + "benchkit.managed=true", + "--label", + f"benchkit.owner={owner_id}", + ] + with tempfile.TemporaryDirectory(prefix="benchkit-patcheval-grade-") as directory: + patch_path = Path(directory) / "submission.patch" + patch_path.write_text(patch, encoding="utf-8") + try: + _run( + [ + docker, + "run", + "--detach", + "--name", + container, + *resource_labels, + "--network", + "none", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + os.environ.get("BENCHKIT_SANDBOX_PIDS", "256"), + "--memory", + os.environ.get("BENCHKIT_SANDBOX_MEMORY", "2g"), + "--cpus", + os.environ.get("BENCHKIT_SANDBOX_CPUS", "2"), + image.image, + "sleep", + "infinity", + ], + timeout=60, + ) + _run([docker, "cp", str(spec.source_archive), f"{container}:/tmp/source"]) + _run([docker, "cp", str(patch_path), f"{container}:/tmp/submission.patch"]) + if hidden_tests: + _run( + [ + docker, + "cp", + str(spec.hidden_test_patch), + f"{container}:/tmp/hidden-tests.patch", + ] + ) + _docker_exec(docker, container, ["mkdir", "-p", "/workspace/repo"]) + _docker_exec( + docker, + container, + ["tar", "-xf", "/tmp/source", "-C", "/workspace/repo"], + ) + if patch: + _docker_exec( + docker, + container, + ["git", "apply", "--binary", "/tmp/submission.patch"], + workdir="/workspace/repo", + ) + if hidden_tests: + _docker_exec( + docker, + container, + ["git", "apply", "--binary", "/tmp/hidden-tests.patch"], + workdir="/workspace/repo", + ) + if spec.setup_command: + _docker_exec( + docker, + container, + list(spec.setup_command), + workdir="/workspace/repo", + timeout=spec.timeout_s + 30, + ) + completed = _docker_exec( + docker, + container, + ["timeout", "--signal=KILL", f"{spec.timeout_s}s", *command], + workdir="/workspace/repo", + timeout=spec.timeout_s + 30, + check=False, + ) + output = (completed.stdout + completed.stderr).strip() + return { + "exit_code": completed.returncode, + "output": output[-8000:], + "output_truncated": len(output) > 8000, + } + finally: + _run([docker, "rm", "--force", container], timeout=30, check=False) + + +class PatchEval: + """Fix real Python repository issues with hidden, deterministic grading.""" + + name = "patcheval" + task_count = 20 + workspace_task = True + evaluation_activity = "running hidden PatchEval checks" + list_note = "pilot-20 ยท requires Pi and Docker" + + def __init__(self, dataset_root: Path | None = None) -> None: + configured = dataset_root or ( + Path(os.environ["BENCHKIT_PATCHEVAL_DATASET"]).expanduser() + if os.environ.get("BENCHKIT_PATCHEVAL_DATASET") + else None + ) + self.dataset_root = configured + self.release = "unloaded" + + def _root(self) -> Path: + if self.dataset_root is None: + raise RuntimeError( + "PatchEval's frozen dataset is not published yet; set " + "BENCHKIT_PATCHEVAL_DATASET to a local dataset snapshot" + ) + return self.dataset_root.resolve() + + @staticmethod + def _spec(task: Task) -> PatchEvalSpec: + spec = task.metadata.get("spec") + if not isinstance(spec, PatchEvalSpec): + raise TypeError("PatchEval task metadata is missing its validated spec") + return spec + + def load_tasks(self) -> list[Task]: + root = self._root() + release_path = root / _RELEASE_FILE + tasks_path = root / _TASKS_FILE + try: + release = json.loads(release_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"could not read PatchEval dataset metadata: {exc}" + ) from exc + if ( + not isinstance(release, dict) + or release.get("schema_version") != _SCHEMA_VERSION + ): + raise RuntimeError( + f"PatchEval dataset must use schema_version {_SCHEMA_VERSION}" + ) + self.release = str(release.get("release") or "unknown") + try: + records = [ + json.loads(line) + for line in tasks_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + specs = [_load_spec(root, record) for record in records] + except (OSError, json.JSONDecodeError, ValueError) as exc: + raise RuntimeError(f"invalid PatchEval dataset: {exc}") from exc + expected = release.get("task_count") + if expected != len(specs): + raise RuntimeError( + f"PatchEval release declares {expected} tasks but contains {len(specs)}" + ) + ids = [spec.id for spec in specs] + if len(ids) != len(set(ids)): + raise RuntimeError("PatchEval task ids must be unique") + self.task_count = len(specs) + return [Task(spec.id, "", {"spec": spec}) for spec in specs] + + def result_metadata(self, _variant: str | None) -> dict: + return {"dataset_release": self.release} + + def build_prompt(self, task: Task) -> str: + spec = self._spec(task) + return ( + "Fix the following issue in the current repository. Inspect the code, " + "make the necessary changes, and verify your solution.\n\n" + f"# {spec.issue_title}\n\n{spec.issue_body}" + ) + + def build_repair_prompt(self, _feedback: str, _attempt: int, _total: int) -> str: + return _GENERIC_REPAIR + + def evaluate(self, _task: Task, _response: str) -> bool: + raise RuntimeError("PatchEval must be evaluated by its external grader") + + def pi_image_for_task(self, task: Task) -> LatestPiImage: + return patcheval_pi_image(self._spec(task).runtime_image) + + def prepare_workspace(self, task: Task, environment: DockerTaskEnvironment) -> None: + spec = self._spec(task) + environment.workdir = "/workspace/repo" + environment.upload(spec.source_archive, "/tmp/patch-eval-source") + environment.exec(["mkdir", "-p", environment.workdir]) + environment.exec( + ["tar", "-xf", "/tmp/patch-eval-source", "-C", environment.workdir] + ) + environment.exec(["rm", "-rf", f"{environment.workdir}/.git"]) + if spec.setup_command: + environment.exec( + list(spec.setup_command), + workdir=environment.workdir, + timeout=spec.timeout_s + 30, + ) + environment.exec(["git", "init", "-q", environment.workdir]) + environment.exec( + ["git", "-C", environment.workdir, "config", "user.name", "BenchKit"] + ) + environment.exec( + [ + "git", + "-C", + environment.workdir, + "config", + "user.email", + "benchkit@localhost", + ] + ) + environment.exec(["git", "-C", environment.workdir, "add", "."]) + environment.exec( + [ + "git", + "-C", + environment.workdir, + "commit", + "-q", + "-m", + "starting state", + ] + ) + + def verify_workspace( + self, + task: Task, + environment: DockerTaskEnvironment, + _tool_trace: list[dict] | None = None, + ) -> EvaluationResult: + spec = self._spec(task) + try: + patch, changed, excluded = _trusted_patch(spec, environment) + f2p = _grade_once( + spec, + environment.image, + patch, + spec.fail_to_pass_command, + hidden_tests=True, + owner_id=environment.owner_id, + ) + regression = _grade_once( + spec, + environment.image, + patch, + spec.regression_command, + hidden_tests=False, + owner_id=environment.owner_id, + ) + except (OSError, RuntimeError, ValueError, SandboxError) as exc: + return EvaluationResult( + 0.0, + error=f"PatchEval grader infrastructure failed: {type(exc).__name__}: {exc}", + ) + passed = f2p["exit_code"] == 0 and regression["exit_code"] == 0 + return EvaluationResult( + 1.0 if passed else 0.0, + feedback="" if passed else _GENERIC_REPAIR, + details={ + "repository": spec.repository, + "f2p_exit_code": f2p["exit_code"], + "regression_exit_code": regression["exit_code"], + "f2p_output": f2p["output"], + "f2p_output_truncated": f2p["output_truncated"], + "regression_output": regression["output"], + "regression_output_truncated": regression["output_truncated"], + "changed_paths": changed, + "excluded_agent_paths": excluded, + "patch_sha256": hashlib.sha256(patch.encode()).hexdigest(), + "patch": patch, + }, + ) diff --git a/src/benchkit/engine.py b/src/benchkit/engine.py index 14a0714..bf9ae7b 100644 --- a/src/benchkit/engine.py +++ b/src/benchkit/engine.py @@ -834,7 +834,18 @@ def emit(self, event: EngineEvent) -> None: with self._emit_lock: self.sink(event) - def _pi(self, bench: object | None = None) -> PiAgentRunner: + def _pi( + self, bench: object | None = None, task: Task | None = None + ) -> PiAgentRunner: + task_image_factory = getattr(bench, "pi_image_for_task", None) + if callable(task_image_factory) and task is not None: + image = task_image_factory(task) + key = f"{getattr(bench, 'name', type(bench).__name__)}:{image.image}" + if key not in self._workspace_pi_runners: + self._workspace_pi_runners[key] = PiAgentRunner( + self.client, image=image + ) + return self._workspace_pi_runners[key] image_factory = getattr(bench, "pi_image", None) if callable(image_factory): key = str(getattr(bench, "name", type(bench).__name__)) @@ -979,10 +990,11 @@ def _run_job( task_id=first.id, entry_point=str(first.metadata.get("entry_point", "")), phase="generating", - activity="preparing latest Pi sandbox image", + activity="preparing pinned Pi sandbox image", ) ) - self._pi(bench).prepare() + for task in tasks: + self._pi(bench, task).prepare() if not tasks: return None, False @@ -1702,10 +1714,11 @@ def on_progress(update: GenerationUpdate) -> None: request_started = time.perf_counter() try: - generator = self._pi(bench) if job.harness == "pi" else self.client + generator = self._pi(bench, task) if job.harness == "pi" else self.client workspace = bool(getattr(bench, "workspace_task", False)) workspace_setup = None workspace_verifier = None + repair_prompt_builder = None if workspace: def workspace_setup(environment): @@ -1714,6 +1727,10 @@ def workspace_setup(environment): def workspace_verifier(environment, tool_trace): return bench.verify_workspace(task, environment, tool_trace) + candidate_builder = getattr(bench, "build_repair_prompt", None) + if callable(candidate_builder): + repair_prompt_builder = candidate_builder + if job.repair_attempts: def verifier(response: str) -> EvaluationResult: @@ -1729,6 +1746,7 @@ def verifier(response: str) -> EvaluationResult: repair_attempts=job.repair_attempts, workspace_setup=workspace_setup, workspace_verifier=workspace_verifier, + repair_prompt_builder=repair_prompt_builder, ) else: gen = self._generate_direct_with_repairs( diff --git a/src/benchkit/pi_agent.py b/src/benchkit/pi_agent.py index fb926de..015be69 100644 --- a/src/benchkit/pi_agent.py +++ b/src/benchkit/pi_agent.py @@ -165,6 +165,7 @@ def generate( [DockerTaskEnvironment, list[dict[str, Any]]], EvaluationResult ] | None = None, + repair_prompt_builder: Callable[[str, int, int], str] | None = None, ) -> dict: self.prepare() started = time.perf_counter() @@ -426,11 +427,8 @@ def send(payload: dict[str, Any]) -> None: if attempt >= repair_attempts: break feedback_sent.append(evaluation.feedback) - message = repair_message( - evaluation.feedback, - attempt + 1, - repair_attempts, - ) + builder = repair_prompt_builder or repair_message + message = builder(evaluation.feedback, attempt + 1, repair_attempts) scaffold_value = environment.pi_scaffold() scaffold = ( dict(scaffold_value) if isinstance(scaffold_value, dict) else {} diff --git a/src/benchkit/pi_package/package-lock.json b/src/benchkit/pi_package/package-lock.json new file mode 100644 index 0000000..ffb728e --- /dev/null +++ b/src/benchkit/pi_package/package-lock.json @@ -0,0 +1,1827 @@ +{ + "name": "benchkit-pi-runtime", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "benchkit-pi-runtime", + "version": "0.0.0", + "dependencies": { + "@earendil-works/pi-coding-agent": "0.84.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.84.2.tgz", + "integrity": "sha512-l4E+B7hgXKWddRo8bC/eSue2aWZjEgJ9xIpf5p0Og+lq8a2TArCwJ0HCoCPCgaBP/tN4zbYH/wOwvx9pJpeLCA==", + "hasShrinkwrap": true, + "license": "MIT", + "dependencies": { + "@earendil-works/pi-agent-core": "^0.84.2", + "@earendil-works/pi-ai": "^0.84.2", + "@earendil-works/pi-client": "^0.84.2", + "@earendil-works/pi-protocol": "^0.84.2", + "@earendil-works/pi-tui": "^0.84.2", + "@silvia-odwyer/photon-node": "0.3.4", + "chalk": "5.6.2", + "cross-spawn": "7.0.6", + "diff": "8.0.4", + "glob": "13.0.6", + "grok-mermaid": "0.2.2", + "highlight.js": "10.7.3", + "hosted-git-info": "9.0.3", + "ignore": "7.0.5", + "jiti": "2.7.0", + "minimatch": "10.2.5", + "proper-lockfile": "4.1.2", + "semver": "7.8.0", + "typebox": "1.3.7", + "undici": "8.9.0", + "yaml": "2.9.0" + }, + "bin": { + "pi": "dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + }, + "optionalDependencies": { + "@mariozechner/clipboard": "0.3.9" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@anthropic-ai/sdk": { + "version": "0.91.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz", + "integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/client-bedrock-runtime": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1048.0.tgz", + "integrity": "sha512-u+NT61JZEkRFtpL0CAw1N1dwxnaLgwVXQl/zjJxTGgLyS/jTIdg2SdoEoCTHxgDyCnqa1HEi9QOoE9/pYRNpOQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/credential-provider-node": "^3.972.42", + "@aws-sdk/eventstream-handler-node": "^3.972.16", + "@aws-sdk/middleware-eventstream": "^3.972.12", + "@aws-sdk/middleware-websocket": "^3.972.19", + "@aws-sdk/token-providers": "3.1048.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/core": { + "version": "3.974.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.974.11.tgz", + "integrity": "sha512-QpnINq5FZH6EOaDEkmHdT7eUunbvD27pDNQypaWjFyYz7Zl1q3UCMQErBZxpmfGfI7MvI2TlK8KTkgNpv8b1ug==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/xml-builder": "^3.972.24", + "@aws/lambda-invoke-store": "^0.2.2", + "@smithy/core": "^3.24.2", + "@smithy/signature-v4": "^5.4.2", + "@smithy/types": "^4.14.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.37", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.37.tgz", + "integrity": "sha512-/jpPvEh6f7ntmIzf7dNxoNX6Q8vt8UpesCjbW6mFfk4V1NW6bIy9qxcQ6WbA8As5yQhsZOe+xeNd4xHX8kdY2Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.39", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.39.tgz", + "integrity": "sha512-pIgTpisWyWg7X1bUbzSjuUYosYTD0Ghz2M0hkSTmb3a6i3qV3uU+NYJPI/E2XSC0HcsZh5rsLPzeXrkb2DS0Cg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.41.tgz", + "integrity": "sha512-u2tyjaxJJzW8UtW4SM1ZcPMDwO6y+kV+llvou+Adts0FAKyzes5jG4izQN+KX3yE8ZROpS5y1LJ//xL2iSf76w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/credential-provider-env": "^3.972.37", + "@aws-sdk/credential-provider-http": "^3.972.39", + "@aws-sdk/credential-provider-login": "^3.972.41", + "@aws-sdk/credential-provider-process": "^3.972.37", + "@aws-sdk/credential-provider-sso": "^3.972.41", + "@aws-sdk/credential-provider-web-identity": "^3.972.41", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/credential-provider-imds": "^4.3.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.41.tgz", + "integrity": "sha512-0LBitxXiAiaE5nlFPfpNIww/8FRY/I7WIndWsc9GmNFOM7cE1wNpVNQEGEk9Outg5l8xl+3vybxFyUy4l9q/LQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.42", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.42.tgz", + "integrity": "sha512-D4oon2zbqqsWOJUM99Gm3/ZyJ0IJvTXVN3PyloGb3kQEyI36fjCZheZj422lAgTWWd6TSHgiImLt3RIaLdv3dQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.37", + "@aws-sdk/credential-provider-http": "^3.972.39", + "@aws-sdk/credential-provider-ini": "^3.972.41", + "@aws-sdk/credential-provider-process": "^3.972.37", + "@aws-sdk/credential-provider-sso": "^3.972.41", + "@aws-sdk/credential-provider-web-identity": "^3.972.41", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/credential-provider-imds": "^4.3.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.37", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.37.tgz", + "integrity": "sha512-7nVaHBUaWIddASYfVaA9O4D5ZVjewU3sCol9WqZPGfW0nR+0WqE0xHZnD/U2L33PlOB8KNXGKZ6wOES/QijKzg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.41.tgz", + "integrity": "sha512-IOWAWEHe5LkjSKkkUUX9ciV6Y1scHTsnfEkdt5yyC4Slrc7AGbkLPrpntjqh18ksJAMOaVhoBsO8p2WyTcY2wQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/token-providers": "3.1048.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.41.tgz", + "integrity": "sha512-mbACk9Yypa8nm4iGZLs0PofOXEcTDOUw6wDnsPXNDNSd2WNXs1tSo+6nc/fh0jLYdfVZThhBL98PHW4aXFsG5A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/eventstream-handler-node": { + "version": "3.972.16", + "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.16.tgz", + "integrity": "sha512-yedpPgKftqjU5SlPFHfqWpOw6xSCRieWRG1euWOlXn4WJxt2VX92VprCa2PpSOXjVCAeK6dTjW9eJRXVig9yGA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/middleware-eventstream": { + "version": "3.972.12", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.12.tgz", + "integrity": "sha512-tHTHHCHNrq6XklQvlzHBDJG4Iuhh7NVPRdtmvP+nHFA+5sxPlIDzlAHHgfoYHGvT3NXP1yVP/L5c3opUn6T3Qg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/middleware-websocket": { + "version": "3.972.19", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-websocket/-/middleware-websocket-3.972.19.tgz", + "integrity": "sha512-mkEhOGYozqKQkbFaVrjwr0faiwwZza1v5/jSY6Tucm3bD+uKTazIUH/4Yo6aMnQD2ua2W9cMP6s8mvwTcjtqHw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/signature-v4": "^5.4.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/nested-clients": { + "version": "3.997.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.9.tgz", + "integrity": "sha512-jPR3rnmRI4hWYyzfmTGBr7NblMp8QYYeflHXba1H6+7CGrWVqWKQzaXFQ4qbExqPRsXN3T3L3JxFhr6aouXUGQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/signature-v4-multi-region": "^3.996.27", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.27", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.27.tgz", + "integrity": "sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/signature-v4": "^5.4.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/token-providers": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1048.0.tgz", + "integrity": "sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/types": { + "version": "3.973.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.8.tgz", + "integrity": "sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/util-locate-window": { + "version": "3.965.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.5.tgz", + "integrity": "sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/xml-builder": { + "version": "3.972.24", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.24.tgz", + "integrity": "sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==", + "license": "Apache-2.0", + "dependencies": { + "@nodable/entities": "2.1.0", + "@smithy/types": "^4.14.1", + "fast-xml-parser": "5.7.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws/lambda-invoke-store": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.4.tgz", + "integrity": "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@babel/runtime": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-agent-core": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.84.2.tgz", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-ai": "^0.84.2", + "@earendil-works/pi-telemetry": "^0.84.2", + "diff": "8.0.4", + "ignore": "7.0.5", + "typebox": "1.3.7", + "yaml": "2.9.0" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-ai": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.84.2.tgz", + "license": "MIT", + "dependencies": { + "@anthropic-ai/sdk": "0.91.1", + "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@earendil-works/pi-telemetry": "^0.84.2", + "@google/genai": "1.52.0", + "@opentelemetry/api": "1.9.0", + "@smithy/node-http-handler": "4.7.3", + "http-proxy-agent": "7.0.2", + "https-proxy-agent": "7.0.6", + "openai": "6.40.0", + "partial-json": "0.1.7", + "typebox": "1.3.7" + }, + "bin": { + "pi-ai": "dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-client": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-client/-/pi-client-0.84.2.tgz", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-protocol": "^0.84.2" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-protocol": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-protocol/-/pi-protocol-0.84.2.tgz", + "license": "MIT", + "dependencies": { + "typebox": "1.3.7" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-telemetry": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-0.84.2.tgz", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-tui": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.84.2.tgz", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "1.6.0", + "marked": "18.0.5" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@google/genai": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.52.0.tgz", + "integrity": "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.9.tgz", + "integrity": "sha512-ABnA53mdfkGZwOFUdZNv2S0CWGO/EIuPj8Vv9xmBFmSYg/qFc7ihO6q5FcQjvoE67kZpWkEc4AhD6B/os04yuA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@mariozechner/clipboard-darwin-arm64": "0.3.9", + "@mariozechner/clipboard-darwin-universal": "0.3.9", + "@mariozechner/clipboard-darwin-x64": "0.3.9", + "@mariozechner/clipboard-linux-arm64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-arm64-musl": "0.3.9", + "@mariozechner/clipboard-linux-riscv64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-x64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-x64-musl": "0.3.9", + "@mariozechner/clipboard-win32-arm64-msvc": "0.3.9", + "@mariozechner/clipboard-win32-x64-msvc": "0.3.9" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-darwin-arm64": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-arm64/-/clipboard-darwin-arm64-0.3.9.tgz", + "integrity": "sha512-BfgV7vCEWZwJwZJw03r6bP5+tf0iI/ANuQYCxi9RNn7FrWB3yzGuMKCrNLRl6V761vXRdL8+OqZ0wd4TqlsNOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-darwin-universal": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-universal/-/clipboard-darwin-universal-0.3.9.tgz", + "integrity": "sha512-BGGR4iA9Z2shAjI65eI5xtyb3LYNlDW9X3gxKxDbqtbnREohsrqznov6zpKoIrsRWpzlYVEdKphS7ksJ0/ndSQ==", + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-darwin-x64": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-x64/-/clipboard-darwin-x64-0.3.9.tgz", + "integrity": "sha512-4kURmCbS6nt8uYhtmWpUcJWyPHfmAr5dTpXD1nO3pIfa+TSQ9DbrGOYCKH+aEFW47XhQ4Vp8ZTszie+wfFvDKg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-arm64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-gnu/-/clipboard-linux-arm64-gnu-0.3.9.tgz", + "integrity": "sha512-g59OkUGP2DDfCOIKypHeYgv2M55u/cKvXa5dSxFbEJ34XvIQMdcVmpKCkGUro3ZgefXiGVdwguvTMQGpHWzIXw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-arm64-musl": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-musl/-/clipboard-linux-arm64-musl-0.3.9.tgz", + "integrity": "sha512-AGuJdgKsmJdm4Pych7kv3sqe591ERRaAHW3xjLooiFzn8J+PxUyof++7YZrB5Y5tpnTO+K18Og3taj2NpluCRQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-riscv64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-riscv64-gnu/-/clipboard-linux-riscv64-gnu-0.3.9.tgz", + "integrity": "sha512-DXBEAiuMpk7dhS1a9NzNxVAFi1vaKoPu7rQNgY8LIDLGrK3lnIp3nT10DUum+PKVJoJppIP+NAA8IZe4DMNDPw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-x64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-gnu/-/clipboard-linux-x64-gnu-0.3.9.tgz", + "integrity": "sha512-WORrMLd6EpElEME7JRKfSaY34nW1P5LbdgK5YNCS1ncG2LqmITsSMEJ8nh2mpvxb3TxqbOOKgY7k9eMJYlW9Mw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-x64-musl": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-musl/-/clipboard-linux-x64-musl-0.3.9.tgz", + "integrity": "sha512-/DHn+1DrfL6oRaPPWXaOKvonFFrni666fxd+zFqiQEfvBH0tsHVWjq9iqBk0oDp0qaPA72lIMy5BptxISBEhZQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-win32-arm64-msvc": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-arm64-msvc/-/clipboard-win32-arm64-msvc-0.3.9.tgz", + "integrity": "sha512-O5FHD3ErkMwMhNzAfu3ggy0ug4z7btZuoQgwwxlzPrwV2bxlD6WDpqBY4NCgICAgZdDKdp+loUEKVAVt8aYnhQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-win32-x64-msvc": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-x64-msvc/-/clipboard-win32-x64-msvc-0.3.9.tgz", + "integrity": "sha512-ihQC3EufqEY81vhXBgVBtK4prL+wc62zJsSvxrgz7K1hsdt6OObz6v9p3Rn1OG3GJksTTKMJF0u/guMISHPhSA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@nodable/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@silvia-odwyer/photon-node": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon-node/-/photon-node-0.3.4.tgz", + "integrity": "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA==", + "license": "Apache-2.0" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/core": { + "version": "3.24.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.24.3.tgz", + "integrity": "sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/credential-provider-imds": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.3.3.tgz", + "integrity": "sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/fetch-http-handler": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.4.3.tgz", + "integrity": "sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/node-http-handler": { + "version": "4.7.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.7.3.tgz", + "integrity": "sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/signature-v4": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.4.3.tgz", + "integrity": "sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/types": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.14.2.tgz", + "integrity": "sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@types/node": { + "version": "22.19.19", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.19.tgz", + "integrity": "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/brace-expansion": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/diff": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/fast-xml-builder": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", + "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.5.0", + "xml-naming": "^0.1.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/fast-xml-parser": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.3.tgz", + "integrity": "sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^2.1.0", + "fast-xml-builder": "^1.1.7", + "path-expression-matcher": "^1.5.0", + "strnum": "^2.2.3" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/gaxios": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", + "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/google-auth-library": { + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", + "integrity": "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/grok-mermaid": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/grok-mermaid/-/grok-mermaid-0.2.2.tgz", + "integrity": "sha512-XcJEP5dDC8liHBh52mlLjU18fNvu1ckFsu0QpIG3+APZ270fsj9wxpiA6cOURmbUEuoMVgjbC2+UYgTdCqqgzA==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/hosted-git-info": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.3.tgz", + "integrity": "sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg==", + "license": "ISC", + "dependencies": { + "lru-cache": "^11.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/lru-cache": { + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.4.0.tgz", + "integrity": "sha512-W+R+kFL4HgVxONq2bhXPi3bGpzGe/yEhVOp233qw9wCRtgncJ15P3bC+e4zZMu4Cq7d+WAJjXGW0uUkifhcatA==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/marked": { + "version": "18.0.5", + "resolved": "https://registry.npmjs.org/marked/-/marked-18.0.5.tgz", + "integrity": "sha512-S6GcvALHg6K4ohtu4E7x0a1AqhAjp6cV8KhLSyN9qVapnzJkusVBxZRcIU9AeYsbe6P1hKDusSbEOzGyyuce6w==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/openai": { + "version": "6.40.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.40.0.tgz", + "integrity": "sha512-MWtTjd/gQt4jpbji61NTgFWJLoY/PdRJ6wG9/ZDRMYNMlBKrCrSlkLI+KgHP1vR1qT6LKSAyAqIxno6lcK9JiA==", + "license": "Apache-2.0", + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/p-retry/node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/partial-json": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/partial-json/-/partial-json-0.1.7.tgz", + "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/path-expression-matcher": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", + "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/proper-lockfile/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/strnum": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.3.0.tgz", + "integrity": "sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/typebox": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.7.tgz", + "integrity": "sha512-meKuifc33Pccx0O6PdIzYMq3Og8zvP4TIi/a+Bw3AEMZMxOD0+RHGQvpglEe6Zdy3wZ8nqn/j95h8LUZLk/6Hg==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/undici": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.9.0.tgz", + "integrity": "sha512-aWZpUj7XoGonMClx4gdDRfgBjqeA+F473aDmROQQbM9n6PRfK/u1q/a0X4wMTgcHfT8H6fpbt98PFuDUwFg2YA==", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/xml-naming": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", + "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + } + } +} diff --git a/src/benchkit/pi_package/package.json b/src/benchkit/pi_package/package.json new file mode 100644 index 0000000..7bb73c2 --- /dev/null +++ b/src/benchkit/pi_package/package.json @@ -0,0 +1,8 @@ +{ + "name": "benchkit-pi-runtime", + "private": true, + "version": "0.0.0", + "dependencies": { + "@earendil-works/pi-coding-agent": "0.84.2" + } +} diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index 92f9fa4..b04528b 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -2,8 +2,10 @@ from __future__ import annotations +import hashlib import json import os +import re import shutil import subprocess import tempfile @@ -18,10 +20,19 @@ PI_IMAGE = "benchkit-pi:latest" AIDER_PI_IMAGE = "benchkit-pi-aider-polyglot:latest" GIT_SURGERY_PI_IMAGE = "benchkit-pi-git-surgery:latest" -PI_PACKAGE = "@earendil-works/pi-coding-agent@latest" +PI_PACKAGE = "@earendil-works/pi-coding-agent@0.84.2" +PI_VERSION = "0.84.2" AIDER_POLYGLOT_COMMIT = "7e0611e77b54e2dea774cdc0aa00cf9f7ed6144f" _PROXY_SOURCE = Path(__file__).with_name("_pi_proxy.py") _ANSWER_KEY_GUARD_SOURCE = Path(__file__).with_name("answer_key_guard.ts") +_PI_PACKAGE_ROOT = Path(__file__).with_name("pi_package") + +_PI_INSTALL = """\ +COPY pi-package/package.json pi-package/package-lock.json /opt/benchkit/pi/ +RUN cd /opt/benchkit/pi \\ + && npm ci --omit=dev \\ + && ln -s /opt/benchkit/pi/node_modules/.bin/pi /usr/local/bin/pi +""" PI_DOCKERFILE = f"""\ FROM node:24-bookworm-slim @@ -29,7 +40,7 @@ RUN apt-get update \\ && apt-get install -y --no-install-recommends bash ca-certificates git python3 ripgrep \\ && rm -rf /var/lib/apt/lists/* -RUN npm install -g {PI_PACKAGE} +{_PI_INSTALL} COPY inference_proxy.py /opt/benchkit/inference_proxy.py COPY answer_key_guard.ts /opt/benchkit/answer_key_guard.ts @@ -51,7 +62,7 @@ openjdk-17-jdk \\ python3 ripgrep unzip \\ && rm -rf /var/lib/apt/lists/* -RUN npm install -g {PI_PACKAGE} +{_PI_INSTALL} ARG TARGETARCH RUN curl -fsSL "https://go.dev/dl/go1.21.5.linux-${{TARGETARCH}}.tar.gz" \\ -o /tmp/go.tar.gz \\ @@ -121,7 +132,7 @@ bash ca-certificates git=${{GIT_DEBIAN_VERSION}} python3 ripgrep \\ && test "$(git --version)" = "git version 2.39.5" \\ && rm -rf /var/lib/apt/lists/* -RUN npm install -g {PI_PACKAGE} +{_PI_INSTALL} COPY inference_proxy.py /opt/benchkit/inference_proxy.py COPY answer_key_guard.ts /opt/benchkit/answer_key_guard.ts @@ -222,7 +233,7 @@ def cleanup_owned_resources(docker: str, owner_id: str) -> None: @dataclass class LatestPiImage: - """Build the stock Pi image once per run, resolving npm's latest release.""" + """Build the reproducibly pinned stock Pi image once per run.""" docker: str = field(default_factory=_docker_binary) image: str = PI_IMAGE @@ -259,6 +270,7 @@ def prepare(self) -> str: _ANSWER_KEY_GUARD_SOURCE.read_text(encoding="utf-8"), encoding="utf-8", ) + shutil.copytree(_PI_PACKAGE_ROOT, context / "pi-package") if self.build_assets is not None: shutil.copytree(self.build_assets, context / "git-surgery") command = [self.docker, "build", "--pull"] @@ -271,6 +283,10 @@ def prepare(self) -> str: timeout=30, ) self.version = result.stdout.strip() or "latest" + if self.version != PI_VERSION: + raise SandboxError( + f"Pi image reported {self.version!r}; expected {PI_VERSION!r}" + ) self._ready = True return self.version @@ -312,6 +328,36 @@ def git_surgery_pi_image() -> LatestPiImage: ) +def patcheval_pi_image(runtime_image: str) -> LatestPiImage: + """Wrap one immutable PatchEval runtime with the pinned stock Pi agent.""" + if not re.fullmatch(r"(?:[A-Za-z0-9._/:+@-]+@)?sha256:[0-9a-f]{64}", runtime_image): + raise ValueError( + "PatchEval runtime_image must be an OCI reference pinned by sha256 digest" + ) + image_id = hashlib.sha256(runtime_image.encode()).hexdigest()[:16] + dockerfile = f"""\\ +FROM {runtime_image} + +USER root +{_PI_INSTALL} +COPY inference_proxy.py /opt/benchkit/inference_proxy.py +COPY answer_key_guard.ts /opt/benchkit/answer_key_guard.ts +RUN mkdir -p /workspace /home/node/.pi/agent \\ + && chown -R node:node /workspace /home/node/.pi + +USER node +WORKDIR /workspace +ENV PI_OFFLINE=1 PI_TELEMETRY=0 +CMD ["sleep", "infinity"] +""" + return LatestPiImage( + image=f"benchkit-pi-patcheval:{image_id}", + dockerfile=dockerfile, + no_cache=False, + transient=False, + ) + + @dataclass class DockerTaskEnvironment: """One isolated, persistent workspace shared by every native Pi tool call.""" diff --git a/tests/test_patcheval.py b/tests/test_patcheval.py new file mode 100644 index 0000000..265664f --- /dev/null +++ b/tests/test_patcheval.py @@ -0,0 +1,235 @@ +"""Tests for PatchEval dataset validation and hidden grading boundaries.""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import tarfile +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from benchkit.benchmarks.patcheval import ( + _GENERIC_REPAIR, + PatchEval, + _trusted_patch, +) +from benchkit.engine import Engine +from benchkit.sandbox import LatestPiImage + +_RUNTIME_IMAGE = "example.invalid/patcheval/python@sha256:" + "a" * 64 + + +def _digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _dataset(root: Path, *, validated: bool = True) -> dict: + source_root = root / "source" + source_root.mkdir() + (source_root / "module.py").write_text("VALUE = 1\n", encoding="utf-8") + tests = source_root / "tests" + tests.mkdir() + (tests / "test_existing.py").write_text("def test_old(): pass\n", encoding="utf-8") + archive = root / "source.tar.gz" + with tarfile.open(archive, "w:gz") as handle: + for path in sorted(source_root.rglob("*")): + handle.add(path, arcname=path.relative_to(source_root), recursive=False) + hidden = root / "hidden.patch" + hidden.write_text("hidden grader patch\n", encoding="utf-8") + record = { + "id": "owner-repo-123", + "repository": "owner/repo", + "issue_title": "Handle empty input", + "issue_body": "Calling parse with an empty value should return None.", + "runtime_image": _RUNTIME_IMAGE, + "source_archive": archive.name, + "hidden_test_patch": hidden.name, + "source_sha256": _digest(archive), + "hidden_test_sha256": _digest(hidden), + "fail_to_pass_command": ["python3", "-m", "pytest", "grader_test.py"], + "regression_command": ["python3", "-m", "pytest", "tests"], + "protected_globs": ["tests/**", "grader_test.py"], + "ignored_globs": [".venv/**", "*.egg-info/**"], + "timeout_s": 120, + "validated": validated, + } + (root / "dataset.json").write_text( + json.dumps({"schema_version": 1, "release": "pilot-20", "task_count": 1}), + encoding="utf-8", + ) + (root / "tasks.jsonl").write_text(json.dumps(record) + "\n", encoding="utf-8") + return record + + +class DownloadEnvironment: + def __init__(self, candidate: Path) -> None: + self.candidate = candidate + self.workdir = "/workspace/repo" + + def download(self, _source: str, destination: Path) -> None: + shutil.copytree(self.candidate, destination, dirs_exist_ok=True, symlinks=True) + + +class PatchEvalTests(unittest.TestCase): + def test_loads_only_validated_checksum_pinned_tasks(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + tasks = benchmark.load_tasks() + + self.assertEqual([task.id for task in tasks], ["owner-repo-123"]) + self.assertEqual(benchmark.task_count, 1) + self.assertEqual( + benchmark.result_metadata(None), {"dataset_release": "pilot-20"} + ) + prompt = benchmark.build_prompt(tasks[0]) + self.assertEqual( + prompt, + "Fix the following issue in the current repository. Inspect the code, " + "make the necessary changes, and verify your solution.\n\n" + "# Handle empty input\n\n" + "Calling parse with an empty value should return None.", + ) + self.assertNotIn("benchmark", prompt.lower()) + self.assertNotIn("hidden", prompt.lower()) + + def test_rejects_unvalidated_tasks(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root, validated=False) + with self.assertRaisesRegex(RuntimeError, "not miner-validated"): + PatchEval(root).load_tasks() + + def test_rejects_modified_dataset_assets(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + (root / "hidden.patch").write_text("changed", encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "checksum mismatch"): + PatchEval(root).load_tasks() + + def test_trusted_patch_ignores_agent_git_and_test_changes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + task = PatchEval(root).load_tasks()[0] + spec = task.metadata["spec"] + candidate = root / "candidate" + candidate.mkdir() + (candidate / "module.py").write_text("VALUE = 2\n", encoding="utf-8") + tests = candidate / "tests" + tests.mkdir() + (tests / "test_existing.py").write_text( + "def test_old(): assert False\n", encoding="utf-8" + ) + (tests / "test_agent.py").write_text( + "def test_new(): pass\n", encoding="utf-8" + ) + (candidate / ".git").mkdir() + (candidate / ".git" / "config").write_text( + "malicious metadata", encoding="utf-8" + ) + + submission, changed, excluded = _trusted_patch( + spec, DownloadEnvironment(candidate) + ) + + self.assertIn("module.py", submission) + self.assertNotIn("test_agent.py", submission) + self.assertEqual( + changed, + ["module.py", "tests/test_agent.py", "tests/test_existing.py"], + ) + self.assertEqual(excluded, ["tests/test_agent.py", "tests/test_existing.py"]) + + def test_verifier_runs_hidden_and_regression_graders_separately(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + environment = SimpleNamespace( + image=SimpleNamespace(), owner_id="owner", workdir="/workspace/repo" + ) + hidden_failure = { + "exit_code": 1, + "output": "SECRET_EXPECTATION", + "output_truncated": False, + } + regression_pass = { + "exit_code": 0, + "output": "all old tests passed", + "output_truncated": False, + } + with ( + patch( + "benchkit.benchmarks.patcheval._trusted_patch", + return_value=("diff", ["module.py"], []), + ), + patch( + "benchkit.benchmarks.patcheval._grade_once", + side_effect=[hidden_failure, regression_pass], + ) as grade, + ): + result = benchmark.verify_workspace(task, environment) + + self.assertFalse(result.passed) + self.assertEqual(result.feedback, _GENERIC_REPAIR) + self.assertNotIn("SECRET_EXPECTATION", result.feedback) + self.assertEqual(grade.call_count, 2) + self.assertTrue(grade.call_args_list[0].kwargs["hidden_tests"]) + self.assertFalse(grade.call_args_list[1].kwargs["hidden_tests"]) + self.assertEqual(result.details["f2p_exit_code"], 1) + self.assertEqual(result.details["regression_exit_code"], 0) + + def test_workspace_setup_exposes_source_but_not_hidden_patch(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + environment = Mock(workdir="/workspace", exec=Mock()) + + benchmark.prepare_workspace(task, environment) + + environment.upload.assert_called_once() + uploaded = environment.upload.call_args.args + self.assertEqual(uploaded[0].name, "source.tar.gz") + self.assertNotIn("hidden", str(uploaded[0])) + commands = [call.args[0] for call in environment.exec.call_args_list] + self.assertIn(["git", "init", "-q", "/workspace/repo"], commands) + self.assertTrue( + all("hidden.patch" not in " ".join(command) for command in commands) + ) + + def test_repair_prompt_is_generic_and_does_not_mention_grading(self) -> None: + prompt = PatchEval().build_repair_prompt("secret output", 1, 3) + + self.assertEqual(prompt, _GENERIC_REPAIR) + self.assertNotIn("verifier", prompt.lower()) + self.assertNotIn("test output", prompt.lower()) + + def test_engine_caches_one_runner_per_task_runtime_image(self) -> None: + image = LatestPiImage( + docker="docker", image="benchkit-pi-patcheval:runtime", transient=False + ) + benchmark = SimpleNamespace( + name="patcheval", pi_image_for_task=Mock(return_value=image) + ) + task = SimpleNamespace() + engine = Engine(client=SimpleNamespace(), jobs=[]) + + first = engine._pi(benchmark, task) + second = engine._pi(benchmark, task) + + self.assertIs(first, second) + self.assertEqual(benchmark.pi_image_for_task.call_count, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pi_harness.py b/tests/test_pi_harness.py index 510120c..a5a01e6 100644 --- a/tests/test_pi_harness.py +++ b/tests/test_pi_harness.py @@ -20,6 +20,7 @@ from benchkit.sandbox import ( PI_DOCKERFILE, PI_PACKAGE, + PI_VERSION, DockerTaskEnvironment, LatestPiImage, _docker_upstream, @@ -31,17 +32,19 @@ class LatestPiImageTests(unittest.TestCase): def test_generic_pi_sandbox_keeps_the_restricted_pid_limit(self) -> None: self.assertEqual(LatestPiImage(docker="docker").pids_limit, 256) - def test_package_deliberately_tracks_npm_latest(self) -> None: - self.assertEqual(PI_PACKAGE, "@earendil-works/pi-coding-agent@latest") - self.assertIn(f"npm install -g {PI_PACKAGE}", PI_DOCKERFILE) + def test_package_and_transitive_dependencies_are_pinned(self) -> None: + self.assertEqual(PI_PACKAGE, "@earendil-works/pi-coding-agent@0.84.2") + self.assertEqual(PI_VERSION, "0.84.2") + self.assertIn("npm ci --omit=dev", PI_DOCKERFILE) + self.assertNotIn("@latest", PI_DOCKERFILE) def test_prepare_pulls_and_bypasses_build_cache_once_per_run(self) -> None: image = LatestPiImage(docker="docker") with patch("benchkit.sandbox._run") as run: - run.return_value = SimpleNamespace(stdout="0.83.0\n") - self.assertEqual(image.prepare(), "0.83.0") - self.assertEqual(image.prepare(), "0.83.0") + run.return_value = SimpleNamespace(stdout=f"{PI_VERSION}\n") + self.assertEqual(image.prepare(), PI_VERSION) + self.assertEqual(image.prepare(), PI_VERSION) commands = [call.args[0] for call in run.call_args_list] self.assertEqual(commands[0][:2], ["docker", "version"]) diff --git a/tests/test_repair.py b/tests/test_repair.py index 7aabc5d..c56f98b 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -303,6 +303,7 @@ def test_pi_repair_reuses_one_rpc_process_and_workspace(self) -> None: EvaluationResult(1.0), ] ) + repair_prompt_builder = Mock(return_value="Continue fixing the issue.") with patch( "benchkit.pi_agent.DockerTaskEnvironment", @@ -313,6 +314,7 @@ def test_pi_repair_reuses_one_rpc_process_and_workspace(self) -> None: "original prompt", verifier=verifier, repair_attempts=1, + repair_prompt_builder=repair_prompt_builder, ) task_environment.assert_called_once() @@ -321,7 +323,8 @@ def test_pi_repair_reuses_one_rpc_process_and_workspace(self) -> None: prompts = [item for item in sent if item.get("type") == "prompt"] self.assertEqual(len(prompts), 2) self.assertEqual(prompts[0]["message"], "original prompt") - self.assertIn("Sanitized failure.", prompts[1]["message"]) + self.assertEqual(prompts[1]["message"], "Continue fixing the issue.") + repair_prompt_builder.assert_called_once_with("Sanitized failure.", 1, 1) self.assertEqual(result["response"], "right") self.assertTrue(result["repaired"]) self.assertEqual(result["repair_attempts_used"], 1) From 3581ca239f42640f07041972c9e3a68a49fbae01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 12:36:52 +0300 Subject: [PATCH 02/15] Exclude agent-authored tests from PatchEval scoring --- docs/patcheval.md | 7 +++++-- src/benchkit/benchmarks/patcheval.py | 16 ++++++++++++++- src/benchkit/sandbox.py | 2 +- tests/test_patcheval.py | 29 +++++++++++++++++++++++++++- 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/patcheval.md b/docs/patcheval.md index ea96675..0113421 100644 --- a/docs/patcheval.md +++ b/docs/patcheval.md @@ -14,8 +14,11 @@ restricted inference proxy already used by BenchKit's Pi harness. After each attempt, BenchKit copies the workspace to the trusted host and diffs it against a fresh extraction of the checksummed source archive. Agent Git metadata is ignored. Changes matching `protected_globs` or `ignored_globs` are -left out of the submitted patch. Protected globs must cover every test tree so -an agent may write tests for itself without making those tests part of scoring. +left out of the submitted patch. The runner always protects conventional Python +test paths (`tests/`, `test/`, `test_*.py`, `*_test.py`, and `conftest.py`); +dataset `protected_globs` must cover any repository-specific test locations. +An agent may therefore write tests for itself without making those tests part +of scoring. BenchKit then starts two new containers with `--network none`: diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py index d3de6af..c6fa0e3 100644 --- a/src/benchkit/benchmarks/patcheval.py +++ b/src/benchkit/benchmarks/patcheval.py @@ -36,6 +36,18 @@ _MAX_PATCH_BYTES = 5 * 1024 * 1024 _MAX_CANDIDATE_FILES = 50_000 _MAX_CANDIDATE_BYTES = 512 * 1024 * 1024 +_PYTHON_TEST_GLOBS = ( + "tests/**", + "test/**", + "**/tests/**", + "**/test/**", + "test_*.py", + "*_test.py", + "**/test_*.py", + "**/*_test.py", + "conftest.py", + "**/conftest.py", +) @dataclass(frozen=True) @@ -141,7 +153,9 @@ def _load_spec(root: Path, record: object) -> PatchEvalSpec: fail_to_pass_command=_strings(record, "fail_to_pass_command"), regression_command=_strings(record, "regression_command"), setup_command=_strings(record, "setup_command", required=False), - protected_globs=_strings(record, "protected_globs"), + protected_globs=tuple( + dict.fromkeys(_PYTHON_TEST_GLOBS + _strings(record, "protected_globs")) + ), ignored_globs=_strings(record, "ignored_globs", required=False), timeout_s=timeout_s, ) diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index b04528b..8a2ba60 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -330,7 +330,7 @@ def git_surgery_pi_image() -> LatestPiImage: def patcheval_pi_image(runtime_image: str) -> LatestPiImage: """Wrap one immutable PatchEval runtime with the pinned stock Pi agent.""" - if not re.fullmatch(r"(?:[A-Za-z0-9._/:+@-]+@)?sha256:[0-9a-f]{64}", runtime_image): + if not re.fullmatch(r"[A-Za-z0-9._/:+@-]+@sha256:[0-9a-f]{64}", runtime_image): raise ValueError( "PatchEval runtime_image must be an OCI reference pinned by sha256 digest" ) diff --git a/tests/test_patcheval.py b/tests/test_patcheval.py index 265664f..f066519 100644 --- a/tests/test_patcheval.py +++ b/tests/test_patcheval.py @@ -18,7 +18,7 @@ _trusted_patch, ) from benchkit.engine import Engine -from benchkit.sandbox import LatestPiImage +from benchkit.sandbox import LatestPiImage, patcheval_pi_image _RUNTIME_IMAGE = "example.invalid/patcheval/python@sha256:" + "a" * 64 @@ -147,6 +147,29 @@ def test_trusted_patch_ignores_agent_git_and_test_changes(self) -> None: ) self.assertEqual(excluded, ["tests/test_agent.py", "tests/test_existing.py"]) + def test_standard_root_python_tests_are_always_excluded(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + task = PatchEval(root).load_tasks()[0] + spec = task.metadata["spec"] + candidate = root / "candidate" + candidate.mkdir() + (candidate / "module.py").write_text("VALUE = 2\n", encoding="utf-8") + shutil.copytree(root / "source" / "tests", candidate / "tests") + (candidate / "test_agent.py").write_text( + "def test_agent(): pass\n", encoding="utf-8" + ) + + submission, changed, excluded = _trusted_patch( + spec, DownloadEnvironment(candidate) + ) + + self.assertIn("module.py", submission) + self.assertNotIn("test_agent.py", submission) + self.assertIn("test_agent.py", changed) + self.assertIn("test_agent.py", excluded) + def test_verifier_runs_hidden_and_regression_graders_separately(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -230,6 +253,10 @@ def test_engine_caches_one_runner_per_task_runtime_image(self) -> None: self.assertIs(first, second) self.assertEqual(benchmark.pi_image_for_task.call_count, 2) + def test_runtime_image_requires_a_named_oci_digest(self) -> None: + with self.assertRaisesRegex(ValueError, "pinned by sha256"): + patcheval_pi_image("sha256:" + "a" * 64) + if __name__ == "__main__": unittest.main() From 7ba89c2b991277e46e842448b484bd58936f1f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 16:45:33 +0300 Subject: [PATCH 03/15] Enforce PatchEval runtime image contract --- src/benchkit/benchmarks/patcheval.py | 6 ++++++ src/benchkit/sandbox.py | 2 +- tests/test_patcheval.py | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py index c6fa0e3..3fe32ed 100644 --- a/src/benchkit/benchmarks/patcheval.py +++ b/src/benchkit/benchmarks/patcheval.py @@ -6,6 +6,7 @@ import hashlib import json import os +import re import shutil import stat import subprocess @@ -36,6 +37,7 @@ _MAX_PATCH_BYTES = 5 * 1024 * 1024 _MAX_CANDIDATE_FILES = 50_000 _MAX_CANDIDATE_BYTES = 512 * 1024 * 1024 +_RUNTIME_IMAGE_RE = re.compile(r"[A-Za-z0-9._/:+@-]+@sha256:[0-9a-f]{64}") _PYTHON_TEST_GLOBS = ( "tests/**", "test/**", @@ -116,6 +118,10 @@ def _load_spec(root: Path, record: object) -> PatchEvalSpec: for field in required_strings: if not isinstance(record.get(field), str) or not record[field].strip(): raise ValueError(f"{field} must be a non-empty string") + if not _RUNTIME_IMAGE_RE.fullmatch(record["runtime_image"]): + raise ValueError( + "runtime_image must be a canonical OCI reference pinned by sha256 digest" + ) if record.get("validated") is not True: raise ValueError(f"PatchEval task {record['id']!r} is not miner-validated") diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index 8a2ba60..662230d 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -347,7 +347,7 @@ def patcheval_pi_image(runtime_image: str) -> LatestPiImage: USER node WORKDIR /workspace -ENV PI_OFFLINE=1 PI_TELEMETRY=0 +ENV HOME=/home/node USER=node LOGNAME=node PI_OFFLINE=1 PI_TELEMETRY=0 CMD ["sleep", "infinity"] """ return LatestPiImage( diff --git a/tests/test_patcheval.py b/tests/test_patcheval.py index f066519..3c3db88 100644 --- a/tests/test_patcheval.py +++ b/tests/test_patcheval.py @@ -105,6 +105,17 @@ def test_rejects_unvalidated_tasks(self) -> None: with self.assertRaisesRegex(RuntimeError, "not miner-validated"): PatchEval(root).load_tasks() + def test_rejects_runtime_image_without_registry_digest(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + record = _dataset(root) + record["runtime_image"] = "example.invalid/patcheval/python:latest" + (root / "tasks.jsonl").write_text( + json.dumps(record) + "\n", encoding="utf-8" + ) + with self.assertRaisesRegex(RuntimeError, "runtime_image"): + PatchEval(root).load_tasks() + def test_rejects_modified_dataset_assets(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -257,6 +268,9 @@ def test_runtime_image_requires_a_named_oci_digest(self) -> None: with self.assertRaisesRegex(ValueError, "pinned by sha256"): patcheval_pi_image("sha256:" + "a" * 64) + image = patcheval_pi_image(_RUNTIME_IMAGE) + self.assertIn("HOME=/home/node USER=node LOGNAME=node", image.dockerfile) + if __name__ == "__main__": unittest.main() From cf3bb6305f3f86a3e9043ca0352dc2d755582e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 17:44:23 +0300 Subject: [PATCH 04/15] Build PatchEval runtimes locally --- docs/patcheval.md | 29 ++- src/benchkit/benchmarks/patcheval.py | 116 ++++++++-- src/benchkit/sandbox.py | 326 ++++++++++++++++++++++++--- tests/test_patcheval.py | 200 ++++++++++++++-- 4 files changed, 595 insertions(+), 76 deletions(-) diff --git a/docs/patcheval.md b/docs/patcheval.md index 0113421..a3bb2f0 100644 --- a/docs/patcheval.md +++ b/docs/patcheval.md @@ -46,18 +46,35 @@ artifacts/ Each line of `tasks.jsonl` contains: - `id`, `repository`, `issue_title`, and reviewed `issue_body` -- `runtime_image`, pinned as an OCI `name@sha256:...` reference +- `runtime_recipe`, containing schema version 1, an explicitly version-tagged + Debian-compatible `base_image`, required `sync_command`, and optional + `bootstrap_command` and `environment` - relative `source_archive` and `hidden_test_patch` paths plus SHA-256 hashes - argv arrays for `setup_command`, `fail_to_pass_command`, and `regression_command` - `protected_globs`, `ignored_globs`, `timeout_s`, and `validated: true` Source archives must contain repository contents at their root, exclude `.git`, -and dereference symlinks. Runtime images are immutable, Debian-compatible -Python environments that provide Node 22.19 or newer, npm, Git, GNU tar, -coreutils `timeout`, ripgrep, a non-root `node` user, and all task dependencies. -BenchKit wraps each runtime image with the locked Pi package and verifies that -`pi --version` is exactly 0.84.2. +and dereference symlinks. BenchKit builds each runtime locally at benchmark +start. A generic Dockerfile combines Node 24 and the recipe's version-tagged +Debian base, installs the task dependencies from the verified parent source, +removes that build copy, and installs the locked Pi package. Build commands run +with network access and version tags may drift; this is an explicitly accepted +tradeoff and does not cryptographically bind the runtime to the miner's +validation image. + +Every build uses a uniquely named `benchkit-patcheval-build-*` buildx +docker-container builder with `--no-cache --load`. Its container and private +cache volume are removed immediately after the image is loaded. The generated +task image is transient and is removed after the benchmark, including failure +paths. BenchKit cleanup is label- and exact-name-scoped and does not prune +unrelated Docker resources. + +The build context is allowlisted: it contains only the checksummed parent source +archive and BenchKit's Dockerfile, Pi package, inference proxy, and guard. It +never contains the hidden-test patch, gold source, repository history, dataset +root, or validation attestations. Agent containers retain only the internal +inference network; grader containers continue to use `--network none`. The miner must set `validated: true` only after checking, in clean containers, that the hidden test fails on the parent, passes on the original fix, and the diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py index 3fe32ed..01cffbb 100644 --- a/src/benchkit/benchmarks/patcheval.py +++ b/src/benchkit/benchmarks/patcheval.py @@ -21,6 +21,7 @@ from benchkit.sandbox import ( DockerTaskEnvironment, LatestPiImage, + PatchEvalRuntimeRecipe, SandboxError, _run, patcheval_pi_image, @@ -37,7 +38,8 @@ _MAX_PATCH_BYTES = 5 * 1024 * 1024 _MAX_CANDIDATE_FILES = 50_000 _MAX_CANDIDATE_BYTES = 512 * 1024 * 1024 -_RUNTIME_IMAGE_RE = re.compile(r"[A-Za-z0-9._/:+@-]+@sha256:[0-9a-f]{64}") +_BASE_IMAGE_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/:+-]*") +_ENVIRONMENT_KEY_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") _PYTHON_TEST_GLOBS = ( "tests/**", "test/**", @@ -60,7 +62,7 @@ class PatchEvalSpec: repository: str issue_title: str issue_body: str - runtime_image: str + runtime_recipe: PatchEvalRuntimeRecipe source_archive: Path hidden_test_patch: Path source_sha256: str @@ -103,25 +105,85 @@ def _strings(record: dict, field: str, *, required: bool = True) -> tuple[str, . return tuple(value) +def _runtime_recipe(record: dict) -> PatchEvalRuntimeRecipe: + value = record.get("runtime_recipe") + if not isinstance(value, dict): + raise ValueError("runtime_recipe must be an object") + allowed = { + "schema_version", + "base_image", + "sync_command", + "bootstrap_command", + "environment", + } + unknown = set(value).difference(allowed) + if unknown: + raise ValueError( + "runtime_recipe contains unsupported fields: " + ", ".join(sorted(unknown)) + ) + if ( + isinstance(value.get("schema_version"), bool) + or value.get("schema_version") != 1 + ): + raise ValueError("runtime_recipe schema_version must be 1") + base_image = value.get("base_image") + if not isinstance(base_image, str) or not base_image: + raise ValueError("runtime_recipe base_image must be a versioned image tag") + if ( + "@" in base_image + or any(character.isspace() for character in base_image) + or _BASE_IMAGE_RE.fullmatch(base_image) is None + or ":" not in base_image.rsplit("/", 1)[-1] + ): + raise ValueError("runtime_recipe base_image must be a versioned image tag") + tag = base_image.rsplit(":", 1)[-1] + if tag == "latest" or not any(character.isdigit() for character in tag): + raise ValueError("runtime_recipe base_image must be a versioned image tag") + try: + sync_command = _strings(value, "sync_command") + bootstrap_command = _strings(value, "bootstrap_command", required=False) + environment = _strings(value, "environment", required=False) + except ValueError as exc: + raise ValueError(f"runtime_recipe {exc}") from exc + keys: list[str] = [] + for item in environment: + key, separator, _environment_value = item.partition("=") + if ( + not separator + or _ENVIRONMENT_KEY_RE.fullmatch(key) is None + or "\n" in item + or "\r" in item + or "\0" in item + ): + raise ValueError("runtime_recipe environment entries must use KEY=value") + keys.append(key) + if len(keys) != len(set(keys)): + raise ValueError("runtime_recipe environment keys must be unique") + return PatchEvalRuntimeRecipe( + schema_version=1, + base_image=base_image, + sync_command=sync_command, + bootstrap_command=bootstrap_command, + environment=environment, + ) + + def _load_spec(root: Path, record: object) -> PatchEvalSpec: if not isinstance(record, dict): raise ValueError("each PatchEval task must be a JSON object") + if "runtime_image" in record: + raise ValueError("runtime_image is unsupported; use runtime_recipe") required_strings = ( "id", "repository", "issue_title", "issue_body", - "runtime_image", "source_sha256", "hidden_test_sha256", ) for field in required_strings: if not isinstance(record.get(field), str) or not record[field].strip(): raise ValueError(f"{field} must be a non-empty string") - if not _RUNTIME_IMAGE_RE.fullmatch(record["runtime_image"]): - raise ValueError( - "runtime_image must be a canonical OCI reference pinned by sha256 digest" - ) if record.get("validated") is not True: raise ValueError(f"PatchEval task {record['id']!r} is not miner-validated") @@ -139,6 +201,7 @@ def _load_spec(root: Path, record: object) -> PatchEvalSpec: f"{field} checksum mismatch for task {record['id']!r}: " f"expected {expected}, got {actual}" ) + _validate_source_archive(source) timeout_s = record.get("timeout_s", 300) if isinstance(timeout_s, bool) or not isinstance(timeout_s, int): @@ -151,7 +214,7 @@ def _load_spec(root: Path, record: object) -> PatchEvalSpec: repository=record["repository"], issue_title=record["issue_title"].strip(), issue_body=record["issue_body"].strip(), - runtime_image=record["runtime_image"], + runtime_recipe=_runtime_recipe(record), source_archive=source, hidden_test_patch=hidden, source_sha256=record["source_sha256"], @@ -167,20 +230,28 @@ def _load_spec(root: Path, record: object) -> PatchEvalSpec: ) +def _source_archive_members(handle: tarfile.TarFile) -> list[tarfile.TarInfo]: + members = handle.getmembers() + for member in members: + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts: + raise ValueError(f"unsafe path in source archive: {member.name!r}") + if member.issym() or member.islnk(): + raise ValueError(f"source archives must dereference links: {member.name!r}") + if not member.isdir() and not member.isreg(): + raise ValueError(f"unsupported archive entry: {member.name!r}") + return members + + +def _validate_source_archive(archive: Path) -> None: + with tarfile.open(archive, "r:*") as handle: + _source_archive_members(handle) + + def _safe_extract(archive: Path, destination: Path) -> None: """Extract a miner-produced source archive without following archive links.""" with tarfile.open(archive, "r:*") as handle: - members = handle.getmembers() - for member in members: - path = PurePosixPath(member.name) - if path.is_absolute() or ".." in path.parts: - raise ValueError(f"unsafe path in source archive: {member.name!r}") - if member.issym() or member.islnk(): - raise ValueError( - f"source archives must dereference links: {member.name!r}" - ) - if member.isdev() or member.isfifo(): - raise ValueError(f"unsupported archive entry: {member.name!r}") + members = _source_archive_members(handle) handle.extractall(destination, members=members, filter="fully_trusted") @@ -518,7 +589,12 @@ def evaluate(self, _task: Task, _response: str) -> bool: raise RuntimeError("PatchEval must be evaluated by its external grader") def pi_image_for_task(self, task: Task) -> LatestPiImage: - return patcheval_pi_image(self._spec(task).runtime_image) + spec = self._spec(task) + return patcheval_pi_image( + spec.source_archive, + spec.source_sha256, + spec.runtime_recipe, + ) def prepare_workspace(self, task: Task, environment: DockerTaskEnvironment) -> None: spec = self._spec(task) diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index 662230d..beb11d8 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -12,7 +12,7 @@ import threading import uuid from dataclasses import dataclass, field -from pathlib import Path +from pathlib import Path, PurePosixPath from urllib.parse import urlsplit, urlunsplit from benchkit.client import InferenceClient, _openai_base @@ -26,6 +26,8 @@ _PROXY_SOURCE = Path(__file__).with_name("_pi_proxy.py") _ANSWER_KEY_GUARD_SOURCE = Path(__file__).with_name("answer_key_guard.ts") _PI_PACKAGE_ROOT = Path(__file__).with_name("pi_package") +_PATCHEVAL_IMAGE_SESSION = uuid.uuid4().hex[:12] +_BUILDKIT_DRIVER_IMAGE = "moby/buildkit:buildx-stable-1" _PI_INSTALL = """\ COPY pi-package/package.json pi-package/package-lock.json /opt/benchkit/pi/ @@ -242,6 +244,9 @@ class LatestPiImage: transient: bool = True pids_limit: int = 256 build_assets: Path | None = None + build_files: tuple[tuple[Path, str], ...] = () + ephemeral_buildx: bool = False + always_cleanup_image: bool = False answer_key_guard: bool = False version: str = "" _ready: bool = field(default=False, init=False, repr=False) @@ -273,33 +278,193 @@ def prepare(self) -> str: shutil.copytree(_PI_PACKAGE_ROOT, context / "pi-package") if self.build_assets is not None: shutil.copytree(self.build_assets, context / "git-surgery") - command = [self.docker, "build", "--pull"] - if self.no_cache: - command.append("--no-cache") - command.extend(["--tag", self.image, str(context)]) - _run(command, timeout=1800) - result = _run( - [self.docker, "run", "--rm", self.image, "pi", "--version"], - timeout=30, - ) - self.version = result.stdout.strip() or "latest" - if self.version != PI_VERSION: - raise SandboxError( - f"Pi image reported {self.version!r}; expected {PI_VERSION!r}" + allowed = { + "Dockerfile", + "inference_proxy.py", + "answer_key_guard.ts", + "pi-package", + "git-surgery", + } + for source, destination in self.build_files: + target = PurePosixPath(destination) + if ( + target.is_absolute() + or len(target.parts) != 1 + or target.name in allowed + or source.is_symlink() + or not source.is_file() + ): + raise SandboxError("invalid explicit Pi build-context file") + shutil.copyfile(source, context / target.name) + allowed.add(target.name) + dockerignore = ["*", *sorted(f"!{item}" for item in allowed)] + dockerignore.extend(("!pi-package/**", "!git-surgery/**")) + (context / ".dockerignore").write_text( + "\n".join(dockerignore) + "\n", encoding="utf-8" ) + try: + if self.ephemeral_buildx: + _run( + [ + self.docker, + "image", + "rm", + "--force", + self.image, + ], + timeout=60, + check=False, + ) + _build_with_ephemeral_buildx(self.docker, self.image, context) + else: + command = [self.docker, "build", "--pull"] + if self.no_cache: + command.append("--no-cache") + command.extend(["--tag", self.image, str(context)]) + _run(command, timeout=1800) + if self.ephemeral_buildx: + smoke = f"benchkit-patcheval-smoke-{uuid.uuid4().hex[:16]}" + try: + result = _run( + [ + self.docker, + "run", + "--name", + smoke, + "--rm", + self.image, + "pi", + "--version", + ], + timeout=30, + ) + finally: + _run( + [self.docker, "rm", "--force", smoke], + timeout=30, + check=False, + ) + else: + result = _run( + [ + self.docker, + "run", + "--rm", + self.image, + "pi", + "--version", + ], + timeout=30, + ) + self.version = result.stdout.strip() or "latest" + if self.version != PI_VERSION: + raise SandboxError( + f"Pi image reported {self.version!r}; " + f"expected {PI_VERSION!r}" + ) + except Exception: + if self.transient: + _run( + [self.docker, "image", "rm", "--force", self.image], + timeout=60, + check=False, + ) + self.version = "" + raise self._ready = True return self.version def cleanup(self) -> None: """Remove the transient Pi image after its benchmark run.""" - if not self._ready or not self.transient: + if not self.transient or (not self._ready and not self.always_cleanup_image): return + if self.always_cleanup_image: + _run( + [self.docker, "image", "rm", "--force", self.image], + timeout=60, + check=False, + ) + else: + _run( + [self.docker, "image", "rm", "--force", self.image], + timeout=60, + ) + self._ready = False + self.version = "" + + +def _build_with_ephemeral_buildx(docker: str, image: str, context: Path) -> None: + """Build without shared cache, then remove the private buildkit session.""" + builder = f"benchkit-patcheval-build-{uuid.uuid4().hex[:16]}" + buildkit_container = f"buildx_buildkit_{builder}0" + cache_volume = f"buildx_buildkit_{builder}0_state" + driver_was_present = bool( _run( - [self.docker, "image", "rm", "--force", self.image], + [ + docker, + "image", + "inspect", + "--format", + "{{.Id}}", + _BUILDKIT_DRIVER_IMAGE, + ], + timeout=30, + check=False, + ).stdout.strip() + ) + try: + _run( + [ + docker, + "buildx", + "create", + "--name", + builder, + "--driver", + "docker-container", + "--driver-opt", + f"image={_BUILDKIT_DRIVER_IMAGE}", + ], timeout=60, ) - self._ready = False - self.version = "" + _run( + [ + docker, + "buildx", + "build", + "--builder", + builder, + "--pull", + "--no-cache", + "--load", + "--tag", + image, + str(context), + ], + timeout=1800, + ) + finally: + _run( + [docker, "buildx", "rm", "--force", builder], + timeout=60, + check=False, + ) + _run( + [docker, "rm", "--force", buildkit_container], + timeout=30, + check=False, + ) + _run( + [docker, "volume", "rm", "--force", cache_volume], + timeout=30, + check=False, + ) + if not driver_was_present: + _run( + [docker, "image", "rm", "--force", _BUILDKIT_DRIVER_IMAGE], + timeout=60, + check=False, + ) def aider_pi_image() -> LatestPiImage: @@ -328,33 +493,130 @@ def git_surgery_pi_image() -> LatestPiImage: ) -def patcheval_pi_image(runtime_image: str) -> LatestPiImage: - """Wrap one immutable PatchEval runtime with the pinned stock Pi agent.""" - if not re.fullmatch(r"[A-Za-z0-9._/:+@-]+@sha256:[0-9a-f]{64}", runtime_image): - raise ValueError( - "PatchEval runtime_image must be an OCI reference pinned by sha256 digest" - ) - image_id = hashlib.sha256(runtime_image.encode()).hexdigest()[:16] - dockerfile = f"""\\ -FROM {runtime_image} +@dataclass(frozen=True) +class PatchEvalRuntimeRecipe: + """Trusted build instructions shipped with one validated PatchEval task.""" + + base_image: str + sync_command: tuple[str, ...] + bootstrap_command: tuple[str, ...] = () + environment: tuple[str, ...] = () + schema_version: int = 1 + + +def _regular_file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _tree_sha256(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ValueError("Pi package build assets must not contain symlinks") + if not path.is_file(): + continue + relative = path.relative_to(root).as_posix().encode() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(bytes.fromhex(_regular_file_sha256(path))) + return digest.hexdigest() + + +def _docker_run(command: tuple[str, ...]) -> str: + return f"RUN {json.dumps(list(command), separators=(',', ':'))}\n" + + +def _patcheval_dockerfile(recipe: PatchEvalRuntimeRecipe) -> str: + bootstrap = ( + _docker_run(recipe.bootstrap_command) if recipe.bootstrap_command else "" + ) + environment = "".join( + f"ENV {key}={json.dumps(value)}\n" + for key, value in (item.split("=", 1) for item in recipe.environment) + ) + return f"""\\ +FROM node:24-bookworm-slim AS benchkit-node +FROM {recipe.base_image} USER root +COPY --from=benchkit-node /usr/local/ /usr/local/ +RUN apt-get update \\ + && apt-get install -y --no-install-recommends \\ + bash ca-certificates coreutils git passwd ripgrep tar \\ + && rm -rf /var/lib/apt/lists/* \\ + && (getent group node >/dev/null || groupadd --gid 1000 node) \\ + && (id --user node >/dev/null 2>&1 || \\ + useradd --uid 1000 --gid node --create-home --shell /bin/bash node) +ENV UV_PROJECT_ENVIRONMENT=/opt/venv UV_LINK_MODE=copy +WORKDIR /opt/project +COPY parent-source.tar /tmp/patcheval-parent.tar +RUN tar -xf /tmp/patcheval-parent.tar -C /opt/project \\ + && rm /tmp/patcheval-parent.tar +{_docker_run(recipe.sync_command)}{bootstrap}RUN rm -rf /opt/project \\ + && test -d /opt/venv \\ + && chmod -R a+rwX /opt/venv {_PI_INSTALL} COPY inference_proxy.py /opt/benchkit/inference_proxy.py COPY answer_key_guard.ts /opt/benchkit/answer_key_guard.ts RUN mkdir -p /workspace /home/node/.pi/agent \\ && chown -R node:node /workspace /home/node/.pi -USER node +{environment}USER node WORKDIR /workspace -ENV HOME=/home/node USER=node LOGNAME=node PI_OFFLINE=1 PI_TELEMETRY=0 +ENV PATH=/opt/venv/bin:$PATH HOME=/home/node USER=node LOGNAME=node \\ + PI_OFFLINE=1 PI_TELEMETRY=0 CMD ["sleep", "infinity"] """ + + +def patcheval_pi_image( + source_archive: Path, + source_sha256: str, + recipe: PatchEvalRuntimeRecipe, +) -> LatestPiImage: + """Build one task runtime locally without retaining buildkit state.""" + if source_archive.is_symlink() or not source_archive.is_file(): + raise ValueError("PatchEval source archive must be a regular file") + if not re.fullmatch(r"[0-9a-f]{64}", source_sha256): + raise ValueError("PatchEval source_sha256 must be a lowercase SHA-256") + if _regular_file_sha256(source_archive) != source_sha256: + raise ValueError("PatchEval source archive checksum mismatch") + dockerfile = _patcheval_dockerfile(recipe) + recipe_json = json.dumps( + { + "schema_version": recipe.schema_version, + "base_image": recipe.base_image, + "sync_command": list(recipe.sync_command), + "bootstrap_command": list(recipe.bootstrap_command), + "environment": list(recipe.environment), + }, + sort_keys=True, + separators=(",", ":"), + ) + identity = hashlib.sha256() + for value in ( + dockerfile, + source_sha256, + recipe_json, + _tree_sha256(_PI_PACKAGE_ROOT), + _regular_file_sha256(_PROXY_SOURCE), + _regular_file_sha256(_ANSWER_KEY_GUARD_SOURCE), + ): + identity.update(value.encode()) + identity.update(b"\0") + image_id = identity.hexdigest()[:16] return LatestPiImage( - image=f"benchkit-pi-patcheval:{image_id}", + image=f"benchkit-pi-patcheval:{_PATCHEVAL_IMAGE_SESSION}-{image_id}", dockerfile=dockerfile, - no_cache=False, - transient=False, + no_cache=True, + transient=True, + build_files=((source_archive, "parent-source.tar"),), + ephemeral_buildx=True, + always_cleanup_image=True, ) diff --git a/tests/test_patcheval.py b/tests/test_patcheval.py index 3c3db88..6b8a0b0 100644 --- a/tests/test_patcheval.py +++ b/tests/test_patcheval.py @@ -18,9 +18,21 @@ _trusted_patch, ) from benchkit.engine import Engine -from benchkit.sandbox import LatestPiImage, patcheval_pi_image +from benchkit.sandbox import ( + PI_VERSION, + LatestPiImage, + PatchEvalRuntimeRecipe, + SandboxError, + patcheval_pi_image, +) -_RUNTIME_IMAGE = "example.invalid/patcheval/python@sha256:" + "a" * 64 +_RUNTIME_RECIPE = { + "schema_version": 1, + "base_image": "ghcr.io/astral-sh/uv:0.12.1-python3.14-trixie-slim", + "sync_command": ["uv", "sync", "--frozen", "--no-install-project"], + "bootstrap_command": ["uv", "pip", "install", "setuptools==80.9.0"], + "environment": ["UV_OFFLINE=1", "PYTHONHASHSEED=0"], +} def _digest(path: Path) -> str: @@ -45,7 +57,7 @@ def _dataset(root: Path, *, validated: bool = True) -> dict: "repository": "owner/repo", "issue_title": "Handle empty input", "issue_body": "Calling parse with an empty value should return None.", - "runtime_image": _RUNTIME_IMAGE, + "runtime_recipe": _RUNTIME_RECIPE, "source_archive": archive.name, "hidden_test_patch": hidden.name, "source_sha256": _digest(archive), @@ -105,16 +117,33 @@ def test_rejects_unvalidated_tasks(self) -> None: with self.assertRaisesRegex(RuntimeError, "not miner-validated"): PatchEval(root).load_tasks() - def test_rejects_runtime_image_without_registry_digest(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - record = _dataset(root) - record["runtime_image"] = "example.invalid/patcheval/python:latest" - (root / "tasks.jsonl").write_text( - json.dumps(record) + "\n", encoding="utf-8" - ) - with self.assertRaisesRegex(RuntimeError, "runtime_image"): - PatchEval(root).load_tasks() + def test_rejects_invalid_runtime_recipes(self) -> None: + invalid_recipes = ( + {**_RUNTIME_RECIPE, "schema_version": True}, + {**_RUNTIME_RECIPE, "schema_version": 2}, + {**_RUNTIME_RECIPE, "base_image": "python"}, + {**_RUNTIME_RECIPE, "base_image": "python:latest"}, + { + **_RUNTIME_RECIPE, + "base_image": "python@sha256:" + "a" * 64, + }, + {**_RUNTIME_RECIPE, "sync_command": []}, + {**_RUNTIME_RECIPE, "environment": ["NOT-VALID"]}, + {**_RUNTIME_RECIPE, "unexpected": True}, + ) + for runtime_recipe in invalid_recipes: + with ( + self.subTest(runtime_recipe=runtime_recipe), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + record = _dataset(root) + record["runtime_recipe"] = runtime_recipe + (root / "tasks.jsonl").write_text( + json.dumps(record) + "\n", encoding="utf-8" + ) + with self.assertRaisesRegex(RuntimeError, "runtime_recipe"): + PatchEval(root).load_tasks() def test_rejects_modified_dataset_assets(self) -> None: with tempfile.TemporaryDirectory() as directory: @@ -248,7 +277,7 @@ def test_repair_prompt_is_generic_and_does_not_mention_grading(self) -> None: self.assertNotIn("verifier", prompt.lower()) self.assertNotIn("test output", prompt.lower()) - def test_engine_caches_one_runner_per_task_runtime_image(self) -> None: + def test_engine_caches_one_runner_per_locally_built_task_image(self) -> None: image = LatestPiImage( docker="docker", image="benchkit-pi-patcheval:runtime", transient=False ) @@ -264,12 +293,147 @@ def test_engine_caches_one_runner_per_task_runtime_image(self) -> None: self.assertIs(first, second) self.assertEqual(benchmark.pi_image_for_task.call_count, 2) - def test_runtime_image_requires_a_named_oci_digest(self) -> None: - with self.assertRaisesRegex(ValueError, "pinned by sha256"): - patcheval_pi_image("sha256:" + "a" * 64) + def test_runtime_image_is_local_content_addressed_and_context_is_minimal( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + spec = task.metadata["spec"] + + image = benchmark.pi_image_for_task(task) + same = benchmark.pi_image_for_task(task) + changed_recipe = PatchEvalRuntimeRecipe( + **{ + **spec.runtime_recipe.__dict__, + "environment": (*spec.runtime_recipe.environment, "TZ=UTC"), + } + ) + changed = patcheval_pi_image( + spec.source_archive, spec.source_sha256, changed_recipe + ) - image = patcheval_pi_image(_RUNTIME_IMAGE) + self.assertEqual(image.image, same.image) + self.assertNotEqual(image.image, changed.image) + self.assertTrue(image.image.startswith("benchkit-pi-patcheval:")) + self.assertEqual( + image.build_files, + ((spec.source_archive, "parent-source.tar"),), + ) + self.assertTrue(image.no_cache) + self.assertTrue(image.transient) + self.assertTrue(image.ephemeral_buildx) + self.assertTrue(image.always_cleanup_image) + self.assertIn("FROM node:24-bookworm-slim AS benchkit-node", image.dockerfile) + self.assertIn(f"FROM {_RUNTIME_RECIPE['base_image']}", image.dockerfile) + self.assertIn("COPY parent-source.tar", image.dockerfile) + self.assertIn("rm -rf /opt/project", image.dockerfile) self.assertIn("HOME=/home/node USER=node LOGNAME=node", image.dockerfile) + self.assertGreater( + image.dockerfile.index('ENV UV_OFFLINE="1"'), + image.dockerfile.index('RUN ["uv","sync"'), + ) + self.assertNotIn("hidden.patch", image.dockerfile) + self.assertNotIn("gold", image.dockerfile.lower()) + + def test_ephemeral_builder_and_image_are_removed_after_prepare(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + image = benchmark.pi_image_for_task(task) + observed_context: set[str] = set() + + def run(args, **_kwargs): + if args[1:3] == ["image", "inspect"]: + return SimpleNamespace(stdout="") + if args[1:3] == ["buildx", "build"]: + context = Path(args[-1]) + observed_context.update( + path.relative_to(context).as_posix() + for path in context.rglob("*") + if path.is_file() + ) + return SimpleNamespace(stdout=f"{PI_VERSION}\n") + + with patch("benchkit.sandbox._run", side_effect=run) as docker_run: + self.assertEqual(image.prepare(), PI_VERSION) + image.cleanup() + + commands = [call.args[0] for call in docker_run.call_args_list] + create = next( + command for command in commands if command[1:3] == ["buildx", "create"] + ) + builder = create[create.index("--name") + 1] + docker = commands[0][0] + self.assertTrue(builder.startswith("benchkit-patcheval-build-")) + self.assertIn([docker, "buildx", "rm", "--force", builder], commands) + self.assertIn( + [docker, "rm", "--force", f"buildx_buildkit_{builder}0"], commands + ) + self.assertIn( + [ + docker, + "volume", + "rm", + "--force", + f"buildx_buildkit_{builder}0_state", + ], + commands, + ) + self.assertIn( + [docker, "image", "rm", "--force", "moby/buildkit:buildx-stable-1"], + commands, + ) + self.assertEqual(commands[-1], [docker, "image", "rm", "--force", image.image]) + self.assertIn("parent-source.tar", observed_context) + self.assertIn("Dockerfile", observed_context) + self.assertFalse(any("hidden" in path for path in observed_context)) + self.assertFalse(any("gold" in path for path in observed_context)) + + def test_failed_build_removes_builder_cache_and_partial_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + image = benchmark.pi_image_for_task(benchmark.load_tasks()[0]) + + def run(args, **_kwargs): + if args[1:3] == ["buildx", "build"]: + raise SandboxError("build failed") + return SimpleNamespace(stdout=f"{PI_VERSION}\n") + + with patch("benchkit.sandbox._run", side_effect=run) as docker_run: + with self.assertRaisesRegex(SandboxError, "build failed"): + image.prepare() + image.cleanup() + + commands = [call.args[0] for call in docker_run.call_args_list] + create = next( + command for command in commands if command[1:3] == ["buildx", "create"] + ) + builder = create[create.index("--name") + 1] + docker = commands[0][0] + self.assertIn([docker, "buildx", "rm", "--force", builder], commands) + self.assertIn( + [ + docker, + "volume", + "rm", + "--force", + f"buildx_buildkit_{builder}0_state", + ], + commands, + ) + image_removals = [ + command + for command in commands + if command == [docker, "image", "rm", "--force", image.image] + ] + self.assertGreaterEqual(len(image_removals), 2) if __name__ == "__main__": From 666343b877d9195c7396080061b3070360f428c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 18:17:43 +0300 Subject: [PATCH 05/15] Clean up Pi runtimes on termination --- src/benchkit/runner.py | 22 ++++++++++++---- tests/test_pi_harness.py | 16 ++++++++++++ tests/test_runner.py | 55 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/src/benchkit/runner.py b/src/benchkit/runner.py index 979f555..03c21ef 100644 --- a/src/benchkit/runner.py +++ b/src/benchkit/runner.py @@ -727,9 +727,21 @@ def on_interrupt(signum: int, frame: FrameType | None) -> None: Text("Interrupted, cancelling active requestsโ€ฆ", style="bold yellow") ) - previous = None - with contextlib.suppress(ValueError): # not on the main thread - previous = signal.signal(signal.SIGINT, on_interrupt) + def on_terminate(signum: int, frame: FrameType | None) -> None: + # SIGHUP (for example, a dropped SSH session) and SIGTERM do not offer + # an interactive second signal. Unwind immediately so Engine.finally + # can remove its containers, networks, images, builders, and volumes. + controls.stop() + signal.signal(signum, signal.SIG_IGN) + raise SystemExit(128 + signum) + + handlers = [(signal.SIGINT, on_interrupt), (signal.SIGTERM, on_terminate)] + if hasattr(signal, "SIGHUP"): + handlers.append((signal.SIGHUP, on_terminate)) + previous: dict[int, signal.Handlers] = {} + for signum, handler in handlers: + with contextlib.suppress(ValueError): # not on the main thread + previous[signum] = signal.signal(signum, handler) try: with reporter: @@ -738,7 +750,7 @@ def on_interrupt(signum: int, frame: FrameType | None) -> None: console.print(Text("Aborted.", style="bold yellow")) return [], "interrupted" finally: - if previous is not None: - signal.signal(signal.SIGINT, previous) + for signum, handler in previous.items(): + signal.signal(signum, handler) return results, engine.failure diff --git a/tests/test_pi_harness.py b/tests/test_pi_harness.py index a5a01e6..0004545 100644 --- a/tests/test_pi_harness.py +++ b/tests/test_pi_harness.py @@ -785,6 +785,22 @@ def test_engine_cleans_the_image_after_a_failed_pi_job(self) -> None: self.assertEqual(results, []) runner.cleanup.assert_called_once_with() + def test_engine_cleans_the_image_when_process_termination_unwinds(self) -> None: + runner = Mock() + engine = Engine( + client=SimpleNamespace(), + jobs=[JobSpec("model", "sanity", "1", harness="pi")], + ) + engine._pi_runner = runner + + with ( + patch.object(engine, "_run_job", side_effect=SystemExit(143)), + self.assertRaisesRegex(SystemExit, "143"), + ): + engine.run() + + runner.cleanup.assert_called_once_with() + def test_cli_both_creates_matching_direct_and_pi_jobs(self) -> None: args = _parse_args( [ diff --git a/tests/test_runner.py b/tests/test_runner.py index 0e41c05..6013d87 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -3,7 +3,10 @@ from __future__ import annotations import io +import signal import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch from rich.console import Console @@ -18,7 +21,15 @@ TaskPhase, TaskRecord, ) -from benchkit.runner import _bar, _counters, _Glyphs, _LiveStats, _Reporter, _spread +from benchkit.runner import ( + _bar, + _counters, + _Glyphs, + _LiveStats, + _Reporter, + _spread, + run, +) def _record( @@ -265,5 +276,47 @@ def test_parallel_job_summary_keeps_parallel_throughput_metrics(self) -> None: self.assertIn("1.87x effective", output) +class HeadlessSignalTests(unittest.TestCase): + def test_sigterm_unwinds_engine_cleanup_and_restores_handlers(self) -> None: + console = _console(record=True) + installed: dict[int, object] = {} + original = object() + engine = Mock() + + def install(signum: int, handler: object) -> object: + installed[signum] = handler + return original + + def interrupt_run() -> list[dict]: + handler = installed[signal.SIGTERM] + assert callable(handler) + handler(signal.SIGTERM, None) + return [] + + engine.run.side_effect = interrupt_run + with ( + patch("benchkit.runner.Engine", return_value=engine), + patch("benchkit.runner.signal.signal", side_effect=install) as set_signal, + self.assertRaisesRegex(SystemExit, str(128 + signal.SIGTERM)), + ): + run( + SimpleNamespace(), + [JobSpec("model", "sanity", "1")], + console, + ) + + self.assertIn( + (signal.SIGTERM, signal.SIG_IGN), + [call.args for call in set_signal.call_args_list], + ) + restored = { + call.args[0]: call.args[1] + for call in set_signal.call_args_list + if call.args[1] is original + } + self.assertEqual(restored[signal.SIGINT], original) + self.assertEqual(restored[signal.SIGTERM], original) + + if __name__ == "__main__": unittest.main() From a2d22633c3f9b098b19f100375eaac5eaf090831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 18:50:04 +0300 Subject: [PATCH 06/15] Make Pi Docker builds zero-retention --- .env.example | 2 +- src/benchkit/_pi_proxy.py | 9 +- src/benchkit/runner.py | 1 + src/benchkit/sandbox.py | 168 ++++++++++++++++++++++---------------- tests/test_pi_harness.py | 129 ++++++++++++++++++++++++++--- 5 files changed, 225 insertions(+), 84 deletions(-) diff --git a/.env.example b/.env.example index 5c0888a..1f1f254 100644 --- a/.env.example +++ b/.env.example @@ -16,7 +16,7 @@ BENCHKIT_HARDWARE=RTX 3060 12GB # Hard per-task generation deadline in seconds BENCHKIT_TIMEOUT=300 -# Stock Pi harness (requires Docker). Pi itself is always installed from latest. +# Stock Pi harness (requires Docker). The Pi package version is pinned in BenchKit. # BENCHKIT_PI_TIMEOUT=900 # BENCHKIT_SANDBOX_MEMORY=2g # BENCHKIT_SANDBOX_CPUS=2 diff --git a/src/benchkit/_pi_proxy.py b/src/benchkit/_pi_proxy.py index df7055d..28cb84f 100644 --- a/src/benchkit/_pi_proxy.py +++ b/src/benchkit/_pi_proxy.py @@ -101,6 +101,13 @@ def _models_payload() -> bytes: ).encode() +def _upstream_timeout() -> float: + try: + return max(0.1, float(os.environ.get("BENCHKIT_UPSTREAM_TIMEOUT", "600"))) + except ValueError: + return 600.0 + + class ProxyHandler(BaseHTTPRequestHandler): """Stream a deliberately tiny subset of an OpenAI-compatible API.""" @@ -151,7 +158,7 @@ def _forward(self) -> None: if scheme == "https" else http.client.HTTPConnection ) - connection = connection_type(host, port, timeout=600) + connection = connection_type(host, port, timeout=_upstream_timeout()) headers = { key: value for key, value in self.headers.items() diff --git a/src/benchkit/runner.py b/src/benchkit/runner.py index 03c21ef..7edb2f8 100644 --- a/src/benchkit/runner.py +++ b/src/benchkit/runner.py @@ -720,6 +720,7 @@ def run( def on_interrupt(signum: int, frame: FrameType | None) -> None: if controls.stopped: + signal.signal(signal.SIGINT, signal.SIG_IGN) raise KeyboardInterrupt reporter.state.stopping = True controls.stop() diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index beb11d8..f59843a 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import hashlib import json import os @@ -17,9 +18,10 @@ from benchkit.client import InferenceClient, _openai_base -PI_IMAGE = "benchkit-pi:latest" -AIDER_PI_IMAGE = "benchkit-pi-aider-polyglot:latest" -GIT_SURGERY_PI_IMAGE = "benchkit-pi-git-surgery:latest" +_PI_IMAGE_SESSION = uuid.uuid4().hex[:12] +PI_IMAGE = f"benchkit-pi:{_PI_IMAGE_SESSION}" +AIDER_PI_IMAGE = f"benchkit-pi-aider-polyglot:{_PI_IMAGE_SESSION}" +GIT_SURGERY_PI_IMAGE = f"benchkit-pi-git-surgery:{_PI_IMAGE_SESSION}" PI_PACKAGE = "@earendil-works/pi-coding-agent@0.84.2" PI_VERSION = "0.84.2" AIDER_POLYGLOT_COMMIT = "7e0611e77b54e2dea774cdc0aa00cf9f7ed6144f" @@ -209,28 +211,34 @@ def _docker_upstream(url: str) -> str: def cleanup_owned_resources(docker: str, owner_id: str) -> None: """Remove every container and network belonging to one Pi runner.""" label = f"benchkit.owner={owner_id}" - containers = _run( - [docker, "ps", "--all", "--quiet", "--filter", f"label={label}"], - timeout=30, - check=False, - ).stdout.split() - if containers: - _run( - [docker, "rm", "--force", *containers], + containers: list[str] = [] + with contextlib.suppress(Exception): + containers = _run( + [docker, "ps", "--all", "--quiet", "--filter", f"label={label}"], timeout=30, check=False, - ) - networks = _run( - [docker, "network", "ls", "--quiet", "--filter", f"label={label}"], - timeout=30, - check=False, - ).stdout.split() - if networks: - _run( - [docker, "network", "rm", *networks], + ).stdout.split() + if containers: + with contextlib.suppress(Exception): + _run( + [docker, "rm", "--force", *containers], + timeout=30, + check=False, + ) + networks: list[str] = [] + with contextlib.suppress(Exception): + networks = _run( + [docker, "network", "ls", "--quiet", "--filter", f"label={label}"], timeout=30, check=False, - ) + ).stdout.split() + if networks: + with contextlib.suppress(Exception): + _run( + [docker, "network", "rm", *networks], + timeout=30, + check=False, + ) @dataclass @@ -245,8 +253,9 @@ class LatestPiImage: pids_limit: int = 256 build_assets: Path | None = None build_files: tuple[tuple[Path, str], ...] = () - ephemeral_buildx: bool = False - always_cleanup_image: bool = False + ephemeral_buildx: bool = True + always_cleanup_image: bool = True + resource_scope: str = "pi" answer_key_guard: bool = False version: str = "" _ready: bool = field(default=False, init=False, repr=False) @@ -315,7 +324,12 @@ def prepare(self) -> str: timeout=60, check=False, ) - _build_with_ephemeral_buildx(self.docker, self.image, context) + _build_with_ephemeral_buildx( + self.docker, + self.image, + context, + self.resource_scope, + ) else: command = [self.docker, "build", "--pull"] if self.no_cache: @@ -323,7 +337,10 @@ def prepare(self) -> str: command.extend(["--tag", self.image, str(context)]) _run(command, timeout=1800) if self.ephemeral_buildx: - smoke = f"benchkit-patcheval-smoke-{uuid.uuid4().hex[:16]}" + smoke = ( + f"benchkit-{self.resource_scope}-smoke-" + f"{uuid.uuid4().hex[:16]}" + ) try: result = _run( [ @@ -339,11 +356,12 @@ def prepare(self) -> str: timeout=30, ) finally: - _run( - [self.docker, "rm", "--force", smoke], - timeout=30, - check=False, - ) + with contextlib.suppress(Exception): + _run( + [self.docker, "rm", "--force", smoke], + timeout=30, + check=False, + ) else: result = _run( [ @@ -362,13 +380,20 @@ def prepare(self) -> str: f"Pi image reported {self.version!r}; " f"expected {PI_VERSION!r}" ) - except Exception: + except BaseException: if self.transient: - _run( - [self.docker, "image", "rm", "--force", self.image], - timeout=60, - check=False, - ) + with contextlib.suppress(Exception): + _run( + [ + self.docker, + "image", + "rm", + "--force", + self.image, + ], + timeout=60, + check=False, + ) self.version = "" raise self._ready = True @@ -393,9 +418,16 @@ def cleanup(self) -> None: self.version = "" -def _build_with_ephemeral_buildx(docker: str, image: str, context: Path) -> None: +def _build_with_ephemeral_buildx( + docker: str, + image: str, + context: Path, + resource_scope: str, +) -> None: """Build without shared cache, then remove the private buildkit session.""" - builder = f"benchkit-patcheval-build-{uuid.uuid4().hex[:16]}" + if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", resource_scope): + raise SandboxError("invalid Pi Docker resource scope") + builder = f"benchkit-{resource_scope}-build-{uuid.uuid4().hex[:16]}" buildkit_container = f"buildx_buildkit_{builder}0" cache_volume = f"buildx_buildkit_{builder}0_state" driver_was_present = bool( @@ -444,27 +476,21 @@ def _build_with_ephemeral_buildx(docker: str, image: str, context: Path) -> None timeout=1800, ) finally: - _run( - [docker, "buildx", "rm", "--force", builder], - timeout=60, - check=False, - ) - _run( - [docker, "rm", "--force", buildkit_container], - timeout=30, - check=False, - ) - _run( - [docker, "volume", "rm", "--force", cache_volume], - timeout=30, - check=False, + cleanup_commands = ( + ([docker, "buildx", "rm", "--force", builder], 60), + ([docker, "rm", "--force", buildkit_container], 30), + ([docker, "volume", "rm", "--force", cache_volume], 30), ) + for command, timeout in cleanup_commands: + with contextlib.suppress(Exception): + _run(command, timeout=timeout, check=False) if not driver_was_present: - _run( - [docker, "image", "rm", "--force", _BUILDKIT_DRIVER_IMAGE], - timeout=60, - check=False, - ) + with contextlib.suppress(Exception): + _run( + [docker, "image", "rm", "--force", _BUILDKIT_DRIVER_IMAGE], + timeout=60, + check=False, + ) def aider_pi_image() -> LatestPiImage: @@ -479,6 +505,7 @@ def aider_pi_image() -> LatestPiImage: # sandbox limit of 256 makes the official test suite impossible. pids_limit=2048, answer_key_guard=True, + resource_scope="aider-polyglot", ) @@ -490,6 +517,7 @@ def git_surgery_pi_image() -> LatestPiImage: no_cache=False, transient=True, build_assets=Path(__file__).with_name("git_surgery"), + resource_scope="git-surgery", ) @@ -617,6 +645,7 @@ def patcheval_pi_image( build_files=((source_archive, "parent-source.tar"),), ephemeral_buildx=True, always_cleanup_image=True, + resource_scope="patcheval", ) @@ -695,6 +724,8 @@ def start(self) -> None: f"BENCHKIT_UPSTREAM={_docker_upstream(self.client.host)}", "--env", f"BENCHKIT_MODEL={self.model}", + "--env", + f"BENCHKIT_UPSTREAM_TIMEOUT={self.client.timeout}", ] if self.client.api_key: proxy_args.extend( @@ -851,15 +882,14 @@ def download(self, source: str, destination: Path) -> None: _run([self.docker, "cp", f"{self.container_name}:{source}", str(destination)]) def stop(self) -> None: - for name in (self.container_name, self.proxy_name): - _run( - [self.docker, "rm", "--force", name], - timeout=30, - check=False, - ) - _run( - [self.docker, "network", "rm", self.network_name], - timeout=30, - check=False, - ) - self._started = False + try: + commands = [ + [self.docker, "rm", "--force", self.container_name], + [self.docker, "rm", "--force", self.proxy_name], + [self.docker, "network", "rm", self.network_name], + ] + for command in commands: + with contextlib.suppress(Exception): + _run(command, timeout=30, check=False) + finally: + self._started = False diff --git a/tests/test_pi_harness.py b/tests/test_pi_harness.py index 0004545..f751e30 100644 --- a/tests/test_pi_harness.py +++ b/tests/test_pi_harness.py @@ -11,7 +11,12 @@ from types import SimpleNamespace from unittest.mock import Mock, patch -from benchkit._pi_proxy import _capture_scaffold, _models_payload, _upstream_target +from benchkit._pi_proxy import ( + _capture_scaffold, + _models_payload, + _upstream_target, + _upstream_timeout, +) from benchkit.cli import _headless_jobs, _parse_args from benchkit.client import _openai_metrics from benchkit.engine import Engine, JobSpec, annotate_harness_effect @@ -23,6 +28,7 @@ PI_VERSION, DockerTaskEnvironment, LatestPiImage, + SandboxError, _docker_upstream, cleanup_owned_resources, ) @@ -41,29 +47,45 @@ def test_package_and_transitive_dependencies_are_pinned(self) -> None: def test_prepare_pulls_and_bypasses_build_cache_once_per_run(self) -> None: image = LatestPiImage(docker="docker") - with patch("benchkit.sandbox._run") as run: - run.return_value = SimpleNamespace(stdout=f"{PI_VERSION}\n") + def docker_run(args, **_kwargs): + if args[1:3] == ["image", "inspect"]: + return SimpleNamespace(stdout="") + if args[1] == "run": + return SimpleNamespace(stdout=f"{PI_VERSION}\n") + return SimpleNamespace(stdout="") + + with patch("benchkit.sandbox._run", side_effect=docker_run) as run: self.assertEqual(image.prepare(), PI_VERSION) self.assertEqual(image.prepare(), PI_VERSION) + image.cleanup() commands = [call.args[0] for call in run.call_args_list] self.assertEqual(commands[0][:2], ["docker", "version"]) - build = commands[1] - self.assertEqual(build[:2], ["docker", "build"]) + build = next( + command for command in commands if command[1:3] == ["buildx", "build"] + ) self.assertIn("--pull", build) self.assertIn("--no-cache", build) - self.assertEqual( - commands[2], + self.assertIn("--load", build) + create = next( + command for command in commands if command[1:3] == ["buildx", "create"] + ) + builder = create[create.index("--name") + 1] + self.assertIn(["docker", "buildx", "rm", "--force", builder], commands) + self.assertIn( [ "docker", - "run", - "--rm", - "benchkit-pi:latest", - "pi", - "--version", + "volume", + "rm", + "--force", + f"buildx_buildkit_{builder}0_state", ], + commands, + ) + self.assertIn( + ["docker", "image", "rm", "--force", image.image], + commands, ) - self.assertEqual(len(commands), 3) def test_cleanup_removes_the_transient_image(self) -> None: image = LatestPiImage( @@ -79,6 +101,7 @@ def test_cleanup_removes_the_transient_image(self) -> None: run.assert_called_once_with( ["docker", "image", "rm", "--force", "benchkit-pi:latest"], timeout=60, + check=False, ) self.assertFalse(image._ready) self.assertEqual(image.version, "") @@ -118,6 +141,69 @@ def test_cleanup_removes_every_resource_owned_by_the_runner(self) -> None: ], ) + def test_private_builder_cleanup_continues_after_one_docker_error(self) -> None: + image = LatestPiImage(docker="docker") + + def docker_run(args, **_kwargs): + if args[1:3] == ["image", "inspect"]: + return SimpleNamespace(stdout="") + if args[1:3] == ["buildx", "rm"]: + raise SandboxError("builder removal failed") + if args[1] == "run": + return SimpleNamespace(stdout=f"{PI_VERSION}\n") + return SimpleNamespace(stdout="") + + with patch("benchkit.sandbox._run", side_effect=docker_run) as run: + self.assertEqual(image.prepare(), PI_VERSION) + image.cleanup() + + commands = [call.args[0] for call in run.call_args_list] + create = next( + command for command in commands if command[1:3] == ["buildx", "create"] + ) + builder = create[create.index("--name") + 1] + self.assertIn( + ["docker", "rm", "--force", f"buildx_buildkit_{builder}0"], + commands, + ) + self.assertIn( + [ + "docker", + "volume", + "rm", + "--force", + f"buildx_buildkit_{builder}0_state", + ], + commands, + ) + + def test_environment_stop_attempts_every_owned_resource(self) -> None: + environment = DockerTaskEnvironment( + SimpleNamespace(), + "model", + LatestPiImage(docker="docker", version="test"), + docker="docker", + ) + environment._started = True + responses = [ + SandboxError("agent removal failed"), + SimpleNamespace(stdout=""), + SimpleNamespace(stdout=""), + ] + + with patch("benchkit.sandbox._run", side_effect=responses) as run: + environment.stop() + + self.assertEqual( + [call.args[0] for call in run.call_args_list], + [ + ["docker", "rm", "--force", environment.container_name], + ["docker", "rm", "--force", environment.proxy_name], + ["docker", "network", "rm", environment.network_name], + ], + ) + self.assertFalse(environment._started) + def test_local_inference_host_is_rewritten_for_docker(self) -> None: self.assertEqual( _docker_upstream("http://localhost:11434"), @@ -210,6 +296,7 @@ def test_task_container_has_no_host_mount_or_direct_egress(self) -> None: host="http://localhost:11434", api_key="real-secret", provider="openai", + timeout=60000.0, context_length=lambda _model: 32768, ) image = LatestPiImage(docker="docker", version="test") @@ -231,6 +318,7 @@ def test_task_container_has_no_host_mount_or_direct_egress(self) -> None: self.assertIn("benchkit.managed=true", command) self.assertIn(f"benchkit.owner={environment.owner_id}", command) self.assertIn("BENCHKIT_MODEL=selected/model", proxy) + self.assertIn("BENCHKIT_UPSTREAM_TIMEOUT=60000.0", proxy) self.assertIn("BENCHKIT_UPSTREAM_API_KEY=real-secret", proxy) self.assertNotIn("BENCHKIT_UPSTREAM_API_KEY=real-secret", agent) self.assertIn(environment.network_name, agent) @@ -250,6 +338,21 @@ def test_task_container_has_no_host_mount_or_direct_egress(self) -> None: ], ) + def test_proxy_uses_the_forwarded_inference_timeout(self) -> None: + with patch.dict( + os.environ, + {"BENCHKIT_UPSTREAM_TIMEOUT": "60000"}, + clear=False, + ): + self.assertEqual(_upstream_timeout(), 60000.0) + + with patch.dict( + os.environ, + {"BENCHKIT_UPSTREAM_TIMEOUT": "invalid"}, + clear=False, + ): + self.assertEqual(_upstream_timeout(), 600.0) + class InferenceProxyTests(unittest.TestCase): def test_proxy_captures_exact_system_prompt_and_available_tools(self) -> None: From dfef7c521d2150ff9d9f297d82a35600203a647b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 19:22:35 +0300 Subject: [PATCH 07/15] Document PatchEval pilot-20 dataset setup Update PatchEval docs to reflect the published immutable dataset layout (`sources/`, `hidden-tests/`, `attestations/`, `SHA256SUMS`) and add explicit Hugging Face download + checksum verification steps. Also improve the benchmark runtime error message so missing configuration points users to the `pilot-20` release and `BENCHKIT_PATCHEVAL_DATASET` root. --- docs/patcheval.md | 28 +++++++++++++++++++++++----- src/benchkit/benchmarks/patcheval.py | 5 +++-- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/patcheval.md b/docs/patcheval.md index a3bb2f0..f75a37a 100644 --- a/docs/patcheval.md +++ b/docs/patcheval.md @@ -37,9 +37,13 @@ the generic instruction to re-examine the issue and continue. ```text dataset.json tasks.jsonl -artifacts/ - /source.tar.gz - /hidden-tests.patch +SHA256SUMS +sources/ + .tar +hidden-tests/ + .patch +attestations/ + .json ``` `dataset.json` uses schema version 1 and contains `release` and `task_count`. @@ -95,5 +99,19 @@ Fix the following issue in the current repository. Inspect the code, make the ne {issue_body} ``` -Set `BENCHKIT_PATCHEVAL_DATASET` to an immutable local snapshot of the published -dataset before running `patcheval`. +Download and verify the immutable release, then point BenchKit at its root: + +```console +hf download DogukanUrker/PatchEval \ + --repo-type dataset \ + --revision pilot-20 \ + --local-dir /path/to/PatchEval +cd /path/to/PatchEval +sha256sum --check SHA256SUMS +export BENCHKIT_PATCHEVAL_DATASET=/path/to/PatchEval +``` + +The release is published at + and the `pilot-20` tag +is the immutable 20-task snapshot. Do not run against a moving branch when +recording benchmark results. diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py index 01cffbb..4bcaec6 100644 --- a/src/benchkit/benchmarks/patcheval.py +++ b/src/benchkit/benchmarks/patcheval.py @@ -521,8 +521,9 @@ def __init__(self, dataset_root: Path | None = None) -> None: def _root(self) -> Path: if self.dataset_root is None: raise RuntimeError( - "PatchEval's frozen dataset is not published yet; set " - "BENCHKIT_PATCHEVAL_DATASET to a local dataset snapshot" + "PatchEval's frozen dataset is not configured; download the " + "pilot-20 release from DogukanUrker/PatchEval and set " + "BENCHKIT_PATCHEVAL_DATASET to its local root" ) return self.dataset_root.resolve() From 11dc463a3836b0d1623c526ed69cf81e7ea65111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 19:25:40 +0300 Subject: [PATCH 08/15] Harden path and Docker cleanup checks Tightens PatchEval dataset path validation by rejecting symlink traversal and requiring resolved paths to stay under the dataset root. Also makes Pi sandbox cleanup fail loudly: Docker resource listing now captures inspection errors, cleanup verifies owned containers/networks are actually removed, and transient image/buildx teardown explicitly checks that images, builders, containers, and volumes are absent. This prevents silent cleanup failures and improves isolation guarantees. --- src/benchkit/benchmarks/patcheval.py | 13 ++- src/benchkit/sandbox.py | 127 ++++++++++++++++++++------- 2 files changed, 106 insertions(+), 34 deletions(-) diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py index 4bcaec6..9d0f11f 100644 --- a/src/benchkit/benchmarks/patcheval.py +++ b/src/benchkit/benchmarks/patcheval.py @@ -89,7 +89,18 @@ def _safe_relative(root: Path, value: object, field: str) -> Path: path = PurePosixPath(value) if path.is_absolute() or ".." in path.parts: raise ValueError(f"{field} must stay inside the dataset root") - return root.joinpath(*path.parts) + resolved_root = root.resolve(strict=True) + candidate = root + for part in path.parts: + candidate = candidate / part + if candidate.is_symlink(): + raise ValueError(f"{field} must not traverse symlinks") + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(resolved_root) + except (FileNotFoundError, RuntimeError, ValueError) as exc: + raise ValueError(f"{field} must stay inside the dataset root") from exc + return resolved def _strings(record: dict, field: str, *, required: bool = True) -> tuple[str, ...]: diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index f59843a..26307ed 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -211,34 +211,66 @@ def _docker_upstream(url: str) -> str: def cleanup_owned_resources(docker: str, owner_id: str) -> None: """Remove every container and network belonging to one Pi runner.""" label = f"benchkit.owner={owner_id}" - containers: list[str] = [] - with contextlib.suppress(Exception): - containers = _run( - [docker, "ps", "--all", "--quiet", "--filter", f"label={label}"], - timeout=30, - check=False, - ).stdout.split() + errors: list[str] = [] + + def listed(command: list[str], kind: str) -> list[str]: + try: + result = _run(command, timeout=30, check=False) + except Exception as exc: + errors.append(f"could not inspect owned {kind}: {exc}") + return [] + if result.returncode: + detail = _tail(result.stderr or result.stdout) or "unknown Docker error" + errors.append(f"could not inspect owned {kind}: {detail}") + return [] + return result.stdout.split() + + container_query = [ + docker, + "ps", + "--all", + "--quiet", + "--filter", + f"label={label}", + ] + network_query = [ + docker, + "network", + "ls", + "--quiet", + "--filter", + f"label={label}", + ] + containers = listed(container_query, "containers") if containers: with contextlib.suppress(Exception): - _run( - [docker, "rm", "--force", *containers], - timeout=30, - check=False, - ) - networks: list[str] = [] - with contextlib.suppress(Exception): - networks = _run( - [docker, "network", "ls", "--quiet", "--filter", f"label={label}"], - timeout=30, - check=False, - ).stdout.split() + _run([docker, "rm", "--force", *containers], timeout=30, check=False) + networks = listed(network_query, "networks") if networks: with contextlib.suppress(Exception): - _run( - [docker, "network", "rm", *networks], - timeout=30, - check=False, - ) + _run([docker, "network", "rm", *networks], timeout=30, check=False) + + remaining_containers = listed(container_query, "containers after cleanup") + remaining_networks = listed(network_query, "networks after cleanup") + if remaining_containers: + errors.append("owned containers remain: " + ", ".join(remaining_containers)) + if remaining_networks: + errors.append("owned networks remain: " + ", ".join(remaining_networks)) + if errors: + raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors)) + + +def _verify_absent( + command: list[str], description: str, errors: list[str], *, timeout: int = 30 +) -> None: + """Record an error unless an exact Docker resource is confirmed absent.""" + try: + result = _run(command, timeout=timeout, check=False) + except Exception as exc: + errors.append(f"could not verify {description}: {exc}") + return + if result.returncode == 0: + errors.append(f"{description} remains") @dataclass @@ -403,19 +435,24 @@ def cleanup(self) -> None: """Remove the transient Pi image after its benchmark run.""" if not self.transient or (not self._ready and not self.always_cleanup_image): return - if self.always_cleanup_image: - _run( - [self.docker, "image", "rm", "--force", self.image], - timeout=60, - check=False, - ) - else: + try: _run( [self.docker, "image", "rm", "--force", self.image], timeout=60, + check=not self.always_cleanup_image, ) - self._ready = False - self.version = "" + if self.always_cleanup_image: + errors: list[str] = [] + _verify_absent( + [self.docker, "image", "inspect", self.image], + f"transient image {self.image}", + errors, + ) + if errors: + raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors)) + finally: + self._ready = False + self.version = "" def _build_with_ephemeral_buildx( @@ -491,6 +528,30 @@ def _build_with_ephemeral_buildx( timeout=60, check=False, ) + errors: list[str] = [] + _verify_absent( + [docker, "buildx", "inspect", builder], + f"Buildx builder {builder}", + errors, + ) + _verify_absent( + [docker, "container", "inspect", buildkit_container], + f"BuildKit container {buildkit_container}", + errors, + ) + _verify_absent( + [docker, "volume", "inspect", cache_volume], + f"BuildKit volume {cache_volume}", + errors, + ) + if not driver_was_present: + _verify_absent( + [docker, "image", "inspect", _BUILDKIT_DRIVER_IMAGE], + f"BuildKit driver image {_BUILDKIT_DRIVER_IMAGE}", + errors, + ) + if errors: + raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors)) def aider_pi_image() -> LatestPiImage: From ebe1f040d0ef25a9f0b61e518cdc60400e7677d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 20:14:53 +0300 Subject: [PATCH 09/15] Update Pi cleanup tests for the stricter checks The hardened cleanup path now reads returncode on every Docker call and verifies each resource is really gone. The fake _run results had no returncode, so six tests crashed with AttributeError. Give the fakes a returncode, let inspect report a missing resource the way real Docker does, and expect the extra verification calls. --- tests/test_patcheval.py | 28 +++++++++--- tests/test_pi_harness.py | 93 +++++++++++++++++++++++++--------------- 2 files changed, 81 insertions(+), 40 deletions(-) diff --git a/tests/test_patcheval.py b/tests/test_patcheval.py index 6b8a0b0..bcf5c60 100644 --- a/tests/test_patcheval.py +++ b/tests/test_patcheval.py @@ -86,6 +86,16 @@ def download(self, _source: str, destination: Path) -> None: shutil.copytree(self.candidate, destination, dirs_exist_ok=True, symlinks=True) +def _docker_result(stdout: str = "", returncode: int = 0) -> SimpleNamespace: + """Stand in for the CompletedProcess that sandbox._run returns.""" + return SimpleNamespace(stdout=stdout, stderr="", returncode=returncode) + + +def _absent() -> SimpleNamespace: + """Docker's answer when an inspected resource does not exist.""" + return _docker_result(returncode=1) + + class PatchEvalTests(unittest.TestCase): def test_loads_only_validated_checksum_pinned_tasks(self) -> None: with tempfile.TemporaryDirectory() as directory: @@ -348,8 +358,8 @@ def test_ephemeral_builder_and_image_are_removed_after_prepare(self) -> None: observed_context: set[str] = set() def run(args, **_kwargs): - if args[1:3] == ["image", "inspect"]: - return SimpleNamespace(stdout="") + if args[2:3] == ["inspect"]: + return _absent() if args[1:3] == ["buildx", "build"]: context = Path(args[-1]) observed_context.update( @@ -357,7 +367,7 @@ def run(args, **_kwargs): for path in context.rglob("*") if path.is_file() ) - return SimpleNamespace(stdout=f"{PI_VERSION}\n") + return _docker_result(f"{PI_VERSION}\n") with patch("benchkit.sandbox._run", side_effect=run) as docker_run: self.assertEqual(image.prepare(), PI_VERSION) @@ -388,7 +398,13 @@ def run(args, **_kwargs): [docker, "image", "rm", "--force", "moby/buildkit:buildx-stable-1"], commands, ) - self.assertEqual(commands[-1], [docker, "image", "rm", "--force", image.image]) + self.assertEqual( + commands[-2:], + [ + [docker, "image", "rm", "--force", image.image], + [docker, "image", "inspect", image.image], + ], + ) self.assertIn("parent-source.tar", observed_context) self.assertIn("Dockerfile", observed_context) self.assertFalse(any("hidden" in path for path in observed_context)) @@ -404,7 +420,9 @@ def test_failed_build_removes_builder_cache_and_partial_image(self) -> None: def run(args, **_kwargs): if args[1:3] == ["buildx", "build"]: raise SandboxError("build failed") - return SimpleNamespace(stdout=f"{PI_VERSION}\n") + if args[2:3] == ["inspect"]: + return _absent() + return _docker_result(f"{PI_VERSION}\n") with patch("benchkit.sandbox._run", side_effect=run) as docker_run: with self.assertRaisesRegex(SandboxError, "build failed"): diff --git a/tests/test_pi_harness.py b/tests/test_pi_harness.py index f751e30..eeb2dc7 100644 --- a/tests/test_pi_harness.py +++ b/tests/test_pi_harness.py @@ -34,6 +34,23 @@ ) +def _docker_result(stdout: str = "", returncode: int = 0) -> SimpleNamespace: + """Stand in for the CompletedProcess that sandbox._run returns.""" + return SimpleNamespace(stdout=stdout, stderr="", returncode=returncode) + + +def _absent() -> SimpleNamespace: + """Docker's answer when an inspected resource does not exist.""" + return _docker_result(returncode=1) + + +def _docker_without_resources(args, **_kwargs) -> SimpleNamespace: + """Fake Docker where every command works and no resource is left behind.""" + if args[2:3] == ["inspect"]: + return _absent() + return _docker_result() + + class LatestPiImageTests(unittest.TestCase): def test_generic_pi_sandbox_keeps_the_restricted_pid_limit(self) -> None: self.assertEqual(LatestPiImage(docker="docker").pids_limit, 256) @@ -48,11 +65,9 @@ def test_prepare_pulls_and_bypasses_build_cache_once_per_run(self) -> None: image = LatestPiImage(docker="docker") def docker_run(args, **_kwargs): - if args[1:3] == ["image", "inspect"]: - return SimpleNamespace(stdout="") if args[1] == "run": - return SimpleNamespace(stdout=f"{PI_VERSION}\n") - return SimpleNamespace(stdout="") + return _docker_result(f"{PI_VERSION}\n") + return _docker_without_resources(args) with patch("benchkit.sandbox._run", side_effect=docker_run) as run: self.assertEqual(image.prepare(), PI_VERSION) @@ -95,49 +110,59 @@ def test_cleanup_removes_the_transient_image(self) -> None: ) image._ready = True - with patch("benchkit.sandbox._run") as run: + with patch( + "benchkit.sandbox._run", side_effect=_docker_without_resources + ) as run: image.cleanup() - run.assert_called_once_with( - ["docker", "image", "rm", "--force", "benchkit-pi:latest"], - timeout=60, - check=False, + self.assertEqual( + [call.args[0] for call in run.call_args_list], + [ + ["docker", "image", "rm", "--force", "benchkit-pi:latest"], + ["docker", "image", "inspect", "benchkit-pi:latest"], + ], ) self.assertFalse(image._ready) self.assertEqual(image.version, "") def test_cleanup_removes_every_resource_owned_by_the_runner(self) -> None: + container_query = [ + "docker", + "ps", + "--all", + "--quiet", + "--filter", + "label=benchkit.owner=owner-123", + ] + network_query = [ + "docker", + "network", + "ls", + "--quiet", + "--filter", + "label=benchkit.owner=owner-123", + ] responses = [ - SimpleNamespace(stdout="container-one\ncontainer-two\n"), - SimpleNamespace(stdout=""), - SimpleNamespace(stdout="network-one\n"), - SimpleNamespace(stdout=""), + _docker_result("container-one\ncontainer-two\n"), + _docker_result(), + _docker_result("network-one\n"), + _docker_result(), + # Cleanup lists both kinds again to prove nothing is left. + _docker_result(), + _docker_result(), ] with patch("benchkit.sandbox._run", side_effect=responses) as run: cleanup_owned_resources("docker", "owner-123") - commands = [call.args[0] for call in run.call_args_list] self.assertEqual( - commands, + [call.args[0] for call in run.call_args_list], [ - [ - "docker", - "ps", - "--all", - "--quiet", - "--filter", - "label=benchkit.owner=owner-123", - ], + container_query, ["docker", "rm", "--force", "container-one", "container-two"], - [ - "docker", - "network", - "ls", - "--quiet", - "--filter", - "label=benchkit.owner=owner-123", - ], + network_query, ["docker", "network", "rm", "network-one"], + container_query, + network_query, ], ) @@ -145,13 +170,11 @@ def test_private_builder_cleanup_continues_after_one_docker_error(self) -> None: image = LatestPiImage(docker="docker") def docker_run(args, **_kwargs): - if args[1:3] == ["image", "inspect"]: - return SimpleNamespace(stdout="") if args[1:3] == ["buildx", "rm"]: raise SandboxError("builder removal failed") if args[1] == "run": - return SimpleNamespace(stdout=f"{PI_VERSION}\n") - return SimpleNamespace(stdout="") + return _docker_result(f"{PI_VERSION}\n") + return _docker_without_resources(args) with patch("benchkit.sandbox._run", side_effect=docker_run) as run: self.assertEqual(image.prepare(), PI_VERSION) From 0b1f1eca0375213dab5337451e53e3d48aba1869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 21:00:02 +0300 Subject: [PATCH 10/15] Fix Git path quoting and Docker build flags Three review findings from the pull request: - Read the workspace diff with -z. Git quotes non-ASCII, quote, backslash, and control characters by default, and a quoted path matches neither its protected glob nor the pathspec of the second diff, so the agent's edits to such a file were dropped from the submission without a word. - Honor no_cache in the buildx path. The flag only reached the plain docker build path, so aider-polyglot and git-surgery asked for a cached build and got --no-cache anyway. The ephemeral builder starts with an empty cache either way, so this makes the field honest, not faster. - Drop the stray backslash from the generated PatchEval Dockerfile. It emitted a lone backslash line before the first FROM. Adds a regression test for the quoted-path case. --- src/benchkit/benchmarks/patcheval.py | 15 +++++++++++--- src/benchkit/sandbox.py | 26 +++++++++---------------- tests/test_patcheval.py | 29 ++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py index 9d0f11f..4476a07 100644 --- a/src/benchkit/benchmarks/patcheval.py +++ b/src/benchkit/benchmarks/patcheval.py @@ -364,9 +364,18 @@ def _trusted_patch( _remove_entry(child) _copy_candidate(candidate, worktree) _git(worktree, ["add", "--intent-to-add", "--", "."]) - changed = _git( - worktree, ["diff", "--name-only", "--no-renames", "HEAD", "--"] - ).stdout.splitlines() + # -z keeps paths verbatim. Git otherwise quotes non-ASCII, quote, + # backslash, and control characters, and a quoted path matches neither + # its protected glob nor the pathspec of the second diff, so the + # agent's real edits are silently dropped from the submission. + changed = [ + path + for path in _git( + worktree, + ["diff", "-z", "--name-only", "--no-renames", "HEAD", "--"], + ).stdout.split("\0") + if path + ] excluded = [ path for path in changed diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index 26307ed..492e4f9 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -361,6 +361,7 @@ def prepare(self) -> str: self.image, context, self.resource_scope, + no_cache=self.no_cache, ) else: command = [self.docker, "build", "--pull"] @@ -460,6 +461,8 @@ def _build_with_ephemeral_buildx( image: str, context: Path, resource_scope: str, + *, + no_cache: bool = True, ) -> None: """Build without shared cache, then remove the private buildkit session.""" if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", resource_scope): @@ -496,22 +499,11 @@ def _build_with_ephemeral_buildx( ], timeout=60, ) - _run( - [ - docker, - "buildx", - "build", - "--builder", - builder, - "--pull", - "--no-cache", - "--load", - "--tag", - image, - str(context), - ], - timeout=1800, - ) + build = [docker, "buildx", "build", "--builder", builder, "--pull"] + if no_cache: + build.append("--no-cache") + build.extend(["--load", "--tag", image, str(context)]) + _run(build, timeout=1800) finally: cleanup_commands = ( ([docker, "buildx", "rm", "--force", builder], 60), @@ -627,7 +619,7 @@ def _patcheval_dockerfile(recipe: PatchEvalRuntimeRecipe) -> str: f"ENV {key}={json.dumps(value)}\n" for key, value in (item.split("=", 1) for item in recipe.environment) ) - return f"""\\ + return f"""\ FROM node:24-bookworm-slim AS benchkit-node FROM {recipe.base_image} diff --git a/tests/test_patcheval.py b/tests/test_patcheval.py index bcf5c60..174ce3a 100644 --- a/tests/test_patcheval.py +++ b/tests/test_patcheval.py @@ -197,6 +197,35 @@ def test_trusted_patch_ignores_agent_git_and_test_changes(self) -> None: ) self.assertEqual(excluded, ["tests/test_agent.py", "tests/test_existing.py"]) + def test_protected_tests_with_unusual_names_are_still_excluded(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + task = PatchEval(root).load_tasks()[0] + spec = task.metadata["spec"] + candidate = root / "candidate" + candidate.mkdir() + (candidate / "module.py").write_text("VALUE = 2\n", encoding="utf-8") + tests = candidate / "tests" + tests.mkdir() + # Git quotes non-ASCII paths unless the diff is read with -z, and a + # quoted path matches no glob and no pathspec, so it is neither + # excluded nor actually diffed. + (tests / "test_caf\u00e9.py").write_text( + "def test_agent(): pass # AGENT_AUTHORED_MARKER\n", encoding="utf-8" + ) + + submission, changed, excluded = _trusted_patch( + spec, DownloadEnvironment(candidate) + ) + + self.assertIn("module.py", submission) + self.assertNotIn("AGENT_AUTHORED_MARKER", submission) + unusual = [path for path in excluded if path.startswith("tests/test_caf")] + self.assertEqual(len(unusual), 1) + self.assertNotIn('"', unusual[0]) + self.assertTrue(all('"' not in path for path in changed)) + def test_standard_root_python_tests_are_always_excluded(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 7f3e48cd015aad6ccb49ad9b626326047d7952fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Sun, 23 Aug 2026 21:24:59 +0300 Subject: [PATCH 11/15] cache pi and patcheval builds within one run builds used --no-cache and a throwaway buildx builder per image, so every task rebuilt the same layers and stored them separately. the isolation comes from the sandbox boundary and --network none, not from --no-cache. - one docker-container builder per run instead of per build, with cache on - pull base images once per run, not once per build - run-scoped uv cache mount for dependency installs - split the patcheval runtime into three stages: shared pi assets, the runtime recipe install, then the per-task final stage - label every image, container, network and volume with one benchkit.run label and tear them down by that label at the end of the run - removing the run builder drops its state volume and all run build cache - run the f2p and regression graders at the same time the build context allowlist is unchanged: no dataset root, hidden patch, gold source, git history, host mount or docker socket. --- docs/patcheval.md | 22 +- src/benchkit/benchmarks/patcheval.py | 50 ++-- src/benchkit/engine.py | 8 + src/benchkit/sandbox.py | 392 ++++++++++++++++++--------- tests/test_patcheval.py | 168 ++++++++---- tests/test_pi_harness.py | 136 ++++++---- 6 files changed, 507 insertions(+), 269 deletions(-) diff --git a/docs/patcheval.md b/docs/patcheval.md index f75a37a..1339044 100644 --- a/docs/patcheval.md +++ b/docs/patcheval.md @@ -60,19 +60,23 @@ Each line of `tasks.jsonl` contains: Source archives must contain repository contents at their root, exclude `.git`, and dereference symlinks. BenchKit builds each runtime locally at benchmark -start. A generic Dockerfile combines Node 24 and the recipe's version-tagged -Debian base, installs the task dependencies from the verified parent source, -removes that build copy, and installs the locked Pi package. Build commands run +start. The generated Dockerfile has three stages: shared Pi assets on top of +the recipe's version-tagged base, the runtime that installs task dependencies +from the verified parent source and then removes that build copy, and a final +stage that adds only the task environment and the agent user. Build commands run with network access and version tags may drift; this is an explicitly accepted tradeoff and does not cryptographically bind the runtime to the miner's validation image. -Every build uses a uniquely named `benchkit-patcheval-build-*` buildx -docker-container builder with `--no-cache --load`. Its container and private -cache volume are removed immediately after the image is loaded. The generated -task image is transient and is removed after the benchmark, including failure -paths. BenchKit cleanup is label- and exact-name-scoped and does not prune -unrelated Docker resources. +One `benchkit-build-*` buildx docker-container builder serves the whole run, so +every task reuses the shared Pi asset layer and one run-scoped uv cache instead +of rebuilding them. Base images are pulled once per run, not once per build. +Removing that builder at the end of the run drops its container, its private +state volume, and with them every layer and cache entry the run created. Task +images, containers, networks, and volumes all carry one run-scoped +`benchkit.run` label and are removed by that label on normal exit, error, +timeout, Ctrl+C, SIGHUP, and SIGTERM. BenchKit cleanup stays label- and +exact-name-scoped and never prunes unrelated Docker resources. The build context is allowlisted: it contains only the checksummed parent source archive and BenchKit's Dockerfile, Pi package, inference proxy, and guard. It diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py index 4476a07..770b601 100644 --- a/src/benchkit/benchmarks/patcheval.py +++ b/src/benchkit/benchmarks/patcheval.py @@ -13,6 +13,7 @@ import tarfile import tempfile import uuid +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path, PurePosixPath @@ -25,6 +26,7 @@ SandboxError, _run, patcheval_pi_image, + resource_labels, ) _SCHEMA_VERSION = 1 @@ -427,12 +429,7 @@ def _grade_once( docker = image.docker label = "f2p" if hidden_tests else "regression" container = f"benchkit-patcheval-{label}-{uuid.uuid4().hex[:12]}" - resource_labels = [ - "--label", - "benchkit.managed=true", - "--label", - f"benchkit.owner={owner_id}", - ] + labels = resource_labels(owner_id) with tempfile.TemporaryDirectory(prefix="benchkit-patcheval-grade-") as directory: patch_path = Path(directory) / "submission.patch" patch_path.write_text(patch, encoding="utf-8") @@ -444,7 +441,7 @@ def _grade_once( "--detach", "--name", container, - *resource_labels, + *labels, "--network", "none", "--cap-drop", @@ -668,22 +665,29 @@ def verify_workspace( spec = self._spec(task) try: patch, changed, excluded = _trusted_patch(spec, environment) - f2p = _grade_once( - spec, - environment.image, - patch, - spec.fail_to_pass_command, - hidden_tests=True, - owner_id=environment.owner_id, - ) - regression = _grade_once( - spec, - environment.image, - patch, - spec.regression_command, - hidden_tests=False, - owner_id=environment.owner_id, - ) + # The two graders share nothing but the read-only task image and + # the submitted patch, so they run at the same time. + with ThreadPoolExecutor(max_workers=2) as pool: + f2p_grade = pool.submit( + _grade_once, + spec, + environment.image, + patch, + spec.fail_to_pass_command, + hidden_tests=True, + owner_id=environment.owner_id, + ) + regression_grade = pool.submit( + _grade_once, + spec, + environment.image, + patch, + spec.regression_command, + hidden_tests=False, + owner_id=environment.owner_id, + ) + f2p = f2p_grade.result() + regression = regression_grade.result() except (OSError, RuntimeError, ValueError, SandboxError) as exc: return EvaluationResult( 0.0, diff --git a/src/benchkit/engine.py b/src/benchkit/engine.py index bf9ae7b..e56a457 100644 --- a/src/benchkit/engine.py +++ b/src/benchkit/engine.py @@ -33,6 +33,7 @@ from benchkit.metrics import throughput_metrics from benchkit.perturbations import annotate_robustness, perturb_task from benchkit.pi_agent import PiAgentRunner +from benchkit.sandbox import cleanup_run_resources MAX_REPAIR_ATTEMPTS = 10 @@ -892,12 +893,19 @@ def run(self) -> list[dict]: results.append(result) self._maybe_unload(index, job) finally: + used_pi = self._pi_runner is not None or bool(self._workspace_pi_runners) if self._pi_runner is not None: with contextlib.suppress(Exception): self._pi_runner.cleanup() for runner in self._workspace_pi_runners.values(): with contextlib.suppress(Exception): runner.cleanup() + if used_pi: + # Removes the run's shared builder, its build cache, and any + # image, container, network, or volume still carrying the run + # label. Scoped by label, never a global prune. + with contextlib.suppress(Exception): + cleanup_run_resources() annotate_robustness(results) annotate_harness_effect(results) diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index 492e4f9..343a216 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -31,6 +31,33 @@ _PATCHEVAL_IMAGE_SESSION = uuid.uuid4().hex[:12] _BUILDKIT_DRIVER_IMAGE = "moby/buildkit:buildx-stable-1" +# One label scopes every image, container, network, volume, and build-cache +# entry that this process creates, so cleanup can remove exactly its own +# resources instead of pruning Docker globally. +RUN_ID = uuid.uuid4().hex[:16] +RUN_LABEL = f"benchkit.run={RUN_ID}" +MANAGED_LABEL = "benchkit.managed=true" +_BUILDER = f"benchkit-build-{RUN_ID}" +_BUILDKIT_CONTAINER = f"buildx_buildkit_{_BUILDER}0" +_BUILDKIT_VOLUME = f"buildx_buildkit_{_BUILDER}0_state" +_UV_CACHE_ID = f"benchkit-uv-{RUN_ID}" +_UV_CACHE_DIR = "/root/.cache/uv" +_BUILD_LOCK = threading.Lock() +_BUILD_STATE: dict[str, object] = { + "builder": False, + "driver_was_present": True, + "pulled": set(), +} + + +def resource_labels(owner_id: str | None = None) -> list[str]: + """Return the label flags every managed Docker resource must carry.""" + labels = ["--label", MANAGED_LABEL, "--label", RUN_LABEL] + if owner_id is not None: + labels.extend(["--label", f"benchkit.owner={owner_id}"]) + return labels + + _PI_INSTALL = """\ COPY pi-package/package.json pi-package/package-lock.json /opt/benchkit/pi/ RUN cd /opt/benchkit/pi \\ @@ -280,12 +307,10 @@ class LatestPiImage: docker: str = field(default_factory=_docker_binary) image: str = PI_IMAGE dockerfile: str = PI_DOCKERFILE - no_cache: bool = True transient: bool = True pids_limit: int = 256 build_assets: Path | None = None build_files: tuple[tuple[Path, str], ...] = () - ephemeral_buildx: bool = True always_cleanup_image: bool = True resource_scope: str = "pi" answer_key_guard: bool = False @@ -344,69 +369,37 @@ def prepare(self) -> str: "\n".join(dockerignore) + "\n", encoding="utf-8" ) try: - if self.ephemeral_buildx: - _run( - [ - self.docker, - "image", - "rm", - "--force", - self.image, - ], - timeout=60, - check=False, - ) - _build_with_ephemeral_buildx( - self.docker, - self.image, - context, - self.resource_scope, - no_cache=self.no_cache, - ) - else: - command = [self.docker, "build", "--pull"] - if self.no_cache: - command.append("--no-cache") - command.extend(["--tag", self.image, str(context)]) - _run(command, timeout=1800) - if self.ephemeral_buildx: - smoke = ( - f"benchkit-{self.resource_scope}-smoke-" - f"{uuid.uuid4().hex[:16]}" - ) - try: - result = _run( - [ - self.docker, - "run", - "--name", - smoke, - "--rm", - self.image, - "pi", - "--version", - ], - timeout=30, - ) - finally: - with contextlib.suppress(Exception): - _run( - [self.docker, "rm", "--force", smoke], - timeout=30, - check=False, - ) - else: + _build_with_run_builder( + self.docker, + self.image, + self.dockerfile, + context, + ) + smoke = ( + f"benchkit-{self.resource_scope}-smoke-{uuid.uuid4().hex[:16]}" + ) + try: result = _run( [ self.docker, "run", + "--name", + smoke, "--rm", + *resource_labels(), self.image, "pi", "--version", ], timeout=30, ) + finally: + with contextlib.suppress(Exception): + _run( + [self.docker, "rm", "--force", smoke], + timeout=30, + check=False, + ) self.version = result.stdout.strip() or "latest" if self.version != PI_VERSION: raise SandboxError( @@ -456,59 +449,171 @@ def cleanup(self) -> None: self.version = "" -def _build_with_ephemeral_buildx( +def _base_image_refs(dockerfile: str) -> tuple[str, ...]: + """Return the external images a Dockerfile pulls, ignoring its own stages.""" + stages: set[str] = set() + refs: list[str] = [] + for line in dockerfile.splitlines(): + match = re.match(r"\s*FROM\s+(\S+)(?:\s+[Aa][Ss]\s+(\S+))?\s*$", line) + if match is None: + continue + reference, alias = match.group(1), match.group(2) + if reference not in stages and reference not in refs: + refs.append(reference) + if alias: + stages.add(alias) + return tuple(refs) + + +def _run_builder(docker: str) -> str: + """Create this run's single private Buildx builder on first use. + + Every build in the run shares one builder, so layers and cache mounts are + reused instead of rebuilt per task. The builder keeps its BuildKit state in + its own container and volume, which cleanup removes by exact name. That is + what keeps a run's build cache separable from the host's own cache without + ever running a global prune. + """ + with _BUILD_LOCK: + if not _BUILD_STATE["builder"]: + _BUILD_STATE["driver_was_present"] = bool( + _run( + [ + docker, + "image", + "inspect", + "--format", + "{{.Id}}", + _BUILDKIT_DRIVER_IMAGE, + ], + timeout=30, + check=False, + ).stdout.strip() + ) + _run( + [ + docker, + "buildx", + "create", + "--name", + _BUILDER, + "--driver", + "docker-container", + "--driver-opt", + f"image={_BUILDKIT_DRIVER_IMAGE}", + ], + timeout=60, + ) + _BUILD_STATE["builder"] = True + return _BUILDER + + +def _build_with_run_builder( docker: str, image: str, + dockerfile: str, context: Path, - resource_scope: str, - *, - no_cache: bool = True, ) -> None: - """Build without shared cache, then remove the private buildkit session.""" - if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", resource_scope): - raise SandboxError("invalid Pi Docker resource scope") - builder = f"benchkit-{resource_scope}-build-{uuid.uuid4().hex[:16]}" - buildkit_container = f"buildx_buildkit_{builder}0" - cache_volume = f"buildx_buildkit_{builder}0_state" - driver_was_present = bool( - _run( - [ - docker, - "image", - "inspect", - "--format", - "{{.Id}}", - _BUILDKIT_DRIVER_IMAGE, - ], - timeout=30, - check=False, - ).stdout.strip() + """Build one image on the run's shared builder, pulling bases once.""" + builder = _run_builder(docker) + with _BUILD_LOCK: + pulled = _BUILD_STATE["pulled"] + assert isinstance(pulled, set) + fresh = [ + reference + for reference in _base_image_refs(dockerfile) + if reference not in pulled + ] + pulled.update(fresh) + _run( + [docker, "image", "rm", "--force", image], + timeout=60, + check=False, ) - try: - _run( - [ - docker, - "buildx", - "create", - "--name", - builder, - "--driver", - "docker-container", - "--driver-opt", - f"image={_BUILDKIT_DRIVER_IMAGE}", - ], - timeout=60, - ) - build = [docker, "buildx", "build", "--builder", builder, "--pull"] - if no_cache: - build.append("--no-cache") - build.extend(["--load", "--tag", image, str(context)]) - _run(build, timeout=1800) - finally: + build = [docker, "buildx", "build", "--builder", builder] + if fresh: + # Recipe and harness tags are pinned, so one pull per run is enough. + build.append("--pull") + build.extend( + [ + "--label", + MANAGED_LABEL, + "--label", + RUN_LABEL, + "--load", + "--tag", + image, + str(context), + ] + ) + _run(build, timeout=1800) + + +def _remove_labelled(docker: str, errors: list[str]) -> None: + """Remove every container, network, volume, and image of this run.""" + queries = ( + ( + "containers", + [docker, "ps", "--all", "--quiet", "--filter", f"label={RUN_LABEL}"], + lambda ids: [docker, "rm", "--force", *ids], + ), + ( + "networks", + [docker, "network", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + lambda ids: [docker, "network", "rm", *ids], + ), + ( + "volumes", + [docker, "volume", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + lambda ids: [docker, "volume", "rm", "--force", *ids], + ), + ( + "images", + [docker, "image", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + lambda ids: [docker, "image", "rm", "--force", *ids], + ), + ) + for kind, query, removal in queries: + try: + listed = _run(query, timeout=30, check=False) + except Exception as exc: + errors.append(f"could not inspect run {kind}: {exc}") + continue + if listed.returncode: + detail = _tail(listed.stderr or listed.stdout) or "unknown Docker error" + errors.append(f"could not inspect run {kind}: {detail}") + continue + identifiers = sorted(set(listed.stdout.split())) + if not identifiers: + continue + with contextlib.suppress(Exception): + _run(removal(identifiers), timeout=120, check=False) + remaining = _run(query, timeout=30, check=False) + if remaining.returncode == 0 and remaining.stdout.split(): + errors.append( + f"run {kind} remain: " + + ", ".join(sorted(set(remaining.stdout.split()))) + ) + + +def cleanup_run_resources(docker: str | None = None) -> None: + """Tear down every Docker resource this run created, and nothing else.""" + docker = docker or _docker_binary() + errors: list[str] = [] + with _BUILD_LOCK: + builder_created = bool(_BUILD_STATE["builder"]) + driver_was_present = bool(_BUILD_STATE["driver_was_present"]) + _BUILD_STATE["builder"] = False + pulled = _BUILD_STATE["pulled"] + assert isinstance(pulled, set) + pulled.clear() + if builder_created: + # Removing the builder drops its BuildKit state volume, and with it + # every layer and uv cache entry the run created. cleanup_commands = ( - ([docker, "buildx", "rm", "--force", builder], 60), - ([docker, "rm", "--force", buildkit_container], 30), - ([docker, "volume", "rm", "--force", cache_volume], 30), + ([docker, "buildx", "rm", "--force", _BUILDER], 60), + ([docker, "rm", "--force", _BUILDKIT_CONTAINER], 30), + ([docker, "volume", "rm", "--force", _BUILDKIT_VOLUME], 30), ) for command, timeout in cleanup_commands: with contextlib.suppress(Exception): @@ -520,20 +625,21 @@ def _build_with_ephemeral_buildx( timeout=60, check=False, ) - errors: list[str] = [] + _remove_labelled(docker, errors) + if builder_created: _verify_absent( - [docker, "buildx", "inspect", builder], - f"Buildx builder {builder}", + [docker, "buildx", "inspect", _BUILDER], + f"Buildx builder {_BUILDER}", errors, ) _verify_absent( - [docker, "container", "inspect", buildkit_container], - f"BuildKit container {buildkit_container}", + [docker, "container", "inspect", _BUILDKIT_CONTAINER], + f"BuildKit container {_BUILDKIT_CONTAINER}", errors, ) _verify_absent( - [docker, "volume", "inspect", cache_volume], - f"BuildKit volume {cache_volume}", + [docker, "volume", "inspect", _BUILDKIT_VOLUME], + f"BuildKit volume {_BUILDKIT_VOLUME}", errors, ) if not driver_was_present: @@ -542,8 +648,8 @@ def _build_with_ephemeral_buildx( f"BuildKit driver image {_BUILDKIT_DRIVER_IMAGE}", errors, ) - if errors: - raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors)) + if errors: + raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors)) def aider_pi_image() -> LatestPiImage: @@ -551,7 +657,6 @@ def aider_pi_image() -> LatestPiImage: return LatestPiImage( image=AIDER_PI_IMAGE, dockerfile=AIDER_PI_DOCKERFILE, - no_cache=False, transient=True, # cpp/bank-account creates 1,000 simultaneous std::threads. Linux # accounts threads against Docker's PID cgroup, so the generic Pi @@ -567,7 +672,6 @@ def git_surgery_pi_image() -> LatestPiImage: return LatestPiImage( image=GIT_SURGERY_PI_IMAGE, dockerfile=GIT_SURGERY_PI_DOCKERFILE, - no_cache=False, transient=True, build_assets=Path(__file__).with_name("git_surgery"), resource_scope="git-surgery", @@ -611,9 +715,27 @@ def _docker_run(command: tuple[str, ...]) -> str: return f"RUN {json.dumps(list(command), separators=(',', ':'))}\n" -def _patcheval_dockerfile(recipe: PatchEvalRuntimeRecipe) -> str: +def _docker_run_cached(command: tuple[str, ...], uv_cache_id: str) -> str: + """Run a build command with the run's shared, private uv cache mounted.""" + mount = f"--mount=type=cache,target={_UV_CACHE_DIR},id={uv_cache_id},sharing=locked" + return f"RUN {mount} {json.dumps(list(command), separators=(',', ':'))}\n" + + +def _patcheval_dockerfile(recipe: PatchEvalRuntimeRecipe, uv_cache_id: str) -> str: + """Generate the three-stage task runtime. + + The stages are split by what actually changes between tasks: + + 1. ``benchkit-pi-assets`` depends only on the base image and the pinned Pi + assets, so the run's shared builder builds it once. + 2. ``benchkit-runtime`` adds the parent source and the frozen sync and + bootstrap commands, and reuses the run's uv cache. + 3. the final stage only adds task environment and the agent user. + """ bootstrap = ( - _docker_run(recipe.bootstrap_command) if recipe.bootstrap_command else "" + _docker_run_cached(recipe.bootstrap_command, uv_cache_id) + if recipe.bootstrap_command + else "" ) environment = "".join( f"ENV {key}={json.dumps(value)}\n" @@ -621,7 +743,8 @@ def _patcheval_dockerfile(recipe: PatchEvalRuntimeRecipe) -> str: ) return f"""\ FROM node:24-bookworm-slim AS benchkit-node -FROM {recipe.base_image} + +FROM {recipe.base_image} AS benchkit-pi-assets USER root COPY --from=benchkit-node /usr/local/ /usr/local/ @@ -632,19 +755,24 @@ def _patcheval_dockerfile(recipe: PatchEvalRuntimeRecipe) -> str: && (getent group node >/dev/null || groupadd --gid 1000 node) \\ && (id --user node >/dev/null 2>&1 || \\ useradd --uid 1000 --gid node --create-home --shell /bin/bash node) +{_PI_INSTALL} +COPY inference_proxy.py /opt/benchkit/inference_proxy.py +COPY answer_key_guard.ts /opt/benchkit/answer_key_guard.ts +RUN mkdir -p /workspace /home/node/.pi/agent \\ + && chown -R node:node /workspace /home/node/.pi + +FROM benchkit-pi-assets AS benchkit-runtime + ENV UV_PROJECT_ENVIRONMENT=/opt/venv UV_LINK_MODE=copy WORKDIR /opt/project COPY parent-source.tar /tmp/patcheval-parent.tar RUN tar -xf /tmp/patcheval-parent.tar -C /opt/project \\ && rm /tmp/patcheval-parent.tar -{_docker_run(recipe.sync_command)}{bootstrap}RUN rm -rf /opt/project \\ +{_docker_run_cached(recipe.sync_command, uv_cache_id)}{bootstrap}RUN rm -rf /opt/project \\ && test -d /opt/venv \\ && chmod -R a+rwX /opt/venv -{_PI_INSTALL} -COPY inference_proxy.py /opt/benchkit/inference_proxy.py -COPY answer_key_guard.ts /opt/benchkit/answer_key_guard.ts -RUN mkdir -p /workspace /home/node/.pi/agent \\ - && chown -R node:node /workspace /home/node/.pi + +FROM benchkit-runtime {environment}USER node WORKDIR /workspace @@ -666,7 +794,10 @@ def patcheval_pi_image( raise ValueError("PatchEval source_sha256 must be a lowercase SHA-256") if _regular_file_sha256(source_archive) != source_sha256: raise ValueError("PatchEval source archive checksum mismatch") - dockerfile = _patcheval_dockerfile(recipe) + # Hash a run-independent Dockerfile so the same task keeps one identity + # across runs even though the uv cache mount is run-scoped. + dockerfile = _patcheval_dockerfile(recipe, _UV_CACHE_ID) + identity_dockerfile = _patcheval_dockerfile(recipe, "benchkit-uv") recipe_json = json.dumps( { "schema_version": recipe.schema_version, @@ -680,7 +811,7 @@ def patcheval_pi_image( ) identity = hashlib.sha256() for value in ( - dockerfile, + identity_dockerfile, source_sha256, recipe_json, _tree_sha256(_PI_PACKAGE_ROOT), @@ -693,10 +824,8 @@ def patcheval_pi_image( return LatestPiImage( image=f"benchkit-pi-patcheval:{_PATCHEVAL_IMAGE_SESSION}-{image_id}", dockerfile=dockerfile, - no_cache=True, transient=True, build_files=((source_archive, "parent-source.tar"),), - ephemeral_buildx=True, always_cleanup_image=True, resource_scope="patcheval", ) @@ -736,19 +865,14 @@ def start(self) -> None: if not self.image.version: raise SandboxError("Pi image must be prepared before starting a task") try: - resource_labels = [ - "--label", - "benchkit.managed=true", - "--label", - f"benchkit.owner={self.owner_id}", - ] + labels = resource_labels(self.owner_id) _run( [ self.docker, "network", "create", "--internal", - *resource_labels, + *labels, self.network_name, ] ) @@ -758,7 +882,7 @@ def start(self) -> None: "--detach", "--name", self.proxy_name, - *resource_labels, + *labels, "--network", self.network_name, "--network-alias", @@ -800,7 +924,7 @@ def start(self) -> None: "--detach", "--name", self.container_name, - *resource_labels, + *labels, "--network", self.network_name, "--cap-drop", diff --git a/tests/test_patcheval.py b/tests/test_patcheval.py index 174ce3a..feeaad3 100644 --- a/tests/test_patcheval.py +++ b/tests/test_patcheval.py @@ -7,6 +7,7 @@ import shutil import tarfile import tempfile +import threading import unittest from pathlib import Path from types import SimpleNamespace @@ -15,11 +16,16 @@ from benchkit.benchmarks.patcheval import ( _GENERIC_REPAIR, PatchEval, + _grade_once, _trusted_patch, ) from benchkit.engine import Engine from benchkit.sandbox import ( + _BUILD_STATE, + _BUILDER, + _UV_CACHE_ID, PI_VERSION, + RUN_LABEL, LatestPiImage, PatchEvalRuntimeRecipe, SandboxError, @@ -96,6 +102,13 @@ def _absent() -> SimpleNamespace: return _docker_result(returncode=1) +def _reset_run_build_state() -> None: + """Forget the shared builder so each test starts from a clean run.""" + _BUILD_STATE["builder"] = False + _BUILD_STATE["driver_was_present"] = True + _BUILD_STATE["pulled"] = set() + + class PatchEvalTests(unittest.TestCase): def test_loads_only_validated_checksum_pinned_tasks(self) -> None: with tempfile.TemporaryDirectory() as directory: @@ -268,6 +281,10 @@ def test_verifier_runs_hidden_and_regression_graders_separately(self) -> None: "output": "all old tests passed", "output_truncated": False, } + + def grade(*_args, hidden_tests: bool, **_kwargs): + return hidden_failure if hidden_tests else regression_pass + with ( patch( "benchkit.benchmarks.patcheval._trusted_patch", @@ -275,17 +292,21 @@ def test_verifier_runs_hidden_and_regression_graders_separately(self) -> None: ), patch( "benchkit.benchmarks.patcheval._grade_once", - side_effect=[hidden_failure, regression_pass], - ) as grade, + side_effect=grade, + ) as grade_once, ): result = benchmark.verify_workspace(task, environment) self.assertFalse(result.passed) self.assertEqual(result.feedback, _GENERIC_REPAIR) self.assertNotIn("SECRET_EXPECTATION", result.feedback) - self.assertEqual(grade.call_count, 2) - self.assertTrue(grade.call_args_list[0].kwargs["hidden_tests"]) - self.assertFalse(grade.call_args_list[1].kwargs["hidden_tests"]) + self.assertEqual(grade_once.call_count, 2) + # The graders run at the same time, so identity comes from the + # keyword, not the call order. + self.assertEqual( + sorted(call.kwargs["hidden_tests"] for call in grade_once.call_args_list), + [False, True], + ) self.assertEqual(result.details["f2p_exit_code"], 1) self.assertEqual(result.details["regression_exit_code"], 0) @@ -356,28 +377,45 @@ def test_runtime_image_is_local_content_addressed_and_context_is_minimal( self.assertEqual(image.image, same.image) self.assertNotEqual(image.image, changed.image) + self.assertNotIn(_UV_CACHE_ID, image.image) self.assertTrue(image.image.startswith("benchkit-pi-patcheval:")) self.assertEqual( image.build_files, ((spec.source_archive, "parent-source.tar"),), ) - self.assertTrue(image.no_cache) self.assertTrue(image.transient) - self.assertTrue(image.ephemeral_buildx) self.assertTrue(image.always_cleanup_image) self.assertIn("FROM node:24-bookworm-slim AS benchkit-node", image.dockerfile) - self.assertIn(f"FROM {_RUNTIME_RECIPE['base_image']}", image.dockerfile) + self.assertIn( + f"FROM {_RUNTIME_RECIPE['base_image']} AS benchkit-pi-assets", + image.dockerfile, + ) + self.assertIn("FROM benchkit-pi-assets AS benchkit-runtime", image.dockerfile) + self.assertIn("FROM benchkit-runtime\n", image.dockerfile) self.assertIn("COPY parent-source.tar", image.dockerfile) + # Shared Pi assets come before any per-task input, so every task in a + # run reuses that layer instead of rebuilding it. + self.assertLess( + image.dockerfile.index("npm ci --omit=dev"), + image.dockerfile.index("COPY parent-source.tar"), + ) + # The dependency install reuses one run-scoped uv cache. + self.assertIn( + f"--mount=type=cache,target=/root/.cache/uv,id={_UV_CACHE_ID}," + 'sharing=locked ["uv","sync"', + image.dockerfile, + ) self.assertIn("rm -rf /opt/project", image.dockerfile) self.assertIn("HOME=/home/node USER=node LOGNAME=node", image.dockerfile) self.assertGreater( image.dockerfile.index('ENV UV_OFFLINE="1"'), - image.dockerfile.index('RUN ["uv","sync"'), + image.dockerfile.index('["uv","sync"'), ) self.assertNotIn("hidden.patch", image.dockerfile) self.assertNotIn("gold", image.dockerfile.lower()) - def test_ephemeral_builder_and_image_are_removed_after_prepare(self) -> None: + def test_task_builds_share_the_run_builder_with_a_minimal_context(self) -> None: + _reset_run_build_state() with tempfile.TemporaryDirectory() as directory: root = Path(directory) _dataset(root) @@ -403,30 +441,13 @@ def run(args, **_kwargs): image.cleanup() commands = [call.args[0] for call in docker_run.call_args_list] - create = next( - command for command in commands if command[1:3] == ["buildx", "create"] - ) - builder = create[create.index("--name") + 1] docker = commands[0][0] - self.assertTrue(builder.startswith("benchkit-patcheval-build-")) - self.assertIn([docker, "buildx", "rm", "--force", builder], commands) - self.assertIn( - [docker, "rm", "--force", f"buildx_buildkit_{builder}0"], commands - ) - self.assertIn( - [ - docker, - "volume", - "rm", - "--force", - f"buildx_buildkit_{builder}0_state", - ], - commands, - ) - self.assertIn( - [docker, "image", "rm", "--force", "moby/buildkit:buildx-stable-1"], - commands, + build = next( + command for command in commands if command[1:3] == ["buildx", "build"] ) + self.assertEqual(build[build.index("--builder") + 1], _BUILDER) + self.assertNotIn("--no-cache", build) + self.assertIn(RUN_LABEL, build) self.assertEqual( commands[-2:], [ @@ -439,7 +460,8 @@ def run(args, **_kwargs): self.assertFalse(any("hidden" in path for path in observed_context)) self.assertFalse(any("gold" in path for path in observed_context)) - def test_failed_build_removes_builder_cache_and_partial_image(self) -> None: + def test_failed_build_removes_the_partial_image(self) -> None: + _reset_run_build_state() with tempfile.TemporaryDirectory() as directory: root = Path(directory) _dataset(root) @@ -459,28 +481,78 @@ def run(args, **_kwargs): image.cleanup() commands = [call.args[0] for call in docker_run.call_args_list] - create = next( - command for command in commands if command[1:3] == ["buildx", "create"] - ) - builder = create[create.index("--name") + 1] docker = commands[0][0] - self.assertIn([docker, "buildx", "rm", "--force", builder], commands) - self.assertIn( - [ - docker, - "volume", - "rm", - "--force", - f"buildx_buildkit_{builder}0_state", - ], - commands, - ) image_removals = [ command for command in commands if command == [docker, "image", "rm", "--force", image.image] ] self.assertGreaterEqual(len(image_removals), 2) + self.assertFalse(any("prune" in command for command in commands)) + + def test_graders_run_concurrently_on_the_same_task_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + image = LatestPiImage( + docker="docker", image="benchkit-pi-patcheval:test", version="0.84.2" + ) + environment = SimpleNamespace( + image=image, owner_id="owner-123", workdir="/workspace/repo" + ) + started = threading.Barrier(2, timeout=5) + + def grade(_spec, _image, _patch, command, **_kwargs): + # Both graders must be in flight at the same time. + started.wait() + return {"exit_code": 0, "output": "", "output_truncated": False} + + with ( + patch( + "benchkit.benchmarks.patcheval._trusted_patch", + return_value=("patch", ["module.py"], []), + ), + patch( + "benchkit.benchmarks.patcheval._grade_once", side_effect=grade + ) as grade_once, + ): + result = benchmark.verify_workspace(task, environment) + + self.assertEqual(result.score, 1.0) + self.assertEqual(grade_once.call_count, 2) + + def test_grader_containers_carry_the_run_label(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + spec = task.metadata["spec"] + image = LatestPiImage( + docker="docker", image="benchkit-pi-patcheval:test", version="0.84.2" + ) + + with patch( + "benchkit.benchmarks.patcheval._run", + return_value=_docker_result(), + ) as run: + _grade_once( + spec, + image, + "patch", + spec.regression_command, + hidden_tests=False, + owner_id="owner-123", + ) + + create = next( + call.args[0] for call in run.call_args_list if call.args[0][1] == "run" + ) + self.assertIn(RUN_LABEL, create) + self.assertIn("benchkit.owner=owner-123", create) + self.assertIn("none", create) if __name__ == "__main__": diff --git a/tests/test_pi_harness.py b/tests/test_pi_harness.py index eeb2dc7..0c37ed1 100644 --- a/tests/test_pi_harness.py +++ b/tests/test_pi_harness.py @@ -23,14 +23,20 @@ from benchkit.evaluation import EvaluationResult from benchkit.pi_agent import PiAgentRunner, _RpcTrace from benchkit.sandbox import ( + _BUILD_STATE, + _BUILDER, + _BUILDKIT_CONTAINER, + _BUILDKIT_VOLUME, PI_DOCKERFILE, PI_PACKAGE, PI_VERSION, + RUN_LABEL, DockerTaskEnvironment, LatestPiImage, SandboxError, _docker_upstream, cleanup_owned_resources, + cleanup_run_resources, ) @@ -44,6 +50,13 @@ def _absent() -> SimpleNamespace: return _docker_result(returncode=1) +def _reset_run_build_state() -> None: + """Forget the shared builder so each test starts from a clean run.""" + _BUILD_STATE["builder"] = False + _BUILD_STATE["driver_was_present"] = True + _BUILD_STATE["pulled"] = set() + + def _docker_without_resources(args, **_kwargs) -> SimpleNamespace: """Fake Docker where every command works and no resource is left behind.""" if args[2:3] == ["inspect"]: @@ -61,8 +74,10 @@ def test_package_and_transitive_dependencies_are_pinned(self) -> None: self.assertIn("npm ci --omit=dev", PI_DOCKERFILE) self.assertNotIn("@latest", PI_DOCKERFILE) - def test_prepare_pulls_and_bypasses_build_cache_once_per_run(self) -> None: - image = LatestPiImage(docker="docker") + def test_run_shares_one_cached_builder_and_pulls_bases_once(self) -> None: + _reset_run_build_state() + first = LatestPiImage(docker="docker", image="benchkit-pi:first") + second = LatestPiImage(docker="docker", image="benchkit-pi:second") def docker_run(args, **_kwargs): if args[1] == "run": @@ -70,37 +85,58 @@ def docker_run(args, **_kwargs): return _docker_without_resources(args) with patch("benchkit.sandbox._run", side_effect=docker_run) as run: - self.assertEqual(image.prepare(), PI_VERSION) - self.assertEqual(image.prepare(), PI_VERSION) - image.cleanup() + self.assertEqual(first.prepare(), PI_VERSION) + self.assertEqual(first.prepare(), PI_VERSION) + self.assertEqual(second.prepare(), PI_VERSION) commands = [call.args[0] for call in run.call_args_list] - self.assertEqual(commands[0][:2], ["docker", "version"]) - build = next( - command for command in commands if command[1:3] == ["buildx", "build"] - ) - self.assertIn("--pull", build) - self.assertIn("--no-cache", build) - self.assertIn("--load", build) - create = next( + creates = [ command for command in commands if command[1:3] == ["buildx", "create"] - ) - builder = create[create.index("--name") + 1] - self.assertIn(["docker", "buildx", "rm", "--force", builder], commands) - self.assertIn( - [ - "docker", - "volume", - "rm", - "--force", - f"buildx_buildkit_{builder}0_state", - ], - commands, - ) - self.assertIn( - ["docker", "image", "rm", "--force", image.image], - commands, - ) + ] + builds = [ + command for command in commands if command[1:3] == ["buildx", "build"] + ] + self.assertEqual(len(creates), 1) + self.assertEqual(len(builds), 2) + self.assertEqual(creates[0][creates[0].index("--name") + 1], _BUILDER) + for build in builds: + self.assertEqual(build[build.index("--builder") + 1], _BUILDER) + self.assertNotIn("--no-cache", build) + self.assertIn(RUN_LABEL, build) + # The base image is pulled for the first build of the run only. + self.assertIn("--pull", builds[0]) + self.assertNotIn("--pull", builds[1]) + + def test_run_cleanup_removes_labelled_resources_and_the_builder(self) -> None: + _reset_run_build_state() + image = LatestPiImage(docker="docker") + + def docker_run(args, **_kwargs): + if args[1] == "run": + return _docker_result(f"{PI_VERSION}\n") + return _docker_without_resources(args) + + with patch("benchkit.sandbox._run", side_effect=docker_run): + image.prepare() + + with patch( + "benchkit.sandbox._run", side_effect=_docker_without_resources + ) as run: + cleanup_run_resources("docker") + + commands = [call.args[0] for call in run.call_args_list] + self.assertIn(["docker", "buildx", "rm", "--force", _BUILDER], commands) + self.assertIn(["docker", "rm", "--force", _BUILDKIT_CONTAINER], commands) + self.assertIn(["docker", "volume", "rm", "--force", _BUILDKIT_VOLUME], commands) + for query in ( + ["docker", "ps", "--all", "--quiet", "--filter", f"label={RUN_LABEL}"], + ["docker", "network", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + ["docker", "volume", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + ["docker", "image", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + ): + self.assertIn(query, commands) + # Cleanup must never reach for a global prune. + self.assertFalse(any("prune" in command for command in commands)) def test_cleanup_removes_the_transient_image(self) -> None: image = LatestPiImage( @@ -166,39 +202,29 @@ def test_cleanup_removes_every_resource_owned_by_the_runner(self) -> None: ], ) - def test_private_builder_cleanup_continues_after_one_docker_error(self) -> None: + def test_run_cleanup_continues_after_one_docker_error(self) -> None: + _reset_run_build_state() image = LatestPiImage(docker="docker") - def docker_run(args, **_kwargs): - if args[1:3] == ["buildx", "rm"]: - raise SandboxError("builder removal failed") + def prepare_docker(args, **_kwargs): if args[1] == "run": return _docker_result(f"{PI_VERSION}\n") return _docker_without_resources(args) - with patch("benchkit.sandbox._run", side_effect=docker_run) as run: - self.assertEqual(image.prepare(), PI_VERSION) - image.cleanup() + with patch("benchkit.sandbox._run", side_effect=prepare_docker): + image.prepare() + + def cleanup_docker(args, **_kwargs): + if args[1:3] == ["buildx", "rm"]: + raise SandboxError("builder removal failed") + return _docker_without_resources(args) + + with patch("benchkit.sandbox._run", side_effect=cleanup_docker) as run: + cleanup_run_resources("docker") commands = [call.args[0] for call in run.call_args_list] - create = next( - command for command in commands if command[1:3] == ["buildx", "create"] - ) - builder = create[create.index("--name") + 1] - self.assertIn( - ["docker", "rm", "--force", f"buildx_buildkit_{builder}0"], - commands, - ) - self.assertIn( - [ - "docker", - "volume", - "rm", - "--force", - f"buildx_buildkit_{builder}0_state", - ], - commands, - ) + self.assertIn(["docker", "rm", "--force", _BUILDKIT_CONTAINER], commands) + self.assertIn(["docker", "volume", "rm", "--force", _BUILDKIT_VOLUME], commands) def test_environment_stop_attempts_every_owned_resource(self) -> None: environment = DockerTaskEnvironment( From a0890742935df95e8a6247dfa0c07100a8853a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Mon, 24 Aug 2026 10:31:28 +0300 Subject: [PATCH 12/15] Normalize sandbox source file mtimes Update the Docker build step in `sandbox.py` to set a fixed timestamp on extracted project files (`1980-01-01`) right after unpacking `parent-source.tar`. This makes file mtimes deterministic, reducing cache churn and improving reproducibility for sandbox image builds. --- src/benchkit/sandbox.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index 343a216..fd2579b 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -767,6 +767,7 @@ def _patcheval_dockerfile(recipe: PatchEvalRuntimeRecipe, uv_cache_id: str) -> s WORKDIR /opt/project COPY parent-source.tar /tmp/patcheval-parent.tar RUN tar -xf /tmp/patcheval-parent.tar -C /opt/project \\ + && find /opt/project -exec touch -t 198001010000 {{}} + \\ && rm /tmp/patcheval-parent.tar {_docker_run_cached(recipe.sync_command, uv_cache_id)}{bootstrap}RUN rm -rf /opt/project \\ && test -d /opt/venv \\ From 9ccceb7b503b40312138b0615ed8b3d5a1ac0252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Mon, 24 Aug 2026 10:39:53 +0300 Subject: [PATCH 13/15] Update patcheval.py --- src/benchkit/benchmarks/patcheval.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py index 770b601..96ba736 100644 --- a/src/benchkit/benchmarks/patcheval.py +++ b/src/benchkit/benchmarks/patcheval.py @@ -622,6 +622,18 @@ def prepare_workspace(self, task: Task, environment: DockerTaskEnvironment) -> N environment.exec( ["tar", "-xf", "/tmp/patch-eval-source", "-C", environment.workdir] ) + environment.exec( + [ + "find", + environment.workdir, + "-exec", + "touch", + "-t", + "198001010000", + "{}", + "+", + ] + ) environment.exec(["rm", "-rf", f"{environment.workdir}/.git"]) if spec.setup_command: environment.exec( From 52eeafe82bff7f573ffeca6d0a88a346d6f008f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Mon, 24 Aug 2026 11:29:39 +0300 Subject: [PATCH 14/15] Normalize Patcheval repo file mtimes After extracting the source tarball in `_grade_once`, this adds a `find ... touch -t 198001010000` step inside the container to reset timestamps across `/workspace/repo`. This makes grading runs more deterministic and avoids mtime-driven behavior differences from archived file metadata. --- src/benchkit/benchmarks/patcheval.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py index 96ba736..1b3ba6f 100644 --- a/src/benchkit/benchmarks/patcheval.py +++ b/src/benchkit/benchmarks/patcheval.py @@ -477,6 +477,20 @@ def _grade_once( container, ["tar", "-xf", "/tmp/source", "-C", "/workspace/repo"], ) + _docker_exec( + docker, + container, + [ + "find", + "/workspace/repo", + "-exec", + "touch", + "-t", + "198001010000", + "{}", + "+", + ], + ) if patch: _docker_exec( docker, From b024fcf900ca0fabb075deaf93d3da54139176ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fukan=20=C3=9Crker?= Date: Mon, 24 Aug 2026 12:13:24 +0300 Subject: [PATCH 15/15] Handle subprocess output decoding errors Add errors="replace" to subprocess.run calls in src/benchkit/benchmarks/patcheval.py (patcheval._git) and src/benchkit/sandbox.py (sandbox._run). This makes decoding of subprocess output tolerant of invalid byte sequences (replacing them) and prevents UnicodeDecodeError crashes when external commands emit malformed or binary data. No other behavior changes. --- src/benchkit/benchmarks/patcheval.py | 1 + src/benchkit/sandbox.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py index 1b3ba6f..5b610d6 100644 --- a/src/benchkit/benchmarks/patcheval.py +++ b/src/benchkit/benchmarks/patcheval.py @@ -279,6 +279,7 @@ def _git( completed = subprocess.run( ["git", "-C", str(worktree), *args], text=True, + errors="replace", capture_output=True, env=environment, timeout=60, diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index fd2579b..d0b2adf 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -210,6 +210,7 @@ def _run( args, input=input_text, text=True, + errors="replace", capture_output=True, timeout=timeout, )