diff --git a/.env.example b/.env.example index 5c0888a..1f1f254 100644 --- a/.env.example +++ b/.env.example @@ -16,7 +16,7 @@ BENCHKIT_HARDWARE=RTX 3060 12GB # Hard per-task generation deadline in seconds BENCHKIT_TIMEOUT=300 -# Stock Pi harness (requires Docker). Pi itself is always installed from latest. +# Stock Pi harness (requires Docker). The Pi package version is pinned in BenchKit. # BENCHKIT_PI_TIMEOUT=900 # BENCHKIT_SANDBOX_MEMORY=2g # BENCHKIT_SANDBOX_CPUS=2 diff --git a/docs/patcheval.md b/docs/patcheval.md new file mode 100644 index 0000000..1339044 --- /dev/null +++ b/docs/patcheval.md @@ -0,0 +1,121 @@ +# PatchEval runner contract + +PatchEval is split deliberately. The one-shot miner and frozen corpus live in a +separate Hugging Face dataset repository. BenchKit contains only the runner, +prompt, sandbox boundary, and deterministic grader. + +## Threat model + +The Pi agent receives a parent-commit source archive in a fresh `/workspace`. +It has no host mount, Docker socket, GitHub network access, real Git history, +gold patch, Hugging Face cache, or grader assets. Its only network route is the +restricted inference proxy already used by BenchKit's Pi harness. + +After each attempt, BenchKit copies the workspace to the trusted host and diffs +it against a fresh extraction of the checksummed source archive. Agent Git +metadata is ignored. Changes matching `protected_globs` or `ignored_globs` are +left out of the submitted patch. The runner always protects conventional Python +test paths (`tests/`, `test/`, `test_*.py`, `*_test.py`, and `conftest.py`); +dataset `protected_globs` must cover any repository-specific test locations. +An agent may therefore write tests for itself without making those tests part +of scoring. + +BenchKit then starts two new containers with `--network none`: + +1. The fail-to-pass grader applies the submitted patch and the hidden test + patch, then runs `fail_to_pass_command`. +2. The regression grader applies only the submitted patch, then runs + `regression_command` against the parent commit's previously passing tests. + +The task passes only when both exit codes are zero. A grader setup failure is a +harness error, not an incorrect answer. Raw output remains in the report for +benchmark maintainers but is never sent to the model. Repair turns receive only +the generic instruction to re-examine the issue and continue. + +## Frozen dataset layout + +```text +dataset.json +tasks.jsonl +SHA256SUMS +sources/ + .tar +hidden-tests/ + .patch +attestations/ + .json +``` + +`dataset.json` uses schema version 1 and contains `release` and `task_count`. +Each line of `tasks.jsonl` contains: + +- `id`, `repository`, `issue_title`, and reviewed `issue_body` +- `runtime_recipe`, containing schema version 1, an explicitly version-tagged + Debian-compatible `base_image`, required `sync_command`, and optional + `bootstrap_command` and `environment` +- relative `source_archive` and `hidden_test_patch` paths plus SHA-256 hashes +- argv arrays for `setup_command`, `fail_to_pass_command`, and + `regression_command` +- `protected_globs`, `ignored_globs`, `timeout_s`, and `validated: true` + +Source archives must contain repository contents at their root, exclude `.git`, +and dereference symlinks. BenchKit builds each runtime locally at benchmark +start. The generated Dockerfile has three stages: shared Pi assets on top of +the recipe's version-tagged base, the runtime that installs task dependencies +from the verified parent source and then removes that build copy, and a final +stage that adds only the task environment and the agent user. Build commands run +with network access and version tags may drift; this is an explicitly accepted +tradeoff and does not cryptographically bind the runtime to the miner's +validation image. + +One `benchkit-build-*` buildx docker-container builder serves the whole run, so +every task reuses the shared Pi asset layer and one run-scoped uv cache instead +of rebuilding them. Base images are pulled once per run, not once per build. +Removing that builder at the end of the run drops its container, its private +state volume, and with them every layer and cache entry the run created. Task +images, containers, networks, and volumes all carry one run-scoped +`benchkit.run` label and are removed by that label on normal exit, error, +timeout, Ctrl+C, SIGHUP, and SIGTERM. BenchKit cleanup stays label- and +exact-name-scoped and never prunes unrelated Docker resources. + +The build context is allowlisted: it contains only the checksummed parent source +archive and BenchKit's Dockerfile, Pi package, inference proxy, and guard. It +never contains the hidden-test patch, gold source, repository history, dataset +root, or validation attestations. Agent containers retain only the internal +inference network; grader containers continue to use `--network none`. + +The miner must set `validated: true` only after checking, in clean containers, +that the hidden test fails on the parent, passes on the original fix, and the +full regression command passes on both. A frozen release never changes in +place; later date windows receive a new release name. + +## Model prompt + +The runner sends the reviewed issue title and full reviewed body, without issue +comments, pull-request text, labels, URLs, commit identifiers, or benchmark +instructions: + +```text +Fix the following issue in the current repository. Inspect the code, make the necessary changes, and verify your solution. + +# {issue_title} + +{issue_body} +``` + +Download and verify the immutable release, then point BenchKit at its root: + +```console +hf download DogukanUrker/PatchEval \ + --repo-type dataset \ + --revision pilot-20 \ + --local-dir /path/to/PatchEval +cd /path/to/PatchEval +sha256sum --check SHA256SUMS +export BENCHKIT_PATCHEVAL_DATASET=/path/to/PatchEval +``` + +The release is published at + and the `pilot-20` tag +is the immutable 20-task snapshot. Do not run against a moving branch when +recording benchmark results. diff --git a/pyproject.toml b/pyproject.toml index 292734c..d08a0a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ benchkit = [ "datasets/*.txt", "git_surgery/**/*.sh", "git_surgery/**/*.md", + "pi_package/*.json", "templates/*.html", "tui/*.tcss", ] diff --git a/src/benchkit/_pi_proxy.py b/src/benchkit/_pi_proxy.py index df7055d..28cb84f 100644 --- a/src/benchkit/_pi_proxy.py +++ b/src/benchkit/_pi_proxy.py @@ -101,6 +101,13 @@ def _models_payload() -> bytes: ).encode() +def _upstream_timeout() -> float: + try: + return max(0.1, float(os.environ.get("BENCHKIT_UPSTREAM_TIMEOUT", "600"))) + except ValueError: + return 600.0 + + class ProxyHandler(BaseHTTPRequestHandler): """Stream a deliberately tiny subset of an OpenAI-compatible API.""" @@ -151,7 +158,7 @@ def _forward(self) -> None: if scheme == "https" else http.client.HTTPConnection ) - connection = connection_type(host, port, timeout=600) + connection = connection_type(host, port, timeout=_upstream_timeout()) headers = { key: value for key, value in self.headers.items() diff --git a/src/benchkit/benchmarks/__init__.py b/src/benchkit/benchmarks/__init__.py index e672bba..b8dd88c 100644 --- a/src/benchkit/benchmarks/__init__.py +++ b/src/benchkit/benchmarks/__init__.py @@ -14,6 +14,7 @@ from benchkit.benchmarks.mmlu import MMLU from benchkit.benchmarks.mmlu_pro import MMLUPro from benchkit.benchmarks.openbookqa import OpenBookQA +from benchkit.benchmarks.patcheval import PatchEval from benchkit.benchmarks.piqa import PIQA from benchkit.benchmarks.ruler import RULER, RULERFull from benchkit.benchmarks.sanity import Sanity @@ -25,6 +26,7 @@ # Generative suites first, then the multiple-choice ones. "aider-polyglot": AiderPolyglot, "git-surgery": GitSurgery, + "patcheval": PatchEval, "sanity": Sanity, "humaneval": HumanEval, "humaneval-plus": HumanEvalPlus, @@ -51,6 +53,7 @@ DESCRIPTIONS: dict[str, str] = { "aider-polyglot": "repository editing across six languages with the Pi agent", "git-surgery": "stateful Git operations in isolated repositories with Pi", + "patcheval": "real Python bug fixes with externally isolated hidden tests", "sanity": "25 curated checks across code, math, instructions, science, and commonsense", "humaneval": "Python function completions with the original unit tests", "humaneval-plus": "HumanEval with 122k+ tougher EvalPlus test inputs", diff --git a/src/benchkit/benchmarks/patcheval.py b/src/benchkit/benchmarks/patcheval.py new file mode 100644 index 0000000..5b610d6 --- /dev/null +++ b/src/benchkit/benchmarks/patcheval.py @@ -0,0 +1,740 @@ +"""Hermetic bug-fix tasks scored by hidden tests outside the agent sandbox.""" + +from __future__ import annotations + +import fnmatch +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import tarfile +import tempfile +import uuid +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path, PurePosixPath + +from benchkit.benchmarks.base import Task +from benchkit.evaluation import EvaluationResult +from benchkit.sandbox import ( + DockerTaskEnvironment, + LatestPiImage, + PatchEvalRuntimeRecipe, + SandboxError, + _run, + patcheval_pi_image, + resource_labels, +) + +_SCHEMA_VERSION = 1 +_TASKS_FILE = "tasks.jsonl" +_RELEASE_FILE = "dataset.json" +_GENERIC_REPAIR = ( + "The issue is not fully resolved yet. Re-examine your changes and continue " + "working on the fix. Use the repository and any tests you can run to validate " + "your next attempt." +) +_MAX_PATCH_BYTES = 5 * 1024 * 1024 +_MAX_CANDIDATE_FILES = 50_000 +_MAX_CANDIDATE_BYTES = 512 * 1024 * 1024 +_BASE_IMAGE_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/:+-]*") +_ENVIRONMENT_KEY_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +_PYTHON_TEST_GLOBS = ( + "tests/**", + "test/**", + "**/tests/**", + "**/test/**", + "test_*.py", + "*_test.py", + "**/test_*.py", + "**/*_test.py", + "conftest.py", + "**/conftest.py", +) + + +@dataclass(frozen=True) +class PatchEvalSpec: + """One frozen PatchEval task and its host-only grading assets.""" + + id: str + repository: str + issue_title: str + issue_body: str + runtime_recipe: PatchEvalRuntimeRecipe + source_archive: Path + hidden_test_patch: Path + source_sha256: str + hidden_test_sha256: str + fail_to_pass_command: tuple[str, ...] + regression_command: tuple[str, ...] + setup_command: tuple[str, ...] + protected_globs: tuple[str, ...] + ignored_globs: tuple[str, ...] + timeout_s: int + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _safe_relative(root: Path, value: object, field: str) -> Path: + if not isinstance(value, str) or not value: + raise ValueError(f"{field} must be a non-empty relative path") + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts: + raise ValueError(f"{field} must stay inside the dataset root") + resolved_root = root.resolve(strict=True) + candidate = root + for part in path.parts: + candidate = candidate / part + if candidate.is_symlink(): + raise ValueError(f"{field} must not traverse symlinks") + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(resolved_root) + except (FileNotFoundError, RuntimeError, ValueError) as exc: + raise ValueError(f"{field} must stay inside the dataset root") from exc + return resolved + + +def _strings(record: dict, field: str, *, required: bool = True) -> tuple[str, ...]: + value = record.get(field) + if value is None and not required: + return () + if ( + not isinstance(value, list) + or not value + or not all(isinstance(item, str) and item for item in value) + ): + raise ValueError(f"{field} must be a non-empty list of strings") + return tuple(value) + + +def _runtime_recipe(record: dict) -> PatchEvalRuntimeRecipe: + value = record.get("runtime_recipe") + if not isinstance(value, dict): + raise ValueError("runtime_recipe must be an object") + allowed = { + "schema_version", + "base_image", + "sync_command", + "bootstrap_command", + "environment", + } + unknown = set(value).difference(allowed) + if unknown: + raise ValueError( + "runtime_recipe contains unsupported fields: " + ", ".join(sorted(unknown)) + ) + if ( + isinstance(value.get("schema_version"), bool) + or value.get("schema_version") != 1 + ): + raise ValueError("runtime_recipe schema_version must be 1") + base_image = value.get("base_image") + if not isinstance(base_image, str) or not base_image: + raise ValueError("runtime_recipe base_image must be a versioned image tag") + if ( + "@" in base_image + or any(character.isspace() for character in base_image) + or _BASE_IMAGE_RE.fullmatch(base_image) is None + or ":" not in base_image.rsplit("/", 1)[-1] + ): + raise ValueError("runtime_recipe base_image must be a versioned image tag") + tag = base_image.rsplit(":", 1)[-1] + if tag == "latest" or not any(character.isdigit() for character in tag): + raise ValueError("runtime_recipe base_image must be a versioned image tag") + try: + sync_command = _strings(value, "sync_command") + bootstrap_command = _strings(value, "bootstrap_command", required=False) + environment = _strings(value, "environment", required=False) + except ValueError as exc: + raise ValueError(f"runtime_recipe {exc}") from exc + keys: list[str] = [] + for item in environment: + key, separator, _environment_value = item.partition("=") + if ( + not separator + or _ENVIRONMENT_KEY_RE.fullmatch(key) is None + or "\n" in item + or "\r" in item + or "\0" in item + ): + raise ValueError("runtime_recipe environment entries must use KEY=value") + keys.append(key) + if len(keys) != len(set(keys)): + raise ValueError("runtime_recipe environment keys must be unique") + return PatchEvalRuntimeRecipe( + schema_version=1, + base_image=base_image, + sync_command=sync_command, + bootstrap_command=bootstrap_command, + environment=environment, + ) + + +def _load_spec(root: Path, record: object) -> PatchEvalSpec: + if not isinstance(record, dict): + raise ValueError("each PatchEval task must be a JSON object") + if "runtime_image" in record: + raise ValueError("runtime_image is unsupported; use runtime_recipe") + required_strings = ( + "id", + "repository", + "issue_title", + "issue_body", + "source_sha256", + "hidden_test_sha256", + ) + for field in required_strings: + if not isinstance(record.get(field), str) or not record[field].strip(): + raise ValueError(f"{field} must be a non-empty string") + if record.get("validated") is not True: + raise ValueError(f"PatchEval task {record['id']!r} is not miner-validated") + + source = _safe_relative(root, record.get("source_archive"), "source_archive") + hidden = _safe_relative(root, record.get("hidden_test_patch"), "hidden_test_patch") + for path, field, expected in ( + (source, "source_archive", record["source_sha256"]), + (hidden, "hidden_test_patch", record["hidden_test_sha256"]), + ): + if not path.is_file(): + raise ValueError(f"{field} does not exist for task {record['id']!r}") + actual = _sha256(path) + if actual != expected: + raise ValueError( + f"{field} checksum mismatch for task {record['id']!r}: " + f"expected {expected}, got {actual}" + ) + _validate_source_archive(source) + + timeout_s = record.get("timeout_s", 300) + if isinstance(timeout_s, bool) or not isinstance(timeout_s, int): + raise ValueError("timeout_s must be an integer") + if not 10 <= timeout_s <= 1800: + raise ValueError("timeout_s must be between 10 and 1800 seconds") + + return PatchEvalSpec( + id=record["id"], + repository=record["repository"], + issue_title=record["issue_title"].strip(), + issue_body=record["issue_body"].strip(), + runtime_recipe=_runtime_recipe(record), + source_archive=source, + hidden_test_patch=hidden, + source_sha256=record["source_sha256"], + hidden_test_sha256=record["hidden_test_sha256"], + fail_to_pass_command=_strings(record, "fail_to_pass_command"), + regression_command=_strings(record, "regression_command"), + setup_command=_strings(record, "setup_command", required=False), + protected_globs=tuple( + dict.fromkeys(_PYTHON_TEST_GLOBS + _strings(record, "protected_globs")) + ), + ignored_globs=_strings(record, "ignored_globs", required=False), + timeout_s=timeout_s, + ) + + +def _source_archive_members(handle: tarfile.TarFile) -> list[tarfile.TarInfo]: + members = handle.getmembers() + for member in members: + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts: + raise ValueError(f"unsafe path in source archive: {member.name!r}") + if member.issym() or member.islnk(): + raise ValueError(f"source archives must dereference links: {member.name!r}") + if not member.isdir() and not member.isreg(): + raise ValueError(f"unsupported archive entry: {member.name!r}") + return members + + +def _validate_source_archive(archive: Path) -> None: + with tarfile.open(archive, "r:*") as handle: + _source_archive_members(handle) + + +def _safe_extract(archive: Path, destination: Path) -> None: + """Extract a miner-produced source archive without following archive links.""" + with tarfile.open(archive, "r:*") as handle: + members = _source_archive_members(handle) + handle.extractall(destination, members=members, filter="fully_trusted") + + +def _git( + worktree: Path, args: list[str], *, check: bool = True +) -> subprocess.CompletedProcess[str]: + environment = os.environ.copy() + environment.update( + GIT_CONFIG_NOSYSTEM="1", + GIT_CONFIG_GLOBAL=os.devnull, + ) + completed = subprocess.run( + ["git", "-C", str(worktree), *args], + text=True, + errors="replace", + capture_output=True, + env=environment, + timeout=60, + ) + if check and completed.returncode: + detail = (completed.stderr or completed.stdout).strip()[-2000:] + raise RuntimeError(f"trusted patch extraction failed: {detail}") + return completed + + +def _remove_entry(path: Path) -> None: + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.exists(): + shutil.rmtree(path) + + +def _copy_candidate(source: Path, destination: Path) -> None: + """Copy only ordinary files, directories, and symlinks from the sandbox.""" + file_count = 0 + byte_count = 0 + + def copy_entry(current: Path, target: Path) -> None: + nonlocal file_count, byte_count + if current.name == ".git": + return + info = current.lstat() + file_count += 1 + if file_count > _MAX_CANDIDATE_FILES: + raise ValueError("candidate workspace contains too many files") + if stat.S_ISLNK(info.st_mode): + target.symlink_to(os.readlink(current)) + elif stat.S_ISDIR(info.st_mode): + target.mkdir(mode=info.st_mode & 0o777, exist_ok=True) + for child in current.iterdir(): + copy_entry(child, target / child.name) + elif stat.S_ISREG(info.st_mode): + byte_count += info.st_size + if byte_count > _MAX_CANDIDATE_BYTES: + raise ValueError("candidate workspace is too large") + shutil.copyfile(current, target, follow_symlinks=False) + target.chmod(info.st_mode & 0o777) + else: + raise ValueError(f"candidate contains unsupported file: {current.name!r}") + + for child in source.iterdir(): + copy_entry(child, destination / child.name) + + +def _matches(path: str, patterns: tuple[str, ...]) -> bool: + return any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns) + + +def _trusted_patch( + spec: PatchEvalSpec, environment: DockerTaskEnvironment +) -> tuple[str, list[str], list[str]]: + """Diff a downloaded workspace against the host's pristine source archive.""" + with tempfile.TemporaryDirectory(prefix="benchkit-patcheval-diff-") as directory: + root = Path(directory) + worktree = root / "trusted" + candidate = root / "candidate" + worktree.mkdir() + candidate.mkdir() + _safe_extract(spec.source_archive, worktree) + environment.download(f"{environment.workdir}/.", candidate) + + _git(worktree, ["init", "-q"]) + _git(worktree, ["add", "-A"]) + _git( + worktree, + [ + "-c", + "user.name=BenchKit", + "-c", + "user.email=benchkit@localhost", + "commit", + "-q", + "--no-gpg-sign", + "-m", + "pristine", + ], + ) + for child in worktree.iterdir(): + if child.name != ".git": + _remove_entry(child) + _copy_candidate(candidate, worktree) + _git(worktree, ["add", "--intent-to-add", "--", "."]) + # -z keeps paths verbatim. Git otherwise quotes non-ASCII, quote, + # backslash, and control characters, and a quoted path matches neither + # its protected glob nor the pathspec of the second diff, so the + # agent's real edits are silently dropped from the submission. + changed = [ + path + for path in _git( + worktree, + ["diff", "-z", "--name-only", "--no-renames", "HEAD", "--"], + ).stdout.split("\0") + if path + ] + excluded = [ + path + for path in changed + if _matches(path, spec.protected_globs + spec.ignored_globs) + ] + included = [path for path in changed if path not in excluded] + if not included: + return "", changed, excluded + patch = _git( + worktree, + ["diff", "--binary", "--no-renames", "HEAD", "--", *included], + ).stdout + if len(patch.encode()) > _MAX_PATCH_BYTES: + raise ValueError("candidate patch exceeds the 5 MiB limit") + return patch, changed, excluded + + +def _docker_exec( + docker: str, + container: str, + command: list[str], + *, + workdir: str | None = None, + timeout: float = 60, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + return _run( + [ + docker, + "exec", + *(["--workdir", workdir] if workdir else []), + container, + *command, + ], + timeout=timeout, + check=check, + ) + + +def _grade_once( + spec: PatchEvalSpec, + image: LatestPiImage, + patch: str, + command: tuple[str, ...], + *, + hidden_tests: bool, + owner_id: str, +) -> dict: + docker = image.docker + label = "f2p" if hidden_tests else "regression" + container = f"benchkit-patcheval-{label}-{uuid.uuid4().hex[:12]}" + labels = resource_labels(owner_id) + with tempfile.TemporaryDirectory(prefix="benchkit-patcheval-grade-") as directory: + patch_path = Path(directory) / "submission.patch" + patch_path.write_text(patch, encoding="utf-8") + try: + _run( + [ + docker, + "run", + "--detach", + "--name", + container, + *labels, + "--network", + "none", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + os.environ.get("BENCHKIT_SANDBOX_PIDS", "256"), + "--memory", + os.environ.get("BENCHKIT_SANDBOX_MEMORY", "2g"), + "--cpus", + os.environ.get("BENCHKIT_SANDBOX_CPUS", "2"), + image.image, + "sleep", + "infinity", + ], + timeout=60, + ) + _run([docker, "cp", str(spec.source_archive), f"{container}:/tmp/source"]) + _run([docker, "cp", str(patch_path), f"{container}:/tmp/submission.patch"]) + if hidden_tests: + _run( + [ + docker, + "cp", + str(spec.hidden_test_patch), + f"{container}:/tmp/hidden-tests.patch", + ] + ) + _docker_exec(docker, container, ["mkdir", "-p", "/workspace/repo"]) + _docker_exec( + docker, + container, + ["tar", "-xf", "/tmp/source", "-C", "/workspace/repo"], + ) + _docker_exec( + docker, + container, + [ + "find", + "/workspace/repo", + "-exec", + "touch", + "-t", + "198001010000", + "{}", + "+", + ], + ) + if patch: + _docker_exec( + docker, + container, + ["git", "apply", "--binary", "/tmp/submission.patch"], + workdir="/workspace/repo", + ) + if hidden_tests: + _docker_exec( + docker, + container, + ["git", "apply", "--binary", "/tmp/hidden-tests.patch"], + workdir="/workspace/repo", + ) + if spec.setup_command: + _docker_exec( + docker, + container, + list(spec.setup_command), + workdir="/workspace/repo", + timeout=spec.timeout_s + 30, + ) + completed = _docker_exec( + docker, + container, + ["timeout", "--signal=KILL", f"{spec.timeout_s}s", *command], + workdir="/workspace/repo", + timeout=spec.timeout_s + 30, + check=False, + ) + output = (completed.stdout + completed.stderr).strip() + return { + "exit_code": completed.returncode, + "output": output[-8000:], + "output_truncated": len(output) > 8000, + } + finally: + _run([docker, "rm", "--force", container], timeout=30, check=False) + + +class PatchEval: + """Fix real Python repository issues with hidden, deterministic grading.""" + + name = "patcheval" + task_count = 20 + workspace_task = True + evaluation_activity = "running hidden PatchEval checks" + list_note = "pilot-20 ยท requires Pi and Docker" + + def __init__(self, dataset_root: Path | None = None) -> None: + configured = dataset_root or ( + Path(os.environ["BENCHKIT_PATCHEVAL_DATASET"]).expanduser() + if os.environ.get("BENCHKIT_PATCHEVAL_DATASET") + else None + ) + self.dataset_root = configured + self.release = "unloaded" + + def _root(self) -> Path: + if self.dataset_root is None: + raise RuntimeError( + "PatchEval's frozen dataset is not configured; download the " + "pilot-20 release from DogukanUrker/PatchEval and set " + "BENCHKIT_PATCHEVAL_DATASET to its local root" + ) + return self.dataset_root.resolve() + + @staticmethod + def _spec(task: Task) -> PatchEvalSpec: + spec = task.metadata.get("spec") + if not isinstance(spec, PatchEvalSpec): + raise TypeError("PatchEval task metadata is missing its validated spec") + return spec + + def load_tasks(self) -> list[Task]: + root = self._root() + release_path = root / _RELEASE_FILE + tasks_path = root / _TASKS_FILE + try: + release = json.loads(release_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"could not read PatchEval dataset metadata: {exc}" + ) from exc + if ( + not isinstance(release, dict) + or release.get("schema_version") != _SCHEMA_VERSION + ): + raise RuntimeError( + f"PatchEval dataset must use schema_version {_SCHEMA_VERSION}" + ) + self.release = str(release.get("release") or "unknown") + try: + records = [ + json.loads(line) + for line in tasks_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + specs = [_load_spec(root, record) for record in records] + except (OSError, json.JSONDecodeError, ValueError) as exc: + raise RuntimeError(f"invalid PatchEval dataset: {exc}") from exc + expected = release.get("task_count") + if expected != len(specs): + raise RuntimeError( + f"PatchEval release declares {expected} tasks but contains {len(specs)}" + ) + ids = [spec.id for spec in specs] + if len(ids) != len(set(ids)): + raise RuntimeError("PatchEval task ids must be unique") + self.task_count = len(specs) + return [Task(spec.id, "", {"spec": spec}) for spec in specs] + + def result_metadata(self, _variant: str | None) -> dict: + return {"dataset_release": self.release} + + def build_prompt(self, task: Task) -> str: + spec = self._spec(task) + return ( + "Fix the following issue in the current repository. Inspect the code, " + "make the necessary changes, and verify your solution.\n\n" + f"# {spec.issue_title}\n\n{spec.issue_body}" + ) + + def build_repair_prompt(self, _feedback: str, _attempt: int, _total: int) -> str: + return _GENERIC_REPAIR + + def evaluate(self, _task: Task, _response: str) -> bool: + raise RuntimeError("PatchEval must be evaluated by its external grader") + + def pi_image_for_task(self, task: Task) -> LatestPiImage: + spec = self._spec(task) + return patcheval_pi_image( + spec.source_archive, + spec.source_sha256, + spec.runtime_recipe, + ) + + def prepare_workspace(self, task: Task, environment: DockerTaskEnvironment) -> None: + spec = self._spec(task) + environment.workdir = "/workspace/repo" + environment.upload(spec.source_archive, "/tmp/patch-eval-source") + environment.exec(["mkdir", "-p", environment.workdir]) + environment.exec( + ["tar", "-xf", "/tmp/patch-eval-source", "-C", environment.workdir] + ) + environment.exec( + [ + "find", + environment.workdir, + "-exec", + "touch", + "-t", + "198001010000", + "{}", + "+", + ] + ) + environment.exec(["rm", "-rf", f"{environment.workdir}/.git"]) + if spec.setup_command: + environment.exec( + list(spec.setup_command), + workdir=environment.workdir, + timeout=spec.timeout_s + 30, + ) + environment.exec(["git", "init", "-q", environment.workdir]) + environment.exec( + ["git", "-C", environment.workdir, "config", "user.name", "BenchKit"] + ) + environment.exec( + [ + "git", + "-C", + environment.workdir, + "config", + "user.email", + "benchkit@localhost", + ] + ) + environment.exec(["git", "-C", environment.workdir, "add", "."]) + environment.exec( + [ + "git", + "-C", + environment.workdir, + "commit", + "-q", + "-m", + "starting state", + ] + ) + + def verify_workspace( + self, + task: Task, + environment: DockerTaskEnvironment, + _tool_trace: list[dict] | None = None, + ) -> EvaluationResult: + spec = self._spec(task) + try: + patch, changed, excluded = _trusted_patch(spec, environment) + # The two graders share nothing but the read-only task image and + # the submitted patch, so they run at the same time. + with ThreadPoolExecutor(max_workers=2) as pool: + f2p_grade = pool.submit( + _grade_once, + spec, + environment.image, + patch, + spec.fail_to_pass_command, + hidden_tests=True, + owner_id=environment.owner_id, + ) + regression_grade = pool.submit( + _grade_once, + spec, + environment.image, + patch, + spec.regression_command, + hidden_tests=False, + owner_id=environment.owner_id, + ) + f2p = f2p_grade.result() + regression = regression_grade.result() + except (OSError, RuntimeError, ValueError, SandboxError) as exc: + return EvaluationResult( + 0.0, + error=f"PatchEval grader infrastructure failed: {type(exc).__name__}: {exc}", + ) + passed = f2p["exit_code"] == 0 and regression["exit_code"] == 0 + return EvaluationResult( + 1.0 if passed else 0.0, + feedback="" if passed else _GENERIC_REPAIR, + details={ + "repository": spec.repository, + "f2p_exit_code": f2p["exit_code"], + "regression_exit_code": regression["exit_code"], + "f2p_output": f2p["output"], + "f2p_output_truncated": f2p["output_truncated"], + "regression_output": regression["output"], + "regression_output_truncated": regression["output_truncated"], + "changed_paths": changed, + "excluded_agent_paths": excluded, + "patch_sha256": hashlib.sha256(patch.encode()).hexdigest(), + "patch": patch, + }, + ) diff --git a/src/benchkit/engine.py b/src/benchkit/engine.py index 14a0714..e56a457 100644 --- a/src/benchkit/engine.py +++ b/src/benchkit/engine.py @@ -33,6 +33,7 @@ from benchkit.metrics import throughput_metrics from benchkit.perturbations import annotate_robustness, perturb_task from benchkit.pi_agent import PiAgentRunner +from benchkit.sandbox import cleanup_run_resources MAX_REPAIR_ATTEMPTS = 10 @@ -834,7 +835,18 @@ def emit(self, event: EngineEvent) -> None: with self._emit_lock: self.sink(event) - def _pi(self, bench: object | None = None) -> PiAgentRunner: + def _pi( + self, bench: object | None = None, task: Task | None = None + ) -> PiAgentRunner: + task_image_factory = getattr(bench, "pi_image_for_task", None) + if callable(task_image_factory) and task is not None: + image = task_image_factory(task) + key = f"{getattr(bench, 'name', type(bench).__name__)}:{image.image}" + if key not in self._workspace_pi_runners: + self._workspace_pi_runners[key] = PiAgentRunner( + self.client, image=image + ) + return self._workspace_pi_runners[key] image_factory = getattr(bench, "pi_image", None) if callable(image_factory): key = str(getattr(bench, "name", type(bench).__name__)) @@ -881,12 +893,19 @@ def run(self) -> list[dict]: results.append(result) self._maybe_unload(index, job) finally: + used_pi = self._pi_runner is not None or bool(self._workspace_pi_runners) if self._pi_runner is not None: with contextlib.suppress(Exception): self._pi_runner.cleanup() for runner in self._workspace_pi_runners.values(): with contextlib.suppress(Exception): runner.cleanup() + if used_pi: + # Removes the run's shared builder, its build cache, and any + # image, container, network, or volume still carrying the run + # label. Scoped by label, never a global prune. + with contextlib.suppress(Exception): + cleanup_run_resources() annotate_robustness(results) annotate_harness_effect(results) @@ -979,10 +998,11 @@ def _run_job( task_id=first.id, entry_point=str(first.metadata.get("entry_point", "")), phase="generating", - activity="preparing latest Pi sandbox image", + activity="preparing pinned Pi sandbox image", ) ) - self._pi(bench).prepare() + for task in tasks: + self._pi(bench, task).prepare() if not tasks: return None, False @@ -1702,10 +1722,11 @@ def on_progress(update: GenerationUpdate) -> None: request_started = time.perf_counter() try: - generator = self._pi(bench) if job.harness == "pi" else self.client + generator = self._pi(bench, task) if job.harness == "pi" else self.client workspace = bool(getattr(bench, "workspace_task", False)) workspace_setup = None workspace_verifier = None + repair_prompt_builder = None if workspace: def workspace_setup(environment): @@ -1714,6 +1735,10 @@ def workspace_setup(environment): def workspace_verifier(environment, tool_trace): return bench.verify_workspace(task, environment, tool_trace) + candidate_builder = getattr(bench, "build_repair_prompt", None) + if callable(candidate_builder): + repair_prompt_builder = candidate_builder + if job.repair_attempts: def verifier(response: str) -> EvaluationResult: @@ -1729,6 +1754,7 @@ def verifier(response: str) -> EvaluationResult: repair_attempts=job.repair_attempts, workspace_setup=workspace_setup, workspace_verifier=workspace_verifier, + repair_prompt_builder=repair_prompt_builder, ) else: gen = self._generate_direct_with_repairs( diff --git a/src/benchkit/pi_agent.py b/src/benchkit/pi_agent.py index fb926de..015be69 100644 --- a/src/benchkit/pi_agent.py +++ b/src/benchkit/pi_agent.py @@ -165,6 +165,7 @@ def generate( [DockerTaskEnvironment, list[dict[str, Any]]], EvaluationResult ] | None = None, + repair_prompt_builder: Callable[[str, int, int], str] | None = None, ) -> dict: self.prepare() started = time.perf_counter() @@ -426,11 +427,8 @@ def send(payload: dict[str, Any]) -> None: if attempt >= repair_attempts: break feedback_sent.append(evaluation.feedback) - message = repair_message( - evaluation.feedback, - attempt + 1, - repair_attempts, - ) + builder = repair_prompt_builder or repair_message + message = builder(evaluation.feedback, attempt + 1, repair_attempts) scaffold_value = environment.pi_scaffold() scaffold = ( dict(scaffold_value) if isinstance(scaffold_value, dict) else {} diff --git a/src/benchkit/pi_package/package-lock.json b/src/benchkit/pi_package/package-lock.json new file mode 100644 index 0000000..ffb728e --- /dev/null +++ b/src/benchkit/pi_package/package-lock.json @@ -0,0 +1,1827 @@ +{ + "name": "benchkit-pi-runtime", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "benchkit-pi-runtime", + "version": "0.0.0", + "dependencies": { + "@earendil-works/pi-coding-agent": "0.84.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.84.2.tgz", + "integrity": "sha512-l4E+B7hgXKWddRo8bC/eSue2aWZjEgJ9xIpf5p0Og+lq8a2TArCwJ0HCoCPCgaBP/tN4zbYH/wOwvx9pJpeLCA==", + "hasShrinkwrap": true, + "license": "MIT", + "dependencies": { + "@earendil-works/pi-agent-core": "^0.84.2", + "@earendil-works/pi-ai": "^0.84.2", + "@earendil-works/pi-client": "^0.84.2", + "@earendil-works/pi-protocol": "^0.84.2", + "@earendil-works/pi-tui": "^0.84.2", + "@silvia-odwyer/photon-node": "0.3.4", + "chalk": "5.6.2", + "cross-spawn": "7.0.6", + "diff": "8.0.4", + "glob": "13.0.6", + "grok-mermaid": "0.2.2", + "highlight.js": "10.7.3", + "hosted-git-info": "9.0.3", + "ignore": "7.0.5", + "jiti": "2.7.0", + "minimatch": "10.2.5", + "proper-lockfile": "4.1.2", + "semver": "7.8.0", + "typebox": "1.3.7", + "undici": "8.9.0", + "yaml": "2.9.0" + }, + "bin": { + "pi": "dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + }, + "optionalDependencies": { + "@mariozechner/clipboard": "0.3.9" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@anthropic-ai/sdk": { + "version": "0.91.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz", + "integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/client-bedrock-runtime": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1048.0.tgz", + "integrity": "sha512-u+NT61JZEkRFtpL0CAw1N1dwxnaLgwVXQl/zjJxTGgLyS/jTIdg2SdoEoCTHxgDyCnqa1HEi9QOoE9/pYRNpOQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/credential-provider-node": "^3.972.42", + "@aws-sdk/eventstream-handler-node": "^3.972.16", + "@aws-sdk/middleware-eventstream": "^3.972.12", + "@aws-sdk/middleware-websocket": "^3.972.19", + "@aws-sdk/token-providers": "3.1048.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/core": { + "version": "3.974.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.974.11.tgz", + "integrity": "sha512-QpnINq5FZH6EOaDEkmHdT7eUunbvD27pDNQypaWjFyYz7Zl1q3UCMQErBZxpmfGfI7MvI2TlK8KTkgNpv8b1ug==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/xml-builder": "^3.972.24", + "@aws/lambda-invoke-store": "^0.2.2", + "@smithy/core": "^3.24.2", + "@smithy/signature-v4": "^5.4.2", + "@smithy/types": "^4.14.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.37", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.37.tgz", + "integrity": "sha512-/jpPvEh6f7ntmIzf7dNxoNX6Q8vt8UpesCjbW6mFfk4V1NW6bIy9qxcQ6WbA8As5yQhsZOe+xeNd4xHX8kdY2Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.39", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.39.tgz", + "integrity": "sha512-pIgTpisWyWg7X1bUbzSjuUYosYTD0Ghz2M0hkSTmb3a6i3qV3uU+NYJPI/E2XSC0HcsZh5rsLPzeXrkb2DS0Cg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.41.tgz", + "integrity": "sha512-u2tyjaxJJzW8UtW4SM1ZcPMDwO6y+kV+llvou+Adts0FAKyzes5jG4izQN+KX3yE8ZROpS5y1LJ//xL2iSf76w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/credential-provider-env": "^3.972.37", + "@aws-sdk/credential-provider-http": "^3.972.39", + "@aws-sdk/credential-provider-login": "^3.972.41", + "@aws-sdk/credential-provider-process": "^3.972.37", + "@aws-sdk/credential-provider-sso": "^3.972.41", + "@aws-sdk/credential-provider-web-identity": "^3.972.41", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/credential-provider-imds": "^4.3.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.41.tgz", + "integrity": "sha512-0LBitxXiAiaE5nlFPfpNIww/8FRY/I7WIndWsc9GmNFOM7cE1wNpVNQEGEk9Outg5l8xl+3vybxFyUy4l9q/LQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.42", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.42.tgz", + "integrity": "sha512-D4oon2zbqqsWOJUM99Gm3/ZyJ0IJvTXVN3PyloGb3kQEyI36fjCZheZj422lAgTWWd6TSHgiImLt3RIaLdv3dQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.37", + "@aws-sdk/credential-provider-http": "^3.972.39", + "@aws-sdk/credential-provider-ini": "^3.972.41", + "@aws-sdk/credential-provider-process": "^3.972.37", + "@aws-sdk/credential-provider-sso": "^3.972.41", + "@aws-sdk/credential-provider-web-identity": "^3.972.41", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/credential-provider-imds": "^4.3.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.37", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.37.tgz", + "integrity": "sha512-7nVaHBUaWIddASYfVaA9O4D5ZVjewU3sCol9WqZPGfW0nR+0WqE0xHZnD/U2L33PlOB8KNXGKZ6wOES/QijKzg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.41.tgz", + "integrity": "sha512-IOWAWEHe5LkjSKkkUUX9ciV6Y1scHTsnfEkdt5yyC4Slrc7AGbkLPrpntjqh18ksJAMOaVhoBsO8p2WyTcY2wQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/token-providers": "3.1048.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.41.tgz", + "integrity": "sha512-mbACk9Yypa8nm4iGZLs0PofOXEcTDOUw6wDnsPXNDNSd2WNXs1tSo+6nc/fh0jLYdfVZThhBL98PHW4aXFsG5A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/eventstream-handler-node": { + "version": "3.972.16", + "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.16.tgz", + "integrity": "sha512-yedpPgKftqjU5SlPFHfqWpOw6xSCRieWRG1euWOlXn4WJxt2VX92VprCa2PpSOXjVCAeK6dTjW9eJRXVig9yGA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/middleware-eventstream": { + "version": "3.972.12", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.12.tgz", + "integrity": "sha512-tHTHHCHNrq6XklQvlzHBDJG4Iuhh7NVPRdtmvP+nHFA+5sxPlIDzlAHHgfoYHGvT3NXP1yVP/L5c3opUn6T3Qg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/middleware-websocket": { + "version": "3.972.19", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-websocket/-/middleware-websocket-3.972.19.tgz", + "integrity": "sha512-mkEhOGYozqKQkbFaVrjwr0faiwwZza1v5/jSY6Tucm3bD+uKTazIUH/4Yo6aMnQD2ua2W9cMP6s8mvwTcjtqHw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/signature-v4": "^5.4.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/nested-clients": { + "version": "3.997.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.9.tgz", + "integrity": "sha512-jPR3rnmRI4hWYyzfmTGBr7NblMp8QYYeflHXba1H6+7CGrWVqWKQzaXFQ4qbExqPRsXN3T3L3JxFhr6aouXUGQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/signature-v4-multi-region": "^3.996.27", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.27", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.27.tgz", + "integrity": "sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/signature-v4": "^5.4.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/token-providers": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1048.0.tgz", + "integrity": "sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/types": { + "version": "3.973.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.8.tgz", + "integrity": "sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/util-locate-window": { + "version": "3.965.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.5.tgz", + "integrity": "sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/xml-builder": { + "version": "3.972.24", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.24.tgz", + "integrity": "sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==", + "license": "Apache-2.0", + "dependencies": { + "@nodable/entities": "2.1.0", + "@smithy/types": "^4.14.1", + "fast-xml-parser": "5.7.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws/lambda-invoke-store": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.4.tgz", + "integrity": "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@babel/runtime": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-agent-core": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.84.2.tgz", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-ai": "^0.84.2", + "@earendil-works/pi-telemetry": "^0.84.2", + "diff": "8.0.4", + "ignore": "7.0.5", + "typebox": "1.3.7", + "yaml": "2.9.0" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-ai": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.84.2.tgz", + "license": "MIT", + "dependencies": { + "@anthropic-ai/sdk": "0.91.1", + "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@earendil-works/pi-telemetry": "^0.84.2", + "@google/genai": "1.52.0", + "@opentelemetry/api": "1.9.0", + "@smithy/node-http-handler": "4.7.3", + "http-proxy-agent": "7.0.2", + "https-proxy-agent": "7.0.6", + "openai": "6.40.0", + "partial-json": "0.1.7", + "typebox": "1.3.7" + }, + "bin": { + "pi-ai": "dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-client": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-client/-/pi-client-0.84.2.tgz", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-protocol": "^0.84.2" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-protocol": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-protocol/-/pi-protocol-0.84.2.tgz", + "license": "MIT", + "dependencies": { + "typebox": "1.3.7" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-telemetry": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-0.84.2.tgz", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-tui": { + "version": "0.84.2", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.84.2.tgz", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "1.6.0", + "marked": "18.0.5" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@google/genai": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.52.0.tgz", + "integrity": "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.9.tgz", + "integrity": "sha512-ABnA53mdfkGZwOFUdZNv2S0CWGO/EIuPj8Vv9xmBFmSYg/qFc7ihO6q5FcQjvoE67kZpWkEc4AhD6B/os04yuA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@mariozechner/clipboard-darwin-arm64": "0.3.9", + "@mariozechner/clipboard-darwin-universal": "0.3.9", + "@mariozechner/clipboard-darwin-x64": "0.3.9", + "@mariozechner/clipboard-linux-arm64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-arm64-musl": "0.3.9", + "@mariozechner/clipboard-linux-riscv64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-x64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-x64-musl": "0.3.9", + "@mariozechner/clipboard-win32-arm64-msvc": "0.3.9", + "@mariozechner/clipboard-win32-x64-msvc": "0.3.9" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-darwin-arm64": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-arm64/-/clipboard-darwin-arm64-0.3.9.tgz", + "integrity": "sha512-BfgV7vCEWZwJwZJw03r6bP5+tf0iI/ANuQYCxi9RNn7FrWB3yzGuMKCrNLRl6V761vXRdL8+OqZ0wd4TqlsNOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-darwin-universal": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-universal/-/clipboard-darwin-universal-0.3.9.tgz", + "integrity": "sha512-BGGR4iA9Z2shAjI65eI5xtyb3LYNlDW9X3gxKxDbqtbnREohsrqznov6zpKoIrsRWpzlYVEdKphS7ksJ0/ndSQ==", + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-darwin-x64": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-x64/-/clipboard-darwin-x64-0.3.9.tgz", + "integrity": "sha512-4kURmCbS6nt8uYhtmWpUcJWyPHfmAr5dTpXD1nO3pIfa+TSQ9DbrGOYCKH+aEFW47XhQ4Vp8ZTszie+wfFvDKg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-arm64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-gnu/-/clipboard-linux-arm64-gnu-0.3.9.tgz", + "integrity": "sha512-g59OkUGP2DDfCOIKypHeYgv2M55u/cKvXa5dSxFbEJ34XvIQMdcVmpKCkGUro3ZgefXiGVdwguvTMQGpHWzIXw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-arm64-musl": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-musl/-/clipboard-linux-arm64-musl-0.3.9.tgz", + "integrity": "sha512-AGuJdgKsmJdm4Pych7kv3sqe591ERRaAHW3xjLooiFzn8J+PxUyof++7YZrB5Y5tpnTO+K18Og3taj2NpluCRQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-riscv64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-riscv64-gnu/-/clipboard-linux-riscv64-gnu-0.3.9.tgz", + "integrity": "sha512-DXBEAiuMpk7dhS1a9NzNxVAFi1vaKoPu7rQNgY8LIDLGrK3lnIp3nT10DUum+PKVJoJppIP+NAA8IZe4DMNDPw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-x64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-gnu/-/clipboard-linux-x64-gnu-0.3.9.tgz", + "integrity": "sha512-WORrMLd6EpElEME7JRKfSaY34nW1P5LbdgK5YNCS1ncG2LqmITsSMEJ8nh2mpvxb3TxqbOOKgY7k9eMJYlW9Mw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-x64-musl": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-musl/-/clipboard-linux-x64-musl-0.3.9.tgz", + "integrity": "sha512-/DHn+1DrfL6oRaPPWXaOKvonFFrni666fxd+zFqiQEfvBH0tsHVWjq9iqBk0oDp0qaPA72lIMy5BptxISBEhZQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-win32-arm64-msvc": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-arm64-msvc/-/clipboard-win32-arm64-msvc-0.3.9.tgz", + "integrity": "sha512-O5FHD3ErkMwMhNzAfu3ggy0ug4z7btZuoQgwwxlzPrwV2bxlD6WDpqBY4NCgICAgZdDKdp+loUEKVAVt8aYnhQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-win32-x64-msvc": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-x64-msvc/-/clipboard-win32-x64-msvc-0.3.9.tgz", + "integrity": "sha512-ihQC3EufqEY81vhXBgVBtK4prL+wc62zJsSvxrgz7K1hsdt6OObz6v9p3Rn1OG3GJksTTKMJF0u/guMISHPhSA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@nodable/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@silvia-odwyer/photon-node": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon-node/-/photon-node-0.3.4.tgz", + "integrity": "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA==", + "license": "Apache-2.0" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/core": { + "version": "3.24.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.24.3.tgz", + "integrity": "sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/credential-provider-imds": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.3.3.tgz", + "integrity": "sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/fetch-http-handler": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.4.3.tgz", + "integrity": "sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/node-http-handler": { + "version": "4.7.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.7.3.tgz", + "integrity": "sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/signature-v4": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.4.3.tgz", + "integrity": "sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/types": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.14.2.tgz", + "integrity": "sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@types/node": { + "version": "22.19.19", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.19.tgz", + "integrity": "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/brace-expansion": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/diff": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/fast-xml-builder": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", + "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.5.0", + "xml-naming": "^0.1.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/fast-xml-parser": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.3.tgz", + "integrity": "sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^2.1.0", + "fast-xml-builder": "^1.1.7", + "path-expression-matcher": "^1.5.0", + "strnum": "^2.2.3" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/gaxios": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", + "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/google-auth-library": { + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", + "integrity": "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/grok-mermaid": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/grok-mermaid/-/grok-mermaid-0.2.2.tgz", + "integrity": "sha512-XcJEP5dDC8liHBh52mlLjU18fNvu1ckFsu0QpIG3+APZ270fsj9wxpiA6cOURmbUEuoMVgjbC2+UYgTdCqqgzA==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/hosted-git-info": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.3.tgz", + "integrity": "sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg==", + "license": "ISC", + "dependencies": { + "lru-cache": "^11.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/lru-cache": { + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.4.0.tgz", + "integrity": "sha512-W+R+kFL4HgVxONq2bhXPi3bGpzGe/yEhVOp233qw9wCRtgncJ15P3bC+e4zZMu4Cq7d+WAJjXGW0uUkifhcatA==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/marked": { + "version": "18.0.5", + "resolved": "https://registry.npmjs.org/marked/-/marked-18.0.5.tgz", + "integrity": "sha512-S6GcvALHg6K4ohtu4E7x0a1AqhAjp6cV8KhLSyN9qVapnzJkusVBxZRcIU9AeYsbe6P1hKDusSbEOzGyyuce6w==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/openai": { + "version": "6.40.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.40.0.tgz", + "integrity": "sha512-MWtTjd/gQt4jpbji61NTgFWJLoY/PdRJ6wG9/ZDRMYNMlBKrCrSlkLI+KgHP1vR1qT6LKSAyAqIxno6lcK9JiA==", + "license": "Apache-2.0", + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/p-retry/node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/partial-json": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/partial-json/-/partial-json-0.1.7.tgz", + "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/path-expression-matcher": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", + "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/proper-lockfile/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/strnum": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.3.0.tgz", + "integrity": "sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/typebox": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.7.tgz", + "integrity": "sha512-meKuifc33Pccx0O6PdIzYMq3Og8zvP4TIi/a+Bw3AEMZMxOD0+RHGQvpglEe6Zdy3wZ8nqn/j95h8LUZLk/6Hg==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/undici": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.9.0.tgz", + "integrity": "sha512-aWZpUj7XoGonMClx4gdDRfgBjqeA+F473aDmROQQbM9n6PRfK/u1q/a0X4wMTgcHfT8H6fpbt98PFuDUwFg2YA==", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/xml-naming": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", + "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + } + } +} diff --git a/src/benchkit/pi_package/package.json b/src/benchkit/pi_package/package.json new file mode 100644 index 0000000..7bb73c2 --- /dev/null +++ b/src/benchkit/pi_package/package.json @@ -0,0 +1,8 @@ +{ + "name": "benchkit-pi-runtime", + "private": true, + "version": "0.0.0", + "dependencies": { + "@earendil-works/pi-coding-agent": "0.84.2" + } +} diff --git a/src/benchkit/runner.py b/src/benchkit/runner.py index 979f555..7edb2f8 100644 --- a/src/benchkit/runner.py +++ b/src/benchkit/runner.py @@ -720,6 +720,7 @@ def run( def on_interrupt(signum: int, frame: FrameType | None) -> None: if controls.stopped: + signal.signal(signal.SIGINT, signal.SIG_IGN) raise KeyboardInterrupt reporter.state.stopping = True controls.stop() @@ -727,9 +728,21 @@ def on_interrupt(signum: int, frame: FrameType | None) -> None: Text("Interrupted, cancelling active requestsโ€ฆ", style="bold yellow") ) - previous = None - with contextlib.suppress(ValueError): # not on the main thread - previous = signal.signal(signal.SIGINT, on_interrupt) + def on_terminate(signum: int, frame: FrameType | None) -> None: + # SIGHUP (for example, a dropped SSH session) and SIGTERM do not offer + # an interactive second signal. Unwind immediately so Engine.finally + # can remove its containers, networks, images, builders, and volumes. + controls.stop() + signal.signal(signum, signal.SIG_IGN) + raise SystemExit(128 + signum) + + handlers = [(signal.SIGINT, on_interrupt), (signal.SIGTERM, on_terminate)] + if hasattr(signal, "SIGHUP"): + handlers.append((signal.SIGHUP, on_terminate)) + previous: dict[int, signal.Handlers] = {} + for signum, handler in handlers: + with contextlib.suppress(ValueError): # not on the main thread + previous[signum] = signal.signal(signum, handler) try: with reporter: @@ -738,7 +751,7 @@ def on_interrupt(signum: int, frame: FrameType | None) -> None: console.print(Text("Aborted.", style="bold yellow")) return [], "interrupted" finally: - if previous is not None: - signal.signal(signal.SIGINT, previous) + for signum, handler in previous.items(): + signal.signal(signum, handler) return results, engine.failure diff --git a/src/benchkit/sandbox.py b/src/benchkit/sandbox.py index 92f9fa4..d0b2adf 100644 --- a/src/benchkit/sandbox.py +++ b/src/benchkit/sandbox.py @@ -2,26 +2,68 @@ from __future__ import annotations +import contextlib +import hashlib import json import os +import re import shutil import subprocess import tempfile import threading import uuid from dataclasses import dataclass, field -from pathlib import Path +from pathlib import Path, PurePosixPath from urllib.parse import urlsplit, urlunsplit from benchkit.client import InferenceClient, _openai_base -PI_IMAGE = "benchkit-pi:latest" -AIDER_PI_IMAGE = "benchkit-pi-aider-polyglot:latest" -GIT_SURGERY_PI_IMAGE = "benchkit-pi-git-surgery:latest" -PI_PACKAGE = "@earendil-works/pi-coding-agent@latest" +_PI_IMAGE_SESSION = uuid.uuid4().hex[:12] +PI_IMAGE = f"benchkit-pi:{_PI_IMAGE_SESSION}" +AIDER_PI_IMAGE = f"benchkit-pi-aider-polyglot:{_PI_IMAGE_SESSION}" +GIT_SURGERY_PI_IMAGE = f"benchkit-pi-git-surgery:{_PI_IMAGE_SESSION}" +PI_PACKAGE = "@earendil-works/pi-coding-agent@0.84.2" +PI_VERSION = "0.84.2" AIDER_POLYGLOT_COMMIT = "7e0611e77b54e2dea774cdc0aa00cf9f7ed6144f" _PROXY_SOURCE = Path(__file__).with_name("_pi_proxy.py") _ANSWER_KEY_GUARD_SOURCE = Path(__file__).with_name("answer_key_guard.ts") +_PI_PACKAGE_ROOT = Path(__file__).with_name("pi_package") +_PATCHEVAL_IMAGE_SESSION = uuid.uuid4().hex[:12] +_BUILDKIT_DRIVER_IMAGE = "moby/buildkit:buildx-stable-1" + +# One label scopes every image, container, network, volume, and build-cache +# entry that this process creates, so cleanup can remove exactly its own +# resources instead of pruning Docker globally. +RUN_ID = uuid.uuid4().hex[:16] +RUN_LABEL = f"benchkit.run={RUN_ID}" +MANAGED_LABEL = "benchkit.managed=true" +_BUILDER = f"benchkit-build-{RUN_ID}" +_BUILDKIT_CONTAINER = f"buildx_buildkit_{_BUILDER}0" +_BUILDKIT_VOLUME = f"buildx_buildkit_{_BUILDER}0_state" +_UV_CACHE_ID = f"benchkit-uv-{RUN_ID}" +_UV_CACHE_DIR = "/root/.cache/uv" +_BUILD_LOCK = threading.Lock() +_BUILD_STATE: dict[str, object] = { + "builder": False, + "driver_was_present": True, + "pulled": set(), +} + + +def resource_labels(owner_id: str | None = None) -> list[str]: + """Return the label flags every managed Docker resource must carry.""" + labels = ["--label", MANAGED_LABEL, "--label", RUN_LABEL] + if owner_id is not None: + labels.extend(["--label", f"benchkit.owner={owner_id}"]) + return labels + + +_PI_INSTALL = """\ +COPY pi-package/package.json pi-package/package-lock.json /opt/benchkit/pi/ +RUN cd /opt/benchkit/pi \\ + && npm ci --omit=dev \\ + && ln -s /opt/benchkit/pi/node_modules/.bin/pi /usr/local/bin/pi +""" PI_DOCKERFILE = f"""\ FROM node:24-bookworm-slim @@ -29,7 +71,7 @@ RUN apt-get update \\ && apt-get install -y --no-install-recommends bash ca-certificates git python3 ripgrep \\ && rm -rf /var/lib/apt/lists/* -RUN npm install -g {PI_PACKAGE} +{_PI_INSTALL} COPY inference_proxy.py /opt/benchkit/inference_proxy.py COPY answer_key_guard.ts /opt/benchkit/answer_key_guard.ts @@ -51,7 +93,7 @@ openjdk-17-jdk \\ python3 ripgrep unzip \\ && rm -rf /var/lib/apt/lists/* -RUN npm install -g {PI_PACKAGE} +{_PI_INSTALL} ARG TARGETARCH RUN curl -fsSL "https://go.dev/dl/go1.21.5.linux-${{TARGETARCH}}.tar.gz" \\ -o /tmp/go.tar.gz \\ @@ -121,7 +163,7 @@ bash ca-certificates git=${{GIT_DEBIAN_VERSION}} python3 ripgrep \\ && test "$(git --version)" = "git version 2.39.5" \\ && rm -rf /var/lib/apt/lists/* -RUN npm install -g {PI_PACKAGE} +{_PI_INSTALL} COPY inference_proxy.py /opt/benchkit/inference_proxy.py COPY answer_key_guard.ts /opt/benchkit/answer_key_guard.ts @@ -168,6 +210,7 @@ def _run( args, input=input_text, text=True, + errors="replace", capture_output=True, timeout=timeout, ) @@ -196,41 +239,81 @@ def _docker_upstream(url: str) -> str: def cleanup_owned_resources(docker: str, owner_id: str) -> None: """Remove every container and network belonging to one Pi runner.""" label = f"benchkit.owner={owner_id}" - containers = _run( - [docker, "ps", "--all", "--quiet", "--filter", f"label={label}"], - timeout=30, - check=False, - ).stdout.split() + errors: list[str] = [] + + def listed(command: list[str], kind: str) -> list[str]: + try: + result = _run(command, timeout=30, check=False) + except Exception as exc: + errors.append(f"could not inspect owned {kind}: {exc}") + return [] + if result.returncode: + detail = _tail(result.stderr or result.stdout) or "unknown Docker error" + errors.append(f"could not inspect owned {kind}: {detail}") + return [] + return result.stdout.split() + + container_query = [ + docker, + "ps", + "--all", + "--quiet", + "--filter", + f"label={label}", + ] + network_query = [ + docker, + "network", + "ls", + "--quiet", + "--filter", + f"label={label}", + ] + containers = listed(container_query, "containers") if containers: - _run( - [docker, "rm", "--force", *containers], - timeout=30, - check=False, - ) - networks = _run( - [docker, "network", "ls", "--quiet", "--filter", f"label={label}"], - timeout=30, - check=False, - ).stdout.split() + with contextlib.suppress(Exception): + _run([docker, "rm", "--force", *containers], timeout=30, check=False) + networks = listed(network_query, "networks") if networks: - _run( - [docker, "network", "rm", *networks], - timeout=30, - check=False, - ) + with contextlib.suppress(Exception): + _run([docker, "network", "rm", *networks], timeout=30, check=False) + + remaining_containers = listed(container_query, "containers after cleanup") + remaining_networks = listed(network_query, "networks after cleanup") + if remaining_containers: + errors.append("owned containers remain: " + ", ".join(remaining_containers)) + if remaining_networks: + errors.append("owned networks remain: " + ", ".join(remaining_networks)) + if errors: + raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors)) + + +def _verify_absent( + command: list[str], description: str, errors: list[str], *, timeout: int = 30 +) -> None: + """Record an error unless an exact Docker resource is confirmed absent.""" + try: + result = _run(command, timeout=timeout, check=False) + except Exception as exc: + errors.append(f"could not verify {description}: {exc}") + return + if result.returncode == 0: + errors.append(f"{description} remains") @dataclass class LatestPiImage: - """Build the stock Pi image once per run, resolving npm's latest release.""" + """Build the reproducibly pinned stock Pi image once per run.""" docker: str = field(default_factory=_docker_binary) image: str = PI_IMAGE dockerfile: str = PI_DOCKERFILE - no_cache: bool = True transient: bool = True pids_limit: int = 256 build_assets: Path | None = None + build_files: tuple[tuple[Path, str], ...] = () + always_cleanup_image: bool = True + resource_scope: str = "pi" answer_key_guard: bool = False version: str = "" _ready: bool = field(default=False, init=False, repr=False) @@ -259,31 +342,315 @@ def prepare(self) -> str: _ANSWER_KEY_GUARD_SOURCE.read_text(encoding="utf-8"), encoding="utf-8", ) + shutil.copytree(_PI_PACKAGE_ROOT, context / "pi-package") if self.build_assets is not None: shutil.copytree(self.build_assets, context / "git-surgery") - command = [self.docker, "build", "--pull"] - if self.no_cache: - command.append("--no-cache") - command.extend(["--tag", self.image, str(context)]) - _run(command, timeout=1800) - result = _run( - [self.docker, "run", "--rm", self.image, "pi", "--version"], - timeout=30, - ) - self.version = result.stdout.strip() or "latest" + allowed = { + "Dockerfile", + "inference_proxy.py", + "answer_key_guard.ts", + "pi-package", + "git-surgery", + } + for source, destination in self.build_files: + target = PurePosixPath(destination) + if ( + target.is_absolute() + or len(target.parts) != 1 + or target.name in allowed + or source.is_symlink() + or not source.is_file() + ): + raise SandboxError("invalid explicit Pi build-context file") + shutil.copyfile(source, context / target.name) + allowed.add(target.name) + dockerignore = ["*", *sorted(f"!{item}" for item in allowed)] + dockerignore.extend(("!pi-package/**", "!git-surgery/**")) + (context / ".dockerignore").write_text( + "\n".join(dockerignore) + "\n", encoding="utf-8" + ) + try: + _build_with_run_builder( + self.docker, + self.image, + self.dockerfile, + context, + ) + smoke = ( + f"benchkit-{self.resource_scope}-smoke-{uuid.uuid4().hex[:16]}" + ) + try: + result = _run( + [ + self.docker, + "run", + "--name", + smoke, + "--rm", + *resource_labels(), + self.image, + "pi", + "--version", + ], + timeout=30, + ) + finally: + with contextlib.suppress(Exception): + _run( + [self.docker, "rm", "--force", smoke], + timeout=30, + check=False, + ) + self.version = result.stdout.strip() or "latest" + if self.version != PI_VERSION: + raise SandboxError( + f"Pi image reported {self.version!r}; " + f"expected {PI_VERSION!r}" + ) + except BaseException: + if self.transient: + with contextlib.suppress(Exception): + _run( + [ + self.docker, + "image", + "rm", + "--force", + self.image, + ], + timeout=60, + check=False, + ) + self.version = "" + raise self._ready = True return self.version def cleanup(self) -> None: """Remove the transient Pi image after its benchmark run.""" - if not self._ready or not self.transient: + if not self.transient or (not self._ready and not self.always_cleanup_image): return - _run( - [self.docker, "image", "rm", "--force", self.image], - timeout=60, + try: + _run( + [self.docker, "image", "rm", "--force", self.image], + timeout=60, + check=not self.always_cleanup_image, + ) + if self.always_cleanup_image: + errors: list[str] = [] + _verify_absent( + [self.docker, "image", "inspect", self.image], + f"transient image {self.image}", + errors, + ) + if errors: + raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors)) + finally: + self._ready = False + self.version = "" + + +def _base_image_refs(dockerfile: str) -> tuple[str, ...]: + """Return the external images a Dockerfile pulls, ignoring its own stages.""" + stages: set[str] = set() + refs: list[str] = [] + for line in dockerfile.splitlines(): + match = re.match(r"\s*FROM\s+(\S+)(?:\s+[Aa][Ss]\s+(\S+))?\s*$", line) + if match is None: + continue + reference, alias = match.group(1), match.group(2) + if reference not in stages and reference not in refs: + refs.append(reference) + if alias: + stages.add(alias) + return tuple(refs) + + +def _run_builder(docker: str) -> str: + """Create this run's single private Buildx builder on first use. + + Every build in the run shares one builder, so layers and cache mounts are + reused instead of rebuilt per task. The builder keeps its BuildKit state in + its own container and volume, which cleanup removes by exact name. That is + what keeps a run's build cache separable from the host's own cache without + ever running a global prune. + """ + with _BUILD_LOCK: + if not _BUILD_STATE["builder"]: + _BUILD_STATE["driver_was_present"] = bool( + _run( + [ + docker, + "image", + "inspect", + "--format", + "{{.Id}}", + _BUILDKIT_DRIVER_IMAGE, + ], + timeout=30, + check=False, + ).stdout.strip() + ) + _run( + [ + docker, + "buildx", + "create", + "--name", + _BUILDER, + "--driver", + "docker-container", + "--driver-opt", + f"image={_BUILDKIT_DRIVER_IMAGE}", + ], + timeout=60, + ) + _BUILD_STATE["builder"] = True + return _BUILDER + + +def _build_with_run_builder( + docker: str, + image: str, + dockerfile: str, + context: Path, +) -> None: + """Build one image on the run's shared builder, pulling bases once.""" + builder = _run_builder(docker) + with _BUILD_LOCK: + pulled = _BUILD_STATE["pulled"] + assert isinstance(pulled, set) + fresh = [ + reference + for reference in _base_image_refs(dockerfile) + if reference not in pulled + ] + pulled.update(fresh) + _run( + [docker, "image", "rm", "--force", image], + timeout=60, + check=False, + ) + build = [docker, "buildx", "build", "--builder", builder] + if fresh: + # Recipe and harness tags are pinned, so one pull per run is enough. + build.append("--pull") + build.extend( + [ + "--label", + MANAGED_LABEL, + "--label", + RUN_LABEL, + "--load", + "--tag", + image, + str(context), + ] + ) + _run(build, timeout=1800) + + +def _remove_labelled(docker: str, errors: list[str]) -> None: + """Remove every container, network, volume, and image of this run.""" + queries = ( + ( + "containers", + [docker, "ps", "--all", "--quiet", "--filter", f"label={RUN_LABEL}"], + lambda ids: [docker, "rm", "--force", *ids], + ), + ( + "networks", + [docker, "network", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + lambda ids: [docker, "network", "rm", *ids], + ), + ( + "volumes", + [docker, "volume", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + lambda ids: [docker, "volume", "rm", "--force", *ids], + ), + ( + "images", + [docker, "image", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + lambda ids: [docker, "image", "rm", "--force", *ids], + ), + ) + for kind, query, removal in queries: + try: + listed = _run(query, timeout=30, check=False) + except Exception as exc: + errors.append(f"could not inspect run {kind}: {exc}") + continue + if listed.returncode: + detail = _tail(listed.stderr or listed.stdout) or "unknown Docker error" + errors.append(f"could not inspect run {kind}: {detail}") + continue + identifiers = sorted(set(listed.stdout.split())) + if not identifiers: + continue + with contextlib.suppress(Exception): + _run(removal(identifiers), timeout=120, check=False) + remaining = _run(query, timeout=30, check=False) + if remaining.returncode == 0 and remaining.stdout.split(): + errors.append( + f"run {kind} remain: " + + ", ".join(sorted(set(remaining.stdout.split()))) + ) + + +def cleanup_run_resources(docker: str | None = None) -> None: + """Tear down every Docker resource this run created, and nothing else.""" + docker = docker or _docker_binary() + errors: list[str] = [] + with _BUILD_LOCK: + builder_created = bool(_BUILD_STATE["builder"]) + driver_was_present = bool(_BUILD_STATE["driver_was_present"]) + _BUILD_STATE["builder"] = False + pulled = _BUILD_STATE["pulled"] + assert isinstance(pulled, set) + pulled.clear() + if builder_created: + # Removing the builder drops its BuildKit state volume, and with it + # every layer and uv cache entry the run created. + cleanup_commands = ( + ([docker, "buildx", "rm", "--force", _BUILDER], 60), + ([docker, "rm", "--force", _BUILDKIT_CONTAINER], 30), + ([docker, "volume", "rm", "--force", _BUILDKIT_VOLUME], 30), + ) + for command, timeout in cleanup_commands: + with contextlib.suppress(Exception): + _run(command, timeout=timeout, check=False) + if not driver_was_present: + with contextlib.suppress(Exception): + _run( + [docker, "image", "rm", "--force", _BUILDKIT_DRIVER_IMAGE], + timeout=60, + check=False, + ) + _remove_labelled(docker, errors) + if builder_created: + _verify_absent( + [docker, "buildx", "inspect", _BUILDER], + f"Buildx builder {_BUILDER}", + errors, + ) + _verify_absent( + [docker, "container", "inspect", _BUILDKIT_CONTAINER], + f"BuildKit container {_BUILDKIT_CONTAINER}", + errors, ) - self._ready = False - self.version = "" + _verify_absent( + [docker, "volume", "inspect", _BUILDKIT_VOLUME], + f"BuildKit volume {_BUILDKIT_VOLUME}", + errors, + ) + if not driver_was_present: + _verify_absent( + [docker, "image", "inspect", _BUILDKIT_DRIVER_IMAGE], + f"BuildKit driver image {_BUILDKIT_DRIVER_IMAGE}", + errors, + ) + if errors: + raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors)) def aider_pi_image() -> LatestPiImage: @@ -291,13 +658,13 @@ def aider_pi_image() -> LatestPiImage: return LatestPiImage( image=AIDER_PI_IMAGE, dockerfile=AIDER_PI_DOCKERFILE, - no_cache=False, transient=True, # cpp/bank-account creates 1,000 simultaneous std::threads. Linux # accounts threads against Docker's PID cgroup, so the generic Pi # sandbox limit of 256 makes the official test suite impossible. pids_limit=2048, answer_key_guard=True, + resource_scope="aider-polyglot", ) @@ -306,9 +673,163 @@ def git_surgery_pi_image() -> LatestPiImage: return LatestPiImage( image=GIT_SURGERY_PI_IMAGE, dockerfile=GIT_SURGERY_PI_DOCKERFILE, - no_cache=False, transient=True, build_assets=Path(__file__).with_name("git_surgery"), + resource_scope="git-surgery", + ) + + +@dataclass(frozen=True) +class PatchEvalRuntimeRecipe: + """Trusted build instructions shipped with one validated PatchEval task.""" + + base_image: str + sync_command: tuple[str, ...] + bootstrap_command: tuple[str, ...] = () + environment: tuple[str, ...] = () + schema_version: int = 1 + + +def _regular_file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _tree_sha256(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ValueError("Pi package build assets must not contain symlinks") + if not path.is_file(): + continue + relative = path.relative_to(root).as_posix().encode() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(bytes.fromhex(_regular_file_sha256(path))) + return digest.hexdigest() + + +def _docker_run(command: tuple[str, ...]) -> str: + return f"RUN {json.dumps(list(command), separators=(',', ':'))}\n" + + +def _docker_run_cached(command: tuple[str, ...], uv_cache_id: str) -> str: + """Run a build command with the run's shared, private uv cache mounted.""" + mount = f"--mount=type=cache,target={_UV_CACHE_DIR},id={uv_cache_id},sharing=locked" + return f"RUN {mount} {json.dumps(list(command), separators=(',', ':'))}\n" + + +def _patcheval_dockerfile(recipe: PatchEvalRuntimeRecipe, uv_cache_id: str) -> str: + """Generate the three-stage task runtime. + + The stages are split by what actually changes between tasks: + + 1. ``benchkit-pi-assets`` depends only on the base image and the pinned Pi + assets, so the run's shared builder builds it once. + 2. ``benchkit-runtime`` adds the parent source and the frozen sync and + bootstrap commands, and reuses the run's uv cache. + 3. the final stage only adds task environment and the agent user. + """ + bootstrap = ( + _docker_run_cached(recipe.bootstrap_command, uv_cache_id) + if recipe.bootstrap_command + else "" + ) + environment = "".join( + f"ENV {key}={json.dumps(value)}\n" + for key, value in (item.split("=", 1) for item in recipe.environment) + ) + return f"""\ +FROM node:24-bookworm-slim AS benchkit-node + +FROM {recipe.base_image} AS benchkit-pi-assets + +USER root +COPY --from=benchkit-node /usr/local/ /usr/local/ +RUN apt-get update \\ + && apt-get install -y --no-install-recommends \\ + bash ca-certificates coreutils git passwd ripgrep tar \\ + && rm -rf /var/lib/apt/lists/* \\ + && (getent group node >/dev/null || groupadd --gid 1000 node) \\ + && (id --user node >/dev/null 2>&1 || \\ + useradd --uid 1000 --gid node --create-home --shell /bin/bash node) +{_PI_INSTALL} +COPY inference_proxy.py /opt/benchkit/inference_proxy.py +COPY answer_key_guard.ts /opt/benchkit/answer_key_guard.ts +RUN mkdir -p /workspace /home/node/.pi/agent \\ + && chown -R node:node /workspace /home/node/.pi + +FROM benchkit-pi-assets AS benchkit-runtime + +ENV UV_PROJECT_ENVIRONMENT=/opt/venv UV_LINK_MODE=copy +WORKDIR /opt/project +COPY parent-source.tar /tmp/patcheval-parent.tar +RUN tar -xf /tmp/patcheval-parent.tar -C /opt/project \\ + && find /opt/project -exec touch -t 198001010000 {{}} + \\ + && rm /tmp/patcheval-parent.tar +{_docker_run_cached(recipe.sync_command, uv_cache_id)}{bootstrap}RUN rm -rf /opt/project \\ + && test -d /opt/venv \\ + && chmod -R a+rwX /opt/venv + +FROM benchkit-runtime + +{environment}USER node +WORKDIR /workspace +ENV PATH=/opt/venv/bin:$PATH HOME=/home/node USER=node LOGNAME=node \\ + PI_OFFLINE=1 PI_TELEMETRY=0 +CMD ["sleep", "infinity"] +""" + + +def patcheval_pi_image( + source_archive: Path, + source_sha256: str, + recipe: PatchEvalRuntimeRecipe, +) -> LatestPiImage: + """Build one task runtime locally without retaining buildkit state.""" + if source_archive.is_symlink() or not source_archive.is_file(): + raise ValueError("PatchEval source archive must be a regular file") + if not re.fullmatch(r"[0-9a-f]{64}", source_sha256): + raise ValueError("PatchEval source_sha256 must be a lowercase SHA-256") + if _regular_file_sha256(source_archive) != source_sha256: + raise ValueError("PatchEval source archive checksum mismatch") + # Hash a run-independent Dockerfile so the same task keeps one identity + # across runs even though the uv cache mount is run-scoped. + dockerfile = _patcheval_dockerfile(recipe, _UV_CACHE_ID) + identity_dockerfile = _patcheval_dockerfile(recipe, "benchkit-uv") + recipe_json = json.dumps( + { + "schema_version": recipe.schema_version, + "base_image": recipe.base_image, + "sync_command": list(recipe.sync_command), + "bootstrap_command": list(recipe.bootstrap_command), + "environment": list(recipe.environment), + }, + sort_keys=True, + separators=(",", ":"), + ) + identity = hashlib.sha256() + for value in ( + identity_dockerfile, + source_sha256, + recipe_json, + _tree_sha256(_PI_PACKAGE_ROOT), + _regular_file_sha256(_PROXY_SOURCE), + _regular_file_sha256(_ANSWER_KEY_GUARD_SOURCE), + ): + identity.update(value.encode()) + identity.update(b"\0") + image_id = identity.hexdigest()[:16] + return LatestPiImage( + image=f"benchkit-pi-patcheval:{_PATCHEVAL_IMAGE_SESSION}-{image_id}", + dockerfile=dockerfile, + transient=True, + build_files=((source_archive, "parent-source.tar"),), + always_cleanup_image=True, + resource_scope="patcheval", ) @@ -346,19 +867,14 @@ def start(self) -> None: if not self.image.version: raise SandboxError("Pi image must be prepared before starting a task") try: - resource_labels = [ - "--label", - "benchkit.managed=true", - "--label", - f"benchkit.owner={self.owner_id}", - ] + labels = resource_labels(self.owner_id) _run( [ self.docker, "network", "create", "--internal", - *resource_labels, + *labels, self.network_name, ] ) @@ -368,7 +884,7 @@ def start(self) -> None: "--detach", "--name", self.proxy_name, - *resource_labels, + *labels, "--network", self.network_name, "--network-alias", @@ -387,6 +903,8 @@ def start(self) -> None: f"BENCHKIT_UPSTREAM={_docker_upstream(self.client.host)}", "--env", f"BENCHKIT_MODEL={self.model}", + "--env", + f"BENCHKIT_UPSTREAM_TIMEOUT={self.client.timeout}", ] if self.client.api_key: proxy_args.extend( @@ -408,7 +926,7 @@ def start(self) -> None: "--detach", "--name", self.container_name, - *resource_labels, + *labels, "--network", self.network_name, "--cap-drop", @@ -543,15 +1061,14 @@ def download(self, source: str, destination: Path) -> None: _run([self.docker, "cp", f"{self.container_name}:{source}", str(destination)]) def stop(self) -> None: - for name in (self.container_name, self.proxy_name): - _run( - [self.docker, "rm", "--force", name], - timeout=30, - check=False, - ) - _run( - [self.docker, "network", "rm", self.network_name], - timeout=30, - check=False, - ) - self._started = False + try: + commands = [ + [self.docker, "rm", "--force", self.container_name], + [self.docker, "rm", "--force", self.proxy_name], + [self.docker, "network", "rm", self.network_name], + ] + for command in commands: + with contextlib.suppress(Exception): + _run(command, timeout=30, check=False) + finally: + self._started = False diff --git a/tests/test_patcheval.py b/tests/test_patcheval.py new file mode 100644 index 0000000..feeaad3 --- /dev/null +++ b/tests/test_patcheval.py @@ -0,0 +1,559 @@ +"""Tests for PatchEval dataset validation and hidden grading boundaries.""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import tarfile +import tempfile +import threading +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from benchkit.benchmarks.patcheval import ( + _GENERIC_REPAIR, + PatchEval, + _grade_once, + _trusted_patch, +) +from benchkit.engine import Engine +from benchkit.sandbox import ( + _BUILD_STATE, + _BUILDER, + _UV_CACHE_ID, + PI_VERSION, + RUN_LABEL, + LatestPiImage, + PatchEvalRuntimeRecipe, + SandboxError, + patcheval_pi_image, +) + +_RUNTIME_RECIPE = { + "schema_version": 1, + "base_image": "ghcr.io/astral-sh/uv:0.12.1-python3.14-trixie-slim", + "sync_command": ["uv", "sync", "--frozen", "--no-install-project"], + "bootstrap_command": ["uv", "pip", "install", "setuptools==80.9.0"], + "environment": ["UV_OFFLINE=1", "PYTHONHASHSEED=0"], +} + + +def _digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _dataset(root: Path, *, validated: bool = True) -> dict: + source_root = root / "source" + source_root.mkdir() + (source_root / "module.py").write_text("VALUE = 1\n", encoding="utf-8") + tests = source_root / "tests" + tests.mkdir() + (tests / "test_existing.py").write_text("def test_old(): pass\n", encoding="utf-8") + archive = root / "source.tar.gz" + with tarfile.open(archive, "w:gz") as handle: + for path in sorted(source_root.rglob("*")): + handle.add(path, arcname=path.relative_to(source_root), recursive=False) + hidden = root / "hidden.patch" + hidden.write_text("hidden grader patch\n", encoding="utf-8") + record = { + "id": "owner-repo-123", + "repository": "owner/repo", + "issue_title": "Handle empty input", + "issue_body": "Calling parse with an empty value should return None.", + "runtime_recipe": _RUNTIME_RECIPE, + "source_archive": archive.name, + "hidden_test_patch": hidden.name, + "source_sha256": _digest(archive), + "hidden_test_sha256": _digest(hidden), + "fail_to_pass_command": ["python3", "-m", "pytest", "grader_test.py"], + "regression_command": ["python3", "-m", "pytest", "tests"], + "protected_globs": ["tests/**", "grader_test.py"], + "ignored_globs": [".venv/**", "*.egg-info/**"], + "timeout_s": 120, + "validated": validated, + } + (root / "dataset.json").write_text( + json.dumps({"schema_version": 1, "release": "pilot-20", "task_count": 1}), + encoding="utf-8", + ) + (root / "tasks.jsonl").write_text(json.dumps(record) + "\n", encoding="utf-8") + return record + + +class DownloadEnvironment: + def __init__(self, candidate: Path) -> None: + self.candidate = candidate + self.workdir = "/workspace/repo" + + def download(self, _source: str, destination: Path) -> None: + shutil.copytree(self.candidate, destination, dirs_exist_ok=True, symlinks=True) + + +def _docker_result(stdout: str = "", returncode: int = 0) -> SimpleNamespace: + """Stand in for the CompletedProcess that sandbox._run returns.""" + return SimpleNamespace(stdout=stdout, stderr="", returncode=returncode) + + +def _absent() -> SimpleNamespace: + """Docker's answer when an inspected resource does not exist.""" + return _docker_result(returncode=1) + + +def _reset_run_build_state() -> None: + """Forget the shared builder so each test starts from a clean run.""" + _BUILD_STATE["builder"] = False + _BUILD_STATE["driver_was_present"] = True + _BUILD_STATE["pulled"] = set() + + +class PatchEvalTests(unittest.TestCase): + def test_loads_only_validated_checksum_pinned_tasks(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + tasks = benchmark.load_tasks() + + self.assertEqual([task.id for task in tasks], ["owner-repo-123"]) + self.assertEqual(benchmark.task_count, 1) + self.assertEqual( + benchmark.result_metadata(None), {"dataset_release": "pilot-20"} + ) + prompt = benchmark.build_prompt(tasks[0]) + self.assertEqual( + prompt, + "Fix the following issue in the current repository. Inspect the code, " + "make the necessary changes, and verify your solution.\n\n" + "# Handle empty input\n\n" + "Calling parse with an empty value should return None.", + ) + self.assertNotIn("benchmark", prompt.lower()) + self.assertNotIn("hidden", prompt.lower()) + + def test_rejects_unvalidated_tasks(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root, validated=False) + with self.assertRaisesRegex(RuntimeError, "not miner-validated"): + PatchEval(root).load_tasks() + + def test_rejects_invalid_runtime_recipes(self) -> None: + invalid_recipes = ( + {**_RUNTIME_RECIPE, "schema_version": True}, + {**_RUNTIME_RECIPE, "schema_version": 2}, + {**_RUNTIME_RECIPE, "base_image": "python"}, + {**_RUNTIME_RECIPE, "base_image": "python:latest"}, + { + **_RUNTIME_RECIPE, + "base_image": "python@sha256:" + "a" * 64, + }, + {**_RUNTIME_RECIPE, "sync_command": []}, + {**_RUNTIME_RECIPE, "environment": ["NOT-VALID"]}, + {**_RUNTIME_RECIPE, "unexpected": True}, + ) + for runtime_recipe in invalid_recipes: + with ( + self.subTest(runtime_recipe=runtime_recipe), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + record = _dataset(root) + record["runtime_recipe"] = runtime_recipe + (root / "tasks.jsonl").write_text( + json.dumps(record) + "\n", encoding="utf-8" + ) + with self.assertRaisesRegex(RuntimeError, "runtime_recipe"): + PatchEval(root).load_tasks() + + def test_rejects_modified_dataset_assets(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + (root / "hidden.patch").write_text("changed", encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "checksum mismatch"): + PatchEval(root).load_tasks() + + def test_trusted_patch_ignores_agent_git_and_test_changes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + task = PatchEval(root).load_tasks()[0] + spec = task.metadata["spec"] + candidate = root / "candidate" + candidate.mkdir() + (candidate / "module.py").write_text("VALUE = 2\n", encoding="utf-8") + tests = candidate / "tests" + tests.mkdir() + (tests / "test_existing.py").write_text( + "def test_old(): assert False\n", encoding="utf-8" + ) + (tests / "test_agent.py").write_text( + "def test_new(): pass\n", encoding="utf-8" + ) + (candidate / ".git").mkdir() + (candidate / ".git" / "config").write_text( + "malicious metadata", encoding="utf-8" + ) + + submission, changed, excluded = _trusted_patch( + spec, DownloadEnvironment(candidate) + ) + + self.assertIn("module.py", submission) + self.assertNotIn("test_agent.py", submission) + self.assertEqual( + changed, + ["module.py", "tests/test_agent.py", "tests/test_existing.py"], + ) + self.assertEqual(excluded, ["tests/test_agent.py", "tests/test_existing.py"]) + + def test_protected_tests_with_unusual_names_are_still_excluded(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + task = PatchEval(root).load_tasks()[0] + spec = task.metadata["spec"] + candidate = root / "candidate" + candidate.mkdir() + (candidate / "module.py").write_text("VALUE = 2\n", encoding="utf-8") + tests = candidate / "tests" + tests.mkdir() + # Git quotes non-ASCII paths unless the diff is read with -z, and a + # quoted path matches no glob and no pathspec, so it is neither + # excluded nor actually diffed. + (tests / "test_caf\u00e9.py").write_text( + "def test_agent(): pass # AGENT_AUTHORED_MARKER\n", encoding="utf-8" + ) + + submission, changed, excluded = _trusted_patch( + spec, DownloadEnvironment(candidate) + ) + + self.assertIn("module.py", submission) + self.assertNotIn("AGENT_AUTHORED_MARKER", submission) + unusual = [path for path in excluded if path.startswith("tests/test_caf")] + self.assertEqual(len(unusual), 1) + self.assertNotIn('"', unusual[0]) + self.assertTrue(all('"' not in path for path in changed)) + + def test_standard_root_python_tests_are_always_excluded(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + task = PatchEval(root).load_tasks()[0] + spec = task.metadata["spec"] + candidate = root / "candidate" + candidate.mkdir() + (candidate / "module.py").write_text("VALUE = 2\n", encoding="utf-8") + shutil.copytree(root / "source" / "tests", candidate / "tests") + (candidate / "test_agent.py").write_text( + "def test_agent(): pass\n", encoding="utf-8" + ) + + submission, changed, excluded = _trusted_patch( + spec, DownloadEnvironment(candidate) + ) + + self.assertIn("module.py", submission) + self.assertNotIn("test_agent.py", submission) + self.assertIn("test_agent.py", changed) + self.assertIn("test_agent.py", excluded) + + def test_verifier_runs_hidden_and_regression_graders_separately(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + environment = SimpleNamespace( + image=SimpleNamespace(), owner_id="owner", workdir="/workspace/repo" + ) + hidden_failure = { + "exit_code": 1, + "output": "SECRET_EXPECTATION", + "output_truncated": False, + } + regression_pass = { + "exit_code": 0, + "output": "all old tests passed", + "output_truncated": False, + } + + def grade(*_args, hidden_tests: bool, **_kwargs): + return hidden_failure if hidden_tests else regression_pass + + with ( + patch( + "benchkit.benchmarks.patcheval._trusted_patch", + return_value=("diff", ["module.py"], []), + ), + patch( + "benchkit.benchmarks.patcheval._grade_once", + side_effect=grade, + ) as grade_once, + ): + result = benchmark.verify_workspace(task, environment) + + self.assertFalse(result.passed) + self.assertEqual(result.feedback, _GENERIC_REPAIR) + self.assertNotIn("SECRET_EXPECTATION", result.feedback) + self.assertEqual(grade_once.call_count, 2) + # The graders run at the same time, so identity comes from the + # keyword, not the call order. + self.assertEqual( + sorted(call.kwargs["hidden_tests"] for call in grade_once.call_args_list), + [False, True], + ) + self.assertEqual(result.details["f2p_exit_code"], 1) + self.assertEqual(result.details["regression_exit_code"], 0) + + def test_workspace_setup_exposes_source_but_not_hidden_patch(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + environment = Mock(workdir="/workspace", exec=Mock()) + + benchmark.prepare_workspace(task, environment) + + environment.upload.assert_called_once() + uploaded = environment.upload.call_args.args + self.assertEqual(uploaded[0].name, "source.tar.gz") + self.assertNotIn("hidden", str(uploaded[0])) + commands = [call.args[0] for call in environment.exec.call_args_list] + self.assertIn(["git", "init", "-q", "/workspace/repo"], commands) + self.assertTrue( + all("hidden.patch" not in " ".join(command) for command in commands) + ) + + def test_repair_prompt_is_generic_and_does_not_mention_grading(self) -> None: + prompt = PatchEval().build_repair_prompt("secret output", 1, 3) + + self.assertEqual(prompt, _GENERIC_REPAIR) + self.assertNotIn("verifier", prompt.lower()) + self.assertNotIn("test output", prompt.lower()) + + def test_engine_caches_one_runner_per_locally_built_task_image(self) -> None: + image = LatestPiImage( + docker="docker", image="benchkit-pi-patcheval:runtime", transient=False + ) + benchmark = SimpleNamespace( + name="patcheval", pi_image_for_task=Mock(return_value=image) + ) + task = SimpleNamespace() + engine = Engine(client=SimpleNamespace(), jobs=[]) + + first = engine._pi(benchmark, task) + second = engine._pi(benchmark, task) + + self.assertIs(first, second) + self.assertEqual(benchmark.pi_image_for_task.call_count, 2) + + def test_runtime_image_is_local_content_addressed_and_context_is_minimal( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + spec = task.metadata["spec"] + + image = benchmark.pi_image_for_task(task) + same = benchmark.pi_image_for_task(task) + changed_recipe = PatchEvalRuntimeRecipe( + **{ + **spec.runtime_recipe.__dict__, + "environment": (*spec.runtime_recipe.environment, "TZ=UTC"), + } + ) + changed = patcheval_pi_image( + spec.source_archive, spec.source_sha256, changed_recipe + ) + + self.assertEqual(image.image, same.image) + self.assertNotEqual(image.image, changed.image) + self.assertNotIn(_UV_CACHE_ID, image.image) + self.assertTrue(image.image.startswith("benchkit-pi-patcheval:")) + self.assertEqual( + image.build_files, + ((spec.source_archive, "parent-source.tar"),), + ) + self.assertTrue(image.transient) + self.assertTrue(image.always_cleanup_image) + self.assertIn("FROM node:24-bookworm-slim AS benchkit-node", image.dockerfile) + self.assertIn( + f"FROM {_RUNTIME_RECIPE['base_image']} AS benchkit-pi-assets", + image.dockerfile, + ) + self.assertIn("FROM benchkit-pi-assets AS benchkit-runtime", image.dockerfile) + self.assertIn("FROM benchkit-runtime\n", image.dockerfile) + self.assertIn("COPY parent-source.tar", image.dockerfile) + # Shared Pi assets come before any per-task input, so every task in a + # run reuses that layer instead of rebuilding it. + self.assertLess( + image.dockerfile.index("npm ci --omit=dev"), + image.dockerfile.index("COPY parent-source.tar"), + ) + # The dependency install reuses one run-scoped uv cache. + self.assertIn( + f"--mount=type=cache,target=/root/.cache/uv,id={_UV_CACHE_ID}," + 'sharing=locked ["uv","sync"', + image.dockerfile, + ) + self.assertIn("rm -rf /opt/project", image.dockerfile) + self.assertIn("HOME=/home/node USER=node LOGNAME=node", image.dockerfile) + self.assertGreater( + image.dockerfile.index('ENV UV_OFFLINE="1"'), + image.dockerfile.index('["uv","sync"'), + ) + self.assertNotIn("hidden.patch", image.dockerfile) + self.assertNotIn("gold", image.dockerfile.lower()) + + def test_task_builds_share_the_run_builder_with_a_minimal_context(self) -> None: + _reset_run_build_state() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + image = benchmark.pi_image_for_task(task) + observed_context: set[str] = set() + + def run(args, **_kwargs): + if args[2:3] == ["inspect"]: + return _absent() + if args[1:3] == ["buildx", "build"]: + context = Path(args[-1]) + observed_context.update( + path.relative_to(context).as_posix() + for path in context.rglob("*") + if path.is_file() + ) + return _docker_result(f"{PI_VERSION}\n") + + with patch("benchkit.sandbox._run", side_effect=run) as docker_run: + self.assertEqual(image.prepare(), PI_VERSION) + image.cleanup() + + commands = [call.args[0] for call in docker_run.call_args_list] + docker = commands[0][0] + build = next( + command for command in commands if command[1:3] == ["buildx", "build"] + ) + self.assertEqual(build[build.index("--builder") + 1], _BUILDER) + self.assertNotIn("--no-cache", build) + self.assertIn(RUN_LABEL, build) + self.assertEqual( + commands[-2:], + [ + [docker, "image", "rm", "--force", image.image], + [docker, "image", "inspect", image.image], + ], + ) + self.assertIn("parent-source.tar", observed_context) + self.assertIn("Dockerfile", observed_context) + self.assertFalse(any("hidden" in path for path in observed_context)) + self.assertFalse(any("gold" in path for path in observed_context)) + + def test_failed_build_removes_the_partial_image(self) -> None: + _reset_run_build_state() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + image = benchmark.pi_image_for_task(benchmark.load_tasks()[0]) + + def run(args, **_kwargs): + if args[1:3] == ["buildx", "build"]: + raise SandboxError("build failed") + if args[2:3] == ["inspect"]: + return _absent() + return _docker_result(f"{PI_VERSION}\n") + + with patch("benchkit.sandbox._run", side_effect=run) as docker_run: + with self.assertRaisesRegex(SandboxError, "build failed"): + image.prepare() + image.cleanup() + + commands = [call.args[0] for call in docker_run.call_args_list] + docker = commands[0][0] + image_removals = [ + command + for command in commands + if command == [docker, "image", "rm", "--force", image.image] + ] + self.assertGreaterEqual(len(image_removals), 2) + self.assertFalse(any("prune" in command for command in commands)) + + def test_graders_run_concurrently_on_the_same_task_image(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + image = LatestPiImage( + docker="docker", image="benchkit-pi-patcheval:test", version="0.84.2" + ) + environment = SimpleNamespace( + image=image, owner_id="owner-123", workdir="/workspace/repo" + ) + started = threading.Barrier(2, timeout=5) + + def grade(_spec, _image, _patch, command, **_kwargs): + # Both graders must be in flight at the same time. + started.wait() + return {"exit_code": 0, "output": "", "output_truncated": False} + + with ( + patch( + "benchkit.benchmarks.patcheval._trusted_patch", + return_value=("patch", ["module.py"], []), + ), + patch( + "benchkit.benchmarks.patcheval._grade_once", side_effect=grade + ) as grade_once, + ): + result = benchmark.verify_workspace(task, environment) + + self.assertEqual(result.score, 1.0) + self.assertEqual(grade_once.call_count, 2) + + def test_grader_containers_carry_the_run_label(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _dataset(root) + benchmark = PatchEval(root) + task = benchmark.load_tasks()[0] + spec = task.metadata["spec"] + image = LatestPiImage( + docker="docker", image="benchkit-pi-patcheval:test", version="0.84.2" + ) + + with patch( + "benchkit.benchmarks.patcheval._run", + return_value=_docker_result(), + ) as run: + _grade_once( + spec, + image, + "patch", + spec.regression_command, + hidden_tests=False, + owner_id="owner-123", + ) + + create = next( + call.args[0] for call in run.call_args_list if call.args[0][1] == "run" + ) + self.assertIn(RUN_LABEL, create) + self.assertIn("benchkit.owner=owner-123", create) + self.assertIn("none", create) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pi_harness.py b/tests/test_pi_harness.py index 510120c..0c37ed1 100644 --- a/tests/test_pi_harness.py +++ b/tests/test_pi_harness.py @@ -11,56 +11,132 @@ from types import SimpleNamespace from unittest.mock import Mock, patch -from benchkit._pi_proxy import _capture_scaffold, _models_payload, _upstream_target +from benchkit._pi_proxy import ( + _capture_scaffold, + _models_payload, + _upstream_target, + _upstream_timeout, +) from benchkit.cli import _headless_jobs, _parse_args from benchkit.client import _openai_metrics from benchkit.engine import Engine, JobSpec, annotate_harness_effect from benchkit.evaluation import EvaluationResult from benchkit.pi_agent import PiAgentRunner, _RpcTrace from benchkit.sandbox import ( + _BUILD_STATE, + _BUILDER, + _BUILDKIT_CONTAINER, + _BUILDKIT_VOLUME, PI_DOCKERFILE, PI_PACKAGE, + PI_VERSION, + RUN_LABEL, DockerTaskEnvironment, LatestPiImage, + SandboxError, _docker_upstream, cleanup_owned_resources, + cleanup_run_resources, ) +def _docker_result(stdout: str = "", returncode: int = 0) -> SimpleNamespace: + """Stand in for the CompletedProcess that sandbox._run returns.""" + return SimpleNamespace(stdout=stdout, stderr="", returncode=returncode) + + +def _absent() -> SimpleNamespace: + """Docker's answer when an inspected resource does not exist.""" + return _docker_result(returncode=1) + + +def _reset_run_build_state() -> None: + """Forget the shared builder so each test starts from a clean run.""" + _BUILD_STATE["builder"] = False + _BUILD_STATE["driver_was_present"] = True + _BUILD_STATE["pulled"] = set() + + +def _docker_without_resources(args, **_kwargs) -> SimpleNamespace: + """Fake Docker where every command works and no resource is left behind.""" + if args[2:3] == ["inspect"]: + return _absent() + return _docker_result() + + class LatestPiImageTests(unittest.TestCase): def test_generic_pi_sandbox_keeps_the_restricted_pid_limit(self) -> None: self.assertEqual(LatestPiImage(docker="docker").pids_limit, 256) - def test_package_deliberately_tracks_npm_latest(self) -> None: - self.assertEqual(PI_PACKAGE, "@earendil-works/pi-coding-agent@latest") - self.assertIn(f"npm install -g {PI_PACKAGE}", PI_DOCKERFILE) + def test_package_and_transitive_dependencies_are_pinned(self) -> None: + self.assertEqual(PI_PACKAGE, "@earendil-works/pi-coding-agent@0.84.2") + self.assertEqual(PI_VERSION, "0.84.2") + self.assertIn("npm ci --omit=dev", PI_DOCKERFILE) + self.assertNotIn("@latest", PI_DOCKERFILE) + + def test_run_shares_one_cached_builder_and_pulls_bases_once(self) -> None: + _reset_run_build_state() + first = LatestPiImage(docker="docker", image="benchkit-pi:first") + second = LatestPiImage(docker="docker", image="benchkit-pi:second") - def test_prepare_pulls_and_bypasses_build_cache_once_per_run(self) -> None: + def docker_run(args, **_kwargs): + if args[1] == "run": + return _docker_result(f"{PI_VERSION}\n") + return _docker_without_resources(args) + + with patch("benchkit.sandbox._run", side_effect=docker_run) as run: + self.assertEqual(first.prepare(), PI_VERSION) + self.assertEqual(first.prepare(), PI_VERSION) + self.assertEqual(second.prepare(), PI_VERSION) + + commands = [call.args[0] for call in run.call_args_list] + creates = [ + command for command in commands if command[1:3] == ["buildx", "create"] + ] + builds = [ + command for command in commands if command[1:3] == ["buildx", "build"] + ] + self.assertEqual(len(creates), 1) + self.assertEqual(len(builds), 2) + self.assertEqual(creates[0][creates[0].index("--name") + 1], _BUILDER) + for build in builds: + self.assertEqual(build[build.index("--builder") + 1], _BUILDER) + self.assertNotIn("--no-cache", build) + self.assertIn(RUN_LABEL, build) + # The base image is pulled for the first build of the run only. + self.assertIn("--pull", builds[0]) + self.assertNotIn("--pull", builds[1]) + + def test_run_cleanup_removes_labelled_resources_and_the_builder(self) -> None: + _reset_run_build_state() image = LatestPiImage(docker="docker") - with patch("benchkit.sandbox._run") as run: - run.return_value = SimpleNamespace(stdout="0.83.0\n") - self.assertEqual(image.prepare(), "0.83.0") - self.assertEqual(image.prepare(), "0.83.0") + def docker_run(args, **_kwargs): + if args[1] == "run": + return _docker_result(f"{PI_VERSION}\n") + return _docker_without_resources(args) + + with patch("benchkit.sandbox._run", side_effect=docker_run): + image.prepare() + + with patch( + "benchkit.sandbox._run", side_effect=_docker_without_resources + ) as run: + cleanup_run_resources("docker") commands = [call.args[0] for call in run.call_args_list] - self.assertEqual(commands[0][:2], ["docker", "version"]) - build = commands[1] - self.assertEqual(build[:2], ["docker", "build"]) - self.assertIn("--pull", build) - self.assertIn("--no-cache", build) - self.assertEqual( - commands[2], - [ - "docker", - "run", - "--rm", - "benchkit-pi:latest", - "pi", - "--version", - ], - ) - self.assertEqual(len(commands), 3) + self.assertIn(["docker", "buildx", "rm", "--force", _BUILDER], commands) + self.assertIn(["docker", "rm", "--force", _BUILDKIT_CONTAINER], commands) + self.assertIn(["docker", "volume", "rm", "--force", _BUILDKIT_VOLUME], commands) + for query in ( + ["docker", "ps", "--all", "--quiet", "--filter", f"label={RUN_LABEL}"], + ["docker", "network", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + ["docker", "volume", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + ["docker", "image", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"], + ): + self.assertIn(query, commands) + # Cleanup must never reach for a global prune. + self.assertFalse(any("prune" in command for command in commands)) def test_cleanup_removes_the_transient_image(self) -> None: image = LatestPiImage( @@ -70,51 +146,113 @@ def test_cleanup_removes_the_transient_image(self) -> None: ) image._ready = True - with patch("benchkit.sandbox._run") as run: + with patch( + "benchkit.sandbox._run", side_effect=_docker_without_resources + ) as run: image.cleanup() - run.assert_called_once_with( - ["docker", "image", "rm", "--force", "benchkit-pi:latest"], - timeout=60, + self.assertEqual( + [call.args[0] for call in run.call_args_list], + [ + ["docker", "image", "rm", "--force", "benchkit-pi:latest"], + ["docker", "image", "inspect", "benchkit-pi:latest"], + ], ) self.assertFalse(image._ready) self.assertEqual(image.version, "") def test_cleanup_removes_every_resource_owned_by_the_runner(self) -> None: + container_query = [ + "docker", + "ps", + "--all", + "--quiet", + "--filter", + "label=benchkit.owner=owner-123", + ] + network_query = [ + "docker", + "network", + "ls", + "--quiet", + "--filter", + "label=benchkit.owner=owner-123", + ] responses = [ - SimpleNamespace(stdout="container-one\ncontainer-two\n"), - SimpleNamespace(stdout=""), - SimpleNamespace(stdout="network-one\n"), - SimpleNamespace(stdout=""), + _docker_result("container-one\ncontainer-two\n"), + _docker_result(), + _docker_result("network-one\n"), + _docker_result(), + # Cleanup lists both kinds again to prove nothing is left. + _docker_result(), + _docker_result(), ] with patch("benchkit.sandbox._run", side_effect=responses) as run: cleanup_owned_resources("docker", "owner-123") - commands = [call.args[0] for call in run.call_args_list] self.assertEqual( - commands, + [call.args[0] for call in run.call_args_list], [ - [ - "docker", - "ps", - "--all", - "--quiet", - "--filter", - "label=benchkit.owner=owner-123", - ], + container_query, ["docker", "rm", "--force", "container-one", "container-two"], - [ - "docker", - "network", - "ls", - "--quiet", - "--filter", - "label=benchkit.owner=owner-123", - ], + network_query, ["docker", "network", "rm", "network-one"], + container_query, + network_query, ], ) + def test_run_cleanup_continues_after_one_docker_error(self) -> None: + _reset_run_build_state() + image = LatestPiImage(docker="docker") + + def prepare_docker(args, **_kwargs): + if args[1] == "run": + return _docker_result(f"{PI_VERSION}\n") + return _docker_without_resources(args) + + with patch("benchkit.sandbox._run", side_effect=prepare_docker): + image.prepare() + + def cleanup_docker(args, **_kwargs): + if args[1:3] == ["buildx", "rm"]: + raise SandboxError("builder removal failed") + return _docker_without_resources(args) + + with patch("benchkit.sandbox._run", side_effect=cleanup_docker) as run: + cleanup_run_resources("docker") + + commands = [call.args[0] for call in run.call_args_list] + self.assertIn(["docker", "rm", "--force", _BUILDKIT_CONTAINER], commands) + self.assertIn(["docker", "volume", "rm", "--force", _BUILDKIT_VOLUME], commands) + + def test_environment_stop_attempts_every_owned_resource(self) -> None: + environment = DockerTaskEnvironment( + SimpleNamespace(), + "model", + LatestPiImage(docker="docker", version="test"), + docker="docker", + ) + environment._started = True + responses = [ + SandboxError("agent removal failed"), + SimpleNamespace(stdout=""), + SimpleNamespace(stdout=""), + ] + + with patch("benchkit.sandbox._run", side_effect=responses) as run: + environment.stop() + + self.assertEqual( + [call.args[0] for call in run.call_args_list], + [ + ["docker", "rm", "--force", environment.container_name], + ["docker", "rm", "--force", environment.proxy_name], + ["docker", "network", "rm", environment.network_name], + ], + ) + self.assertFalse(environment._started) + def test_local_inference_host_is_rewritten_for_docker(self) -> None: self.assertEqual( _docker_upstream("http://localhost:11434"), @@ -207,6 +345,7 @@ def test_task_container_has_no_host_mount_or_direct_egress(self) -> None: host="http://localhost:11434", api_key="real-secret", provider="openai", + timeout=60000.0, context_length=lambda _model: 32768, ) image = LatestPiImage(docker="docker", version="test") @@ -228,6 +367,7 @@ def test_task_container_has_no_host_mount_or_direct_egress(self) -> None: self.assertIn("benchkit.managed=true", command) self.assertIn(f"benchkit.owner={environment.owner_id}", command) self.assertIn("BENCHKIT_MODEL=selected/model", proxy) + self.assertIn("BENCHKIT_UPSTREAM_TIMEOUT=60000.0", proxy) self.assertIn("BENCHKIT_UPSTREAM_API_KEY=real-secret", proxy) self.assertNotIn("BENCHKIT_UPSTREAM_API_KEY=real-secret", agent) self.assertIn(environment.network_name, agent) @@ -247,6 +387,21 @@ def test_task_container_has_no_host_mount_or_direct_egress(self) -> None: ], ) + def test_proxy_uses_the_forwarded_inference_timeout(self) -> None: + with patch.dict( + os.environ, + {"BENCHKIT_UPSTREAM_TIMEOUT": "60000"}, + clear=False, + ): + self.assertEqual(_upstream_timeout(), 60000.0) + + with patch.dict( + os.environ, + {"BENCHKIT_UPSTREAM_TIMEOUT": "invalid"}, + clear=False, + ): + self.assertEqual(_upstream_timeout(), 600.0) + class InferenceProxyTests(unittest.TestCase): def test_proxy_captures_exact_system_prompt_and_available_tools(self) -> None: @@ -782,6 +937,22 @@ def test_engine_cleans_the_image_after_a_failed_pi_job(self) -> None: self.assertEqual(results, []) runner.cleanup.assert_called_once_with() + def test_engine_cleans_the_image_when_process_termination_unwinds(self) -> None: + runner = Mock() + engine = Engine( + client=SimpleNamespace(), + jobs=[JobSpec("model", "sanity", "1", harness="pi")], + ) + engine._pi_runner = runner + + with ( + patch.object(engine, "_run_job", side_effect=SystemExit(143)), + self.assertRaisesRegex(SystemExit, "143"), + ): + engine.run() + + runner.cleanup.assert_called_once_with() + def test_cli_both_creates_matching_direct_and_pi_jobs(self) -> None: args = _parse_args( [ diff --git a/tests/test_repair.py b/tests/test_repair.py index 7aabc5d..c56f98b 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -303,6 +303,7 @@ def test_pi_repair_reuses_one_rpc_process_and_workspace(self) -> None: EvaluationResult(1.0), ] ) + repair_prompt_builder = Mock(return_value="Continue fixing the issue.") with patch( "benchkit.pi_agent.DockerTaskEnvironment", @@ -313,6 +314,7 @@ def test_pi_repair_reuses_one_rpc_process_and_workspace(self) -> None: "original prompt", verifier=verifier, repair_attempts=1, + repair_prompt_builder=repair_prompt_builder, ) task_environment.assert_called_once() @@ -321,7 +323,8 @@ def test_pi_repair_reuses_one_rpc_process_and_workspace(self) -> None: prompts = [item for item in sent if item.get("type") == "prompt"] self.assertEqual(len(prompts), 2) self.assertEqual(prompts[0]["message"], "original prompt") - self.assertIn("Sanitized failure.", prompts[1]["message"]) + self.assertEqual(prompts[1]["message"], "Continue fixing the issue.") + repair_prompt_builder.assert_called_once_with("Sanitized failure.", 1, 1) self.assertEqual(result["response"], "right") self.assertTrue(result["repaired"]) self.assertEqual(result["repair_attempts_used"], 1) diff --git a/tests/test_runner.py b/tests/test_runner.py index 0e41c05..6013d87 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -3,7 +3,10 @@ from __future__ import annotations import io +import signal import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch from rich.console import Console @@ -18,7 +21,15 @@ TaskPhase, TaskRecord, ) -from benchkit.runner import _bar, _counters, _Glyphs, _LiveStats, _Reporter, _spread +from benchkit.runner import ( + _bar, + _counters, + _Glyphs, + _LiveStats, + _Reporter, + _spread, + run, +) def _record( @@ -265,5 +276,47 @@ def test_parallel_job_summary_keeps_parallel_throughput_metrics(self) -> None: self.assertIn("1.87x effective", output) +class HeadlessSignalTests(unittest.TestCase): + def test_sigterm_unwinds_engine_cleanup_and_restores_handlers(self) -> None: + console = _console(record=True) + installed: dict[int, object] = {} + original = object() + engine = Mock() + + def install(signum: int, handler: object) -> object: + installed[signum] = handler + return original + + def interrupt_run() -> list[dict]: + handler = installed[signal.SIGTERM] + assert callable(handler) + handler(signal.SIGTERM, None) + return [] + + engine.run.side_effect = interrupt_run + with ( + patch("benchkit.runner.Engine", return_value=engine), + patch("benchkit.runner.signal.signal", side_effect=install) as set_signal, + self.assertRaisesRegex(SystemExit, str(128 + signal.SIGTERM)), + ): + run( + SimpleNamespace(), + [JobSpec("model", "sanity", "1")], + console, + ) + + self.assertIn( + (signal.SIGTERM, signal.SIG_IGN), + [call.args for call in set_signal.call_args_list], + ) + restored = { + call.args[0]: call.args[1] + for call in set_signal.call_args_list + if call.args[1] is original + } + self.assertEqual(restored[signal.SIGINT], original) + self.assertEqual(restored[signal.SIGTERM], original) + + if __name__ == "__main__": unittest.main()