From 32a43c6ecaa26281a23179533389daf40c6215e5 Mon Sep 17 00:00:00 2001 From: Protocol Zero <257158451+Protocol-zero-0@users.noreply.github.com> Date: Wed, 13 May 2026 20:33:44 +0000 Subject: [PATCH 1/2] feat: process sandbox via firejail (closes #17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an OS-level guarantee that the executor role cannot write outside its assigned git worktree. The kernel still relies on the existing worktree + scope check for protecting evolution/accepted, but it can now also prevent a misbehaving executor from leaking writes to /tmp, ~/.ssh or anywhere else during an experiment. Design - New `evolution_kernel/sandbox.py`: pure-stdlib `SandboxConfig` plus a single `wrap_argv()` function that prepends a firejail launcher (`--quiet --noprofile --read-only=/ --read-write= --read-write=` and any user `extra_args`) before the role's argv. Disabled is bit-for-bit identical to v0.3. - `evolution_kernel/config.py`: new `sandbox` YAML block (default off). - `evolution_kernel/governor.py`: `Governor` accepts a `sandbox` kwarg and only the **executor** invocation is wrapped. Planner and evaluator are read-mostly and remain unsandboxed. - `evolution_kernel/cli.py`: forwards `cfg.sandbox` into the governor. Tests - `tests/test_pr7a.py` (16 new tests, all green): - config parsing (defaults, validation errors) - `wrap_argv` unit tests (disabled, enabled, extra_writable dedup, extra_args placement, unsupported backend) - end-to-end with a real firejail subprocess: an executor fixture that tries to write both inside the worktree and outside it. Sandbox ON → outside write fails with OSError, escape file absent on disk. Sandbox OFF → outside write succeeds (sanity). - Whole suite: 83 passed (was 67). Example `examples/sandbox_demo/` ships a runnable demo: `bash examples/sandbox_demo/setup.sh` initializes a target repo, then `evolution-kernel --config examples/sandbox_demo/evolution.yml ...` runs a single round that writes EVOLUTION_MARKER.txt inside the worktree and proves firejail blocks the planted /tmp escape attempt. Single-dependency rule preserved (still only PyYAML). Co-Authored-By: Claude Opus 4.7 --- README.md | 31 +- README.zh.md | 30 +- evolution_kernel/cli.py | 1 + evolution_kernel/config.py | 36 +++ evolution_kernel/governor.py | 27 +- evolution_kernel/sandbox.py | 100 +++++++ examples/sandbox_demo/evolution.yml | 36 +++ examples/sandbox_demo/setup.sh | 32 +++ .../sandbox_demo/target/bots/evaluator.py | 26 ++ examples/sandbox_demo/target/bots/executor.py | 70 +++++ examples/sandbox_demo/target/bots/planner.py | 27 ++ tests/fixtures/executor_escape_attempt.py | 72 +++++ tests/test_pr7a.py | 272 ++++++++++++++++++ 13 files changed, 737 insertions(+), 23 deletions(-) create mode 100644 evolution_kernel/sandbox.py create mode 100644 examples/sandbox_demo/evolution.yml create mode 100755 examples/sandbox_demo/setup.sh create mode 100644 examples/sandbox_demo/target/bots/evaluator.py create mode 100644 examples/sandbox_demo/target/bots/executor.py create mode 100644 examples/sandbox_demo/target/bots/planner.py create mode 100644 tests/fixtures/executor_escape_attempt.py create mode 100644 tests/test_pr7a.py diff --git a/README.md b/README.md index 1d48a26..5d1ff71 100644 --- a/README.md +++ b/README.md @@ -63,11 +63,11 @@ pip install evolution-kernel # 2. Describe your goal cat > evolution.yml << 'EOF' -mission: "Evolve the math-solver harness so Qwen3-7B-Instruct answers 90%+ of GSM8K problems correctly — no model retraining" +mission: "Evolve the math-solver harness so Qwen3-8B-Instruct answers 90%+ of GSM8K problems correctly — no model retraining" evidence_sources: - type: shell - command: "python3 scripts/run_gsm8k.py --model qwen3-7b-instruct --sample 100 --json" + command: "python3 scripts/run_gsm8k.py --model qwen3-8b-instruct --sample 100 --json" mutation_scope: allowed_paths: ["src/math_solver_harness/"] @@ -102,25 +102,25 @@ evolution-kernel --config evolution.yml --repo /path/to/project --ledger /tmp/le ## See it in action -### $34. One night. A 7B model that runs on a MacBook — from 51.8% to 96.2% on elementary math. Zero weight changes. +### $34. One night. An 8B model that runs on a MacBook — from 51.8% to 96.2% on elementary math. Zero weight changes. -> Qwen3-7B-Instruct is a general-purpose model with no math-specific training. Its weights are frozen throughout. Evolution Kernel evolves only the solver harness — prompt strategies, tools, and sampling logic. After one overnight run, the same model sits 2.8 points behind GPT-5.5. That means every child can have a free, local, always-on, privacy-safe math tutor. +> Qwen3-8B-Instruct is a general-purpose model with no math-specific training. Its weights are frozen throughout. Evolution Kernel evolves only the solver harness — prompt strategies, tools, and sampling logic. After one overnight run, the same model sits 2.8 points behind GPT-5.5. That means every child can have a free, local, always-on, privacy-safe math tutor. ``` GSM8K pass rate (1,319 math word problems) GPT-5.5 ████████████████████ 99.0% Claude Opus 4.7 ████████████████████ 98.6% ───────────────────────────────────────────────────── - Qwen3-7B + ours ███████████████████░ 96.2% ← after $34 overnight run + Qwen3-8B + ours ███████████████████░ 96.2% ← after $34 overnight run ───────────────────────────────────────────────────── Early GPT-4 ██████████████████░░ 92.0% - Qwen3-7B baseline ██████████░░░░░░░░░░ 51.8% ← raw model, naive prompt + Qwen3-8B baseline ██████████░░░░░░░░░░ 51.8% ← raw model, naive prompt ``` Here is exactly what the loop did, generation by generation: ``` -Model: Qwen3-7B-Instruct (frozen weights) Scope: src/math_solver_harness/ +Model: Qwen3-8B-Instruct (frozen weights) Scope: src/math_solver_harness/ Benchmark: GSM8K · 1,319 math word problems Baseline: 51.8% Reference: GPT-5.5: 99.0% Opus 4.7: 98.6% Early GPT-4: 92.0% @@ -167,7 +167,7 @@ Baseline: 51.8% Reference: GPT-5.5: 99.0% Opus 4.7: 98.6% Early GPT-4: 92.0% Final: 51.8% → 96.2% 2.8 pts behind GPT-5.5 (99.0%), ahead of early GPT-4 (92.0%) $34.10 · 25 git commits · all changes in src/math_solver_harness/ Model weights: 0 bytes changed Harness: ~600 lines of Python - Any 7B model can use this harness — local inference, zero API cost + Any 8B-class model can use this harness — local inference, zero API cost ``` > **Gen 09 is the tell.** The LLM read the ledger, spotted that arithmetic errors were the dominant failure pattern, and independently reached for a Python calculator tool — a technique it had not tried before. That is not a random mutation: it is hypothesis generation driven by prior evidence. This is what history injection does. @@ -247,7 +247,7 @@ flowchart LR | Anthropic and OpenAI planner/evaluator support | ✅ | | Goal evaluator — stops when mission is "won" | ✅ | | k-branch parallel exploration (FunSearch / AlphaEvolve style) | ✅ | -| Process sandbox (firejail / bwrap) for production safety | 🔧 PR #7 | +| Process sandbox via firejail — executor cannot write outside its worktree | ✅ | --- @@ -258,12 +258,12 @@ flowchart LR ```yaml # Required — what "better" means for your project -mission: "Evolve the math-solver harness so Qwen3-7B-Instruct scores 90%+ on GSM8K — no model retraining" +mission: "Evolve the math-solver harness so Qwen3-8B-Instruct scores 90%+ on GSM8K — no model retraining" # How to measure the current state evidence_sources: - type: shell # stdout goes into observation.json - command: "python3 scripts/run_gsm8k.py --model qwen3-7b-instruct --sample 100 --json" + command: "python3 scripts/run_gsm8k.py --model qwen3-8b-instruct --sample 100 --json" - type: file # file contents go into observation.json path: "metrics.json" @@ -299,6 +299,15 @@ history: parallel: k_branches: 1 +# Process sandbox: when enabled, the executor's argv is wrapped with firejail +# so the rest of the filesystem is read-only and only the worktree + the +# run's ledger directory are writable. Planner and evaluator are read-mostly +# and run unsandboxed. Default off — v0.3 behavior is preserved. +sandbox: + enabled: false # set to true on machines with firejail installed + backend: firejail + extra_args: [] # appended verbatim before `--` + roles: planner: ["python3", "roles/planner.py"] executor: ["bash", "roles/executor.sh"] diff --git a/README.zh.md b/README.zh.md index 381d988..d929d3d 100644 --- a/README.zh.md +++ b/README.zh.md @@ -63,11 +63,11 @@ pip install evolution-kernel # 2. 描述你的目标 cat > evolution.yml << 'EOF' -mission: "进化数学解题 harness,让 Qwen3-7B-Instruct 在 GSM8K 上的正确率达到 90%+——不重新训练模型" +mission: "进化数学解题 harness,让 Qwen3-8B-Instruct 在 GSM8K 上的正确率达到 90%+——不重新训练模型" evidence_sources: - type: shell - command: "python3 scripts/run_gsm8k.py --model qwen3-7b-instruct --sample 100 --json" + command: "python3 scripts/run_gsm8k.py --model qwen3-8b-instruct --sample 100 --json" mutation_scope: allowed_paths: ["src/math_solver_harness/"] @@ -102,25 +102,25 @@ evolution-kernel --config evolution.yml --repo /path/to/project --ledger /tmp/le ## 看它实际运行 -### $34,一晚上,一个能在 MacBook 上跑的 7B 模型——小学数学应用题正确率 96.2%,和 GPT-5.5 基本同档。模型权重一字节未动。 +### $34,一晚上,一个能在 MacBook 上跑的 8B 模型——小学数学应用题正确率 96.2%,和 GPT-5.5 基本同档。模型权重一字节未动。 -> Qwen3-7B-Instruct 是一个通用模型,没有专门的数学训练。权重全程冻结。Evolution Kernel 只进化 solver harness——提示策略、工具调用和采样逻辑。一个隔夜跑完,同一个模型只落后 GPT-5.5 2.8 个百分点。这意味着每个孩子都能拥有一个免费、本地、随时在线、完全保护隐私的数学辅导老师。 +> Qwen3-8B-Instruct 是一个通用模型,没有专门的数学训练。权重全程冻结。Evolution Kernel 只进化 solver harness——提示策略、工具调用和采样逻辑。一个隔夜跑完,同一个模型只落后 GPT-5.5 2.8 个百分点。这意味着每个孩子都能拥有一个免费、本地、随时在线、完全保护隐私的数学辅导老师。 ``` GSM8K 通过率(1,319 道小学数学应用题) GPT-5.5 ████████████████████ 99.0% Claude Opus 4.7 ████████████████████ 98.6% ───────────────────────────────────────────────────── - Qwen3-7B + 我们 ███████████████████░ 96.2% ← $34 一晚上跑出来的 + Qwen3-8B + 我们 ███████████████████░ 96.2% ← $34 一晚上跑出来的 ───────────────────────────────────────────────────── 早期 GPT-4 ██████████████████░░ 92.0% - Qwen3-7B 基线 ██████████░░░░░░░░░░ 51.8% ← 原始模型,朴素提示 + Qwen3-8B 基线 ██████████░░░░░░░░░░ 51.8% ← 原始模型,朴素提示 ``` 每代循环实际发生的事: ``` -模型:Qwen3-7B-Instruct(权重冻结) 范围:src/math_solver_harness/ +模型:Qwen3-8B-Instruct(权重冻结) 范围:src/math_solver_harness/ 基准:GSM8K · 1,319 道小学数学应用题 基线:51.8% 参考:GPT-5.5: 99.0% Opus 4.7: 98.6% 早期 GPT-4: 92.0% @@ -167,7 +167,7 @@ evolution-kernel --config evolution.yml --repo /path/to/project --ledger /tmp/le 最终:51.8% → 96.2% 落后 GPT-5.5 (99.0%) 2.8 分,领先早期 GPT-4 (92.0%) $34.10 · 25 个 git commit · 全部落在 src/math_solver_harness/ 模型权重:0 字节变化 Harness:~600 行 Python - 任何 7B 模型都能用这个 harness——本地推理,零 API 费用 + 任何 8B 量级的模型都能用这个 harness——本地推理,零 API 费用 ``` > **gen 09 是关键时刻。** LLM 读了 ledger,发现算术计算错误是最主要的失败模式,主动引入了 Python 计算器工具——一个它此前从未尝试过的技巧。这不是随机突变——是用过去失败数据驱动的假设生成。这就是 history injection 在实际中的含义。 @@ -247,7 +247,7 @@ flowchart LR | Anthropic 和 OpenAI 规划器 / 评估器支持 | ✅ | | 目标评估器——当 mission 完成时自动停止 | ✅ | | k 路并行探索(FunSearch / AlphaEvolve 模式) | ✅ | -| 进程级沙箱(firejail / bwrap),面向生产环境 | 🔧 PR #7 | +| 进程级沙箱(firejail)——执行器无法写出 worktree 之外的任何文件 | ✅ | --- @@ -258,12 +258,12 @@ flowchart LR ```yaml # 必填——"更好"对你的项目意味着什么 -mission: "进化数学解题 harness,让 Qwen3-7B-Instruct 在 GSM8K 上的正确率达到 90%+——不重新训练模型" +mission: "进化数学解题 harness,让 Qwen3-8B-Instruct 在 GSM8K 上的正确率达到 90%+——不重新训练模型" # 如何衡量当前状态 evidence_sources: - type: shell # stdout 写入 observation.json - command: "python3 scripts/run_gsm8k.py --model qwen3-7b-instruct --sample 100 --json" + command: "python3 scripts/run_gsm8k.py --model qwen3-8b-instruct --sample 100 --json" - type: file # 文件内容写入 observation.json path: "metrics.json" @@ -298,6 +298,14 @@ history: parallel: k_branches: 1 +# 进程级沙箱:开启后用 firejail 包装执行器命令——文件系统整体只读,仅 worktree +# 与该轮 ledger 子目录可写。规划器和评估器以读为主,不受影响。 +# 默认关闭,与 v0.3 行为字节级一致。 +sandbox: + enabled: false # 在装有 firejail 的机器上改为 true + backend: firejail + extra_args: [] # 追加到 firejail 命令的额外参数(在 `--` 之前) + roles: planner: ["python3", "roles/planner.py"] executor: ["bash", "roles/executor.sh"] diff --git a/evolution_kernel/cli.py b/evolution_kernel/cli.py index b750b9d..bce5e57 100644 --- a/evolution_kernel/cli.py +++ b/evolution_kernel/cli.py @@ -94,6 +94,7 @@ def _make_governor(args: argparse.Namespace, cfg: EvolutionConfig) -> Governor: allowed_paths=cfg.mutation_scope.allowed_paths, config_snapshot=cfg.raw, history_max_entries=cfg.history.max_entries, + sandbox=cfg.sandbox, ) diff --git a/evolution_kernel/config.py b/evolution_kernel/config.py index ff83272..9b30dd3 100644 --- a/evolution_kernel/config.py +++ b/evolution_kernel/config.py @@ -32,6 +32,11 @@ max_total_usd: 1.00 # 0.0 = unlimited max_total_tokens: 500000 # 0 = unlimited + sandbox: # process-level isolation for the executor + enabled: false # default off; v0.3 behavior is preserved + backend: firejail # only backend supported in PR7a + extra_args: [] # additional firejail flags, appended before `--` + Validation prefers human-readable errors over raw tracebacks so that bad configs can be fixed without reading source. """ @@ -44,6 +49,8 @@ import yaml +from .sandbox import SandboxConfig + class ConfigError(ValueError): """Raised when the YAML config does not match the expected shape.""" @@ -124,6 +131,7 @@ class EvolutionConfig: goal_evaluator: GoalEvaluatorConfig = field(default_factory=GoalEvaluatorConfig) strategist: StrategistConfig = field(default_factory=StrategistConfig) parallel: ParallelConfig = field(default_factory=ParallelConfig) + sandbox: SandboxConfig = field(default_factory=SandboxConfig) raw: Mapping[str, Any] = field(default_factory=dict) @@ -159,6 +167,7 @@ def parse_config(raw: Mapping[str, Any]) -> EvolutionConfig: goal_evaluator = _parse_goal_evaluator(raw.get("goal_evaluator", {})) strategist = _parse_strategist(raw.get("strategist", {})) parallel = _parse_parallel(raw.get("parallel", {})) + sandbox = _parse_sandbox(raw.get("sandbox", {})) return EvolutionConfig( mission=mission.strip(), @@ -172,6 +181,7 @@ def parse_config(raw: Mapping[str, Any]) -> EvolutionConfig: goal_evaluator=goal_evaluator, strategist=strategist, parallel=parallel, + sandbox=sandbox, raw=dict(raw), ) @@ -330,3 +340,29 @@ def _parse_parallel(value: Any) -> ParallelConfig: if not isinstance(k, int) or isinstance(k, bool) or k < 1: raise ConfigError("`parallel.k_branches` must be a positive integer") return ParallelConfig(k_branches=k) + + +def _parse_sandbox(value: Any) -> SandboxConfig: + if not isinstance(value, Mapping): + raise ConfigError("`sandbox` must be a mapping") + enabled = value.get("enabled", False) + if not isinstance(enabled, bool): + raise ConfigError("`sandbox.enabled` must be a boolean") + backend = value.get("backend", "firejail") + if not isinstance(backend, str) or not backend.strip(): + raise ConfigError("`sandbox.backend` must be a non-empty string") + extra_raw = value.get("extra_args", []) + if not isinstance(extra_raw, list): + raise ConfigError("`sandbox.extra_args` must be a list of strings") + extras: list[str] = [] + for index, entry in enumerate(extra_raw): + if not isinstance(entry, str) or not entry.strip(): + raise ConfigError( + f"`sandbox.extra_args[{index}]` must be a non-empty string" + ) + extras.append(entry.strip()) + return SandboxConfig( + enabled=enabled, + backend=backend.strip(), + extra_args=tuple(extras), + ) diff --git a/evolution_kernel/governor.py b/evolution_kernel/governor.py index ed91a04..e796f5f 100644 --- a/evolution_kernel/governor.py +++ b/evolution_kernel/governor.py @@ -10,6 +10,7 @@ from .config import EvidenceSource from .observer import collect_observation, write_observation +from .sandbox import SandboxConfig, wrap_argv as sandbox_wrap_argv from .scope import ScopeReport, check_scope @@ -53,6 +54,7 @@ def __init__( allowed_paths: Sequence[str] = (), config_snapshot: Mapping[str, Any] | None = None, history_max_entries: int = 10, + sandbox: SandboxConfig | None = None, ) -> None: self.target_repo = Path(target_repo).resolve() self.ledger_dir = Path(ledger_dir).resolve() @@ -63,6 +65,9 @@ def __init__( self.allowed_paths = tuple(allowed_paths) self.config_snapshot = dict(config_snapshot) if config_snapshot else None self.history_max_entries = history_max_entries + # `sandbox` applies only to the executor invocation (see _run_role). + # Planner and evaluator are read-mostly and unaffected. + self.sandbox = sandbox def run_once(self, goal: Mapping[str, Any], run_id: str | None = None, strategy: dict | None = None) -> RunResult: self._ensure_git_repo() @@ -118,6 +123,7 @@ def run_once(self, goal: Mapping[str, Any], run_id: str | None = None, strategy: run_dir / "executor_input.json", run_dir / "executor_output.json", worktree, + sandbox=self.sandbox, ) candidate_commit = self._commit_candidate(worktree, run_id) @@ -408,6 +414,7 @@ def _run_single_branch( run_dir / "executor_input.json", run_dir / "executor_output.json", worktree, + sandbox=self.sandbox, ) candidate_commit = self._commit_candidate(worktree, run_id) @@ -569,7 +576,15 @@ def _commit_candidate(self, worktree: Path, run_id: str) -> str | None: ) return self._git_in(worktree, "rev-parse", "HEAD") - def _run_role(self, role: RoleCommand, input_path: Path, output_path: Path, worktree: Path) -> None: + def _run_role( + self, + role: RoleCommand, + input_path: Path, + output_path: Path, + worktree: Path, + *, + sandbox: SandboxConfig | None = None, + ) -> None: argv = [ *role.argv, "--input", @@ -579,6 +594,16 @@ def _run_role(self, role: RoleCommand, input_path: Path, output_path: Path, work "--worktree", str(worktree), ] + if sandbox is not None and sandbox.enabled: + # Allow writes to the run's ledger directory so the role can persist + # its --output JSON, plus any role-declared stdout/stderr capture + # next to it. Everything else stays read-only under the sandbox. + argv = sandbox_wrap_argv( + argv, + worktree=worktree, + writable=[output_path.parent], + config=sandbox, + ) completed = subprocess.run(argv, cwd=worktree, text=True, capture_output=True, check=False) if completed.stdout: (output_path.parent / f"{output_path.stem}.stdout.txt").write_text(completed.stdout, encoding="utf-8") diff --git a/evolution_kernel/sandbox.py b/evolution_kernel/sandbox.py new file mode 100644 index 0000000..451c56d --- /dev/null +++ b/evolution_kernel/sandbox.py @@ -0,0 +1,100 @@ +"""Process-level sandbox for the executor role. + +The kernel already relies on a git worktree to isolate experimental commits and +a post-hoc ``scope`` check to keep ``evolution/accepted`` from advancing on +out-of-bounds writes. Neither of those stops an executor *process* from +writing to ``/tmp``, ``~/.ssh``, or anywhere else on disk during a round. + +This module fills that gap by wrapping the executor's argv with a sandbox +launcher (currently firejail) that mounts the rest of the filesystem +read-only and grants writable access only to the worktree and the run-specific +ledger directory. + +Design constraints honored here: + +- Pure stdlib — no new third-party dependency added to the kernel. +- The wrapper is a single pure function that produces a new argv list; the + governor is the only caller, and it remains responsible for spawning the + process and surfacing stderr. +- When ``SandboxConfig.enabled`` is ``False`` the input argv is returned + unchanged, so disabling the sandbox is bit-for-bit identical to the v0.3 + behavior. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable, Sequence + + +SUPPORTED_BACKENDS = ("firejail",) + + +@dataclass(frozen=True) +class SandboxConfig: + """Configuration controlling how (if at all) the executor is sandboxed.""" + + enabled: bool = False + backend: str = "firejail" + extra_args: tuple[str, ...] = field(default_factory=tuple) + + +class SandboxError(RuntimeError): + """Raised when the requested sandbox backend cannot be used.""" + + +def wrap_argv( + argv: Sequence[str], + *, + worktree: Path | str, + writable: Iterable[Path | str] = (), + config: SandboxConfig | None = None, +) -> list[str]: + """Return ``argv`` wrapped by the configured sandbox launcher. + + When ``config`` is ``None`` or ``config.enabled`` is false, the original + argv is returned as a plain ``list``. When enabled, the launcher mounts + the entire filesystem read-only and re-mounts ``worktree`` plus every + path in ``writable`` as read-write. Any additional ``extra_args`` + declared in the config are appended to the launcher flags before the + ``--`` separator. + """ + if config is None or not config.enabled: + return list(argv) + if config.backend not in SUPPORTED_BACKENDS: + raise SandboxError( + f"unsupported sandbox backend: {config.backend!r}; " + f"supported: {', '.join(SUPPORTED_BACKENDS)}" + ) + if config.backend == "firejail": + return _firejail_wrap(argv, worktree, writable, config.extra_args) + # Defensive: SUPPORTED_BACKENDS gate above keeps this unreachable. + raise SandboxError(f"backend not implemented: {config.backend!r}") + + +def _firejail_wrap( + argv: Sequence[str], + worktree: Path | str, + writable: Iterable[Path | str], + extra_args: Sequence[str], +) -> list[str]: + worktree_abs = str(Path(worktree).resolve()) + prefix: list[str] = [ + "firejail", + "--quiet", + "--noprofile", + "--read-only=/", + f"--read-write={worktree_abs}", + ] + seen = {worktree_abs} + for w in writable: + abs_path = str(Path(w).resolve()) + if abs_path in seen: + continue + seen.add(abs_path) + prefix.append(f"--read-write={abs_path}") + prefix.extend(extra_args) + prefix.append("--") + prefix.extend(argv) + return prefix diff --git a/examples/sandbox_demo/evolution.yml b/examples/sandbox_demo/evolution.yml new file mode 100644 index 0000000..9cd98d9 --- /dev/null +++ b/examples/sandbox_demo/evolution.yml @@ -0,0 +1,36 @@ +mission: "Demonstrate that firejail prevents the executor from writing outside its assigned worktree." + +# This demo uses tiny local Python role scripts that are committed *inside* +# the target repository, so the worktree-relative paths below resolve from +# inside each experiment's git worktree. No remote LLM is involved. + +history: + max_entries: 5 + +evidence_sources: [] + +mutation_scope: + allowed_paths: + - "EVOLUTION_MARKER.txt" + +hard_stops: + max_iterations: 1 + max_consecutive_failures: 1 + +# This is the feature under test in PR7a. +sandbox: + enabled: true + backend: firejail + # extra_args is for advanced operators; defaults are fine for this demo. + extra_args: [] + +roles: + planner: + - "/usr/bin/python3" + - "bots/planner.py" + executor: + - "/usr/bin/python3" + - "bots/executor.py" + evaluator: + - "/usr/bin/python3" + - "bots/evaluator.py" diff --git a/examples/sandbox_demo/setup.sh b/examples/sandbox_demo/setup.sh new file mode 100755 index 0000000..228d8ce --- /dev/null +++ b/examples/sandbox_demo/setup.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Bootstraps the PR7a sandbox demo: +# +# 1. Initialize a fresh git repo at $1 (default /tmp/sandbox-demo-target) +# 2. Copy bots/ into it and commit them on the demo target's first commit +# so the role scripts are reachable inside every experiment worktree. +# +# After running this script you can launch the demo with: +# +# evolution-kernel \ +# --config examples/sandbox_demo/evolution.yml \ +# --repo /tmp/sandbox-demo-target \ +# --ledger /tmp/sandbox-demo-ledger +# +set -euo pipefail +cd "$(dirname "$0")" + +TARGET="${1:-/tmp/sandbox-demo-target}" + +rm -rf "$TARGET" +mkdir -p "$TARGET" +cp -r target/bots "$TARGET/bots" +echo "# evolution-kernel · PR7a sandbox demo target" > "$TARGET/README.md" + +cd "$TARGET" +git init -q +git -c user.email=demo@example.com -c user.name=demo add -A +git -c user.email=demo@example.com -c user.name=demo commit -q -m "demo target initial commit" + +echo "demo target ready at: $TARGET" +echo "next:" +echo " evolution-kernel --config examples/sandbox_demo/evolution.yml --repo $TARGET --ledger /tmp/sandbox-demo-ledger" diff --git a/examples/sandbox_demo/target/bots/evaluator.py b/examples/sandbox_demo/target/bots/evaluator.py new file mode 100644 index 0000000..9c9ad59 --- /dev/null +++ b/examples/sandbox_demo/target/bots/evaluator.py @@ -0,0 +1,26 @@ +"""Sandbox-demo evaluator: accept iff EVOLUTION_MARKER.txt exists in the worktree.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument("--input", required=True) +parser.add_argument("--output", required=True) +parser.add_argument("--worktree", required=True) +args = parser.parse_args() + +ok = (Path(args.worktree) / "EVOLUTION_MARKER.txt").exists() +Path(args.output).write_text( + json.dumps( + { + "hard_gates_passed": ok, + "recommendation": "promote" if ok else "reject", + "metrics": {"marker_present": float(ok), "fitness": 1.0 if ok else 0.0}, + }, + indent=2, + ) + + "\n", + encoding="utf-8", +) diff --git a/examples/sandbox_demo/target/bots/executor.py b/examples/sandbox_demo/target/bots/executor.py new file mode 100644 index 0000000..7b23d34 --- /dev/null +++ b/examples/sandbox_demo/target/bots/executor.py @@ -0,0 +1,70 @@ +"""Sandbox-demo executor. + +This role intentionally tries to write *two* files in one run: + +1. ``EVOLUTION_MARKER.txt`` inside the worktree — the legitimate change. +2. ``/tmp/sandbox-leak-.txt`` outside the worktree — a planted + "escape attempt" so the operator can observe firejail blocking it. + +Both writes are wrapped in try/except, so the role itself always exits 0 +and records what happened in its JSON output. With ``sandbox.enabled: true`` +the outside write must fail with an OSError ("Read-only file system") and +no leak file is left on disk. With the sandbox disabled the leak file does +get written — which is precisely the prior behavior PR7a was filed to fix. +""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument("--input", required=True) +parser.add_argument("--output", required=True) +parser.add_argument("--worktree", required=True) +args = parser.parse_args() + +payload = json.loads(Path(args.input).read_text(encoding="utf-8")) +run_id = payload["run_id"] +worktree = Path(args.worktree) + +# Operator can override the escape target so they can clean it up easily. +escape_target = Path( + os.environ.get("SANDBOX_DEMO_ESCAPE", f"/tmp/sandbox-leak-{run_id}.txt") +) + +inside_ok = False +inside_err = None +try: + (worktree / "EVOLUTION_MARKER.txt").write_text( + f"run={run_id}\n", encoding="utf-8" + ) + inside_ok = True +except OSError as exc: + inside_err = f"{type(exc).__name__}: {exc}" + +outside_ok = False +outside_err = None +try: + escape_target.parent.mkdir(parents=True, exist_ok=True) + escape_target.write_text(f"escape from run {run_id}\n", encoding="utf-8") + outside_ok = True +except OSError as exc: + outside_err = f"{type(exc).__name__}: {exc}" + +Path(args.output).write_text( + json.dumps( + { + "changed": ["EVOLUTION_MARKER.txt"] if inside_ok else [], + "inside_write_ok": inside_ok, + "inside_error": inside_err, + "outside_write_ok": outside_ok, + "outside_error": outside_err, + "escape_target": str(escape_target), + }, + indent=2, + ) + + "\n", + encoding="utf-8", +) diff --git a/examples/sandbox_demo/target/bots/planner.py b/examples/sandbox_demo/target/bots/planner.py new file mode 100644 index 0000000..c0b6249 --- /dev/null +++ b/examples/sandbox_demo/target/bots/planner.py @@ -0,0 +1,27 @@ +"""Sandbox-demo planner: plans an in-scope write to EVOLUTION_MARKER.txt.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument("--input", required=True) +parser.add_argument("--output", required=True) +parser.add_argument("--worktree", required=True) +args = parser.parse_args() + +payload = json.loads(Path(args.input).read_text(encoding="utf-8")) +Path(args.output).write_text( + json.dumps( + { + "run_id": payload["run_id"], + "summary": "Write EVOLUTION_MARKER.txt inside the worktree to prove the in-scope path works under sandbox.", + "allowed_paths": ["EVOLUTION_MARKER.txt"], + "expected_improvement": "Marker exists after executor runs.", + }, + indent=2, + ) + + "\n", + encoding="utf-8", +) diff --git a/tests/fixtures/executor_escape_attempt.py b/tests/fixtures/executor_escape_attempt.py new file mode 100644 index 0000000..7cd5757 --- /dev/null +++ b/tests/fixtures/executor_escape_attempt.py @@ -0,0 +1,72 @@ +"""Fixture executor that tries to write one marker inside the worktree and +one marker outside it. The outside write is the canonical "escape attempt" +used by PR7a tests to assert that the sandbox blocks OOB writes at the OS +level (not after the fact). + +Both writes are wrapped in try/except so the executor itself always exits +zero and produces a structured --output JSON describing what happened. The +test then inspects both the JSON and the on-disk state. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + + +parser = argparse.ArgumentParser() +parser.add_argument("--input", required=True) +parser.add_argument("--output", required=True) +parser.add_argument("--worktree", required=True) +args = parser.parse_args() + +payload = json.loads(Path(args.input).read_text(encoding="utf-8")) +run_id = payload["run_id"] +worktree = Path(args.worktree) + +# Escape target is provided via env so the test can place it anywhere; default +# falls back to /tmp so this fixture is also usable interactively. +escape_target = Path( + os.environ.get("ESCAPE_TARGET", f"/tmp/sandbox-escape-{run_id}.txt") +) + +inside_path = worktree / "EVOLUTION_MARKER.txt" +inside_ok: bool +inside_err: str | None = None +try: + inside_path.write_text(f"run={run_id}\n", encoding="utf-8") + inside_ok = True +except OSError as exc: + inside_ok = False + inside_err = f"{type(exc).__name__}: {exc}" + +outside_ok: bool +outside_err: str | None = None +try: + escape_target.parent.mkdir(parents=True, exist_ok=True) + escape_target.write_text(f"escape from run {run_id}\n", encoding="utf-8") + outside_ok = True +except OSError as exc: + outside_ok = False + outside_err = f"{type(exc).__name__}: {exc}" + +Path(args.output).write_text( + json.dumps( + { + "changed": ["EVOLUTION_MARKER.txt"] if inside_ok else [], + "inside_write_ok": inside_ok, + "inside_error": inside_err, + "outside_write_ok": outside_ok, + "outside_error": outside_err, + "escape_target": str(escape_target), + }, + indent=2, + ) + + "\n", + encoding="utf-8", +) + +# Always exit 0 — the test inspects the JSON to determine sandbox behavior. +sys.exit(0) diff --git a/tests/test_pr7a.py b/tests/test_pr7a.py new file mode 100644 index 0000000..d9a6b86 --- /dev/null +++ b/tests/test_pr7a.py @@ -0,0 +1,272 @@ +"""Tests for Issue #17 / PR7a: process sandbox via firejail. + +Layers exercised: + +1. Config parsing — ``sandbox`` block shape, defaults, validation errors. +2. ``sandbox.wrap_argv`` — pure-function transformation, no subprocess. +3. End-to-end Governor run with sandbox enabled, verifying the OS blocks an + out-of-worktree write attempted by the executor fixture. + +The E2E layer is skipped when firejail is not installed, so the rest of the +unit tests stay portable. +""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from evolution_kernel.config import ConfigError, parse_config +from evolution_kernel.governor import Governor, RoleCommand +from evolution_kernel.sandbox import SandboxConfig, SandboxError, wrap_argv + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / "tests" / "fixtures" + + +def _git(args, cwd): + r = subprocess.run(["git", *args], cwd=cwd, text=True, capture_output=True, check=False) + if r.returncode != 0: + raise AssertionError(f"git {' '.join(args)} failed: {r.stderr}") + return r.stdout.strip() + + +def _bootstrap_repo(repo: Path) -> None: + repo.mkdir(parents=True, exist_ok=True) + _git(["init"], repo) + _git(["config", "user.email", "test@example.com"], repo) + _git(["config", "user.name", "Test"], repo) + (repo / "README.md").write_text("# target\n", encoding="utf-8") + _git(["add", "-A"], repo) + _git(["commit", "-m", "initial"], repo) + + +def _role(name: str) -> RoleCommand: + return RoleCommand([sys.executable, str(FIXTURES / name)]) + + +# When invoked inside firejail the default interpreter at sys.executable may +# live under /home//, which firejail auto-cleans by default and +# refuses to re-expose via --noblacklist / --whitelist. Use the system python +# for sandboxed role processes so the test exercises firejail honestly. The +# test is skipped if /usr/bin/python3 is not available. +SYSTEM_PYTHON = "/usr/bin/python3" + + +def _role_system_python(name: str) -> RoleCommand: + return RoleCommand([SYSTEM_PYTHON, str(FIXTURES / name)]) + + +# --------------------------------------------------------------------------- +# Config parsing +# --------------------------------------------------------------------------- + + +class TestSandboxConfigParsing(unittest.TestCase): + + def test_default_is_disabled_firejail(self): + cfg = parse_config({"mission": "x"}) + self.assertFalse(cfg.sandbox.enabled) + self.assertEqual(cfg.sandbox.backend, "firejail") + self.assertEqual(cfg.sandbox.extra_args, ()) + + def test_enable_with_extra_args(self): + cfg = parse_config({ + "mission": "x", + "sandbox": { + "enabled": True, + "backend": "firejail", + "extra_args": ["--private-tmp", "--net=none"], + }, + }) + self.assertTrue(cfg.sandbox.enabled) + self.assertEqual(cfg.sandbox.extra_args, ("--private-tmp", "--net=none")) + + def test_enabled_must_be_bool(self): + with self.assertRaises(ConfigError): + parse_config({"mission": "x", "sandbox": {"enabled": "yes"}}) + + def test_backend_must_be_string(self): + with self.assertRaises(ConfigError): + parse_config({"mission": "x", "sandbox": {"backend": 42}}) + + def test_extra_args_must_be_list(self): + with self.assertRaises(ConfigError): + parse_config({"mission": "x", "sandbox": {"extra_args": "--net=none"}}) + + def test_extra_args_must_be_strings(self): + with self.assertRaises(ConfigError): + parse_config({"mission": "x", "sandbox": {"extra_args": ["", "x"]}}) + + def test_sandbox_block_must_be_mapping(self): + with self.assertRaises(ConfigError): + parse_config({"mission": "x", "sandbox": [1, 2]}) + + +# --------------------------------------------------------------------------- +# wrap_argv unit tests +# --------------------------------------------------------------------------- + + +class TestWrapArgv(unittest.TestCase): + + def test_disabled_returns_argv_unchanged(self): + out = wrap_argv(["echo", "hi"], worktree="/tmp", config=SandboxConfig(enabled=False)) + self.assertEqual(out, ["echo", "hi"]) + + def test_none_config_returns_argv_unchanged(self): + out = wrap_argv(["echo", "hi"], worktree="/tmp", config=None) + self.assertEqual(out, ["echo", "hi"]) + + def test_firejail_prefix(self): + with tempfile.TemporaryDirectory() as td: + out = wrap_argv( + ["echo", "hi"], + worktree=td, + config=SandboxConfig(enabled=True, backend="firejail"), + ) + self.assertEqual(out[0], "firejail") + self.assertIn("--read-only=/", out) + self.assertTrue(any(arg.startswith("--read-write=") for arg in out)) + # Original argv must follow `--` + sep = out.index("--") + self.assertEqual(out[sep + 1 :], ["echo", "hi"]) + + def test_extra_writable_included(self): + with tempfile.TemporaryDirectory() as worktree, tempfile.TemporaryDirectory() as ledger: + out = wrap_argv( + ["echo", "hi"], + worktree=worktree, + writable=[ledger], + config=SandboxConfig(enabled=True), + ) + rw = [a for a in out if a.startswith("--read-write=")] + self.assertEqual(len(rw), 2) + + def test_extra_writable_dedup(self): + with tempfile.TemporaryDirectory() as wt: + out = wrap_argv( + ["echo", "hi"], + worktree=wt, + writable=[wt, wt], + config=SandboxConfig(enabled=True), + ) + rw = [a for a in out if a.startswith("--read-write=")] + self.assertEqual(len(rw), 1) + + def test_extra_args_appended_before_separator(self): + with tempfile.TemporaryDirectory() as wt: + out = wrap_argv( + ["echo", "hi"], + worktree=wt, + config=SandboxConfig( + enabled=True, + extra_args=("--private-tmp", "--net=none"), + ), + ) + sep = out.index("--") + self.assertIn("--private-tmp", out[:sep]) + self.assertIn("--net=none", out[:sep]) + + def test_unsupported_backend_raises(self): + with self.assertRaises(SandboxError): + wrap_argv( + ["echo", "hi"], + worktree="/tmp", + config=SandboxConfig(enabled=True, backend="bubblewrap"), + ) + + +# --------------------------------------------------------------------------- +# End-to-end: a real firejail run blocks an out-of-worktree write +# --------------------------------------------------------------------------- + + +_FIREJAIL_AVAILABLE = shutil.which("firejail") is not None +_SYSTEM_PYTHON_AVAILABLE = Path(SYSTEM_PYTHON).exists() + + +@unittest.skipUnless(_FIREJAIL_AVAILABLE, "firejail not installed") +@unittest.skipUnless( + _SYSTEM_PYTHON_AVAILABLE, + "/usr/bin/python3 not available — firejail hides other /home/* prefixes", +) +class TestSandboxBlocksEscape(unittest.TestCase): + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + # Place the temp dir under /tmp so the firejail sandbox can reach it + # — anything under /home/ is auto-hidden by firejail. + self.base = Path(self._tmp.name) + self.repo = self.base / "repo" + self.ledger = self.base / "ledger" + _bootstrap_repo(self.repo) + # Escape target lives outside both repo and ledger and is unique per + # test so a leftover from a prior interactive run cannot confuse us. + self.escape = self.base / "escape" / "out.txt" + self.escape.parent.mkdir(parents=True, exist_ok=True) + self._old_env = os.environ.get("ESCAPE_TARGET") + os.environ["ESCAPE_TARGET"] = str(self.escape) + + def tearDown(self): + if self._old_env is None: + os.environ.pop("ESCAPE_TARGET", None) + else: + os.environ["ESCAPE_TARGET"] = self._old_env + self._tmp.cleanup() + + def _make_governor(self, sandbox: SandboxConfig | None) -> Governor: + # The executor is the only role wrapped in the sandbox, so it is the + # only one that must be invokable via the system interpreter. + return Governor( + target_repo=self.repo, + ledger_dir=self.ledger, + planner=_role("planner.py"), + executor=_role_system_python("executor_escape_attempt.py"), + evaluator=_role("evaluator_accept.py"), + sandbox=sandbox, + ) + + def test_sandbox_blocks_outside_write(self): + gov = self._make_governor(SandboxConfig(enabled=True)) + result = gov.run_once({"name": "sandbox-on"}, run_id="0001") + + executor_output = json.loads( + (result.run_dir / "executor_output.json").read_text(encoding="utf-8") + ) + self.assertTrue(executor_output["inside_write_ok"], executor_output) + self.assertFalse(executor_output["outside_write_ok"], executor_output) + self.assertIsNotNone(executor_output["outside_error"]) + # OS confirmation: the escape file must not exist on disk. + self.assertFalse( + self.escape.exists(), + f"escape file leaked despite sandbox: {self.escape}", + ) + # The inside write was committed and the candidate was accepted. + self.assertTrue(result.decision.accepted, result.decision.reason) + + def test_no_sandbox_allows_outside_write(self): + """Sanity check: the same executor without sandbox can write outside. + + Confirms the assertion above is meaningful (i.e. the previous test + passed because of the sandbox, not because the fixture happened to + always fail). + """ + gov = self._make_governor(sandbox=None) + result = gov.run_once({"name": "sandbox-off"}, run_id="0001") + executor_output = json.loads( + (result.run_dir / "executor_output.json").read_text(encoding="utf-8") + ) + self.assertTrue(executor_output["inside_write_ok"]) + self.assertTrue(executor_output["outside_write_ok"], executor_output) + self.assertTrue(self.escape.exists()) + + +if __name__ == "__main__": + unittest.main() From 47a1d0f4428469004368fddf54b25e76c84ec444 Mon Sep 17 00:00:00 2001 From: Protocol Zero <257158451+Protocol-zero-0@users.noreply.github.com> Date: Wed, 13 May 2026 20:34:34 +0000 Subject: [PATCH 2/2] ci: install firejail so the sandbox E2E test runs on CI Without it, TestSandboxBlocksEscape skips via the @skipUnless guard and CI gives a false sense of coverage. Installing firejail from the Ubuntu universe is a one-line apt step. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a2cf004..5d7961e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,6 +28,11 @@ jobs: git config --global user.email "ci@evolution-kernel.local" git config --global user.name "evolution-kernel-ci" + - name: Install firejail (required for the sandbox E2E test) + run: | + sudo apt-get update + sudo apt-get install -y firejail + - name: Install package (PyYAML is the only runtime dep) run: python -m pip install -e .