diff --git a/apn/redteam.py b/apn/redteam.py index d119d7c..ec66d66 100644 --- a/apn/redteam.py +++ b/apn/redteam.py @@ -138,7 +138,7 @@ def _apn_codebase_tar() -> bytes: return buf.getvalue() -def _sandbox_with_agent_internet(backend: SandboxBackend) -> tuple[str, str]: +def _sandbox_with_agent_internet(backend: SandboxBackend | None) -> tuple[str, str]: """The standard sandbox config, but with the **agent** service (``default``) given internet access -- red-team only. @@ -157,7 +157,7 @@ def _sandbox_with_agent_internet(backend: SandboxBackend) -> tuple[str, str]: backend_type, path = get_sandbox_config(_FC_PIN, literature=False, backend=backend) config = yaml.safe_load(Path(path).read_text()) agent = config["services"]["default"] - if backend == "docker": + if backend_type == "docker": # Drop `network_mode: none` -> the compose project's default bridge # network, which NATs to the host (internet). agent.pop("network_mode", None) @@ -174,7 +174,9 @@ def _sandbox_with_agent_internet(backend: SandboxBackend) -> tuple[str, str]: config["allowEntities"] = ["world"] src = Path(path) out = src.with_name( - "redteam-internet.compose.yaml" if backend == "docker" else "redteam-internet-values.yaml" + "redteam-internet.compose.yaml" + if backend_type == "docker" + else "redteam-internet-values.yaml" ) content = yaml.safe_dump(config, sort_keys=False) if not out.exists() or out.read_text() != content: @@ -230,7 +232,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: @task def apn_redteam_collatz( gated: bool = True, - sandbox_backend: SandboxBackend = "docker", + sandbox_backend: SandboxBackend | None = None, ) -> Task: """Red-team Comparator's soundness: an architecture-aware adversary tries to get a Collatz "proof" accepted. ``accuracy`` is the break rate; the desired diff --git a/apn/software_test.py b/apn/software_test.py index 33eec2c..f2b3364 100644 --- a/apn/software_test.py +++ b/apn/software_test.py @@ -280,7 +280,7 @@ def _samples(groups: list[str]) -> list[Sample]: ] -def _agent_only_sandbox(backend: SandboxBackend) -> tuple[str, str]: +def _agent_only_sandbox(backend: SandboxBackend | None) -> tuple[str, str]: """The standard sandbox config with the ``comparator`` verifier service removed: nothing is scored by proof-checking here, so the second container (and its memory reservation) would be waste. Post-processed from the shared @@ -292,7 +292,7 @@ def _agent_only_sandbox(backend: SandboxBackend) -> tuple[str, str]: config["services"].pop("comparator", None) src = Path(path) out = src.with_name( - "software-test.compose.yaml" if backend == "docker" else "software-test-values.yaml" + "software-test.compose.yaml" if backend_type == "docker" else "software-test-values.yaml" ) content = yaml.safe_dump(config, sort_keys=False) if not out.exists() or out.read_text() != content: @@ -387,7 +387,7 @@ async def score(state: TaskState, target: Target) -> Score: @task def apn_math_software_test( group: str | None = None, - sandbox_backend: SandboxBackend = "docker", + sandbox_backend: SandboxBackend | None = None, ) -> Task: """Exercise the agent image's advertised math software and report problems. diff --git a/apn/task.py b/apn/task.py index 160ee3a..59d9087 100644 --- a/apn/task.py +++ b/apn/task.py @@ -36,6 +36,24 @@ SandboxBackend = Literal["docker", "k8s"] + +def resolve_sandbox_backend(backend: SandboxBackend | None) -> SandboxBackend: + """``backend`` if given, else the backend native to where the task loads. + + In a Hawk runner that is ``k8s``. Hawk also accepts the docker compose + file, but only by auto-converting it to chart values, and the result lacks + everything the compose file cannot say: the ephemeral-storage requests + (compose has no such resource) and the comparator ``runtimeClassName`` pin + from :func:`get_values_file_content`. Everywhere else (local runs, CI, + ``hawk local``) it is ``docker``. + """ + if backend is not None: + return backend + # Hawk's runner Job sets this unconditionally, next to the flag that turns + # on its docker->k8s sandbox conversion; `hawk local` sets neither. + return "k8s" if "HAWK_JOB_ID" in os.environ else "docker" + + # --------------------------------------------------------------------------- # # Shared sandbox constants. Both backend writers draw from these so the two # # artifacts cannot drift semantically. Each config is written in its backend's # @@ -190,15 +208,17 @@ def resources(memory_gib: int, ephemeral_storage_gib: int) -> dict[str, Any]: def get_sandbox_config( - fc_commit: str, literature: bool, backend: SandboxBackend -) -> tuple[str, str]: - """The Inspect ``sandbox`` spec ``(type, config-file path)`` for a backend. + fc_commit: str, literature: bool, backend: SandboxBackend | None +) -> tuple[SandboxBackend, str]: + """The Inspect ``sandbox`` spec ``(type, config-file path)`` for a backend + (``None``: see :func:`resolve_sandbox_backend`). Each backend gets its own generated artifact (``compose.yaml`` for docker, ``values.yaml`` for k8s -- k8s_sandbox treats any config file NOT named ``*compose.yaml``/``*compose.yml`` as chart values). Files are isolated in per-(version, FC pin, variant) subdirs so they don't clobber each other. """ + backend = resolve_sandbox_backend(backend) variant = "corpus" if literature else "closed-book" directory = ( SANDBOX_FILES_DIR @@ -234,7 +254,7 @@ def apn_oeis( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - sandbox_backend: SandboxBackend = "docker", + sandbox_backend: SandboxBackend | None = None, ) -> Task: """The Formal Conjectures autoformalized OEIS conjectures (492 samples). @@ -264,7 +284,7 @@ def apn_fc100open( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - sandbox_backend: SandboxBackend = "docker", + sandbox_backend: SandboxBackend | None = None, ) -> Task: name_list = load_subset(FC100_DIR, subset) if subset is not None else None pin = fc_commit(FC100_DIR) @@ -287,7 +307,7 @@ def apn_erdos( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - sandbox_backend: SandboxBackend = "docker", + sandbox_backend: SandboxBackend | None = None, ) -> Task: """The Bloom statement selection of Erdős problems.""" name_list = load_subset(ERDOS_DIR, subset) if subset is not None else None @@ -311,7 +331,7 @@ def apn_personal_corresp( gated: bool = True, literature: bool = False, agent_type: AgentType = "react", - sandbox_backend: SandboxBackend = "docker", + sandbox_backend: SandboxBackend | None = None, ) -> Task: """Open conjectures sent to us in personal correspondence, formalized by their contributors (the finitistic dimension and Nakayama conjectures from diff --git a/tests/test_sandbox_backend.py b/tests/test_sandbox_backend.py new file mode 100644 index 0000000..70f6787 --- /dev/null +++ b/tests/test_sandbox_backend.py @@ -0,0 +1,45 @@ +"""Tasks pick the sandbox config native to where they load: chart values in a +Hawk runner, compose elsewhere. Fast, no docker.""" + +from pathlib import Path + +import pytest +import yaml + +import apn.task +from apn.dataset import ERDOS_DIR, fc_commit +from apn.task import get_sandbox_config, resolve_sandbox_backend + + +@pytest.fixture(autouse=True) +def sandbox_files_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(apn.task, "SANDBOX_FILES_DIR", tmp_path) + + +def test_defaults_to_docker_outside_hawk(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("HAWK_JOB_ID", raising=False) + assert resolve_sandbox_backend(None) == "docker" + + +def test_defaults_to_k8s_in_hawk_runner(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HAWK_JOB_ID", "bloom68-alv3junt54xqrybd") + assert resolve_sandbox_backend(None) == "k8s" + + +def test_explicit_backend_wins(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HAWK_JOB_ID", "bloom68-alv3junt54xqrybd") + assert resolve_sandbox_backend("docker") == "docker" + + +def test_hawk_default_carries_values_only_settings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The settings Hawk's compose conversion would drop reach the pods.""" + monkeypatch.setenv("HAWK_JOB_ID", "bloom68-alv3junt54xqrybd") + backend, path = get_sandbox_config(fc_commit(ERDOS_DIR), literature=True, backend=None) + assert backend == "k8s" + assert Path(path).name == "values.yaml" + services = yaml.safe_load(Path(path).read_text())["services"] + for name in ("default", "comparator"): + assert "ephemeral-storage" in services[name]["resources"]["requests"] + assert services["comparator"]["runtimeClassName"] == "CLUSTER_DEFAULT"