diff --git a/README.md b/README.md index 626b078..9406642 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

⛪ Cathedral CLI

-

Two ways to mine. One way to validate.

+

Two early miner paths. Validator control is quarantined.

Watch on YouTube · EARLY BETA

@@ -13,7 +13,7 @@ One interface for Cathedral miners and validators. |---|---| | Compute miner | Runs Intel TDX CPU work and returns measured evidence with receipts. | | Distill miner | Finds exploit proofs for fixed vulnerabilities. Validators rerun both builds before scoring. | -| Validator | Verifies evidence and composes weights. Chain broadcasts stay off by default. | +| Validator | Reports a contract mismatch and launches no test, serving loop, or write operation. At reviewed upstream commit `d225e8758ca02627cced800b7de0c79464d89aee`, its entrypoint is a direct chain writer with no non-writing mode. | ## Quickstart @@ -37,8 +37,26 @@ python3.11 ./cathedral agent-brief compute python3.11 ./cathedral capabilities --json ``` -Replace `compute` with `distill` or `validator`. - -Everything above works on a fresh clone today. - -`setup` and `test` additionally need a signed Cathedral release, which is not published yet — so they fail closed until it is. That is the design, not a fault: the CLI installs no engine it cannot verify. Rewards and chain writes stay off by default in early beta. +Replace `compute` with `distill`. `explain validator` and `capabilities` are +available for diagnostics, but Validator setup, config writes, tests, and starts +fail closed with `contract.engine_incompatible`. + +The discovery and explanation commands above work on a fresh clone today. + +Miner `setup` and `test` additionally need a signed Cathedral release, which is +not published yet, so they fail closed until it is. The Validator needs more +than a new pin: an owner must first choose a diagnostics-only interface or an +explicit chain-controller design. This CLI will not infer that authority. + +Signed releases are node-wide. Miner setup or update may therefore install the +retained Validator package and verify its signed files. After wheel installation, +no runtime, import, client-entrypoint, or server-entrypoint probe intentionally +executes newly installed Validator code; all public Validator launch, test, setup, +and config-write paths remain quarantined. + +Engine receipt v6 records `runtime_checked` or `static_quarantine`. A static +generation exposes no executable path, and an older v5 generation is not inferred +safe. The verifier binds that tier to the current adapter policy, so rewriting a +receipt cannot promote static evidence. Enabling a future Validator contract +therefore requires an owner-specified migration and a newly runtime-checked, +higher signed release; this draft intentionally does not invent that transition. diff --git a/cathedral b/cathedral index f0d26fe..24e7aa4 100755 --- a/cathedral +++ b/cathedral @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Cathedral node — one command for Distill mining, Compute mining, and validation. +"""Cathedral node — Distill mining, Compute mining, and Validator diagnostics. Zero dependencies beyond the standard library, on purpose: `git clone` then run this file. Nothing to install before you can find out whether your machine diff --git a/cathedral.lock.json b/cathedral.lock.json index 61f26c4..7e1e16d 100644 --- a/cathedral.lock.json +++ b/cathedral.lock.json @@ -1,7 +1,7 @@ { "schema": "cathedral.node.lock.v1", "generated_at": "2026-07-30T23:00:00Z", - "note": "Pinned upstream engine revisions. A node installs exactly these commits. Changing a pin is a reviewed change: run `cathedral update --check` to see what a newer pin would bring, and `cathedral update --to ` to apply one.", + "note": "Pinned upstream engine revisions. A node installs only releases that are signed and authorized by these exact pins. Changing a pin requires an owner-reviewed signed whole-node release: use `cathedral update --check` to inspect the configured release, and `cathedral update --release --yes` to apply an explicit verified bundle.", "engines": { "distill": { "repository": "https://github.com/cathedralai/cathedral-distill.git", diff --git a/cathedral_node/cli.py b/cathedral_node/cli.py index 74c094f..37f9857 100644 --- a/cathedral_node/cli.py +++ b/cathedral_node/cli.py @@ -9,10 +9,11 @@ templatable surface — ``cathedral test distill --json`` and ``cathedral test compute --json`` differ by one token. -Upstream command names are not preserved. ``cathedral-cybergym-agent --local``, -``cathedral worker serve``, and ``cathedral-validator serve --dry-run --offline`` -are three unrelated spellings of "try this safely"; here they are all -``cathedral test ``. +Upstream command names are adapted only when their semantics still match. The +Distill and Compute engines expose safe local tests. At the content-addressed +Validator revision recorded by its adapter, the entry point is a direct chain +writer with no non-writing mode, so its historical test/start translation is +quarantined instead of being approximated with different argv. """ from __future__ import annotations @@ -64,9 +65,9 @@ def error(self, message: str): # noqa: A003 - argparse's own name cathedral status what is running, and what happened validator - cathedral setup validator - cathedral test validator safe dry run against the signed feed - cathedral start validator begin validating (never broadcasts by default) + cathedral explain validator inspect the contract mismatch + cathedral test validator fail closed; launches nothing + cathedral start validator fail closed; launches nothing agents every command takes --json versioned envelope on stdout; diagnostics on stderr @@ -79,7 +80,10 @@ def error(self, message: str): # noqa: A003 - argparse's own name def build_parser() -> argparse.ArgumentParser: parser = ContractParser( prog="cathedral", - description="Cathedral node — one command for Distill mining, Compute mining, and validation.", + description=( + "Cathedral node — Distill mining, Compute mining, and quarantined " + "Validator diagnostics." + ), epilog=_EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -118,7 +122,7 @@ def add_release_args(sub: argparse.ArgumentParser) -> None: help="root-owned allowed_signers trust file (default: $CATHEDRAL_HOME/allowed_signers)") # --- discovery and orientation --------------------------------------------- - quickstart_cmd = add("quickstart", "Guided path from a clean machine to a verified local test", + quickstart_cmd = add("quickstart", "Guided path for an available role to a verified local test", role="optional") add_release_args(quickstart_cmd) add("doctor", "Check whether this machine and identity qualify") @@ -126,7 +130,7 @@ def add_release_args(sub: argparse.ArgumentParser) -> None: add("explain", "What a role does, what it needs, and what it pays", role="required") # --- setup ------------------------------------------------------------------- - setup = add("setup", "Install the signed release and write configuration", role="required") + setup = add("setup", "Install a signed release for an available role", role="required") setup.add_argument("--force", action="store_true", help="reinstall even if already correct") add_release_args(setup) @@ -159,12 +163,13 @@ def add_release_args(sub: argparse.ArgumentParser) -> None: secret_rm.add_argument("name") # --- running ------------------------------------------------------------------- - test_cmd = add("test", "Run the verified local test — pays nothing, touches no chain", role="required") + test_cmd = add("test", "Run an available engine's local test — pays nothing, touches no chain", + role="required") test_cmd.add_argument("--timeout", type=float, default=0, help="seconds before giving up") - start = add("start", "Start mining or validating", role="required") + start = add("start", "Start an available miner engine", role="required") start.add_argument("--broadcast", action="store_true", - help="validator only: allow chain writes. Requires --yes as well.") + help="validator-only refusal sentinel; chain writes are never enabled") start.add_argument("--foreground", action="store_true", default=True, help="run in this terminal (the default)") start.add_argument("--once", action="store_true", help="one cycle, then exit") @@ -181,7 +186,7 @@ def add_release_args(sub: argparse.ArgumentParser) -> None: logs.add_argument("--lines", "-n", type=int, default=40) logs.add_argument("--raw", action="store_true", help="engine output rather than node events") - resume = add("resume", "Continue an interrupted run", role=None) + resume = add("resume", "Continue an interrupted run when its current contract permits", role=None) resume.add_argument("run", help="run id") cancel = add("cancel", "Cancel a run, preserving its state", role=None) @@ -200,13 +205,19 @@ def add_release_args(sub: argparse.ArgumentParser) -> None: # --- updates ------------------------------------------------------------------- update = subparsers.add_parser("update", parents=[common], - help="Move to new pinned engine revisions, safely") - update.add_argument("--check", action="store_true", help="report what would change; change nothing") + help="Inspect or apply a signed node-wide release") + update.add_argument("--check", action="store_true", + help="inspect the signed node-wide release; change nothing") update.add_argument("--to", metavar="LOCKFILE", default=None, - help="adopt pins from another lockfile. NOT verified: review it, and " - "the repositories it names, before using it") + help="refused compatibility sentinel; unsigned lockfile adoption is disabled") add_release_args(update) - update.add_argument("role", nargs="?", choices=lockfile.ROLES, default=None) + update.add_argument( + "role", + nargs="?", + choices=lockfile.ROLES, + default=None, + help="refused compatibility sentinel; signed releases are node-wide", + ) # Rollback is node-wide (there is no per-role rollback): it undoes an interrupted # transaction, or explains that a deliberate rollback is a newly-signed release. diff --git a/cathedral_node/commands/agent_brief.py b/cathedral_node/commands/agent_brief.py index d79afd0..fce3573 100644 --- a/cathedral_node/commands/agent_brief.py +++ b/cathedral_node/commands/agent_brief.py @@ -39,7 +39,15 @@ def _brief(ctx: Context, lock, roles, states, group) -> Envelope: for name in roles: engine = engines.load(name, lock, group) installed = states[name] - cfg = config.load(name) + static_blocker = engine.operation_blocker() + legacy_config_error: str | None = None + try: + cfg = config.load(name) + except config.ConfigError as exc: + if static_blocker is None: + raise + cfg = config.defaults(name) + legacy_config_error = str(exc) qualification = engine.qualify(cfg) facts[name] = { "title": engine.title, @@ -48,6 +56,7 @@ def _brief(ctx: Context, lock, roles, states, group) -> Envelope: "can_local_test": qualification.can_local_test, "can_operate": qualification.can_operate, "blockers": qualification.blockers, + "legacy_config_error": legacy_config_error, "owner_gated": [ {"capability": k, "detail": v.get("detail", "")} for k, v in engine.capabilities().items() @@ -88,6 +97,44 @@ def _markdown(facts: dict[str, Any], roles: list[str]) -> str: ] blocked_block = "\n".join(blocked) if blocked else "- Nothing additional." + quarantined = [ + name + for name in roles + if any( + blocker.get("code") == "contract.engine_incompatible" + for blocker in facts[name]["blockers"] + ) + ] + available = [name for name in roles if name not in quarantined] + operation_sections: list[str] = [] + if available: + role_arg = available[0] if len(available) == 1 else "" + operation_sections.append( + f"""For available role(s): {', '.join(available)}. + +```bash +cathedral capabilities --json # discovery; confirm the protocol MAJOR +cathedral doctor {role_arg} --json # qualification +cathedral setup {role_arg} --json # install the signed release +cathedral test {role_arg} --json # verified local test; no chain access +cathedral status --json # what is running and what happened +```""" + ) + if quarantined: + operation_sections.append( + f"""For quarantined role(s): {', '.join(quarantined)}. Diagnostics only. + +```bash +cathedral capabilities --json +cathedral explain validator --json +cathedral doctor validator --json # exits 32; owner action is required +``` + +Do not run setup, test, start, or config set for the Validator. Retrying or +installing the retired pin cannot create a non-writing contract.""" + ) + operation_block = "\n\n".join(operation_sections) + return f"""\ # Operating a Cathedral node @@ -105,8 +152,8 @@ def _markdown(facts: dict[str, Any], roles: list[str]) -> str: - `error.remediation.command`, when present, is runnable verbatim. - `error.remediation.requires_operator: true` means **no command fixes this**. Stop and report it. Do not retry, and do not look for a way around it. -- Every command is safe to re-run. `setup`, `config set`, `secret set`, `stop`, and - `cleanup` are idempotent. +- Every command is safe to re-run. For available roles, `setup` and `config set` + are idempotent; `secret set`, `stop`, and `cleanup` are also idempotent. - Add `--yes` where a confirmation is required. A required confirmation is never hidden inside an interactive prompt — you will always get exit `{int(Exit.USAGE)}` with the flag named. @@ -121,13 +168,7 @@ def _markdown(facts: dict[str, Any], roles: list[str]) -> str: ## Order of operations -```bash -cathedral capabilities --json # discovery; confirm the protocol MAJOR -cathedral doctor --json # qualification; read .data.roles..can_local_test -cathedral setup --json # idempotent install of the pinned engine -cathedral test --json # verified local test; pays nothing, no chain access -cathedral status --json # what is running and what happened -``` +{operation_block} {role_sections} @@ -150,8 +191,8 @@ def _markdown(facts: dict[str, Any], roles: list[str]) -> str: ## Reporting Report the envelope's `run_id`, `status`, `exit_code`, and any identifiers under -`data.identifiers` — they are the exact challenge, receipt, vector, and submission ids the -operator needs. Include `error.code` verbatim when something failed. +`data.identifiers`. Include `error.code` verbatim when something failed. The +quarantined Validator creates no run and returns no vector or submission identifiers. """ @@ -159,14 +200,36 @@ def _role_section(role: str, facts: dict[str, Any]) -> str: lines = [f"### {facts['title']} (`{role}`)", ""] if facts["installed"]: lines.append(f"- Engine installed at `{(facts['revision'] or '')[:12]}`.") + elif any( + blocker.get("code") == "contract.engine_incompatible" + for blocker in facts["blockers"] + ): + lines.append("- Engine execution is quarantined. Installation is not a compatibility fix.") else: lines.append(f"- Engine not installed. Run `cathedral setup {role} --json` first.") - lines.append( - f"- Local test: {'available' if facts['can_local_test'] else 'blocked'} — `cathedral test {role} --json`" - ) - lines.append( - f"- Live operation: {'available' if facts['can_operate'] else 'blocked'} — `cathedral start {role} --json`" + contract_quarantined = any( + blocker.get("code") == "contract.engine_incompatible" + for blocker in facts["blockers"] ) + if contract_quarantined: + lines.append("- Local test: quarantined; do not install or retry.") + lines.append("- Live operation: quarantined; this CLI launches no Validator operation.") + lines.append( + "- Node-wide miner setup/update may install and statically verify the retained package, " + "but no post-install runtime, import, or entrypoint probe executes its code." + ) + if facts.get("legacy_config_error"): + lines.append( + "- Legacy config is unreadable but inert while quarantined: " + + facts["legacy_config_error"] + ) + else: + lines.append( + f"- Local test: {'available' if facts['can_local_test'] else 'blocked'} — `cathedral test {role} --json`" + ) + lines.append( + f"- Live operation: {'available' if facts['can_operate'] else 'blocked'} — `cathedral start {role} --json`" + ) if facts["blockers"]: lines.append("- Blockers:") for blocker in facts["blockers"]: diff --git a/cathedral_node/commands/capabilities.py b/cathedral_node/commands/capabilities.py index 3667884..439d594 100644 --- a/cathedral_node/commands/capabilities.py +++ b/cathedral_node/commands/capabilities.py @@ -21,7 +21,6 @@ EVENT_SCHEMA, PROTOCOL_VERSION, RESULT_SCHEMA, - schema_id, ) from cathedral_node.engines import installer from cathedral_node.runner import Context, command, registry @@ -46,13 +45,37 @@ def _engine_reports(lock, states, group) -> dict[str, Any]: for role in lockfile.ROLES: engine = engines.load(role, lock, group) installed = states[role] - cfg = config.load(role) + static_blocker = engine.operation_blocker() + legacy_config: dict[str, Any] | None = None + try: + cfg = config.load(role) + except config.ConfigError as exc: + if static_blocker is None: + raise + # A malformed, inert relay file must not suppress the permanent + # contract result—or take down discovery for usable miner roles. + cfg = config.defaults(role) + legacy_config = {"readable": False, "detail": str(exc)} qualification = engine.qualify(cfg) engine_reports[role] = { "title": engine.title, "tagline": engine.tagline, "installed": installed.to_dict(), "pinned": lock.pin(role).to_dict(), + "contract_status": { + "execution_status": ( + "not_statically_blocked" + if static_blocker is None + else "quarantined" + ), + "pin_status": ( + "retained signed-release member" + if static_blocker is None + else "retained signed-release member; CLI execution quarantined" + ), + "blocker": static_blocker, + "legacy_config": legacy_config, + }, "can_local_test": qualification.can_local_test, "can_operate": qualification.can_operate, "capabilities": engine.capabilities(), @@ -87,7 +110,7 @@ def _envelope(ctx: Context, lock, engine_reports: dict[str, Any]) -> Envelope: "diagnostic_stream": "stderr", "non_interactive": "every command runs unattended; --yes supplies any confirmation", "idempotent": ["setup", "config set", "secret set", "cleanup", "stop"], - "resumable": ["mine", "validate"], + "resumable": list(lockfile.MINER_ROLES), "secrets": "never in argv, output, logs, or committed config", }, "machine": machine.summary(), @@ -130,11 +153,26 @@ def _render(console: Console, data: dict[str, Any], env: Envelope) -> None: console.blank() installed = report["installed"] + contract = report["contract_status"] + quarantined = contract["execution_status"] == "quarantined" if installed["installed"]: drift = " (differs from pin)" if installed["revision_drift"] else "" - console.ok("engine", f"{installed['short_revision']}{drift}") + if quarantined: + console.info( + "engine", + console.join( + f"retained pin {installed['short_revision']}{drift}", + "execution quarantined", + ), + ) + else: + console.ok("engine", f"{installed['short_revision']}{drift}") else: - console.info("engine", console.join("not installed", f"pinned at {report['pinned']['short_revision']}")) + label = "retained pin" if quarantined else "pinned at" + console.info( + "engine", + console.join("not installed", f"{label} {report['pinned']['short_revision']}"), + ) for name, capability in report["capabilities"].items(): if not isinstance(capability, dict) or "available" not in capability: diff --git a/cathedral_node/commands/cleanup.py b/cathedral_node/commands/cleanup.py index cf8259e..6c3780c 100644 --- a/cathedral_node/commands/cleanup.py +++ b/cathedral_node/commands/cleanup.py @@ -11,7 +11,7 @@ import time from typing import Any -from cathedral_node import lockfile, paths, state +from cathedral_node import engines, lockfile, paths, state from cathedral_node.contracts import Envelope, Exit, Remediation from cathedral_node.contracts import codes as C from cathedral_node.contracts.version import schema_id @@ -123,8 +123,19 @@ def cleanup(ctx: Context) -> Envelope: detail=data) env = Envelope.ok("cleanup", data) env.data_schema = schema_id("cleanup") - if removable_engines: - env.then("Reinstall when you need it", f"cathedral setup {removable_engines[0]['role']}") + lock = lockfile.load() + for item in removable_engines: + role = item["role"] + blocker = engines.load(role, lock).operation_blocker() + if blocker is None: + env.then(f"Reinstall {role} when you need it", f"cathedral setup {role}") + else: + env.warn( + "contract.engine_incompatible", + f"{role} was removed, but this build cannot recommend reinstalling its " + "quarantined execution contract", + role=role, + ) return env diff --git a/cathedral_node/commands/config_cmd.py b/cathedral_node/commands/config_cmd.py index e83df73..dd2582d 100644 --- a/cathedral_node/commands/config_cmd.py +++ b/cathedral_node/commands/config_cmd.py @@ -7,34 +7,33 @@ from __future__ import annotations -import re from typing import Any from cathedral_node import config, engines, lockfile, paths from cathedral_node import redact -from cathedral_node.engines import installer from cathedral_node.contracts import Envelope, Exit, Remediation from cathedral_node.contracts import codes as C from cathedral_node.contracts.version import schema_id +from cathedral_node.engines.validator import legacy_config_contract from cathedral_node.runner import Context, command from cathedral_node.ui.console import Console from cathedral_node.ui.render import renders def _keep_public_fields_readable(role: str, values: dict[str, Any]) -> None: - """Exempt ONLY the validator's `weight_policy_key` from value-shape redaction, - and only when it is exactly 64 hex characters. + """Keep the retired validator's public key readable for audit/migration. - The redaction backstop masks anything 64-hex-shaped, which would blank this - PUBLIC 32-byte key in `--json` while the human view shows it — the operator is - told to read and verify it. The exemption is deliberately narrow: exempting + The redaction backstop masks anything 64-hex-shaped, which would blank the + one published 32-byte key in `--json` while the human view shows it. The + exemption is deliberately narrow: exempting every non-secret field would leak an embedded credential, e.g. an `api_base` - holding `https://host/?api_key=SECRET`. Requiring an exact 64-hex value means - the exempted string cannot carry a credential, and no other field is touched. + holding `https://host/?api_key=SECRET`. Requiring the Validator role and the + exact compiled public value prevents a same-named miner field or arbitrary + 64-hex credential from becoming globally public. Known-secret values always take precedence over this (see redact.redact_value). """ key = values.get("weight_policy_key") - if isinstance(key, str) and re.fullmatch(r"[0-9a-fA-F]{64}", key): + if role == "validator" and key == config.SN39_WEIGHT_POLICY_PUBLIC_KEY: redact.register_public_values([key]) @@ -50,6 +49,7 @@ def config_show(ctx: Context) -> Envelope: "file": str(paths.config_file(role)), "exists": paths.config_file(role).exists(), "values": values, + "contract_status": legacy_config_contract() if role == "validator" else None, } found = config.validate(role, values) if found: @@ -65,6 +65,12 @@ def config_show(ctx: Context) -> Envelope: env.data_schema = schema_id("config") if problems: env.warn("config.invalid", f"{sum(len(v) for v in problems.values())} configuration problem(s)") + if "validator" in roles: + env.warn( + "contract.legacy_config", + payload["validator"]["contract_status"]["warning"], + role="validator", + ) return env @@ -80,8 +86,22 @@ def config_get(ctx: Context) -> Envelope: remediation=Remediation(summary="List the fields and what they mean.", command=f"cathedral config schema {role}"), ) - env = Envelope.ok("config.get", {"role": role, "field": field, "value": values[field]}) + legacy_contract = legacy_config_contract() if role == "validator" else None + env = Envelope.ok("config.get", { + "role": role, + "field": field, + "value": values[field], + "legacy_inert": role == "validator", + "contract_status": legacy_contract, + }) env.data_schema = schema_id("config_value") + if legacy_contract is not None: + env.warn( + "contract.legacy_config", + legacy_contract["warning"], + role="validator", + field=field, + ) return env @@ -90,6 +110,23 @@ def config_set(ctx: Context) -> Envelope: role, field, raw = ctx.args.role, ctx.args.field, ctx.args.value if field.lower() in config.FORBIDDEN_FIELDS: + if role == "validator": + return Envelope.fail( + "config.set", + C.E_COLDKEY_REFUSED, + f"`{field}` is never accepted", + exit_code=Exit.USAGE, + remediation=Remediation( + summary=( + "Cathedral never needs a coldkey, seed, or mnemonic. The Validator " + "runtime contract is also quarantined; no replacement config command " + "is available in this build." + ), + command=None, + docs="cathedral explain validator", + requires_operator=True, + ), + ) return Envelope.fail( "config.set", C.E_COLDKEY_REFUSED, @@ -113,6 +150,28 @@ def config_set(ctx: Context) -> Envelope: command=f"cathedral config schema {role}"), ) + # Known Validator fields describe the retired relay. Refuse both applied and + # dry-run writes before parsing a value: a dry run must not imply that this + # build knows how the reviewed direct-writer contract should be configured. + static_blocker = engines.load(role, lockfile.load()).operation_blocker() + if static_blocker is not None: + return Envelope.blocked( + "config.set", + static_blocker.get("code", C.E_ENGINE_INCOMPATIBLE), + f"this build cannot change {role} runtime configuration: " + f"{static_blocker.get('what', 'the engine contract is incompatible')}", + exit_code=Exit.INCOMPATIBLE, + remediation=Remediation( + summary=( + "Legacy values remain readable for audit and migration, but nothing was written." + ), + command=static_blocker.get("fix"), + docs=f"cathedral explain {role}", + requires_operator=bool(static_blocker.get("requires_operator")), + ), + detail={"blockers": [static_blocker]}, + ) + field_spec = next(f for f in config.schema(role) if f.name == field) if _looks_like_a_secret_value(field_spec, raw): return Envelope.fail( @@ -152,49 +211,7 @@ def config_set(ctx: Context) -> Envelope: env.dry_run = True return env - if role == "validator": - # The validator's engine TOML is *derived* from this value, and deriving it - # reads the verified generation's shipped default. Three things follow. - # - # The adapter must be bound to its verified role: an unbound one renders - # from an empty base and silently drops whatever the signed release - # shipped. The lease must be held while it renders, for the same reason - # every other installed-code path holds it. And the derived file must be - # produced and checked BEFORE the node config is committed — otherwise a - # render that fails leaves the node config saying one thing and the engine - # config another, which is a partial success reported as a whole one. - try: - with installer.active_view(lockfile.load()) as (_states, group, detail): - if group is None: - return Envelope.blocked( - "config.set", C.E_ENGINE_NOT_INSTALLED, - "the validator engine is not installed as a verified signed release", - exit_code=Exit.NOT_READY, - remediation=Remediation( - summary=(f"Nothing was written: {detail}. The engine configuration is " - f"derived from the verified generation, so it cannot be " - f"written without one."), - command="cathedral setup validator")) - bound = engines.load(role, lockfile.load(), group) - derived = bound.render_engine_config(values) - except (installer.ActiveStateError, installer.InstallError) as exc: - return Envelope.blocked( - "config.set", C.E_ENGINE_NOT_INSTALLED, - "the validator engine configuration could not be derived", exit_code=Exit.NOT_READY, - remediation=Remediation(summary=f"Nothing was written: {exc}", - command="cathedral status validator")) - except ValueError as exc: - return Envelope.fail( - "config.set", C.E_CONFIG_INVALID, - "the derived validator engine configuration is not valid", - exit_code=Exit.CONFIG_INVALID, - remediation=Remediation(summary=f"Nothing was written: {exc}", - command="cathedral config show validator")) - # Both writes are atomic and only happen once the derived form is known good. - config.save(role, values) - bound.commit_engine_config(derived) - else: - config.save(role, values) + config.save(role, values) data = { "role": role, "field": field, "from": previous, "to": values[field], @@ -272,6 +289,8 @@ def _render_show(console: Console, data: dict[str, Any], env: Envelope) -> None: console.rule(role) if not report["exists"]: console.info("file", "not written yet — showing defaults") + if report.get("contract_status"): + console.warn("contract", report["contract_status"]["warning"]) pairs = [(k, v if v != "" else console.style.dim("(empty)")) for k, v in report["values"].items()] console.kv_block(pairs, indent=4) for problem in data["problems"].get(role, []): @@ -314,4 +333,6 @@ def _render_get(console: Console, data: dict[str, Any], env: Envelope) -> None: if "field" not in data: return console.blank() + if data.get("contract_status"): + console.warn("contract", data["contract_status"]["warning"]) console.info(data["field"], str(data.get("value", ""))) diff --git a/cathedral_node/commands/doctor.py b/cathedral_node/commands/doctor.py index b831db1..b03e7c1 100644 --- a/cathedral_node/commands/doctor.py +++ b/cathedral_node/commands/doctor.py @@ -49,7 +49,7 @@ def doctor(ctx: Context) -> Envelope: purpose="needed to fetch pinned engines"), required=True, fix="Install git.", code=C.E_TOOL_MISSING)) checks.append(_check("network", machine.network_probe(), required=False, - fix="Local tests work offline. Live operation needs the feed.", + fix="Local tests work offline. Live operation may need network access.", code=C.E_NETWORK)) for problem in config.secrets_file_problems(): @@ -67,18 +67,28 @@ def doctor(ctx: Context) -> Envelope: def _diagnose(ctx: Context, lock, roles, checks, states, group) -> Envelope: role_reports: dict[str, Any] = {} + scoped_to_one_role = getattr(ctx.args, "role", None) is not None for name in roles: engine = engines.load(name, lock, group) installed = states[name] - cfg = config.load(name) - problems = config.validate(name, cfg) + static_blocker = engine.operation_blocker() + legacy_config_error: str | None = None + try: + cfg = config.load(name) + except config.ConfigError as exc: + if static_blocker is None: + raise + cfg = config.defaults(name) + legacy_config_error = str(exc) + problems = [] if static_blocker is not None else config.validate(name, cfg) qualification = engine.qualify(cfg) - role_checks: list[dict[str, Any]] = [ - { + role_checks: list[dict[str, Any]] = [] + if static_blocker is None: + role_checks.append({ "name": "engine", "verdict": "yes" if installed.installed and not installed.drift else "no", - "required": True, + "required": scoped_to_one_role, "passed": installed.installed and not installed.drift, "detail": ( f"{lock.pin(name).distribution} at {installed.revision[:12]}" @@ -89,11 +99,11 @@ def _diagnose(ctx: Context, lock, roles, checks, states, group) -> Envelope: "code": str(C.E_ENGINE_REVISION_DRIFT if installed.drift else C.E_ENGINE_NOT_INSTALLED), "fix": f"cathedral setup {name}", "requires_operator": False, - } - ] + }) for problem in problems: role_checks.append({ - "name": f"{name} configuration", "verdict": "no", "required": True, "passed": False, + "name": f"{name} configuration", "verdict": "no", + "required": scoped_to_one_role, "passed": False, "detail": problem, "code": str(C.E_CONFIG_INVALID), "fix": f"cathedral config set {name} ", "requires_operator": False, }) @@ -111,7 +121,11 @@ def _diagnose(ctx: Context, lock, roles, checks, states, group) -> Envelope: role_checks.append({ "name": _blocker_label(blocker), "verdict": "no", - "required": stops_everything, + # A role-scoped doctor must fail on the quarantined Validator. + # A whole-node doctor still reports that owner blocker, but it + # may pass for usable miner roles instead of becoming + # permanently red because an unrelated role is unavailable. + "required": stops_everything and scoped_to_one_role, "passed": False, "blocks_local_test": stops_everything, "detail": blocker.get("what", ""), @@ -120,6 +134,20 @@ def _diagnose(ctx: Context, lock, roles, checks, states, group) -> Envelope: "requires_operator": bool(blocker.get("requires_operator")), "blocks": blocker.get("blocks", []), }) + if legacy_config_error is not None: + role_checks.append({ + "name": "legacy validator configuration", + "verdict": "unknown", + "required": False, + "passed": False, + "detail": ( + f"{legacy_config_error}; ignored because the execution contract is quarantined" + ), + "code": str(C.E_CONFIG_INVALID), + "fix": None, + "requires_operator": True, + "blocks": [], + }) role_reports[name] = { "role": name, @@ -137,7 +165,22 @@ def _diagnose(ctx: Context, lock, roles, checks, states, group) -> Envelope: "container_runtime": machine.container_runtime_probe().to_dict(), } - blocking = [c for c in checks if c["required"] and not c["passed"]] + machine_blocking = [c for c in checks if c["required"] and not c["passed"]] + role_blocking = [ + check + for report in role_reports.values() + for check in report["checks"] + if check["required"] and not check["passed"] + ] + # For `doctor validator`, the permanent contract mismatch is the + # authoritative first answer even if this particular host also lacks disk, + # Python, or another repairable prerequisite. Whole-node doctor retains its + # broad machine-first diagnostic behavior for available miner roles. + blocking = ( + role_blocking + machine_blocking + if scoped_to_one_role + else machine_blocking + role_blocking + ) ready_roles = [r for r, report in role_reports.items() if report["can_local_test"]] data = { @@ -152,13 +195,22 @@ def _diagnose(ctx: Context, lock, roles, checks, states, group) -> Envelope: if blocking: first = blocking[0] + exit_code = ( + Exit.INCOMPATIBLE + if first.get("code") == str(C.E_ENGINE_INCOMPATIBLE) + else Exit.NOT_READY + ) env = Envelope.blocked( "doctor", first.get("code") or C.E_TOOL_MISSING, f"{first['name']}: {first['detail']}", - exit_code=Exit.NOT_READY, + exit_code=exit_code, remediation=Remediation( - summary=f"{len(blocking)} check(s) must pass before anything can run.", + summary=( + "The Validator contract needs an owner decision; no command can make it pass." + if exit_code == Exit.INCOMPATIBLE + else f"{len(blocking)} check(s) must pass before anything can run." + ), command=first.get("fix"), requires_operator=bool(first.get("requires_operator")), ), @@ -170,7 +222,10 @@ def _diagnose(ctx: Context, lock, roles, checks, states, group) -> Envelope: env = Envelope.ok("doctor", data) env.data_schema = schema_id("doctor") if not ready_roles: - env.warn("doctor.no_role_ready", "No role can run a local test yet. Run `cathedral setup `.") + env.warn( + "doctor.no_role_ready", + "No available miner role can run a local test yet. Set up Distill or Compute.", + ) else: for name in ready_roles: env.then(f"Run the {name} local test", f"cathedral test {name}") @@ -183,6 +238,7 @@ def _diagnose(ctx: Context, lock, roles, checks, states, group) -> Envelope: "config.field_required": "config", "install.engine_missing": "engine", "install.revision_drift": "engine", + "contract.engine_incompatible": "validator contract", } diff --git a/cathedral_node/commands/evidence.py b/cathedral_node/commands/evidence.py index e08206f..2bfb2f8 100644 --- a/cathedral_node/commands/evidence.py +++ b/cathedral_node/commands/evidence.py @@ -14,6 +14,7 @@ from cathedral_node.contracts import Envelope, Exit, Remediation from cathedral_node.contracts import codes as C from cathedral_node.contracts.version import schema_id +from cathedral_node.engines.validator import historical_run_contract from cathedral_node.runner import Context, command from cathedral_node.ui.console import Console from cathedral_node.ui.render import renders @@ -26,18 +27,33 @@ def evidence(ctx: Context) -> Envelope: # A run id resolves directly. record = state.load_run(needle) if record is not None: - record = state.reconcile(record) + if record.role != "validator": + record = state.reconcile(record) events = list(state.read_events(needle)) + shown_record = record.to_dict() + historical = record.role == "validator" + if historical: + shown_record["historical_contract"] = True + shown_record["contract_status"] = historical_run_contract() data = { "identifier": needle, "kind": "run", - "run": record.to_dict(), + "run": shown_record, "matches": events, "match_count": len(events), "run_dir": paths.relative_to_home(paths.run_dir(needle)), + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None, } env = Envelope.ok("evidence", data) env.data_schema = schema_id("evidence") + if historical: + env.warn( + "contract.historical_run", + data["contract_status"]["warning"], + role=record.role, + run_id=record.run_id, + ) return env # Otherwise search every run's events and artifacts for the string. @@ -45,8 +61,14 @@ def evidence(ctx: Context) -> Envelope: for candidate in state.list_runs(limit=500): for event in state.read_events(candidate.run_id): if _contains(event, needle): - matches.append({"run_id": candidate.run_id, "role": candidate.role, - "event": event}) + historical = candidate.role == "validator" + matches.append({ + "run_id": candidate.run_id, + "role": candidate.role, + "event": event, + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None, + }) if len(matches) >= 50: break if len(matches) >= 50: @@ -64,15 +86,24 @@ def evidence(ctx: Context) -> Envelope: ) runs = sorted({m["run_id"] for m in matches}) + historical = any(m["historical_contract"] for m in matches) data = { "identifier": needle, "kind": "identifier", "matches": matches, "match_count": len(matches), "runs": runs, + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None, } env = Envelope.ok("evidence", data) env.data_schema = schema_id("evidence") + if historical: + env.warn( + "contract.historical_run", + data["contract_status"]["warning"], + runs=[m["run_id"] for m in matches if m["historical_contract"]], + ) env.then("See the whole run", f"cathedral status --run {runs[0]}") return env @@ -90,6 +121,10 @@ def _contains(value: Any, needle: str) -> bool: @renders("evidence") def _render(console: Console, data: dict[str, Any], env: Envelope) -> None: console.title("Evidence", data["identifier"]) + if data.get("historical_contract"): + console.blank() + console.info("scope", "historical contract") + console.warn("contract", data["contract_status"]["warning"]) if data["kind"] == "run": record = data["run"] console.blank() @@ -104,8 +139,9 @@ def _render(console: Console, data: dict[str, Any], env: Envelope) -> None: console.info("found in", f"{data['match_count']} event(s) across {len(data['runs'])} run(s)") console.blank() console.table( - ["run", "event", "detail"], - [[m["run_id"], m["event"].get("event", ""), m["event"].get("detail", "")] + ["run", "contract", "event", "detail"], + [[m["run_id"], "historical" if m["historical_contract"] else "current", + m["event"].get("event", ""), m["event"].get("detail", "")] for m in data["matches"][:12]], indent=4, ) diff --git a/cathedral_node/commands/explain.py b/cathedral_node/commands/explain.py index 03b612c..55af7d2 100644 --- a/cathedral_node/commands/explain.py +++ b/cathedral_node/commands/explain.py @@ -25,8 +25,12 @@ def explain(ctx: Context) -> Envelope: data["pinned_revision"] = lock.pin(ctx.args.role).short_revision env = Envelope.ok("explain", data) env.data_schema = schema_id("explain") - env.then(f"Check whether this machine qualifies", f"cathedral doctor {ctx.args.role}") - env.then(f"Install and test it", f"cathedral setup {ctx.args.role} && cathedral test {ctx.args.role}") + env.then("Check whether this machine qualifies", f"cathedral doctor {ctx.args.role}") + if engine.operation_blocker() is None: + env.then( + "Install and test it", + f"cathedral setup {ctx.args.role} && cathedral test {ctx.args.role}", + ) return env @@ -62,6 +66,20 @@ def _render(console: Console, data: dict[str, Any], env: Envelope) -> None: console.rule("what it never does") console.bullets(data["what_it_never_does"]) + evidence = data.get("reviewed_contract_evidence") + if evidence: + console.blank() + console.rule("reviewed contract evidence") + console.kv_block( + [ + ("repository", evidence["repository"]), + ("commit", evidence["commit"]), + ("entrypoint", evidence["entrypoint"]), + ("source sha256", evidence["entrypoint_source_sha256"]), + ], + indent=4, + ) + console.blank() console.rule("what you need") console.bullets(data.get("what_you_need", [])) diff --git a/cathedral_node/commands/logs.py b/cathedral_node/commands/logs.py index b408a8f..331691b 100644 --- a/cathedral_node/commands/logs.py +++ b/cathedral_node/commands/logs.py @@ -11,10 +11,11 @@ from pathlib import Path from typing import Any -from cathedral_node import lockfile, paths, state +from cathedral_node import engines, lockfile, paths, state from cathedral_node.contracts import Envelope, Exit, Remediation from cathedral_node.contracts import codes as C from cathedral_node.contracts.version import schema_id +from cathedral_node.engines.validator import historical_run_contract from cathedral_node.runner import Context, command from cathedral_node.ui.console import Console from cathedral_node.ui.render import renders @@ -31,13 +32,24 @@ def logs(ctx: Context) -> Envelope: if not run_id: candidates = state.list_runs(role, limit=1) if not candidates: + blocker = ( + engines.load(role, lockfile.load()).operation_blocker() + if role is not None + else None + ) return Envelope.fail( "logs", C.E_RUN_NOT_FOUND, f"no runs recorded{' for ' + role if role else ''}", exit_code=Exit.NOT_FOUND, remediation=Remediation( - summary="Run something first.", - command=f"cathedral test {role or 'distill'}", + summary=( + "No Validator run can be created while its contract is quarantined." + if blocker is not None + else "Run something first." + ), + command=(None if blocker is not None else f"cathedral test {role or 'distill'}"), + docs=(f"cathedral explain {role}" if blocker is not None else None), + requires_operator=bool(blocker and blocker.get("requires_operator")), ), ) run_id = candidates[0].run_id @@ -48,21 +60,26 @@ def logs(ctx: Context) -> Envelope: "logs", C.E_RUN_NOT_FOUND, f"no run named {run_id}", exit_code=Exit.NOT_FOUND, remediation=Remediation(summary="List recent runs.", command="cathedral status"), ) - record = state.reconcile(record) + historical = record.role == "validator" + if not historical: + record = state.reconcile(record) if raw: - return _raw(ctx, record, lines, follow) + return _raw(ctx, record, lines, follow, historical=historical) events = list(state.read_events(run_id)) shown = events[-lines:] if not ctx.json_mode: - ctx.console.title(f"Run {run_id}", f"{record.role} · {record.status}") + suffix = " · historical contract" if historical else "" + ctx.console.title(f"Run {run_id}", f"{record.role} · {record.status}{suffix}") + if historical: + ctx.console.warn("contract", historical_run_contract()["warning"]) ctx.console.blank() for event in shown: - _print_event(ctx.console, event) + _print_event(ctx.console, event, historical=historical) - if follow and record.status == "running": + if follow and record.status == "running" and not historical: seen = len(events) try: while True: @@ -71,7 +88,7 @@ def logs(ctx: Context) -> Envelope: for event in fresh: seen += 1 if not ctx.json_mode: - _print_event(ctx.console, event) + _print_event(ctx.console, event, historical=historical) current = state.load_run(run_id) if current is None or state.reconcile(current).status != "running": break @@ -87,13 +104,36 @@ def logs(ctx: Context) -> Envelope: "event_count": len(events), "events": shown, "run_dir": paths.relative_to_home(paths.run_dir(run_id)), + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None, } env = Envelope.ok("logs", data) env.data_schema = schema_id("logs") + if historical: + env.warn( + "contract.historical_run", + data["contract_status"]["warning"], + role=record.role, + run_id=record.run_id, + ) + if follow: + env.warn( + "contract.historical_follow_disabled", + "Follow is disabled for quarantined Validator history; this is a static snapshot.", + role=record.role, + run_id=record.run_id, + ) return env -def _raw(ctx: Context, record: state.RunRecord, lines: int, follow: bool) -> Envelope: +def _raw( + ctx: Context, + record: state.RunRecord, + lines: int, + follow: bool, + *, + historical: bool, +) -> Envelope: path = paths.run_dir(record.run_id) / "engine.log" if not path.exists(): return Envelope.fail( @@ -105,16 +145,38 @@ def _raw(ctx: Context, record: state.RunRecord, lines: int, follow: bool) -> Env text = path.read_text(errors="replace").splitlines() tail = text[-lines:] if not ctx.json_mode: - ctx.console.title(f"Engine output · {record.run_id}", paths.relative_to_home(path)) + suffix = " · historical contract" if historical else "" + ctx.console.title( + f"Engine output · {record.run_id}", + paths.relative_to_home(path) + suffix, + ) + if historical: + ctx.console.warn("contract", historical_run_contract()["warning"]) ctx.console.blank() for line in tail: ctx.console.write(" " + line) - if follow: + if follow and record.status == "running" and not historical: _follow_file(ctx, path) data = {"run_id": record.run_id, "raw": True, "lines": tail, - "file": paths.relative_to_home(path), "total_lines": len(text)} + "file": paths.relative_to_home(path), "total_lines": len(text), + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None} env = Envelope.ok("logs", data) env.data_schema = schema_id("logs") + if historical: + env.warn( + "contract.historical_run", + data["contract_status"]["warning"], + role=record.role, + run_id=record.run_id, + ) + if follow: + env.warn( + "contract.historical_follow_disabled", + "Follow is disabled for quarantined Validator history; this is a static snapshot.", + role=record.role, + run_id=record.run_id, + ) return env @@ -133,13 +195,20 @@ def _follow_file(ctx: Context, path: Path) -> None: pass -def _print_event(console: Console, event: dict[str, Any]) -> None: +def _print_event( + console: Console, event: dict[str, Any], *, historical: bool = False +) -> None: status = str(event.get("status", "INFO")).upper() name = str(event.get("event", "")).lower()[:11] detail = event.get("detail", "") timestamp = str(event.get("ts", ""))[11:19] prefix = console.style.dim(timestamp + " ") if timestamp else "" - if status == "PASS": + if historical: + console.write( + f" {prefix}{console.style.dim(console.glyphs.info)} " + f"{console.style.dim('historical'.ljust(11))} {status} {name} · {detail}" + ) + elif status == "PASS": console.write(f" {prefix}{console.style.green(console.glyphs.ok)} " f"{console.style.dim(name.ljust(11))} {detail}") elif status in ("FAIL", "ERROR"): @@ -158,4 +227,6 @@ def _render(console: Console, data: dict[str, Any], env: Envelope) -> None: console.info("lines", f"{len(data['lines'])} of {data['total_lines']}") else: console.info("events", str(data["event_count"])) + if data.get("historical_contract"): + console.warn("contract", "historical Validator evidence; execution is quarantined") console.info("status", data.get("status", "")) diff --git a/cathedral_node/commands/quickstart.py b/cathedral_node/commands/quickstart.py index a6f186b..4710167 100644 --- a/cathedral_node/commands/quickstart.py +++ b/cathedral_node/commands/quickstart.py @@ -8,7 +8,6 @@ from __future__ import annotations -import sys from typing import Any from cathedral_node import config, engines, lockfile, machine, paths, state @@ -35,6 +34,26 @@ def quickstart(ctx: Context) -> Envelope: steps: list[dict[str, Any]] = [] console = ctx.console + static_blocker = engine.operation_blocker() + if static_blocker is not None: + env = Envelope.blocked( + "quickstart", + static_blocker.get("code", C.E_ENGINE_INCOMPATIBLE), + "Not yet true: " + + static_blocker.get("what", "the engine contract is incompatible"), + exit_code=Exit.INCOMPATIBLE, + remediation=Remediation( + summary="Quickstart stopped before installation or execution.", + command=static_blocker.get("fix"), + docs=f"cathedral explain {role}", + requires_operator=bool(static_blocker.get("requires_operator")), + ), + detail={"blockers": [static_blocker]}, + ) + env.data = {"role": role, "steps": steps, "verified": False} + env.data_schema = schema_id("quickstart") + return env + console.title(console.join("Cathedral", engine.title), engine.tagline) # 1 — what this is @@ -231,7 +250,15 @@ def _choose(ctx: Context, lock: lockfile.Lock) -> Envelope: options = [] for role in lockfile.ROLES: engine = engines.load(role, lock) - cfg = config.load(role) + blocker = engine.operation_blocker() + legacy_config_error: str | None = None + try: + cfg = config.load(role) + except config.ConfigError as exc: + if blocker is None: + raise + cfg = config.defaults(role) + legacy_config_error = str(exc) qualification = engine.qualify(cfg) explanation = engine.explain() options.append({ @@ -240,11 +267,16 @@ def _choose(ctx: Context, lock: lockfile.Lock) -> Envelope: "tagline": engine.tagline, "needs": explanation.get("what_you_need", []), "can_local_test_now": qualification.can_local_test, - "command": f"cathedral quickstart {role}", + "legacy_config_error": legacy_config_error, + "command": ( + f"cathedral explain {role}" + if blocker is not None + else f"cathedral quickstart {role}" + ), }) console = ctx.console - console.title("Cathedral", "one node · two ways to mine · one way to validate") + console.title("Cathedral", "one node · two miner paths · validator quarantined") for option in options: console.blank() console.rule(option["title"].lower()) diff --git a/cathedral_node/commands/run.py b/cathedral_node/commands/run.py index 676994d..dce96f1 100644 --- a/cathedral_node/commands/run.py +++ b/cathedral_node/commands/run.py @@ -7,9 +7,10 @@ than racing. * Ctrl-C or SIGTERM sends the engine a TERM, waits for it to flush, records the run as interrupted, and exits with ``CANCELLED``. -* ``resume`` continues from the recorded state; the engines already keep durable - fences and journals, so this restarts them against the same run directory - rather than inventing a checkpoint they do not have. +* ``resume`` continues an available miner contract from recorded state; those + engines keep durable fences and journals, so this restarts them against the + same run directory rather than inventing a checkpoint they do not have. + Runs from a now-quarantined contract remain historical and cannot resume. """ from __future__ import annotations @@ -24,6 +25,7 @@ from cathedral_node.contracts import codes as C from cathedral_node.contracts.version import schema_id from cathedral_node.engines import installer +from cathedral_node.engines.validator import historical_run_contract from cathedral_node.proc import stream from cathedral_node.runner import Context, command from cathedral_node.ui.console import Console @@ -51,12 +53,11 @@ def start(ctx: Context) -> Envelope: exit_code=Exit.UNSUPPORTED, remediation=Remediation( summary=( - "Weight submission needs a registered validator wallet with a permit and is an " - "owner decision, and it is gated behind Gate 6 of the launch specification. " - "There is deliberately no way to do it from here and no supported way around " - "this node: the single-publisher fence, the pending-attempt journal and the " - "signed authorization all live on this path, and an engine invoked directly " - "would hold none of them. Do not retry." + "This flag is retained only as a permanent refusal sentinel. The reviewed " + "Validator entrypoint is a separate direct writer, and this CLI exposes neither its " + "authorization nor its chain-write path. Do not retry or invoke that runtime " + "directly without an explicit owner-controlled procedure. " + "There is no supported way around this refusal from the CLI." ), docs="cathedral explain validator", requires_operator=True, @@ -64,6 +65,21 @@ def start(ctx: Context) -> Envelope: ) lock = lockfile.load() + static_blocker = engines.load(role, lock).operation_blocker() + if static_blocker is not None: + return Envelope.blocked( + "start", + static_blocker.get("code", C.E_ENGINE_INCOMPATIBLE), + f"this build cannot start {role}: {static_blocker.get('what', 'the engine contract is incompatible')}", + exit_code=Exit.INCOMPATIBLE, + remediation=Remediation( + summary="Nothing was started and no publisher fence was acquired.", + command=static_blocker.get("fix"), + docs=f"cathedral explain {role}", + requires_operator=bool(static_blocker.get("requires_operator")), + ), + detail={"blockers": [static_blocker]}, + ) # The whole verify-to-launch sequence happens under the SHARED lifecycle lock, # against one sealed group. An install, update, recovery or rollback takes the # exclusive form of the same lock, so it cannot interleave; and the launch @@ -154,12 +170,9 @@ def _start_verified(ctx: Context, role: str, lock, active) -> Envelope: return env record = state.create_run(ctx.run_id, role, "operate", f"{engine.title} running") - env_extra = engine.operate_env(cfg) log_path = paths.run_dir(ctx.run_id) / "engine.log" ctx.console.title(f"{engine.title} running", f"run {ctx.run_id}") - if role == "validator": - ctx.console.info("chain", "dry run — no weights are submitted") ctx.console.info("stop", "ctrl-c, or `cathedral stop " + role + "` from another terminal") ctx.console.blank() @@ -370,26 +383,64 @@ def own(child) -> None: @command("stop") def stop(ctx: Context) -> Envelope: role = ctx.args.role + historical = role == "validator" holder = state.running_run(role) if holder is None: return Envelope.ok("stop", {"role": role, "was_running": False, "detail": "not running"}) if ctx.dry_run: - env = Envelope.ok("stop", {"role": role, "was_running": True, "would_stop_pid": holder["pid"]}) + env = Envelope.ok("stop", { + "role": role, + "was_running": True, + "would_stop_pid": holder["pid"], + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None, + }) env.dry_run = True + if historical: + env.warn( + "contract.incompatible_live_process", + "The owned Validator process belongs to a quarantined historical contract.", + role=role, + run_id=holder.get("run_id"), + ) return env - stopped, detail = state.stop_role(role) + expected_run_id = holder.get("run_id") + if not expected_run_id: + return Envelope.fail( + "stop", + C.E_NOT_RUNNING, + f"could not stop {role}: the current owner has no provable run id", + exit_code=Exit.WORK_FAILED, + remediation=Remediation( + summary="The ownership record requires operator inspection; nothing was signalled.", + command=f"cathedral status {role}", + requires_operator=historical, + ), + detail={"role": role, "was_running": True, "stopped": False}, + ) + stopped, detail = state.stop_role(role, expected_run_id=expected_run_id) data = {"role": role, "was_running": True, "stopped": stopped, "detail": detail, - "pid": holder.get("pid"), "run_id": holder.get("run_id")} + "pid": holder.get("pid"), "run_id": holder.get("run_id"), + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None} if not stopped: return Envelope.fail( "stop", C.E_NOT_RUNNING, f"could not stop {role}: {detail}", exit_code=Exit.WORK_FAILED, - remediation=Remediation(summary=detail, command=f"cathedral status {role}"), + remediation=Remediation(summary=detail, command=f"cathedral status {role}", + requires_operator=historical), detail=data, ) env = Envelope.ok("stop", data) env.data_schema = schema_id("stop") + if historical: + env.warn( + "contract.historical_run", + env.data["contract_status"]["warning"], + role=role, + run_id=holder.get("run_id"), + ) return env @@ -402,6 +453,48 @@ def resume(ctx: Context) -> Envelope: "resume", C.E_RUN_NOT_FOUND, f"no run named {run_id}", exit_code=Exit.NOT_FOUND, remediation=Remediation(summary="List recent runs.", command="cathedral status"), ) + if record.role not in lockfile.ROLES: + return Envelope.fail( + "resume", + C.E_CONFIG_INVALID, + f"run {run_id} records unknown role {record.role!r}; refusing to interpret it", + exit_code=Exit.CONFIG_INVALID, + remediation=Remediation( + summary="The stored run record requires operator inspection.", + command=None, + requires_operator=True, + ), + ) + + # Consult the current role contract before interpreting ANY historical + # status. Otherwise an old live record looks retryable and an old completed + # record looks like present-day success, even though neither can be resumed + # under this build. + static_blocker = engines.load(record.role, lockfile.load()).operation_blocker() + if static_blocker is not None: + return Envelope.blocked( + "resume", + static_blocker.get("code", C.E_ENGINE_INCOMPATIBLE), + f"this build cannot resume {record.role}: " + f"{static_blocker.get('what', 'the engine contract is incompatible')}", + exit_code=Exit.INCOMPATIBLE, + remediation=Remediation( + summary=( + "The historical run remains recorded; no replacement process was started. " + "If its status is running, inspect status and let an operator decide whether " + "to stop the owned process." + ), + command=static_blocker.get("fix"), + docs=f"cathedral explain {record.role}", + requires_operator=bool(static_blocker.get("requires_operator")), + ), + detail={ + "run_id": run_id, + "previous_status": record.status, + "historical_contract": True, + }, + ) + record = state.reconcile(record) if record.status == "running": @@ -443,19 +536,93 @@ def cancel(ctx: Context) -> Envelope: "cancel", C.E_RUN_NOT_FOUND, f"no run named {run_id}", exit_code=Exit.NOT_FOUND, remediation=Remediation(summary="List recent runs.", command="cathedral status"), ) - record = state.reconcile(record) + if record.role not in lockfile.ROLES: + return Envelope.fail( + "cancel", + C.E_CONFIG_INVALID, + f"run {run_id} records unknown role {record.role!r}; refusing ownership access", + exit_code=Exit.CONFIG_INVALID, + remediation=Remediation( + summary="The stored run record requires operator inspection.", + command=None, + requires_operator=True, + ), + ) + historical = record.role == "validator" + if not historical: + record = state.reconcile(record) if record.status != "running": - return Envelope.ok("cancel", {"run_id": run_id, "status": record.status, - "cancelled": False, "detail": "already finished"}) + env = Envelope.ok("cancel", { + "run_id": run_id, + "status": record.status, + "cancelled": False, + "detail": "already finished", + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None, + }) + if historical: + env.warn( + "contract.historical_run", + env.data["contract_status"]["warning"], + role=record.role, + run_id=record.run_id, + ) + return env if ctx.dry_run: - env = Envelope.ok("cancel", {"run_id": run_id, "would_cancel": True}) + env = Envelope.ok("cancel", { + "run_id": run_id, + "would_attempt_cancel": True, + "expected_owned_run_id": run_id, + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None, + }) env.dry_run = True + if historical: + env.warn( + "contract.historical_run", + env.data["contract_status"]["warning"], + role=record.role, + run_id=record.run_id, + ) return env - stopped, detail = state.stop_role(record.role) + stopped, detail = state.stop_role(record.role, expected_run_id=run_id) + if not stopped: + return Envelope.fail( + "cancel", + C.E_NOT_RUNNING, + f"could not cancel {record.role}: {detail}", + exit_code=Exit.WORK_FAILED, + remediation=Remediation( + summary=( + "The run record was not changed because the owned process was not proven stopped." + ), + command=f"cathedral status {record.role}", + requires_operator=historical, + ), + detail={ + "run_id": run_id, + "cancelled": False, + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None, + }, + ) state.finish_run(record, "cancelled", int(Exit.CANCELLED), "cancelled by request") - env = Envelope.ok("cancel", {"run_id": run_id, "cancelled": stopped, "detail": detail, - "state_preserved": True}) + env = Envelope.ok("cancel", { + "run_id": run_id, + "cancelled": True, + "detail": detail, + "state_preserved": True, + "historical_contract": historical, + "contract_status": historical_run_contract() if historical else None, + }) env.data_schema = schema_id("cancel") + if historical: + env.warn( + "contract.historical_run", + env.data["contract_status"]["warning"], + role=record.role, + run_id=record.run_id, + ) return env diff --git a/cathedral_node/commands/setup.py b/cathedral_node/commands/setup.py index 2da9e82..1ea2f95 100644 --- a/cathedral_node/commands/setup.py +++ b/cathedral_node/commands/setup.py @@ -1,9 +1,10 @@ -"""`cathedral setup ` — install the signed release and write config. +"""`cathedral setup ` — install a signed release for an available role. A release always covers the whole node (compute + distill + validator), so setup installs and activates the entire signed group in one transaction; the named role is what its config and next steps are reported for. Idempotent: re-running the same -signed release is a no-op that still returns success. +signed release is a no-op that still returns success. A role with a static contract +mismatch is refused before release resolution or filesystem setup. """ from __future__ import annotations @@ -29,8 +30,23 @@ def setup(ctx: Context) -> Envelope: return Envelope.fail("setup", C.E_UNKNOWN_ROLE, f"unknown role {role!r}", exit_code=Exit.USAGE, remediation=Remediation(summary=f"Known roles: {', '.join(lockfile.ROLES)}.", command="cathedral capabilities --json")) + static_blocker = engines.load(role, lock).operation_blocker() + if static_blocker is not None: + return Envelope.blocked( + "setup", + static_blocker.get("code", C.E_ENGINE_INCOMPATIBLE), + f"this build cannot set up {role}: " + f"{static_blocker.get('what', 'the engine contract is incompatible')}", + exit_code=Exit.INCOMPATIBLE, + remediation=Remediation( + summary="Nothing was installed or configured; installation is not a compatibility fix.", + command=static_blocker.get("fix"), + docs=f"cathedral explain {role}", + requires_operator=bool(static_blocker.get("requires_operator")), + ), + detail={"blockers": [static_blocker]}, + ) paths.ensure_layout() - pin = lock.pin(role) # Read the current state under the lease. The install itself takes the # exclusive form of the same lock and re-verifies, so the lease is released # first rather than held across the transaction it would deadlock with. @@ -114,7 +130,20 @@ def progress(label: str, detail: str) -> None: qualification = engine.qualify(cfg) data = { "role": role, "installed": installed.to_dict(), "release": result, - "roles": {r: after[r].to_dict() for r in lockfile.ROLES}, + "roles": { + r: { + **after[r].to_dict(), + "contract_status": { + "execution_status": ( + "quarantined" + if engines.load(r, lock).operation_blocker() is not None + else "not_statically_blocked" + ), + "blocker": engines.load(r, lock).operation_blocker(), + }, + } + for r in lockfile.ROLES + }, "config_file": str(paths.config_file(role)), "config_problems": config.validate(role, cfg), "can_local_test": qualification.can_local_test, "can_operate": qualification.can_operate, "blockers": qualification.blockers, "notes": qualification.notes, "detail": detail, @@ -145,7 +174,15 @@ def _render(console: Console, data: dict[str, Any], env: Envelope) -> None: console.ok("installed", f"release v{release.get('release_version')} at {installed['short_revision']}") console.info("signed by", release.get("signer_identity", "?")) for other, st in data.get("roles", {}).items(): - console.info(other, "active" if st["installed"] else "not active") + if st.get("contract_status", {}).get("execution_status") == "quarantined": + label = ( + "retained package; execution quarantined" + if st["installed"] + else "not installed; execution quarantined" + ) + console.info(other, label) + else: + console.info(other, "active" if st["installed"] else "not active") console.info("config", data["config_file"]) for problem in data["config_problems"]: console.warn("config", problem) diff --git a/cathedral_node/commands/status.py b/cathedral_node/commands/status.py index 68765d1..3140aae 100644 --- a/cathedral_node/commands/status.py +++ b/cathedral_node/commands/status.py @@ -5,11 +5,12 @@ import datetime as _dt from typing import Any -from cathedral_node import config, lockfile, paths, revocation, state +from cathedral_node import config, engines, lockfile, paths, revocation, state from cathedral_node.contracts import Envelope, Exit, Remediation from cathedral_node.contracts import codes as C from cathedral_node.contracts.version import schema_id from cathedral_node.engines import installer +from cathedral_node.engines.validator import historical_run_contract from cathedral_node.runner import Context, command from cathedral_node.ui.console import Console from cathedral_node.ui.render import renders @@ -30,16 +31,35 @@ def status(ctx: Context) -> Envelope: # is held for the whole report. Releasing it first would let an activation commit # between the verification and the lines that describe it, so the report would # describe a node that no longer exists. - with installer.active_view(lock) as (states, _group, active_detail): - return _report(lock, roles, limit, states, active_detail) + with installer.active_view(lock) as (states, group, active_detail): + return _report(lock, roles, limit, states, group, active_detail) -def _report(lock, roles, limit, states, active_detail) -> Envelope: +def _report(lock, roles, limit, states, group, active_detail) -> Envelope: reports: dict[str, Any] = {} for role in roles: installed = states[role] holder = state.running_run(role) - recent = [state.reconcile(r).to_dict() for r in state.list_runs(role, limit=limit)] + blocker = engines.load(role, lock, group).operation_blocker() + legacy_config: dict[str, Any] | None = None + if blocker is not None: + legacy_config = { + "present": paths.config_file(role).exists(), + "inert": True, + "readable": True, + } + try: + config.load(role) + except config.ConfigError as exc: + # The retired relay configuration is evidence, not executable + # input. Report that it is unreadable without letting it take + # down status for the usable miner roles—or pretending it is OK. + legacy_config["readable"] = False + legacy_config["detail"] = str(exc) + recent = [ + _record_for_display(_observed_record(r)) + for r in state.list_runs(role, limit=limit) + ] last_test = next( (r for r in recent if r["kind"] == "test"), None ) @@ -48,7 +68,29 @@ def _report(lock, roles, limit, states, active_detail) -> Envelope: "running": holder is not None, "run": holder, "configured": paths.config_file(role).exists(), - "config_problems": config.validate(role, config.load(role)), + "config_problems": ( + [] if blocker is not None else config.validate(role, config.load(role)) + ), + "contract_status": { + "execution_status": ( + "not_statically_blocked" if blocker is None else "quarantined" + ), + "blocker": blocker, + "legacy_config": legacy_config, + }, + "incompatible_live_process": ( + { + "present": True, + "detail": ( + "A process is recorded under the quarantined Validator role. Status did " + "not signal it; an operator must inspect and decide whether to stop it." + ), + "command": "cathedral stop validator", + "requires_operator": True, + } + if holder is not None and blocker is not None + else None + ), "last_test": last_test, "recent_runs": recent[:limit], } @@ -62,6 +104,31 @@ def _report(lock, roles, limit, states, active_detail) -> Envelope: } env = Envelope.ok("status", data) env.data_schema = schema_id("status") + incompatible_live = [ + role + for role, report in reports.items() + if report["incompatible_live_process"] is not None + ] + for role in incompatible_live: + env.warn( + "contract.incompatible_live_process", + f"{role} has a live ownership record under a quarantined contract", + role=role, + ) + env.then( + f"Have an operator inspect and stop the incompatible {role} process", + f"cathedral stop {role}", + safe=False, + ) + for role, report in reports.items(): + legacy = report["contract_status"].get("legacy_config") + if legacy is not None and not legacy.get("readable", True): + env.warn( + "contract.legacy_config_unreadable", + "The quarantined Validator config is inert but unreadable; it was not treated " + "as valid configuration.", + role=role, + ) return env @@ -72,24 +139,54 @@ def _one_run(run_id: str) -> Envelope: "status", C.E_RUN_NOT_FOUND, f"no run named {run_id}", exit_code=Exit.NOT_FOUND, remediation=Remediation(summary="List recent runs.", command="cathedral status"), ) - record = state.reconcile(record) + record = _observed_record(record) events = list(state.read_events(run_id)) + shown_record = _record_for_display(record) data = { - "run": record.to_dict(), + "run": shown_record, "event_count": len(events), "events": events[-40:], "run_dir": paths.relative_to_home(paths.run_dir(run_id)), } env = Envelope.ok("status", data) env.data_schema = schema_id("run_status") + if shown_record.get("historical_contract"): + env.warn( + "contract.historical_run", + shown_record["contract_status"]["warning"], + role=record.role, + run_id=record.run_id, + ) return env +def _record_for_display(record: state.RunRecord) -> dict[str, Any]: + """Annotate old Validator evidence without rewriting the stored record.""" + displayed = record.to_dict() + if record.role == "validator": + displayed["historical_contract"] = True + displayed["contract_status"] = historical_run_contract() + return displayed + + +def _observed_record(record: state.RunRecord) -> state.RunRecord: + """Reconcile active contracts, but preserve quarantined evidence byte-for-byte.""" + if record.role == "validator": + return record + return state.reconcile(record) + + @renders("status") def _render(console: Console, data: dict[str, Any], env: Envelope) -> None: if "run" in data and "roles" not in data: record = data["run"] - console.title(f"Run {record['run_id']}", record["role"] + " · " + record["kind"]) + historical = bool(record.get("historical_contract")) + subtitle = record["role"] + " · " + record["kind"] + if historical: + subtitle += " · historical contract" + console.title(f"Run {record['run_id']}", subtitle) + if historical: + console.warn("contract", record["contract_status"]["warning"]) console.blank() console.kv_block( [ @@ -106,10 +203,17 @@ def _render(console: Console, data: dict[str, Any], env: Envelope) -> None: console.blank() console.rule("events") for event in data["events"]: - glyph = {"PASS": console.ok, "FAIL": console.fail, "ERROR": console.fail}.get( - event.get("status", "INFO"), console.info - ) - glyph(event.get("event", "").lower()[:11], event.get("detail", "")) + if historical: + console.info( + "historical", + f"{event.get('status', 'INFO')} " + f"{event.get('event', '').lower()[:11]} · {event.get('detail', '')}", + ) + else: + glyph = {"PASS": console.ok, "FAIL": console.fail, "ERROR": console.fail}.get( + event.get("status", "INFO"), console.info + ) + glyph(event.get("event", "").lower()[:11], event.get("detail", "")) return console.title("Status", data["home"]) @@ -117,7 +221,11 @@ def _render(console: Console, data: dict[str, Any], env: Envelope) -> None: console.blank() console.rule(role) installed = report["installed"] - if not installed["installed"]: + contract = report["contract_status"] + if contract["execution_status"] == "quarantined": + revision = installed["short_revision"] or installed["expected_short_revision"] + console.info("engine", f"retained pin {revision} · execution quarantined") + elif not installed["installed"]: console.info("engine", f"not installed · pinned {installed['expected_short_revision']}") elif installed["revision_drift"]: console.warn("engine", f"{installed['short_revision']} differs from pin " @@ -125,16 +233,31 @@ def _render(console: Console, data: dict[str, Any], env: Envelope) -> None: else: console.ok("engine", installed["short_revision"]) + legacy = contract.get("legacy_config") + if legacy is not None and not legacy.get("readable", True): + console.warn("legacy config", "inert and unreadable · " + legacy.get("detail", "")) + if report["running"]: run = report["run"] or {} - console.ok("running", f"pid {run.get('pid')} · since {run.get('since')} · run {run.get('run_id')}") + detail = f"pid {run.get('pid')} · since {run.get('since')} · run {run.get('run_id')}" + if report["incompatible_live_process"] is not None: + console.warn("running", detail + " · incompatible quarantined contract") + console.command("cathedral stop validator", indent=6) + else: + console.ok("running", detail) else: console.info("running", "no") last = report["last_test"] if last: - glyph = console.ok if last["status"] == "completed" else console.fail - glyph("last test", f"{last['status']} · {last['started_at']} · {last['detail']}") + if last.get("historical_contract"): + console.info( + "last test", + f"historical · {last['status']} · {last['started_at']} · {last['detail']}", + ) + else: + glyph = console.ok if last["status"] == "completed" else console.fail + glyph("last test", f"{last['status']} · {last['started_at']} · {last['detail']}") else: console.info("last test", "never run") diff --git a/cathedral_node/commands/test.py b/cathedral_node/commands/test.py index 3aa6d2c..f294d1a 100644 --- a/cathedral_node/commands/test.py +++ b/cathedral_node/commands/test.py @@ -35,6 +35,22 @@ def test(ctx: Context) -> Envelope: remediation=Remediation(summary=f"Known roles: {', '.join(lockfile.ROLES)}."), ) + static_blocker = engines.load(role, lock).operation_blocker() + if static_blocker is not None: + return Envelope.blocked( + "test", + static_blocker.get("code", C.E_ENGINE_INCOMPATIBLE), + f"this build cannot test {role}: {static_blocker.get('what', 'the engine contract is incompatible')}", + exit_code=Exit.INCOMPATIBLE, + remediation=Remediation( + summary="Nothing was run and no run record was created.", + command=static_blocker.get("fix"), + docs=f"cathedral explain {role}", + requires_operator=bool(static_blocker.get("requires_operator")), + ), + detail={"blockers": [static_blocker]}, + ) + # One strict verification, held under the shared lifecycle lock for the whole # test, and revalidated immediately before the engine subprocess starts. try: diff --git a/cathedral_node/commands/update.py b/cathedral_node/commands/update.py index 1ffba33..94e3b69 100644 --- a/cathedral_node/commands/update.py +++ b/cathedral_node/commands/update.py @@ -13,7 +13,7 @@ from typing import Any -from cathedral_node import lockfile, paths, state +from cathedral_node import engines, lockfile, paths, state from cathedral_node.commands import _release from cathedral_node.contracts import Envelope, Exit, Remediation from cathedral_node.contracts import codes as C @@ -27,6 +27,22 @@ @command("update") def update(ctx: Context) -> Envelope: + requested_role = getattr(ctx.args, "role", None) + if requested_role is not None: + return Envelope.fail( + "update", + C.E_USAGE, + "signed releases are node-wide; a role-qualified update would be misleading", + exit_code=Exit.USAGE, + remediation=Remediation( + summary=( + f"Nothing changed. `{requested_role}` cannot be updated independently; " + "review the complete node-wide plan first." + ), + command="cathedral update --check", + ), + detail={"requested_role": requested_role, "scope": "node-wide"}, + ) if getattr(ctx.args, "to", None): return Envelope.fail( "update", C.E_CONFIG_INVALID, @@ -75,9 +91,23 @@ def update(ctx: Context) -> Envelope: remediation=Remediation(summary=vreason, command="cathedral status")) available = bundle.authorization.release_version - plan = [{"role": r, "from_version": current[r].release_version, "to_version": available, - "changes": (current[r].release_version != available) or not current[r].installed} - for r in lockfile.ROLES] + plan = [ + { + "role": r, + "from_version": current[r].release_version, + "to_version": available, + "changes": (current[r].release_version != available) or not current[r].installed, + "contract_status": { + "execution_status": ( + "quarantined" + if engines.load(r, lock).operation_blocker() is not None + else "not_statically_blocked" + ), + "blocker": engines.load(r, lock).operation_blocker(), + }, + } + for r in lockfile.ROLES + ] data = {"plan": plan, "current_version": current_version, "available_version": available, "signer": source.identity, "applied": False, "recovery_required": paths.recovery_required(), "active_release": active_detail} @@ -177,14 +207,32 @@ def _render_update(console: Console, data: dict[str, Any], env: Envelope) -> Non console.title("Update", f"signed by {data.get('signer', '?')}" if data.get("signer") else "") console.info("current", f"v{data.get('current_version')}" if data.get("current_version") else "none") console.info("available", f"v{data.get('available_version')}" if data.get("available_version") else "?") - changing = [p["role"] for p in data.get("plan", []) if p.get("changes")] + changing = [p for p in data.get("plan", []) if p.get("changes")] if data.get("applied"): console.ok("applied", "the signed release is active" if changing else "already up to date") elif changing: - console.info("would change", ", ".join(changing)) + console.info( + "would change", + ", ".join( + p["role"] + + ( + " (retained package; execution quarantined)" + if p.get("contract_status", {}).get("execution_status") == "quarantined" + else "" + ) + for p in changing + ), + ) else: console.ok("up to date", "no change") + for item in data.get("plan", []): + if item.get("contract_status", {}).get("execution_status") == "quarantined": + console.info( + item["role"], + "retained signed-release member; operational execution remains quarantined", + ) + @renders("rollback") def _render_rollback(console: Console, data: dict[str, Any], env: Envelope) -> None: diff --git a/cathedral_node/config.py b/cathedral_node/config.py index 4715f99..d56565b 100644 --- a/cathedral_node/config.py +++ b/cathedral_node/config.py @@ -37,9 +37,9 @@ ) -# Cathedral's published weight-policy signing key (key_id "cathedral-weight-policy"). -# Public by design: a validator's control is which key it will accept, so the value -# has to be readable in both the human and --json views rather than masked. +# Public key retained from the retired signed-feed adapter for deterministic +# audit and migration. It is public by design and remains readable in human and +# JSON views, but it is not authority for the reviewed direct Validator. SN39_WEIGHT_POLICY_PUBLIC_KEY = "10890a66aa752479cb3b634f366d7bd27c374324d83f88d2d6b69ab066f25e26" _WEIGHT_POLICY_KEY = re.compile(r"[0-9a-fA-F]{64}") @@ -70,10 +70,20 @@ def is_forbidden_secret(name: str) -> bool: class ConfigError(Exception): """Raised with a message written for the operator, not the developer.""" - def __init__(self, message: str, field: str | None = None, remedy: str | None = None) -> None: + def __init__( + self, + message: str, + field: str | None = None, + remedy: str | None = None, + *, + role: str | None = None, + requires_operator: bool = False, + ) -> None: super().__init__(message) self.field = field self.remedy = remedy + self.role = role + self.requires_operator = requires_operator @dataclasses.dataclass(slots=True) @@ -129,37 +139,36 @@ def to_dict(self) -> dict[str, Any]: secret_ref=True, default="COMPUTE_BEARER_TOKEN"), ), "validator": ( - Field("network", "Bittensor network label.", default="finney", choices=("finney", "test")), - Field("netuid", "Subnet id.", default=39), - Field("wallet_name", "Bittensor wallet name holding your validator hotkey.", default="validator"), - Field("wallet_hotkey", "Validator hotkey name inside that wallet.", default="default"), - Field("publisher_url", "Signed score feed.", default="https://api.cathedral.computer"), - Field("interval_secs", "Seconds between ticks.", default=1500), - Field("provenance", "Audit mode.", default="shadow", + # Preserved for deterministic migration from the retired signed-feed + # adapter. The Validator execution paths are statically quarantined; + # these values are readable but do not configure the reviewed direct + # writer. + Field("network", "Legacy Validator network label; inert while quarantined.", + default="finney", choices=("finney", "test")), + Field("netuid", "Legacy Validator subnet id; inert while quarantined.", default=39), + Field("wallet_name", "Legacy wallet name; never launched by this CLI.", default="validator"), + Field("wallet_hotkey", "Legacy hotkey name; never launched by this CLI.", default="default"), + Field("publisher_url", "Retired signed-feed URL; preserved for migration only.", + default="https://api.cathedral.computer"), + Field("interval_secs", "Retired relay interval; preserved for migration only.", default=1500), + Field("provenance", "Retired relay audit mode; preserved for migration only.", default="shadow", choices=("off", "shadow", "authority", "full", "thin")), - # Burn and allocation are deliberately ABSENT. They arrive inside the - # Cathedral-signed weight vector and from Cathedral-signed burn and - # allocation documents; nothing local changes them. Offering them here - # would let an operator believe they had changed the economics when - # nothing had changed -- the worst kind of setting. - # - # What an operator genuinely controls is what they will ACCEPT: + # Burn and allocation remain absent. These last fields describe the + # retired signed-feed acceptance contract, not current owner controls. Field("require_policy", - "The weight-policy contract this validator will accept. Finney SN39 " - "broadcast requires `validated_supply_v1`.", + "Retired signed-feed policy contract; not direct-Validator authority.", default="validated_supply_v1"), Field("weight_policy_key", - "Public signing key (64 hex chars) whose signed weight vectors you " - "will accept. Public by design: read it aloud, check it against the " - "published key.", + "Retired signed-feed public key (64 hex chars), retained for migration only.", default=SN39_WEIGHT_POLICY_PUBLIC_KEY), ), } -# Owner policy an update must never silently change. `cathedral update` diffs -# these and refuses to proceed if a new default would move one. +# No Validator field is current owner authority while its runtime contract is +# quarantined. Keep the key so callers can distinguish "known role, no active +# controls" from an unknown role. OWNER_CONTROLLED = { - "validator": ("wallet_name", "wallet_hotkey", "require_policy", "weight_policy_key"), + "validator": (), } @@ -184,14 +193,18 @@ def load(role: str) -> dict[str, Any]: except (OSError, tomllib.TOMLDecodeError) as exc: raise ConfigError( f"{paths.relative_to_home(path)} is not valid TOML: {exc}", - remedy=f"cathedral config reset {role}", + remedy=( + f"Edit or move {paths.relative_to_home(path)} manually; this CLI has no " + "config-reset command and did not change the file." + ), + role=role, + requires_operator=True, ) from exc - # The weight-policy key is declared public: register the literal so the - # redaction heuristics never mask it. A masked key would hide the - # operator's signing-key control from an agent reading --json, while the - # human view showed it in full -- one envelope, two answers. + # The retired weight-policy key is public: register the literal so the + # redaction heuristics do not make audit/migration output differ between + # human and JSON views. key = values.get("weight_policy_key") - if key: + if role == "validator" and key == SN39_WEIGHT_POLICY_PUBLIC_KEY: redact.register_public_values([key]) return values @@ -216,7 +229,6 @@ def save(role: str, values: dict[str, Any]) -> Path: def validate(role: str, values: dict[str, Any]) -> list[str]: """Problems with this configuration, in operator language. Empty means good.""" problems: list[str] = [] - fields = {f.name: f for f in schema(role)} for key in values: if key.lower() in FORBIDDEN_FIELDS: @@ -260,8 +272,18 @@ def validate(role: str, values: dict[str, Any]) -> list[str]: f"key); found {len(str(key))} characters" ) interval = values.get("interval_secs") - if interval is not None and int(interval) < 60: - problems.append("`interval_secs` below 60 will be throttled by the chain's write cooldown") + if interval is not None: + try: + parsed_interval = int(interval) + except (TypeError, ValueError): + problems.append( + "`interval_secs` must be an integer in the retired relay configuration" + ) + else: + if parsed_interval < 60: + problems.append( + "`interval_secs` below 60 was invalid for the retired relay's chain cooldown" + ) return problems diff --git a/cathedral_node/contracts/codes.py b/cathedral_node/contracts/codes.py index 658bc95..adf1851 100644 --- a/cathedral_node/contracts/codes.py +++ b/cathedral_node/contracts/codes.py @@ -55,7 +55,7 @@ class Exit(IntEnum): # 50+ — the operator or the OS interrupted us. CANCELLED = 50 - """Interrupted. Durable state was flushed; ``resume`` will continue.""" + """Interrupted. State was flushed; an available miner contract may resume it.""" INTERNAL = 70 """A bug in this CLI. Always includes a diagnostic bundle path.""" @@ -86,7 +86,9 @@ def retryable(code: Exit | int) -> bool: Exit.INCOMPATIBLE: "Protocol or engine version mismatch. Stop and re-discover.", Exit.NETWORK: "A required remote was unreachable. Always safe to retry.", Exit.UPSTREAM: "A pinned engine failed in a way this layer does not model.", - Exit.CANCELLED: "Interrupted. Durable state was flushed; `resume` will continue.", + Exit.CANCELLED: ( + "Interrupted. Durable state was flushed; available miner contracts may resume it." + ), Exit.INTERNAL: "A bug in this CLI. Always includes a diagnostics bundle path.", } diff --git a/cathedral_node/engines/base.py b/cathedral_node/engines/base.py index 18de76b..0221d87 100644 --- a/cathedral_node/engines/base.py +++ b/cathedral_node/engines/base.py @@ -119,6 +119,19 @@ def capabilities(self) -> dict[str, Any]: def qualify(self, cfg: dict[str, Any]) -> Qualification: """Can this machine and identity do the work? Never optimistic.""" + def operation_blocker(self) -> dict[str, Any] | None: + """Return a static fail-closed blocker before release verification. + + Most adapters need their verified generation before deciding whether an + operation can run. A known contract mismatch is different: asking an + operator to install or verify an already-incompatible engine is itself + misleading. Such adapters override this hook so command handlers can + refuse before creating a run, acquiring a publisher fence, or resolving + an executable. + """ + + return None + # ---- operations ----------------------------------------------------------- @abc.abstractmethod @@ -154,6 +167,11 @@ def bin(self, name: str) -> Path: raise UnverifiedEngine( f"the {self.role} engine has no verified generation bound; refusing to resolve " f"an executable path") + if self._verified.receipt_data.get("execution_validation") != "runtime_checked": + raise UnverifiedEngine( + f"the {self.role} generation was verified as static quarantine only; " + "refusing to resolve an executable path" + ) return self._verified.bin(name) def has_bin(self, name: str) -> bool: @@ -161,6 +179,8 @@ def has_bin(self, name: str) -> bool: no — honest, because nothing is runnable until the group verifies.""" if self._verified is None: return False + if self._verified.receipt_data.get("execution_validation") != "runtime_checked": + return False return self._verified.has_bin(name) def python(self) -> Path: @@ -168,6 +188,11 @@ def python(self) -> Path: raise UnverifiedEngine( f"the {self.role} engine has no verified generation bound; refusing to resolve " f"an interpreter path") + if self._verified.receipt_data.get("execution_validation") != "runtime_checked": + raise UnverifiedEngine( + f"the {self.role} generation was verified as static quarantine only; " + "refusing to resolve an interpreter path" + ) return self._verified.python def child_env(self, cfg: dict[str, Any] | None = None) -> dict[str, str]: diff --git a/cathedral_node/engines/installer.py b/cathedral_node/engines/installer.py index ed83f4b..d89959b 100644 --- a/cathedral_node/engines/installer.py +++ b/cathedral_node/engines/installer.py @@ -11,8 +11,11 @@ --only-binary=:all:`` from the already-verified local wheels only — no index, no dependency resolution, no source build, no build backend. A malicious PEP 517 backend is never invoked because source is never built. -3. Each generation is verified in place (interpreter is a byte-copy of the trusted - parent; declared entrypoints run and any nonzero exit fails). The receipt is +3. Each available generation is verified in place (interpreter is a byte-copy of the + trusted parent; declared entrypoints run and any nonzero exit fails). A statically + quarantined role is verified from files and signed metadata only; after wheel + installation no runtime/import/entrypoint probe executes its newly installed + package code. The receipt is written, the *complete* generation — source, venv, receipt and the generation root — is frozen read-only, and only then is the whole thing re-verified. Any failure at any step removes the incomplete generation before it can be named by anything. @@ -75,7 +78,9 @@ from cathedral_node.release_lock import AuthorizedBundle from cathedral_node.verified import VerifiedActiveGroup, VerifiedRole -RECEIPT_SCHEMA = "cathedral.node.engine_receipt.v5" +RECEIPT_SCHEMA = "cathedral.node.engine_receipt.v6" +EXECUTION_RUNTIME_CHECKED = "runtime_checked" +EXECUTION_STATIC_QUARANTINE = "static_quarantine" POINTER_SCHEMA = "cathedral.node.active_release.v1" FLOOR_SCHEMA = "cathedral.node.release_floor.v2" _MIN = (3, 11) @@ -93,13 +98,13 @@ "parent_base_executable", "parent_base_sha256", "venv_python", "venv_python_sha256", "venv_python_stat", "manifest_sha256", "source_sha256", "release_version", "signer_identity", "lock_digest", "extras", "entrypoints", "server_entrypoints", - "launch_mode", "protocol", "installed_at", + "launch_mode", "protocol", "execution_validation", "installed_at", } _RECEIPT_STRING_FIELDS = ( "schema", "role", "generation", "repository", "revision", "distribution", "version", "parent_base_executable", "parent_base_sha256", "venv_python", "venv_python_sha256", "manifest_sha256", "source_sha256", "signer_identity", "lock_digest", "launch_mode", - "protocol", "installed_at", + "protocol", "execution_validation", "installed_at", ) _RECEIPT_LIST_FIELDS = ("extras", "entrypoints", "server_entrypoints") _STAT_KEYS = ("uid", "gid", "mode", "device", "inode", "size") @@ -544,7 +549,12 @@ def _hash_tree(root: Path) -> tuple[list[list[Any]], str]: return entries, "" -def _local_manifest(gen_dir: Path, pin: EnginePin) -> tuple[bool, str, str]: +def _local_manifest( + gen_dir: Path, + pin: EnginePin, + *, + execution_validation: str = EXECUTION_RUNTIME_CHECKED, +) -> tuple[bool, str, str]: source_entries, reason = _hash_tree(gen_dir / "source") if reason: return False, "", reason @@ -553,7 +563,8 @@ def _local_manifest(gen_dir: Path, pin: EnginePin) -> tuple[bool, str, str]: return False, "", reason profile = {"extras": sorted(pin.extras), "entrypoints": sorted(pin.entrypoints), "server_entrypoints": sorted(pin.server_entrypoints), - "launch_mode": pin.launch_mode, "protocol": pin.protocol} + "launch_mode": pin.launch_mode, "protocol": pin.protocol, + "execution_validation": execution_validation} return True, _digest({"source": source_entries, "venv": venv_entries, "profile": profile}), "" @@ -764,7 +775,19 @@ def _scrubbed_engine_env(gen_dir: Path) -> dict[str, str]: _SERVER_READY_WINDOW = 2.0 # a server must still be up after this; a crash exits sooner -def _self_check(venv: Path, pin: EnginePin, gen_dir: Path) -> tuple[bool, str]: +def _self_check( + venv: Path, + pin: EnginePin, + gen_dir: Path, + *, + allow_server_start: bool = True, +) -> tuple[bool, str]: + if not allow_server_start: + # Historical parameter name retained for compatibility. For a statically + # blocked role it means *no installed runtime execution at all*. Even + # `--help`, an import probe, or `python -c` can execute sitecustomize/.pth + # code from the package environment; argv is not a sandbox. + return True, "static quarantine check only; no installed code executed" env = _scrubbed_engine_env(gen_dir) run = proc.probe([str(venv / "bin" / "python"), "-I", "-B", "-c", "print('ok')"], timeout=30, env=env, inherit_env=False) @@ -800,7 +823,7 @@ def _self_check(venv: Path, pin: EnginePin, gen_dir: Path) -> tuple[bool, str]: def _prepare_generation(role: str, spec: release_lock.RoleRelease, bundle: AuthorizedBundle, pin: EnginePin, base_exec: Path, base_sha: str, node_abi: str, node_platform: str, on_progress: Callable[[str, str], None], - log_path: Path | None) -> str: + log_path: Path | None, *, allow_server_start: bool = True) -> str: """Build, verify, receipt, freeze and re-verify one generation. Any failure removes the incomplete generation before returning, so a half-built @@ -811,7 +834,8 @@ def _prepare_generation(role: str, spec: release_lock.RoleRelease, bundle: Autho gen_dir.mkdir(parents=True, exist_ok=False) try: return _build_generation(role, generation, gen_dir, spec, bundle, pin, base_exec, - base_sha, node_abi, node_platform, on_progress, log_path) + base_sha, node_abi, node_platform, on_progress, log_path, + allow_server_start=allow_server_start) except BaseException: _force_rmtree(gen_dir) _journal("PREPARE_FAILED", role=role, generation=generation) @@ -821,7 +845,8 @@ def _prepare_generation(role: str, spec: release_lock.RoleRelease, bundle: Autho def _build_generation(role: str, generation: str, gen_dir: Path, spec: release_lock.RoleRelease, bundle: AuthorizedBundle, pin: EnginePin, base_exec: Path, base_sha: str, node_abi: str, node_platform: str, - on_progress: Callable[[str, str], None], log_path: Path | None) -> str: + on_progress: Callable[[str, str], None], log_path: Path | None, *, + allow_server_start: bool = True) -> str: # Inert source archive: provenance only. Copied, RE-HASHED at the destination # (a copy could be raced), and never built. source = gen_dir / "source" @@ -850,6 +875,20 @@ def _build_generation(role: str, generation: str, gen_dir: Path, if not ok: raise InstallError(f"{role}: {detail}") + # Evaluate the signed wheel closure while the venv is still pristine. This + # uses only the interpreter/pip seeded by the trusted parent and inert + # METADATA read from the already hash-verified wheels; none of the candidate + # packages has been installed yet, so no candidate .pth/import hook can run. + # It is therefore also the dependency/marker check for statically + # quarantined roles, whose installed interpreter must never be started. + closure_ok, closure_detail = _verify_signed_wheel_closure( + venv, wheelhouse, spec, pin, gen_dir + ) + if not closure_ok: + raise InstallError( + f"{role}: dependency closure/markers not satisfied: {closure_detail}" + ) + # Offline, hash-pinned, wheels-only install. Requirements reference each wheel by # its validated single-component filename relative to the wheelhouse (cwd), so no # attacker-influenced path is ever interpolated into the requirements file. @@ -871,16 +910,23 @@ def _build_generation(role: str, generation: str, gen_dir: Path, if not pip.ok: raise InstallError(f"{role}: offline install failed: {pip.tail(8)}") - # The declared distribution must import; the receipt records the VERIFIED - # installed version, cross-checked against the signed release. - dist_ok, installed_version, dist_detail = _verify_distribution(venv, pin, gen_dir) + # Available roles prove runtime importability. Once a statically quarantined + # role's signed wheels have been installed, no interpreter/import/entrypoint + # probe may execute their code, so it receives a file-only check instead. + if allow_server_start: + dist_ok, installed_version, dist_detail = _verify_distribution(venv, pin, gen_dir) + else: + dist_ok, installed_version, dist_detail = _verify_distribution_files(venv, pin) if not dist_ok: raise InstallError(f"{role}: {dist_detail}") if installed_version != spec.version: raise InstallError(f"{role}: installed {pin.distribution} {installed_version!r} != signed {spec.version!r}") - closure_ok, closure_detail = _verify_closure(venv, pin, gen_dir) - if not closure_ok: - raise InstallError(f"{role}: dependency closure/markers not satisfied: {closure_detail}") + if allow_server_start: + closure_ok, closure_detail = _verify_closure(venv, pin, gen_dir) + if not closure_ok: + raise InstallError( + f"{role}: dependency closure/markers not satisfied: {closure_detail}" + ) # Strip the venv's own precompiled bytecode, then require none remains, then # freeze the trees read-only so nothing can be written while the generation is live. @@ -891,8 +937,20 @@ def _build_generation(role: str, generation: str, gen_dir: Path, _freeze_readonly(venv) _freeze_readonly(source) - on_progress("self-check", f"{role}: running declared entrypoints") - ok, detail = _self_check(venv, pin, gen_dir) + on_progress( + "self-check", + ( + f"{role}: running declared entrypoints" + if allow_server_start + else f"{role}: static quarantine check; no installed code is executed" + ), + ) + ok, detail = _self_check( + venv, + pin, + gen_dir, + allow_server_start=allow_server_start, + ) if not ok: raise InstallError(f"{role}: {detail}") # Post-health full-tree reverify: the self-check must not have produced bytecode. @@ -900,7 +958,12 @@ def _build_generation(role: str, generation: str, gen_dir: Path, if bytecode is not None: raise InstallError(f"{role}: bytecode appeared during the health check at {bytecode}") - ok, manifest_sha, detail = _local_manifest(gen_dir, pin) + execution_validation = ( + EXECUTION_RUNTIME_CHECKED if allow_server_start else EXECUTION_STATIC_QUARANTINE + ) + ok, manifest_sha, detail = _local_manifest( + gen_dir, pin, execution_validation=execution_validation + ) if not ok: raise InstallError(f"{role}: {detail}") @@ -916,7 +979,9 @@ def _build_generation(role: str, generation: str, gen_dir: Path, "signer_identity": bundle.authorization.identity, "lock_digest": bundle.authorization.lock_digest, "extras": list(spec.extras), "entrypoints": list(spec.entrypoints), "server_entrypoints": list(spec.server_entrypoints), "launch_mode": spec.launch_mode, - "protocol": spec.protocol, "installed_at": utcnow(), + "protocol": spec.protocol, + "execution_validation": execution_validation, + "installed_at": utcnow(), } _write_json_atomic(gen_dir / "receipt.json", receipt) @@ -925,8 +990,13 @@ def _build_generation(role: str, generation: str, gen_dir: Path, # runtime read uses. A generation that cannot pass its own verifier is removed # by the caller rather than published. _freeze_generation(gen_dir) - expected = _expected(spec, bundle.authorization.release_version, - bundle.authorization.lock_digest, bundle.authorization.identity) + expected = _expected( + spec, + bundle.authorization.release_version, + bundle.authorization.lock_digest, + bundle.authorization.identity, + execution_validation, + ) ok, _data, reason = _verify_generation(pin, generation, expected, base_exec=base_exec, base_sha=base_sha) if not ok: @@ -951,6 +1021,37 @@ def _verify_distribution(venv: Path, pin: EnginePin, gen_dir: Path) -> tuple[boo return True, version, "ok" +def _verify_distribution_files(venv: Path, pin: EnginePin) -> tuple[bool, str, str]: + """Verify an inert installed distribution without starting its interpreter.""" + metadata_files = [ + *venv.glob("lib/python*/site-packages/*.dist-info/METADATA"), + *(venv / "Lib" / "site-packages").glob("*.dist-info/METADATA"), + ] + matches: list[tuple[Path, str]] = [] + for metadata_path in metadata_files: + if metadata_path.is_symlink() or metadata_path.parent.is_symlink(): + continue + try: + metadata = email.parser.Parser().parsestr( + metadata_path.read_text(encoding="utf-8", errors="replace") + ) + except OSError: + continue + if release_lock.normalized_name(metadata.get("Name", "")) == release_lock.normalized_name( + pin.distribution + ): + matches.append((metadata_path, metadata.get("Version", ""))) + if len(matches) != 1: + return False, "", ( + f"{pin.distribution} has {len(matches)} installed metadata records; expected exactly one" + ) + for entrypoint in (*pin.entrypoints, *pin.server_entrypoints): + script = venv / "bin" / entrypoint + if not script.is_file() or script.is_symlink(): + return False, "", f"required inert entrypoint {entrypoint!r} is missing or unsafe" + return True, matches[0][1], "ok" + + _IMPORT_PROBE = r""" import importlib.metadata as m, importlib.util as u, sys try: @@ -1038,8 +1139,6 @@ def check(distname, extras): def _verify_wheels(wheelhouse: Path, spec: release_lock.RoleRelease, node_abi: str, node_platform: str) -> tuple[bool, str]: - closure = {release_lock.normalized_name(a.name) for a in spec.artifacts} - requested = {e.lower() for e in spec.extras} for artifact in spec.artifacts: wheel = wheelhouse / artifact.file try: @@ -1059,20 +1158,186 @@ def _verify_wheels(wheelhouse: Path, spec: release_lock.RoleRelease, return False, f"{artifact.file}: internal Version disagrees with the signed release" if not _tags_compatible(wheel_md.get_all("Tag") or [], node_abi, node_platform): return False, f"{artifact.file}: wheel tags are not compatible with {node_abi}/{node_platform}" - for req in (metadata.get_all("Requires-Dist") or []): - name, extra, has_marker = _parse_requires(req) - if name is None: - continue - if extra is not None: - if extra.lower() not in requested: - continue # gated by an extra we did not request - elif has_marker: - continue # a platform/python conditional; do not force it into the closure - if release_lock.normalized_name(name) not in closure: - return False, f"incomplete closure: {artifact.name} requires {name!r}, absent from the signed set" return True, "ok" +def _verify_signed_wheel_closure( + venv: Path, + wheelhouse: Path, + spec: release_lock.RoleRelease, + pin: EnginePin, + gen_dir: Path, +) -> tuple[bool, str]: + """Evaluate versions, markers and requested extras without candidate code. + + The probe runs before ``pip install`` in the newly created venv. Its only + executable dependency is pip's packaging parser supplied by the trusted + Python/venv bootstrap. Candidate wheels are opened as zip files by this + process and represented to the probe as JSON metadata; they are never added + to ``sys.path``. + """ + distributions: list[dict[str, Any]] = [] + for artifact in spec.artifacts: + wheel = wheelhouse / artifact.file + try: + with zipfile.ZipFile(wheel) as archive: + info = _wheel_dist_info(archive, artifact.name) + if info is None: + return False, f"{artifact.file} has no matching .dist-info/METADATA" + metadata = email.parser.Parser().parsestr( + archive.read(f"{info}/METADATA").decode("utf-8", "replace") + ) + except (zipfile.BadZipFile, KeyError, OSError, RuntimeError, ValueError): + return False, f"{artifact.file} is not a readable wheel" + distributions.append({ + "name": metadata.get("Name", ""), + "version": metadata.get("Version", ""), + "requires": metadata.get_all("Requires-Dist") or [], + "provides_extras": metadata.get_all("Provides-Extra") or [], + }) + + payload = { + "target": pin.distribution, + "requested_extras": list(pin.extras), + "distributions": distributions, + } + fd, raw_path = tempfile.mkstemp(prefix=".closure-metadata-", suffix=".json", dir=gen_dir) + payload_path = Path(raw_path) + try: + with os.fdopen(fd, "wb") as stream: + stream.write(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(payload_path, 0o600) + probe = proc.run( + [str(venv / "bin" / "python"), "-I", "-B", "-c", _SIGNED_CLOSURE_PROBE, + str(payload_path)], + timeout=120, + env=_scrubbed_engine_env(gen_dir), + inherit_env=False, + ) + finally: + with contextlib.suppress(OSError): + payload_path.unlink() + + try: + result = json.loads(probe.stdout.strip() or "{}") + except json.JSONDecodeError: + return False, "the signed-metadata closure probe did not report" + if result.get("error"): + return False, str(result["error"]) + problems = result.get("problems") or [] + if problems: + return False, "; ".join(str(problem) for problem in problems[:4]) + if not probe.ok: + return False, "the signed-metadata closure probe failed" + return True, "ok" + + +_SIGNED_CLOSURE_PROBE = r''' +import json, re, sys +try: + from pip._vendor.packaging.requirements import Requirement + from pip._vendor.packaging.markers import default_environment +except Exception as e: + print(json.dumps({"error": "packaging is unavailable to evaluate markers: %s" % e})); raise SystemExit +def norm(n): return re.sub(r"[-_.]+", "-", n).strip("-").lower() +try: + with open(sys.argv[1], "r", encoding="utf-8") as stream: + payload = json.load(stream) +except Exception as e: + print(json.dumps({"error": "cannot read signed wheel metadata: %s" % e})); raise SystemExit +records = {} +problems = [] +for item in payload.get("distributions", []): + name = norm(item.get("name", "")) + if not name or name in records: + problems.append("duplicate or empty distribution name %r" % item.get("name", "")) + continue + records[name] = item +target = norm(payload.get("target", "")) +if target not in records: + problems.append("target distribution %s is absent" % payload.get("target", "")) +selected = {name: set() for name in records} +def extra_aliases(raw): + text = str(raw) + canonical = norm(text) + return { + text, text.lower(), canonical, + canonical.replace("-", "_"), canonical.replace("-", "."), + } +declared = { + name: { + norm(raw): extra_aliases(raw) + for raw in item.get("provides_extras", []) + if norm(raw) + } + for name, item in records.items() +} +if target in selected: + requested = {norm(x) for x in payload.get("requested_extras", []) if norm(x)} + unknown = sorted(requested - set(declared[target])) + for extra in unknown: + problems.append("target requests undeclared extra %s" % extra) + selected[target].update(requested & set(declared[target])) +parsed = {} +for name, item in records.items(): + parsed[name] = [] + for text in item.get("requires", []): + try: + req = Requirement(text) + if req.url: + problems.append("%s uses unsupported direct reference %r" % (item.get("name", name), text)) + continue + parsed[name].append(req) + except Exception: + problems.append("%s has an invalid Requires-Dist entry %r" % (item.get("name", name), text)) +marker_errors = set() +def applicable(owner, req, extras): + if req.marker is None: return True + values = [""] + for extra in sorted(extras): + values.extend(sorted(declared[owner].get(extra, {extra}))) + for extra in values: + env = default_environment(); env["extra"] = extra + try: + if req.marker.evaluate(env): return True + except Exception as e: + marker_errors.add("%s marker %r could not be evaluated: %s" % (owner, str(req.marker), e)) + return False + return False +# Propagate dependency-requested extras to a fixed point. Base requirements of +# every signed distribution remain active; optional requirements become active +# only when their owning distribution's extra was requested through the graph. +changed = True +while changed: + changed = False + for owner, requirements in parsed.items(): + for req in requirements: + if not applicable(owner, req, selected[owner]): continue + dependency = norm(req.name) + if dependency not in records: continue + additions = {norm(extra) for extra in req.extras if norm(extra)} + unknown = sorted(additions - set(declared[dependency])) + for extra in unknown: + problems.append("%s requests undeclared extra %s from %s" % (owner, extra, dependency)) + additions &= set(declared[dependency]) + if not additions.issubset(selected[dependency]): + selected[dependency].update(additions); changed = True +for owner, requirements in parsed.items(): + for req in requirements: + if not applicable(owner, req, selected[owner]): continue + dependency = norm(req.name) + if dependency not in records: + problems.append("missing dependency %s" % req.name); continue + version = str(records[dependency].get("version", "")) + if req.specifier and not req.specifier.contains(version, prereleases=True): + problems.append("%s %s does not satisfy %s" % (req.name, version, req.specifier)) +problems.extend(sorted(marker_errors)) +print(json.dumps({"problems": problems})) +''' + + def _wheel_dist_info(archive: zipfile.ZipFile, name: str) -> str | None: want = release_lock.normalized_name(name) for entry in archive.namelist(): @@ -1086,15 +1351,6 @@ def _wheel_dist_info(archive: zipfile.ZipFile, name: str) -> str | None: return None -def _parse_requires(req: str) -> tuple[str | None, str | None, bool]: - head, _, marker = req.partition(";") - match = re.match(r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)", head) - if not match: - return None, None, False - extra_match = re.search(r"""extra\s*==\s*["']([^"']+)["']""", marker) - return match.group(1), (extra_match.group(1) if extra_match else None), bool(marker.strip()) - - def _tags_compatible(tags: list[str], node_abi: str, node_platform: str) -> bool: plat_norm = re.sub(r"[-.]", "_", node_platform) # A Linux node accepts the standard portable-Linux wheel tags for its own @@ -1267,8 +1523,25 @@ def _install_release(bundle_dir, lock, allowed_signers_path, *, identity, superv if not authorized: raise InstallError(why) spec = auth.role(role) - prepared[role] = _prepare_generation(role, spec, bundle, pin, base_exec, base_sha, - node_abi, node_platform, on_progress, log_path) + # A node-wide miner install may retain a statically quarantined + # package, but after installation no runtime/import/entrypoint probe + # may execute code from that quarantined environment. + allow_server_start = ( + _role_execution_validation(role, lock) == EXECUTION_RUNTIME_CHECKED + ) + prepared[role] = _prepare_generation( + role, + spec, + bundle, + pin, + base_exec, + base_sha, + node_abi, + node_platform, + on_progress, + log_path, + allow_server_start=allow_server_start, + ) except Exception: _cleanup_uncommitted(prepared) _discard_unreferenced_release(auth.lock_digest) @@ -1747,15 +2020,36 @@ def _trust_names_identity(allowed_signers: str, identity: str) -> bool: return False -def _expected(spec: release_lock.RoleRelease, release_version: int, lock_digest: str, - identity: str) -> dict[str, Any]: - """Every launch-relevant fact the receipt must equal, taken from the SIGNED role - spec and the verified pointer — the receipt's own claims are never trusted.""" +def _expected( + spec: release_lock.RoleRelease, + release_version: int, + lock_digest: str, + identity: str, + execution_validation: str, +) -> dict[str, Any]: + """Every launch-relevant fact the receipt must equal. + + Release facts come from the signed role spec and verified pointer. The + execution tier comes from this CLI's fail-closed role policy. Neither is + inferred from the receipt itself. + """ return {"repository": spec.repository, "revision": spec.revision, "distribution": spec.distribution, "version": spec.version, "source_sha256": spec.source_sha256, "extras": sorted(spec.extras), "entrypoints": sorted(spec.entrypoints), "server_entrypoints": sorted(spec.server_entrypoints), "launch_mode": spec.launch_mode, "protocol": spec.protocol, - "release_version": release_version, "signer_identity": identity, "lock_digest": lock_digest} + "release_version": release_version, "signer_identity": identity, "lock_digest": lock_digest, + "execution_validation": execution_validation} + + +def _role_execution_validation(role: str, lock: lockfile.Lock) -> str: + """The validation tier required by the current fail-closed adapter policy.""" + from cathedral_node import engines as engine_adapters + + return ( + EXECUTION_RUNTIME_CHECKED + if engine_adapters.load(role, lock).operation_blocker() is None + else EXECUTION_STATIC_QUARANTINE + ) def verify_group_pointer(pointer: dict | None, expected_state: str, lock: lockfile.Lock, @@ -1909,7 +2203,17 @@ def verify_group_pointer(pointer: dict | None, expected_state: str, lock: lockfi server_entrypoints=list(pin.server_entrypoints), protocol=pin.protocol, launch_mode=pin.launch_mode) if not aok: return False, f"{role}: {areason}", None - expected = _expected(spec, version, digest, identity) + # The receipt cannot promote itself from static evidence to executable + # evidence: the expected tier comes from the current adapter policy. + # A future policy transition therefore requires an explicit migration + # design, not a local receipt rewrite. + expected = _expected( + spec, + version, + digest, + identity, + _role_execution_validation(role, lock), + ) generation = pointer["generations"][role] gok, data, greason = _verify_generation(pin, generation, expected, base_exec=base_exec, base_sha=base_sha) @@ -2144,7 +2448,7 @@ def _state_from(pin: EnginePin, role_value: VerifiedRole, group: VerifiedActiveG revision = data.get("revision") return InstallState( role=pin.role, installed=True, revision=revision, expected_revision=pin.revision, - installed_at=data.get("installed_at"), python=str(role_value.python), + installed_at=data.get("installed_at"), python=str(role_value.python_path), drift=bool(revision) and revision != pin.revision, generation=role_value.generation, release_version=group.release_version, signer_identity=group.identity) @@ -2195,6 +2499,11 @@ def _receipt_types_ok(data: dict[str, Any]) -> str | None: return f"receipt {key} is not a lowercase sha256 digest" if _aware_utc(data["installed_at"]) is None: return "receipt installed_at is not an aware UTC timestamp" + if data["execution_validation"] not in ( + EXECUTION_RUNTIME_CHECKED, + EXECUTION_STATIC_QUARANTINE, + ): + return "receipt execution_validation is unknown" stat_block = data.get("venv_python_stat") if not isinstance(stat_block, dict) or set(stat_block.keys()) != set(_STAT_KEYS): return "receipt venv_python_stat has an unknown or missing key" @@ -2250,10 +2559,13 @@ def _verify_generation(pin: EnginePin, generation: str | None, expected: dict[st # --- bound to the signed release ------------------------------------------ if _canonical_name(str(data["distribution"])) != _canonical_name(str(expected["distribution"])): return False, None, "receipt distribution does not match the signed release" - for key in ("repository", "revision", "version", "source_sha256", "launch_mode", "protocol", - "release_version", "signer_identity", "lock_digest"): + for key in ( + "repository", "revision", "version", "source_sha256", "launch_mode", "protocol", + "release_version", "signer_identity", "lock_digest", "execution_validation", + ): if data.get(key) != expected[key]: - return False, None, f"receipt {key} does not match the signed release" + authority = "current role policy" if key == "execution_validation" else "signed release" + return False, None, f"receipt {key} does not match the {authority}" for key in ("extras", "entrypoints", "server_entrypoints"): if sorted(data.get(key) or []) != expected[key]: return False, None, f"receipt {key} does not match the signed release" @@ -2289,7 +2601,11 @@ def _verify_generation(pin: EnginePin, generation: str | None, expected: dict[st return False, None, f"venv python {key} changed" if current["mode"] & 0o022: return False, None, "venv python is writable by others" - ok, manifest_sha, reason = _local_manifest(gen_dir, pin) + ok, manifest_sha, reason = _local_manifest( + gen_dir, + pin, + execution_validation=expected["execution_validation"], + ) if not ok: return False, None, reason if manifest_sha != data["manifest_sha256"]: diff --git a/cathedral_node/engines/validator.py b/cathedral_node/engines/validator.py index b9af3c3..22e51c9 100644 --- a/cathedral_node/engines/validator.py +++ b/cathedral_node/engines/validator.py @@ -1,41 +1,104 @@ -"""The Validator. - -Fetches Cathedral's signed score feed, verifies it cryptographically before -every write, composes one weight vector across lanes under an owner-signed burn -and allocation policy, and submits weights only when explicitly told to. - -Two properties of this engine drive the whole design here: - -* It is safe by default. Without ``--broadcast`` nothing reaches the chain, and - ``--offline`` additionally removes all chain access. -* It already emits a good JSONL event stream. The node reads that stream rather - than inventing a second one, so what an operator sees and what an agent parses - come from the same source. +"""Fail-closed adapter for the reviewed, incompatible Validator contract. + +The pinned CLI adapter describes a retired signed-feed relay. At the reviewed +upstream commit recorded below, the Validator console entry point is a direct +chain writer with a different command contract and no non-writing mode. +Adapting argv by adding its direct-write acknowledgement would cross the CLI's +explicit no-chain-write boundary, so every operational execution path is +quarantined pending an owner decision. """ from __future__ import annotations -import json from pathlib import Path from typing import Any -from cathedral_node import paths, proc +from cathedral_node import paths, redact from cathedral_node.engines.base import Engine, Progress, Qualification, TestOutcome -# The engine's derived-copy status, stated plainly wherever it matters. This is -# not editorial: deploying production weight authority from the derived mirror -# is an owner cutover decision, and the node must not imply otherwise. -DERIVED_NOTICE = ( - "cathedral-validator is a derived copy of the validator, extracted from cathedralai/cathedral. " - "Running it locally, in dry run, and against the live signed feed is supported here. Making it " - "your production weight authority is a separate owner cutover decision." +REVIEWED_CONTRACT_EVIDENCE = { + "repository": "https://github.com/cathedralai/cathedral-validator", + "reviewed_ref": "origin/main", + "commit": "d225e8758ca02627cced800b7de0c79464d89aee", + "reviewed_on": "2026-09-01", + "entrypoint": "cathedral_thin.independent_runtime.direct_validator:main", + "entrypoint_source": "cathedral_thin/independent_runtime/direct_validator.py", + "entrypoint_source_sha256": ( + "ac258f889192a88eb9269a48bb3de5f488419c8645bff59b7cb2e9aa335204eb" + ), + "pyproject_sha256": ( + "acb8355a17c7011d4718805755c6dd70f5f42126fbb415ca294a1664fadad6df" + ), + "parser_contract": { + "requires_before_chain_access": "--confirm-direct-write", + "non_writing_flags": [], + }, +} + + +def reviewed_contract_evidence() -> dict[str, Any]: + """Return the public evidence fixture without generic digest redaction. + + The redaction backstop correctly masks arbitrary 64-hex values. These two + exact compiled source digests are public audit identifiers, so exempt only + those literals—never an arbitrary same-shaped value. + """ + redact.register_public_values([ + REVIEWED_CONTRACT_EVIDENCE["entrypoint_source_sha256"], + REVIEWED_CONTRACT_EVIDENCE["pyproject_sha256"], + ]) + evidence = dict(REVIEWED_CONTRACT_EVIDENCE) + evidence["parser_contract"] = dict(REVIEWED_CONTRACT_EVIDENCE["parser_contract"]) + return evidence + + +def historical_run_contract() -> dict[str, Any]: + """Read-time annotation for records created under the retired adapter. + + The quarantine prevents new Validator runs, so every record that can already + exist belongs to a different execution contract. The record and its events + remain untouched; callers add this metadata when presenting them. + """ + return { + "historical_contract": True, + "execution_status": "quarantined", + "reviewed_contract_evidence": reviewed_contract_evidence(), + "warning": ( + "This record predates the Validator contract quarantine and is not evidence " + "that the reviewed direct-writer contract was tested or operated safely." + ), + } + + +def legacy_config_contract() -> dict[str, Any]: + """Read-time annotation for inert settings from the retired relay.""" + return { + "legacy_inert": True, + "execution_status": "quarantined", + "reviewed_contract_evidence": reviewed_contract_evidence(), + "warning": ( + "These values belong to the retired signed-feed adapter. They are readable for " + "audit/migration only and do not configure the reviewed direct Validator." + ), + } + +CONTRACT_NOTICE = ( + "This CLI's validator adapter targets the retired signed-feed relay contract. " + "At reviewed cathedral-validator commit d225e8758ca0, the console entry point is a " + "direct chain writer, requires explicit direct-write acknowledgement before chain " + "access, and exposes no non-writing mode. This CLI therefore cannot test or start " + "that contract safely." ) +class ValidatorContractIncompatible(RuntimeError): + """The CLI cannot translate its no-write contract into the reviewed runtime.""" + + class ValidatorEngine(Engine): role = "validator" title = "Validator" - tagline = "Verify what miners claim. Decide what goes on chain." + tagline = "Compatibility quarantined; no Validator operation is launched." # ---- description ---------------------------------------------------------- @@ -45,276 +108,143 @@ def explain(self) -> dict[str, Any]: "title": self.title, "tagline": self.tagline, "what_you_do": ( - "You fetch Cathedral's signed score feed, verify its signature, freshness, and " - "replay fence, check the burn contract, compose one weight vector, and — only when " - "you explicitly allow it — submit that vector to the chain." + "This build reports a validator contract mismatch and stops before execution. " + "It does not translate the retired signed-feed arguments into the reviewed direct " + "writer because doing so would turn a no-write CLI into a chain controller." ), "what_you_verify": [ - "The feed is signed by the key you pinned, and nothing else is accepted", - "It is fresh, unexpired, and newer than anything already applied", - "The burn contract holds and the burn destination is not taken from the feed on faith", - "Target UIDs stay provably stable for the whole lifetime of the write", + "Validator execution commands stop before resolving an operational argv", + "Test and start remain unavailable even when an obsolete engine is installed", + "No direct-write acknowledgement is added or inferred", ], "what_it_never_does": [ - "Never writes to the chain without --broadcast", - "Never writes twice for one attempt: a durable attempt journal prevents it", - "Never guesses — an unprovable outcome halts and re-proves rather than resubmitting", + "Never launches a Validator test, serving loop, or write operation", + "Never adds the reviewed Validator's direct-write acknowledgement", + "Never labels a direct validator process as a dry run", ], "what_you_need": [ - "A Bittensor wallet registered on SN39 with a validator permit and stake", - "A machine that stays on", - "Python 3.11-3.13", + "An owner decision selecting the Validator interface and authority boundary", + "A signed validator interface that preserves the selected authority boundary", + "A separately reviewed configuration and release migration", ], - "not_yet_true": [DERIVED_NOTICE], - "who_sets_the_burn": ( - "Not you, and not this node. The burn share comes from the signed weight vector " - "under the pinned validated_supply_v1 contract, and the integration lane reads " - "Cathedral-signed burn and allocation documents. What is yours is which contract " - "and which signing key you will accept — `require_policy` and `weight_policy_key` " - "— plus your wallet and network. Updates never change those." + "reviewed_contract_evidence": reviewed_contract_evidence(), + "not_yet_true": [CONTRACT_NOTICE], + "safety": ( + "Do not repin the lock or append a direct-write confirmation as a compatibility fix. " + "Neither action creates a non-writing validator contract. Signed node releases are " + "node-wide, so miner setup or update may install the retained Validator package and " + "verify its signed files. After wheel installation, no runtime, import, or " + "entrypoint probe intentionally executes newly installed Validator code." ), } def capabilities(self) -> dict[str, Any]: return { + "local_test": { + "available": False, + "requires_credentials": False, + "requires_network": False, + "detail": CONTRACT_NOTICE, + }, "dry_run": { - "available": True, + "available": False, "requires_credentials": False, - "requires_network": True, - "what_it_proves": ( - "The full verify path runs against the real signed feed and prints the vector " - "it would write. Nothing is submitted." - ), + "requires_network": False, + "detail": "The reviewed Validator entrypoint has no non-writing mode.", }, "offline_dry_run": { - "available": True, + "available": False, "requires_credentials": False, - "requires_network": True, - "detail": ( - "--offline removes all chain access and uses a synthetic UID map. It does not " - "remove the HTTPS fetch of the signed feed, which is what is being verified." - ), + "requires_network": False, + "detail": "The reviewed Validator entrypoint has no offline or synthetic-chain mode.", + }, + "operate": { + "available": False, + "detail": CONTRACT_NOTICE, }, "broadcast": { "available": False, "detail": ( - "Submitting weights needs a registered validator wallet with a permit and an " - "explicit --broadcast. This node will not enable it for you." + "This CLI permanently refuses validator chain writes. Adding the current " + "runtime's direct-write confirmation is not a compatibility repair." ), "requires_operator": True, }, "production_authority": { "available": False, - "detail": DERIVED_NOTICE, + "detail": CONTRACT_NOTICE, "requires_operator": True, }, + "contract_diagnostics": { + "available": True, + "detail": ( + "Explain, capabilities, and qualification report the mismatch pinned to " + "reviewed commit d225e8758ca0 without launching a Validator operation." + ), + "evidence": reviewed_contract_evidence(), + }, } # ---- readiness ------------------------------------------------------------ def qualify(self, cfg: dict[str, Any]) -> Qualification: - blockers: list[dict[str, Any]] = [] - notes: list[str] = [] - - if not self.has_bin("cathedral-validator"): - blockers.append( - { - "code": "install.engine_missing", - "what": "the validator engine is not installed", - "fix": "cathedral setup validator", - "blocks": ["local_test", "operate"], - } - ) - - if not cfg.get("wallet_name") or not cfg.get("wallet_hotkey"): - notes.append( - "No wallet configured. Dry runs work without one; broadcasting does not." - ) - - runtime_root = Path(str(cfg.get("runtime_root") or paths.home() / "validator-runtime")) - notes.append(f"Runtime root {paths.relative_to_home(runtime_root)} holds the cross-mode lock and journal.") - notes.append(DERIVED_NOTICE) - + blocker = self.operation_blocker() + assert blocker is not None return Qualification( - can_local_test=not any("local_test" in b["blocks"] for b in blockers), - can_operate=not any("operate" in b["blocks"] for b in blockers), - blockers=blockers, - notes=notes, + can_local_test=False, + can_operate=False, + blockers=[blocker], + notes=[CONTRACT_NOTICE], ) + def operation_blocker(self) -> dict[str, Any]: + return { + "code": "contract.engine_incompatible", + "what": CONTRACT_NOTICE, + "fix": None, + "blocks": ["local_test", "operate"], + "requires_operator": True, + } + # ---- operations ----------------------------------------------------------- def local_test( self, cfg: dict[str, Any], run_id: str, *, progress: Progress, timeout: float ) -> TestOutcome: - """One offline tick: verify the signed feed and print the vector it would - write. No chain access, no broadcast, nothing consumed.""" - run_root = paths.run_dir(run_id) - runtime_root = run_root / "runtime" - runtime_root.mkdir(parents=True, exist_ok=True) - runtime_root.chmod(0o700) - events = run_root / "validator-events.jsonl" - - progress("verify", "fetching and verifying the signed score feed") - argv = [ - str(self.bin("cathedral-validator")), - "serve", - "--config", - str(self._config_path()), - "--once", - "--dry-run", - "--offline", - "--provenance", - "off", - "--runtime-root", - str(runtime_root), - "--state-file", - str(run_root / "state.json"), - "--jsonl", - str(events), - "--network", - str(cfg.get("network", "finney")), - "--netuid", - str(cfg.get("netuid", 39)), - ] - result = proc.run(argv, timeout=timeout, log_path=run_root / "engine.log", - inherit_env=False, env=self.child_env(cfg)) - - parsed = _read_events(events) - checks = _checks_from_events(parsed) - - if not checks: - offline_note = ( - "the validator produced no events — it may not have reached the signed feed" - if result.ok - else f"the validator exited {result.returncode}" - ) - return TestOutcome( - passed=False, - summary=offline_note, - checks=[], - failure_code="network.unreachable" if result.ok else "upstream.failed", - remediation="cathedral doctor validator", - identifiers={"engine_stderr": result.tail(8)}, - ) - - passed = result.ok and all(c["passed"] for c in checks) - vector = next((e for e in parsed if e.get("event") == "WEIGHTS_DRY_RUN"), {}) - accepted = next((e for e in parsed if e.get("event") == "VECTOR_ACCEPTED"), {}) - - return TestOutcome( - passed=passed, - summary=( - "the signed feed verified and a weight vector was composed; nothing was written" - if passed - else "the validator refused to compose a vector" - ), - checks=checks, - identifiers={ - "vector_id": vector.get("vector_id") or accepted.get("artifact"), - "policy_version": vector.get("policy_version"), - "signed_vector_sha256": vector.get("signed_vector_sha256"), - "uid_count": vector.get("uid_count"), - "burn_uid": vector.get("burn_uid"), - "burn_share": vector.get("burn_share"), - "uid_weights": vector.get("uid_weights"), - "events": str(events), - }, - failure_code=None if passed else "verify.signature_failed", - remediation=None if passed else "cathedral logs validator --run " + run_id, - ) + """Refuse even when called outside the normal qualification path.""" + raise ValidatorContractIncompatible(CONTRACT_NOTICE) def operate_argv(self, cfg: dict[str, Any], *, dry_run: bool) -> list[str]: - """The real validator command. - - ``--broadcast`` is never added here. Turning on chain writes is an - explicit operator action through ``cathedral validate --broadcast``, - which requires a separate confirmation, so it can never be reached by - an agent that merely retried a start. - """ - runtime_root = Path(str(cfg.get("runtime_root") or paths.home() / "validator-runtime")) - runtime_root.mkdir(parents=True, exist_ok=True) - runtime_root.chmod(0o700) - argv = [ - str(self.bin("cathedral-validator")), - "serve", - "--config", - str(self._config_path()), - "--network", - str(cfg.get("network", "finney")), - "--netuid", - str(cfg.get("netuid", 39)), - "--publisher-url", - str(cfg.get("publisher_url", "https://api.cathedral.computer")), - "--interval-secs", - str(cfg.get("interval_secs", 1500)), - "--provenance", - str(cfg.get("provenance", "shadow")), - "--runtime-root", - str(runtime_root), - "--state-file", - str(paths.state_dir() / "validator-thin-state.json"), - ] - if cfg.get("wallet_name"): - argv += ["--wallet-name", str(cfg["wallet_name"])] - if cfg.get("wallet_hotkey"): - argv += ["--wallet-hotkey", str(cfg["wallet_hotkey"])] - if dry_run: - argv.append("--dry-run") - return argv + """Refuse before resolving an executable or creating runtime state.""" + raise ValidatorContractIncompatible(CONTRACT_NOTICE) def operate_env(self, cfg: dict[str, Any]) -> dict[str, str]: - from cathedral_node import config as config_module - - return dict(config_module.secret_environment(self.role, cfg)) + """Refuse every execution hook, including direct adapter use.""" + raise ValidatorContractIncompatible(CONTRACT_NOTICE) def interpret_line(self, line: str) -> dict[str, Any] | None: - """The validator renders its own excellent status view. We pass it - through untouched rather than re-styling it — it is already the - clearest presentation of what it is doing.""" - stripped = line.rstrip() - if not stripped.strip(): - return None - return {"event": "ENGINE", "stage": "run", "status": "INFO", "detail": stripped, "passthrough": True} - - # ---- internals ------------------------------------------------------------ - - def _config_path(self) -> Path: - """The engine's own TOML. The node writes a validator.toml derived from - its config, falling back to the engine's shipped default.""" - managed = paths.config_dir() / "validator-engine.toml" - if managed.exists(): - return managed - # The verified generation's inert source tree, or nothing. There is no - # pointer re-read here: an unbound adapter simply has no engine default. - if self.verified is None: - return managed - return self.source_dir() / "config" / "validator.toml" + """Refuse rather than retain a parser for the retired event contract.""" + raise ValidatorContractIncompatible(CONTRACT_NOTICE) - def render_engine_config(self, cfg: dict[str, Any]) -> str: - """Project node config onto the engine's TOML and return the TEXT. + # ---- retired configuration migration ------------------------------------- - Rendering is separated from writing so a failure to render cannot leave a - half-updated pair of files behind: the caller commits the node config and - this derived config only after both are known good. + def render_engine_config(self, cfg: dict[str, Any]) -> str: + """Render the retired relay config for offline audit/migration only. - Owner-controlled burn and allocation settings are projected here and - nowhere else, so an engine upgrade cannot silently reset them. + No public command calls this while the contract is quarantined. Keeping + the deterministic renderer lets existing signed generations be inspected + without presenting their fields as current runtime authority. """ source = (self.source_dir() / "config" / "validator.toml" if self.verified is not None else None) base = source.read_text() if source is not None and source.exists() else "" lines = [ - "# Cathedral node — validator engine configuration.", - "# Generated from $CATHEDRAL_HOME/config/validator.toml by `cathedral config set`.", - "# Edit the node's config, not this file: this one is rewritten.", + "# Cathedral node — RETIRED validator relay configuration.", + "# Audit/migration artifact only; the reviewed direct Validator does not read this file.", + "# Validator execution and config writes are quarantined in this CLI.", "#", - "# Burn fraction, burn destination, and lane allocation are NOT here and are", - "# not operator settings. The validator takes them from Cathedral-signed", - "# cathedral_burn_config_v1 / cathedral_lane_allocation_v1 documents, and the", - "# serving path takes the burn share from the signed vector under the pinned", - "# weight contract below.", + "# Fields below preserve the historical signed-feed contract only.", "", ] replaced = _substitute( @@ -340,10 +270,8 @@ def render_engine_config(self, cfg: dict[str, Any]) -> str: ) rendered = "\n".join(lines) + replaced - # Parse what was actually produced, and require the fields this projection - # exists to carry. Writing a file that does not parse, or that silently - # lost the owner-controlled settings, is worse than refusing: the engine - # would start and quietly use its own defaults. + # Parse what was actually produced so a migration artifact cannot silently + # lose fields or become ambiguous. import tomllib try: parsed = tomllib.loads(rendered) @@ -357,23 +285,20 @@ def render_engine_config(self, cfg: dict[str, Any]) -> str: f"{'.'.join(filter(None, (section, field)))}") return rendered - # What the projection must carry through, checked against the PARSED result. - # Checking the parsed document rather than the rendered text is the point: a - # substitution that silently failed to land still produces plausible-looking - # text, and only parsing shows the field is not there. + # Historical fields that a deterministic migration artifact must retain. REQUIRED_PROJECTED_FIELDS = (("network", "name"), ("network", "netuid"), ("network", "wallet_name"), ("network", "validator_hotkey"), ("weight_policy", "public_key_hex")) def commit_engine_config(self, rendered: str) -> Path: - """Write the already-validated derived configuration atomically.""" + """Write an already-validated legacy artifact atomically for migration.""" from cathedral_node import safeio managed = paths.config_dir() / "validator-engine.toml" safeio.secure_write_atomic(managed, rendered.encode("utf-8"), mode=0o600) return managed def write_engine_config(self, cfg: dict[str, Any]) -> Path: - """Render, validate, then commit. The single-call form, for setup paths.""" + """Legacy migration helper; no public command calls it while quarantined.""" return self.commit_engine_config(self.render_engine_config(cfg)) @@ -381,8 +306,9 @@ def _substitute(toml_text: str, sections: dict[str, dict[str, Any]]) -> str: """Replace known scalar keys inside known sections, leaving everything else — comments, pinned keys, provenance bundles — byte-identical. - Deliberately conservative: the engine's config carries security-critical - pins we must not rewrite, so anything not named here is passed through. + Deliberately conservative: the historical config carries signed-feed pins + whose original values must remain inspectable, so unnamed content passes + through unchanged. """ out: list[str] = [] current: str | None = None @@ -405,9 +331,8 @@ def _substitute(toml_text: str, sections: dict[str, dict[str, Any]]) -> str: # Anything the base did not already contain is APPENDED rather than dropped. # Substitution alone was only ever correct when the base was the engine's own - # shipped default; with an empty or trimmed base it silently produced a config - # carrying none of the projection, which the engine would then start against - # using its own defaults. Emitting the section is what makes the projection a + # shipped default; with an empty or trimmed base it silently produced an + # incomplete migration artifact. Emitting the section makes preservation a # guarantee instead of a best effort. missing = {section: {k: v for k, v in values.items() if k not in written[section]} for section, values in sections.items()} @@ -422,84 +347,11 @@ def _substitute(toml_text: str, sections: dict[str, dict[str, Any]]) -> str: def _toml_scalar(value: Any) -> str: - """Render a scalar for the engine's TOML. + """Render a scalar for the retired engine's migration TOML. Shares config.py's escaping deliberately: an unescaped value here could - inject keys into a security-critical section, or break the file so the - engine will not start. + inject keys into the historical security-policy sections. """ from cathedral_node.config import _toml_value return _toml_value(value) - - -def _read_events(path: Path) -> list[dict[str, Any]]: - if not path.exists(): - return [] - events = [] - for line in path.read_text().splitlines(): - line = line.strip() - if not line: - continue - try: - events.append(json.loads(line)) - except json.JSONDecodeError: - continue - return events - - -def _checks_from_events(events: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Turn the engine's own events into the node's uniform check shape. - - Only events that represent a real verification decision become checks; - startup and bookkeeping lines are diagnostics, not results. - """ - # (label, description, verdict) where verdict is the meaning of the event - # NAME itself. The name is authoritative: VECTOR_REJECTED means the feed was - # refused whatever the status field happens to say, or fails to say. - interesting: dict[str, tuple[str, str, bool]] = { - "VECTOR_ACCEPTED": ("feed", "the signed score feed verified", True), - "VECTOR_REJECTED": ("feed", "the signed score feed was refused", False), - "WEIGHTS_DRY_RUN": ("vector", "a weight vector was composed without writing", True), - "WEIGHTS_SUBMITTED": ("submit", "weights were submitted", True), - "PROVENANCE_AUDIT_PASS": ("audit", "the independent provenance audit agreed", True), - "PROVENANCE_AUDIT_NOT_PROVEN": ("audit", "the provenance audit could not prove the epoch", False), - "PROVENANCE_VECTOR_MISMATCH": ("audit", "the independent audit computed a different vector", False), - "PROVENANCE_DIVERGENCE": ("audit", "the independent provenance audit disagreed", False), - "TICK_FAILED": ("tick", "the validator could not complete the tick", False), - "PENDING_RECEIPT_CONTRADICTION": ("receipt", "a pending submission contradicts the chain", False), - "PENDING_RECEIPT_NOT_PROVEN": ("receipt", "a pending submission could not be proven", False), - } - - checks = [] - for event in events: - name = event.get("event") - if name not in interesting: - continue - label, description, name_says_pass = interesting[name] - - # Fail closed on the status field. Only an explicit affirmative counts; - # a missing, empty, or unrecognised status is NOT a pass. Reading "" as - # success meant a refused vector reported as verified — the engine's - # verdict inverted by a field the node neither sets nor requires. - raw_status = str(event.get("status", "")).strip().upper() - if raw_status in ("PASS", "INFO"): - status_says_pass = True - elif raw_status in ("FAIL", "NOT_PROVEN", "ERROR", "WARN"): - status_says_pass = False - else: - status_says_pass = False # unknown or absent: refuse - - checks.append( - { - "label": label, - "name": description, - # Both must agree. Either signal alone saying "no" is a no. - "passed": bool(name_says_pass and status_says_pass), - "detail": event.get("detail", ""), - "event": name, - "event_status": raw_status or None, - "artifact": event.get("artifact"), - } - ) - return checks diff --git a/cathedral_node/lockfile.py b/cathedral_node/lockfile.py index 1782562..093c33a 100644 --- a/cathedral_node/lockfile.py +++ b/cathedral_node/lockfile.py @@ -22,10 +22,10 @@ class EnginePin: branch: str distribution: str description: str - # The launch install profile — part of the pinned, reviewable contract and - # bound into the generation receipt and the signed release lock. A generation - # that omits a required extra (e.g. the validator's `integration` seam) is not - # launch-correct. + # The historical install profile — part of the pinned, reviewable contract + # and bound into the generation receipt and signed release lock. Adapter + # compatibility is a separate gate: a retained profile can verify exactly + # and still be statically quarantined from execution. extras: tuple[str, ...] = () entrypoints: tuple[str, ...] = () server_entrypoints: tuple[str, ...] = () diff --git a/cathedral_node/paths.py b/cathedral_node/paths.py index 1b621fc..4a900fe 100644 --- a/cathedral_node/paths.py +++ b/cathedral_node/paths.py @@ -272,6 +272,11 @@ def role_lock(role: str) -> Path: return state_dir() / f"{role}.lock" +def role_control_lock(role: str) -> Path: + """Stable advisory lock serialising one role's ownership-file lifecycle.""" + return state_dir() / f"{role}.control.lock" + + def logs_dir() -> Path: return home() / "logs" diff --git a/cathedral_node/runner.py b/cathedral_node/runner.py index 560bbd5..398c9d2 100644 --- a/cathedral_node/runner.py +++ b/cathedral_node/runner.py @@ -296,6 +296,8 @@ def _classify(name: str, exc: Exception, ctx: Context) -> Envelope: ) if isinstance(exc, ConfigError): + problem_role = getattr(exc, "role", None) or getattr(ctx.args, "role", None) + requires_operator = bool(getattr(exc, "requires_operator", False)) return Envelope.fail( name, C.E_CONFIG_INVALID, @@ -303,9 +305,19 @@ def _classify(name: str, exc: Exception, ctx: Context) -> Envelope: exit_code=Exit.CONFIG_INVALID, remediation=Remediation( summary=getattr(exc, "remedy", None) or "Correct the configuration and retry.", - command=f"cathedral config show {getattr(ctx.args, 'role', '') or ''}".strip(), + command=( + None + if requires_operator + else f"cathedral config show {problem_role or ''}".strip() + ), + docs=( + "cathedral explain validator" + if problem_role == "validator" + else (f"cathedral config schema {problem_role}" if problem_role else None) + ), + requires_operator=requires_operator, ), - detail={"field": getattr(exc, "field", None)}, + detail={"field": getattr(exc, "field", None), "role": problem_role}, run_id=ctx.run_id, ) diff --git a/cathedral_node/state.py b/cathedral_node/state.py index f6e538e..97115b3 100644 --- a/cathedral_node/state.py +++ b/cathedral_node/state.py @@ -61,6 +61,15 @@ def to_dict(self) -> dict[str, Any]: } +_RUN_RECORD_KEYS = { + "schema", "run_id", "role", "kind", "status", "started_at", "finished_at", + "pid", "exit_code", "detail", "artifacts", +} +_RUN_ROLES = frozenset({"distill", "compute", "validator"}) +_RUN_KINDS = frozenset({"test", "operate", "mine", "validate"}) +_RUN_STATUSES = frozenset({"running", "completed", "failed", "cancelled", "interrupted"}) + + def _record_path(run_id: str) -> Path: return paths.run_dir(run_id) / "run.json" @@ -117,17 +126,49 @@ def load_run(run_id: str) -> RunRecord | None: data = json.loads(path.read_text()) except (OSError, json.JSONDecodeError): return None + if not isinstance(data, dict) or set(data) != _RUN_RECORD_KEYS: + return None + if data.get("schema") != "cathedral.node.run.v1" or data.get("run_id") != run_id: + # The directory name is the lookup authority. Trusting a different id + # embedded in its document lets `cancel A` stop A and then finish/write B. + # A missing or foreign schema is likewise not a run this build may mutate. + return None + if ( + data.get("role") not in _RUN_ROLES + or data.get("kind") not in _RUN_KINDS + or data.get("status") not in _RUN_STATUSES + or not isinstance(data.get("started_at"), str) + or not data["started_at"] + or not ( + data.get("finished_at") is None + or (isinstance(data.get("finished_at"), str) and bool(data["finished_at"])) + ) + or not ( + data.get("pid") is None + or (isinstance(data.get("pid"), int) and not isinstance(data["pid"], bool)) + ) + or not ( + data.get("exit_code") is None + or ( + isinstance(data.get("exit_code"), int) + and not isinstance(data["exit_code"], bool) + ) + ) + or not isinstance(data.get("detail"), str) + or not isinstance(data.get("artifacts"), dict) + ): + return None return RunRecord( run_id=data["run_id"], - role=data.get("role", ""), - kind=data.get("kind", ""), - status=data.get("status", "unknown"), - started_at=data.get("started_at", ""), - finished_at=data.get("finished_at"), - pid=data.get("pid"), - exit_code=data.get("exit_code"), - detail=data.get("detail", ""), - artifacts=data.get("artifacts", {}), + role=data["role"], + kind=data["kind"], + status=data["status"], + started_at=data["started_at"], + finished_at=data["finished_at"], + pid=data["pid"], + exit_code=data["exit_code"], + detail=data["detail"], + artifacts=data["artifacts"], ) @@ -884,9 +925,10 @@ def read_ownership(role: str) -> ChildOwnership | None: if not ok or data is None: return None try: - return ChildOwnership.parse(json.loads(data.decode("utf-8"))) + parsed = ChildOwnership.parse(json.loads(data.decode("utf-8"))) except (UnicodeDecodeError, ValueError): return None + return parsed if parsed is not None and parsed.role == role else None def ownership_status(role: str) -> tuple[str, ChildOwnership | None, str]: @@ -913,6 +955,11 @@ def ownership_status(role: str) -> tuple[str, ChildOwnership | None, str]: # what. Fail closed rather than guess it away. return OWNERSHIP_UNVERIFIABLE, None, ( f"the {role} ownership record is malformed; refusing to assume nothing is running") + if parsed.role != role: + return OWNERSHIP_UNVERIFIABLE, parsed, ( + f"the {role} ownership record embeds foreign role {parsed.role!r}; refusing to " + "signal or mutate either role" + ) if not parsed.boot_identity_known(): # We could not learn which boot this is, or the record does not say. That is # not the same as knowing the record is stale — it is knowing nothing — and @@ -1050,7 +1097,28 @@ def __enter__(self) -> "RoleLock": def __exit__(self, *_exc: Any) -> None: self.release() - def claim_child(self, child_pid: int, *, generation: str = "", lock_digest: str = "") -> ChildOwnership: + def claim_child( + self, child_pid: int, *, generation: str = "", lock_digest: str = "" + ) -> ChildOwnership: + from cathedral_node import safeio + try: + with safeio.secure_lock( + paths.role_control_lock(self.role), + exclusive=True, + timeout=30.0, + busy_message=f"another operation is changing {self.role} ownership", + ): + return self._claim_child_locked( + child_pid, generation=generation, lock_digest=lock_digest + ) + except safeio.SecureOpenError as exc: + raise OwnershipLost( + f"the {self.role} ownership control lock could not be taken safely: {exc}" + ) from exc + + def _claim_child_locked( + self, child_pid: int, *, generation: str = "", lock_digest: str = "" + ) -> ChildOwnership: """Record durable ownership of the signed child, before start is reported. Called from the launcher's ``on_start`` hook, so there is no window in @@ -1096,6 +1164,21 @@ def child_reaped(self) -> None: self._reaped = True def begin_spawn(self, *, generation: str = "", lock_digest: str = "") -> None: + from cathedral_node import safeio + try: + with safeio.secure_lock( + paths.role_control_lock(self.role), + exclusive=True, + timeout=30.0, + busy_message=f"another operation is changing {self.role} ownership", + ): + self._begin_spawn_locked(generation=generation, lock_digest=lock_digest) + except safeio.SecureOpenError as exc: + raise OwnershipLost( + f"the {self.role} ownership control lock could not be taken safely: {exc}" + ) from exc + + def _begin_spawn_locked(self, *, generation: str = "", lock_digest: str = "") -> None: """Record that a spawn is about to happen, BEFORE the child can exist. Without this, a launcher SIGKILLed between `Popen` returning and ownership @@ -1123,6 +1206,21 @@ def begin_spawn(self, *, generation: str = "", lock_digest: str = "") -> None: lock_digest=lock_digest or current.lock_digest, euid=os.geteuid()) def acquire(self) -> None: + from cathedral_node import safeio + try: + with safeio.secure_lock( + paths.role_control_lock(self.role), + exclusive=True, + timeout=30.0, + busy_message=f"another operation is changing {self.role} ownership", + ): + self._acquire_locked() + except safeio.SecureOpenError as exc: + raise OwnershipLost( + f"the {self.role} ownership control lock could not be taken safely: {exc}" + ) from exc + + def _acquire_locked(self) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) payload = {"schema": OWNERSHIP_SCHEMA, "role": self.role, "run_id": self.run_id, "parent_pid": os.getpid(), "child_pid": -1, "pgid": -1, @@ -1133,7 +1231,7 @@ def acquire(self) -> None: # read-then-replace left a window in which two starts both saw "free" and # the second clobbered the first. Reclaim a dead owner, then create. while True: - holder = self.holder() + holder = self._holder_locked() if holder is not None: raise LockHeld(holder["pid"], holder.get("run_id", "unknown"), holder.get("since", "")) from cathedral_node import safeio @@ -1148,6 +1246,30 @@ def acquire(self) -> None: return def holder(self) -> dict[str, Any] | None: + from cathedral_node import safeio + try: + with safeio.secure_lock( + paths.role_control_lock(self.role), + exclusive=True, + timeout=30.0, + busy_message=f"another operation is changing {self.role} ownership", + ): + return self._holder_locked() + except safeio.SecureOpenError as exc: + # A failed control lock is not evidence that the role is free. + return { + "pid": None, + "parent_pid": None, + "pgid": None, + "run_id": "", + "since": "", + "role": self.role, + "generation": "", + "detail": f"the ownership control lock could not be trusted: {exc}", + "unresolved": "control_lock", + } + + def _holder_locked(self) -> dict[str, Any] | None: """Who holds this lock, or None if free. Clears a lock nobody owns. A record is only a holder when the **child** it names is provably still the @@ -1216,6 +1338,22 @@ def _blocking_lease(self) -> tuple[dict[str, Any] | None, str]: return None, "" def release(self) -> None: + from cathedral_node import safeio + try: + with safeio.secure_lock( + paths.role_control_lock(self.role), + exclusive=True, + timeout=30.0, + busy_message=f"another operation is changing {self.role} ownership", + ): + self._release_locked() + except safeio.SecureOpenError: + # Leave both ownership and lease evidence in place. A later operator + # can resolve them; guessing them away would make a second start unsafe. + self._acquired = False + self._owned = False + + def _release_locked(self) -> None: """Drop the claim, but only ours. The token, not the pid, decides: after `claim_child` the record names the @@ -1496,27 +1634,76 @@ def _ownership_from_lease(role: str) -> tuple["ChildOwnership | None", str]: return None, "" -def stop_role(role: str, grace: float = 10.0) -> tuple[bool, str]: +def stop_role( + role: str, + grace: float = 10.0, + *, + expected_run_id: str | None = None, +) -> tuple[bool, str]: + """Serialise a stop against acquisition, ownership publication, and release.""" + from cathedral_node import safeio + try: + with safeio.secure_lock( + paths.role_control_lock(role), + exclusive=True, + timeout=30.0, + busy_message=f"another operation is changing {role} ownership", + ): + return _stop_role_locked( + role, grace=grace, expected_run_id=expected_run_id + ) + except safeio.SecureOpenError as exc: + return False, f"the {role} ownership control lock could not be taken safely: {exc}" + + +def _stop_role_locked( + role: str, + grace: float = 10.0, + *, + expected_run_id: str | None = None, +) -> tuple[bool, str]: """Stop a running role and prove the whole process group is gone. Ownership is validated before anything is signalled. Signalling a stored pgid without checking that the recorded child still occupies it is how a stop command reaches an unrelated process that happened to inherit the number. + When ``expected_run_id`` is supplied, the ownership record must name that + exact run before this function clears state or sends any signal. This binds + ``cancel RUN_ID`` to its target instead of merely stopping whichever run now + happens to own the same role. "Stopped" then means every member of the group has exited — not that the one pid we held a handle to did. A surviving descendant is still executing the generation the caller is about to prune. """ verdict, ownership, detail = ownership_status(role) + recovered_from_lease = False if verdict == OWNERSHIP_ABSENT: recovered, why = _ownership_from_lease(role) if recovered is None: + if expected_run_id is not None: + return False, ( + f"the current {role} owner cannot be bound to requested run " + f"{expected_run_id}; nothing was signalled or reconciled" + ) closed, close_why = _close_finished_leases(role) if not closed: return False, close_why return False, "not running" # The record is gone but the ledger still names a child. Stop what it names. ownership, verdict, detail = recovered, OWNERSHIP_LIVE, why + recovered_from_lease = True + if expected_run_id is not None: + if ownership is None: + return False, ( + f"the current {role} owner cannot be bound to requested run " + f"{expected_run_id}; nothing was signalled" + ) + if ownership.run_id != expected_run_id: + return False, ( + f"the current {role} owner is run {ownership.run_id}, not requested run " + f"{expected_run_id}; nothing was signalled" + ) if verdict == OWNERSHIP_UNVERIFIABLE: # Nothing here can be signalled — there is no trustworthy pgid to signal — # and the record must not be removed on a guess either. That covers a @@ -1561,6 +1748,52 @@ def stop_role(role: str, grace: float = 10.0) -> tuple[bool, str]: def _members() -> list[int]: return [pid for pid in ownership.group_members() if pid != os.getpid()] + def _signal_target_state() -> tuple[str, str]: + """Re-prove the exact target immediately before each group signal.""" + fresh_verdict, fresh, fresh_detail = ownership_status(role) + if recovered_from_lease and fresh_verdict == OWNERSHIP_ABSENT: + # No Cathedral owner can be admitted while the open lease remains. + # Prove the lease's recorded process identity and group again anyway, + # because the kernel may have recycled their numeric ids. + leader = ownership.leader_state() + if leader == ChildOwnership.LEADER_OURS: + return "live", fresh_detail + if leader == ChildOwnership.LEADER_GONE: + try: + return ("live", fresh_detail) if _members() else ("gone", fresh_detail) + except ProbeUnavailable as exc: + return "unsafe", f"the target process group could not be probed: {exc}" + return "unsafe", ( + f"the recorded leader is {leader}; its process group may have been recycled" + ) + if fresh is None: + return "unsafe", f"{fresh_detail}; the stop target is no longer identifiable" + original_identity = ( + ownership.run_id, + ownership.token, + ownership.child_pid, + ownership.pgid, + ownership.start_identity, + ownership.boot_id, + ownership.euid, + ) + fresh_identity = ( + fresh.run_id, + fresh.token, + fresh.child_pid, + fresh.pgid, + fresh.start_identity, + fresh.boot_id, + fresh.euid, + ) + if fresh_identity != original_identity: + return "unsafe", "the ownership identity changed; nothing was signalled" + if fresh_verdict == OWNERSHIP_LIVE: + return "live", fresh_detail + if fresh_verdict == OWNERSHIP_TERMINATED: + return "gone", fresh_detail + return "unsafe", fresh_detail + try: _members() except ProbeUnavailable as exc: @@ -1569,6 +1802,16 @@ def _members() -> list[int]: try: for sig in (signal.SIGTERM, signal.SIGKILL): + target_state, target_detail = _signal_target_state() + if target_state == "unsafe": + return False, target_detail + if target_state == "gone": + if not recovered_from_lease: + paths.role_lock(role).unlink(missing_ok=True) + closed, why = _close_finished_leases(role) + if not closed: + return False, why + return True, "the owned process group ended before it was signalled" try: os.killpg(ownership.pgid, sig) except ProcessLookupError: diff --git a/cathedral_node/verified.py b/cathedral_node/verified.py index 50971a9..c6444fc 100644 --- a/cathedral_node/verified.py +++ b/cathedral_node/verified.py @@ -62,7 +62,7 @@ class VerifiedRole: comes from here, never from the mutable pointer.""" __slots__ = ("role", "generation", "generation_dir", "source_dir", "venv_dir", - "python", "receipt", "receipt_data", "entrypoints") + "_python", "receipt", "receipt_data", "entrypoints") def __init__(self, seal: Any, *, role: str, generation: str, generation_dir: Path, source_dir: Path, venv_dir: Path, python: Path, receipt: Path, @@ -74,7 +74,7 @@ def __init__(self, seal: Any, *, role: str, generation: str, generation_dir: Pat object.__setattr__(self, "generation_dir", Path(generation_dir)) object.__setattr__(self, "source_dir", Path(source_dir)) object.__setattr__(self, "venv_dir", Path(venv_dir)) - object.__setattr__(self, "python", Path(python)) + object.__setattr__(self, "_python", Path(python)) object.__setattr__(self, "receipt", Path(receipt)) object.__setattr__(self, "receipt_data", freeze(receipt_data)) object.__setattr__(self, "entrypoints", tuple(sorted({str(e) for e in entrypoints}))) @@ -82,6 +82,24 @@ def __init__(self, seal: Any, *, role: str, generation: str, generation_dir: Pat __setattr__ = _immutable __delattr__ = _immutable + @property + def execution_validation(self) -> str: + return str(self.receipt_data.get("execution_validation", "")) + + @property + def python_path(self) -> Path: + """Interpreter location for diagnostics only; this grants no execution.""" + return self._python + + @property + def python(self) -> Path: + """The interpreter only when this generation passed runtime validation.""" + if self.execution_validation != "runtime_checked": + raise SealError( + f"{self.role} is static-quarantine evidence, not an executable generation" + ) + return self._python + def bin(self, name: str) -> Path: """An executable inside this verified venv. @@ -89,6 +107,10 @@ def bin(self, name: str) -> Path: interpreter itself) and must be a single path component, so a caller can never steer execution outside the generation that was verified. """ + if self.execution_validation != "runtime_checked": + raise SealError( + f"{self.role} is static-quarantine evidence, not an executable generation" + ) if not isinstance(name, str) or not name or "/" in name or name in (".", ".."): raise ValueError(f"{name!r} is not a single executable name") if name not in self.entrypoints: @@ -99,14 +121,14 @@ def bin(self, name: str) -> Path: def has_bin(self, name: str) -> bool: try: return self.bin(name).is_file() - except ValueError: + except (ValueError, SealError): return False def to_dict(self) -> dict[str, Any]: """A reporting view. Not authorization — the sealed value is.""" return {"role": self.role, "generation": self.generation, "generation_dir": str(self.generation_dir), "venv": str(self.venv_dir), - "python": str(self.python), "receipt": str(self.receipt), + "python": str(self.python_path), "receipt": str(self.receipt), "entrypoints": list(self.entrypoints)} def __repr__(self) -> str: # pragma: no cover - diagnostics only diff --git a/docs/AGENT_CONTRACT.md b/docs/AGENT_CONTRACT.md index c64de2b..76b49b3 100644 --- a/docs/AGENT_CONTRACT.md +++ b/docs/AGENT_CONTRACT.md @@ -82,7 +82,7 @@ grouped so an unfamiliar code is still classifiable. | 32 | `INCOMPATIBLE` | Protocol or engine mismatch. Stop and re-discover. | | 40 | `NETWORK` | A required remote was unreachable. Always safe to retry. | | 41 | `UPSTREAM` | A pinned engine failed in a way this layer does not model. | -| 50 | `CANCELLED` | Interrupted. Durable state was flushed; `resume` will continue. | +| 50 | `CANCELLED` | Interrupted. State was flushed; an available miner contract may resume it. | | 70 | `INTERNAL` | A bug in the node. Includes a diagnostics bundle path. | **Retry** `40`, `22`, `14`, `41` with backoff. **Never retry** `11`, or anything @@ -148,8 +148,18 @@ Every command is safe to re-run. `setup`, `config set`, `secret set`, `stop`, and `cleanup` are idempotent by design: calling them again when they have already taken effect returns success without changing anything. -`--dry-run` on any command reports what would happen and changes nothing. The -envelope's `dry_run` field is `true` and no filesystem write occurs. +`--dry-run` on an available command reports what would happen and changes +nothing. It does not make an unavailable engine contract available. Validator +setup, test, non-broadcast start, quickstart, and valid legacy config writes +return exit `32` with `contract.engine_incompatible` before installation, a run, +or a publisher fence. Unknown fields and forbidden coldkey/seed material still +fail earlier as usage/security errors. The explicit `--broadcast` sentinel +remains the stronger, permanent chain-write refusal at exit `11`. + +Signed release installation is node-wide. Miner setup/update may install and +statically verify the retained Validator package. After wheel installation, no +runtime, import, client-entrypoint, or server-entrypoint probe intentionally +executes newly installed Validator code. ## Confirmations @@ -171,9 +181,8 @@ Results carry the exact identifiers the operator needs, under `cathedral evidence --json` and get the run and events that produced it. A Distill test returns `batch_nonce`, `task_ids`, `solved`, `score`; each check -carries its `task_id` and `poc_sha256`. A validator dry run returns `vector_id`, -`policy_version`, `signed_vector_sha256`, `uid_count`, `burn_uid`, `burn_share`, -and `uid_weights`. +carries its `task_id` and `poc_sha256`. The quarantined Validator creates no test +run and therefore returns no vector identifiers. ## Events @@ -200,9 +209,11 @@ happened, and `cathedral status --run ` tells the truth afterwards — a run whose process died without finishing is reconciled to `interrupted` rather than being reported as still running. -`cathedral resume ` continues. The engines keep their own durable fences -and journals, so resuming restarts the engine against that state; the node does -not claim a checkpoint the engines do not have, and says so in its response. +`cathedral resume ` continues only when the run's current engine contract +is available. Distill and Compute keep their own durable fences and journals, so +resuming restarts the engine against that state; the node does not claim a +checkpoint the engines do not have. Historical Validator runs return exit `32` +and remain readable without being restarted. ## Secrets @@ -220,6 +231,8 @@ The node will not: - accept a coldkey, seed, or mnemonic; - enable chain writes — `--broadcast` is refused, with or without `--yes`; +- test or start the reviewed Validator contract — its direct-write-only interface is not + compatible with this CLI's no-write boundary; - install or expose anything from the legacy `cathedralai/cathedral` repository; - spend money, provision hardware, or register an identity. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f850ec3..b9c8b9a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -25,7 +25,7 @@ bearing. ┌─────────────────────┼─────────────────────┐ │ │ │ cathedral-distill cathedral-compute cathedral-validator - pinned, own venv pinned, own venv pinned, own venv + pinned, own venv pinned, own venv pin retained; execution quarantined ``` The engines are preserved exactly as their owners built them. Nothing here @@ -51,12 +51,13 @@ goes both ways. ### Isolated environments per engine, not tidiness `cathedral-compute` publishes a top-level Python package named `cathedral` and a -console script named `cathedral-validator`. `cathedral-validator` publishes a -console script of that same name pointing at completely different code: +console script named `cathedral-validator`. The historical Validator pin also +publishes a console script of that name pointing at different code: ``` -cathedral-compute → cathedral.neuron.validator:main (runtime epoch operator) -cathedral-validator → scaffold.cli:main (SN39 weight validator) +cathedral-compute → cathedral.neuron.validator:main (runtime epoch operator) +retained CLI pin → scaffold.cli:main (retired signed-feed relay) +reviewed d225e8758ca0 → direct_validator:main (direct chain writer; quarantined) ``` Installed into one environment, whichever landed last silently wins and an @@ -64,8 +65,9 @@ operator gets the wrong program with no error. So each engine gets its own virtualenv under `$CATHEDRAL_HOME/engines//venv`, and the node never resolves an engine command through `PATH` — always an absolute path. -This also means installing Distill does not drag in `bittensor`, so its local -journey remains smaller than the validator installation. +This also means installing Distill does not drag in `bittensor`. Isolation does +not make the historical Validator pin launch-compatible; the static quarantine +supersedes its retained launch metadata. ### Zero dependencies in the orchestrator @@ -84,17 +86,19 @@ The engines have real dependencies. They get them, in their own environments. ``` cathedral test distill cathedral test compute -cathedral test validator +cathedral test validator # contract.engine_incompatible; launches nothing ``` The first decision an operator makes is what they are doing; the role is the object it acts on. For an agent it is a flat templatable surface where role is one token. -Upstream names were not preserved. `cathedral-cybergym-agent --local`, -`cathedral worker serve`, and `cathedral-validator serve --dry-run --offline` -are three unrelated spellings of "try this safely". Keeping them would have -meant the operator learning three products. +Upstream names are adapted only when the semantics still match. The historical +Validator adapter translated a signed-feed relay's dry-run and offline flags. +At reviewed `cathedral-validator` commit +`d225e8758ca02627cced800b7de0c79464d89aee`, that entrypoint is a direct chain +writer and has neither mode, so the translation is quarantined instead of being +approximated with new argv. ### Pinned revisions in a lockfile @@ -108,26 +112,26 @@ half-state that looks fine. makes `rollback` reinstall a known commit rather than hoping a reinstall of "the old version" resolves the same way. -### Owner settings are node state, not engine state +### Retired validator settings are preserved, not treated as authority -Burn fraction, burn destination, and lane allocation live in the node's config -and are projected onto the engine's TOML. An engine upgrade replaces engine -files; it cannot touch node config. `update` additionally snapshots those fields -before and after and warns if any moved — a defect alarm for something that -should be structurally impossible. +The repository still recognizes the old Validator configuration so existing +files remain readable and update comparisons remain deterministic. Those fields +belong to the retired signed-feed adapter and are not an active control plane. +No projection, pin change, or argv edit can authorize the current direct writer. The projection is deliberately conservative: `_substitute` replaces only named scalar keys inside named sections and passes everything else through -byte-identical, because the engine's config carries security-critical key pins -the node must not rewrite. +byte-identical, so historical signed-feed pins remain reproducible for offline +audit. No public command writes this projection while the contract is quarantined. ### Secrets travel through the environment only -`operate_argv()` builds a command line without ever reading a credential; -`operate_env()` supplies them separately. A secret therefore cannot reach `argv`, -which means it cannot appear in `ps` output for other users on the host. A test -asserts this per engine by putting a sentinel in every secret-shaped config -field and checking the resulting argv. +For available engines, `operate_argv()` builds a command line without ever +reading a credential and `operate_env()` supplies them separately. A secret +therefore cannot reach `argv`, which means it cannot appear in `ps` output for +other users on the host. Validator execution hooks all refuse. A test asserts +the separation for every available engine by putting a sentinel in a +secret-shaped config field and checking the resulting argv. `redact.py` is a backstop for an engine that prints one anyway, applied before anything reaches a log, the terminal, or an envelope. @@ -143,8 +147,9 @@ The Compute local test asks the engine's quote verifier four questions, three of which must be refused: an unlisted measurement, a stale TCB, and an empty policy. Proving the gate is closed is the point; proving a happy path exists is not. -The validator local test is one real offline tick against the live signed feed, -producing a real vector id and policy version. +The Validator local test is unavailable. Qualification returns +`contract.engine_incompatible`, and both the public command and adapter method +refuse before resolving an executable or creating a run. ### Honest capability reporting @@ -163,29 +168,32 @@ failure at a time. |---|---| | `cathedralai/cathedral-distill` | Installed. Owns Distill mining and verification. | | `cathedralai/cathedral-compute` | Installed. Owns verified Compute mining and evidence. | -| `cathedralai/cathedral-validator` | Installed. Owns validation, composition, burn handling, guarded submission. | +| `cathedralai/cathedral-validator` | Retained signed-release member; public execution, setup, and config-write paths are quarantined. Historical inspection and operator-controlled stop remain available. A node-wide install statically verifies it and executes none of its installed code. | | `cathedralai/cathedral` | **Never installed.** Legacy. Audited only. | The node contains no product logic that belongs in an engine. If a behaviour would be wrong for a direct user of `cathedral-distill`, it does not belong here either — it belongs upstream. -Two honest boundaries the node states rather than papers over: - -- `cathedral-validator` describes itself as a derived copy, not yet the - authoritative validator source, and says not to deploy from it without an - explicit cutover decision. The node installs it, supports local and dry-run - use, and says exactly this wherever it matters. -- The validator's `--offline` removes chain access, not network access. The - signed feed is still fetched over HTTPS, because verifying that feed is the - thing being tested. The node says so rather than implying an air gap. +The principal honest boundary is explicit: the old CLI adapter and the current +Validator do not share a command or authority contract. The CLI exposes +diagnostics only, reports every Validator execution capability unavailable, and +never adds a direct-write confirmation to make an argv superficially work. +Receipt v6 preserves that boundary across upgrades: `static_quarantine` remains +inspectable but cannot resolve an interpreter or entrypoint, and the verifier +binds the receipt tier to the current adapter policy rather than trusting the +receipt's own claim. A future compatible adapter needs an owner-specified +migration through a higher signed release that creates a `runtime_checked` +generation; v5 is not grandfathered, and this draft does not define that +transition. ## Adding an engine 1. Add a pin to `cathedral.lock.json`. 2. Add an adapter subclassing `engines.base.Engine`. Implement `explain`, `capabilities`, `qualify`, `local_test`, `operate_argv`, `operate_env`, and - optionally `interpret_line`. + optionally `interpret_line`. Override `operation_blocker` when a known static + contract mismatch must be refused before release verification. 3. Register it in `engines/__init__.py` and add the role to `lockfile.ROLES`. 4. Add its configuration fields to `config.SCHEMAS`. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 668be7c..e73a851 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -8,7 +8,7 @@ wrong, and what the node will not do for you. ``` $CATHEDRAL_HOME/ default ~/.cathedral, mode 0700 config/.toml safe to read; holds no credentials - config/validator-engine.toml generated; owner burn/lane settings live here + config/validator-engine.toml legacy projection; inert while Validator is quarantined secrets.env mode 0600; never printed engines//src the pinned checkout engines//venv that engine's isolated environment @@ -66,55 +66,50 @@ need the trail. ## Updating ```bash -cathedral update --check # what a newer pin would change -cathedral update --yes # apply -cathedral rollback --yes # return to the previous revisions +cathedral update --check # inspect the configured signed release +cathedral update --release --yes # apply one node-wide signed release +cathedral rollback # explain signed rollback / recover interruption ``` -An update refuses to run while a role is running — stop it first. It records the -outgoing revision before changing anything, which is what makes rollback -reinstall a known commit rather than a guess. +An update refuses to run while any role is running—stop it first. Updates are +node-wide; `update ` is rejected rather than silently changing other +roles. Unsigned `--to ` adoption is also refused. -If an update fails partway, that engine is left **uninstalled** rather than -half-updated, and the failure names `cathedral rollback ` as the fix. A -node with a missing engine is a state the CLI understands; a node with a -half-installed one is not. +If an update is interrupted, the durable transaction record makes +`cathedral recover` the explicit next action. A deliberate rollback is not a +local pointer reversal: it is a newer signed release selecting the retained +prior generation set. -### Owner-controlled settings +### Quarantined Validator settings -What is yours is what you will **accept**: `require_policy` (which weight-policy -contract) and `weight_policy_key` (whose signature), plus your wallet and -network. They live in the node's configuration, not in engine files, so an -engine upgrade cannot reach them. `update` snapshots them before and after and -warns loudly if any moved — an alarm for something that should be structurally -impossible. +The CLI preserves the retired Validator configuration so existing nodes can be +inspected and update diffs remain deterministic. It does not project those +values into a supported runtime or treat them as current authority. In +particular, the old signed-feed policy key does not configure the reviewed direct +writer. ```bash cathedral config get validator require_policy cathedral config get validator weight_policy_key ``` -The signing key is public by design. It is meant to be read aloud and checked -against Cathedral's published key, so it is shown in full in both the human and -`--json` views rather than masked. - -**The burn share and the lane allocation are not yours, and are not settings.** -They arrive inside the Cathedral-signed weight vector and from Cathedral-signed -burn and allocation documents; nothing local changes them. Earlier versions of -this document — and of the config schema — offered `burn_fraction`, -`burn_destination` and `lane_allocation` as editable fields, which was worse than -useless: it let an operator believe they had changed the economics when nothing -had changed. If you want different economics, the lever is which contract and -which key you accept, above. +These values are readable for migration and audit only. An owner decision and a +new signed interface are required before any Validator configuration can be +described as operational. ## Recovering -**"engine is not installed"** — `cathedral setup `. Idempotent; safe to run -any time. +**"engine is not installed"** — `cathedral setup `. +Idempotent; safe to run for Distill and Compute. **"differs from the pinned revision"** — the checkout drifted. `cathedral setup --force` reinstalls from the pin. +**`contract.engine_incompatible` (exit 32)** — stop. An identical retry, setup, +lockfile repin, or argv edit cannot make the reviewed direct Validator a +non-writing process. Read `cathedral explain validator` and escalate the +authority decision. + **Verification failed (exit 20)** — the node worked correctly and the answer was no. Nothing was submitted anywhere. Read the checks: `cathedral logs --run `. Retrying will not change the result. @@ -126,7 +121,7 @@ no. Nothing was submitted anywhere. Read the checks: names a diagnostics bundle under `logs/`. Set `CATHEDRAL_TRACEBACK=1` to also print the traceback. -**Interrupted run** — `cathedral resume `. The engines keep their own +**Interrupted run** — `cathedral resume `. Available miner engines keep their own durable fences and journals, so resuming restarts the engine against that state. The node tells you that is what it is doing rather than implying a checkpoint the engines do not have. @@ -147,9 +142,8 @@ deserves its own explicit act. - **Ask for a coldkey.** Not for mining, not for validating, not ever. Only a hotkey address. A coldkey, seed, or mnemonic is refused in every field. - **Write to the chain.** `--broadcast` is refused, with or without `--yes`. - Submitting weights needs a registered validator wallet with a permit and is an - owner action taken against the engine directly, deliberately outside this - node's reach. + This CLI also provides no supported direct-engine bypass. Chain authority and + the reviewed direct-Validator bootstrap remain a separate owner-controlled workflow. - **Install anything from the legacy `cathedralai/cathedral` repository.** - **Spend money, provision hardware, or register an identity.** diff --git a/tests/_bundle_fixture.py b/tests/_bundle_fixture.py index cbf11ec..f69c43d 100644 --- a/tests/_bundle_fixture.py +++ b/tests/_bundle_fixture.py @@ -169,7 +169,8 @@ def stage_revocation_cache(self, raw: bytes, signature: bytes) -> None: # ---- wheels --------------------------------------------------------------- def _wheel(self, name: str, module: str, scripts: dict[str, str], extra_dep: str | None = None, - server_exit: int | None = None) -> Path: + server_exit: int | None = None, + declared_extras: tuple[str, ...] = ()) -> Path: src = self.root / f"src-{name}" (src / module).mkdir(parents=True, exist_ok=True) (src / module / "__init__.py").write_text("") @@ -243,7 +244,17 @@ def _wheel(self, name: str, module: str, scripts: dict[str, str], extra_dep: str elif kind == "server_bad": body += f"def {fn}():\n sys.exit({server_exit})\n" (src / module / "cli.py").write_text(body) - dep = f'[project.optional-dependencies]\nintegration=["{extra_dep}"]\n' if extra_dep else "" + optional: dict[str, list[str]] = {} + if extra_dep: + optional["integration"] = [extra_dep] + for extra in declared_extras: + optional.setdefault(extra, []) + dep = "" + if optional: + dep = "[project.optional-dependencies]\n" + "".join( + f"{json.dumps(extra)}={json.dumps(requirements)}\n" + for extra, requirements in sorted(optional.items()) + ) eps = "".join(f'{func}="{module}.cli:{func.replace("-", "_")}"\n' for func in scripts) (src / "pyproject.toml").write_text( f'[build-system]\nrequires=["setuptools"]\nbuild-backend="setuptools.build_meta"\n' @@ -328,7 +339,12 @@ def plan_from_lock(self, lock: Lock) -> dict: scripts.update({ep: "server_ok" for ep in pin.server_entrypoints}) if not scripts: scripts = {f"{pin.distribution}-noop": "client"} - wheel = self._wheel(pin.distribution, module, scripts) + wheel = self._wheel( + pin.distribution, + module, + scripts, + declared_extras=tuple(pin.extras), + ) plan[role] = {"dist": pin.distribution, "extras": list(pin.extras), "eps": list(pin.entrypoints), "seps": list(pin.server_entrypoints), "mode": pin.launch_mode, "wheels": [wheel], diff --git a/tests/test_contract.py b/tests/test_contract.py index 57ad457..2c8c049 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -28,6 +28,10 @@ from cathedral_node.contracts.codes import retryable # noqa: E402 from cathedral_node.contracts.version import PROTOCOL_VERSION, RESULT_SCHEMA, compatible # noqa: E402 from cathedral_node.engines.base import Engine, UnverifiedEngine # noqa: E402 +from cathedral_node.engines.validator import ( # noqa: E402 + REVIEWED_CONTRACT_EVIDENCE, + ValidatorContractIncompatible, +) from cathedral_node.redact import redact_text, redact_value # noqa: E402 @@ -453,7 +457,12 @@ def test_an_unbound_adapter_refuses_to_resolve_an_executable_path(self): adapter = engines.load(role, self.lock) self.assertIsNone(adapter.verified) self.assertFalse(adapter.has_bin("python")) - with self.assertRaises(UnverifiedEngine): + expected = ( + ValidatorContractIncompatible + if role == "validator" + else UnverifiedEngine + ) + with self.assertRaises(expected): adapter.operate_argv(cfg, dry_run=True) def test_pins_are_full_commit_shas(self): @@ -490,6 +499,775 @@ def test_trusted_repository_parses_the_origin_not_a_substring(self): self.assertFalse(lockfile.is_trusted_repository(url), url) +class TestValidatorContractQuarantine(CliCase): + """The stale relay adapter must never become an accidental chain controller.""" + + def test_discovery_reports_the_current_incompatibility(self): + adapter = engines.load("validator", lockfile.load()) + explanation = adapter.explain() + rendered = json.dumps(explanation, sort_keys=True) + self.assertIn("direct chain writer", rendered) + self.assertIn("no non-writing mode", rendered) + evidence = explanation["reviewed_contract_evidence"] + self.assertEqual( + evidence["commit"], + "d225e8758ca02627cced800b7de0c79464d89aee", + ) + self.assertEqual( + evidence["entrypoint"], + "cathedral_thin.independent_runtime.direct_validator:main", + ) + self.assertEqual( + evidence["entrypoint_source_sha256"], + "ac258f889192a88eb9269a48bb3de5f488419c8645bff59b7cb2e9aa335204eb", + ) + self.assertEqual(evidence, REVIEWED_CONTRACT_EVIDENCE) + + proc, public_explanation = self.json_cli("explain", "validator") + self.assertEqual(proc.returncode, 0) + public_evidence = public_explanation["data"]["reviewed_contract_evidence"] + for field in ("entrypoint_source_sha256", "pyproject_sha256"): + self.assertEqual(public_evidence[field], REVIEWED_CONTRACT_EVIDENCE[field]) + self.assertNotEqual(public_evidence[field], "[redacted]") + human_explanation = self.run_cli("explain", "validator") + self.assertEqual(human_explanation.returncode, 0) + self.assertIn( + REVIEWED_CONTRACT_EVIDENCE["entrypoint_source_sha256"], + human_explanation.stdout, + ) + + capabilities = adapter.capabilities() + for name in ("local_test", "dry_run", "offline_dry_run", "operate"): + with self.subTest(capability=name): + self.assertFalse(capabilities[name]["available"]) + self.assertTrue(capabilities["contract_diagnostics"]["available"]) + + qualification = adapter.qualify(config.defaults("validator")) + self.assertFalse(qualification.can_local_test) + self.assertFalse(qualification.can_operate) + self.assertEqual( + qualification.blockers[0]["code"], + "contract.engine_incompatible", + ) + self.assertTrue(qualification.blockers[0]["requires_operator"]) + + help_result = self.run_cli("--help") + self.assertEqual(help_result.returncode, 0) + self.assertIn("fail closed; launches nothing", help_result.stdout) + self.assertNotIn("safe dry run against the signed feed", help_result.stdout) + + def test_direct_adapter_calls_refuse_before_creating_runtime_state(self): + adapter = engines.load("validator", lockfile.load()) + run_root = self.home / "runs" / "must-not-exist" + + with self.assertRaises(ValidatorContractIncompatible): + adapter.local_test( + config.defaults("validator"), + "must-not-exist", + progress=lambda *_args: self.fail("progress must not run"), + timeout=1, + ) + with self.assertRaises(ValidatorContractIncompatible): + adapter.operate_argv(config.defaults("validator"), dry_run=True) + with self.assertRaises(ValidatorContractIncompatible): + adapter.operate_env(config.defaults("validator")) + with self.assertRaises(ValidatorContractIncompatible): + adapter.interpret_line("WEIGHTS_SUBMITTED") + + self.assertFalse(run_root.exists()) + + import dataclasses + from types import SimpleNamespace + from unittest import mock + + from cathedral_node.engines import installer + + pin = dataclasses.replace( + lockfile.load().pin("validator"), + entrypoints=("validator-client",), + server_entrypoints=(), + ) + probe_result = SimpleNamespace(returncode=0, stdout="ok\n", timed_out=False) + with mock.patch.object(installer.proc, "probe", return_value=probe_result) as probe: + ok, detail = installer._self_check( + Path("/sealed/venv"), + pin, + Path("/sealed/generation"), + allow_server_start=False, + ) + self.assertTrue(ok, detail) + self.assertIn("no installed code executed", detail) + probe.assert_not_called() + + static_verified = SimpleNamespace( + receipt_data={"execution_validation": "static_quarantine"}, + python=Path("/sealed/venv/bin/python"), + bin=lambda name: Path("/sealed/venv/bin") / name, + has_bin=lambda _name: True, + ) + available_adapter = engines.load("distill", lockfile.load()) + available_adapter._verified = static_verified + with self.assertRaises(UnverifiedEngine): + available_adapter.python() + with self.assertRaises(UnverifiedEngine): + available_adapter.bin("cathedral-cybergym-agent") + self.assertFalse(available_adapter.has_bin("cathedral-cybergym-agent")) + + server_pin = dataclasses.replace( + pin, + server_entrypoints=("validator-server",), + ) + with mock.patch.object(installer.proc, "probe", return_value=probe_result) as probe: + ok, detail = installer._self_check( + Path("/sealed/venv"), + server_pin, + Path("/sealed/generation"), + allow_server_start=False, + ) + self.assertTrue(ok, detail) + self.assertIn("no installed code executed", detail) + probe.assert_not_called() + + def test_all_public_test_and_start_variants_refuse_without_a_run(self): + runs = self.home / "runs" + before = set(runs.iterdir()) if runs.exists() else set() + cases = ( + ("test", "validator"), + ("test", "validator", "--dry-run"), + ("start", "validator"), + ("start", "validator", "--dry-run"), + ("start", "validator", "--once"), + ("start", "validator", "--once", "--dry-run"), + ("quickstart", "validator"), + ("quickstart", "validator", "--dry-run"), + ("setup", "validator"), + ("setup", "validator", "--dry-run"), + ) + for args in cases: + with self.subTest(command=" ".join(args)): + proc, payload = self.json_cli(*args) + self.assertEqual(proc.returncode, int(Exit.INCOMPATIBLE)) + self.assertEqual( + payload["error"]["code"], + "contract.engine_incompatible", + ) + self.assertTrue(payload["error"]["remediation"]["requires_operator"]) + self.assertNotIn("--confirm-direct-write", proc.stdout + proc.stderr) + + after = set(runs.iterdir()) if runs.exists() else set() + self.assertEqual(after, before, "quarantined commands must create no run") + + def test_diagnostics_and_legacy_config_do_not_prescribe_a_retry(self): + proc, explanation = self.json_cli("explain", "validator") + self.assertEqual(proc.returncode, 0) + next_commands = [step.get("command", "") for step in explanation["next"]] + self.assertFalse(any("setup validator" in command for command in next_commands)) + + proc, diagnosis = self.json_cli("doctor", "validator") + self.assertEqual(proc.returncode, int(Exit.INCOMPATIBLE)) + self.assertEqual(diagnosis["error"]["code"], "contract.engine_incompatible") + self.assertTrue(diagnosis["error"]["remediation"]["requires_operator"]) + self.assertNotIn("setup validator", proc.stdout + proc.stderr) + + prior_disk = os.environ.get("CATHEDRAL_TEST_ASSUME_DISK_GB") + os.environ["CATHEDRAL_TEST_ASSUME_DISK_GB"] = "0" + try: + proc, diagnosis = self.json_cli("doctor", "validator") + finally: + if prior_disk is None: + os.environ.pop("CATHEDRAL_TEST_ASSUME_DISK_GB", None) + else: + os.environ["CATHEDRAL_TEST_ASSUME_DISK_GB"] = prior_disk + self.assertEqual(proc.returncode, int(Exit.INCOMPATIBLE)) + self.assertEqual(diagnosis["error"]["code"], "contract.engine_incompatible") + + proc, brief = self.json_cli("agent-brief", "validator") + self.assertEqual(proc.returncode, 0) + text = brief["data"]["brief"] + self.assertIn("Diagnostics only", text) + for command in ( + "cathedral setup validator", + "cathedral test validator", + "cathedral start validator", + ): + self.assertNotIn(command, text) + + config_path = self.home / "config" / "validator.toml" + self.assertFalse(config_path.exists()) + for args in ( + ("config", "set", "validator", "netuid", "7"), + ("config", "set", "validator", "netuid", "7", "--dry-run"), + ): + with self.subTest(command=" ".join(args)): + proc, blocked = self.json_cli(*args) + self.assertEqual(proc.returncode, int(Exit.INCOMPATIBLE)) + self.assertEqual(blocked["error"]["code"], "contract.engine_incompatible") + self.assertFalse(config_path.exists()) + + proc, unknown = self.json_cli( + "config", "set", "validator", "not_a_validator_field", "value" + ) + self.assertEqual(proc.returncode, int(Exit.USAGE)) + self.assertEqual(unknown["error"]["code"], "config.field_required") + + proc, forbidden = self.json_cli( + "config", "set", "validator", "coldkey", "do-not-store" + ) + self.assertEqual(proc.returncode, int(Exit.USAGE)) + self.assertEqual(forbidden["error"]["code"], "identity.coldkey_refused") + self.assertIsNone(forbidden["error"]["remediation"]["command"]) + self.assertEqual( + forbidden["error"]["remediation"]["docs"], + "cathedral explain validator", + ) + self.assertTrue(forbidden["error"]["remediation"]["requires_operator"]) + self.assertFalse(config_path.exists()) + + with tempfile.TemporaryDirectory(prefix="cathedral-malformed-legacy-") as home: + isolated = Path(home) + config_dir = isolated / "config" + config_dir.mkdir(parents=True) + (config_dir / "validator.toml").write_text( + 'network = "unterminated\n', encoding="utf-8" + ) + prior_disk = os.environ.get("CATHEDRAL_TEST_ASSUME_DISK_GB") + os.environ["CATHEDRAL_TEST_ASSUME_DISK_GB"] = "999" + try: + for args, expected in ( + (("capabilities",), 0), + (("doctor",), 0), + (("doctor", "validator"), int(Exit.INCOMPATIBLE)), + (("agent-brief",), 0), + (("agent-brief", "validator"), 0), + (("quickstart",), 0), + ): + with self.subTest(malformed_legacy_config=" ".join(args)): + proc, result = self.json_cli(*args, home=isolated) + self.assertEqual(proc.returncode, expected, proc.stderr) + self.assertNotEqual( + (result.get("error") or {}).get("code"), + "config.invalid", + ) + for args in ( + ("config", "show", "validator"), + ("config", "get", "validator", "netuid"), + ): + with self.subTest(malformed_legacy_config=" ".join(args)): + proc, result = self.json_cli(*args, home=isolated) + self.assertEqual(proc.returncode, int(Exit.CONFIG_INVALID)) + self.assertEqual(result["error"]["code"], "config.invalid") + remediation = result["error"]["remediation"] + self.assertIsNone(remediation["command"]) + self.assertEqual( + remediation["docs"], "cathedral explain validator" + ) + self.assertTrue(remediation["requires_operator"]) + + proc, status = self.json_cli("status", "validator", home=isolated) + self.assertEqual(proc.returncode, 0, proc.stderr) + contract = status["data"]["roles"]["validator"]["contract_status"] + self.assertTrue(contract["legacy_config"]["inert"]) + self.assertTrue(contract["legacy_config"]["present"]) + self.assertFalse(contract["legacy_config"]["readable"]) + self.assertIn("not valid TOML", contract["legacy_config"]["detail"]) + self.assertIn( + "contract.legacy_config_unreadable", + {warning["code"] for warning in status["warnings"]}, + ) + human_status = self.run_cli("status", "validator", home=isolated) + self.assertEqual(human_status.returncode, 0, human_status.stderr) + self.assertIn("inert and unreadable", human_status.stdout.lower()) + + (config_dir / "validator.toml").write_text( + 'interval_secs = "abc"\n', encoding="utf-8" + ) + proc, wrong_type = self.json_cli( + "config", "show", "validator", home=isolated + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn( + "must be an integer", + " ".join(wrong_type["data"]["problems"]["validator"]), + ) + self.assertIn( + "config.invalid", + {warning["code"] for warning in wrong_type["warnings"]}, + ) + finally: + if prior_disk is None: + os.environ.pop("CATHEDRAL_TEST_ASSUME_DISK_GB", None) + else: + os.environ["CATHEDRAL_TEST_ASSUME_DISK_GB"] = prior_disk + + with tempfile.TemporaryDirectory(prefix="cathedral-historical-validator-") as home: + isolated = Path(home) + runs_dir = isolated / "runs" + statuses = ("running", "completed", "failed", "cancelled", "interrupted") + originals: dict[str, str] = {} + for index, status_name in enumerate(statuses): + run_id = f"test-validator-{status_name}" + run_dir = runs_dir / run_id + run_dir.mkdir(parents=True) + # Completed is newest so role-scoped status/logs select it. + minute = 59 if status_name == "completed" else index + record = { + "schema": "cathedral.node.run.v1", + "run_id": run_id, + "role": "validator", + "kind": "test", + "status": status_name, + "started_at": f"2026-08-30T12:{minute:02d}:00.000Z", + "finished_at": ( + None if status_name == "running" else "2026-08-30T13:00:00.000Z" + ), + # A dead pid would normally make reconcile() rewrite this + # record. Quarantined Validator history must remain byte-exact. + "pid": 2_147_483_647 if status_name == "running" else None, + "exit_code": 0 if status_name == "completed" else None, + "detail": "retired signed-feed result", + "artifacts": {}, + } + record_text = json.dumps(record, indent=2) + "\n" + (run_dir / "run.json").write_text(record_text, encoding="utf-8") + originals[run_id] = record_text + (run_dir / "events.jsonl").write_text( + json.dumps({ + "schema": "cathedral.node.event.v1", + "ts": "2026-08-30T12:59:01.000Z", + "run_id": run_id, + "event": "WEIGHTS_DRY_RUN", + "stage": "verify", + "status": "PASS", + "detail": "retired relay accepted", + }) + "\n", + encoding="utf-8", + ) + (run_dir / "engine.log").write_text( + "retired relay output\n", encoding="utf-8" + ) + + proc, summary = self.json_cli("status", "validator", home=isolated) + self.assertEqual(proc.returncode, 0) + last_test = summary["data"]["roles"]["validator"]["last_test"] + self.assertTrue(last_test["historical_contract"]) + self.assertEqual( + last_test["contract_status"]["execution_status"], "quarantined" + ) + + completed_id = "test-validator-completed" + running_id = "test-validator-running" + for args in ( + ("status", "--run", completed_id), + ("status", "--run", running_id), + ("logs", "validator"), + ("logs", "--run", completed_id), + ("logs", "--run", running_id), + ("logs", "--run", completed_id, "--raw"), + ("logs", "--run", running_id, "--raw", "--follow"), + ("evidence", completed_id), + ("evidence", running_id), + ("evidence", "retired relay accepted"), + ): + with self.subTest(historical_validator=" ".join(args)): + proc, result = self.json_cli(*args, home=isolated) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertTrue( + result["data"].get("historical_contract") + or result["data"].get("run", {}).get("historical_contract") + ) + warning_codes = {warning["code"] for warning in result["warnings"]} + self.assertIn( + "contract.historical_run", + warning_codes, + ) + if "--follow" in args: + self.assertIn( + "contract.historical_follow_disabled", warning_codes + ) + + human_evidence = self.run_cli("evidence", completed_id, home=isolated) + self.assertEqual(human_evidence.returncode, 0) + self.assertIn("historical", human_evidence.stdout.lower()) + + proc, cancelled = self.json_cli("cancel", completed_id, home=isolated) + self.assertEqual(proc.returncode, 0) + self.assertTrue(cancelled["data"]["historical_contract"]) + self.assertIn( + "contract.historical_run", + {warning["code"] for warning in cancelled["warnings"]}, + ) + + for status_name in statuses: + run_id = f"test-validator-{status_name}" + with self.subTest(resume_historical_status=status_name): + proc, result = self.json_cli("resume", run_id, home=isolated) + self.assertEqual(proc.returncode, int(Exit.INCOMPATIBLE)) + self.assertEqual( + result["error"]["code"], "contract.engine_incompatible" + ) + self.assertTrue( + result["error"]["detail"]["historical_contract"] + ) + + for run_id, original in originals.items(): + self.assertEqual( + (runs_dir / run_id / "run.json").read_text(encoding="utf-8"), + original, + "read-time annotations must not rewrite historical evidence", + ) + + # A live ownership record under a quarantined role is a warning, never a + # green running state. Exercise the report builder without signalling it. + import importlib + from types import SimpleNamespace + from unittest import mock + + status_module = importlib.import_module("cathedral_node.commands.status") + installed = SimpleNamespace( + to_dict=lambda: { + "installed": True, + "revision_drift": False, + "short_revision": "deadbeef1234", + "expected_short_revision": "deadbeef1234", + } + ) + live = { + "pid": 4242, + "since": "2026-08-30T12:00:00.000Z", + "run_id": "start-validator-old", + } + with ( + mock.patch.object(status_module.state, "running_run", return_value=live), + mock.patch.object(status_module.state, "list_runs", return_value=[]), + ): + report = status_module._report( + lockfile.load(), + ["validator"], + 10, + {"validator": installed}, + None, + {}, + ) + self.assertIsNotNone( + report.data["roles"]["validator"]["incompatible_live_process"] + ) + self.assertIn( + "contract.incompatible_live_process", + {warning.code for warning in report.warnings}, + ) + self.assertFalse(report.next_steps[0].safe) + + run_module = importlib.import_module("cathedral_node.commands.run") + live_record = run_module.state.RunRecord( + "test-validator-unstoppable", + "validator", + "test", + "running", + "2026-08-30T12:00:00.000Z", + pid=os.getpid(), + ) + cancel_ctx = SimpleNamespace( + args=SimpleNamespace(run=live_record.run_id), + dry_run=False, + ) + with ( + mock.patch.object(run_module.state, "load_run", return_value=live_record), + mock.patch.object(run_module.state, "reconcile") as reconcile, + mock.patch.object( + run_module.state, + "stop_role", + return_value=(False, "ownership could not be proven"), + ) as stop_role, + mock.patch.object(run_module.state, "finish_run") as finish_run, + ): + failed_cancel = run_module.cancel(cancel_ctx) + self.assertEqual(failed_cancel.exit_code, Exit.WORK_FAILED) + self.assertFalse(failed_cancel.error.detail["cancelled"]) + self.assertTrue(failed_cancel.error.detail["historical_contract"]) + reconcile.assert_not_called() + stop_role.assert_called_once_with( + "validator", expected_run_id=live_record.run_id + ) + finish_run.assert_not_called() + + # The lower-level stop primitive performs the target binding before it + # can signal a process group. A stale run record must never cancel the + # newer run that currently owns the same role. + foreign_owner = SimpleNamespace(run_id="start-validator-new-owner") + with ( + mock.patch.object( + run_module.state, + "ownership_status", + return_value=(run_module.state.OWNERSHIP_LIVE, foreign_owner, "live"), + ), + mock.patch.object(run_module.state.os, "killpg") as killpg, + ): + stopped, detail = run_module.state.stop_role( + "validator", expected_run_id=live_record.run_id + ) + self.assertFalse(stopped) + self.assertIn("not requested run", detail) + self.assertIn("nothing was signalled", detail) + killpg.assert_not_called() + + with ( + mock.patch.object( + run_module.state, + "ownership_status", + return_value=( + run_module.state.OWNERSHIP_ABSENT, + None, + "no ownership record", + ), + ), + mock.patch.object( + run_module.state, + "_ownership_from_lease", + return_value=(None, "no matching lease"), + ), + mock.patch.object( + run_module.state, "_close_finished_leases" + ) as close_leases, + ): + stopped, detail = run_module.state._stop_role_locked( + "validator", expected_run_id=live_record.run_id + ) + self.assertFalse(stopped) + self.assertIn("nothing was signalled or reconciled", detail) + close_leases.assert_not_called() + + signal_owner = run_module.state.ChildOwnership( + role="validator", + run_id=live_record.run_id, + parent_pid=os.getpid(), + child_pid=424_242, + pgid=424_242, + start_identity="linux-startticks:1", + boot_id=run_module.state.boot_identity(), + euid=os.geteuid(), + generation="generation-a", + lock_digest="digest-a", + token="owner-token-a", + since="2026-08-30T12:00:00.000Z", + spawn_state=run_module.state.SPAWN_OWNED, + ) + with ( + mock.patch.object( + run_module.state, + "ownership_status", + side_effect=( + (run_module.state.OWNERSHIP_LIVE, signal_owner, "live"), + ( + run_module.state.OWNERSHIP_UNVERIFIABLE, + signal_owner, + "leader identity is no longer proven", + ), + ), + ), + mock.patch.object(run_module.state.os, "killpg") as killpg, + ): + stopped, detail = run_module.state._stop_role_locked( + "validator", expected_run_id=live_record.run_id + ) + self.assertFalse(stopped) + self.assertIn("no longer proven", detail) + killpg.assert_not_called() + + import dataclasses as dc + + foreign_role_owner = dc.replace(signal_owner, role="compute") + validator_ownership = run_module.state.paths.role_lock("validator") + run_module.state.write_ownership_document( + validator_ownership, + json.dumps(foreign_role_owner.to_dict()).encode("utf-8"), + ) + with ( + mock.patch.object(run_module.state.os, "killpg") as killpg, + mock.patch.object( + run_module.state, "_close_finished_leases" + ) as close_leases, + ): + stopped, detail = run_module.state.stop_role( + "validator", expected_run_id=foreign_role_owner.run_id + ) + self.assertFalse(stopped) + self.assertIn("foreign role", detail) + self.assertTrue(validator_ownership.exists()) + killpg.assert_not_called() + close_leases.assert_not_called() + + stop_ctx = SimpleNamespace( + args=SimpleNamespace(role="validator"), + dry_run=False, + ) + current_holder = { + "pid": 4242, + "run_id": live_record.run_id, + "since": "2026-08-30T12:00:00.000Z", + } + with ( + mock.patch.object( + run_module.state, "running_run", return_value=current_holder + ), + mock.patch.object( + run_module.state, + "stop_role", + return_value=(False, "owner changed; nothing was signalled"), + ) as stop_role, + ): + refused_stop = run_module.stop(stop_ctx) + self.assertEqual(refused_stop.exit_code, Exit.WORK_FAILED) + stop_role.assert_called_once_with( + "validator", expected_run_id=live_record.run_id + ) + + corrupt_record = run_module.state.RunRecord( + "test-validator-corrupt-role", + "../../outside", + "test", + "running", + "2026-08-30T12:00:00.000Z", + pid=os.getpid(), + ) + corrupt_ctx = SimpleNamespace( + args=SimpleNamespace(run=corrupt_record.run_id), + dry_run=False, + ) + with ( + mock.patch.object(run_module.state, "load_run", return_value=corrupt_record), + mock.patch.object(run_module.engines, "load") as load_engine, + mock.patch.object(run_module.state, "stop_role") as stop_role, + ): + refused_resume = run_module.resume(corrupt_ctx) + refused_cancel = run_module.cancel(corrupt_ctx) + for refused in (refused_resume, refused_cancel): + self.assertEqual(refused.exit_code, Exit.CONFIG_INVALID) + self.assertEqual(refused.error.code, "config.invalid") + self.assertTrue(refused.error.remediation.requires_operator) + load_engine.assert_not_called() + stop_role.assert_not_called() + + alias_id = "test-validator-alias" + embedded_id = "test-validator-other" + alias_dir = self.home / "runs" / alias_id + alias_dir.mkdir(parents=True) + (alias_dir / "run.json").write_text( + json.dumps({ + "schema": "cathedral.node.run.v1", + "run_id": embedded_id, + "role": "validator", + "kind": "test", + "status": "running", + "started_at": "2026-08-30T12:00:00.000Z", + "pid": os.getpid(), + "artifacts": {}, + }) + "\n", + encoding="utf-8", + ) + alias_ctx = SimpleNamespace( + args=SimpleNamespace(run=alias_id), + dry_run=False, + ) + with ( + mock.patch.object(run_module.state, "stop_role") as stop_role, + mock.patch.object(run_module.state, "finish_run") as finish_run, + ): + refused_alias = run_module.cancel(alias_ctx) + self.assertEqual(refused_alias.exit_code, Exit.NOT_FOUND) + self.assertEqual(refused_alias.error.code, "run.not_found") + stop_role.assert_not_called() + finish_run.assert_not_called() + self.assertFalse((self.home / "runs" / embedded_id).exists()) + + proc, no_logs = self.json_cli("logs", "validator") + self.assertEqual(proc.returncode, int(Exit.NOT_FOUND)) + self.assertIsNone(no_logs["error"]["remediation"]["command"]) + self.assertTrue(no_logs["error"]["remediation"]["requires_operator"]) + + def test_capabilities_label_the_validator_pin_as_historical(self): + proc, payload = self.json_cli("capabilities") + self.assertEqual(proc.returncode, 0) + report = payload["data"]["engines"]["validator"] + self.assertEqual(report["contract_status"]["execution_status"], "quarantined") + self.assertEqual( + report["contract_status"]["pin_status"], + "retained signed-release member; CLI execution quarantined", + ) + self.assertEqual( + report["capabilities"]["contract_diagnostics"]["evidence"]["commit"], + "d225e8758ca02627cced800b7de0c79464d89aee", + ) + self.assertNotIn("validator", payload["data"]["conventions"]["resumable"]) + + proc, status = self.json_cli("status", "validator") + self.assertEqual(proc.returncode, 0) + self.assertEqual( + status["data"]["roles"]["validator"]["contract_status"]["execution_status"], + "quarantined", + ) + + import importlib + import io + + from cathedral_node.ui.console import Console + from cathedral_node.ui.theme import Style + + contract_status = { + "execution_status": "quarantined", + "blocker": {"code": "contract.engine_incompatible"}, + } + setup_module = importlib.import_module("cathedral_node.commands.setup") + setup_data = { + "installed": {"short_revision": "deadbeef1234"}, + "release": {"release_version": 7, "signer_identity": "test-signer"}, + "roles": { + "compute": { + "installed": True, + "contract_status": {"execution_status": "not_statically_blocked"}, + }, + "validator": { + "installed": True, + "contract_status": contract_status, + }, + }, + "config_file": "/tmp/compute.toml", + "config_problems": [], + "notes": [], + "can_local_test": True, + "can_operate": True, + } + buffer = io.StringIO() + setup_module._render( + Console(stream=buffer, style=Style(enabled=False)), + setup_data, + Envelope.ok("setup", setup_data), + ) + rendered_setup = buffer.getvalue() + self.assertIn("retained package; execution quarantined", rendered_setup) + self.assertNotIn("validator active", rendered_setup) + + update_module = importlib.import_module("cathedral_node.commands.update") + update_data = { + "current_version": 6, + "available_version": 7, + "applied": False, + "plan": [{ + "role": "validator", + "changes": True, + "contract_status": contract_status, + }], + } + buffer = io.StringIO() + update_module._render_update( + Console(stream=buffer, style=Style(enabled=False)), + update_data, + Envelope.ok("update", update_data), + ) + rendered_update = buffer.getvalue() + self.assertIn("retained package; execution quarantined", rendered_update) + self.assertNotIn("validator active", rendered_update) + + class TestConfigSafety(unittest.TestCase): def test_forbidden_fields_cannot_be_saved(self): with self.assertRaises(config.ConfigError): @@ -509,22 +1287,19 @@ def test_non_loopback_without_tls_is_refused(self): self.assertTrue(any("TLS" in p for p in problems), problems) def test_owner_controlled_fields_are_declared(self): - owner = config.OWNER_CONTROLLED["validator"] - for field in ("wallet_name", "wallet_hotkey", "require_policy", "weight_policy_key"): - self.assertIn(field, owner) + """The known Validator role deliberately declares no active controls.""" + self.assertEqual(config.OWNER_CONTROLLED["validator"], ()) def test_burn_is_not_offered_as_an_operator_setting(self): - """The validator takes burn and allocation from Cathedral-signed - documents and from the signed vector, so an editable `burn_fraction` - would let an operator believe they had changed the economics when - nothing had changed.""" + """Neither the retired schema nor this quarantined CLI exposes economics + as an operator setting.""" names = {f.name for f in config.schema("validator")} for field in ("burn_fraction", "burn_destination", "lane_allocation"): self.assertNotIn(field, names, f"`{field}` is not the operator's to set") self.assertNotIn(field, config.OWNER_CONTROLLED["validator"]) def test_the_weight_contract_and_signing_key_are_the_operators(self): - """What an operator genuinely controls is what they will *accept*.""" + """Legacy public fields stay parseable without becoming active authority.""" names = {f.name for f in config.schema("validator")} self.assertIn("require_policy", names) self.assertIn("weight_policy_key", names) @@ -1214,8 +1989,7 @@ def test_ascii_fallback_is_pure_ascii_including_dynamic_text(self): def test_public_key_reads_the_same_in_json_and_human(self): """Regression: the public weight-policy key was masked in --json but shown in full by the human view, inverting the one-envelope-two-renderers - invariant and hiding the operator's signing-key control from an agent.""" - self.json_cli("setup", "validator") + invariant and hiding a migration/audit value from an agent.""" _, payload = self.json_cli("config", "show", "validator") json_value = payload["data"]["roles"]["validator"]["values"]["weight_policy_key"] self.assertNotEqual(json_value, "[redacted]", @@ -1225,8 +1999,26 @@ def test_public_key_reads_the_same_in_json_and_human(self): self.assertIn(json_value, human, "the human view and the --json envelope must show the same key") + # A same-named field in another role is not public. The exception is for + # exactly the published, retired Validator key—not arbitrary 64-hex data. + with tempfile.TemporaryDirectory(prefix="cathedral-public-key-scope-") as home: + isolated = Path(home) + config_dir = isolated / "config" + config_dir.mkdir(parents=True) + secret = "9f" * 32 + (config_dir / "compute.toml").write_text( + f'weight_policy_key = "{secret}"\n', + encoding="utf-8", + ) + proc, cross_role = self.json_cli("config", "show", "compute", home=isolated) + self.assertEqual(proc.returncode, 0) + self.assertNotIn(secret, proc.stdout) + self.assertEqual( + cross_role["data"]["roles"]["compute"]["values"]["weight_policy_key"], + "[redacted]", + ) + def test_config_get_returns_the_public_key_not_a_mask(self): - self.json_cli("setup", "validator") proc, payload = self.json_cli("config", "get", "validator", "weight_policy_key") self.assertEqual(proc.returncode, 0) self.assertRegex(payload["data"]["value"], r"^[0-9a-fA-F]{64}$") @@ -1462,6 +2254,8 @@ def test_a_config_problem_is_not_reported_as_an_internal_error(self): self.assertEqual(proc.returncode, int(Exit.CONFIG_INVALID)) self.assertEqual(payload["error"]["code"], "config.invalid") self.assertIsNotNone(payload["error"]["remediation"]) + self.assertIsNone(payload["error"]["remediation"]["command"]) + self.assertTrue(payload["error"]["remediation"]["requires_operator"]) finally: import shutil shutil.rmtree(fresh, ignore_errors=True) @@ -1495,6 +2289,34 @@ def test_a_corrupt_run_record_is_reported_as_missing(self): proc, payload = self.json_cli("status", "--run", "test-broken", home=home) self.assertEqual(proc.returncode, int(Exit.NOT_FOUND)) self.assertEqual(payload["error"]["code"], "run.not_found") + + (run_dir / "run.json").write_text(json.dumps({ + "schema": "cathedral.node.run.v1", + "run_id": "test-broken", + "role": "compute", + "kind": "test", + "status": "running", + "started_at": "2026-01-01T00:00:00.000Z", + "finished_at": None, + "pid": "oops", + "exit_code": None, + "detail": "malformed current-schema record", + "artifacts": {}, + })) + for args in ( + ("status", "--run", "test-broken"), + ("logs", "--run", "test-broken"), + ("evidence", "test-broken"), + ("resume", "test-broken"), + ("cancel", "test-broken"), + ): + with self.subTest(malformed_current_schema=" ".join(args)): + proc, payload = self.json_cli(*args, home=home) + self.assertEqual(proc.returncode, int(Exit.NOT_FOUND), proc.stderr) + expected_code = ( + "run.identifier_unknown" if args[0] == "evidence" else "run.not_found" + ) + self.assertEqual(payload["error"]["code"], expected_code) finally: import shutil shutil.rmtree(home, ignore_errors=True) @@ -1505,6 +2327,7 @@ def test_corrupt_event_lines_are_skipped_not_fatal(self): run_dir = home / "runs" / "test-partial" run_dir.mkdir(parents=True) (run_dir / "run.json").write_text(json.dumps({ + "schema": "cathedral.node.run.v1", "run_id": "test-partial", "role": "distill", "kind": "test", "status": "completed", "started_at": "2026-01-01T00:00:00.000Z", "finished_at": "2026-01-01T00:00:01.000Z", "pid": None, @@ -1702,6 +2525,12 @@ def _write(self, repository: str, revision: str) -> Path: return path def test_an_unsigned_lockfile_update_is_refused(self): + checked_in = json.loads((REPO / "cathedral.lock.json").read_text()) + note = checked_in.get("note", "") + self.assertNotIn("update --to", note) + self.assertIn("signed whole-node release", note) + self.assertIn("update --release --yes", note) + for repository in ("https://attacker.example/evil.git", "https://github.com/attacker/evil.git", "https://github.com/cathedralai/cathedral-distill.git"): @@ -1716,3 +2545,10 @@ def test_update_check_reports_without_a_configured_release(self): self.assertEqual(proc.returncode, 0) self.assertEqual(payload["status"], "ok") self.assertIn("available_version", payload["data"]) + + for role in lockfile.ROLES: + with self.subTest(role_qualified_update=role): + proc, scoped = self.json_cli("update", role, "--check") + self.assertEqual(proc.returncode, int(Exit.USAGE)) + self.assertEqual(scoped["error"]["code"], "usage.invalid") + self.assertEqual(scoped["error"]["detail"]["scope"], "node-wide") diff --git a/tests/test_gate0.py b/tests/test_gate0.py index f6aea8b..a913dd4 100644 --- a/tests/test_gate0.py +++ b/tests/test_gate0.py @@ -774,6 +774,82 @@ def test_an_unsatisfiable_dependency_version_constraint_is_rejected(self): self.assertNoSuccessfulNoOp(ok, detail) self.assertIn("closure", detail.lower()) + def test_static_closure_probe_handles_markers_extras_and_direct_references(self): + """The quarantined path uses the pristine bootstrap parser, not target code.""" + venv = self.home / "closure-probe-venv" + made = subprocess.run( + [str(self.fx.trusted), "-I", "-m", "venv", "--copies", str(venv)], + capture_output=True, + text=True, + ) + self.assertEqual(made.returncode, 0, made.stderr) + + def evaluate(payload: dict) -> list[str]: + payload_path = self.home / "closure-probe.json" + payload_path.write_text(json.dumps(payload)) + result = proc_module.run( + [str(venv / "bin" / "python"), "-I", "-B", "-c", + installer._SIGNED_CLOSURE_PROBE, str(payload_path)], + timeout=120, + inherit_env=False, + env=proc_module.signed_child_env(home=self.home), + ) + self.assertTrue(result.ok, result.stderr) + return json.loads(result.stdout)["problems"] + + root = {"name": "root", "version": "1.0", "provides_extras": ["Foo.Bar"]} + active_extra = { + "target": "root", + "requested_extras": ["foo_bar"], + "distributions": [{ + **root, + "requires": ["missing>=9; extra == 'foo-bar'"], + }], + } + self.assertTrue(any("missing dependency missing" in problem + for problem in evaluate(active_extra))) + + inactive_platform = { + "target": "root", + "requested_extras": [], + "distributions": [{ + **root, + "requires": ["missing>=9; sys_platform == '__cathedral_never__'"], + }], + } + self.assertEqual(evaluate(inactive_platform), []) + + transitive_extra = { + "target": "root", + "requested_extras": [], + "distributions": [ + {**root, "requires": ["dep[foo_bar]"]}, + {"name": "dep", "version": "1.0", "provides_extras": ["Foo.Bar"], + "requires": ["missing>=9; extra == 'foo.bar'"]}, + ], + } + self.assertTrue(any("missing dependency missing" in problem + for problem in evaluate(transitive_extra))) + + direct_reference = { + "target": "root", + "requested_extras": [], + "distributions": [ + {**root, "requires": ["dep @ https://evil.invalid/dep.whl"]}, + {"name": "dep", "version": "1.0", "provides_extras": [], "requires": []}, + ], + } + self.assertTrue(any("unsupported direct reference" in problem + for problem in evaluate(direct_reference))) + + unknown_extra = { + "target": "root", + "requested_extras": ["not-declared"], + "distributions": [{**root, "requires": []}], + } + self.assertTrue(any("undeclared extra" in problem + for problem in evaluate(unknown_extra))) + def test_role_readiness_and_liveness_matrix(self): """Every named readiness failure fails the install and leaves nothing active: immediate clean exit, immediate nonzero exit, exit after readiness, a client @@ -819,7 +895,7 @@ def test_generations_are_frozen_read_only_including_the_receipt(self): self.assertTrue(ok, detail) _states, group, _r = installer.install_states(self.lock) role = group.role("validator") - self.assertFalse(role.python.stat().st_mode & 0o222, + self.assertFalse(role.python_path.stat().st_mode & 0o222, "the venv python must not be writable after prepare") self.assertFalse(role.receipt.stat().st_mode & 0o222, "the receipt must be frozen with the rest of the generation") @@ -957,7 +1033,9 @@ def failing(path, document): def test_an_incomplete_generation_is_removed_when_preparation_fails(self): original = installer._self_check - installer._self_check = lambda venv, pin, gen_dir: (False, "injected self-check failure") + installer._self_check = lambda venv, pin, gen_dir, **_kwargs: ( + False, "injected self-check failure" + ) try: ok, detail, _ = self.install(self.bundle("prepfail")) finally: @@ -1090,6 +1168,12 @@ def __init__(self, role, verified): def qualify(self, _cfg): return _StubQualification() + def operation_blocker(self): + """Mirror each role's current static execution policy in runtime tests.""" + if self.role == "validator": + return {"code": "contract.engine_incompatible"} + return None + def capabilities(self): return {"local_test": {"available": True, "what_it_proves": "nothing"}} @@ -1233,7 +1317,7 @@ def test_state_python_path_comes_from_verified_role_not_second_pointer_read(self self.assertTrue(ok, reason) for role in ROLES: state = installer.state(self.lock.pin(role)) - self.assertEqual(state.python, str(group.role(role).python)) + self.assertEqual(state.python, str(group.role(role).python_path)) self.assertIn(f"/generations/{group.role(role).generation}/venv/bin/python", state.python) # The pointer-reading execution-path helpers no longer exist at all, so a # consumer cannot re-resolve a path from mutable state even by accident. @@ -1245,7 +1329,7 @@ def test_state_python_path_comes_from_verified_role_not_second_pointer_read(self def test_start_pointer_swap_after_verify_starts_no_process(self): from cathedral_node import state as run_state from cathedral_node.commands import run as run_command - stubs = self._stub("validator") + stubs = self._stub("distill") before = self.weakening_snapshot() original_create = run_state.create_run @@ -1258,19 +1342,19 @@ def swap_then_create(*a, **kw): with ProcessSpy() as spy: with installer.verified_active_group(self.lock) as active: envelope = run_command._start_verified( - self._context("validator", "start-swap"), "validator", self.lock, active) + self._context("distill", "start-swap"), "distill", self.lock, active) finally: run_state.create_run = original_create self.assertNotEqual(envelope.status, "ok", "a post-verification swap must not start") self.assertStartsNoProcess(spy) - self.assertFalse(stubs["validator"].ran) + self.assertFalse(stubs["distill"].ran) self.assertNoWeakening(before, pointer_may_change=True) # --- 7 ----------------------------------------------------------------- def test_start_generation_tamper_after_qualification_starts_no_process(self): from cathedral_node import state as run_state from cathedral_node.commands import run as run_command - self._stub("validator") + self._stub("distill") before = self.weakening_snapshot() original_create = run_state.create_run @@ -1283,7 +1367,7 @@ def tamper_then_create(*a, **kw): with ProcessSpy() as spy: with installer.verified_active_group(self.lock) as active: envelope = run_command._start_verified( - self._context("validator", "start-tamper"), "validator", self.lock, active) + self._context("distill", "start-tamper"), "distill", self.lock, active) finally: run_state.create_run = original_create self.assertNotEqual(envelope.status, "ok") @@ -1446,14 +1530,24 @@ def test_runtime_executes_only_paths_from_the_verified_group_snapshot(self): from cathedral_node.engines.base import UnverifiedEngine ok, reason, group = self.verify_active() self.assertTrue(ok, reason) - bound = engines_module.load("validator", self.lock, group) - role = group.role("validator") + # Executable-path binding is exercised by a runtime-checked role. The + # Validator member of this same sealed group is deliberately static-only. + bound = engines_module.load("distill", self.lock, group) + role = group.role("distill") self.assertEqual(bound.python(), role.python) self.assertTrue(str(bound.python()).startswith(str(role.venv_dir))) with self.assertRaises(ValueError): role.bin("evil") # not an entrypoint the signed release authorizes with self.assertRaises(ValueError): role.bin("../../bin/sh") # not a single component + + validator = engines_module.load("validator", self.lock, group) + with self.assertRaises(UnverifiedEngine): + validator.python() + with self.assertRaises(UnverifiedEngine): + validator.bin("pv-server") + self.assertFalse(validator.has_bin("pv-server")) + unbound = engines_module.load("validator", self.lock) with self.assertRaises(UnverifiedEngine): unbound.python() @@ -1902,6 +1996,7 @@ def test_every_receipt_provenance_field_changed_individually_is_refused(self): "server_entrypoints": ["evil"], "launch_mode": "worker", "protocol": "9.9", + "execution_validation": "runtime_checked", "installed_at": "2026-07-31T00:00:00", } self.assertEqual(set(forgeries), installer._RECEIPT_KEYS, @@ -1914,6 +2009,27 @@ def test_every_receipt_provenance_field_changed_individually_is_refused(self): finally: self.write_receipt(path, original) + def test_static_validation_cannot_be_promoted_with_a_recomputed_manifest(self): + """A receipt is not its own authority for becoming executable evidence.""" + path = self.receipt_path("validator") + original = json.loads(path.read_text()) + self.protect(path) + generation = self.generation_dir("validator") + ok, forged_manifest, reason = installer._local_manifest( + generation, + self.lock.pin("validator"), + execution_validation=installer.EXECUTION_RUNTIME_CHECKED, + ) + self.assertTrue(ok, reason) + forged = { + **original, + "execution_validation": installer.EXECUTION_RUNTIME_CHECKED, + "manifest_sha256": forged_manifest, + } + self.write_receipt(path, forged) + rejection = self.assertRejected("self-promoted execution validation") + self.assertIn("current role policy", rejection) + def test_an_unknown_receipt_field_is_refused(self): path = self.receipt_path("validator") original = json.loads(path.read_text()) @@ -2740,6 +2856,14 @@ def test_operate_argv_and_env_come_from_the_verified_generation_and_carry_no_sec cfg.update({"hotkey": "5F3sa2TJAWMqDhXG6jhV4N8ko9SxwGy8TpaNS1repo5EYjQX", "api_key_secret": secret, "bearer_token_secret": secret}) adapter = engines_module.load(role, self.lock, group) + if role == "validator": + from cathedral_node.engines.validator import ( + ValidatorContractIncompatible, + ) + + with self.assertRaises(ValidatorContractIncompatible): + adapter.operate_argv(cfg, dry_run=True) + continue argv = adapter.operate_argv(cfg, dry_run=True) joined = " ".join(argv) self.assertNotIn(secret, joined, "a credential reached argv") @@ -2751,6 +2875,7 @@ def test_operate_argv_and_env_come_from_the_verified_generation_and_carry_no_sec def test_every_adapter_binary_resolves_inside_its_own_verified_generation(self): from cathedral_node import engines as engines_module + from cathedral_node.engines.base import UnverifiedEngine ok, reason, group = self.verify_active() self.assertTrue(ok, reason) for role in ROLES: @@ -2758,15 +2883,21 @@ def test_every_adapter_binary_resolves_inside_its_own_verified_generation(self): verified = group.role(role) adapter = engines_module.load(role, self.lock, group) pin = self.lock.pin(role) - for name in (*pin.entrypoints, *pin.server_entrypoints, "python"): - resolved = adapter.bin(name) - self.assertEqual(resolved.parent.parent, verified.venv_dir) - self.assertTrue(resolved.is_file(), f"{role}:{name} is missing") # A role may never reach into another role's verified generation. for other in ROLES: if other == role: continue self.assertNotEqual(verified.venv_dir, group.role(other).venv_dir) + if role == "validator": + for name in (*pin.entrypoints, *pin.server_entrypoints, "python"): + with self.assertRaises(UnverifiedEngine): + adapter.bin(name) + self.assertFalse(adapter.has_bin(name)) + continue + for name in (*pin.entrypoints, *pin.server_entrypoints, "python"): + resolved = adapter.bin(name) + self.assertEqual(resolved.parent.parent, verified.venv_dir) + self.assertTrue(resolved.is_file(), f"{role}:{name} is missing") def test_the_validator_engine_config_comes_from_the_verified_source_tree(self): from cathedral_node import engines as engines_module @@ -5667,11 +5798,14 @@ def _crash_public_start(self, role: str, run_id: str): from cathedral_node import state as run_state script = ( "import os, sys, types\n" + "from pathlib import Path\n" f"sys.path.insert(0, {str(Path.cwd())!r})\n" f"sys.path.insert(0, {str(Path(__file__).resolve().parent)!r})\n" "from cathedral_node import config as config_module, engines as engines_module, runner\n" + "from cathedral_node.engines import installer as installer_module\n" "from cathedral_node.commands import run as run_command\n" "from test_gate0 import _StubEngine\n" + f"installer_module.trusted_base_executable = lambda: (Path({str(self.trusted)!r}), '')\n" "class _LongRunning(_StubEngine):\n" " def operate_argv(self, _cfg, *, dry_run=False):\n" # Run through the verified generation's regular-file interpreter, just @@ -6545,11 +6679,14 @@ class TestDetachedDescendantThroughPublicCommands(LeaseCase): def _public_start(self, role: str, run_id: str, body: str): script = ( "import os, sys, types\n" + "from pathlib import Path\n" f"sys.path.insert(0, {str(Path.cwd())!r})\n" f"sys.path.insert(0, {str(Path(__file__).resolve().parent)!r})\n" "from cathedral_node import config as config_module, engines as engines_module, runner\n" + "from cathedral_node.engines import installer as installer_module\n" "from cathedral_node.commands import run as run_command\n" "from test_gate0 import _StubEngine\n" + f"installer_module.trusted_base_executable = lambda: (Path({str(self.trusted)!r}), '')\n" "class _Detaching(_StubEngine):\n" " def operate_argv(self, _cfg, *, dry_run=False):\n" # Keep the fixture inside the same verified-generation boundary as the @@ -6939,7 +7076,7 @@ class TestVerifyToExecBinding(RuntimeBindingCase): def test_replacing_the_program_after_revalidation_starts_nothing_that_survives(self): from cathedral_node import state as run_state from cathedral_node.commands import run as run_command - stubs = self._stub("validator") + stubs = self._stub("distill") before = self.weakening_snapshot() swapped: list = [] @@ -6976,7 +7113,7 @@ def swap_then_stream(argv, **kwargs): with ProcessSpy() as spy: with installer.verified_active_group(self.lock) as active: envelope = run_command._start_verified( - self._context("validator", "exec-gap"), "validator", self.lock, active) + self._context("distill", "exec-gap"), "distill", self.lock, active) finally: run_command.stream = original_stream @@ -6993,15 +7130,15 @@ def swap_then_stream(argv, **kwargs): for pid in spawned: self.assertFalse(_pid_is_live(pid), f"pid {pid} from the swapped program is still running") - self.assertFalse(stubs["validator"].ran) + self.assertFalse(stubs["distill"].ran) self.assertNoWeakening(before, pointer_may_change=True) def test_the_binding_accepts_an_untouched_program(self): from cathedral_node.commands import run as run_command - self._stub("validator") + self._stub("distill") with installer.verified_active_group(self.lock) as active: envelope = run_command._start_verified( - self._context("validator", "exec-clean"), "validator", self.lock, active) + self._context("distill", "exec-clean"), "distill", self.lock, active) self.assertNotIn("replaced between verification and execution", str(envelope.error))