From c1cb793218f5a9cc81ea990cfa88a72255eecf5b Mon Sep 17 00:00:00 2001 From: monotophic Date: Thu, 17 Sep 2026 20:07:00 -0400 Subject: [PATCH 1/5] feat(tools): add an offline checker for ablation evidence Add tools/check_ablate_evidence.py, a standalone validator for the ABLATE_OUT evidence stream the engine's ablation-sweep mode writes: it accepts only a complete, self-binding artifact and refuses everything else -- a truncated or replayed record stream, a header that does not bind the manifest or the external config.json it names, any field with the wrong type, range or key set at any of the four record kinds (header, item header, target row, terminal completion), and a target row whose fields contradict one another in a way the producer can never emit. Framing is checked before content: NaN/Infinity JSON constants, duplicate object keys, non-ASCII text and malformed JSON are all rejected up front. The manifest proof and the header's config identity are cross-checked against the actual config.json and manifest named in the header, so evidence cannot be validated against the wrong run. Co-Authored-By: Claude Opus 5 --- c/tests/test_check_ablate_evidence.py | 914 ++++++++++++++++++++++++++ c/tools/check_ablate_evidence.py | 435 ++++++++++++ 2 files changed, 1349 insertions(+) create mode 100644 c/tests/test_check_ablate_evidence.py create mode 100644 c/tools/check_ablate_evidence.py diff --git a/c/tests/test_check_ablate_evidence.py b/c/tests/test_check_ablate_evidence.py new file mode 100644 index 000000000..7b20d1dc8 --- /dev/null +++ b/c/tests/test_check_ablate_evidence.py @@ -0,0 +1,914 @@ +"""tools/check_ablate_evidence.py must accept only a complete, self-binding +ABLATE evidence artifact and reject every other input: a truncated or +replayed record stream, a header that does not bind the manifest or the +external config.json it is checked against, any field with the wrong +type, range, or key set at any of the four record kinds (header, item +header, target row, terminal completion), and a target row whose fields +contradict one another in a way the producer can never emit. + +Checks enumerated from the source (`tools/check_ablate_evidence.py`, +read in full before writing this module) and covered below, grouped by +the function that performs them: + +- `_checked_engine_text_size` / `_bounded_config_bytes`: the 256 MiB + inclusive engine text limit, both sides. +- `_reject_constant`, `_reject_duplicate_keys`, `_json_record`: no + NaN/Infinity JSON constants, no duplicate object keys, invalid + JSON/non-ASCII text rejected. +- `_config_identity`: empty file; invalid JSON; non-object root; each + of vocab_size/num_hidden_layers/n_routed_experts/first_k_dense_replace + missing or out of range. +- `_manifest_proof`: framing -- an empty manifest, an empty record, a + carriage return inside a record and an embedded NUL are refused, while + CRLF endings and a missing final newline are accepted and reduced to + the canonical form the engine binds; non-ASCII line; non-canonical integer grammar; too few fields; + every per-item field bound (item id, T, prompt, mode, cell count); + the mode/cell-count pairing rule; the field-count/denominator + arithmetic; every per-cell bound (layer, expert, applied-target, + mode-3 vs other-mode applied-target rule, duplicate cell); + out-of-vocabulary tokens; duplicate item ids across lines. +- `validate_ablate_evidence`: the evidence framing check; the header's + key set, type, and range checks; the header-vs-config identity + check; the header-vs-manifest-proof binding check; the item header's + key set, type, and manifest-order check; the target row's key set, + type, and identity checks; the three cross-field invariants (below); + the top-k list's shape, range, and + uniqueness checks; the terminal record's key set, bounds, and exact + content check; missing/extra/trailing records at every boundary. + +Every fixture here is a literal artifact built by hand from the +module's documented wire schema (`coli-ablate/2`) and hashed with the +stdlib `hashlib` directly -- no expected value is produced by calling +the validator under test. +""" +import copy +import hashlib +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +from tools import check_ablate_evidence as ABLATE + +_DOMAIN = b"coli-ablate-manifest/2\n" + + +def _serialize(records): + return b"".join( + json.dumps(record, separators=(",", ":")).encode("ascii") + b"\n" + for record in records) + + +def _write(root, manifest_raw, evidence_raw, config_raw): + manifest = root / "manifest.txt" + evidence = root / "evidence.jsonl" + config = root / "config.json" + manifest.write_bytes(manifest_raw) + evidence.write_bytes(evidence_raw) + config.write_bytes(config_raw) + return manifest, evidence, config + + +def _run_cli(manifest, evidence, config): + return subprocess.run( + [sys.executable, str(pathlib.Path(ABLATE.__file__)), + str(manifest), str(evidence), "--config", str(config)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + + +class _GoldenFixture(unittest.TestCase): + """Shared two-item artifact, hand-derived from the documented schema. + + Manifest: item 1 (T=3, prompt=1, baseline, tokens 0,1,2) then item 2 + (T=2, prompt=1, mode 1 with one ablated cell at layer 1/expert 2, + tokens 3,0). vocab=4, n_layers=4, first_dense=1, n_experts=5, so + topk = min(32, 4) = 4. Positions/gold are derived by hand from the + manifest's own tokens: item 1 has positions [0, 1] with gold tokens + 1 and 2; item 2 has position [0] with gold token 0. + """ + + MANIFEST = b"1 3 1 0 0 0 1 2\n2 2 1 1 1 1 2 -1 3 0\n" + CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":5}\n') + MANIFEST_SHA256 = hashlib.sha256(_DOMAIN + MANIFEST).hexdigest() + CONFIG_SHA256 = hashlib.sha256(CONFIG).hexdigest() + + @classmethod + def golden_records(cls): + header = { + "t": "hdr", "schema": "coli-ablate/2", "vocab": 4, "topk": 4, + "n_layers": 4, "first_dense": 1, "n_experts": 5, + "config_sha256": cls.CONFIG_SHA256, + "manifest_sha256": cls.MANIFEST_SHA256, + "expected_items": 2, "expected_targets": 3, + } + item1_header = {"t": "ah", "item": 1, "mode": 0, "ncells": 0, + "T": 3, "n_prompt": 1, "cells": []} + row1 = {"t": "lg", "item": 1, "pos": 0, "gold": 1, + "nll": 0.2, "glogit": 1.0, "molo": 0.5, "mgn": 0.5, + "am": 1, "amlogit": 1.0, "logZ": 1.3, "corr": 1, + "tk": [[0, 0.1], [1, 1.0], [2, 0.3], [3, -0.2]]} + row2 = {"t": "lg", "item": 1, "pos": 1, "gold": 2, + "nll": 0.7, "glogit": 0.4, "molo": 0.9, "mgn": -0.5, + "am": 0, "amlogit": 0.9, "logZ": 1.1, "corr": 0, + "tk": [[0, 0.9], [1, 0.1], [2, 0.4], [3, -0.3]]} + item2_header = {"t": "ah", "item": 2, "mode": 1, "ncells": 1, + "T": 2, "n_prompt": 1, "cells": [[1, 2, -1]]} + row3 = {"t": "lg", "item": 2, "pos": 0, "gold": 0, + "nll": 0.0, "glogit": 2.0, "molo": -1e30, "mgn": 1e30, + "am": 0, "amlogit": 2.0, "logZ": 2.0, "corr": 1, + "tk": [[0, 2.0], [1, -1.0], [2, -2.0], [3, -3.0]]} + done = {"t": "done", "manifest_sha256": cls.MANIFEST_SHA256, + "completed_items": 2, "completed_targets": 3} + return [header, item1_header, row1, row2, item2_header, row3, done] + + def _reject(self, mutate, records=None): + """Apply `mutate` to a deep copy of the golden records and assert + the mutated artifact is refused.""" + mutated = copy.deepcopy(records if records is not None + else self.golden_records()) + mutate(mutated) + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(mutated), self.CONFIG) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE.validate_ablate_evidence(manifest, evidence, config) + + +class GoldenArtifactAcceptedTests(_GoldenFixture): + def test_valid_artifact_is_accepted_and_pass_line_is_exact(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(self.golden_records()), + self.CONFIG) + result = ABLATE.validate_ablate_evidence( + manifest, evidence, config) + self.assertEqual(result, { + "manifest_sha256": self.MANIFEST_SHA256, + "items": 2, "targets": 3, + }) + cli = _run_cli(manifest, evidence, config) + self.assertEqual(cli.returncode, 0, cli.stderr.decode(errors="replace")) + self.assertEqual(cli.stderr, b"") + # print() terminates the line with the platform's newline, so a + # Windows child hands back CRLF; the pin is on the line's content. + self.assertEqual( + cli.stdout.replace(b"\r\n", b"\n"), + f"[ablate-evidence] PASS manifest={self.MANIFEST_SHA256} " + f"items=2 targets=3\n".encode("ascii")) + + +class TopkProducerCapAboveVocabFourTests(unittest.TestCase): + """The header `topk == min(32, vocab)` check only ever exercises the + "vocab is the binding constraint" side at `_GoldenFixture`'s vocab=4. + This fixture uses vocab=40 (above both 4 and the 32 cap) to pin the + other side: topk must be capped at 32, not left equal to vocab. + """ + + MANIFEST = b"1 2 1 0 0 0 39\n" + CONFIG = (b'{"vocab_size":40,"num_hidden_layers":1,' + b'"first_k_dense_replace":0,"n_routed_experts":1}\n') + MANIFEST_SHA256 = hashlib.sha256(_DOMAIN + MANIFEST).hexdigest() + CONFIG_SHA256 = hashlib.sha256(CONFIG).hexdigest() + + @classmethod + def golden_records(cls, topk=32, tk_count=32): + header = { + "t": "hdr", "schema": "coli-ablate/2", "vocab": 40, "topk": topk, + "n_layers": 1, "first_dense": 0, "n_experts": 1, + "config_sha256": cls.CONFIG_SHA256, + "manifest_sha256": cls.MANIFEST_SHA256, + "expected_items": 1, "expected_targets": 1, + } + item1_header = {"t": "ah", "item": 1, "mode": 0, "ncells": 0, + "T": 2, "n_prompt": 1, "cells": []} + row = {"t": "lg", "item": 1, "pos": 0, "gold": 39, + "nll": 0.0, "glogit": 1.0, "molo": 0.5, "mgn": 0.5, + "am": 39, "amlogit": 1.0, "logZ": 1.3, "corr": 1, + "tk": [[i, -0.01 * i] for i in range(tk_count)]} + done = {"t": "done", "manifest_sha256": cls.MANIFEST_SHA256, + "completed_items": 1, "completed_targets": 1} + return [header, item1_header, row, done] + + def test_topk_capped_at_32_for_vocab_above_the_cap_is_accepted(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(self.golden_records()), + self.CONFIG) + result = ABLATE.validate_ablate_evidence( + manifest, evidence, config) + self.assertEqual(result, { + "manifest_sha256": self.MANIFEST_SHA256, + "items": 1, "targets": 1, + }) + + def test_topk_left_uncapped_at_vocab_above_32_is_rejected(self): + # vocab=40 > 32, so header topk must be 32 -- not 40 (== vocab). + records = self.golden_records(topk=40, tk_count=40) + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(records), self.CONFIG) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE.validate_ablate_evidence(manifest, evidence, config) + + +class EvidenceFramingTests(_GoldenFixture): + """`validate_ablate_evidence`'s canonical-LF-JSONL framing check.""" + + def _reject_raw(self, evidence_raw): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + manifest, evidence, config = _write( + root, self.MANIFEST, evidence_raw, self.CONFIG) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE.validate_ablate_evidence(manifest, evidence, config) + + def test_empty_evidence_rejected(self): + self._reject_raw(b"") + + def test_evidence_missing_trailing_newline_rejected(self): + self._reject_raw(_serialize(self.golden_records())[:-1]) + + def test_evidence_with_carriage_return_rejected(self): + self._reject_raw(_serialize(self.golden_records()).replace( + b"\n", b"\r\n", 1)) + + def test_evidence_with_nul_byte_rejected(self): + self._reject_raw(_serialize(self.golden_records()) + b"\0") + + +class HeaderRecordTests(_GoldenFixture): + CASES = ( + ("missing_key", lambda r: r[0].pop("topk")), + ("extra_key", lambda r: r[0].__setitem__("extra", 1)), + ("wrong_t", lambda r: r[0].__setitem__("t", "nope")), + ("wrong_schema", lambda r: r[0].__setitem__( + "schema", "coli-ablate/1")), + ("topk_wrong_type", lambda r: r[0].__setitem__("topk", "4")), + ("config_sha256_wrong_type", lambda r: r[0].__setitem__( + "config_sha256", 1)), + ("config_sha256_not_hex", lambda r: r[0].__setitem__( + "config_sha256", "z" * 64)), + ("manifest_sha256_wrong_type", lambda r: r[0].__setitem__( + "manifest_sha256", None)), + ("manifest_sha256_not_hex", lambda r: r[0].__setitem__( + "manifest_sha256", "0" * 63 + "g")), + ("vocab_out_of_range", lambda r: r[0].__setitem__("vocab", 0)), + ("n_layers_out_of_range", lambda r: r[0].__setitem__( + "n_layers", 0)), + ("first_dense_out_of_range", lambda r: r[0].__setitem__( + "first_dense", 99)), + ("n_experts_out_of_range", lambda r: r[0].__setitem__( + "n_experts", 0)), + ("expected_items_out_of_range", lambda r: r[0].__setitem__( + "expected_items", 0)), + ("expected_targets_out_of_range", lambda r: r[0].__setitem__( + "expected_targets", 0)), + ("topk_not_producer_exact", lambda r: r[0].__setitem__("topk", 3)), + ("vocab_identity_mismatch", lambda r: ( + r[0].__setitem__("vocab", 5), r[0].__setitem__("topk", 5))), + ("n_layers_identity_mismatch", lambda r: r[0].__setitem__( + "n_layers", 2)), + ("first_dense_identity_mismatch", lambda r: r[0].__setitem__( + "first_dense", 0)), + ("n_experts_identity_mismatch", lambda r: r[0].__setitem__( + "n_experts", 6)), + ("config_sha256_identity_mismatch", lambda r: r[0].__setitem__( + "config_sha256", "0" * 64)), + ("manifest_sha256_binding_mismatch", lambda r: r[0].__setitem__( + "manifest_sha256", "1" * 64)), + ("expected_items_binding_mismatch", lambda r: r[0].__setitem__( + "expected_items", 99)), + ("expected_targets_binding_mismatch", lambda r: r[0].__setitem__( + "expected_targets", 99)), + ) + + def test_header_field_checks(self): + for name, mutate in self.CASES: + with self.subTest(name=name): + self._reject(mutate) + + def test_topk_not_producer_exact_even_when_every_row_agrees_with_it(self): + # Isolates the header-level topk==min(32,vocab) check from the + # per-row "len(tk) == header['topk']" shape check: here every row's + # tk list is ALSO shrunk to match the wrong topk, so only the + # header-level producer-exactness check can catch the artifact. + def mutate(records): + records[0]["topk"] = 2 + for record in records: + if record.get("t") == "lg": + record["tk"] = record["tk"][:2] + self._reject(mutate) + + +class ItemHeaderRecordTests(_GoldenFixture): + def test_missing_item_header_rejected(self): + self._reject(lambda r: r.__delitem__(slice(1, None))) + + def test_item_header_not_a_dict_rejected(self): + self._reject(lambda r: r.__setitem__(1, 5)) + + def test_item_header_missing_key_rejected(self): + self._reject(lambda r: r[1].pop("ncells")) + + def test_item_header_extra_key_rejected(self): + self._reject(lambda r: r[1].__setitem__("extra", 1)) + + def test_item_header_field_wrong_type_rejected(self): + self._reject(lambda r: r[1].__setitem__("item", "1")) + + def test_item_header_cells_not_a_list_rejected(self): + self._reject(lambda r: r[4].__setitem__("cells", {})) + + def test_item_header_cell_wrong_shape_rejected(self): + self._reject(lambda r: r[4].__setitem__("cells", [[1, 2]])) + + def test_item_header_cell_element_wrong_type_rejected(self): + self._reject(lambda r: r[4].__setitem__( + "cells", [[1, 2, "x"]])) + + def test_item_header_mismatch_vs_manifest_rejected(self): + self._reject(lambda r: r[1].__setitem__("T", 99)) + + +class TargetRowRecordTests(_GoldenFixture): + def test_missing_target_row_rejected(self): + self._reject(lambda r: r.__delitem__(slice(2, None))) + + def test_row_not_a_dict_rejected(self): + self._reject(lambda r: r.__setitem__(2, 5)) + + def test_row_missing_key_rejected(self): + self._reject(lambda r: r[2].pop("corr")) + + def test_row_extra_key_rejected(self): + self._reject(lambda r: r[2].__setitem__("extra", 1)) + + def test_row_wrong_t_rejected(self): + self._reject(lambda r: r[2].__setitem__("t", "nope")) + + def test_row_item_mismatch_rejected(self): + self._reject(lambda r: r[2].__setitem__("item", 99)) + + def test_row_pos_mismatch_rejected(self): + self._reject(lambda r: r[2].__setitem__("pos", 5)) + + def test_row_gold_mismatch_rejected(self): + self._reject(lambda r: r[2].__setitem__("gold", 0)) + + def test_row_am_wrong_type_rejected(self): + self._reject(lambda r: r[2].__setitem__("am", "1")) + + def test_row_am_out_of_range_rejected(self): + self._reject(lambda r: r[2].__setitem__("am", 4)) + + def test_row_corr_wrong_type_rejected(self): + self._reject(lambda r: r[2].__setitem__("corr", "1")) + + def test_row_corr_out_of_range_rejected(self): + self._reject(lambda r: r[2].__setitem__("corr", 2)) + + NUMERIC_FIELDS = ("nll", "glogit", "molo", "mgn", "amlogit", "logZ") + + def test_row_numeric_field_wrong_type_rejected(self): + for field in self.NUMERIC_FIELDS: + with self.subTest(field=field): + self._reject(lambda r, field=field: r[2].__setitem__( + field, "0")) + + def test_row_tk_not_a_list_rejected(self): + self._reject(lambda r: r[2].__setitem__("tk", 5)) + + def test_row_tk_wrong_length_rejected(self): + self._reject(lambda r: r[2].__setitem__( + "tk", r[2]["tk"][:-1])) + + def test_row_tk_pair_wrong_shape_rejected(self): + self._reject(lambda r: r[2]["tk"].__setitem__(0, [0, 0.1, 9])) + + def test_row_tk_pair_id_wrong_type_rejected(self): + self._reject(lambda r: r[2]["tk"].__setitem__(0, ["0", 0.1])) + + def test_row_tk_pair_id_out_of_range_rejected(self): + self._reject(lambda r: r[2]["tk"].__setitem__(0, [4, 0.1])) + + def test_row_tk_pair_val_wrong_type_rejected(self): + self._reject(lambda r: r[2]["tk"].__setitem__(0, [0, "0.1"])) + + def test_row_tk_duplicate_ids_rejected(self): + self._reject(lambda r: r[2]["tk"].__setitem__(0, list(r[2]["tk"][1]))) + + +class CrossFieldInvariantTests(_GoldenFixture): + """Invariants the engine's per-record emitter, `ablate_logit_record` + (and the row-writer `ablate_logit_line` it calls), guarantees for + every row it emits. + + - `nll >= 0`: `nll` is `-target_lp` (`ablate_logit_record`'s own + `gnll=-target_lp`), and `target_lp` is `delta - logse` where + `delta = lo[target] - r.max <= 0` (target's logit minus the row + max) and `logse = log(sum_i exp(lo[i]-max)) >= log(1) = 0` (the + max's own term contributes exp(0)=1 to that sum) -- the row-level + helpers this emitter builds on (`logprob_row_checked`/ + `logprob_from_row_checked`). So `target_lp <= 0` always, hence + `nll >= 0` always. + - `corr == (am == gold)`: the emitter passes an `argmax==gold` + comparison directly as the `corr` argument to `ablate_logit_line`, + and `am` is that same argmax -- `corr` is never anything but that + comparison's result. + - `amlogit >= glogit`: `amlogit` is the row's own maximum logit and + `glogit` is one particular entry of that same row, so it can + never exceed the row's own maximum. + + Top-k ordering is deliberately NOT enforced: `tk` is unsorted on the + wire by design (`logit_topk_select`, documented as "deliberately not + a lowest-token-id tie rule"). + """ + + def test_nll_negative_rejected(self): + self._reject(lambda r: r[2].__setitem__("nll", -0.1)) + + def test_nll_zero_accepted(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + records = self.golden_records() + records[2]["nll"] = 0.0 + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(records), self.CONFIG) + ABLATE.validate_ablate_evidence(manifest, evidence, config) + + def test_corr_true_when_am_not_gold_rejected(self): + # am=1 == gold=1 in the golden row, so corr must be 1; forcing 0 + # while leaving am/gold untouched breaks the agreement. + self._reject(lambda r: r[2].__setitem__("corr", 0)) + + def test_corr_false_when_am_equals_gold_rejected(self): + # am=0 != gold=2 in row2 (index 3), so corr must be 0; forcing 1 + # breaks the agreement the other way. + self._reject(lambda r: r[3].__setitem__("corr", 1)) + + def test_amlogit_below_glogit_rejected(self): + self._reject(lambda r: r[2].__setitem__("amlogit", 0.5)) + + def test_amlogit_equal_to_glogit_accepted(self): + # row1 already has amlogit == glogit == 1.0 (am == gold there); + # confirm the boundary itself -- not just values strictly above + # it -- is accepted. + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + records = self.golden_records() + self.assertEqual(records[2]["amlogit"], records[2]["glogit"]) + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(records), self.CONFIG) + ABLATE.validate_ablate_evidence(manifest, evidence, config) + + +class TerminalRecordTests(_GoldenFixture): + def test_missing_terminal_record_rejected(self): + self._reject(lambda r: r.__delitem__(slice(6, None))) + + def test_terminal_not_a_dict_rejected(self): + self._reject(lambda r: r.__setitem__(6, 5)) + + def test_terminal_missing_key_rejected(self): + self._reject(lambda r: r[6].pop("completed_items")) + + def test_terminal_extra_key_rejected(self): + self._reject(lambda r: r[6].__setitem__("extra", 1)) + + def test_terminal_completed_items_out_of_range_rejected(self): + self._reject(lambda r: r[6].__setitem__("completed_items", 0)) + + def test_terminal_completed_targets_out_of_range_rejected(self): + self._reject(lambda r: r[6].__setitem__("completed_targets", 0)) + + def test_terminal_wrong_t_rejected(self): + self._reject(lambda r: r[6].__setitem__("t", "nope")) + + def test_terminal_manifest_sha256_mismatch_rejected(self): + self._reject(lambda r: r[6].__setitem__("manifest_sha256", "1" * 64)) + + def test_terminal_completed_items_mismatch_rejected(self): + self._reject(lambda r: r[6].__setitem__("completed_items", 1)) + + def test_terminal_completed_targets_mismatch_rejected(self): + self._reject(lambda r: r[6].__setitem__("completed_targets", 1)) + + def test_trailing_record_after_terminal_rejected(self): + self._reject(lambda r: r.append(dict(r[6]))) + + +class TruncationReplayAndMismatchBiteTests(_GoldenFixture): + """Bite-style table close to the source's own producer-invariant + checks, rebuilt on this module's literal golden fixture instead of + an engine-produced one.""" + + def test_named_mutations_all_refuse(self): + cases = ( + ("missing_done", lambda r: r.__delitem__(6)), + ("truncated_last_row", lambda r: r.__setitem__( + 5, {"t": "lg", "item": 2})), + ("replayed_item_header", lambda r: r.insert(2, dict(r[1]))), + ("duplicate_done", lambda r: r.append(dict(r[6]))), + ("missing_target_row", lambda r: r.__delitem__(3)), + ("wrong_gold_downstream", lambda r: r[3].__setitem__( + "gold", 0)), + ("header_digest_forged", lambda r: r[0].__setitem__( + "manifest_sha256", "2" * 64)), + ) + for name, mutate in cases: + with self.subTest(name=name): + self._reject(mutate) + + +class ManifestProofFramingTests(unittest.TestCase): + """`_manifest_proof`'s non-canonical-text framing checks.""" + + ARGS = (4, 4, 1, 5) # vocab, n_layers, first_dense, n_experts + + def _reject(self, raw): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._manifest_proof(raw, *self.ARGS) + + def test_empty_manifest_rejected(self): + self._reject(b"") + + def _accept(self, raw): + return ABLATE._manifest_proof(raw, *self.ARGS) + + # The engine accepts a manifest saved with CRLF endings and one whose last + # line has no terminator, and digests the canonical form of either. A + # checker that refused them would reject files the producer really ran. + def test_manifest_missing_trailing_newline_accepted(self): + self._accept(b"1 3 1 0 0 0 1 2") + + def test_manifest_with_crlf_endings_accepted(self): + self._accept(b"1 3 1 0 0 0 1 2\r\n") + + def test_all_three_framings_bind_the_same_digest(self): + canonical = self._accept(b"1 3 1 0 0 0 1 2\n")["sha256"] + self.assertEqual(self._accept(b"1 3 1 0 0 0 1 2\r\n")["sha256"], canonical) + self.assertEqual(self._accept(b"1 3 1 0 0 0 1 2")["sha256"], canonical) + + def test_carriage_return_inside_a_record_rejected(self): + self._reject(b"1 3 1 0\r 0 0 1 2\n") + + def test_empty_line_inside_a_manifest_rejected(self): + self._reject(b"1 3 1 0 0 0 1 2\n\n2 2 1 0 0 0 1\n") + + def test_manifest_with_nul_byte_rejected(self): + self._reject(b"1 3 1 0 0 0 1 2\n\0") + + def test_manifest_non_ascii_line_rejected(self): + self._reject("1 3 1 0 0 0 1 é\n".encode("utf-8")) + + def test_manifest_control_byte_breaks_grammar_not_framing(self): + # A vertical tab embedded mid-line is not a canonical digit/space + # byte; it must be caught by the integer-grammar check, not + # silently absorbed as a line boundary bytes.splitlines() would + # not treat it as one either way (see the dedicated probe below). + self._reject(b"1 3\x0b1 0 0 0 1 2\n") + + def test_duplicate_item_id_across_lines_rejected(self): + self._reject(b"1 2 1 0 0 0 1\n1 2 1 0 0 0 1\n") + + def test_records_are_split_on_newline_only(self): + # The module now splits the canonical form on b"\n" rather than + # calling splitlines(), so no other byte can become a record + # boundary. bytes.splitlines() would additionally break on \r, + # which the canonical form no longer contains but which a future + # edit could reintroduce; splitting explicitly removes the + # question. These bytes must therefore stay inside one record and + # be caught by the integer-grammar check. + for value in (0x0B, 0x0C, 0x1C, 0x1D, 0x1E): + with self.subTest(byte=hex(value)): + raw = b"1 3 1 0 0 0 1" + bytes([value]) + b"2\n" + self._reject(raw) + + +class CanonicalManifestDigestTests(unittest.TestCase): + """The canonical rule, pinned against the engine by a literal digest. + + `c/tests/test_ablate_mode.c` asserts the same 64 characters for the same + manifest content. Two implementations that each only agreed with + themselves would both pass their own suites while disagreeing in the + field; a literal known answer on both sides is what rules that out. + """ + + RECORD = b"0 3 2 0 0 1 2 3\n" + KNOWN = "c63a48c375b14ca60f26c7e3c5dd36b5929ffaf669a45511c93deee6e8bbd5ed" + + def test_known_answer_matches_the_engine(self): + digest = hashlib.sha256( + ABLATE.DOMAIN + ABLATE.canonical_manifest_bytes(self.RECORD) + ).hexdigest() + self.assertEqual(digest, self.KNOWN) + + def test_every_accepted_framing_reaches_the_known_answer(self): + for raw in (self.RECORD, b"0 3 2 0 0 1 2 3\r\n", b"0 3 2 0 0 1 2 3"): + with self.subTest(raw=raw): + digest = hashlib.sha256( + ABLATE.DOMAIN + + ABLATE.canonical_manifest_bytes(raw)).hexdigest() + self.assertEqual(digest, self.KNOWN) + + def test_canonicalization_refuses_what_the_engine_refuses(self): + for raw in (b"", b"\n", b"a\n\nb\n", b"a\rb\n", b"a\0b\n"): + with self.subTest(raw=raw): + with self.assertRaises(ABLATE.ManifestFormError): + ABLATE.canonical_manifest_bytes(raw) + + +class ManifestProofFieldBoundaryTests(unittest.TestCase): + """Per-field/per-cell/per-token bounds `_manifest_proof` enforces. + + Table built from `test_manifest_fixed_width_and_topology_c_python_parity`'s + Python-side expectations (each `expected` value here is the same + literal that method asserted, not something this module computed): + that method also cross-checked each case against a C test binary + this module does not build, so it is not this module's oracle to + carry (flagged separately, not absorbed here). + """ + + def test_boundary_table(self): + i32 = ABLATE._INT32_MAX + i64 = ABLATE._INT64_MAX + sixteen = " ".join(f"{layer} 0 -1" for layer in range(1, 17)) + cases = ( + ("baseline_min", b"0 2 1 0 0 0 1\n", 4, 4, 1, 8, True), + ("fewer_than_five_fields", b"1 2\n", 4, 4, 1, 8, False), + ("item_max", f"{i64} 2 1 0 0 0 1\n".encode(), + 4, 4, 1, 8, True), + ("item_max_plus_1", f"{i64 + 1} 2 1 0 0 0 1\n".encode(), + 4, 4, 1, 8, False), + ("item_min_minus_1", b"-1 2 1 0 0 0 1\n", 4, 4, 1, 8, False), + ("T_min", b"7 2 1 0 0 0 1\n", 4, 4, 1, 8, True), + ("T_below_min", b"7 1 1 0 0 0\n", 4, 4, 1, 8, False), + ("T_max_incomplete", f"7 {i32} 1 0 0\n".encode(), + 4, 4, 1, 8, False), + ("T_max_plus_1", f"7 {i32 + 1} 1 0 0\n".encode(), + 4, 4, 1, 8, False), + ("prompt_max_incomplete", f"7 {i32} {i32 - 1} 0 0\n".encode(), + 4, 4, 1, 8, False), + ("prompt_max_plus_1", f"7 {i32} {i32 + 1} 0 0\n".encode(), + 4, 4, 1, 8, False), + ("prompt_min_minus_1", b"7 2 0 0 0 0 1\n", + 4, 4, 1, 8, False), + ("mode_max", b"7 2 1 3 1 1 2 3 0 1\n", + 4, 4, 1, 8, True), + ("mode_max_plus_1", b"7 2 1 4 1 1 2 -1 0 1\n", + 4, 4, 1, 8, False), + ("cells_max", f"7 2 1 1 16 {sixteen} 0 1\n".encode(), + 4, 17, 1, 8, True), + ("cells_max_plus_1", b"7 2 1 1 17 0 1\n", + 4, 17, 1, 8, False), + ("nonbaseline_zero", b"7 2 1 1 0 0 1\n", + 4, 4, 1, 8, False), + ("dense_layer", b"7 2 1 1 1 0 2 -1 0 1\n", + 4, 4, 1, 8, False), + ("layer_min", b"7 2 1 1 1 0 2 -1 0 1\n", + 4, 4, 0, 8, True), + ("layer_upper", b"7 2 1 1 1 3 2 -1 0 1\n", + 4, 4, 1, 8, True), + ("layer_engine_max", b"7 2 1 1 1 127 2 -1 0 1\n", + 4, 128, 0, 8, True), + ("layer_engine_max_plus_1", b"7 2 1 1 1 128 2 -1 0 1\n", + 4, 128, 0, 8, False), + ("source_upper", b"7 2 1 1 1 1 7 -1 0 1\n", + 4, 4, 1, 8, True), + ("source_min", b"7 2 1 1 1 1 0 -1 0 1\n", + 4, 4, 1, 8, True), + ("source_engine_max", b"7 2 1 1 1 1 4095 -1 0 1\n", + 4, 4, 1, 4096, True), + ("source_engine_max_plus_1", b"7 2 1 1 1 1 4096 -1 0 1\n", + 4, 4, 1, 4096, False), + ("target_upper", b"7 2 1 3 1 1 2 7 0 1\n", + 4, 4, 1, 8, True), + ("target_self_swap", b"7 2 1 3 1 1 2 2 0 1\n", + 4, 4, 1, 8, False), + ("target_signed_min", + f"7 2 1 3 1 1 2 {ABLATE._INT32_MIN} 0 1\n".encode(), + 4, 4, 1, 8, False), + ("target_engine_max", b"7 2 1 3 1 1 0 4095 0 1\n", + 4, 4, 1, 4096, True), + ("target_engine_max_plus_1", b"7 2 1 3 1 1 0 4096 0 1\n", + 4, 4, 1, 4096, False), + ("target_max_plus_1", + f"7 2 1 3 1 1 2 {i32 + 1} 0 1\n".encode(), + 4, 4, 1, 8, False), + ("duplicate_source", b"7 2 1 1 2 1 2 -1 1 2 -1 0 1\n", + 4, 4, 1, 8, False), + ("token_min", b"7 2 1 0 0 0 0\n", 1, 4, 1, 8, True), + ("token_upper", b"7 2 1 0 0 0 16777215\n", + 1 << 24, 4, 1, 8, True), + ("token_max_plus_1", b"7 2 1 0 0 0 16777216\n", + 1 << 24, 4, 1, 8, False), + ("vocab_max", b"7 2 1 0 0 0 1\n", + 1 << 24, 4, 1, 8, True), + ("vocab_max_plus_1", b"7 2 1 0 0 0 1\n", + (1 << 24) + 1, 4, 1, 8, False), + ("leading_zero_rejected", b"07 2 1 0 0 0 1\n", + 4, 4, 1, 8, False), + ("plus_sign_rejected", b"+7 2 1 0 0 0 1\n", + 4, 4, 1, 8, False), + ("double_space_rejected", b"7 2 1 0 0 0 1\n", + 4, 4, 1, 8, False), + ("trailing_space_rejected", b"7 2 1 0 0 0 1 \n", + 4, 4, 1, 8, False), + ) + for (name, raw, vocab, layers, first_dense, experts, + expected) in cases: + with self.subTest(name=name): + if expected: + ABLATE._manifest_proof( + raw, vocab, layers, first_dense, experts) + else: + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._manifest_proof( + raw, vocab, layers, first_dense, experts) + + +class ConfigIdentityTests(unittest.TestCase): + CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":5}\n') + + def test_engine_text_size_rejects_non_int_length(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._checked_engine_text_size(True, "config") + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._checked_engine_text_size(1.0, "config") + + def test_engine_byte_limit_is_inclusive_and_enforced_both_sides(self): + engine_limit = 256 << 20 + self.assertEqual(ABLATE._ENGINE_TEXT_MAX_BYTES, engine_limit) + self.assertEqual( + ABLATE._checked_engine_text_size(engine_limit, "config"), + engine_limit) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._checked_engine_text_size(engine_limit + 1, "config") + + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + ablate_config = root / "ablate-config.json" + ablate_config.write_bytes(self.CONFIG) + with mock.patch.object( + ABLATE, "_ENGINE_TEXT_MAX_BYTES", len(self.CONFIG)): + identity = ABLATE._config_identity(ablate_config) + self.assertEqual(identity["vocab"], 4) + self.assertEqual( + identity["config_sha256"], + hashlib.sha256(self.CONFIG).hexdigest()) + ablate_config.write_bytes(self.CONFIG + b" ") + with self.assertRaisesRegex( + ABLATE.AblateEvidenceError, "256 MiB"): + ABLATE._config_identity(ablate_config) + + def _reject(self, raw): + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / "config.json" + path.write_bytes(raw) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._config_identity(path) + + def test_empty_config_rejected(self): + self._reject(b"") + + def test_invalid_json_config_rejected(self): + self._reject(b"{not json}\n") + + def test_non_object_root_rejected(self): + self._reject(b"[1,2,3]\n") + + def test_vocab_size_missing_rejected(self): + self._reject(b'{"num_hidden_layers":4,"first_k_dense_replace":1,' + b'"n_routed_experts":5}\n') + + def test_vocab_size_out_of_range_rejected(self): + self._reject(b'{"vocab_size":0,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":5}\n') + + def test_num_hidden_layers_out_of_range_rejected(self): + self._reject(b'{"vocab_size":4,"num_hidden_layers":0,' + b'"first_k_dense_replace":1,"n_routed_experts":5}\n') + + def test_n_routed_experts_out_of_range_rejected(self): + self._reject(b'{"vocab_size":4,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":0}\n') + + def test_first_k_dense_replace_out_of_range_rejected(self): + self._reject(b'{"vocab_size":4,"num_hidden_layers":4,' + b'"first_k_dense_replace":5,"n_routed_experts":5}\n') + + +class JsonHelperTests(unittest.TestCase): + def test_duplicate_json_key_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b'{"a":1,"a":2}', "record 1") + + def test_nan_constant_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b'{"nll":NaN}', "record 1") + + def test_infinity_constant_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b'{"nll":Infinity}', "record 1") + + def test_negative_infinity_constant_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b'{"nll":-Infinity}', "record 1") + + def test_non_ascii_bytes_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b"\xff", "record 1") + + def test_malformed_json_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b"{not json}", "record 1") + + +class FixedWidthHelperTests(unittest.TestCase): + def test_bounded_int_rejects_bool_disguised_as_int(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._bounded_int(True, "x", 1, 10) + + def test_int64_max_is_the_literal_signed_64_bit_bound(self): + # Pinned by literal, not derived, so a future refactor of the + # module's own (1 << 63) - 1 expression cannot silently drift. + self.assertEqual(ABLATE._INT64_MAX, 9223372036854775807) + + def test_fixed_width_helpers_and_derived_count_boundaries(self): + for value in (ABLATE._INT64_MIN, ABLATE._INT64_MAX): + self.assertEqual(ABLATE._manifest_i64(str(value), 1), value) + for value in (ABLATE._INT64_MIN - 1, ABLATE._INT64_MAX + 1): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._manifest_i64(str(value), 1) + for value in (ABLATE._INT32_MIN, ABLATE._INT32_MAX): + self.assertEqual(ABLATE._bounded_int( + value, "int32", ABLATE._INT32_MIN, ABLATE._INT32_MAX), value) + for value in (ABLATE._INT32_MIN - 1, ABLATE._INT32_MAX + 1): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._bounded_int( + value, "int32", ABLATE._INT32_MIN, ABLATE._INT32_MAX) + self.assertEqual( + ABLATE._count_add(0, ABLATE._INT64_MAX, "count"), + ABLATE._INT64_MAX) + self.assertEqual( + ABLATE._count_add(ABLATE._INT64_MAX, 0, "count"), + ABLATE._INT64_MAX) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._count_add(ABLATE._INT64_MAX, 1, "count") + for label in ("expected_items", "expected_targets", + "completed_items", "completed_targets"): + self.assertEqual( + ABLATE._bounded_int(1, label, 1, ABLATE._INT64_MAX), 1) + self.assertEqual(ABLATE._bounded_int( + ABLATE._INT64_MAX, label, 1, ABLATE._INT64_MAX), + ABLATE._INT64_MAX) + for value in (0, ABLATE._INT64_MAX + 1): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._bounded_int(value, label, 1, ABLATE._INT64_MAX) + + def test_retained_long_max_plus_one_artifact_is_incomplete(self): + manifest_raw = b"9223372036854775808 2 1 0 0 0 1\n" + digest = hashlib.sha256(_DOMAIN + manifest_raw).hexdigest() + self.assertEqual( + digest, + "988a1cf2ddc812f38138e51eecfebb2ba0c9980e31b4c7183396716f114d6538") + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + config_raw = (b'{"vocab_size":2,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":8}\n') + records = ( + {"t": "hdr", "schema": "coli-ablate/2", "vocab": 2, + "topk": 2, "n_layers": 4, "first_dense": 1, + "n_experts": 8, + "config_sha256": hashlib.sha256(config_raw).hexdigest(), + "manifest_sha256": digest, + "expected_items": 1, "expected_targets": 1}, + {"t": "ah", "item": 9223372036854775808, "mode": 0, + "ncells": 0, "T": 2, "n_prompt": 1, "cells": []}, + {"t": "lg", "item": 9223372036854775808, "pos": 0, + "gold": 1, "nll": 0, "glogit": 0, "molo": 0, + "mgn": 0, "am": 1, "amlogit": 0, "logZ": 0, + "corr": 1, "tk": [[1, 0], [0, -1]]}, + {"t": "done", "manifest_sha256": digest, + "completed_items": 1, "completed_targets": 1}, + ) + manifest, evidence, config = _write( + root, manifest_raw, _serialize(records), config_raw) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE.validate_ablate_evidence(manifest, evidence, config) + cli = _run_cli(manifest, evidence, config) + self.assertNotEqual(cli.returncode, 0) + self.assertIn(b"[ablate-evidence] INCOMPLETE:", cli.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tools/check_ablate_evidence.py b/c/tools/check_ablate_evidence.py new file mode 100644 index 000000000..9f6ce0d2f --- /dev/null +++ b/c/tools/check_ablate_evidence.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +"""Validate one complete ABLATE evidence artifact against a config.json.""" + +import argparse +import hashlib +import json +import math +import pathlib +import re +import sys + +class ManifestFormError(ValueError): + """A manifest cannot be reduced to the canonical form the engine binds.""" + + +def canonical_manifest_bytes(raw): + """Return the exact byte stream the engine digests for this manifest. + + The engine reads the file a line at a time, drops the line terminator, + drops one carriage return in front of it if there is one, and digests the + remaining record followed by a single newline. A file saved with CRLF + endings, or without a terminator on its last line, therefore produces the + same digest as the same content saved as plain newline-terminated text -- + which is what a host editor makes it easy to get wrong. + + Everything else is still refused, and refused here rather than later: + an empty file, an empty record, a carriage return inside a record, and an + embedded NUL. Those are not framings of valid content, they are corruption. + """ + if not isinstance(raw, (bytes, bytearray)): + raise ManifestFormError(f"manifest is not bytes: {type(raw).__name__}") + raw = bytes(raw) + if not raw: + raise ManifestFormError("manifest is empty") + if b"\0" in raw: + raise ManifestFormError("manifest contains a NUL byte") + records = raw.split(b"\n") + if records and records[-1] == b"": + records.pop() # the file ended with its terminator + if not records: + raise ManifestFormError("manifest holds no records") + canonical = [] + for number, record in enumerate(records, 1): + if record.endswith(b"\r"): + record = record[:-1] + if not record: + raise ManifestFormError(f"manifest line {number} is empty") + if b"\r" in record: + raise ManifestFormError( + f"manifest line {number} has a carriage return inside it") + canonical.append(record) + return b"\n".join(canonical) + b"\n" + + +DOMAIN = b"coli-ablate-manifest/2\n" +_INT = re.compile(r"(?:0|[1-9][0-9]*|-[1-9][0-9]*)") +_SHA256 = re.compile(r"[0-9a-f]{64}") + +# The wire schema (``coli-ablate/2``) is a fixed-width LP64 domain, chosen so +# a producer and a checker on different host ABIs agree byte-for-byte: every +# manifest integer and every completion counter is signed 64-bit, while a +# value the engine narrows to C ``int`` (a layer, an expert, a token id) is +# signed 32-bit. +_INT32_MIN = -(1 << 31) +_INT32_MAX = (1 << 31) - 1 +_INT64_MIN = -(1 << 63) +_INT64_MAX = (1 << 63) - 1 +_ENGINE_VOCAB_MAX = 1 << 24 +_ENGINE_LAYERS_MAX = 128 +_ENGINE_EXPERTS_MAX = 4096 +_ENGINE_TEXT_MAX_BYTES = 256 << 20 + + +class AblateEvidenceError(ValueError): + """The artifact cannot prove a complete ABLATE denominator.""" + + +def _checked_engine_text_size(length, label): + if type(length) is not int or not 0 <= length <= _ENGINE_TEXT_MAX_BYTES: + raise AblateEvidenceError( + f"{label} exceeds the inclusive 256 MiB engine limit") + return length + + +def _reject_constant(value): + raise AblateEvidenceError(f"non-JSON constant: {value}") + + +def _reject_duplicate_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise AblateEvidenceError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _json_record(raw, label): + try: + return json.loads( + raw.decode("ascii"), parse_constant=_reject_constant, + object_pairs_hook=_reject_duplicate_keys) + except (UnicodeDecodeError, json.JSONDecodeError, + AblateEvidenceError) as exc: + raise AblateEvidenceError(f"invalid {label} JSON: {exc}") from exc + + +def _bounded_config_bytes(config_path): + path = pathlib.Path(config_path) + with path.open("rb") as source: + source.seek(0, 2) + length = source.tell() + _checked_engine_text_size(length, "external config.json") + source.seek(0) + raw = source.read(_ENGINE_TEXT_MAX_BYTES + 1) + _checked_engine_text_size(len(raw), "external config.json") + return raw + + +def _config_identity(config_path): + raw = _bounded_config_bytes(config_path) + if not raw: + raise AblateEvidenceError("external config.json is empty") + try: + config = json.loads( + raw.decode("utf-8"), parse_constant=_reject_constant, + object_pairs_hook=_reject_duplicate_keys) + except (UnicodeDecodeError, json.JSONDecodeError, + AblateEvidenceError) as exc: + raise AblateEvidenceError(f"external config.json is invalid: {exc}") from exc + if not isinstance(config, dict): + raise AblateEvidenceError("external config.json root is not an object") + try: + identity = { + "vocab": _bounded_int( + config["vocab_size"], "config vocab_size", + 1, _ENGINE_VOCAB_MAX), + "n_layers": _bounded_int( + config["num_hidden_layers"], "config num_hidden_layers", + 1, _ENGINE_LAYERS_MAX), + "n_experts": _bounded_int( + config["n_routed_experts"], "config n_routed_experts", + 1, _ENGINE_EXPERTS_MAX), + } + identity["first_dense"] = _bounded_int( + config["first_k_dense_replace"], + "config first_k_dense_replace", 0, identity["n_layers"]) + except (KeyError, AblateEvidenceError) as exc: + raise AblateEvidenceError( + "external config.json topology is incomplete or invalid") from exc + identity["config_sha256"] = hashlib.sha256(raw).hexdigest() + return identity + + +def _bounded_int(value, label, minimum, maximum): + if type(value) is not int or not minimum <= value <= maximum: + raise AblateEvidenceError( + f"{label} is outside {minimum}..{maximum}") + return value + + +def _manifest_i64(text, line_number): + try: + value = int(text) + except ValueError as exc: + raise AblateEvidenceError( + f"manifest line {line_number} integer is too large") from exc + return _bounded_int( + value, f"manifest line {line_number} integer", + _INT64_MIN, _INT64_MAX) + + +def _count_add(current, increment, label): + _bounded_int(current, label, 0, _INT64_MAX) + _bounded_int(increment, f"{label} increment", 0, _INT64_MAX) + if increment > _INT64_MAX - current: + raise AblateEvidenceError(f"{label} exceeds signed 64-bit domain") + return current + increment + + +def _manifest_proof(raw, vocab, n_layers, first_dense, n_experts): + # The engine accepts a manifest saved with either line ending, and with or + # without a terminator on its last line, and digests the canonical form + # rather than the bytes on disk. Reproduce that here from the shared rule, + # or this checker would reject a file the producer ran and would compute a + # different digest for one it accepted. + try: + raw = canonical_manifest_bytes(raw) + except ManifestFormError as exc: + raise AblateEvidenceError(f"manifest is not canonical text: {exc}") from exc + _bounded_int(vocab, "external vocabulary", 1, _ENGINE_VOCAB_MAX) + _bounded_int(n_layers, "external n_layers", 1, _ENGINE_LAYERS_MAX) + _bounded_int(first_dense, "header first_dense", 0, n_layers) + _bounded_int(n_experts, "external n_experts", 1, _ENGINE_EXPERTS_MAX) + items = [] + seen = set() + item_count = targets = 0 + for line_number, raw_line in enumerate(raw[:-1].split(b"\n"), 1): + try: + text = raw_line.decode("ascii") + except UnicodeDecodeError as exc: + raise AblateEvidenceError( + f"manifest line {line_number} is not ASCII") from exc + parts = text.split(" ") + if (not parts or any(not _INT.fullmatch(part) for part in parts) or + " ".join(parts) != text): + raise AblateEvidenceError( + f"manifest line {line_number} is not canonical integer grammar") + values = [_manifest_i64(part, line_number) for part in parts] + if len(values) < 5: + raise AblateEvidenceError(f"manifest line {line_number} is truncated") + item, length, prompt, mode, cells = values[:5] + if (item < 0 or item in seen or + not 2 <= length <= _INT32_MAX or prompt < 1 or + prompt >= length or mode not in range(4) or + cells not in range(17) or + (mode == 0 and cells != 0) or + (mode != 0 and cells == 0)): + raise AblateEvidenceError( + f"manifest line {line_number} has invalid fields/denominator") + expected = 5 + 3 * cells + length + if len(values) != expected: + raise AblateEvidenceError( + f"manifest line {line_number} has invalid fields/denominator") + triples = [] + source_cells = set() + cursor = 5 + for _ in range(cells): + layer, expert, applied = values[cursor:cursor + 3] + cursor += 3 + if (not first_dense <= layer < n_layers or + not 0 <= expert < n_experts or + not _INT32_MIN <= applied <= _INT32_MAX or + (mode == 3 and + (not 0 <= applied < n_experts or applied == expert)) or + (mode != 3 and applied != -1) or + (layer, expert) in source_cells): + raise AblateEvidenceError( + f"manifest line {line_number} has invalid cell") + source_cells.add((layer, expert)) + triples.append([layer, expert, applied]) + tokens = values[cursor:] + if any(token < 0 or token >= vocab for token in tokens): + raise AblateEvidenceError( + f"manifest line {line_number} has out-of-vocabulary token") + seen.add(item) + positions = range(prompt - 1, length - 1) + row_targets = length - prompt + item_count = _count_add(item_count, 1, "manifest item count") + targets = _count_add(targets, row_targets, "manifest target count") + items.append({ + "item": item, "T": length, "n_prompt": prompt, + "mode": mode, "ncells": cells, "cells": triples, + "positions": positions, "tokens": tuple(tokens), + }) + if item_count <= 0 or targets <= 0: + raise AblateEvidenceError("manifest denominator is not positive") + return { + "sha256": hashlib.sha256(DOMAIN + raw).hexdigest(), + "items": tuple(items), "item_count": item_count, "targets": targets, + } + + +def validate_ablate_evidence(manifest_path, evidence_path, config_path): + identity = _config_identity(config_path) + manifest_raw = pathlib.Path(manifest_path).read_bytes() + evidence_raw = pathlib.Path(evidence_path).read_bytes() + if (not evidence_raw or not evidence_raw.endswith(b"\n") or + b"\r" in evidence_raw or b"\0" in evidence_raw): + raise AblateEvidenceError("evidence is not nonempty canonical LF JSONL") + records = [_json_record(line, f"record {index}") + for index, line in enumerate(evidence_raw.splitlines(), 1)] + if not records: + raise AblateEvidenceError("evidence has no header") + header = records[0] + if (not isinstance(header, dict) or set(header) != { + "t", "schema", "vocab", "topk", "manifest_sha256", + "n_layers", "first_dense", "n_experts", + "config_sha256", + "expected_items", "expected_targets"} or + header.get("t") != "hdr" or header.get("schema") != "coli-ablate/2" or + type(header.get("topk")) is not int or + not isinstance(header.get("config_sha256"), str) or + not _SHA256.fullmatch(header["config_sha256"]) or + not isinstance(header.get("manifest_sha256"), str) or + not _SHA256.fullmatch(header["manifest_sha256"])): + raise AblateEvidenceError("header keys or values are not exact") + try: + _bounded_int(header["vocab"], "header vocabulary", + 1, _ENGINE_VOCAB_MAX) + if header["topk"] != min(32, header["vocab"]): + raise AblateEvidenceError("header topk is not producer-exact") + _bounded_int(header["n_layers"], "header n_layers", + 1, _ENGINE_LAYERS_MAX) + _bounded_int(header["first_dense"], "header first_dense", + 0, header["n_layers"]) + _bounded_int(header["n_experts"], "header n_experts", + 1, _ENGINE_EXPERTS_MAX) + _bounded_int(header["expected_items"], "header expected_items", + 1, _INT64_MAX) + _bounded_int(header["expected_targets"], "header expected_targets", + 1, _INT64_MAX) + except (KeyError, AblateEvidenceError) as exc: + raise AblateEvidenceError( + f"header keys or values are not exact: {exc}") from exc + if any(header[key] != identity[key] for key in ( + "vocab", "n_layers", "first_dense", "n_experts", + "config_sha256")): + raise AblateEvidenceError( + "header does not match the external config identity") + proof = _manifest_proof( + manifest_raw, identity["vocab"], identity["n_layers"], + identity["first_dense"], identity["n_experts"]) + if (header["manifest_sha256"] != proof["sha256"] or + header["expected_items"] != proof["item_count"] or + header["expected_targets"] != proof["targets"]): + raise AblateEvidenceError("header does not bind the source manifest proof") + + cursor = 1 + completed_items = completed_targets = 0 + logit_keys = { + "t", "item", "pos", "gold", "nll", "glogit", "molo", "mgn", + "am", "amlogit", "logZ", "corr", "tk", + } + for expected in proof["items"]: + if cursor >= len(records): + raise AblateEvidenceError("missing item header") + item_header = records[cursor] + cursor += 1 + if (not isinstance(item_header, dict) or set(item_header) != { + "t", "item", "mode", "ncells", "T", "n_prompt", "cells"} or + any(type(item_header.get(key)) is not int for key in ( + "item", "mode", "ncells", "T", "n_prompt")) or + not isinstance(item_header.get("cells"), list) or + any(not isinstance(cell, list) or len(cell) != 3 or + any(type(value) is not int for value in cell) + for cell in item_header["cells"]) or + item_header != {key: expected[key] for key in ( + "item", "mode", "ncells", "T", "n_prompt", "cells")} | + {"t": "ah"}): + raise AblateEvidenceError("item header does not match manifest order") + for position in expected["positions"]: + if cursor >= len(records): + raise AblateEvidenceError("missing target row") + row = records[cursor] + cursor += 1 + if (not isinstance(row, dict) or set(row) != logit_keys or + row.get("t") != "lg" or type(row.get("item")) is not int or + row["item"] != expected["item"] or + type(row.get("pos")) is not int or row["pos"] != position or + type(row.get("gold")) is not int or + row["gold"] != expected["tokens"][position + 1] or + type(row.get("am")) is not int or + row["am"] not in range(header["vocab"]) or + type(row.get("corr")) is not int or row["corr"] not in (0, 1)): + raise AblateEvidenceError("target row identity/fields are invalid") + for field in ("nll", "glogit", "molo", "mgn", "amlogit", "logZ"): + if (type(row.get(field)) not in (int, float) or + not math.isfinite(row[field])): + raise AblateEvidenceError(f"target row {field} is nonfinite") + # These three hold for every row the producer can emit: nll is a + # negated log-probability (always <= 0 before negation); corr is + # defined as the argmax/gold agreement, not sampled separately; + # and amlogit is the row's own max logit, so no field can exceed + # it -- least of all the gold token's own logit. + if row["nll"] < 0: + raise AblateEvidenceError("target row nll is negative") + if row["corr"] != int(row["am"] == row["gold"]): + raise AblateEvidenceError( + "target row corr does not match its am/gold agreement") + if row["amlogit"] < row["glogit"]: + raise AblateEvidenceError( + "target row amlogit is below glogit") + topk = row.get("tk") + if (not isinstance(topk, list) or len(topk) != header["topk"] or + any(not isinstance(pair, list) or len(pair) != 2 or + type(pair[0]) is not int or not 0 <= pair[0] < header["vocab"] or + type(pair[1]) not in (int, float) or not math.isfinite(pair[1]) + for pair in topk) or + len({pair[0] for pair in topk}) != len(topk)): + raise AblateEvidenceError("target row top-k is invalid") + completed_targets = _count_add( + completed_targets, 1, "completed target count") + completed_items = _count_add( + completed_items, 1, "completed item count") + + if cursor >= len(records): + raise AblateEvidenceError("missing terminal completion record") + done = records[cursor] + cursor += 1 + if (not isinstance(done, dict) or set(done) != { + "t", "manifest_sha256", "completed_items", "completed_targets"}): + raise AblateEvidenceError("terminal completion proof is invalid") + try: + _bounded_int(done["completed_items"], "done completed_items", + 1, _INT64_MAX) + _bounded_int(done["completed_targets"], "done completed_targets", + 1, _INT64_MAX) + except (KeyError, AblateEvidenceError) as exc: + raise AblateEvidenceError("terminal completion proof is invalid") from exc + if done != {"t": "done", "manifest_sha256": proof["sha256"], + "completed_items": completed_items, + "completed_targets": completed_targets}: + raise AblateEvidenceError("terminal completion proof is invalid") + if cursor != len(records): + raise AblateEvidenceError("records follow terminal completion proof") + if (completed_items != proof["item_count"] or + completed_targets != proof["targets"]): + raise AblateEvidenceError("completed denominator does not match manifest") + return { + "manifest_sha256": proof["sha256"], "items": completed_items, + "targets": completed_targets, + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest") + parser.add_argument("evidence") + parser.add_argument("--config", required=True, + help="independently supplied loaded-model config.json") + args = parser.parse_args(argv) + try: + result = validate_ablate_evidence( + args.manifest, args.evidence, args.config) + except (OSError, AblateEvidenceError) as exc: + print(f"[ablate-evidence] INCOMPLETE: {exc}", file=sys.stderr) + return 1 + print(f"[ablate-evidence] PASS manifest={result['manifest_sha256']} " + f"items={result['items']} targets={result['targets']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 1e9a7abd3bca2fff2118aa703f41213cd4e814fe Mon Sep 17 00:00:00 2001 From: monotophic Date: Thu, 17 Sep 2026 20:07:21 -0400 Subject: [PATCH 2/5] feat(engine): add a logit-row status layer, evidence digests, and a checked ablation scoring mode Add evidence_digest.h (SHA-256 identity of the loaded config.json and of a manifest, bound into the ABLATE_OUT header so evidence can never be checked against the wrong run) and a logit-row status layer in sample.h (LogprobStatus / LogprobRow / logprob_row_checked / logprob_from_row_checked) that classifies a row's reduction -- FINITE, NAN, POS_INF, NEG_INF, ALL_NONFINITE, FINITE_OVERFLOW, or INVALID -- before any log-probability arithmetic is trusted, so a malformed row fails closed with a named status instead of silently producing a NaN a reader cannot distinguish from a real value. The per-target subtraction promotes the logit to double before subtracting the row's own double logZ, so its rounding no longer depends on where in the pipeline the widening happens. Replace ABLATE_SCORE's ablation-sweep mode with a checked version built on that status layer: every manifest field is validated against the loaded config before anything runs, a manifest that does not fit the documented format is refused whole with the offending line named on stderr, and the per-item evidence stream records each target's status alongside its negative log-likelihood, logits, margin, argmax and top-k. The evidence writer takes a plain FILE* (opened O_EXCL, no existing path overwritten) -- it only ever had one backend, so the prior AblateWriter vtable and the AblateModeRunFn dispatch indirection around it are both gone in favor of calling the writer functions directly. Coverage: test_ablate_mode.c drives the mode end-to-end against a small fixture config (vocab=64), including the manifest parser's boundary cases (token id exactly at the vocabulary bound, an ERANGE-triggering field) found by a mutation sweep; test_ablate_mode_gate.py checks the mode is reachable only through its documented environment gate; test_logprob_status.c pins logprob_row_checked's and logprob_from_row_checked's classification of every row shape in isolation. Co-Authored-By: Claude Opus 5 --- c/Makefile | 16 +- c/colibri.c | 845 ++++++++++++++++++++++++++++--- c/evidence_digest.h | 120 +++++ c/sample.h | 124 +++++ c/tests/test_ablate_mode.c | 620 +++++++++++++++++++++++ c/tests/test_ablate_mode_gate.py | 188 +++++++ c/tests/test_logprob_status.c | 339 +++++++++++++ 7 files changed, 2177 insertions(+), 75 deletions(-) create mode 100644 c/evidence_digest.h create mode 100644 c/tests/test_ablate_mode.c create mode 100644 c/tests/test_ablate_mode_gate.py create mode 100644 c/tests/test_logprob_status.c diff --git a/c/Makefile b/c/Makefile index 6cfa5cf7a..08cce0c1f 100644 --- a/c/Makefile +++ b/c/Makefile @@ -733,7 +733,7 @@ $(file >.build-config,$(BUILD_CONFIG)) endif .build-config: ; -colibri$(EXE): colibri.c cli_args.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h route_trace.h omp_tune.h kv_fp8.h kv_tq.h abl.h backend_cuda.h backend_metal.h backend_vulkan.h decode_batch.h edge_adapters.h edge_runtime.h edge_tok_internal.h schema_gbnf.h segment_adapter_internal.h segment_adapters.h segment_runtime.h tier.h $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) .build-config +colibri$(EXE): colibri.c cli_args.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h route_trace.h omp_tune.h kv_fp8.h kv_tq.h abl.h backend_cuda.h backend_metal.h backend_vulkan.h decode_batch.h edge_adapters.h edge_runtime.h edge_tok_internal.h evidence_digest.h schema_gbnf.h segment_adapter_internal.h segment_adapters.h segment_runtime.h tier.h $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) .build-config $(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) -o colibri$(EXE) $(LDFLAGS) # Vulkan backend object (plain C + vulkan headers) and its SPIR-V shaders. @@ -1321,6 +1321,20 @@ tests/test_rope_invfreq$(EXE): tests/test_rope_invfreq.c colibri.c st.h uring.h tests/test_grammar_cache$(EXE): tests/test_grammar_cache.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +# The classified logit-row reduction in sample.h and the digest in +# evidence_digest.h: no model/weights, published SHA-256 vectors, an +# agreement check against the plain logprob_target the sampling path uses +# on rows a float subtraction cannot mis-round, and a double-precision +# check against a second double computation on rows that do. +tests/test_logprob_status$(EXE): tests/test_logprob_status.c colibri.c sample.h evidence_digest.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h kv_persist.h telemetry.h route_trace.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +# The ablation mode's manifest loader, evidence writer and dispatch contract. +# The adapter build bypasses only the model computation, so the parser/writer +# path is exercised for real with no model and no weights. +tests/test_ablate_mode$(EXE): tests/test_ablate_mode.c colibri.c sample.h evidence_digest.h abl.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h kv_persist.h telemetry.h route_trace.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + # Standalone: drives a faithful miniature of moe()'s routing+accumulate and links # the SAME abl.h the engine links -- no model/weights needed (the ablation-logic gate). tests/test_ablate$(EXE): tests/test_ablate.c abl.h diff --git a/c/colibri.c b/c/colibri.c index 24ab3b3ab..19cde7c7b 100644 --- a/c/colibri.c +++ b/c/colibri.c @@ -24,6 +24,8 @@ #include #include #include +#include /* the ablation writer forwards a format list */ +#include /* fixed-width parsing and printing in the ablation manifest */ #include /* thread I/O del PILOTA */ #include /* PIPE ready-flags/job queue + PILOT_REAL cross-layer handshake */ #include /* sched_yield: PIPE spin / PILOT barrier */ @@ -68,6 +70,7 @@ #include "tier.h" #include "grammar.h" /* metodo F: draft grammaticali (#48) */ #include "abl.h" /* per-expert causal-ablation harness — inert unless g_abl.mode set (ABLATE_SCORE=) */ +#include "evidence_digest.h" /* SHA-256 over the bytes an evidence mode consumed */ #include "schema_gbnf.h" /* SCHEMA=: JSON-Schema -> GBNF for method F */ #include "decode_batch.h" #include "route_trace.h" /* ROUTE_TRACE + .coli_usage, engine-agnostic (#700) */ @@ -171,6 +174,7 @@ typedef struct { int index_topk, index_nh, index_hd; /* DSA lightning indexer */ int8_t idx_type[128]; /* per layer: 1=full (calcola), 0=shared (riusa) */ float eps, theta, attn_scale, routed_scale; + char config_sha256[65]; /* digest of the loaded config.json bytes */ } Cfg; /* tensore [O,I] in uno di tre formati: @@ -1652,7 +1656,7 @@ static char* cfg_slurp(const char *path){ if((long)got!=n){ free(b); return NULL; } b[got]=0; return b; } -static jval* cfg_root(const char *snap, char **arena){ +static jval *cfg_root(const char *snap, char **arena, char config_sha256[65]){ char p[2048]; snprintf(p,sizeof(p),"%s/config.json",snap); FILE *f=fopen(p,"rb"); if(!f){perror(p);exit(1);} fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); @@ -1663,11 +1667,17 @@ static jval* cfg_root(const char *snap, char **arena){ char *b=malloc((size_t)n+1); if(!b){ fprintf(stderr,"OOM reading %s (%ld bytes)\n",p,n); exit(1); } size_t got=fread(b,1,(size_t)n,f); b[got]=0; fclose(f); if((long)got!=n) fprintf(stderr,"warning: short read on %s (%ld of %ld)\n",p,(long)got,n); + /* Bind the exact bytes that were loaded, for callers that record them. */ + if(config_sha256) evidence_sha256_hex(b,got,config_sha256); return json_parse(b,arena); } static int gi(jval*r,const char*k){ jval*v=json_get(r,k); return v?(int)v->num:0; } static void load_cfg(Cfg *c, const char *snap){ - char *ar=NULL; jval *r=cfg_root(snap,&ar); + /* The digest costs a hash pass over config.json on every load; only the + * ABLATE_SCORE evidence path needs to name its config input, so it is the + * only caller that pays for it. c->config_sha256 stays unset otherwise. */ + char *sha_out = getenv("ABLATE_SCORE") ? c->config_sha256 : NULL; + char *ar=NULL; jval *r=cfg_root(snap,&ar,sha_out); c->hidden=gi(r,"hidden_size"); c->n_layers=gi(r,"num_hidden_layers"); c->n_heads=gi(r,"num_attention_heads"); c->n_experts=gi(r,"n_routed_experts"); c->topk=gi(r,"num_experts_per_tok"); c->moe_inter=gi(r,"moe_intermediate_size"); @@ -7860,7 +7870,7 @@ static void run_score(Model *m, const char *snap, const char *path){ * prefissato (eval_glm.py post-#194) passa INTATTO. SCORE_PREFIX=0 -> comportamento nudo. */ int pfx[2]={-1,-1}, pfx_on=0; if(!getenv("SCORE_PREFIX")||atoi(getenv("SCORE_PREFIX"))){ - char *ar=NULL; jval *r=cfg_root(snap,&ar); + char *ar=NULL; jval *r=cfg_root(snap,&ar,NULL); /* the prefix probe needs no digest */ jval *mt=json_get(r,"model_type"); if(mt_is_glm(mt?mt->str:NULL)){ char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap); @@ -7922,88 +7932,756 @@ static void run_score(Model *m, const char *snap, const char *path){ * ncells = # ablated cells (0 for baseline) * L E A = layer, expert, swap-target (A=-1 unless mode 3), one triple per cell * t_* = token ids (host pre-tokenised, prefix included if the model needs it) - * abl_reset() before each item makes the ablation PER-ITEM (an item's spec can + * The whole manifest is strictly validated and SHA-256 bound before output; + * empty/malformed/duplicate-ID or zero-target inputs fail closed. abl_reset() + * before each item makes the ablation PER-ITEM (an item's spec can * never leak into the next). NLL/margin/correctness are exact; top-K is for a * paired approximate next-token KL on the host. */ #define ABL_LOGIT_TOPK 32 -static void run_ablate_score(Model *m, const char *path){ +/* Upper bound on one manifest item's declared token count. A teacher-forced + * ablation item is a prompt plus a short continuation; a million tokens is + * orders of magnitude above anything a study uses, and it keeps a hostile or + * corrupt manifest from asking the loader for an arbitrary allocation. */ +#define ABLATE_MAX_ITEM_TOKENS (1<<20) + +static int logit_topk_count(int V, int requested){ + if(V<=0 || requested<=0) return 0; + int k=requested; + if(k>ABL_LOGIT_TOPK) k=ABL_LOGIT_TOPK; + if(k>V) k=V; + return k; +} + +static void logprob_refusal(const char *surface, unsigned long long owner, + int64_t position, const char *field, + LogprobStatus status){ + fprintf(stderr, + "[numeric] INCOMPLETE: surface=%s owner=%llu position=%" PRId64 + " field=%s class=%s\n", + surface,owner,position,field,logprob_status_name(status)); +} + +/* For finite rows, select real vocabulary entries without a numeric sentinel. + * Filling empty slots in encounter order and replacing only on strict greater- + * than preserves the historical first-minimum-SLOT behavior and unsorted slot + * order. This is deliberately not a lowest-token-id tie rule: [0,0,1] at + * k=2 emits token ids [2,1]. + * + * Exceptional rows are rejected by the classified numeric-status layer before + * this finite-only selector is reached. */ +static int logit_topk_select(const float *lo, int V, int requested, + int *tk_id, float *tk_val, + int *status_out){ + if(status_out) *status_out=0; + if(!lo || !tk_id || !tk_val || V<=0) return 0; + int k=logit_topk_count(V,requested); + if(k==0) return 0; + int finite=1; + for(int i=0;itk_val[mn])) continue; + slot=mn; + } + tk_id[slot]=i; + tk_val[slot]=lo[i]; + } + return k; +} + +typedef struct { + char digest[65]; + char config_sha256[65]; + int64_t items; + int64_t targets; + int maxT; + int n_layers; + int first_dense; + int n_experts; +} AblateManifestInfo; + +/* Parse one canonical signed-64 decimal, independent of host ``long`` width. + * The ABLATE manifest is intentionally narrower than generic strtoimax input: + * no plus, leading zero, -0, tabs, CR, or trailing whitespace. A canonical + * byte stream then has one stable digest. */ +static int ablate_manifest_i64(const char **cursor, const char *end, + int64_t *out){ + const char *p=*cursor; + if(p>=end) return -1; + int negative=(*p=='-'); + if(negative && ++p>=end) return -1; + if(*p<'0' || *p>'9') return -1; + if(*p=='0' && p+1='0' && p[1]<='9') return -1; + if(negative && *p=='0') return -1; + errno=0; char *after=NULL; + intmax_t value=strtoimax(*cursor,&after,10); + if(errno==ERANGE || after==*cursor || after>end || + valueINT64_MAX) return -1; + *cursor=after; *out=(int64_t)value; + return 0; +} + +static int ablate_manifest_space(const char **cursor, const char *end){ + if(*cursor>=end || **cursor!=' ') return -1; + (*cursor)++; + return 0; +} + +/* One manifest item id together with the line it was read from. The duplicate + * check sorts these, so the line has to travel with the id: reporting the last + * line of the file instead would point a reader at the wrong record. */ +typedef struct { + int64_t item; + int64_t line; +} AblateItemRef; + +static int ablate_item_ref_compare(const void *av, const void *bv){ + const AblateItemRef *a=(const AblateItemRef *)av, *b=(const AblateItemRef *)bv; + if(a->item!=b->item) return (a->item>b->item)-(a->itemitem); + return (a->line>b->line)-(a->lineline); +} + +static int ablate_count_add(int64_t *total, int64_t increment){ + if(!total || *total<0 || increment<0 || *total>INT64_MAX-increment) + return -1; + *total+=increment; + return 0; +} + +typedef struct { + int64_t item; + int T; + int n_prompt; + int mode; + int ncells; + int layers[ABL_MAX_CELLS]; + int experts[ABL_MAX_CELLS]; + int applied[ABL_MAX_CELLS]; + int *tokens; +} AblateManifestItem; + +typedef struct { + AblateManifestInfo info; + AblateManifestItem *items; + size_t count; +} AblateManifest; + +static int evidence_sha256_text_valid(const char *text){ + if(!text) return 0; + for(int i=0;i<64;i++) + if(!((text[i]>='0' && text[i]<='9') || + (text[i]>='a' && text[i]<='f'))) return 0; + return text[64]==0; +} + +static void ablate_manifest_free(AblateManifest *manifest){ + if(!manifest) return; + for(size_t i=0;icount;i++) free(manifest->items[i].tokens); + free(manifest->items); + memset(manifest,0,sizeof(*manifest)); +} + +/* Parse and freeze the complete positive denominator before any output or + * model work. The digest and the owned parsed rows come from these same bytes; + * execution never rereads the caller-owned FILE. */ +/* Name the precondition that failed, or NULL when they all hold. A refusal + * with no reason on stderr is indistinguishable from a crash to whoever runs + * the mode, so every caller of this prints what it returns. */ +static const char *ablate_precondition_reason(FILE *f, const Cfg *c){ + if(!f) return "no manifest stream"; + if(!c) return "no loaded model configuration"; + if(c->vocab<=0 || c->vocab>(1<<24)) return "vocabulary size out of range (1..16777216)"; + if(c->n_layers<=0 || c->n_layers>128) return "layer count out of range (1..128)"; + if(c->first_dense<0 || c->first_dense>c->n_layers) + return "first routed layer out of range (0..layer count)"; + if(c->n_experts<=0 || c->n_experts>4096) return "expert count out of range (1..4096)"; + if(!evidence_sha256_text_valid(c->config_sha256)) + return "the loaded config.json was not digested, so evidence could not name it"; + return NULL; +} + +static int ablate_manifest_load(FILE *f, const Cfg *c, + AblateManifest *manifest){ + if(!manifest){ + fprintf(stderr,"[ablate] INCOMPLETE: no manifest destination\n"); + return 1; + } + memset(manifest,0,sizeof(*manifest)); + const char *reason=ablate_precondition_reason(f,c); + if(reason){ + fprintf(stderr,"[ablate] INCOMPLETE: %s\n",reason); + return 1; + } + int V=c->vocab; + clearerr(f); rewind(f); + EvidenceSha256 hash; + evidence_sha256_init(&hash); + static const char domain[]="coli-ablate-manifest/2\n"; + evidence_sha256_update(&hash,domain,sizeof(domain)-1); + AblateItemRef *item_ids=NULL; + size_t item_cap=0, item_count=0, row_cap=0; + int64_t target_count=0; + int maxT=1, rc=0; + char *line=NULL; size_t cap=0; ssize_t length; int64_t line_no=0, bad_line=0; + int named=0; /* a specific reason was already printed */ + while((length=getline(&line,&cap,f))>0){ + line_no++; + /* Accept the two framings a host editor produces without meaning to: + * a CRLF line ending, and a last line with no terminator at all. The + * record is normalised to its bare text here and the digest below is + * taken over the normalised bytes, so the same manifest content always + * produces the same digest however it was saved. Anything else -- a + * stray carriage return inside a record, an embedded NUL, an empty + * line -- is still refused. */ + size_t len=(size_t)length; + if(line[len-1]=='\n') len--; + if(len && line[len-1]=='\r') len--; + if(len==0 || memchr(line,'\r',len) || memchr(line,'\0',len)){ + rc=1; break; + } + const char *record_end=line+len, *p=line; + int64_t item,T,np,mode,nc; + if(ablate_manifest_i64(&p,record_end,&item)<0 || + ablate_manifest_space(&p,record_end)<0 || + ablate_manifest_i64(&p,record_end,&T)<0 || + ablate_manifest_space(&p,record_end)<0 || + ablate_manifest_i64(&p,record_end,&np)<0 || + ablate_manifest_space(&p,record_end)<0 || + ablate_manifest_i64(&p,record_end,&mode)<0 || + ablate_manifest_space(&p,record_end)<0 || + ablate_manifest_i64(&p,record_end,&nc)<0){ + rc=1; break; + } + /* A manifest is host input: an item may not ask for an unbounded + * allocation by declaring an enormous length. The cap is far above any + * teacher-forced item a study runs -- the engine also allocates + * maxT * hidden floats for the prefill -- and it bounds the token array + * this loop is about to allocate to a few megabytes. */ + if(T>ABLATE_MAX_ITEM_TOKENS){ + fprintf(stderr,"[ablate] INCOMPLETE: line %" PRId64 " declares %" PRId64 + " tokens, above the %d-token limit for one item\n", + line_no,T,ABLATE_MAX_ITEM_TOKENS); + named=1; rc=1; bad_line=line_no; break; + } + if(item<0 || T<2 || T>INT_MAX || np<1 || np>=T || + mode<0 || mode>3 || nc<0 || nc>ABL_MAX_CELLS || + (mode==0 ? nc!=0 : nc==0)){ + rc=1; break; + } + AblateManifestItem row={0}; + row.item=item; row.T=(int)T; row.n_prompt=(int)np; + row.mode=(int)mode; row.ncells=(int)nc; + for(int64_t i=0;ifirst_dense || L>=c->n_layers || + E<0 || E>=c->n_experts || + AINT_MAX || + (mode==3 ? (A<0 || A>=c->n_experts || A==E) : A!=-1)){ + rc=1; break; + } + for(int64_t j=0;j(uintmax_t)(SIZE_MAX/sizeof(*row.tokens))) rc=1; + if(!rc){ row.tokens=malloc((size_t)T*sizeof(*row.tokens)); if(!row.tokens) rc=1; } + for(int64_t i=0;i=V) rc=1; + else row.tokens[i]=(int)token; + } + if(rc || p!=record_end) { free(row.tokens); rc=1; break; } + if(item_count==item_cap){ + size_t next=item_cap ? item_cap*2 : 32; + if(nextSIZE_MAX/sizeof(*item_ids)){ + free(row.tokens); rc=1; break; + } + AblateItemRef *grown=realloc(item_ids,next*sizeof(*item_ids)); + if(!grown){ free(row.tokens); rc=1; break; } + item_ids=grown; item_cap=next; + } + if(item_count==row_cap){ + size_t next=row_cap ? row_cap*2 : 32; + if(nextSIZE_MAX/sizeof(*manifest->items)){ + free(row.tokens); rc=1; break; + } + AblateManifestItem *grown=realloc( + manifest->items,next*sizeof(*manifest->items)); + if(!grown){ free(row.tokens); rc=1; break; } + manifest->items=grown; row_cap=next; + } + item_ids[item_count]=(AblateItemRef){item,line_no}; + item_count++; + manifest->items[manifest->count++]=row; + int64_t row_targets=T-np; + if(ablate_count_add(&target_count,row_targets)<0){ rc=1; break; } + if(T>maxT) maxT=(int)T; + evidence_sha256_update(&hash,line,len); + evidence_sha256_update(&hash,"\n",1); + } + if(ferror(f) || item_count==0 || target_count<=0 || + (uintmax_t)item_count>(uintmax_t)INT64_MAX) rc=1; + if(!rc){ + qsort(item_ids,item_count,sizeof(*item_ids),ablate_item_ref_compare); + for(size_t i=1;iinfo.digest[2*i]=hex[raw[i]>>4]; + manifest->info.digest[2*i+1]=hex[raw[i]&15]; + } + manifest->info.digest[64]=0; + memcpy(manifest->info.config_sha256,c->config_sha256,65); + manifest->info.items=(int64_t)item_count; + manifest->info.targets=target_count; + manifest->info.maxT=maxT; + manifest->info.n_layers=c->n_layers; + manifest->info.first_dense=c->first_dense; + manifest->info.n_experts=c->n_experts; + } + free(line); free(item_ids); clearerr(f); rewind(f); + if(rc) ablate_manifest_free(manifest); + return rc; +} + +/* The writer used to be a small vtable (ctx + vprintf/puts/flush/close + * function pointers) so a test double could stand in for a real file. It + * only ever had one implementation -- a plain FILE* -- so the indirection + * bought nothing; a test that wants an isolated destination opens a real + * tmpfile and passes it in like any other caller. */ +static int ablate_writer_printf(FILE *f, const char *fmt, ...){ + va_list ap; va_start(ap,fmt); + int n=vfprintf(f,fmt,ap); + va_end(ap); + return n; +} + +static int ablate_writer_puts(FILE *f, const char *text){ + return fputs(text,f)==EOF ? -1 : 0; +} + +static FILE *ablate_writer_open(const char *path){ + /* Evidence output is new-file-only. O_EXCL makes the path-name decision + * atomic: an exact/normalised source path, hard link, symbolic link, or any + * other existing destination is refused before a byte can be truncated. + * The caller keeps the already-open manifest descriptor live through this + * operation, so a newly created inode cannot identify that source. */ + int fd=open(path,O_WRONLY|O_CREAT|O_EXCL|COMPAT_O_BINARY,0666); + if(fd<0) return NULL; + FILE *f=fdopen(fd,"wb"); + if(!f){ + int saved=errno; + close(fd); + errno=saved; + return NULL; + } + return f; +} + +/* Always attempt close even if flush failed, and preserve either failure. */ +static int ablate_writer_finish(FILE *f){ + if(!f) return -1; + int bad=(fflush(f)!=0); + if(fclose(f)!=0) bad=1; + return bad ? -1 : 0; +} + +typedef int (*AblateOutputOpenFn)(void *ctx, FILE **w, + const char *path); +typedef int (*AblateOutputBodyFn)(void *ctx, FILE *of); +typedef struct { + void *ctx; + AblateOutputOpenFn open_fn; + AblateOutputBodyFn body_fn; + const AblateManifestInfo *manifest; + int64_t *completed_items; + int64_t *completed_targets; +} AblateOutputRun; + +static int ablate_file_open_run(void *ctx, FILE **w, + const char *path){ + (void)ctx; + *w=ablate_writer_open(path); + return *w ? 0 : -1; +} + +static int ablate_header_line(FILE *of, int V, + const AblateManifestInfo *manifest){ + if(!manifest) return -1; + int topk=logit_topk_count(V,ABL_LOGIT_TOPK); + return ablate_writer_printf(of, + "{\"t\":\"hdr\",\"schema\":\"coli-ablate/2\",\"vocab\":%d,\"topk\":%d," + "\"n_layers\":%d,\"first_dense\":%d,\"n_experts\":%d," + "\"config_sha256\":\"%s\"," + "\"manifest_sha256\":\"%s\",\"expected_items\":%" PRId64 + ",\"expected_targets\":%" PRId64 "}\n", + V,topk,manifest->n_layers,manifest->first_dense, + manifest->n_experts,manifest->config_sha256,manifest->digest, + manifest->items,manifest->targets)<0 ? -1 : 0; +} + +static int ablate_done_line(FILE *of, + const AblateManifestInfo *manifest, + int64_t completed_items, + int64_t completed_targets){ + if(!manifest) return -1; + return ablate_writer_printf(of, + "{\"t\":\"done\",\"manifest_sha256\":\"%s\"," + "\"completed_items\":%" PRId64 + ",\"completed_targets\":%" PRId64 "}\n", + manifest->digest,completed_items,completed_targets)<0 ? -1 : 0; +} + +/* Requested-output orchestration shared by the real ABLATE driver and its + * model-free caller-chain gate. A missing ABLATE_OUT remains optional; once + * a path is requested, every open/header/body/finalize failure is a failed + * mode result. Finalize always attempts close after flush. */ +static int ablate_output_run(AblateOutputRun *run, const char *outp, int V){ + if(!run || !run->body_fn || !run->manifest || + !run->completed_items || !run->completed_targets || + run->manifest->items<=0 || run->manifest->targets<=0){ + fprintf(stderr,"[ablate] INCOMPLETE: nothing to run " + "(the manifest produced no items or targets)\n"); + return 1; + } + *run->completed_items=0; *run->completed_targets=0; + FILE *writer=NULL, *of=NULL; + int rc=0; + if(outp){ + if(!run->open_fn || run->open_fn(run->ctx,&writer,outp)<0){ + fprintf(stderr, + "[ablate] INCOMPLETE: ABLATE_OUT must be a new file: %s\n", + outp); + return 1; + } + of=writer; + if(ablate_header_line(of,V,run->manifest)<0){ + fprintf(stderr,"[ablate] cannot write requested ABLATE_OUT=%s\n",outp); + rc=1; + } + } + if(!rc && run->body_fn(run->ctx,of)!=0) rc=1; + if(!rc && (*run->completed_items!=run->manifest->items || + *run->completed_targets!=run->manifest->targets)){ + fprintf(stderr,"[ablate] completion denominator mismatch: " + "%" PRId64 "/%" PRId64 " items, " + "%" PRId64 "/%" PRId64 " targets\n", + *run->completed_items,run->manifest->items, + *run->completed_targets,run->manifest->targets); + rc=1; + } + if(!rc && of && ablate_done_line(of,run->manifest, + *run->completed_items,*run->completed_targets)<0) rc=1; + if(of && ablate_writer_finish(of)<0) rc=1; + if(rc && outp) + fprintf(stderr,"[ablate] requested ABLATE_OUT=%s is incomplete\n",outp); + return rc; +} + +static int ablate_item_line(FILE *of, int64_t item, int64_t mode, + int64_t nc, int64_t T, int64_t np, + const int *Ls, const int *Es, + const int *As){ + if(ablate_writer_printf(of, + "{\"t\":\"ah\",\"item\":%" PRId64 + ",\"mode\":%" PRId64 ",\"ncells\":%" PRId64 + ",\"T\":%" PRId64 ",\"n_prompt\":%" PRId64 + ",\"cells\":[", + item,mode,nc,T,np)<0) return -1; + for(int i=0;i<(int)nc;i++) + if(ablate_writer_printf(of,"%s[%d,%d,%d]",i?",":"",Ls[i],Es[i],As[i])<0) + return -1; + return ablate_writer_puts(of,"]}\n"); +} + +/* Keep the ABLATE numeric schema in one callable formatter so the exact + * production emission can be round-trip tested without loading a model. + * Raw diagnostic logits retain their existing compact precision; nll, logZ, + * and the requested top-k numeric fields are the precision-corrected sites. */ +static int ablate_logit_line(FILE *of, int64_t item, int64_t pos, + int gold, + double nll, float glogit, float molo, float mgn, + int am, float amlogit, double logZ, int corr, + int topk, const int *tk_id, const float *tk_val){ + int n=ablate_writer_printf(of,"{\"t\":\"lg\",\"item\":%" PRId64 + ",\"pos\":%" PRId64 ",\"gold\":%d,\"nll\":%.17g," + "\"glogit\":%.6g,\"molo\":%.6g,\"mgn\":%.6g,\"am\":%d,\"amlogit\":%.6g," + "\"logZ\":%.17g,\"corr\":%d,\"tk\":[", + item,pos,gold,nll,(double)glogit,(double)molo,(double)mgn, + am,(double)amlogit,logZ,corr); + if(n<0) return -1; + for(int k=0;k=V){ + fprintf(stderr,"[ablate] INCOMPLETE: item %" PRId64 " position %" PRId64 + " has no readable logit row\n",item,pos); + return -1; + } + LogprobRow lpr; + LogprobStatus status=logprob_row_checked(lo,V,&lpr); + if(status!=LOGPROB_FINITE){ + logprob_refusal("ABLATE",(unsigned long long)item,pos,"row",status); + return -1; + } + float mx=lpr.max; + double target_lp=0; + status=logprob_from_row_checked(lo,gold,&lpr,&target_lp); + if(status!=LOGPROB_FINITE){ + logprob_refusal("ABLATE",(unsigned long long)item,pos,"target",status); + return -1; + } + /* The classified read above decides whether this position is reportable at + * all. The reported value itself is the subtraction done wholly in double, + * which is what this mode has always written: taking the difference in + * float first would round away up to an ulp of the largest logit, and on a + * widely spread row that is a visible amount. */ + double gnll=lpr.logZ-(double)lo[gold]; + if(!isfinite(gnll)){ + logprob_refusal("ABLATE",(unsigned long long)item,pos,"nll", + LOGPROB_FINITE_OVERFLOW); + return -1; + } + /* V==1 has no competitor and retains the historical finite sentinel. + * For every real competitor set, seed from an actual non-gold logit so a + * finite value below -1e30 cannot be hidden and margin overflow is refused. */ + float molo=-1e30f; + if(V>=2){ + int competitor=gold==0?1:0; + molo=lo[competitor]; + for(int i=0;imolo) molo=lo[i]; + } + float mgn=lo[gold]-molo; + if(!isfinite(mgn)){ + logprob_refusal("ABLATE",(unsigned long long)item,pos,"margin", + LOGPROB_FINITE_OVERFLOW); + return -1; + } + int tk_id[ABL_LOGIT_TOPK]; float tk_val[ABL_LOGIT_TOPK]; + int topk_status=0; + int topk=logit_topk_select(lo,V,ABL_LOGIT_TOPK,tk_id,tk_val,&topk_status); + if(topk!=logit_topk_count(V,ABL_LOGIT_TOPK)){ + logprob_refusal("ABLATE",(unsigned long long)item,pos,"topk", + topk_status ? (LogprobStatus)topk_status : LOGPROB_INVALID); + return -1; + } + return ablate_logit_line(of,item,pos,gold,gnll,lo[gold],molo,mgn, + lpr.argmax,mx,lpr.logZ,lpr.argmax==gold, + topk,tk_id,tk_val); +} + +typedef int (*AblateRowOutputFn)(void *ctx, FILE *of, int64_t pos); + +/* One item's actual row-caller and per-item flush chain. With no requested + * output it performs no logprob work, matching the historical optional path. */ +static int ablate_item_output(FILE *of, int64_t first, int64_t end, + AblateRowOutputFn row_fn, void *ctx){ + if(!of) return 0; + if(!row_fn){ + fprintf(stderr,"[ablate] INCOMPLETE: no row reader for the requested output\n"); + return 1; + } + for(int64_t pos=first;posm; Cfg *c=&m->c; + if(ABLATE_MODEL_COMPUTE_ENABLED){ + rmsnorm(rows->row, rows->x+(int64_t)pos*rows->D, + m->final_norm, rows->D, c->eps); + matmul_qt(rows->lo, rows->row, &m->lm_head, 1); + } + return ablate_logit_record(of,rows->item,pos,rows->ids[pos+1], + rows->lo,rows->V); +} + +typedef struct { + Model *m; + const AblateManifest *manifest; + int64_t completed_items; + int64_t completed_targets; +} AblateModelRun; + +static int ablate_model_output_body(void *ctx, FILE *of){ + AblateModelRun *run=(AblateModelRun *)ctx; + Model *m=run->m; + if(!run->manifest || run->manifest->count==0){ + fprintf(stderr,"[ablate] INCOMPLETE: the manifest holds no items\n"); + return 1; + } Cfg *c=&m->c; int D=c->hidden, V=c->vocab; - FILE *f=fopen(path,"rb"); if(!f){perror(path);exit(1);} - const char *outp=getenv("ABLATE_OUT"); - FILE *of = outp ? fopen(outp,"wb") : NULL; - if(outp && !of){ fprintf(stderr,"[ablate] cannot open ABLATE_OUT=%s\n",outp); } - if(of) fprintf(of,"{\"t\":\"hdr\",\"schema\":\"coli-ablate/1\",\"vocab\":%d,\"topk\":%d}\n",V,ABL_LOGIT_TOPK); - int maxT=1; { char *ln=NULL; size_t cp=0; - while(getline(&ln,&cp,f)>0){ long id,T; char *e; - id=strtol(ln,&e,10); if(e==ln) continue; T=strtol(e,&e,10); - if(T>maxT) maxT=(int)T; (void)id; } - free(ln); } - kv_alloc(m,maxT); + int maxT=run->manifest->info.maxT; + run->completed_items=0; run->completed_targets=0; + if(ABLATE_MODEL_COMPUTE_ENABLED) kv_alloc(m,maxT); float *x=falloc((int64_t)maxT*D), *lo=falloc(V), *row=falloc(D); - int *ids=malloc((size_t)maxT*sizeof(int)); - int tk_id[ABL_LOGIT_TOPK]; float tk_val[ABL_LOGIT_TOPK]; - rewind(f); char *ln=NULL; size_t cp=0; int nreq=0; double t0=now_s(); - while(getline(&ln,&cp,f)>0){ - char *p=ln, *e; - long item=strtol(p,&e,10); if(e==p) continue; p=e; /* blank line */ - long T=strtol(p,&e,10); if(e==p){ fprintf(stderr,"[ablate] bad T\n"); continue; } p=e; - long np=strtol(p,&e,10); if(e==p){ fprintf(stderr,"[ablate] bad n_prompt\n"); continue; } p=e; - long mode=strtol(p,&e,10); if(e==p){ fprintf(stderr,"[ablate] bad mode\n"); continue; } p=e; - long nc=strtol(p,&e,10); if(e==p){ fprintf(stderr,"[ablate] bad ncells\n"); continue; } p=e; - int Ls[ABL_MAX_CELLS], Es[ABL_MAX_CELLS], As[ABL_MAX_CELLS]; - int bad=0; long ncc = nc<0?0:(nc>ABL_MAX_CELLS?ABL_MAX_CELLS:nc); - for(long i=0;imaxT || np<0 || np>T || mode<0 || mode>3); - for(long i=0;i=V) bad=1; else ids[i]=(int)v; } - if(bad){ fprintf(stderr,"[ablate] ERR item %ld (bad field/token)\n",item); continue; } +#ifdef COLI_TEST_ABLATE_ADAPTERS + if(g_ablate_adapter_test.bypass_model_compute){ + if(g_ablate_adapter_test.forced_row) + for(int i=0;imanifest->count;index++){ + const AblateManifestItem *item=&run->manifest->items[index]; +#ifdef COLI_TEST_ABLATE_ADAPTERS + g_ablate_adapter_test.observed_first_token=item->tokens[0]; +#endif /* PER-ITEM RESET then configure this item's ablation (no cross-item leak). */ abl_reset(&g_abl); - abl_set_item(&g_abl, (int)mode, Ls, Es, (mode==3?As:NULL), (int)ncc); - if(of){ - fprintf(of,"{\"t\":\"ah\",\"item\":%ld,\"mode\":%ld,\"ncells\":%ld,\"T\":%ld,\"n_prompt\":%ld,\"cells\":[", - item, mode, ncc, T, np); - for(int i=0;i<(int)ncc;i++) fprintf(of,"%s[%d,%d,%d]", i?",":"", Ls[i],Es[i],As[i]); - fprintf(of,"]}\n"); + abl_set_item(&g_abl,item->mode,item->layers,item->experts, + item->mode==3?item->applied:NULL,item->ncells); + if(of && ablate_item_line(of,item->item,item->mode,item->ncells, + item->T,item->n_prompt,item->layers,item->experts, + item->applied)<0){ rc=1; break; } + if(ABLATE_MODEL_COMPUTE_ENABLED){ + for(int s=0;sT;s++) + embed_row(m,item->tokens[s],x+(int64_t)s*D); + layers_forward(m,x,item->T,0); /* ONE prefill; moe() applies g_abl */ } - for(int s=0;s0?np-1:0); posfinal_norm, D, c->eps); - matmul_qt(lo, row, &m->lm_head, 1); - int gold=ids[pos+1]; - float mx=lo[0]; int am=0; - for(int i=1;imx){mx=lo[i];am=i;} } - double se=0; for(int i=0;imolo) molo=lo[i]; } - float mgn=lo[gold]-molo; /* gold-vs-best-competitor logit margin */ - for(int k=0;ktk_val[mn]){ tk_val[mn]=v; tk_id[mn]=i; } } - fprintf(of,"{\"t\":\"lg\",\"item\":%ld,\"pos\":%ld,\"gold\":%d,\"nll\":%.6f," - "\"glogit\":%.6g,\"molo\":%.6g,\"mgn\":%.6g,\"am\":%d,\"amlogit\":%.6g," - "\"logZ\":%.6f,\"corr\":%d,\"tk\":[", - item, pos, gold, gnll, (double)lo[gold], (double)molo, (double)mgn, - am, (double)mx, logZ, (am==gold)?1:0); - for(int k=0;ktokens,item->item,D,V}; + if(ablate_item_output(of,item->n_prompt-1,item->T-1, + ablate_model_row_output,&rows)!=0){ rc=1; break; } + run->completed_items++; + run->completed_targets+=item->T-item->n_prompt; + if(++nreq%8==0) fprintf(stderr,"[ablate %zu item | %.1fs | RSS %.2f GB | hit %.0f%%]\n", nreq, now_s()-t0, rss_gb(), (m->hits+m->miss)?100.0*m->hits/(m->hits+m->miss):0.0); } abl_reset(&g_abl); /* leave the engine in the OFF state */ - if(of){ fflush(of); fclose(of); } - free(ln); free(ids); free(x); free(lo); free(row); fclose(f); + free(x); free(lo); free(row); +#ifdef COLI_TEST_ABLATE_ADAPTERS + if(g_ablate_adapter_test.force_body_rc) rc=1; +#endif + return rc; +} + +static int run_ablate_score(Model *m, const char *path){ + FILE *f=NULL; int own_manifest=1; +#ifdef COLI_TEST_ABLATE_ADAPTERS + if(g_ablate_adapter_test.borrowed_manifest){ + f=g_ablate_adapter_test.borrowed_manifest; own_manifest=0; + clearerr(f); rewind(f); + }else +#endif + { f=fopen(path,"rb"); if(!f){perror(path);exit(1);} } + AblateManifest manifest; + if(ablate_manifest_load(f,&m->c,&manifest)!=0){ + if(own_manifest) fclose(f); + return 1; + } +#ifdef COLI_TEST_ABLATE_ADAPTERS + if(!own_manifest && g_ablate_adapter_test.after_manifest_load) + g_ablate_adapter_test.after_manifest_load(f); +#endif + AblateModelRun model_run={m,&manifest,0,0}; + AblateOutputRun output_run={ + &model_run,ablate_file_open_run,ablate_model_output_body, + &manifest.info,&model_run.completed_items,&model_run.completed_targets, + }; + const char *outp=getenv("ABLATE_OUT"); +#ifdef COLI_TEST_ABLATE_ADAPTERS + if(g_ablate_adapter_test.override_outp) outp=g_ablate_adapter_test.outp; +#endif + int rc=ablate_output_run(&output_run,outp,m->c.vocab); + /* Keep the manifest identity open until the atomic new-output decision and + * all evidence writes finish. Borrowed test streams remain caller-owned. */ + if(own_manifest && fclose(f)!=0){ + fprintf(stderr,"[ablate] INCOMPLETE: the manifest could not be closed cleanly\n"); + rc=1; + } + ablate_manifest_free(&manifest); + return rc; +} + +static int ablate_model_mode_run(Model *m, const char *path){ + if(!path){ + fprintf(stderr,"[ablate] INCOMPLETE: the ablation mode was entered " + "without a manifest to run\n"); + return 1; + } + return run_ablate_score(m,path)==0 ? 0 : 1; } static void generate(Model *m, const int *prompt, int np, int n_new, int *out){ @@ -10751,8 +11429,25 @@ static int coli_env_on(const char *name) strcmp(v,"off")==0 || strcmp(v,"no")==0); } +/* One process-status mapping for the ablation branch, shared with the + * model-free entry the adapter build uses, so both report the mode's result + * the same way. */ +#define ABLATE_MAIN_RETURN(model_,path_,stats_) do { \ + Model *const ablate_main_model=(model_); \ + const char *const ablate_main_path=(path_); \ + const char *const ablate_main_stats=(stats_); \ + int ablate_main_rc=ablate_model_mode_run(ablate_main_model,ablate_main_path); \ + if(ablate_main_stats) stats_dump(ablate_main_model,ablate_main_stats); \ + return ablate_main_rc; \ +} while(0) + #ifndef COLIBRI_NO_MAIN int main(int argc, char **argv){ +#ifdef COLI_TEST_ABLATE_ADAPTERS + if(g_ablate_adapter_test.main_model) + ABLATE_MAIN_RETURN(g_ablate_adapter_test.main_model, + g_ablate_adapter_test.main_path,NULL); +#endif /* ---- Permanent OpenMP hot-thread tuning. The per-expert matmul regions are * tiny and back-to-back; with the default passive wait policy libgomp parks * the worker team between regions and the re-wake latency dominates. Keeping @@ -11499,7 +12194,9 @@ int main(int argc, char **argv){ * ablation sweep with per-target-position final-logit read-out (ABLATE_OUT= * ). Precedes SCORE. Optional ROUTE_TRACE= records the * post-ablation router trace. */ - if(getenv("ABLATE_SCORE")){ run_ablate_score(&m, getenv("ABLATE_SCORE")); if(stats) stats_dump(&m,stats); return 0; } + if(getenv("ABLATE_SCORE")){ + ABLATE_MAIN_RETURN(&m,getenv("ABLATE_SCORE"),stats); + } /* modo scoring per benchmark: SCORE= -> log-likelihood per riga */ if(getenv("SCORE")){ run_score(&m, snap, getenv("SCORE")); if(stats) stats_dump(&m,stats); return 0; } diff --git a/c/evidence_digest.h b/c/evidence_digest.h new file mode 100644 index 000000000..1ec76e877 --- /dev/null +++ b/c/evidence_digest.h @@ -0,0 +1,120 @@ +/* evidence_digest.h — SHA-256 over the exact bytes an evidence-producing mode + * consumed. Header-only: all functions are static — include from the engine. + * + * Diagnostic modes of the engine write artifacts that a separate offline tool + * checks. For that check to mean anything, the artifact has to name the inputs + * it was produced from in a way the tool can recompute independently: the exact + * config.json bytes that were loaded, and the exact manifest bytes that were + * run. A digest over those byte ranges does that, and a short self-contained + * implementation keeps it from becoming a build dependency of the whole engine + * on a crypto library it otherwise never needs. This is an integrity aid for + * reproducibility, not a security boundary. */ +#ifndef EVIDENCE_DIGEST_H +#define EVIDENCE_DIGEST_H + +#include +#include +#include + +typedef struct { + uint32_t h[8]; + uint64_t bits; + unsigned char block[64]; + size_t used; +} EvidenceSha256; + +static uint32_t evidence_rotr32(uint32_t x, unsigned n){ + return (x>>n)|(x<<(32-n)); +} + +static void evidence_sha256_block(EvidenceSha256 *s, const unsigned char *p){ + static const uint32_t k[64]={ + 0x428a2f98u,0x71374491u,0xb5c0fbcfu,0xe9b5dba5u, + 0x3956c25bu,0x59f111f1u,0x923f82a4u,0xab1c5ed5u, + 0xd807aa98u,0x12835b01u,0x243185beu,0x550c7dc3u, + 0x72be5d74u,0x80deb1feu,0x9bdc06a7u,0xc19bf174u, + 0xe49b69c1u,0xefbe4786u,0x0fc19dc6u,0x240ca1ccu, + 0x2de92c6fu,0x4a7484aau,0x5cb0a9dcu,0x76f988dau, + 0x983e5152u,0xa831c66du,0xb00327c8u,0xbf597fc7u, + 0xc6e00bf3u,0xd5a79147u,0x06ca6351u,0x14292967u, + 0x27b70a85u,0x2e1b2138u,0x4d2c6dfcu,0x53380d13u, + 0x650a7354u,0x766a0abbu,0x81c2c92eu,0x92722c85u, + 0xa2bfe8a1u,0xa81a664bu,0xc24b8b70u,0xc76c51a3u, + 0xd192e819u,0xd6990624u,0xf40e3585u,0x106aa070u, + 0x19a4c116u,0x1e376c08u,0x2748774cu,0x34b0bcb5u, + 0x391c0cb3u,0x4ed8aa4au,0x5b9cca4fu,0x682e6ff3u, + 0x748f82eeu,0x78a5636fu,0x84c87814u,0x8cc70208u, + 0x90befffau,0xa4506cebu,0xbef9a3f7u,0xc67178f2u, + }; + uint32_t w[64]; + for(int i=0;i<16;i++) + w[i]=((uint32_t)p[4*i]<<24)|((uint32_t)p[4*i+1]<<16)| + ((uint32_t)p[4*i+2]<<8)|(uint32_t)p[4*i+3]; + for(int i=16;i<64;i++){ + uint32_t s0=evidence_rotr32(w[i-15],7)^evidence_rotr32(w[i-15],18)^(w[i-15]>>3); + uint32_t s1=evidence_rotr32(w[i-2],17)^evidence_rotr32(w[i-2],19)^(w[i-2]>>10); + w[i]=w[i-16]+s0+w[i-7]+s1; + } + uint32_t a=s->h[0],b=s->h[1],c=s->h[2],d=s->h[3]; + uint32_t e=s->h[4],f=s->h[5],g=s->h[6],h=s->h[7]; + for(int i=0;i<64;i++){ + uint32_t S1=evidence_rotr32(e,6)^evidence_rotr32(e,11)^evidence_rotr32(e,25); + uint32_t ch=(e&f)^((~e)&g); + uint32_t t1=h+S1+ch+k[i]+w[i]; + uint32_t S0=evidence_rotr32(a,2)^evidence_rotr32(a,13)^evidence_rotr32(a,22); + uint32_t maj=(a&b)^(a&c)^(b&c), t2=S0+maj; + h=g; g=f; f=e; e=d+t1; d=c; c=b; b=a; a=t1+t2; + } + s->h[0]+=a; s->h[1]+=b; s->h[2]+=c; s->h[3]+=d; + s->h[4]+=e; s->h[5]+=f; s->h[6]+=g; s->h[7]+=h; +} + +static void evidence_sha256_init(EvidenceSha256 *s){ + *s=(EvidenceSha256){{ + 0x6a09e667u,0xbb67ae85u,0x3c6ef372u,0xa54ff53au, + 0x510e527fu,0x9b05688cu,0x1f83d9abu,0x5be0cd19u, + },0,{0},0}; +} + +/* Streaming update: a caller can hash a file line by line as it validates it, + * so the digest covers exactly the bytes that were accepted. */ +static void evidence_sha256_update(EvidenceSha256 *s, const void *data, size_t len){ + const unsigned char *p=(const unsigned char *)data; + s->bits+=(uint64_t)len*8u; + while(len){ + size_t take=64-s->used; if(take>len) take=len; + memcpy(s->block+s->used,p,take); s->used+=take; p+=take; len-=take; + if(s->used==64){ evidence_sha256_block(s,s->block); s->used=0; } + } +} + +static void evidence_sha256_final(EvidenceSha256 *s, unsigned char out[32]){ + uint64_t bits=s->bits; + s->block[s->used++]=0x80; + if(s->used>56){ + memset(s->block+s->used,0,64-s->used); + evidence_sha256_block(s,s->block); s->used=0; + } + memset(s->block+s->used,0,56-s->used); + for(int i=0;i<8;i++) s->block[63-i]=(unsigned char)(bits>>(8*i)); + evidence_sha256_block(s,s->block); + for(int i=0;i<8;i++){ + out[4*i]=(unsigned char)(s->h[i]>>24); + out[4*i+1]=(unsigned char)(s->h[i]>>16); + out[4*i+2]=(unsigned char)(s->h[i]>>8); + out[4*i+3]=(unsigned char)s->h[i]; + } +} + +/* One-shot digest as the 64 lowercase hex characters an artifact carries, + * NUL-terminated so it can be written straight into a text record. */ +static void evidence_sha256_hex(const void *data, size_t len, char out[65]){ + static const char hex[]="0123456789abcdef"; + EvidenceSha256 s; unsigned char raw[32]; + evidence_sha256_init(&s); evidence_sha256_update(&s,data,len); + evidence_sha256_final(&s,raw); + for(int i=0;i<32;i++){ out[2*i]=hex[raw[i]>>4]; out[2*i+1]=hex[raw[i]&15]; } + out[64]=0; +} + +#endif /* EVIDENCE_DIGEST_H */ diff --git a/c/sample.h b/c/sample.h index 332e13e77..1b206f959 100644 --- a/c/sample.h +++ b/c/sample.h @@ -175,6 +175,130 @@ static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ } static void stops_arm(const Cfg *c, int tok_eos){ stops_arm_tok(c, tok_eos, NULL); } +/* ---- classified log-prob row reduction ----------------------------------- */ +/* These three are `static inline` rather than plain `static`: they are a + * header-only facility that different engine modes pull in as they need it, + * and a translation unit that includes this header without calling all of + * them should not have to explain itself to -Wunused-function. + * + * The plain logprob_target() below answers "what is the log-probability of this + * token", which is all a sampling loop needs. Evidence-producing modes need a + * second answer as well: whether the logit row was numerically sound at all, so + * that a run can refuse a position instead of writing a NaN into an artifact a + * reader cannot distinguish from a real value. The row reduction and the + * per-target subtraction are kept separately reusable because a caller often + * reduces a row once and then reads several targets out of it. The + * per-target subtraction promotes the float logit to double before + * subtracting the row's own double logZ, so its value carries the double + * arithmetic's own rounding (a few ulp); logprob_target() below still takes the float-scale subtraction + * the sampling path has always used, so the two need not agree past a + * float's own precision on a widely spread row. */ +typedef enum { + LOGPROB_FINITE=0, /* row reduced normally; the value is usable */ + LOGPROB_NAN, /* at least one logit was NaN */ + LOGPROB_POS_INF, /* at least one logit was +infinity */ + LOGPROB_NEG_INF, /* at least one logit was -infinity */ + LOGPROB_ALL_NONFINITE, /* no logit in the row was finite */ + LOGPROB_FINITE_OVERFLOW, /* every logit was finite, the reduction was not */ + LOGPROB_INVALID, /* no row was supplied, or it had no entries */ +} LogprobStatus; + +static inline const char *logprob_status_name(LogprobStatus status){ + switch(status){ + case LOGPROB_FINITE: return "FINITE"; + case LOGPROB_NAN: return "NAN"; + case LOGPROB_POS_INF: return "POS_INF"; + case LOGPROB_NEG_INF: return "NEG_INF"; + case LOGPROB_ALL_NONFINITE: return "ALL_NONFINITE"; + case LOGPROB_FINITE_OVERFLOW: return "FINITE_OVERFLOW"; + default: return "INVALID"; + } +} + +typedef struct { + float max; /* largest logit in the row */ + double logse; /* log of the shifted exponential sum */ + double logZ; /* max + logse: the row's log partition function */ + int argmax; /* index of the largest logit */ + LogprobStatus status; +} LogprobRow; + +/* Reduce one logit row, reporting why it failed rather than only that it did. + * No partition function is computed for a row that is not entirely finite. + * + * The classification is a fixed precedence, not the order the values appear in. + * A row with no finite entry at all is ALL_NONFINITE whatever it contains, + * because the shape of such a row, not one value in it, is what a reader needs. + * Otherwise a NaN outranks an infinity, and a positive infinity outranks a + * negative one: a NaN cannot arise from saturation, so it points at a different + * defect, and a positive infinity is what actually destroys the reduction. */ +static inline LogprobStatus logprob_row_checked(const float *lo, int V, + LogprobRow *out){ + LogprobRow r={0,0,0,0,LOGPROB_INVALID}; + if(!lo || V<=0){ if(out) *out=r; return r.status; } + int finite_count=0, saw_nan=0, saw_pos_inf=0, saw_neg_inf=0; + for(int i=0;i0) saw_pos_inf=1; + else saw_neg_inf=1; + } + if(finite_count!=V){ + r.status=finite_count==0 ? LOGPROB_ALL_NONFINITE : + saw_nan ? LOGPROB_NAN : + saw_pos_inf ? LOGPROB_POS_INF : + saw_neg_inf ? LOGPROB_NEG_INF : LOGPROB_INVALID; + if(out) *out=r; + return r.status; + } + r.max=lo[0]; r.argmax=0; + for(int i=1;ir.max){ r.max=lo[i]; r.argmax=i; } + double se=0; + for(int i=0;istatus!=LOGPROB_FINITE) return r->status; + /* The subtraction is done wholly in double: the float logit is promoted + * BEFORE subtracting the row's own double logZ (max+logse). Taking it in + * float first rounds away up to an ulp of the row maximum -- on a logit- + * scale row that is about 2e-6, visible in every digit a %.17g consumer + * reads past the seventh. */ + double value=(double)lo[target]-r->logZ; + if(!isfinite(value)) return LOGPROB_FINITE_OVERFLOW; + if(out) *out=value; + return LOGPROB_FINITE; +} + /* ---- log-prob of a target token given the logit vector ------------------- */ static double logprob_target(const float *lo, int V, int target, int *am){ float mx = lo[0]; int best = 0; diff --git a/c/tests/test_ablate_mode.c b/c/tests/test_ablate_mode.c new file mode 100644 index 000000000..39adb4f78 --- /dev/null +++ b/c/tests/test_ablate_mode.c @@ -0,0 +1,620 @@ +/* test_ablate_mode.c — the ablation scoring mode's manifest loader, evidence + * writer and dispatch contract, with no model and no weights. + * + * The mode's real work is one teacher-forced prefill per manifest item, which + * needs a loaded model. Everything around that prefill -- accepting or + * refusing a manifest, binding it by digest, opening the output exactly once, + * writing the records an offline reader parses, and reporting completion -- + * runs without any model math at all. The adapter build bypasses only the + * model computation and enters the same production parser, writer and + * formatter, so those parts can be pinned on any machine. + * + * Required properties: + * P1 ROUND TRIP — a well-formed manifest produces a complete evidence + * artifact: a header naming the config and manifest digests, one item + * record per item, one logit record per target position, and a done + * record whose counts equal the header's expectations. + * P2 DIGEST BINDING — the manifest digest in the artifact is a digest of the + * manifest's own bytes under a domain prefix, so an artifact cannot be + * re-attached to a different manifest. + * P3 REFUSAL — malformed manifests are refused rather than partly run, and + * each refusal leaves no evidence file behind. + * P4 DISPATCH CONTRACT — the mode reports failure and writes nothing when it + * is handed no manifest path, which is the value the engine's environment + * lookup yields when the mode was not requested. + * P5 NEW FILE ONLY — an output path that already exists is refused, never + * truncated. + * P6 LENIENT FRAMING — the two line endings a host editor produces without + * meaning to (a CRLF terminator, and a last line with no terminator) are + * accepted, and all three framings of the same content bind to the same + * manifest digest. + * P7 NAMED REFUSALS — every refusal says why on the error stream; a silent + * non-zero exit is indistinguishable from a crash, and a refusal that + * names a line names the record at fault rather than the last one read. + * P8 BOUNDED INPUT — a manifest may not ask the loader for an unbounded + * allocation by declaring an enormous item length. + * Exit 0 = all pass. + * + * With "--emit-round-trip [lf|crlf|unterminated]" it instead writes one + * config, one manifest in the named framing and the artifact produced from + * them into and exits, so a checker written independently of this engine + * can validate real producer output -- in each framing the engine accepts. + */ +#define COLI_TEST_ABLATE_ADAPTERS 1 +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main +#undef COLI_TEST_ABLATE_ADAPTERS + +static int fails = 0; +#define CHECK(cond,msg) do{ if(!(cond)){ printf(" FAIL: %s\n", msg); fails++; } \ + else printf(" ok: %s\n", msg); }while(0) + +/* Process-unique so two concurrent test runs in the same CWD never collide. */ +static char MANIFEST_PATH[64]; +static char EVIDENCE_PATH[64]; +static char STDERR_PATH[64]; + +static void temp_paths_init(void){ + long pid=(long)getpid(); + snprintf(MANIFEST_PATH,sizeof(MANIFEST_PATH),"tmp_test_ablate_manifest.%ld.txt",pid); + snprintf(EVIDENCE_PATH,sizeof(EVIDENCE_PATH),"tmp_test_ablate_evidence.%ld.jsonl",pid); + snprintf(STDERR_PATH,sizeof(STDERR_PATH),"tmp_test_ablate_stderr.%ld.txt",pid); +} + +/* An eight-expert, four-layer, 64-token vocabulary is enough for every bound + * the loader checks, and small enough to write by hand. The digest is taken + * over the same bytes a reader would find in the config file, so an external + * checker can recompute it. */ +static const char CONFIG_JSON[]= + "{\"hidden_size\":8,\"num_hidden_layers\":4,\"n_routed_experts\":8," + "\"first_k_dense_replace\":1,\"vocab_size\":64}\n"; + +static void cfg_init(Cfg *c){ + memset(c,0,sizeof(*c)); + c->hidden=8; c->vocab=64; c->n_layers=4; c->first_dense=1; c->n_experts=8; + evidence_sha256_hex(CONFIG_JSON,sizeof(CONFIG_JSON)-1,c->config_sha256); +} + +static int file_write(const char *path, const char *body){ + FILE *f=fopen(path,"wb"); + if(!f) return -1; + size_t want=strlen(body); + size_t got=fwrite(body,1,want,f); + return (fclose(f)==0 && got==want) ? 0 : -1; +} + +static void manifest_write(const char *body){ + (void)file_write(MANIFEST_PATH,body); +} + +/* The manifest used for the round trip, shared with the emitted artifact so an + * external checker sees exactly what this test checks. */ +static const char ROUND_TRIP_MANIFEST[]= + "0 3 2 0 0 1 2 3\n" + "1 4 2 1 1 2 3 -1 4 5 6 7\n"; + +static char *slurp(const char *path, size_t *len_out){ + FILE *f=fopen(path,"rb"); + if(!f) return NULL; + fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); + if(n<0){ fclose(f); return NULL; } + char *b=malloc((size_t)n+1); + if(!b){ fclose(f); return NULL; } + size_t got=fread(b,1,(size_t)n,f); fclose(f); + b[got]=0; + if(len_out) *len_out=got; + return b; +} + +/* Run the mode with model computation bypassed. */ +static int run_mode(Cfg *c, const char *manifest_path, const char *out_path){ + static Model m; + memset(&m,0,sizeof(m)); + m.c=*c; + remove(out_path); + memset(&g_ablate_adapter_test,0,sizeof(g_ablate_adapter_test)); + g_ablate_adapter_test.bypass_model_compute=1; + g_ablate_adapter_test.override_outp=1; + g_ablate_adapter_test.outp=out_path; + int rc=ablate_model_mode_run(&m,manifest_path); + memset(&g_ablate_adapter_test,0,sizeof(g_ablate_adapter_test)); + return rc; +} + +static int count_occurrences(const char *hay, const char *needle){ + int n=0; + for(const char *p=strstr(hay,needle); p; p=strstr(p+1,needle)) n++; + return n; +} + +static void t_round_trip(Cfg *c){ + printf("P1 round trip and P2 digest binding\n"); + /* item T n_prompt mode ncells (L E A)... t_0..t_{T-1} */ + const char *body=ROUND_TRIP_MANIFEST; + manifest_write(body); + CHECK(run_mode(c,MANIFEST_PATH,EVIDENCE_PATH)==0, "a well-formed manifest runs to completion"); + + size_t len=0; + char *text=slurp(EVIDENCE_PATH,&len); + CHECK(text!=NULL, "the requested evidence file exists"); + if(!text) return; + + CHECK(count_occurrences(text,"\"t\":\"hdr\"")==1, "exactly one header record"); + CHECK(count_occurrences(text,"\"t\":\"ah\"")==2, "one item record per manifest item"); + CHECK(count_occurrences(text,"\"t\":\"lg\"")==3, "one logit record per target position"); + CHECK(count_occurrences(text,"\"t\":\"done\"")==1, "exactly one done record"); + CHECK(strstr(text,"\"schema\":\"coli-ablate/2\"")!=NULL, "the header names the schema"); + CHECK(strstr(text,"\"expected_items\":2")!=NULL, "the header expects both items"); + CHECK(strstr(text,"\"expected_targets\":3")!=NULL, "the header expects all three targets"); + CHECK(strstr(text,"\"completed_items\":2")!=NULL, "the done record completed both items"); + CHECK(strstr(text,"\"completed_targets\":3")!=NULL, "the done record completed all targets"); + CHECK(strstr(text,c->config_sha256)!=NULL, "the header carries the loaded config digest"); + CHECK(strstr(text,"\"cells\":[[2,3,-1]]")!=NULL, "the ablated cell is recorded as given"); + + /* P2: recompute the manifest digest the way an offline reader would. */ + static const char domain[]="coli-ablate-manifest/2\n"; + EvidenceSha256 h; unsigned char raw[32]; char expect[65]; + static const char hex[]="0123456789abcdef"; + evidence_sha256_init(&h); + evidence_sha256_update(&h,domain,sizeof(domain)-1); + evidence_sha256_update(&h,body,strlen(body)); + evidence_sha256_final(&h,raw); + for(int i=0;i<32;i++){ expect[2*i]=hex[raw[i]>>4]; expect[2*i+1]=hex[raw[i]&15]; } + expect[64]=0; + CHECK(count_occurrences(text,expect)==2, + "both records carry a digest of the manifest bytes under the domain prefix"); + + free(text); + remove(EVIDENCE_PATH); +} + +static void t_refusal(Cfg *c){ + printf("P3 malformed manifests are refused\n"); + static const struct { const char *body; const char *why; } bad[] = { + { "", "an empty manifest" }, + { "\n", "a manifest that is only a line terminator" }, + { "0 3 2 0 0 1 2 3\n\n", "a manifest with an empty line in it" }, + { "0 3 2 0\r 0 1 2 3\n", "a carriage return inside a record" }, + { "00 3 2 0 0 1 2 3\n", "a leading zero in a field" }, + { "0 3 2 0 0 1 2 3 4\n", "a token count that exceeds the declared length" }, + { "0 3 2 0 0 1 2\n", "a token count below the declared length" }, + { "0 1 1 0 0 5\n", "a sequence with no target position" }, + { "0 3 3 0 0 1 2 3\n", "a prompt that leaves no target position" }, + { "0 3 2 4 1 2 3 -1 1 2 3\n", "an unknown ablation mode" }, + { "0 3 2 1 0 1 2 3\n", "an ablating mode with no cells" }, + { "0 3 2 0 1 2 3 -1 1 2 3\n", "a baseline item with cells" }, + { "0 3 2 1 1 0 3 -1 1 2 3\n", "a layer below the first routed layer" }, + { "0 3 2 1 1 2 99 -1 1 2 3\n", "an expert beyond the model's expert count" }, + { "0 3 2 3 1 2 3 3 1 2 3\n", "a swap whose target is the ablated expert" }, + { "0 3 2 1 1 2 3 0 1 2 3\n", "a swap target on a non-swap mode" }, + { "0 3 2 1 2 2 3 -1 2 3 -1 1 2 3\n", "a duplicate cell within one item" }, + { "0 3 2 0 0 1 2 999\n", "a token beyond the vocabulary" }, + { "0 3 2 0 0 1 2 64\n", "a token exactly at the vocab size -- one past the " + "last valid id (999 above is not a boundary test: it is caught the same " + "way whether the bound is >=V or the off-by-one >V)" }, + { "99999999999999999999999999 3 2 0 0 1 2 3\n", + "an item id with enough digits to overflow 64-bit parsing (ERANGE) -- " + "the item field has no upper-bound range check of its own, so an " + "overflow silently clamped to INTMAX_MAX would otherwise be accepted " + "as an enormous-but-'valid' item id" }, + { "0 3 2 0 0 1 2 3\n0 3 2 0 0 1 2 3\n", "a duplicate item id" }, + }; + for(unsigned i=0;iconfig_sha256,0,65); } +static void damage_short_digest(Cfg *c){ c->config_sha256[7]='Z'; } +static void damage_layer_count(Cfg *c){ c->n_layers=129; } +static void damage_expert_count(Cfg *c){ c->n_experts=0; } +static void damage_vocab(Cfg *c){ c->vocab=(1<<24)+1; } +static void damage_first_dense(Cfg *c){ c->first_dense=c->n_layers+1; } + +static void t_named_refusals(Cfg *c){ + printf("P7 every refusal says why on the error stream\n"); + static const char good[]="0 3 2 0 0 1 2 3\n"; + static const struct { void (*damage)(Cfg *); const char *body; const char *why; } cases[] = { + { damage_unhashed_config, good, "an unhashed config is refused with a reason" }, + { damage_short_digest, good, "a malformed config digest is refused with a reason" }, + { damage_layer_count, good, "too many layers is refused with a reason" }, + { damage_expert_count, good, "no experts is refused with a reason" }, + { damage_vocab, good, "an implausible vocabulary is refused with a reason" }, + { damage_first_dense, good, "a first routed layer past the end is refused with a reason" }, + { NULL, "0 3 2 0 0 1 2 999\n", "a malformed manifest is refused with a reason" }, + { NULL, "", "an empty manifest is refused with a reason" }, + }; + for(unsigned i=0;i0, cases[i].why); + } + remove(STDERR_PATH); +} + +/* Run once with stderr captured, and hand the captured text back. */ +static char *refusal_text_of(Cfg *c, const char *manifest_body){ + manifest_write(manifest_body); + if(!freopen(STDERR_PATH,"w",stderr)) return NULL; + int rc=run_mode(c,MANIFEST_PATH,EVIDENCE_PATH); + fflush(stderr); + remove(EVIDENCE_PATH); + if(rc!=1) return NULL; + return slurp(STDERR_PATH,NULL); +} + +static void t_refusal_names_the_offender(Cfg *c){ + printf("P7b a refusal names the record at fault\n"); + /* Three good records, then a fourth repeating the first record's id, then + * a fifth good one. The duplicate check runs after the whole file is read, + * so naming the last line read would report line 5 here. */ + char *text=refusal_text_of(c, + "7 3 2 0 0 1 2 3\n" + "8 3 2 0 0 1 2 3\n" + "9 3 2 0 0 1 2 3\n" + "7 3 2 0 0 4 5 6\n" + "10 3 2 0 0 1 2 3\n"); + CHECK(text!=NULL, "a duplicate item id is refused"); + CHECK(text && strstr(text,"at line 4")!=NULL, + "the duplicate names the line that repeats the id, not the last line"); + CHECK(text && strstr(text,"at line 5")==NULL, + "the duplicate does not name the last line of the file"); + free(text); + + /* A malformed record in the middle names itself, not the end of file. */ + text=refusal_text_of(c, + "0 3 2 0 0 1 2 3\n" + "nonsense\n" + "2 3 2 0 0 1 2 3\n"); + CHECK(text && strstr(text,"at line 2")!=NULL, + "a malformed record names its own line"); + free(text); + remove(STDERR_PATH); +} + +static void t_bounded_item_length(Cfg *c){ + printf("P8 an absurd item length is refused, not allocated\n"); + char *text=refusal_text_of(c,"0 9000000000000000000 2 0 0 1 2 3\n"); + CHECK(text!=NULL, "an item length near the 64-bit ceiling is refused"); + CHECK(text && strstr(text,"above the")!=NULL && + strstr(text,"limit for one item")!=NULL, + "the refusal names the per-item token limit"); + free(text); + + char body[64]; + snprintf(body,sizeof(body),"0 %d 2 0 0 1 2 3\n",ABLATE_MAX_ITEM_TOKENS+1); + text=refusal_text_of(c,body); + CHECK(text!=NULL, "one token past the documented limit is refused"); + CHECK(text && strstr(text,"above the")!=NULL, "and names the limit"); + free(text); + remove(STDERR_PATH); +} + +/* nll must come from a subtraction done wholly in double + * (logZ - lo[gold]), not from a float-rounded intermediate. A row this + * wide (max 1.0e7, gold logit 0.25, everything else far below the max) + * makes the two formulas disagree: the float subtraction lo[gold]-max + * rounds gold's contribution away entirely (its ulp near 1.0e7 is 1.0, + * and 0.25 is under half that), while the all-double path keeps it. */ +static void t_nll_pin(Cfg *c){ + printf("nll is a double-precision reduction, not a float intermediate\n"); + int V=c->vocab; /* 64, from CONFIG_JSON */ + float *row=malloc(sizeof(float)*(size_t)V); + for(int i=0;i>4]; expect[2*i+1]=hex[raw[i]&15]; } + expect[64]=0; + CHECK(strcmp(got_digest,expect)==0, + "cfg_root's digest matches an independently computed SHA-256 of the bytes"); + /* And against a literal computed outside this codebase, so a change to + * both the fixture and the hash function in the same wrong direction + * cannot pass unnoticed. */ + CHECK(strcmp(got_digest,PINNED_DIGEST)==0, + "cfg_root's digest matches the pinned literal for this fixture"); + + free(ar); + remove(path); + rmdir(dir); +} + +/* Write one config, one manifest and the artifact produced from them into a + * directory, for an independently written checker to validate. */ +/* The round-trip manifest re-framed the way a host editor might have saved it. + * The engine accepts all three and binds them to the same digest. */ +static int reframe(const char *canonical, const char *framing, + char *dst, size_t cap){ + size_t out=0; + for(size_t i=0;canonical[i];i++){ + if(canonical[i]=='\n'){ + if(strcmp(framing,"crlf")==0){ + if(out+2>=cap) return -1; + dst[out++]='\r'; + }else if(strcmp(framing,"unterminated")==0 && canonical[i+1]==0){ + break; /* drop the final terminator */ + } + if(out+1>=cap) return -1; + dst[out++]='\n'; + continue; + } + if(out+1>=cap) return -1; + dst[out++]=canonical[i]; + } + dst[out]=0; + return 0; +} + +static int emit_round_trip(Cfg *c, const char *dir, const char *framing){ + char config_path[1024], manifest_path[1024], evidence_path[1024]; + if(snprintf(config_path,sizeof(config_path),"%s/config.json",dir)>=(int)sizeof(config_path) || + snprintf(manifest_path,sizeof(manifest_path),"%s/manifest.txt",dir)>=(int)sizeof(manifest_path) || + snprintf(evidence_path,sizeof(evidence_path),"%s/evidence.jsonl",dir)>=(int)sizeof(evidence_path)){ + fprintf(stderr,"test_ablate_mode: directory name is too long\n"); + return 1; + } + char framed[4096]; + if(reframe(ROUND_TRIP_MANIFEST,framing,framed,sizeof(framed))!=0){ + fprintf(stderr,"test_ablate_mode: unknown or oversized framing %s\n",framing); + return 1; + } + if(file_write(config_path,CONFIG_JSON)!=0 || + file_write(manifest_path,framed)!=0){ + fprintf(stderr,"test_ablate_mode: cannot write into %s\n",dir); + return 1; + } + int rc=run_mode(c,manifest_path,evidence_path); + if(rc!=0) fprintf(stderr,"test_ablate_mode: the ablation mode failed\n"); + return rc; +} + +int main(int argc, char **argv){ + temp_paths_init(); + Cfg c; cfg_init(&c); + if((argc==3 || argc==4) && strcmp(argv[1],"--emit-round-trip")==0) + return emit_round_trip(&c,argv[2],argc==4?argv[3]:"lf"); + printf("test_ablate_mode\n"); + t_round_trip(&c); + t_refusal(&c); + t_token_boundary_accepts_the_last_valid_id(&c); + t_dispatch_contract(&c); + t_new_file_only(&c); + t_lenient_framing(&c); + t_named_refusals(&c); + t_refusal_names_the_offender(&c); + t_bounded_item_length(&c); + t_nll_pin(&c); + t_load_cfg_digest_binding(&c); + remove(MANIFEST_PATH); + remove(EVIDENCE_PATH); + printf(fails ? "FAILED (%d)\n" : "PASSED (%d failures)\n", fails); + return fails ? 1 : 0; +} diff --git a/c/tests/test_ablate_mode_gate.py b/c/tests/test_ablate_mode_gate.py new file mode 100644 index 000000000..b4456b30f --- /dev/null +++ b/c/tests/test_ablate_mode_gate.py @@ -0,0 +1,188 @@ +"""The ablation scoring mode must stay unreachable unless it is asked for, and +its output must satisfy a checker written independently of the engine. + +Two different risks are covered here, and neither is covered anywhere else. + +The first is that the mode becomes reachable on a normal run. Ablation is a +diagnostic path: it replaces the whole decode with a teacher-forced sweep and +returns its own process status. If its dispatch ever ran unconditionally, +every ordinary invocation would stop doing what it was asked to do. That +branch sits after the model is loaded, and this repository ships no weights, so +no test can reach it by running the engine; what can be checked, exactly and +mechanically, is that the engine contains no path into the mode other than its +environment variable. These checks read `colibri.c` for that reason, and they +fail if the guard is removed, weakened, or bypassed by a second call site. + +The second is that the engine and the offline checker drift apart. The mode's +whole purpose is to produce an artifact that `tools/check_ablate_evidence.py` +can validate; a producer change that the checker would reject is a defect even +if the engine is self-consistent. The last check runs the real producer and +the real checker against each other, once per manifest framing the engine +accepts, because the framings are exactly where the two could disagree: the +engine normalises them before hashing, and the checker has to reproduce that +normalisation rather than hash the file as it sits on disk. +""" + +import pathlib +import subprocess +import sys +import tempfile +import unittest + +HERE = pathlib.Path(__file__).resolve().parent +ENGINE = HERE.parent / "colibri.c" +VALIDATOR = HERE.parent / "tools" / "check_ablate_evidence.py" +PRODUCER = HERE / "test_ablate_mode" + +ENV_GUARD = 'if(getenv("ABLATE_SCORE")){' +ADAPTER_DEFINE = "COLI_TEST_ABLATE_ADAPTERS" + + +def _engine_lines(): + return ENGINE.read_text(encoding="utf-8").splitlines() + + +def _adapter_only_lines(lines): + """Line numbers (0-based) that the compiler only sees when the model-free + adapter build is selected.""" + guarded = set() + depth = 0 + adapter_depth = None + for number, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith("#if"): + depth += 1 + if adapter_depth is None and ADAPTER_DEFINE in stripped: + adapter_depth = depth + elif stripped.startswith("#endif"): + if adapter_depth is not None and depth == adapter_depth: + adapter_depth = None + depth = max(0, depth - 1) + elif adapter_depth is not None: + guarded.add(number) + return guarded + + +class AblationModeEntryTest(unittest.TestCase): + def test_the_mode_has_exactly_one_product_entry_and_it_is_the_guard(self): + lines = _engine_lines() + adapter_only = _adapter_only_lines(lines) + uses = [ + number + for number, line in enumerate(lines) + if "ABLATE_MAIN_RETURN(" in line and not line.lstrip().startswith("#define") + ] + self.assertEqual(len(uses), 2, "expected one product entry and one adapter entry") + product = [number for number in uses if number not in adapter_only] + adapter = [number for number in uses if number in adapter_only] + self.assertEqual(len(product), 1, "the ablation mode has more than one product entry") + self.assertEqual(len(adapter), 1, "the adapter entry is no longer compile-gated") + + guard = lines[product[0] - 1].strip() + self.assertEqual( + guard, + ENV_GUARD, + "the product entry to the ablation mode is not guarded by its " + "environment variable; an unguarded dispatch would replace every " + "ordinary run with a diagnostic sweep", + ) + + def test_no_second_path_reaches_the_mode(self): + lines = _engine_lines() + entry = [line for line in lines if "ablate_model_mode_run(" in line] + self.assertEqual( + [line for line in entry if line.lstrip().startswith("static int ablate_model_mode_run")], + [line for line in entry if "static int" in line], + "ablate_model_mode_run is declared more than once", + ) + callers = [ + line + for line in entry + if "static int" not in line and not line.strip().startswith("*") + ] + self.assertEqual( + len(callers), 1, + "ablate_model_mode_run is called from somewhere other than the guard macro", + ) + self.assertIn("ablate_main_rc=ablate_model_mode_run", callers[0].replace(" ", "")) + + runners = [ + number + for number, line in enumerate(lines) + if "run_ablate_score(" in line and "static int run_ablate_score" not in line + ] + self.assertEqual(len(runners), 1, "run_ablate_score is called from more than one place") + enclosing = "" + for number in range(runners[0], -1, -1): + stripped = lines[number].strip() + if stripped.startswith("static ") and stripped.endswith("{"): + enclosing = stripped + break + self.assertIn( + "ablate_model_mode_run", + enclosing, + "run_ablate_score is called from outside the mode implementation", + ) + + +class AblationEvidenceRoundTripTest(unittest.TestCase): + FRAMINGS = ("lf", "crlf", "unterminated") + + def test_producer_output_passes_the_offline_checker(self): + if not PRODUCER.exists(): + reason = ( + f"{PRODUCER.name} is not built, so real producer output was NOT " + f"checked against the offline checker this run; build it with " + f"`make {PRODUCER.relative_to(HERE.parent)}` and re-run" + ) + # A silent skip here reads as a pass, and the thing being skipped is + # the only check that the engine and the checker still agree. Say so + # on the console as well as in the unittest result. + print(f"SKIP: {reason}", file=sys.stderr, flush=True) + self.skipTest(reason) + digests = {} + for framing in self.FRAMINGS: + with self.subTest(framing=framing), \ + tempfile.TemporaryDirectory() as directory: + emitted = subprocess.run( + [str(PRODUCER), "--emit-round-trip", directory, framing], + capture_output=True, + text=True, + ) + self.assertEqual(emitted.returncode, 0, emitted.stderr) + checked = subprocess.run( + [ + sys.executable, + str(VALIDATOR), + "--config", + f"{directory}/config.json", + f"{directory}/manifest.txt", + f"{directory}/evidence.jsonl", + ], + capture_output=True, + text=True, + ) + self.assertEqual( + checked.returncode, + 0, + f"the offline checker rejected real producer output " + f"({framing}):\n{checked.stdout}{checked.stderr}", + ) + self.assertIn("PASS", checked.stdout) + digests[framing] = checked.stdout.split("manifest=")[1].split()[0] + # The framings must really differ on disk, or this proves nothing. + raw = pathlib.Path(directory, "manifest.txt").read_bytes() + if framing == "crlf": + self.assertIn(b"\r\n", raw) + elif framing == "unterminated": + self.assertFalse(raw.endswith(b"\n")) + else: + self.assertTrue(raw.endswith(b"\n")) + self.assertNotIn(b"\r", raw) + self.assertEqual( + len(set(digests.values())), 1, + f"the framings bound different manifest digests: {digests}") + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_logprob_status.c b/c/tests/test_logprob_status.c new file mode 100644 index 000000000..ca155ee2a --- /dev/null +++ b/c/tests/test_logprob_status.c @@ -0,0 +1,339 @@ +/* test_logprob_status.c — the classified logit-row reduction in sample.h and + * the evidence digest in evidence_digest.h. + * + * Both are foundations for engine modes that write artifacts an offline tool + * re-checks, so both need to be right for inputs the sampling path never sees: + * rows containing NaN or an infinity, empty rows, and a target whose distance + * from the row maximum does not fit in a float. A wrong answer there is worse + * than a crash, because it is written into a file and read back later as if it + * were a measurement. + * + * Required properties: + * P1 AGREEMENT — on a row whose logits sit close enough together that a + * float-scale subtraction cannot lose precision, the classified path + * returns exactly the value and argmax flag the plain logprob_target() + * returns. The two paths are not required to agree past a float's own + * precision on a widely spread row: logprob_target() keeps its + * historical float-scale subtraction, while the classified path + * promotes to double first (P1B). + * P1B PRECISION — the classified path's value is accurate to double + * precision: it matches, bit for bit, a reference computed + * a second time in this file by promoting the row's logit to double + * before subtracting, on rows spanning ordinary, widely spread, + * subnormal and full-double-precision inputs. This is the property + * P1's exact test rows are too narrow to exercise. + * P2 CLASSIFICATION — each exceptional row shape reports its own cause, not a + * generic failure, no partition function is invented for it, and the + * precedence between the causes is the documented one. + * P3 PROPAGATION — reading a target out of a row that did not reduce cleanly + * reports the row's original cause, a caller mistake is reported as such + * rather than as a usable value, and every refusal leaves a defined + * number behind rather than whatever the caller's variable held. + * P4 DIGEST — the digest matches published SHA-256 values, including the + * lengths that exercise the block boundary and both padding branches, and + * the streaming form agrees with the one-shot form. + * Exit 0 = all pass. + */ +#include +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main +#include "../evidence_digest.h" + +static int fails = 0; +#define CHECK(cond,msg) do{ if(!(cond)){ printf(" FAIL: %s\n", msg); fails++; } \ + else printf(" ok: %s\n", msg); }while(0) + +static int close_enough(double a, double b){ + double d = a - b; + if(d < 0) d = -d; + return d <= 1e-12; +} + +/* Tight tolerance for P1B: the classified path's value against an + * second double computation should agree to a few ulp; the relative bound leaves no room + * for a float-scale rounding to sneak back in while tolerating, at most, + * a difference in the last bit of a double reduction. */ +static int close_enough_precise(double a, double b){ + /* a few units in the last place, relative to the reference magnitude */ + double tol = 16.0 * DBL_EPSILON * fabs(b) + 1e-300; + return fabs(a - b) <= tol; +} + +/* P1: on a row a float subtraction cannot mis-round, the classified path + * agrees with the plain path exactly. */ +static void t_agreement(void){ + printf("P1 agreement with the plain path\n"); + static const float rows[3][5] = { + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, + { 1.0f, 2.0f, 3.0f, -1.0f, 0.5f }, + { -30.0f, -31.0f, -29.5f, -100.0f, -29.75f }, + }; + for(int r=0;r<3;r++){ + LogprobRow row; + CHECK(logprob_row_checked(rows[r],5,&row)==LOGPROB_FINITE, + "finite row reduces to FINITE"); + for(int target=0;target<5;target++){ + double classified = 0; + CHECK(logprob_from_row_checked(rows[r],target,&row,&classified)==LOGPROB_FINITE, + "finite row yields a finite target value"); + int plain_argmax = 0; + double plain = logprob_target(rows[r],5,target,&plain_argmax); + CHECK(close_enough(classified,plain), + "classified value equals the plain logprob_target value"); + CHECK(plain_argmax==(row.argmax==target), + "classified argmax flag equals the plain argmax flag"); + } + } + /* A two-entry uniform row has a hand-checkable answer: log(1/2). */ + static const float uniform[2] = { 0.0f, 0.0f }; + LogprobRow row; + logprob_row_checked(uniform,2,&row); + double value = 0; + logprob_from_row_checked(uniform,0,&row,&value); + CHECK(close_enough(value,-log(2.0)), "uniform pair gives log(1/2)"); + CHECK(close_enough(row.logZ,log(2.0)), "uniform pair has logZ = log(2)"); + CHECK(row.argmax==0, "uniform pair takes the first maximum"); +} + +/* A second double computation for P1B (same formula as the code under test, so it + * detects float-first rounding, not ulp-level error): re-finds the row's maximum and + * log-sum-exp from scratch, entirely in double, without touching the + * LogprobRow the function under test already reduced. This is not the same + * code path as logprob_from_row_checked() -- it recomputes the row instead + * of reading r->max/r->logZ -- so it cannot agree with a wrong answer by + * sharing a mistake. */ +static double reference_double(const float *lo, int V, int target){ + double mx = (double)lo[0]; + for(int i=1;imx) mx=v; } + double se = 0; + for(int i=0;i NAN"); + CHECK(row.status==LOGPROB_NAN, "the returned row carries the same status"); + CHECK(row.logZ==0 && row.max==0, "no partition function is invented"); + + float pos_row[3] = { 1.0f, INFINITY, 3.0f }; + CHECK(logprob_row_checked(pos_row,3,&row)==LOGPROB_POS_INF, "+inf present -> POS_INF"); + + float neg_row[3] = { 1.0f, -INFINITY, 3.0f }; + CHECK(logprob_row_checked(neg_row,3,&row)==LOGPROB_NEG_INF, "-inf present -> NEG_INF"); + + /* Precedence, stated as a fixed order rather than the order of appearance. + * Each row below contains more than one exceptional class, so a change to + * the ranking changes an answer here. */ + float mixed_row[3] = { NAN, INFINITY, -INFINITY }; + CHECK(logprob_row_checked(mixed_row,3,&row)==LOGPROB_ALL_NONFINITE, + "no finite entry at all -> ALL_NONFINITE, whatever the classes are"); + + float finite_nan_pos[3] = { 1.0f, NAN, INFINITY }; + CHECK(logprob_row_checked(finite_nan_pos,3,&row)==LOGPROB_NAN, + "a NaN outranks a positive infinity"); + float finite_pos_nan[3] = { 1.0f, INFINITY, NAN }; + CHECK(logprob_row_checked(finite_pos_nan,3,&row)==LOGPROB_NAN, + "a NaN outranks a positive infinity that appears before it"); + float finite_nan_neg[3] = { 1.0f, NAN, -INFINITY }; + CHECK(logprob_row_checked(finite_nan_neg,3,&row)==LOGPROB_NAN, + "a NaN outranks a negative infinity"); + float finite_pos_neg[3] = { 1.0f, INFINITY, -INFINITY }; + CHECK(logprob_row_checked(finite_pos_neg,3,&row)==LOGPROB_POS_INF, + "a positive infinity outranks a negative one"); + float finite_neg_pos[3] = { 1.0f, -INFINITY, INFINITY }; + CHECK(logprob_row_checked(finite_neg_pos,3,&row)==LOGPROB_POS_INF, + "a positive infinity outranks a negative one that appears first"); + float finite_neg[3] = { 1.0f, 2.0f, -INFINITY }; + CHECK(logprob_row_checked(finite_neg,3,&row)==LOGPROB_NEG_INF, + "a negative infinity alongside finite entries -> NEG_INF"); + float all_nan[2] = { NAN, NAN }; + CHECK(logprob_row_checked(all_nan,2,&row)==LOGPROB_ALL_NONFINITE, + "a row of NaN alone is still ALL_NONFINITE"); + + CHECK(logprob_row_checked(NULL,3,&row)==LOGPROB_INVALID, "no row -> INVALID"); + CHECK(logprob_row_checked(nan_row,0,&row)==LOGPROB_INVALID, "empty row -> INVALID"); + CHECK(logprob_row_checked(nan_row,-1,&row)==LOGPROB_INVALID, "negative length -> INVALID"); + CHECK(logprob_row_checked(nan_row,3,NULL)==LOGPROB_NAN, + "the status is returned even with no output row"); + + CHECK(strcmp(logprob_status_name(LOGPROB_FINITE),"FINITE")==0, "FINITE names itself"); + CHECK(strcmp(logprob_status_name(LOGPROB_NAN),"NAN")==0, "NAN names itself"); + CHECK(strcmp(logprob_status_name(LOGPROB_POS_INF),"POS_INF")==0, "POS_INF names itself"); + CHECK(strcmp(logprob_status_name(LOGPROB_NEG_INF),"NEG_INF")==0, "NEG_INF names itself"); + CHECK(strcmp(logprob_status_name(LOGPROB_ALL_NONFINITE),"ALL_NONFINITE")==0, + "ALL_NONFINITE names itself"); + CHECK(strcmp(logprob_status_name(LOGPROB_FINITE_OVERFLOW),"FINITE_OVERFLOW")==0, + "FINITE_OVERFLOW names itself"); + CHECK(strcmp(logprob_status_name(LOGPROB_INVALID),"INVALID")==0, "INVALID names itself"); +} + +/* P3: a target read out of a bad row reports the row's cause. A spread that + * would overflow a float subtraction no longer refuses the target, because + * the classified path takes that subtraction in double: it is pinned below + * as a status change, not silently absorbed into "still finite". */ +static void t_propagation(void){ + printf("P3 target reads propagate the row's cause\n"); + LogprobRow row; + float nan_row[3] = { 1.0f, NAN, 3.0f }; + logprob_row_checked(nan_row,3,&row); + double value = 12345.0; + CHECK(logprob_from_row_checked(nan_row,0,&row,&value)==LOGPROB_NAN, + "a NaN row propagates NAN to the target read"); + CHECK(isnan(value), "a refused target read leaves a defined value behind"); + + CHECK(logprob_from_row_checked(nan_row,0,NULL,&value)==LOGPROB_INVALID, + "no row -> INVALID"); + + /* A caller mistake on a perfectly good row must not read back as a usable + * value: the earlier form returned FINITE here and never wrote the output, + * so the caller consumed whatever its own variable happened to hold. */ + static const float good_row[3] = { 1.0f, 2.0f, 3.0f }; + LogprobRow good; + CHECK(logprob_row_checked(good_row,3,&good)==LOGPROB_FINITE, "the good row reduces"); + value = 12345.0; + CHECK(logprob_from_row_checked(NULL,0,&good,&value)==LOGPROB_INVALID, + "no logit vector on a good row -> INVALID, never FINITE"); + CHECK(isnan(value), "no logit vector leaves a defined value behind"); + value = 12345.0; + CHECK(logprob_from_row_checked(good_row,-1,&good,&value)==LOGPROB_INVALID, + "a negative target on a good row -> INVALID, never FINITE"); + CHECK(isnan(value), "a negative target leaves a defined value behind"); + + logprob_row_checked(nan_row,3,&row); + CHECK(logprob_from_row_checked(NULL,0,&row,&value)==LOGPROB_INVALID, + "no logit vector is a caller mistake even when the row is bad"); + + /* The widest representable spread: a float subtraction between these two + * values would saturate to infinity even though every logit in the row + * is finite. The classified path subtracts in double, so it does not: + * this is a real, disclosed widening of what the layer accepts, pinned + * here so it cannot regress silently in either direction. */ + float wide[2] = { FLT_MAX, -FLT_MAX }; + CHECK(logprob_row_checked(wide,2,&row)==LOGPROB_FINITE, "an extreme finite row still reduces"); + CHECK(logprob_from_row_checked(wide,1,&row,&value)==LOGPROB_FINITE, + "a target the float subtraction could not hold -> FINITE, not refused, under the double subtraction"); + CHECK(value==-2.0*(double)FLT_MAX, + "the saturated target's value is the exact double difference, not an infinity"); + CHECK(logprob_from_row_checked(wide,0,&row,&value)==LOGPROB_FINITE, + "the maximum itself still reads back finite"); + CHECK(close_enough(value,0.0), "the maximum of a saturated row has log-probability 0"); + + logprob_row_checked(nan_row,0,&row); + CHECK(logprob_from_row_checked(nan_row,-1,&row,&value)==LOGPROB_INVALID, + "a negative target on an invalid row -> INVALID"); +} + +/* P4: published SHA-256 answers, chosen to cover the block boundary and both + * padding branches (a message whose tail leaves no room for the length field + * needs an extra block). */ +static void t_digest(void){ + printf("P4 evidence digest matches published SHA-256 values\n"); + static const struct { const char *text; const char *hex; } vectors[] = { + { "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, + { "abc", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" }, + { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" }, + { "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmno", + "2ff100b36c386c65a1afc462ad53e25479bec9498ed00aa5a04de584bc25301b" }, + }; + char out[65]; + for(unsigned v=0;v>4]; streamed[2*i+1]=hex[raw[i]&15]; } + streamed[64]=0; + CHECK(strcmp(streamed, + "635361c48bb9eab14198e76ea8ab7f1a41685d6ad62aa9146d301d4f17eb0ae0")==0, + "a byte-at-a-time stream matches the published value"); + evidence_sha256_hex(sixty_five,65,out); + CHECK(strcmp(streamed,out)==0, "the streaming and one-shot forms agree"); +} + +int main(void){ + printf("test_logprob_status\n"); + t_agreement(); + t_precision(); + t_classification(); + t_propagation(); + t_digest(); + printf(fails ? "FAILED (%d)\n" : "PASSED (%d failures)\n", fails); + return fails ? 1 : 0; +} From 6d6d3f3cd8a1f346bdf172218dda5d857ff17d45 Mon Sep 17 00:00:00 2001 From: monotophic Date: Thu, 17 Sep 2026 20:07:56 -0400 Subject: [PATCH 3/5] perf(ablate): stop re-scanning for an already-full top-k slot buffer logit_topk_select measured 2.01x slower than dev's equivalent top-k selection at V=151552 (real GLM-5.2 vocab), topk=32. Cause: after the first k iterations fill all k slots, every one of the remaining V-k iterations still ran an O(k) linear scan of tk_id[] asking whether an empty slot exists, an answer that can only ever be 'no' from that point on -- loop-invariant work paid on almost every iteration for a result invariant after the first k. Fixed with a fill counter: the first k iterations fill slots 0..k-1 in that exact order (as the old empty-slot scan also did, since it always returns the lowest-indexed empty slot), so 'is there an empty slot' is 'have fewer than k been filled', O(1) instead of O(k). The minimum-slot scan for i>=k is unchanged. Co-Authored-By: Claude Opus 5 --- c/colibri.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/c/colibri.c b/c/colibri.c index 19cde7c7b..83d5fb9a5 100644 --- a/c/colibri.c +++ b/c/colibri.c @@ -7968,7 +7968,18 @@ static void logprob_refusal(const char *surface, unsigned long long owner, * k=2 emits token ids [2,1]. * * Exceptional rows are rejected by the classified numeric-status layer before - * this finite-only selector is reached. */ + * this finite-only selector is reached. + * + * The k slots fill in index order 0..k-1 during the first k iterations and + * never empty again, so "is there still an empty slot" is exactly "have + * fewer than k been filled" -- an O(1) counter, not the O(k) linear scan of + * tk_id[] a previous version of this loop re-ran on every one of the V-k + * remaining iterations even though it could only ever find one answer by + * then. That scan was loop-invariant work paid V-k times over for a result + * that is invariant after the first k: replacing it measurably speeds up + * the selection without changing which slot is chosen or when (verified by + * running both forms against several thousand rows, real vocab size + * included, and diffing tk_id[]/tk_val[]/the return value bit for bit). */ static int logit_topk_select(const float *lo, int V, int requested, int *tk_id, float *tk_val, int *status_out){ @@ -7985,11 +7996,12 @@ static int logit_topk_select(const float *lo, int V, int requested, return 0; } - for(int j=0;jtk_val[mn])) continue; From 53279a59593e8f87704a50c81952afe999833d79 Mon Sep 17 00:00:00 2001 From: monotophic Date: Thu, 17 Sep 2026 20:08:06 -0400 Subject: [PATCH 4/5] ci(release): package check_ablate_evidence.py, then prove it actually imports check_ablate_evidence.py is reached by nobody in coli's own import graph -- it is a person-invoked verifier, the same position k3_tokenizer.py has always been in -- so pack_python's import walk cannot carry it into the release archive on its own. Bring in this branch's release.yml hunks: copy the tool into dist/tools, assert it shipped, assert it parses. test -f and ast.parse both only look at the one file, and only at its syntax -- neither executes it, so neither catches a real import-time failure. Add a fourth gate that actually imports the packaged module from the archive root, never executes it, so a real import failure fails the job instead of shipping silently broken. test_pack_python.py's HumanOnlyToolsShipExplicitly pins both halves of this on the real tree: the import walk does not reach check_ablate_evidence.py (if it ever does, the explicit copy became redundant), and release.yml both copies and gates it, the same way it already does for k3_tokenizer.py. Co-Authored-By: Claude Opus 5 --- .github/workflows/release.yml | 20 +++++++++++++++++++ c/tests/test_pack_python.py | 37 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1ef3d3af7..72258d9be 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -136,6 +136,11 @@ jobs: # run it -- the second half of #720. mkdir -p dist/tools cp c/tools/k3_tokenizer.py dist/tools/ + # check_ablate_evidence.py verifies the ABLATE evidence that eval_glm.py + # (packaged above) produces. Nothing in the code names it -- its caller is a + # person with argparse arguments -- so pack_python's import walk cannot reach + # it, exactly like k3_tokenizer.py. Copied explicitly for the same reason. + cp c/tools/check_ablate_evidence.py dist/tools/ cp c/coli dist/ # Windows has nothing to click otherwise: `coli` is an extensionless # Python script, and the .exe files are engines that exit at once @@ -190,6 +195,9 @@ jobs: # Kimi K3 needs tokenizer.json synthesized from tiktoken.model; without this # script in the archive the engine ships but cannot be driven by coli chat. test -f tools/k3_tokenizer.py || { echo "FAIL: k3_tokenizer.py missing from archive"; exit 1; } + # Same position as k3_tokenizer.py: reachable only by a person, so only an + # explicit gate can prove it shipped. + test -f tools/check_ablate_evidence.py || { echo "FAIL: check_ablate_evidence.py missing from archive"; exit 1; } # E tutto cio' che coli raggiunge, import e sottoprocessi. Il controllo # qui sopra guarda l'unico file che il passo di copia aveva appena # scritto: conferma se stesso, ed e' il motivo per cui la v1.10.0 e' @@ -202,6 +210,18 @@ jobs: # fine. The check was wrong, not the artifact. python3 -c "import ast; ast.parse(open('tools/k3_tokenizer.py', encoding='utf-8').read())" \ || { echo "FAIL: packaged k3_tokenizer.py does not parse"; exit 1; } + python3 -c "import ast; ast.parse(open('tools/check_ablate_evidence.py', encoding='utf-8').read())" \ + || { echo "FAIL: packaged check_ablate_evidence.py does not parse"; exit 1; } + # test -f and ast.parse both only look at this one file, and only at its + # syntax -- neither one executes it, so neither catches a real import-time + # failure: an undefined name at module scope, a decorator that raises, or + # (were a local import ever reintroduced here) a dependency pack_python's + # walk did not carry into the archive. Import it for real, from the archive + # root, the same way a person running the shipped `coli` would reach it -- + # never execute it, an import is enough to prove every name it needs at + # module load resolves. + python3 -c "import tools.check_ablate_evidence" \ + || { echo "FAIL: packaged check_ablate_evidence.py does not import"; exit 1; } COLI_EXPECT="$SIBLINGS" python3 - <<'PYCHK' import importlib.machinery, importlib.util, json, os, sys, tempfile from family_registry import all_families diff --git a/c/tests/test_pack_python.py b/c/tests/test_pack_python.py index dc0aaaccd..9a8ea9665 100644 --- a/c/tests/test_pack_python.py +++ b/c/tests/test_pack_python.py @@ -249,5 +249,42 @@ def test_the_summary_line_counts_data_files_separately(self): self.assertIn("1 Python files and 1 data files", buf.getvalue()) +class HumanOnlyToolsShipExplicitly(unittest.TestCase): + """Tools that only a person invokes are invisible to the import walk. + + `pack_python.py` computes the archive from what `coli` reaches, imports and + subprocesses alike. A script whose caller is a person typing argparse + arguments is reached by nobody in the code, so static analysis cannot put + it in the archive -- `k3_tokenizer.py` has always been in that position and + `release.yml` copies it by hand. `check_ablate_evidence.py` joined it: it + verifies the ABLATE evidence that the packaged `eval_glm.py` produces, so a + release that ships the producer without the verifier is half a tool. + + This pins both halves of that decision on the real tree: the walk does NOT + reach the file (if it ever does, the explicit copy became redundant and this + test should be retired), and the workflow copies AND gates it, the way it + does for the tokenizer helper. A copy without a gate is how v1.10.0 shipped + four broken commands green (#1296).""" + + HUMAN_ONLY = ("k3_tokenizer.py", "check_ablate_evidence.py") + + def test_the_walk_does_not_reach_them(self): + paths = PACK.needed(HERE.parent) + for name in self.HUMAN_ONLY: + self.assertTrue((TOOLS / name).is_file(), name) + self.assertNotIn(TOOLS / name, paths, + f"{name} is now reached by the walk; drop its " + f"explicit copy from release.yml and this pin") + + def test_release_copies_and_gates_them(self): + release = (HERE.parent.parent / ".github" / "workflows" + / "release.yml").read_text(encoding="utf-8") + for name in self.HUMAN_ONLY: + self.assertIn(f"cp c/tools/{name} dist/tools/", release, + f"release.yml no longer copies {name}") + self.assertIn(f"test -f tools/{name} ||", release, + f"release.yml copies {name} but does not gate on it") + + if __name__ == "__main__": unittest.main() From 14f299c9daf25dca56be23ea3f3caa1f1705dd29 Mon Sep 17 00:00:00 2001 From: monotophic Date: Thu, 17 Sep 2026 20:08:20 -0400 Subject: [PATCH 5/5] fix(ablate): derive the per-item token cap from CTX, bound all three allocations Re-review found that documenting the prefill buffer this cap did not bound undersold the problem twice. First, docs/ENVIRONMENT.md asserted the hazard closed ('so that a corrupt manifest cannot ask the loader for an arbitrary allocation') while this file's own comment said the opposite; a doc our own comment refutes is worse than the narrower true statement it replaced. Second, measurement found THREE unbounded allocations behind this one constant, not one: kv_alloc's per-layer KV buffers (177.75 GiB at the old 2^20 cap, GLM-5.2 shape), the prefill buffer x (24.00 GiB, the one originally named), and this loop's own resident manifest bookkeeping (240 B/item, unbounded in item count). kv_alloc is 7.41x the prefill buffer, not the smaller of the two. Given the doc already claims the hazard is closed, the honest fix is to make the claim true rather than narrow it further. The bound: a manifest item's T is now checked against min(CTX, ABLATE_MAX_ITEM_TOKENS) via ablate_item_token_limit(), not the bare 2^20 constant. CTX ('Maximum context length (tokens) the KV cache is sized for', docs/ENVIRONMENT.md) is an existing engine limit already read by kv_pool_bytes and expert_avail for exactly this kind of sizing -- no new constant, no new environment variable. An item longer than the context this engine is configured to hold cannot be processed regardless of what a manifest asks for. The check still runs before the item can contribute to any of the three allocations; the backstop constant remains in case CTX itself is set to something absurd. Two tests, both required so the bound is proven exact, not merely strict: a manifest at cap+1 (4097 under the CTX=4096 default) is refused by name; a manifest at the real maximum (4096) loads and runs to completion -- without this second case, a bound that refused everything would also pass the first. Co-Authored-By: Claude Opus 5 --- c/colibri.c | 77 ++++++++++++++++++++++++++++++++------ c/tests/test_ablate_mode.c | 54 ++++++++++++++++++++++++++ docs/ENVIRONMENT.md | 4 +- 3 files changed, 122 insertions(+), 13 deletions(-) diff --git a/c/colibri.c b/c/colibri.c index 83d5fb9a5..6dd62cfc6 100644 --- a/c/colibri.c +++ b/c/colibri.c @@ -7938,12 +7938,63 @@ static void run_score(Model *m, const char *snap, const char *path){ * never leak into the next). NLL/margin/correctness are exact; top-K is for a * paired approximate next-token KL on the host. */ #define ABL_LOGIT_TOPK 32 -/* Upper bound on one manifest item's declared token count. A teacher-forced - * ablation item is a prompt plus a short continuation; a million tokens is - * orders of magnitude above anything a study uses, and it keeps a hostile or - * corrupt manifest from asking the loader for an arbitrary allocation. */ +/* Backstop ceiling on one manifest item's declared token count T. The + * operative bound applied in ablate_manifest_load (below) is + * ablate_item_token_limit() = min(CTX, ABLATE_MAX_ITEM_TOKENS): CTX + * ("Maximum context length (tokens) the KV cache is sized for", + * docs/ENVIRONMENT.md -- the same max_ctx parameter kv_pool_bytes and + * expert_avail already take) is an existing engine limit, not an + * invented one, and an item longer than the context this engine is + * configured to hold cannot be processed regardless of what the + * manifest asks for. This constant only bounds CTX itself, in case an + * operator sets it to something absurd. + * + * That check runs before this item is allowed to contribute to any of + * THREE allocations, largest first -- naming only the one the maintainer + * raised, or getting the ranking backwards, is how an earlier draft of + * this comment was wrong twice over: + * + * 1. kv_alloc's per-layer KV buffers, sized from the manifest-wide + * maxT once parsing finishes (`(n_layers+1)*max_t*(kv_lora+qk_rope)`, + * below; maxT can equal this item's own T). + * 2. ablate_model_output_body's prefill buffer `x`, also sized from + * maxT (`falloc((int64_t)maxT*D)`, D = c->hidden, further below). + * 3. This loop's own resident bookkeeping: one AblateManifestItem plus + * one AblateItemRef appended per accepted item (both defined + * above) -- bounded per item by this same check, but unbounded in + * ITEM COUNT across a whole manifest, a separate axis this cap + * does not close. + * + * Worked at the backstop ceiling ABLATE_MAX_ITEM_TOKENS = 2^20 (reachable + * only if CTX is itself set that high), with GLM-5.2's real config + * (Cfg.hidden=6144, n_layers=78, kv_lora=512, qk_rope=64 -- all loaded + * together in load_cfg at c/colibri.c:1681): + * + * 1. kv_alloc: (n_layers+1) * maxT * (kv_lora+qk_rope) * sizeof(float) + * = 79 * 2^20 * 576 * 4 B = 190,857,609,216 B = 177.75 GiB + * 2. prefill x: maxT * hidden * sizeof(float) + * = 2^20 * 6144 * 4 B = 25,769,803,776 B = 24.00 GiB + * 3. resident: sizeof(AblateManifestItem) + sizeof(AblateItemRef) + * = 224 B + 16 B = 240 B per item (illustrative only -- + * unbounded in item count, not item length) + * + * kv_alloc is 7.41x the prefill buffer, not the smaller of the two. */ #define ABLATE_MAX_ITEM_TOKENS (1<<20) +/* See the comment above: the bound actually applied to one manifest + * item's T is the smaller of CTX and ABLATE_MAX_ITEM_TOKENS, not the + * backstop constant alone. Reads CTX itself (not a cached max_ctx) + * because ablate mode has no serving session to inherit one from; same + * default (4096) and same unvalidated-atoi parsing as every other CTX + * reader in this file, for one consistent meaning of "context length" + * throughout. */ +static int64_t ablate_item_token_limit(void){ + const char *env=getenv("CTX"); + int ctx = env ? atoi(env) : 4096; + if(ctx<=0) ctx=4096; + return ctx