diff --git a/README.md b/README.md index f0973a5..498809b 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Validation reports the first error in each record, with its file and line number Available block files must parse completely and match their transaction merkle roots and applicable witness commitments. CI checks output-value overflow, forward transaction spends and excessive sigop cost directly. -Sigops checks use a verified cache in `.cache/prevouts/`, restored between GitHub Actions runs. +Sigops and missing-parent checks use a verified cache in `.cache/prevouts/`, restored between GitHub Actions runs. Missing entries are fetched from public Esplora-compatible APIs when `--fetch-prevouts` is supplied. API failures, missing evidence and corrupt cached transactions fail validation. After filling the cache, omit the flag for an offline run; `--prevouts-dir` selects another cache and `--api-url` selects an API base. diff --git a/blocks/474294-00000000000000000182acdf5657c93a0769dc6f9004047496b2e15efc6a4232.bin b/blocks/474294-00000000000000000182acdf5657c93a0769dc6f9004047496b2e15efc6a4232.bin new file mode 100644 index 0000000..6e2276f Binary files /dev/null and b/blocks/474294-00000000000000000182acdf5657c93a0769dc6f9004047496b2e15efc6a4232.bin differ diff --git a/ci/block_evidence.py b/ci/block_evidence.py index 1a6e3b5..2708afd 100644 --- a/ci/block_evidence.py +++ b/ci/block_evidence.py @@ -88,6 +88,23 @@ def read_block(data: bytes) -> CBlock: return block +def omitted_prevouts(transactions: Sequence[CTransaction]) -> list[tuple[bytes, int]]: + """Return external prevouts spent by non-coinbase inputs, as (txid, vout).""" + own = {tx.GetTxid() for tx in transactions} + return [(txin.prevout.hash, txin.prevout.n) + for tx in transactions[1:] for txin in tx.vin if txin.prevout.hash not in own] + + +def confirmed_at_or_after(confirmation: tuple[int, str], height: int, block_hash: str) -> bool: + """True if the parent is now confirmed at height or later in a block other than block_hash. + + It was then not in the chain below the candidate, so its output did not + exist at the tip of a canonical previous block. This is not a UTXO lookup. + """ + confirmed_height, confirmed_hash = confirmation + return confirmed_height >= height and confirmed_hash != block_hash + + def establishes_rule(block: CBlock, rule: str) -> bool: """Recognize only failures provable from these committed transactions. diff --git a/ci/prevouts.py b/ci/prevouts.py index 0473a47..ef991b9 100644 --- a/ci/prevouts.py +++ b/ci/prevouts.py @@ -1,107 +1,125 @@ -"""Acquire previous transactions through Esplora-compatible public APIs. +"""Fetch previous transactions, parent confirmations and canonical block hashes. -Cache entries contain stripped transaction bytes, sufficient to authenticate -output scripts against the txid committed in a spending input. Every cache hit -is verified too. Downloads and cache corruption fail explicitly; there is no -fallback to a claimed reject string or an unauthenticated scriptPubKey. +`{txid}.bin` is a stripped transaction, checked against its txid. +`{txid}.status.json` is the Esplora status reply for that transaction, and +`height-{n}.hash` is the block hash Esplora reports at height n. Cache hits +are decoded and checked like downloads. Failed downloads and corrupt files +fail the run, and a reply is stored only after it decodes as evidence, so an +unconfirmed status is never cached. """ -from collections.abc import Sequence +from collections.abc import Callable, Sequence from concurrent.futures import ThreadPoolExecutor, as_completed from http.client import HTTPException, IncompleteRead +import json from pathlib import Path +import re import time import tempfile +from typing import TypeVar from urllib.request import Request, urlopen from bitcoin.core import CTransaction, b2lx -from block_evidence import read_transaction +from block_evidence import omitted_prevouts, read_transaction DEFAULT_APIS = ("https://mempool.space/api", "https://blockstream.info/api") PREVOUTS_DIR = Path(".cache/prevouts") +BLOCK_HASH = re.compile(r"[0-9a-fA-F]{64}") +T = TypeVar("T") -def decode_previous(data: bytes, txid: str) -> CTransaction: - """Authenticate a complete transaction against the requested display txid.""" - transaction = read_transaction(data) - if b2lx(transaction.GetTxid()) != txid: - raise ValueError(f"previous transaction identity mismatch: {txid}") - return transaction +def _get(url: str, limit: int) -> bytes: + """Download at most limit bytes. A truncated body raises.""" + request = Request(url, headers={"User-Agent": "invalid-blocks-evidence-check/1"}) + with urlopen(request, timeout=30) as response: + raw = response.read(limit + 1) + if len(raw) > limit: + raise ValueError(f"oversized response from {url}") + # urlopen can return a short body without raising. response.length is the + # remaining Content-Length. + remaining = getattr(response, "length", None) + if remaining: + raise IncompleteRead(raw, remaining) + return raw -def fetch_previous(txid: str, apis: Sequence[str] = DEFAULT_APIS) -> CTransaction: - """Fetch raw hex with bounded retries, provider fallback and HTTP timeouts. - - Two workers call this function in parallel. Successful downloads are paced - and rate-limit/transient failures back off. Identity failures are terminal: - trying another provider must not conceal a wrong transaction response. - """ +def _fetch(what: str, apis: Sequence[str], path: str, limit: int) -> bytes: + """Try each API up to twice on transport errors and return the first body.""" failures = [] for api in apis: - url = f"{api.rstrip('/')}/tx/{txid}/hex" + url = f"{api.rstrip('/')}/{path}" for attempt in range(2): - request = Request(url, headers={"User-Agent": "invalid-blocks-evidence-check/1"}) try: - with urlopen(request, timeout=30) as response: - # Raw transaction size is bounded for this evidence path; - # the limit also catches oversized error responses. - raw = response.read(8_000_001) - if len(raw) > 8_000_000: - raise ValueError(f"oversized previous transaction response: {txid}") - # read(amt) can return early without raising on a truncated - # Content-Length response. HTTPResponse tracks bytes still due. - remaining = getattr(response, "length", None) - if remaining: - raise IncompleteRead(raw, remaining) - transaction = decode_previous(bytes.fromhex(raw.decode("ascii").strip()), txid) + raw = _get(url, limit) time.sleep(0.25) - return transaction + return raw except (OSError, HTTPException) as exc: # urllib wraps connection setup errors, but resets and short # HTTP reads can escape unwrapped after the response starts. failures.append(f"{url}: {exc}") if attempt == 0: time.sleep(2) - raise ValueError(f"could not download previous transaction {txid}: " + "; ".join(failures)) + raise ValueError(f"could not download {what}: " + "; ".join(failures)) + + +def _replace(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as handle: + temporary = Path(handle.name) + try: + handle.write(data) + handle.close() + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def _cached(what: str, path: Path, decode: Callable[[bytes], T], fetch: bool, + download: Callable[[], bytes]) -> T: + """Decode the cached file, or with fetch download it, decode it and then store it.""" + if path.exists(): + return decode(path.read_bytes()) + if not fetch: + raise ValueError(f"missing cached {what}; run with --fetch-prevouts") + raw = download() + value = decode(raw) + _replace(path, raw) + return value + + +def decode_previous(data: bytes, txid: str) -> CTransaction: + """Parse the transaction and require its txid to match.""" + transaction = read_transaction(data) + if b2lx(transaction.GetTxid()) != txid: + raise ValueError(f"previous transaction identity mismatch: {txid}") + return transaction + + +def fetch_previous(txid: str, apis: Sequence[str] = DEFAULT_APIS) -> CTransaction: + """Download the raw transaction. A txid mismatch is terminal, not retried.""" + raw = _fetch(f"previous transaction {txid}", apis, f"tx/{txid}/hex", 8_000_000) + return decode_previous(bytes.fromhex(raw.decode("ascii").strip()), txid) + + +def load_transaction(txid: str, cache_dir: Path | str = PREVOUTS_DIR, fetch: bool = False, + apis: Sequence[str] = DEFAULT_APIS) -> CTransaction: + """Return the stripped transaction for one hex txid, from cache or the API.""" + return _cached(f"previous transaction {txid}", Path(cache_dir) / f"{txid}.bin", + lambda data: decode_previous(data, txid), fetch, + lambda: fetch_previous(txid, apis).serialize({"include_witness": False})) def load_previous(transactions: Sequence[CTransaction], cache_dir: Path | str = PREVOUTS_DIR, fetch: bool = False, apis: Sequence[str] = DEFAULT_APIS) -> dict[bytes, CTransaction]: - """Resolve external txids; earlier in-block outputs are handled by sigops. - - The complete requested set is required for an exact count. Offline mode - reports missing evidence. Online mode writes only verified stripped bytes - using atomic replacement, so interrupted downloads cannot poison the cache. - """ - own = {tx.GetTxid() for tx in transactions} - needed = sorted({b2lx(txin.prevout.hash) for tx in transactions[1:] - for txin in tx.vin if txin.prevout.hash not in own}) + """Load every external prevout transaction; spends of earlier in-block txs are not fetched.""" + needed = sorted({b2lx(txid) for txid, _ in omitted_prevouts(transactions)}) cache_dir = Path(cache_dir) - - def load(txid: str) -> CTransaction: - path = cache_dir / f"{txid}.bin" - if path.exists(): - return decode_previous(path.read_bytes(), txid) - if not fetch: - raise ValueError(f"missing cached previous transaction {txid}; run with --fetch-prevouts") - transaction = fetch_previous(txid, apis) - cache_dir.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile(dir=cache_dir, delete=False) as handle: - temporary = Path(handle.name) - try: - handle.write(transaction.serialize({"include_witness": False})) - handle.close() - temporary.replace(path) - finally: - temporary.unlink(missing_ok=True) - return transaction - if fetch: missing = sum(not (cache_dir / f"{txid}.bin").exists() for txid in needed) print(f"Previous transactions: {len(needed)} required, {missing} to download", flush=True) pool = ThreadPoolExecutor(max_workers=2) - futures = [pool.submit(load, txid) for txid in needed] + futures = [pool.submit(load_transaction, txid, cache_dir, fetch, apis) for txid in needed] result = {} try: for future in as_completed(futures): @@ -113,3 +131,41 @@ def load(txid: str) -> CTransaction: # Do not keep downloading the remaining catalogue after an error. pool.shutdown(wait=True, cancel_futures=True) return result + + +def decode_confirmation(data: bytes, txid: str) -> tuple[int, str]: + """Read an Esplora status reply as (height, block hash); unconfirmed is not evidence.""" + try: + payload = json.loads(data) + except ValueError as exc: + raise ValueError(f"malformed confirmation: {txid}") from exc + if not isinstance(payload, dict) or payload.get("confirmed") is not True: + raise ValueError(f"parent transaction {txid} is not confirmed") + height, block_hash = payload.get("block_height"), payload.get("block_hash") + if type(height) is not int or height < 0 or not isinstance(block_hash, str) or not BLOCK_HASH.fullmatch(block_hash): + raise ValueError(f"malformed confirmation: {txid}") + return height, block_hash.lower() + + +def load_confirmation(txid: str, cache_dir: Path | str = PREVOUTS_DIR, fetch: bool = False, + apis: Sequence[str] = DEFAULT_APIS) -> tuple[int, str]: + """Return where the API reported txid confirmed, as (height, block hash).""" + what = f"confirmation {txid}" + return _cached(what, Path(cache_dir) / f"{txid}.status.json", lambda data: decode_confirmation(data, txid), + fetch, lambda: _fetch(what, apis, f"tx/{txid}/status", 65_536)) + + +def decode_block_hash(data: bytes, height: int) -> str: + """Read a block-height reply as a lowercase block hash.""" + text = data.decode("ascii", errors="replace").strip() + if not BLOCK_HASH.fullmatch(text): + raise ValueError(f"malformed block hash for height {height}") + return text.lower() + + +def load_canonical_hash(height: int, cache_dir: Path | str = PREVOUTS_DIR, fetch: bool = False, + apis: Sequence[str] = DEFAULT_APIS) -> str: + """Return the block hash the API currently reports at height.""" + what = f"block hash for height {height}" + return _cached(what, Path(cache_dir) / f"height-{height}.hash", lambda data: decode_block_hash(data, height), + fetch, lambda: _fetch(what, apis, f"block-height/{height}", 256)) diff --git a/ci/sanity-check.py b/ci/sanity-check.py index e0895cd..8375d53 100644 --- a/ci/sanity-check.py +++ b/ci/sanity-check.py @@ -16,24 +16,27 @@ from typing import Any from urllib.parse import urlparse -from bitcoin.core import CBlock, CBlockHeader, b2lx +from bitcoin.core import CBlock, CBlockHeader, b2lx, lx from bitcoin.core._bignum import vch2bn from bitcoin.core.script import CScript, CScriptInvalidError, OP_1NEGATE from bitcoin.core.serialize import uint256_from_compact from block_evidence import ( - MAX_BLOCK_SIGOPS_COST, establishes_rule, read_block, - sigop_cost, verify_witness_commitment, + MAX_BLOCK_SIGOPS_COST, confirmed_at_or_after, establishes_rule, omitted_prevouts, + read_block, sigop_cost, verify_witness_commitment, +) +from prevouts import ( + DEFAULT_APIS, PREVOUTS_DIR, load_canonical_hash, load_confirmation, load_previous, load_transaction, ) -from prevouts import DEFAULT_APIS, PREVOUTS_DIR, load_previous DATA_PATH = Path("data/invalid-blocks.jsonl") BLOCKS_DIR = Path("blocks") REQUIRED = {"height", "hash", "header", "prev_hash", "nTime", "core_reject_reason", "rule"} CONTEXT_FIELDS = { "expected_nbits", "parent_mtp", "coinbase_height", "coinbase_scriptsig_hex", - "pool", "parent_kind", + "pool", "parent_kind", "missing_prevout", } +OUTPOINT = re.compile(r"[0-9a-f]{64}:(?:0|[1-9][0-9]*)") OBSERVATION_REQUIRED = {"channel", "source", "provenance"} CHILD_FIELDS = {"child_chain", "child_height", "child_block_hash", "child_block_time", "child_header"} OBSERVATION_FIELDS = OBSERVATION_REQUIRED | CHILD_FIELDS | {"first_seen"} @@ -41,13 +44,15 @@ PARENT_KINDS = {"canonical", "stale", "invalid"} POW_LIMIT = 0xFFFF << (8 * (0x1D - 3)) -# Evidence paths: local = checked header/context predicate; body = a complete, -# merkle-bound body proving the failure; sigops = body plus authenticated prevouts. -# Keep the rule names and reject strings aligned with docs/schema.md. +# Evidence paths: local = header/context only; body = complete block file; +# sigops = body plus previous transactions; missing_parent = body plus API +# evidence for the recorded outpoint. Rule names and reject strings must +# match docs/schema.md. RULES = { "bad-txns-vout-toolarge": ("bad-txns-vout-toolarge", (), "body"), "bad-blk-sigops": ("bad-blk-sigops", (), "sigops"), "bad-txns-inputs-missingorspent": ("bad-txns-inputs-missingorspent", (), "body"), + "missing_unconfirmed_parent": ("bad-txns-inputs-missingorspent", ("missing_prevout",), "missing_parent"), "bip34_v2_coinbase_height_mismatch": ( "bad-cb-height", ("coinbase_height", "coinbase_scriptsig_hex"), "local"), "bip34_coinbase_height_mismatch": ( @@ -158,6 +163,9 @@ def check_context(record: dict[str, Any]) -> None: hex_value(details, name, size) if "pool" in details: string(details, "pool") + if "missing_prevout" in details and not ( + isinstance(details["missing_prevout"], str) and OUTPOINT.fullmatch(details["missing_prevout"])): + raise ValueError("missing_prevout must be txid:vout in lowercase hex") if "parent_kind" in details and details["parent_kind"] not in tuple(PARENT_KINDS): raise ValueError(f"parent_kind must be one of {sorted(PARENT_KINDS)}") required = set(RULES[record["rule"]][1]) @@ -229,11 +237,10 @@ def height_prefix(height: int) -> bytes: def check_local_evidence(record: dict[str, Any], header: CBlockHeader) -> None: - """Check the claimed failure against already type-checked header/context. + """Check the named rule against header and context already in the record. - This proves consistency with supplied height, MTP and expected difficulty; - it does not authenticate those facts against the parent chain or bind an - extracted coinbase script to the header without a complete body. + Parent MTP, expected nBits and parent_kind are not looked up on the chain. + Without a body, a coinbase scriptSig cannot be bound to the header. """ rule = record["rule"] context = record.get("context", {}) @@ -279,6 +286,23 @@ def check_failure_evidence(record: dict[str, Any], block: CBlock | None, prevout raise ValueError(f"{record['rule']} requires a complete block body") if mode == "body" and not establishes_rule(block, record["rule"]): raise ValueError(f"committed body does not demonstrate {record['rule']}") + if mode == "missing_parent": + txid, vout = record["context"]["missing_prevout"].split(":") + if (lx(txid), int(vout)) not in omitted_prevouts(block.vtx): + raise ValueError("missing_prevout is not spent by the body from outside the block") + previous_height = record["height"] - 1 + canonical = load_canonical_hash(previous_height, prevouts_dir, fetch_prevouts, apis) + if canonical != record["prev_hash"]: + raise ValueError(f"prev_hash is not the canonical block at height {previous_height}") + parent = load_transaction(txid, prevouts_dir, fetch_prevouts, apis) + if int(vout) >= len(parent.vout): + raise ValueError(f"missing_prevout output {vout} does not exist in the parent transaction") + confirmation = load_confirmation(txid, prevouts_dir, fetch_prevouts, apis) + if not confirmed_at_or_after(confirmation, record["height"], record["hash"]): + raise ValueError("parent transaction is not confirmed at this height or later in another block") + if fetch_prevouts: + print(f"{record['height']}: parent {txid} currently confirmed at {confirmation[0]} " + f"in {confirmation[1]}; canonical block at {previous_height} is {canonical}", flush=True) if mode == "sigops": # This checker uses BIP16 + BIP141 counting, not pre-SegWit rules. if record["height"] < 481824: @@ -365,7 +389,8 @@ def check_dataset(path: Path | str = DATA_PATH, blocks_dir: Path | str = BLOCKS_ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--fetch-prevouts", action="store_true", help="fetch missing sigops prevout evidence from public APIs") + parser.add_argument("--fetch-prevouts", action="store_true", + help="fetch missing sigops prevouts, parent confirmations and canonical block hashes from public APIs") parser.add_argument("--prevouts-dir", type=Path, default=PREVOUTS_DIR, help="verified transaction cache directory") parser.add_argument("--api-url", action="append", help="Esplora API base URL; repeat for fallback providers") args = parser.parse_args() diff --git a/ci/test_block_evidence.py b/ci/test_block_evidence.py index 6a1e59a..c6eef08 100644 --- a/ci/test_block_evidence.py +++ b/ci/test_block_evidence.py @@ -6,7 +6,8 @@ from bitcoin.core.script import CScript, CScriptWitness, OP_TRUE from block_evidence import ( - MAX_MONEY, establishes_rule, read_block, sha256d, sigop_count, witness_sigops, + MAX_MONEY, confirmed_at_or_after, establishes_rule, omitted_prevouts, read_block, sha256d, + sigop_count, witness_sigops, ) @@ -47,6 +48,19 @@ def test_forward_spend_requires_later_transaction_and_existing_output(self): self.assertFalse(establishes_rule(read_block(block(coinbase, bad_index, producer)), rule)) self.assertFalse(establishes_rule(read_block(block(coinbase, producer)), rule)) + def test_missing_parent_boundaries(self): + """Only outside-block spends are omitted prevouts; the parent must be confirmed at this height or later elsewhere.""" + coinbase = transaction() + parent = transaction(prev_hash=b"\x11" * 32, vout=0) + consumer = transaction(prev_hash=sha256d(parent[1]), vout=0) + self.assertEqual(omitted_prevouts(read_block(block(coinbase, consumer)).vtx), [(sha256d(parent[1]), 0)]) + self.assertEqual(omitted_prevouts(read_block(block(coinbase, parent, consumer)).vtx), [(b"\x11" * 32, 0)]) + candidate = "ab" * 32 + for confirmation, expected in (((99, "cd" * 32), False), ((100, "cd" * 32), True), + ((101, "cd" * 32), True), ((100, candidate), False)): + with self.subTest(confirmation=confirmation): + self.assertEqual(confirmed_at_or_after(confirmation, 100, candidate), expected) + def test_invalid_block_serialization(self): """Reject malformed bodies, transaction commitments and witness encodings.""" tx = transaction() diff --git a/ci/test_prevouts.py b/ci/test_prevouts.py index 8cf787c..a0fd2ba 100644 --- a/ci/test_prevouts.py +++ b/ci/test_prevouts.py @@ -2,13 +2,14 @@ from io import BytesIO from pathlib import Path +import json import tempfile import unittest from unittest.mock import patch from urllib.error import URLError from block_evidence import read_transaction, sha256d -from prevouts import decode_previous, fetch_previous, load_previous +from prevouts import decode_previous, fetch_previous, load_canonical_hash, load_confirmation, load_previous from test_block_evidence import transaction @@ -65,6 +66,28 @@ def test_cache_lifecycle(self, sleep): with self.subTest(case="corrupt entry"), self.assertRaisesRegex(ValueError, "identity mismatch"): load_previous(txs, self.cache, fetch=True) + @patch("prevouts.time.sleep") + def test_unconfirmed_status_is_not_evidence_or_cached(self, sleep): + """Decode a confirmed status reply; an unconfirmed one fails and is not written to the cache.""" + path = self.cache / f"{self.txid}.status.json" + with patch("prevouts.urlopen", return_value=BytesIO(b'{"confirmed":false}')): + with self.assertRaisesRegex(ValueError, "not confirmed"): + load_confirmation(self.txid, self.cache, fetch=True) + self.assertFalse(path.exists()) + confirmed = json.dumps({"confirmed": True, "block_height": 10, "block_hash": "AB" * 32}).encode() + with patch("prevouts.urlopen", return_value=BytesIO(confirmed)): + self.assertEqual(load_confirmation(self.txid, self.cache, fetch=True), (10, "ab" * 32)) + + @patch("prevouts.time.sleep") + def test_malformed_block_hash_is_not_cached(self, sleep): + """Accept only a 64-hex block-height reply; anything else fails and is not written to the cache.""" + with patch("prevouts.urlopen", return_value=BytesIO(b"Block not found")): + with self.assertRaisesRegex(ValueError, "malformed block hash"): + load_canonical_hash(5, self.cache, fetch=True) + self.assertFalse((self.cache / "height-5.hash").exists()) + with patch("prevouts.urlopen", return_value=BytesIO(b"AB" * 32)): + self.assertEqual(load_canonical_hash(5, self.cache, fetch=True), "ab" * 32) + if __name__ == "__main__": unittest.main() diff --git a/ci/test_sanity_check.py b/ci/test_sanity_check.py index 0f9f475..0a0c115 100644 --- a/ci/test_sanity_check.py +++ b/ci/test_sanity_check.py @@ -36,10 +36,10 @@ def copy_body(self, record): path.write_bytes((CHECK.BLOCKS_DIR / name).read_bytes()) return path - def validate(self, records=None): + def validate(self, records=None, prevouts_dir=CHECK.PREVOUTS_DIR): path = self.root / "data.jsonl" path.write_text("".join(json.dumps(record) + "\n" for record in (records or [self.record]))) - return CHECK.check_dataset(path, self.root / "blocks")[0] + return CHECK.check_dataset(path, self.root / "blocks", prevouts_dir)[0] def test_documented_sigops_breakdowns(self): """Reproduce both F2Pool blocks' documented legacy, P2SH and witness costs.""" @@ -119,6 +119,26 @@ def test_sigops_requires_previous_transactions(self): problems, _ = CHECK.check_dataset(path, self.root / "blocks", self.root / "empty-cache") self.assertTrue(any("missing cached previous transaction" in p for p in problems)) + def test_missing_parent_requires_recorded_outpoint_and_cached_evidence(self): + """The body must spend the recorded outpoint, the previous block must be canonical, and evidence must be cached.""" + self.record = self.for_rule("missing_unconfirmed_parent") + self.copy_body(self.record) + cache = self.root / "cache" + cache.mkdir() + canonical = cache / f"height-{self.record['height'] - 1}.hash" + with self.subTest(case="malformed outpoint"), patch.dict(self.record["context"], {"missing_prevout": "abc"}): + self.assertTrue(any("missing_prevout must be" in p for p in self.validate(prevouts_dir=cache))) + with self.subTest(case="outpoint not spent"), patch.dict(self.record["context"], {"missing_prevout": "00" * 32 + ":0"}): + self.assertTrue(any("not spent by the body" in p for p in self.validate(prevouts_dir=cache))) + with self.subTest(case="no cached canonical hash"): + self.assertTrue(any("missing cached block hash" in p for p in self.validate(prevouts_dir=cache))) + canonical.write_text("00" * 32) + with self.subTest(case="previous block not canonical"): + self.assertTrue(any("not the canonical block" in p for p in self.validate(prevouts_dir=cache))) + canonical.write_text(self.record["prev_hash"]) + with self.subTest(case="no cached parent transaction"): + self.assertTrue(any("missing cached previous transaction" in p for p in self.validate(prevouts_dir=cache))) + def test_sigops_limit_and_supported_activation(self): """Require cost above 80000 and reject pre-SegWit records before fetching evidence.""" self.record = self.for_rule("bad-blk-sigops") diff --git a/data/invalid-blocks.jsonl b/data/invalid-blocks.jsonl index 83d3fab..44cce1c 100644 --- a/data/invalid-blocks.jsonl +++ b/data/invalid-blocks.jsonl @@ -31,6 +31,7 @@ {"height":389043,"hash":"00000000000000000306ea979ad487157d2950081413eb9d2dca82060f1b89b2","header":"030000009b8df38b440b4e93de66f1bc04aafd774d9b0f5eada3430900000000000000006e552de3422d394c0ef86c00bfd46e1b90ecc55f2a8c163ff9760d1850d6b782fc1374564fe60d18581bca4e","prev_hash":"00000000000000000943a3ad5e0f9b4d77fdaa04bcf166de934e0b448bf38d9b","nTime":1450447868,"core_reject_reason":"bad-version","rule":"bip65_block_version_below_4","context":{"expected_nbits":"180de64f","coinbase_height":389043,"coinbase_scriptsig_hex":"03b3ef05040314745608fabe6d6d20d6656f5ca5ac4408cbee6ff6d3235c6039015372788a3c0b0c39721502efd001000000000000004000000f873d00860d2f6e6f64655374726174756d2f","pool":"nodeStratum","parent_kind":"canonical"},"observations":[{"channel":"merge_mining","source":"merge-mining-research","child_chain":"namecoin","child_height":263217,"provenance":"https://github.com/deadmanoz/merge-mining-research/blob/f543b1f23c57be6840c7a2a44a59306b5d0180f7/data/error-blocks/error_block_observations.csv#L69","child_block_hash":"20d6656f5ca5ac4408cbee6ff6d3235c6039015372788a3c0b0c39721502efd0","child_block_time":1450447692},{"channel":"merge_mining","source":"mergedmonitor","child_chain":"namecoin","provenance":"https://github.com/NStifter/mergedmonitor/blob/54344d4e355f73eb94bef8d391e8fb6e4a9323a6/fork-analysis/mergedmonitor/mergedmonitor.json"},{"channel":"p2p","source":"kit-mon1","first_seen":1450447890,"provenance":"https://github.com/NStifter/mergedmonitor/blob/54344d4e355f73eb94bef8d391e8fb6e4a9323a6/fork-analysis/kit/mon1.json.tar.gz"},{"channel":"p2p","source":"kit-mon2","first_seen":1450447889,"provenance":"https://github.com/NStifter/mergedmonitor/blob/54344d4e355f73eb94bef8d391e8fb6e4a9323a6/fork-analysis/kit/mon2.json.tar.gz"}]} {"height":402610,"hash":"000000000000000003a1ce220ae97419cc4bdb5d70b90189b8f8a06b0b37e3a2","header":"04000000d29de83f75a5660bdb41268113ef1641ef823386119b6e0200000000000000008d1db810ec599011afd5752e00d557a6875094e9436cf1fd9412e463f37cdf59a9a4e656a8f00618c2eac184","prev_hash":"0000000000000000026e9b11863382ef4116ef13812641db0b66a5753fe89dd2","nTime":1457956009,"core_reject_reason":"bad-cb-height","rule":"bip34_coinbase_height_mismatch","context":{"expected_nbits":"1806f0a8","coinbase_height":402611,"coinbase_scriptsig_hex":"03b3240637e4b883e5bda9e7a59ee4bb99e9b1bcdb8c127393c82df3adac41a9a3a6d30491ade6cab548308a174cc92165515c1f02000000f09f909f124d696e656420627920633633333436313039000000000000000000000000000000000000000000","pool":"c63346109","parent_kind":"stale"},"observations":[{"channel":"merge_mining","source":"merge-mining-research","child_chain":"namecoin","child_height":276713,"provenance":"https://github.com/deadmanoz/merge-mining-research/blob/f543b1f23c57be6840c7a2a44a59306b5d0180f7/data/error-blocks/error_block_observations.csv#L70","child_block_hash":"229b2fa5af33af86559c830b86d4cc90830834e06612e43f0b0f8e8636d6f39a","child_block_time":1457955815,"child_header":"040101009eae642e432113e560f4bb4c8bda6e59fb9f79d1cd2486ba53909535bffff325e6c52c32ce27a17151ee2c7a64bdd3a441f6237ba4e39c7b62089d7db8885abce7a3e656e24b171800000000"},{"channel":"merge_mining","source":"mergedmonitor","child_chain":"namecoin","provenance":"https://github.com/NStifter/mergedmonitor/blob/54344d4e355f73eb94bef8d391e8fb6e4a9323a6/fork-analysis/mergedmonitor/mergedmonitor.json"}]} {"height":422059,"hash":"00000000000000000254ed1e8143f0bcd3c3564db07e7c35631e999d53e81fa7","header":"0000002015b9a5a957588d43fdcbc9c8c8fb5b2d378b74bc54fc1e040000000000000000bb4bf58844f6bec746791e35d4f99e476ec7a2ab9a70b6172087ab8e4b1c0681d27c945769260518385ece24","prev_hash":"0000000000000000041efc54bc748b372d5bfbc8c8c9cbfd438d5857a9a5b915","nTime":1469349074,"core_reject_reason":"bad-cb-height","rule":"bip34_coinbase_height_mismatch","context":{"expected_nbits":"18052669","coinbase_height":422060,"coinbase_scriptsig_hex":"03ac7006162f5669614254432f48656c6c6f2c20576f726c64212f2cfabe6d6d66a8e99ddd64fdd53d2fcba9e0da13d660eb4f3af15c1cc9f4afd712b09729fb01000000000000000cd114e24c6b5af1af8c290400","pool":"ViaBTC","parent_kind":"stale"},"observations":[{"channel":"merge_mining","source":"merge-mining-research","child_chain":"namecoin","child_height":296773,"provenance":"https://github.com/deadmanoz/merge-mining-research/blob/f543b1f23c57be6840c7a2a44a59306b5d0180f7/data/error-blocks/error_block_observations.csv#L71","child_block_hash":"66a8e99ddd64fdd53d2fcba9e0da13d660eb4f3af15c1cc9f4afd712b09729fb","child_block_time":1469348069,"child_header":"04010100498a5a053ae0b56d2397b69808899b1676cb8cbccca1242ee61249b42242bec388fe0e2260ff96162a9e53c8565ee4d286bd7bb4a045b194cf9a0f7f06bd9f3ae57894572fe20d1800000000"},{"channel":"merge_mining","source":"mergedmonitor","child_chain":"namecoin","provenance":"https://github.com/NStifter/mergedmonitor/blob/54344d4e355f73eb94bef8d391e8fb6e4a9323a6/fork-analysis/mergedmonitor/mergedmonitor.json"},{"channel":"p2p","source":"kit-mon1","first_seen":1469348631,"provenance":"https://github.com/NStifter/mergedmonitor/blob/54344d4e355f73eb94bef8d391e8fb6e4a9323a6/fork-analysis/kit/mon1.json.tar.gz"},{"channel":"p2p","source":"kit-mon2","first_seen":1469348631,"provenance":"https://github.com/NStifter/mergedmonitor/blob/54344d4e355f73eb94bef8d391e8fb6e4a9323a6/fork-analysis/kit/mon2.json.tar.gz"}]} +{"height":474294,"hash":"00000000000000000182acdf5657c93a0769dc6f9004047496b2e15efc6a4232","header":"020000203a4b08e0a87bb99712b929dcc7ea45f14c07230c98b4800000000000000000001cd58a6def6a066eb763bff9ff237a098a7a3c19445d12f9d0f392774efae01ead595c59308d0118453bb5d9","prev_hash":"00000000000000000080b4980c23074cf145eac7dc29b91297b97ba8e0084b3a","nTime":1499224493,"core_reject_reason":"bad-txns-inputs-missingorspent","rule":"missing_unconfirmed_parent","context":{"coinbase_height":474294,"coinbase_scriptsig_hex":"03b63c072cfabe6d6dd2ebb1599afbefb038c64430a42ed0057716d5a6f1aa2cd842756f3e1743548d01000000000000002f4e59412f","pool":"1Hash","parent_kind":"canonical","missing_prevout":"b11a78c6c61af1cb37586f639050d74b95c2b0fd525623b6cb6a4bb4fba46a0e:1"},"observations":[{"channel":"merge_mining","source":"merge-mining-research","child_chain":"namecoin","child_height":349887,"child_block_hash":"d2ebb1599afbefb038c64430a42ed0057716d5a6f1aa2cd842756f3e1743548d","child_block_time":1499224108,"provenance":"https://github.com/deadmanoz/merge-mining-research/blob/f543b1f23c57be6840c7a2a44a59306b5d0180f7/data/child-identity/namecoin_child_identity.csv#L1388"},{"channel":"scrape","source":"chainquery.com","provenance":"https://github.com/NStifter/mergedmonitor/blob/54344d4e355f73eb94bef8d391e8fb6e4a9323a6/fork-analysis/chainquery.com/orphans_chainquery.com.json"}]} {"height":477115,"hash":"0000000000000000013ee4a86822d37a061732e04ee5f41fb77168f193363d1b","header":"12000020257adfb6e7e0b3cd645a9458c6fbc4362eff71878cae3301000000000000000047e7d6f16255b474b896c6bf5c7336c1b570b8de4f2367d126bda61e8007c6d41d217459dc5d0118b5a222ce","prev_hash":"00000000000000000133ae8c8771ff2e36c4fbc658945a64cdb3e0e7b6df7a25","nTime":1500782877,"core_reject_reason":"bad-txns-inputs-missingorspent","rule":"bad-txns-inputs-missingorspent","context":{"coinbase_height":477115,"coinbase_scriptsig_hex":"03bb47072cfabe6d6dab936343b3137ca4569b6c9ef7af36bce731965d82b1b9a221da07786260a5df01000000000000002f4e59412f","parent_kind":"canonical"},"observations":[{"channel":"merge_mining","source":"merge-mining-research","child_chain":"namecoin","child_height":352422,"child_block_hash":"ab936343b3137ca4569b6c9ef7af36bce731965d82b1b9a221da07786260a5df","child_block_time":1500782788,"provenance":"https://github.com/deadmanoz/merge-mining-research/blob/f543b1f23c57be6840c7a2a44a59306b5d0180f7/data/child-identity/namecoin_child_identity.csv#L1390"},{"channel":"scrape","source":"chainquery.com","provenance":"https://github.com/NStifter/mergedmonitor/blob/54344d4e355f73eb94bef8d391e8fb6e4a9323a6/fork-analysis/chainquery.com/orphans_chainquery.com.json"}]} {"height":543804,"hash":"0000000000000000000c958fe3563e7e3ecda8e35e6d6ecce4d5693cec1f1918","header":"000000a0e1f0be09a3ee109bc463656178dfcc9b5b3f4bdf857b12000000000000000000b61fd2f54e77727b941724900da0695433083c49a4c6a3374ce652f15e3ec92a5623b15b1f5a2717661d5ac5","prev_hash":"000000000000000000127b85df4b3f5b9bccdf78616563c49b10eea309bef0e1","nTime":1538335574,"core_reject_reason":"bad-version","rule":"bip65_block_version_below_4","context":{"expected_nbits":"17275a1f","coinbase_height":543804,"coinbase_scriptsig_hex":"033c4c08045623b15b622f4254432e434f4d2ffabe6d6dee6689d8fa713e1a935e1b03dd664008ac0a74b753d03c527f6d25800e82299a0100000000000000fc00cfe8cc4d000000000000","pool":"BTC.COM","parent_kind":"canonical"},"observations":[{"channel":"merge_mining","source":"merge-mining-research","child_chain":"namecoin","child_height":419550,"provenance":"https://github.com/deadmanoz/merge-mining-research/blob/f543b1f23c57be6840c7a2a44a59306b5d0180f7/data/error-blocks/error_block_observations.csv#L72","child_block_hash":"ee6689d8fa713e1a935e1b03dd664008ac0a74b753d03c527f6d25800e82299a","child_block_time":1538335419},{"channel":"merge_mining","source":"merge-mining-research","child_chain":"rsk","child_height":789982,"provenance":"https://github.com/deadmanoz/merge-mining-research/blob/f543b1f23c57be6840c7a2a44a59306b5d0180f7/data/error-blocks/error_block_observations.csv#L79","child_block_hash":"7e26d8a3de73a67b4bbe35bfd0763556504ef02ee5eb84e15c1c99024a6a0077","child_block_time":1538335566}]} {"height":544024,"hash":"0000000000000000001ac59b46b44a9e525bc03a5cd2e138a8ae684445fa6ed8","header":"000000e083bcaf0ad9fb201f6f888ee014d8c1e8c317213cb5ba1e000000000000000000fc3d782759e6ade1a1b211bff767cd07d7f33dc0c21f82c8d62359bc3aa2b31ac0fab25b1f5a271719a0fc34","prev_hash":"0000000000000000001ebab53c2117c3e8c1d814e08e886f1f20fbd90aafbc83","nTime":1538456256,"core_reject_reason":"bad-version","rule":"bip65_block_version_below_4","context":{"expected_nbits":"17275a1f","coinbase_height":544024,"coinbase_scriptsig_hex":"03184d0804c9fab25b612f4254432e434f4d2ffabe6d6dcea0fda9be365f0d78b66072ae5cf48ed7cb6338df96d4a951f49bcc9641a6710100000000000000fb0136c12081000000000000","pool":"BTC.COM","parent_kind":"canonical"},"observations":[{"channel":"merge_mining","source":"merge-mining-research","child_chain":"namecoin","child_height":419759,"provenance":"https://github.com/deadmanoz/merge-mining-research/blob/f543b1f23c57be6840c7a2a44a59306b5d0180f7/data/error-blocks/error_block_observations.csv#L74","child_block_hash":"cea0fda9be365f0d78b66072ae5cf48ed7cb6338df96d4a951f49bcc9641a671","child_block_time":1538455676},{"channel":"merge_mining","source":"merge-mining-research","child_chain":"rsk","child_height":793596,"provenance":"https://github.com/deadmanoz/merge-mining-research/blob/f543b1f23c57be6840c7a2a44a59306b5d0180f7/data/error-blocks/error_block_observations.csv#L81","child_block_hash":"9820cd4d8c71156858576c8fb33c1d26acf4a35f6759be6992d20aa34380854c","child_block_time":1538456260}]} diff --git a/docs/notes.md b/docs/notes.md index c047d2d..877e712 100644 --- a/docs/notes.md +++ b/docs/notes.md @@ -8,7 +8,9 @@ Deeper per-block history is documented elsewhere; see the observation `provenanc A `.bin` can be replayed with `bitcoin-cli submitblock`, but what comes back depends on where the violation is caught. 74638 fails the context-free `CheckBlock` checks before anything is stored, so every replay returns `bad-txns-vout-toolarge` on any node. Its header alone is valid, though: `submitheader` accepts it (noted in ), since every header-level check passes and the violation lives entirely in the body. -The transaction-ordering and 2023 sigops blocks fail in `ConnectBlock`, which never runs for a deep side-chain block: a node seeing them fresh returns `inconclusive` and stores the block as a `valid-headers` chain tip, a node that already stores them returns `duplicate`, and only a node that attempted the connect at their original tip and marked them failed returns `duplicate-invalid` (the result reported in ). +The transaction-ordering blocks, 474294 and the 2023 sigops blocks fail in `ConnectBlock`, which a node only runs when it is about to extend its active chain with the block. +Replayed today they sit on a branch with less work than the tip, so a node seeing them fresh stores them after the context-free checks and never connects them: `submitblock` returns `inconclusive` and the block becomes a `valid-headers` chain tip. +A node that already stores them returns `duplicate`, and only a node that attempted the connect at their original tip and marked them failed returns `duplicate-invalid` (the result reported in ). ## Incident notes @@ -41,3 +43,14 @@ This violation is re-derivable from the [preserved full block](../blocks/477115- `ConnectBlock` processes transactions in order, so the first such input fails the coins lookup: `bad-txns-inputs-missingorspent`. This violation is re-derivable from the `.bin` alone. Documented in [b10c observation 07](https://b10c.me/observations/07-invalid-block-809478/). + +### 474294 - missing parent transaction (2017) + +Transaction 110 spends `b11a78c6c61af1cb37586f639050d74b95c2b0fd525623b6cb6a4bb4fba46a0e:1`, and that parent transaction is not in the block. +The record names that outpoint in `missing_prevout`. +Both transactions confirmed in the competing block at the same height, `000000000000000000db2504327e272fe7658fac0dd0741f46b212256e500886`. +The spent output did not exist at the tip of the previous block, so `ConnectBlock` fails with `bad-txns-inputs-missingorspent`. +This is not an in-block ordering error: unlike 477115 and 809478, no later transaction in the body creates the output. +The [preserved full block](../blocks/474294-00000000000000000182acdf5657c93a0769dc6f9004047496b2e15efc6a4232.bin), the parent transaction, its current confirmation and the canonical block hash at 474293 re-derive the violation. + +Contemporaneous discussion is [BitcoinTalk topic 2041607](https://bitcointalk.org/index.php?topic=2041607.0). diff --git a/docs/schema.md b/docs/schema.md index 562e622..939f387 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -6,7 +6,7 @@ Height is `prev + 1`, not a unique key: different blocks at the same height rema A block may enter the dataset only if its header meets its encoded PoW target and its named consensus failure has the evidence required below. Header/context rules use the supplied header and context. -Body rules require a complete block; sigops additionally requires previous transactions fetched from public APIs or verified cache entries. +Body rules require a complete block; sigops and missing-parent rules additionally require evidence fetched from public APIs or verified cache entries. Observations document acquisition and incident history, but an explorer label or reported reject string cannot substitute for the evidence check. JSONL keeps each block's identity, optional context and repeated observations together. @@ -44,7 +44,8 @@ Context is shared across observations; adding another witness does not duplicate | `coinbase_height` | integer | Height decoded from the BIP34 scriptSig prefix. Required for a BIP34 height mismatch. | | `coinbase_scriptsig_hex` | string | Coinbase input scriptSig. Required for BIP34 failures and `coinbase_scriptsig_length_above_100`. | | `pool` | string | Pool identified from the coinbase tag, when known. | -| `parent_kind` | string | `canonical`, `stale`, or `invalid`. `invalid` means the parent is in this dataset. | +| `parent_kind` | string | Chain status of the previous block: `canonical`, `stale`, or `invalid`. `invalid` means the previous block is in this dataset. Descriptive and unverified: CI checks the spelling, not the chain. | +| `missing_prevout` | string | Outpoint as `txid:vout`, spent by a non-coinbase input whose transaction is not in the block. Required for `missing_unconfirmed_parent`. | ## Optional `observations` array @@ -93,6 +94,7 @@ Core functions live in [bitcoin/bitcoin](https://github.com/bitcoin/bitcoin): | `bad-txns-vout-toolarge` | `bad-txns-vout-toolarge` | `CheckTransaction` | | `bad-blk-sigops` | `bad-blk-sigops` | `ConnectBlock` | | `bad-txns-inputs-missingorspent` | `bad-txns-inputs-missingorspent` | `ConnectBlock` via `CheckTxInputs` | +| `missing_unconfirmed_parent` | `bad-txns-inputs-missingorspent` | `ConnectBlock` via `CheckTxInputs` | | `bip34_v2_coinbase_height_mismatch` | `bad-cb-height` | `ContextualCheckBlock` | | `bip34_coinbase_height_mismatch` | `bad-cb-height` | `ContextualCheckBlock` | | `bip34_coinbase_height_missing` | `bad-cb-height` | `ContextualCheckBlock` | @@ -118,7 +120,8 @@ A provenance URL cannot bypass these requirements. | Rule | Required evidence and check | | --- | --- | | `bad-txns-vout-toolarge` | A complete block whose transactions contain an output above 21000000 BTC. | -| `bad-txns-inputs-missingorspent` | A complete block containing a spend of an existing output of a later transaction in that block. Other missing/spent-input cases require an additional evidence checker before admission. | +| `bad-txns-inputs-missingorspent` | A complete block containing a spend of an existing output of a later transaction in that block. A spend of a parent transaction absent from the block uses `missing_unconfirmed_parent`. | +| `missing_unconfirmed_parent` | A complete block extending the block a public API reports at the previous height. The `missing_prevout` outpoint must be spent by an input in the block and created by a transaction not in the block, and the API must currently report that transaction confirmed in another block at this height or later. Unconfirmed or absent status is not evidence. | | `bad-blk-sigops` | A complete block at mainnet height 481824 or later, authenticated previous transactions for every external input, and calculated BIP16/BIP141 sigop cost above 80000. | | `bip34_v2_coinbase_height_mismatch` | Both coinbase context fields, decoded height matching the scriptSig, and a scriptSig that lacks the exact expected BIP34 prefix. Header version must be at least 2 and height below 227931. Applicability of the historical rolling-version threshold still requires review. | | `bip34_coinbase_height_mismatch` | Both coinbase context fields, decoded height matching the scriptSig, and a scriptSig that lacks the exact expected BIP34 prefix, at height 227931 or later. A non-minimal encoding of the right number also fails the prefix check. | @@ -138,7 +141,7 @@ At or after mainnet SegWit activation, witness data must match the coinbase witn The validator also checks JSONL structure, field types, decoded Bitcoin header identity, compact target and PoW, ordering, uniqueness and observation fields. Locally checked predicates establish consistency with the supplied context. -Review must still establish the block height, parent MTP, expected difficulty, historical activation state and the connection between an extracted coinbase script and its header when no complete body is available. +Review must still establish the block height, parent MTP, expected difficulty, historical activation state, the chain status claimed in `parent_kind`, and the connection between an extracted coinbase script and its header when no complete body is available. The retarget check does not reconstruct the previous difficulty or prove that it was reused. CI does not execute scripts, validate child-chain commitments, reconstruct historical chain state or fetch observation provenance. It verifies the named failures, not every consensus rule or the exact first rejection a historical node would return. @@ -178,3 +181,20 @@ The block's witness commitment is verified before counting witness scripts, so c The previous-transaction txid authenticates its output scripts, but does not prove that an output was unspent at the candidate's parent or that a signature is valid. Those are separate consensus checks. This calculation establishes the excessive sigop cost without reconstructing a historical UTXO set or depending on an explorer's rejection label. + +## Missing parent transaction evidence + +Here "parent transaction" is the transaction that created a spent output, and "previous block" is the block the candidate extends. + +`missing_unconfirmed_parent` shares reject string `bad-txns-inputs-missingorspent` with the forward-spend rule, but that checker only sees transactions inside the block. +Most valid blocks spend outputs created in earlier blocks, so a parent transaction absent from the body is not by itself a failure. +The rule requires the parent transaction named by `missing_prevout` to be currently confirmed in another block at the candidate's height or later: it was not in the chain below the candidate, so its output did not exist at the tip of the previous block. +A parent transaction currently confirmed below that height is a normal spend. +That inference holds only if the candidate extends the canonical chain, so CI also requires the block hash the API reports at the previous height to equal `prev_hash`. + +`ci/prevouts.py` caches the parent transaction as `{txid}.bin`, the Esplora `/tx/{txid}/status` reply as `{txid}.status.json`, and the `/block-height/{n}` reply as `height-{n}.hash`; `--fetch-prevouts` permits the downloads. +A failed download or an unconfirmed reply fails validation, and an unconfirmed status is never cached. +Offline validation needs all three files in `.cache/prevouts/`. + +This check does not reconstruct a UTXO set or replay `ConnectBlock`, and it trusts the configured APIs for the confirmation and the canonical hash. +Cached confirmation and block-hash entries are snapshots from their first fetch; delete them to re-check.