Admit 1Hash block 474294 under a new missing-parent-transaction rule

README.md

@@ -56,7 +56,7 @@ Validation reports the first error in each record, with its file and line number
 Available block files must parse completely and match their transaction merkle roots and applicable witness commitments.
 CI checks output-value overflow, forward transaction spends and excessive sigop cost directly.
 
-Sigops checks use a verified cache in `.cache/prevouts/`, restored between GitHub Actions runs.
+Sigops and missing-parent checks use a verified cache in `.cache/prevouts/`, restored between GitHub Actions runs.
 Missing entries are fetched from public Esplora-compatible APIs when `--fetch-prevouts` is supplied.
 API failures, missing evidence and corrupt cached transactions fail validation.
 After filling the cache, omit the flag for an offline run; `--prevouts-dir` selects another cache and `--api-url` selects an API base.

ci/block_evidence.py

@@ -88,6 +88,23 @@ def read_block(data: bytes) -> CBlock:
     return block
 
 
+def omitted_prevouts(transactions: Sequence[CTransaction]) -> list[tuple[bytes, int]]:
+    """Return external prevouts spent by non-coinbase inputs, as (txid, vout)."""
+    own = {tx.GetTxid() for tx in transactions}
+    return [(txin.prevout.hash, txin.prevout.n)
+            for tx in transactions[1:] for txin in tx.vin if txin.prevout.hash not in own]
+
+
+def confirmed_at_or_after(confirmation: tuple[int, str], height: int, block_hash: str) -> bool:
+    """True if the parent is now confirmed at height or later in a block other than block_hash.
+
+    It was then not in the chain below the candidate, so its output did not
+    exist at the tip of a canonical previous block. This is not a UTXO lookup.
+    """
+    confirmed_height, confirmed_hash = confirmation
+    return confirmed_height >= height and confirmed_hash != block_hash
+
+
 def establishes_rule(block: CBlock, rule: str) -> bool:
     """Recognize only failures provable from these committed transactions.
 

ci/prevouts.py

@@ -1,107 +1,125 @@
-"""Acquire previous transactions through Esplora-compatible public APIs.
+"""Fetch previous transactions, parent confirmations and canonical block hashes.
 
-Cache entries contain stripped transaction bytes, sufficient to authenticate
-output scripts against the txid committed in a spending input. Every cache hit
-is verified too. Downloads and cache corruption fail explicitly; there is no
-fallback to a claimed reject string or an unauthenticated scriptPubKey.
+`{txid}.bin` is a stripped transaction, checked against its txid.
+`{txid}.status.json` is the Esplora status reply for that transaction, and
+`height-{n}.hash` is the block hash Esplora reports at height n. Cache hits
+are decoded and checked like downloads. Failed downloads and corrupt files
+fail the run, and a reply is stored only after it decodes as evidence, so an
+unconfirmed status is never cached.
 """
 
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from http.client import HTTPException, IncompleteRead
+import json
 from pathlib import Path
+import re
 import time
 import tempfile
+from typing import TypeVar
 from urllib.request import Request, urlopen
 
 from bitcoin.core import CTransaction, b2lx
 
-from block_evidence import read_transaction
+from block_evidence import omitted_prevouts, read_transaction
 
 DEFAULT_APIS = ("https://mempool.space/api", "https://blockstream.info/api")
 PREVOUTS_DIR = Path(".cache/prevouts")
+BLOCK_HASH = re.compile(r"[0-9a-fA-F]{64}")
+T = TypeVar("T")
 
 
-def decode_previous(data: bytes, txid: str) -> CTransaction:
-    """Authenticate a complete transaction against the requested display txid."""
-    transaction = read_transaction(data)
-    if b2lx(transaction.GetTxid()) != txid:
-        raise ValueError(f"previous transaction identity mismatch: {txid}")
-    return transaction
+def _get(url: str, limit: int) -> bytes:
+    """Download at most limit bytes. A truncated body raises."""
+    request = Request(url, headers={"User-Agent": "invalid-blocks-evidence-check/1"})
+    with urlopen(request, timeout=30) as response:
+        raw = response.read(limit + 1)
+    if len(raw) > limit:
+        raise ValueError(f"oversized response from {url}")
+    # urlopen can return a short body without raising. response.length is the
+    # remaining Content-Length.
+    remaining = getattr(response, "length", None)
+    if remaining:
+        raise IncompleteRead(raw, remaining)
+    return raw
 
 
-def fetch_previous(txid: str, apis: Sequence[str] = DEFAULT_APIS) -> CTransaction:
-    """Fetch raw hex with bounded retries, provider fallback and HTTP timeouts.
-
-    Two workers call this function in parallel. Successful downloads are paced
-    and rate-limit/transient failures back off. Identity failures are terminal:
-    trying another provider must not conceal a wrong transaction response.
-    """
+def _fetch(what: str, apis: Sequence[str], path: str, limit: int) -> bytes:
+    """Try each API up to twice on transport errors and return the first body."""
     failures = []
     for api in apis:
-        url = f"{api.rstrip('/')}/tx/{txid}/hex"
+        url = f"{api.rstrip('/')}/{path}"
         for attempt in range(2):
-            request = Request(url, headers={"User-Agent": "invalid-blocks-evidence-check/1"})
             try:
-                with urlopen(request, timeout=30) as response:
-                    # Raw transaction size is bounded for this evidence path;
-                    # the limit also catches oversized error responses.
-                    raw = response.read(8_000_001)
-                if len(raw) > 8_000_000:
-                    raise ValueError(f"oversized previous transaction response: {txid}")
-                # read(amt) can return early without raising on a truncated
-                # Content-Length response. HTTPResponse tracks bytes still due.
-                remaining = getattr(response, "length", None)
-                if remaining:
-                    raise IncompleteRead(raw, remaining)
-                transaction = decode_previous(bytes.fromhex(raw.decode("ascii").strip()), txid)
+                raw = _get(url, limit)
                 time.sleep(0.25)
-                return transaction
+                return raw
             except (OSError, HTTPException) as exc:
                 # urllib wraps connection setup errors, but resets and short
                 # HTTP reads can escape unwrapped after the response starts.
                 failures.append(f"{url}: {exc}")
                 if attempt == 0:
                     time.sleep(2)
-    raise ValueError(f"could not download previous transaction {txid}: " + "; ".join(failures))
+    raise ValueError(f"could not download {what}: " + "; ".join(failures))
+
+
+def _replace(path: Path, data: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as handle:
+        temporary = Path(handle.name)
+        try:
+            handle.write(data)
+            handle.close()
+            temporary.replace(path)
+        finally:
+            temporary.unlink(missing_ok=True)
+
+
+def _cached(what: str, path: Path, decode: Callable[[bytes], T], fetch: bool,
+            download: Callable[[], bytes]) -> T:
+    """Decode the cached file, or with fetch download it, decode it and then store it."""
+    if path.exists():
+        return decode(path.read_bytes())
+    if not fetch:
+        raise ValueError(f"missing cached {what}; run with --fetch-prevouts")
+    raw = download()
+    value = decode(raw)
+    _replace(path, raw)
+    return value
+
+
+def decode_previous(data: bytes, txid: str) -> CTransaction:
+    """Parse the transaction and require its txid to match."""
+    transaction = read_transaction(data)
+    if b2lx(transaction.GetTxid()) != txid:
+        raise ValueError(f"previous transaction identity mismatch: {txid}")
+    return transaction
+
+
+def fetch_previous(txid: str, apis: Sequence[str] = DEFAULT_APIS) -> CTransaction:
+    """Download the raw transaction. A txid mismatch is terminal, not retried."""
+    raw = _fetch(f"previous transaction {txid}", apis, f"tx/{txid}/hex", 8_000_000)
+    return decode_previous(bytes.fromhex(raw.decode("ascii").strip()), txid)
+
+
+def load_transaction(txid: str, cache_dir: Path | str = PREVOUTS_DIR, fetch: bool = False,
+                     apis: Sequence[str] = DEFAULT_APIS) -> CTransaction:
+    """Return the stripped transaction for one hex txid, from cache or the API."""
+    return _cached(f"previous transaction {txid}", Path(cache_dir) / f"{txid}.bin",
+                   lambda data: decode_previous(data, txid), fetch,
+                   lambda: fetch_previous(txid, apis).serialize({"include_witness": False}))
 
 
 def load_previous(transactions: Sequence[CTransaction], cache_dir: Path | str = PREVOUTS_DIR,
                   fetch: bool = False, apis: Sequence[str] = DEFAULT_APIS) -> dict[bytes, CTransaction]:
-    """Resolve external txids; earlier in-block outputs are handled by sigops.
-
-    The complete requested set is required for an exact count. Offline mode
-    reports missing evidence. Online mode writes only verified stripped bytes
-    using atomic replacement, so interrupted downloads cannot poison the cache.
-    """
-    own = {tx.GetTxid() for tx in transactions}
-    needed = sorted({b2lx(txin.prevout.hash) for tx in transactions[1:]
-                     for txin in tx.vin if txin.prevout.hash not in own})
+    """Load every external prevout transaction; spends of earlier in-block txs are not fetched."""
+    needed = sorted({b2lx(txid) for txid, _ in omitted_prevouts(transactions)})
     cache_dir = Path(cache_dir)
-
-    def load(txid: str) -> CTransaction:
-        path = cache_dir / f"{txid}.bin"
-        if path.exists():
-            return decode_previous(path.read_bytes(), txid)
-        if not fetch:
-            raise ValueError(f"missing cached previous transaction {txid}; run with --fetch-prevouts")
-        transaction = fetch_previous(txid, apis)
-        cache_dir.mkdir(parents=True, exist_ok=True)
-        with tempfile.NamedTemporaryFile(dir=cache_dir, delete=False) as handle:
-            temporary = Path(handle.name)
-            try:
-                handle.write(transaction.serialize({"include_witness": False}))
-                handle.close()
-                temporary.replace(path)
-            finally:
-                temporary.unlink(missing_ok=True)
-        return transaction
-
     if fetch:
         missing = sum(not (cache_dir / f"{txid}.bin").exists() for txid in needed)
         print(f"Previous transactions: {len(needed)} required, {missing} to download", flush=True)
     pool = ThreadPoolExecutor(max_workers=2)
-    futures = [pool.submit(load, txid) for txid in needed]
+    futures = [pool.submit(load_transaction, txid, cache_dir, fetch, apis) for txid in needed]
     result = {}
     try:
         for future in as_completed(futures):
@@ -113,3 +131,41 @@ def load(txid: str) -> CTransaction:
         # Do not keep downloading the remaining catalogue after an error.
         pool.shutdown(wait=True, cancel_futures=True)
     return result
+
+
+def decode_confirmation(data: bytes, txid: str) -> tuple[int, str]:
+    """Read an Esplora status reply as (height, block hash); unconfirmed is not evidence."""
+    try:
+        payload = json.loads(data)
+    except ValueError as exc:
+        raise ValueError(f"malformed confirmation: {txid}") from exc
+    if not isinstance(payload, dict) or payload.get("confirmed") is not True:
+        raise ValueError(f"parent transaction {txid} is not confirmed")
+    height, block_hash = payload.get("block_height"), payload.get("block_hash")
+    if type(height) is not int or height < 0 or not isinstance(block_hash, str) or not BLOCK_HASH.fullmatch(block_hash):
+        raise ValueError(f"malformed confirmation: {txid}")
+    return height, block_hash.lower()
+
+
+def load_confirmation(txid: str, cache_dir: Path | str = PREVOUTS_DIR, fetch: bool = False,
+                      apis: Sequence[str] = DEFAULT_APIS) -> tuple[int, str]:
+    """Return where the API reported txid confirmed, as (height, block hash)."""
+    what = f"confirmation {txid}"
+    return _cached(what, Path(cache_dir) / f"{txid}.status.json", lambda data: decode_confirmation(data, txid),
+                   fetch, lambda: _fetch(what, apis, f"tx/{txid}/status", 65_536))
+
+
+def decode_block_hash(data: bytes, height: int) -> str:
+    """Read a block-height reply as a lowercase block hash."""
+    text = data.decode("ascii", errors="replace").strip()
+    if not BLOCK_HASH.fullmatch(text):
+        raise ValueError(f"malformed block hash for height {height}")
+    return text.lower()
+
+
+def load_canonical_hash(height: int, cache_dir: Path | str = PREVOUTS_DIR, fetch: bool = False,
+                        apis: Sequence[str] = DEFAULT_APIS) -> str:
+    """Return the block hash the API currently reports at height."""
+    what = f"block hash for height {height}"
+    return _cached(what, Path(cache_dir) / f"height-{height}.hash", lambda data: decode_block_hash(data, height),
+                   fetch, lambda: _fetch(what, apis, f"block-height/{height}", 256))

ci/sanity-check.py

@@ -16,38 +16,43 @@
 from typing import Any
 from urllib.parse import urlparse
 
-from bitcoin.core import CBlock, CBlockHeader, b2lx
+from bitcoin.core import CBlock, CBlockHeader, b2lx, lx
 from bitcoin.core._bignum import vch2bn
 from bitcoin.core.script import CScript, CScriptInvalidError, OP_1NEGATE
 from bitcoin.core.serialize import uint256_from_compact
 
 from block_evidence import (
-    MAX_BLOCK_SIGOPS_COST, establishes_rule, read_block,
-    sigop_cost, verify_witness_commitment,
+    MAX_BLOCK_SIGOPS_COST, confirmed_at_or_after, establishes_rule, omitted_prevouts,
+    read_block, sigop_cost, verify_witness_commitment,
+)
+from prevouts import (
+    DEFAULT_APIS, PREVOUTS_DIR, load_canonical_hash, load_confirmation, load_previous, load_transaction,
 )
-from prevouts import DEFAULT_APIS, PREVOUTS_DIR, load_previous
 
 DATA_PATH = Path("data/invalid-blocks.jsonl")
 BLOCKS_DIR = Path("blocks")
 REQUIRED = {"height", "hash", "header", "prev_hash", "nTime", "core_reject_reason", "rule"}
 CONTEXT_FIELDS = {
     "expected_nbits", "parent_mtp", "coinbase_height", "coinbase_scriptsig_hex",
-    "pool", "parent_kind",
+    "pool", "parent_kind", "missing_prevout",
 }
+OUTPOINT = re.compile(r"[0-9a-f]{64}:(?:0|[1-9][0-9]*)")
 OBSERVATION_REQUIRED = {"channel", "source", "provenance"}
 CHILD_FIELDS = {"child_chain", "child_height", "child_block_hash", "child_block_time", "child_header"}
 OBSERVATION_FIELDS = OBSERVATION_REQUIRED | CHILD_FIELDS | {"first_seen"}
 CHANNELS = {"merge_mining", "p2p", "scrape"}
 PARENT_KINDS = {"canonical", "stale", "invalid"}
 POW_LIMIT = 0xFFFF << (8 * (0x1D - 3))
 
-# Evidence paths: local = checked header/context predicate; body = a complete,
-# merkle-bound body proving the failure; sigops = body plus authenticated prevouts.
-# Keep the rule names and reject strings aligned with docs/schema.md.
+# Evidence paths: local = header/context only; body = complete block file;
+# sigops = body plus previous transactions; missing_parent = body plus API
+# evidence for the recorded outpoint. Rule names and reject strings must
+# match docs/schema.md.
 RULES = {
     "bad-txns-vout-toolarge": ("bad-txns-vout-toolarge", (), "body"),
     "bad-blk-sigops": ("bad-blk-sigops", (), "sigops"),
     "bad-txns-inputs-missingorspent": ("bad-txns-inputs-missingorspent", (), "body"),
+    "missing_unconfirmed_parent": ("bad-txns-inputs-missingorspent", ("missing_prevout",), "missing_parent"),
     "bip34_v2_coinbase_height_mismatch": (
         "bad-cb-height", ("coinbase_height", "coinbase_scriptsig_hex"), "local"),
     "bip34_coinbase_height_mismatch": (
@@ -158,6 +163,9 @@ def check_context(record: dict[str, Any]) -> None:
             hex_value(details, name, size)
     if "pool" in details:
         string(details, "pool")
+    if "missing_prevout" in details and not (
+            isinstance(details["missing_prevout"], str) and OUTPOINT.fullmatch(details["missing_prevout"])):
+        raise ValueError("missing_prevout must be txid:vout in lowercase hex")
     if "parent_kind" in details and details["parent_kind"] not in tuple(PARENT_KINDS):
         raise ValueError(f"parent_kind must be one of {sorted(PARENT_KINDS)}")
     required = set(RULES[record["rule"]][1])
@@ -229,11 +237,10 @@ def height_prefix(height: int) -> bytes:
 
 
 def check_local_evidence(record: dict[str, Any], header: CBlockHeader) -> None:
-    """Check the claimed failure against already type-checked header/context.
+    """Check the named rule against header and context already in the record.
 
-    This proves consistency with supplied height, MTP and expected difficulty;
-    it does not authenticate those facts against the parent chain or bind an
-    extracted coinbase script to the header without a complete body.
+    Parent MTP, expected nBits and parent_kind are not looked up on the chain.
+    Without a body, a coinbase scriptSig cannot be bound to the header.
     """
     rule = record["rule"]
     context = record.get("context", {})
@@ -279,6 +286,23 @@ def check_failure_evidence(record: dict[str, Any], block: CBlock | None, prevout
         raise ValueError(f"{record['rule']} requires a complete block body")
     if mode == "body" and not establishes_rule(block, record["rule"]):
         raise ValueError(f"committed body does not demonstrate {record['rule']}")
+    if mode == "missing_parent":
+        txid, vout = record["context"]["missing_prevout"].split(":")
+        if (lx(txid), int(vout)) not in omitted_prevouts(block.vtx):
+            raise ValueError("missing_prevout is not spent by the body from outside the block")
+        previous_height = record["height"] - 1
+        canonical = load_canonical_hash(previous_height, prevouts_dir, fetch_prevouts, apis)
+        if canonical != record["prev_hash"]:
+            raise ValueError(f"prev_hash is not the canonical block at height {previous_height}")
+        parent = load_transaction(txid, prevouts_dir, fetch_prevouts, apis)
+        if int(vout) >= len(parent.vout):
+            raise ValueError(f"missing_prevout output {vout} does not exist in the parent transaction")
+        confirmation = load_confirmation(txid, prevouts_dir, fetch_prevouts, apis)
+        if not confirmed_at_or_after(confirmation, record["height"], record["hash"]):
+            raise ValueError("parent transaction is not confirmed at this height or later in another block")
+        if fetch_prevouts:
+            print(f"{record['height']}: parent {txid} currently confirmed at {confirmation[0]} "
+                  f"in {confirmation[1]}; canonical block at {previous_height} is {canonical}", flush=True)
     if mode == "sigops":
         # This checker uses BIP16 + BIP141 counting, not pre-SegWit rules.
         if record["height"] < 481824:
@@ -365,7 +389,8 @@ def check_dataset(path: Path | str = DATA_PATH, blocks_dir: Path | str = BLOCKS_
 
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--fetch-prevouts", action="store_true", help="fetch missing sigops prevout evidence from public APIs")
+    parser.add_argument("--fetch-prevouts", action="store_true",
+                        help="fetch missing sigops prevouts, parent confirmations and canonical block hashes from public APIs")
     parser.add_argument("--prevouts-dir", type=Path, default=PREVOUTS_DIR, help="verified transaction cache directory")
     parser.add_argument("--api-url", action="append", help="Esplora API base URL; repeat for fallback providers")
     args = parser.parse_args()