From 7733980c411392a6945b2d346127742866eb5048 Mon Sep 17 00:00:00 2001 From: Defenwycke Date: Fri, 11 Sep 2026 10:21:55 +0100 Subject: [PATCH] =?UTF-8?q?fix(wraith):=20the=20coordinator=20roster=20con?= =?UTF-8?q?verges=20=E2=80=94=20no=20local=20verdicts,=20no=20first-look?= =?UTF-8?q?=20freeze=20(#710)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The night all eight mainnet nodes opted in (epoch 6711), rosters read 6/7/5/5/2/2/4/5 and the fleet elected two different coordinators for one epoch. Ten hours later it was still split. Two causes, both removed here. The view was computed once per epoch and held. Whatever the peer table looked like at that instant became the roster for the rest of the day, so a node restarted mid-epoch drew from a half-filled table and kept it. Measured: one node restarted while three others still had coordinating switched off, and its roster was still missing exactly those three ten hours later. The roster is now re-read on every refresh and the view rebuilt when it changes. The beacon and the seat target stay frozen for the epoch, so only the roster can move. Eligibility also read `qualified` and `archive` from this node's own verification ledger. No two nodes hold the same challenge history, so that input could never converge, whatever the timing. Both fields are removed from `NodeFacts`. The module doc said they were "a verdict the network reached together"; they were not. What is given up is a Sybil cost (a verified archive), which is stated in the module rather than implied. A roster that moves mid-epoch can unseat a node while it holds rounds that participants have committed inputs to, and `stop()` aborts them. So a node seated at any point in an epoch keeps serving until the epoch turns (`should_serve`); new wallets follow the current view. The mutation harness targeted the removed archive rule; it now targets the opt-in refusal instead. All 29 mutations caught. --- bins/ghost-pool/src/coordinator_election.rs | 457 ++++++++++++++++---- bins/ghost-pool/src/main.rs | 22 +- crates/wraith-protocol/src/eligibility.rs | 115 ++--- scripts/mutate-wraith.sh | 4 +- 4 files changed, 441 insertions(+), 157 deletions(-) diff --git a/bins/ghost-pool/src/coordinator_election.rs b/bins/ghost-pool/src/coordinator_election.rs index ee62d5185..0fda8dd30 100644 --- a/bins/ghost-pool/src/coordinator_election.rs +++ b/bins/ghost-pool/src/coordinator_election.rs @@ -35,22 +35,8 @@ use ghost_common::rpc::BitcoinRpc; use ghost_consensus::mesh::MeshNetwork; use ghost_common::types::NodeCapabilities; +use tracing::info; use wraith_protocol::eligibility::{eligible_roster, EligibilityPolicy, NodeFacts}; - -/// Verified-capability lookup, matching `CapabilityVerifierCallback` in -/// `ghost-consensus` so the same provider serves both. -pub type QualifiedCapsFn = Arc NodeCapabilities + Send + Sync>; - -/// Did this node pass the qualification gatekeeper at all? -/// -/// Judged on the **verified** capability flags. `coordinator` is deliberately -/// excluded: it is documented as opt-in that "needs no verification challenge", -/// so counting it would let a self-declared flag masquerade as a challenge -/// result — which is exactly the confusion this whole roster change exists to -/// remove. -fn passed_gatekeeper(caps: &NodeCapabilities) -> bool { - caps.archive_mode || caps.ghost_pay || caps.public_mining || caps.reaper || caps.elder_status -} use wraith_protocol::epoch::canonical_roster; use wraith_protocol::roster_snapshot::roster_commitment; use wraith_protocol::service::{CoordinatorView, EndpointMap}; @@ -188,6 +174,71 @@ struct Cached { /// the roster comes from live mesh state, which is the defect this value /// exposes rather than repairs. roster_commitment: [u8; 32], + /// The endpoint map the view was built with, kept so a refresh can tell + /// whether anything a wallet would dial has changed. + endpoints: EndpointMap, + /// Session demand as first read this epoch. Frozen: the roster may still + /// change within an epoch, the seat target may not, or ordinary churn in + /// the session counters would reshuffle seats every few minutes. + demand: u64, + /// This node held a seat in some view of this epoch, not necessarily the + /// current one. See [`CoordinatorElection::should_serve`]. + seated_this_epoch: bool, +} + +/// The view to cache next, or `None` to keep the current one. +/// +/// # Why this runs on every refresh, not once per epoch +/// +/// The view used to be computed once, when the epoch changed, and then held for +/// the rest of the epoch (~a day). Whatever the peer table looked like at that +/// instant became this node's roster until the next flip. A node restarted +/// mid-epoch drew from a table health pings had barely begun to fill; a node +/// that computed before a peer opted in never saw that peer at all. +/// +/// Measured on mainnet the night all eight nodes opted in (2026-09-10, epoch +/// 6711): one node restarted while three others still had coordinating switched +/// off, and ten hours later its roster was still missing exactly those three. +/// The rest of the fleet had filled in theirs, so it elected a different +/// coordinator from everyone else. +/// +/// Recomputing the roster whenever it is asked for lets every node converge on +/// the gossip it now holds. The beacon and the seat target stay frozen for the +/// epoch; only the roster, and with it the draw, may move. +fn next_view( + prev: Option<&Cached>, + self_id: &CoordinatorNodeId, + epoch: u64, + fresh_beacon: Option<[u8; 32]>, + roster: Vec, + endpoints: EndpointMap, + demand: u64, +) -> Option { + let (beacon, demand, seated_before) = match prev.filter(|c| c.epoch == epoch) { + Some(c) => { + if c.roster == roster && c.endpoints == endpoints { + return None; + } + (c.beacon, c.demand, c.seated_this_epoch) + } + // A new epoch needs its own beacon. Without one, keep the last good view + // rather than cache a partial one. + None => (fresh_beacon?, demand, false), + }; + let seats = seats_for_demand(demand, roster.len()); + let view = CoordinatorView::build(epoch, &beacon, &roster, endpoints.clone(), seats); + let anchor_height = anchor_height_for_epoch(epoch); + Some(Cached { + epoch, + seated_this_epoch: seated_before || view.am_i_coordinator(self_id), + view, + beacon, + roster_commitment: roster_commitment(epoch, anchor_height, &roster), + roster, + anchor_height, + endpoints, + demand, + }) } /// Live coordinator-election service for ghost-pool. @@ -211,12 +262,6 @@ pub struct CoordinatorElection { mesh: Arc, /// Ghost Core RPC — source of the beacon anchor (block hash at a height). rpc: Arc, - /// Verified capabilities for a node, from `QualifiedCapabilityProvider`. - /// - /// Returns what a node has **proved** through challenges, not what it - /// claims in its health ping. The distinction is the point: archive mode is - /// the Sybil cost, and a claimed one costs nothing. - qualified_caps: QualifiedCapsFn, /// Cached current-epoch view. cached: RwLock>, } @@ -230,10 +275,8 @@ impl CoordinatorElection { self_endpoint: Option, mesh: Arc, rpc: Arc, - qualified_caps: QualifiedCapsFn, ) -> Self { Self { - qualified_caps, self_id: identity.node_id(), self_coordinator: capabilities.coordinator, self_endpoint, @@ -252,7 +295,6 @@ impl CoordinatorElection { self_endpoint: Option, mesh: Arc, rpc: Arc, - qualified_caps: QualifiedCapsFn, ) -> Option> { if !enabled { return None; @@ -263,17 +305,16 @@ impl CoordinatorElection { self_endpoint, mesh, rpc, - qualified_caps, ))) } /// The eligible coordinator roster for this epoch, plus the endpoint map. /// - /// A peer is eligible iff it opted in, advertises a dialable endpoint, has - /// **verified** archive capability, passed the qualification gatekeeper, is - /// mature, and is not long-absent. Self is judged by the same verified - /// verdict as everyone else. The roster is canonicalised (dedup + sort), so - /// a node's own collection order cannot change the result. + /// A peer is eligible iff it opted in, advertises a dialable endpoint, is + /// mature, and is not long-absent — all of it declared by the peer and + /// gossiped, so nodes holding the same gossip agree. The roster is + /// canonicalised (dedup + sort), so a node's own collection order cannot + /// change the result. /// /// # Declared facts only /// @@ -286,9 +327,11 @@ impl CoordinatorElection { /// order-independent, not two nodes' answers equal. /// /// Eligibility is now `wraith_protocol::eligibility`, over facts a node - /// declared about itself or the network agreed on together: opted in, has - /// an endpoint, qualified, archive, mature, not long-absent. None of it - /// depends on whether *this* node holds a socket. + /// declared about itself: opted in, has an endpoint, mature, not + /// long-absent. None of it depends on whether *this* node holds a socket, + /// or on the verdicts *this* node happens to hold — the verified archive + /// and qualification checks were removed for that reason (see that + /// module). /// /// `Cached::roster_commitment` stays regardless — it is how a split is /// *seen*, and it is the only field in the status response one node cannot @@ -309,21 +352,15 @@ impl CoordinatorElection { // honest nodes disagree. for p in self.mesh.peers().get_all_peers() { let endpoint = p.coordinator_endpoint.clone(); - // Verified, not claimed. `p.capabilities.archive_mode` is what the - // peer says about itself; this is what it proved under challenge, - // and a claimed archive flag costs an attacker nothing. - let verified = (self.qualified_caps)(&p.node_id); let f = NodeFacts { node_id: p.node_id, // Opt-in stays declared: `coordinator` carries no challenge by // design, and a node that has not asked to coordinate should // not be conscripted. opted_in: p.capabilities.coordinator, - archive: verified.archive_mode, endpoint: endpoint.clone(), first_seen_secs: p.first_seen, last_seen_secs: p.last_seen, - qualified: passed_gatekeeper(&verified), }; if let Some(ep) = endpoint { if !ep.trim().is_empty() { @@ -341,18 +378,12 @@ impl CoordinatorElection { .filter(|e| !e.trim().is_empty()) { endpoints.insert(self.self_id, ep.to_string()); - // Self is judged by the same verified verdict as everyone - // else. Trusting our own claim here would make this node the - // one peer that never has to prove anything. - let mine = (self.qualified_caps)(&self.self_id); facts.push(NodeFacts { node_id: self.self_id, opted_in: true, - archive: mine.archive_mode, endpoint: Some(ep.to_string()), first_seen_secs: 0, last_seen_secs: now, - qualified: passed_gatekeeper(&mine), }); demand = demand.saturating_add(self.mesh.coordinator_sessions() as u64); } @@ -374,46 +405,64 @@ impl CoordinatorElection { Some(derive_beacon(epoch, &anchor)) } - /// Recompute and cache the `CoordinatorView` for the epoch `current_height` - /// falls in — but only when the epoch has actually changed since the last - /// cached view (cheap no-op otherwise). Safe to call on every new block / - /// round advance. Returns the (possibly unchanged) current epoch. + /// Recompute the roster and, if it or the epoch has changed, rebuild and + /// cache the `CoordinatorView` for the epoch `current_height` falls in. + /// Safe to call on every new block / round advance: the beacon is fetched + /// once per epoch, and an unchanged roster is a no-op. Returns the current + /// epoch. /// /// On any input failure (no anchor block yet, RPC error) it leaves the /// existing cache untouched and returns the current epoch unchanged — never /// poisons the cache with a partial view. pub async fn refresh_for_height(&self, current_height: u64) -> u64 { let epoch = epoch_for_height(current_height); + let (roster, endpoints, demand) = self.roster_with_endpoints(); - // Fast path: same epoch as the cached view → nothing to do. - if let Some(c) = self.cached.read().as_ref() { - if c.epoch == epoch { + let same_epoch = self + .cached + .read() + .as_ref() + .is_some_and(|c| c.epoch == epoch); + let fresh_beacon = if same_epoch { + None + } else { + let Some(beacon) = self.beacon_for_epoch(epoch).await else { + // Anchor not reachable yet — keep the last good view. return epoch; - } - } + }; + Some(beacon) + }; - let Some(beacon) = self.beacon_for_epoch(epoch).await else { - // Anchor not reachable yet — keep the last good view. + let (next, previous_size) = { + let guard = self.cached.read(); + let prev = guard.as_ref(); + let previous_size = prev.filter(|c| c.epoch == epoch).map(|c| c.roster.len()); + let next = next_view( + prev, + &self.self_id, + epoch, + fresh_beacon, + roster, + endpoints, + demand, + ); + (next, previous_size) + }; + let Some(next) = next else { return epoch; }; - // Roster = opted-in coordinators advertising a reachable endpoint (+ self - // when opted in), with the endpoint map a wallet uses to dial the owner. - // Seats are sized from the frozen, mesh-summed recent session demand — - // this recompute only runs when the epoch flips, so the snapshot is the - // per-epoch freeze. - let (roster, endpoints, demand) = self.roster_with_endpoints(); - let seats = seats_for_demand(demand, roster.len()); - let view = CoordinatorView::build(epoch, &beacon, &roster, endpoints, seats); - let anchor_height = anchor_height_for_epoch(epoch); - let commitment = roster_commitment(epoch, anchor_height, &roster); - *self.cached.write() = Some(Cached { + // Said once per change, so a split can be traced to the moment one + // node's roster moved rather than reconstructed afterwards. + info!( epoch, - view, - beacon, - roster, - anchor_height, - roster_commitment: commitment, - }); + within_epoch = same_epoch, + roster_size = next.roster.len(), + previous_roster_size = ?previous_size, + seats = next.view.seats(), + roster_commitment = %hex::encode(next.roster_commitment), + "Coordinator roster changed" + ); + *self.cached.write() = Some(next); epoch } @@ -428,6 +477,23 @@ impl CoordinatorElection { .unwrap_or(false) } + /// Whether THIS node should be running its coordinator: seated now, **or + /// seated at any point earlier in this epoch**. + /// + /// Distinct from [`Self::am_i_coordinator`] because the roster can now move + /// within an epoch. A node that loses its seat mid-epoch may be holding + /// rounds that participants have already committed inputs to, and stopping + /// the coordinator aborts them. So it keeps serving until the epoch turns: + /// new wallets follow the current view elsewhere, and the rounds it already + /// holds get to finish. The cost is an idle coordinator for the rest of a + /// day, which is cheap. + pub fn should_serve(&self) -> bool { + self.cached + .read() + .as_ref() + .is_some_and(|c| c.seated_this_epoch) + } + /// A JSON snapshot of the cached election for the read-only HTTP endpoint: /// `{enabled, roster_commitment, epoch, seats, my_seat, elected: [hex ids], /// [{node_id, seat, endpoint}]}`. The `coordinators` array is what a wallet @@ -676,6 +742,251 @@ mod tests { assert_eq!(seats_for_demand(10_000_000, 100), MAX_SEATS); } + // ── refresh: the roster converges within an epoch ── + + /// A node outside every test roster, for tests about the roster rather + /// than about this node's own seat. + const OBSERVER: CoordinatorNodeId = [0xEE; 32]; + + fn endpoints_for(roster: &[CoordinatorNodeId]) -> EndpointMap { + roster + .iter() + .map(|id| (*id, format!("10.0.0.{}:9100", id[0]))) + .collect() + } + + fn first_draw_as( + me: &CoordinatorNodeId, + epoch: u64, + roster: &[CoordinatorNodeId], + demand: u64, + ) -> Cached { + next_view( + None, + me, + epoch, + Some(derive_beacon(epoch, &[7u8; 32])), + roster.to_vec(), + endpoints_for(roster), + demand, + ) + .expect("a first draw with a beacon always produces a view") + } + + fn first_draw(epoch: u64, roster: &[CoordinatorNodeId], demand: u64) -> Cached { + first_draw_as(&OBSERVER, epoch, roster, demand) + } + + /// Same epoch, a new roster — what a refresh does once more gossip is in. + fn redraw_as( + me: &CoordinatorNodeId, + prev: &Cached, + roster: &[CoordinatorNodeId], + ) -> Option { + next_view( + Some(prev), + me, + prev.epoch, + None, + roster.to_vec(), + endpoints_for(roster), + prev.demand, + ) + } + + fn winner(c: &Cached) -> CoordinatorNodeId { + c.view.seated()[0].node_id + } + + /// The mainnet failure, epoch 6711: a node that drew while three peers had + /// not yet opted in kept that roster for the whole epoch, and elected a + /// different coordinator from the nodes that drew later. + #[test] + fn a_node_that_drew_early_converges_on_the_late_nodes_election() { + let full: Vec<_> = (1u8..=8).map(node).collect(); + let partial: Vec<_> = full.iter().copied().filter(|id| id[0] > 3).collect(); + + let late = first_draw(6711, &full, 0); + let early = first_draw(6711, &partial, 0); + assert_ne!( + early.roster_commitment, late.roster_commitment, + "precondition: the two nodes start out split" + ); + + let caught_up = redraw_as(&OBSERVER, &early, &full) + .expect("a roster that grew within the epoch must be redrawn, not held"); + + let seating = |c: &Cached| -> Vec<(u32, CoordinatorNodeId, Option)> { + c.view + .seated() + .into_iter() + .map(|s| (s.seat, s.node_id, s.endpoint)) + .collect() + }; + assert_eq!(caught_up.roster_commitment, late.roster_commitment); + assert_eq!(caught_up.beacon, late.beacon); + assert_eq!( + seating(&caught_up), + seating(&late), + "same roster and beacon must seat the same coordinators" + ); + } + + #[test] + fn an_unchanged_roster_is_not_redrawn() { + let roster: Vec<_> = (1u8..=8).map(node).collect(); + let cached = first_draw(10, &roster, 0); + assert!(redraw_as(&OBSERVER, &cached, &roster).is_none()); + } + + #[test] + fn a_moved_endpoint_is_picked_up_within_the_epoch() { + // A wallet dials the endpoint, so a stale one is as wrong as a stale + // roster even when the draw itself is unchanged. + let roster: Vec<_> = (1u8..=5).map(node).collect(); + let cached = first_draw(10, &roster, 0); + let mut moved = endpoints_for(&roster); + moved.insert(node(2), "10.9.9.9:9100".into()); + let next = next_view(Some(&cached), &OBSERVER, 10, None, roster, moved.clone(), 0) + .expect("a changed endpoint must be republished"); + assert_eq!(next.endpoints, moved); + } + + #[test] + fn the_seat_target_is_frozen_for_the_epoch() { + // Session counters move constantly. If a roster change also re-read + // them, seats would be resized mid-epoch by ordinary traffic. + let roster: Vec<_> = (1u8..=8).map(node).collect(); + let cached = first_draw(10, &roster[..6], 0); + assert_eq!(cached.view.seats(), 1); + let busy = TARGET_SESSIONS_PER_SEAT * 4; + let next = next_view( + Some(&cached), + &OBSERVER, + 10, + None, + roster.clone(), + endpoints_for(&roster), + busy, + ) + .unwrap(); + assert_eq!( + next.view.seats(), + 1, + "demand read mid-epoch must not resize seats" + ); + assert_eq!(next.demand, 0); + } + + #[test] + fn a_new_epoch_is_drawn_with_its_own_beacon_and_demand() { + let roster: Vec<_> = (1u8..=8).map(node).collect(); + let cached = first_draw(10, &roster, 0); + let busy = TARGET_SESSIONS_PER_SEAT * 3; + let beacon = derive_beacon(11, &[8u8; 32]); + let next = next_view( + Some(&cached), + &OBSERVER, + 11, + Some(beacon), + roster.clone(), + endpoints_for(&roster), + busy, + ) + .expect("a new epoch is always redrawn"); + assert_eq!(next.epoch, 11); + assert_eq!(next.beacon, beacon); + assert_eq!(next.view.seats(), 3); + } + + #[test] + fn a_new_epoch_without_its_beacon_keeps_the_last_good_view() { + let roster: Vec<_> = (1u8..=8).map(node).collect(); + let cached = first_draw(10, &roster, 0); + assert!(next_view( + Some(&cached), + &OBSERVER, + 11, + None, + roster.clone(), + endpoints_for(&roster), + 0 + ) + .is_none()); + } + + /// An epoch in which the full roster seats a node the partial roster did + /// not — so the partial roster's winner loses its seat on catching up. + /// Searched for rather than hard-coded, so the test states the situation it + /// needs instead of depending on what one beacon happens to rank first. + fn epoch_where_catching_up_unseats( + full: &[CoordinatorNodeId], + partial: &[CoordinatorNodeId], + ) -> (u64, CoordinatorNodeId) { + (1..500u64) + .find_map(|epoch| { + let early = winner(&first_draw(epoch, partial, 0)); + let late = winner(&first_draw(epoch, full, 0)); + (early != late).then_some((epoch, early)) + }) + .expect("some epoch in 500 seats a node outside the partial roster") + } + + #[test] + fn a_seat_lost_within_the_epoch_is_served_until_the_epoch_turns() { + // Stopping the coordinator aborts the rounds it holds, and participants + // may already have committed inputs to them. Losing the seat to a + // roster that filled in is not a reason to do that. + let full: Vec<_> = (1u8..=8).map(node).collect(); + let partial: Vec<_> = full.iter().copied().filter(|id| id[0] > 3).collect(); + let (epoch, me) = epoch_where_catching_up_unseats(&full, &partial); + + let early = first_draw_as(&me, epoch, &partial, 0); + assert!( + early.view.am_i_coordinator(&me), + "precondition: seated early" + ); + assert!(early.seated_this_epoch); + + let caught_up = redraw_as(&me, &early, &full).unwrap(); + assert!( + !caught_up.view.am_i_coordinator(&me), + "the published view moves on — new wallets go to the new seat" + ); + assert!( + caught_up.seated_this_epoch, + "but the coordinator keeps serving what it already holds" + ); + } + + #[test] + fn a_new_epoch_forgets_a_seat_held_in_the_last_one() { + let full: Vec<_> = (1u8..=8).map(node).collect(); + let partial: Vec<_> = full.iter().copied().filter(|id| id[0] > 3).collect(); + let (epoch, me) = epoch_where_catching_up_unseats(&full, &partial); + let held = redraw_as(&me, &first_draw_as(&me, epoch, &partial, 0), &full).unwrap(); + assert!(held.seated_this_epoch); + + // Walk forward to an epoch that does not seat `me`, so the only way it + // could still be serving is a seat carried over from before. + let next = (epoch + 1..epoch + 500) + .find_map(|e| { + let v = next_view( + Some(&held), + &me, + e, + Some(derive_beacon(e, &[7u8; 32])), + full.clone(), + endpoints_for(&full), + 0, + ) + .unwrap(); + (!v.view.am_i_coordinator(&me)).then_some(v) + }) + .expect("some later epoch leaves `me` unseated"); + assert!(!next.seated_this_epoch, "a seat does not outlive its epoch"); + } + #[test] fn seats_for_demand_is_deterministic() { // Same frozen inputs → identical seats on every node (no path dependence). diff --git a/bins/ghost-pool/src/main.rs b/bins/ghost-pool/src/main.rs index a5ed833d3..a45dc1aba 100644 --- a/bins/ghost-pool/src/main.rs +++ b/bins/ghost-pool/src/main.rs @@ -7873,15 +7873,10 @@ async fn main() -> Result<()> { config.coordinator.advertised_endpoint.clone(), Arc::clone(&mesh), Arc::clone(&rpc), - { - // The same verified-capability provider the health handler uses, so - // the election judges a node by what it PROVED under challenge - // rather than by what it claims in its health ping. A claimed - // archive flag costs an attacker nothing; a proved one costs - // storage. - let qp = Arc::clone(&qualification_provider_for_health); - Arc::new(move |node_id: &[u8; 32]| qp.get_qualified(node_id)) - }, + // No verified-capability provider. It read THIS node's own challenge + // ledger, which no two nodes share, so the roster could not converge + // (`wraith_protocol::eligibility`, "Why qualification and archive are + // not here"). ); { let coord_for_api = coordinator_election.clone(); @@ -10761,16 +10756,17 @@ async fn main() -> Result<()> { warn!(round_id = round_id, error = %e, "Failed to persist round at start"); } - // Refresh the coordinator-election view if the epoch has - // changed (cheap no-op within an epoch; a no-op entirely + // Refresh the coordinator-election view: re-read the roster, + // rebuild only if it or the epoch changed (a no-op entirely // when the feature is off). Read-only — activates nothing. if let Some(ref coord) = coord_for_events { coord.refresh_for_height(height).await; // Start/stop the in-process coordinator to match the // freshly-recomputed election (no-op when role activation - // is off or the seat is unchanged). + // is off or the seat is unchanged). A seat lost mid-epoch + // is served until the epoch turns — see `should_serve`. if let Some(ref sup) = supervisor_for_events { - sup.reconcile(coord.am_i_coordinator()).await; + sup.reconcile(coord.should_serve()).await; } } diff --git a/crates/wraith-protocol/src/eligibility.rs b/crates/wraith-protocol/src/eligibility.rs index dc184ef45..72d23bb0a 100644 --- a/crates/wraith-protocol/src/eligibility.rs +++ b/crates/wraith-protocol/src/eligibility.rs @@ -33,18 +33,45 @@ //! //! # Declared, not observed //! -//! Every input here is something the node itself declared and gossiped, or a -//! qualification verdict the network reached together: +//! Every input here is something the node itself declared and gossiped: //! //! - opted in to coordinate //! - advertises an endpoint a wallet can dial -//! - passes qualification — 95% uptime over seven days, ten challenges -//! - runs in archive mode //! - has been known long enough to be mature +//! - has not been absent for days //! //! None of it depends on whether *this* node currently holds a socket to that //! peer. //! +//! # Why qualification and archive are not here +//! +//! Both used to be. Both were described as "a qualification verdict the network +//! reached together", and neither was: the caller filled them from +//! `QualifiedCapabilityProvider`, which reads **this node's own** verification +//! ledger — the challenges it issued and the verdicts it holds. Challenge +//! rotation samples a few peers per round, so no two nodes ever hold the same +//! evidence, and the roster could not converge by construction. +//! +//! The night all eight mainnet nodes opted in (2026-09-10, epoch 6711) the +//! rosters read 6/7/5/5/2/2/4/5 and the fleet elected two different +//! coordinators for one epoch. Most of that was the pool freezing whatever +//! roster it saw first (fixed alongside this, in `ghost-pool`). But a fix for +//! the freeze alone would still leave an input that disagrees for ever, and +//! one such input is enough to split a draw that is otherwise deterministic. +//! +//! Qualification was not load-bearing for safety. A coordinator can deny +//! service but cannot take coins — the round is atomic and blind-signed +//! whichever node runs it. Misbehaviour is answered by the outpoint ban list, +//! unreachability by walking to the next coordinator, and identity by the +//! coordinator challenge. What is given up is a Sybil cost: an identity now +//! needs its proof-of-work, a day of maturity and a dialable endpoint, not a +//! verified archive. That is weaker, and it is stated here rather than implied +//! by a field that could not deliver it. +//! +//! ⛔ **Do not add a verdict back unless every node reads the same one.** A +//! BFT-finalised or chain-anchored verdict would qualify; one assembled from a +//! node's own challenge history never will. +//! //! # Liveness is coarse, on purpose //! //! [`EligibilityPolicy::prune_after_secs`] is the only liveness input, and it is @@ -65,20 +92,18 @@ //! //! [`EligibilityPolicy::maturity_secs`] requires an identity to have been known //! *before* the beacon it is ranked under existed, which makes that grind -//! useless. It does not stop an attacker registering many identities in advance; -//! qualification is what costs them there. +//! useless. It does not stop an attacker registering many identities in advance. use crate::sortition::CoordinatorNodeId; -/// What is known about a candidate coordinator. All declared or network-agreed. +/// What is known about a candidate coordinator. All of it declared by the node +/// and gossiped, so every node holding the same gossip computes the same roster. #[derive(Debug, Clone, PartialEq, Eq)] pub struct NodeFacts { /// Identity. pub node_id: CoordinatorNodeId, /// Declared the coordinator capability. pub opted_in: bool, - /// Declared archive mode. - pub archive: bool, /// Advertised endpoint. `None` or empty means a wallet cannot dial it. pub endpoint: Option, /// When this identity was first seen, unix seconds. @@ -86,21 +111,11 @@ pub struct NodeFacts { /// When it was last heard from, unix seconds. Used only against the /// **coarse** pruning window. pub last_seen_secs: u64, - /// Passes `ghost-verification::qualification`. - pub qualified: bool, } /// Eligibility rules. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct EligibilityPolicy { - /// Require archive mode. - /// - /// Raises a Sybil farm from lightweight VMs to real storage. Worth having - /// and not a barrier: archive capability can be proxied, because remote - /// attestation cannot tell *"I store it"* from *"I can fetch it quickly"*. - pub require_archive: bool, - /// Require passing qualification. - pub require_qualified: bool, /// How long an identity must have been known before it may be elected. pub maturity_secs: u64, /// How long an absent node stays in the roster. **Days, not seconds.** @@ -111,8 +126,6 @@ impl Default for EligibilityPolicy { /// Parameters, not results. None of these is measured. fn default() -> Self { Self { - require_archive: true, - require_qualified: true, // One epoch's worth of days, comfortably longer than the gossip // needed to agree an identity exists. maturity_secs: 24 * 60 * 60, @@ -131,12 +144,6 @@ pub enum Ineligible { /// No endpoint to dial. #[error("node advertises no coordinator endpoint, so no wallet can reach it")] NoEndpoint, - /// Does not pass qualification. - #[error("node does not pass qualification (uptime and challenge history)")] - NotQualified, - /// Not an archive node. - #[error("node does not run in archive mode")] - NotArchive, /// Identity is too new to be ranked under this beacon. #[error("identity has been known for {known_secs}s, below the {required_secs}s maturity; a fresh key could be ground against a beacon already in hand")] TooNew { @@ -170,12 +177,6 @@ pub fn check(facts: &NodeFacts, policy: EligibilityPolicy, now: u64) -> Result<( { return Err(Ineligible::NoEndpoint); } - if policy.require_qualified && !facts.qualified { - return Err(Ineligible::NotQualified); - } - if policy.require_archive && !facts.archive { - return Err(Ineligible::NotArchive); - } let known = now.saturating_sub(facts.first_seen_secs); if known < policy.maturity_secs { @@ -226,19 +227,27 @@ mod tests { NodeFacts { node_id: [id; 32], opted_in: true, - archive: true, endpoint: Some("node.example:8443".into()), first_seen_secs: NOW - 30 * DAY, last_seen_secs: NOW - 60, - qualified: true, } } #[test] - fn a_qualified_archive_node_that_opted_in_is_eligible() { + fn a_mature_node_that_opted_in_with_an_endpoint_is_eligible() { assert_eq!(check(&good(1), EligibilityPolicy::default(), NOW), Ok(())); } + #[test] + fn a_node_that_has_not_opted_in_is_never_conscripted() { + let mut f = good(5); + f.opted_in = false; + assert_eq!( + check(&f, EligibilityPolicy::default(), NOW), + Err(Ineligible::NotOptedIn) + ); + } + #[test] fn nothing_here_depends_on_a_live_connection() { // The whole point. A node this one has no socket to, and has not heard @@ -277,26 +286,6 @@ mod tests { assert!(matches!(check(&f, p, NOW), Err(Ineligible::TooNew { .. }))); } - #[test] - fn an_unqualified_node_cannot_coordinate() { - let mut f = good(5); - f.qualified = false; - assert_eq!( - check(&f, EligibilityPolicy::default(), NOW), - Err(Ineligible::NotQualified) - ); - } - - #[test] - fn a_non_archive_node_cannot_coordinate() { - let mut f = good(6); - f.archive = false; - assert_eq!( - check(&f, EligibilityPolicy::default(), NOW), - Err(Ineligible::NotArchive) - ); - } - #[test] fn an_endpoint_nobody_can_dial_is_no_endpoint() { // Blank and whitespace both mean unreachable; treating either as an @@ -330,21 +319,9 @@ mod tests { fn the_ineligible_are_absent_rather_than_ranked_last() { let p = EligibilityPolicy::default(); let mut bad = good(4); - bad.qualified = false; + bad.endpoint = None; let roster = eligible_roster(&[good(1), bad, good(2)], p, NOW); assert_eq!(roster.len(), 2); assert!(!roster.contains(&[4u8; 32])); } - - #[test] - fn relaxing_archive_admits_a_non_archive_node() { - // The requirement is a policy, so a test network can run without it. - let mut f = good(8); - f.archive = false; - let p = EligibilityPolicy { - require_archive: false, - ..Default::default() - }; - assert_eq!(check(&f, p, NOW), Ok(())); - } } diff --git a/scripts/mutate-wraith.sh b/scripts/mutate-wraith.sh index d45935236..92bfabf39 100755 --- a/scripts/mutate-wraith.sh +++ b/scripts/mutate-wraith.sh @@ -137,8 +137,8 @@ mutate "eligibility ignores maturity" eligibility.rs \ ' if known < policy.maturity_secs {' \ ' if false {' eligibility -mutate "eligibility ignores archive mode" eligibility.rs \ -' if policy.require_archive && !facts.archive {' \ +mutate "eligibility conscripts nodes that did not opt in" eligibility.rs \ +' if !facts.opted_in {' \ ' if false {' eligibility mutate "assignment ignores the modulus check" assignment.rs \