diff --git a/apps/wraith-wallet/daemon/src/coordinator_resolve.rs b/apps/wraith-wallet/daemon/src/coordinator_resolve.rs index cef38e645..e664fcdbd 100644 --- a/apps/wraith-wallet/daemon/src/coordinator_resolve.rs +++ b/apps/wraith-wallet/daemon/src/coordinator_resolve.rs @@ -1,6 +1,6 @@ -//! Resolve which seated Wraith coordinator owns a wallet's mix, from a -//! published election view — so a wallet can mix without being handed a -//! coordinator URL. +//! Resolve which Wraith coordinator leads a wallet's tier, and where to fall +//! back to, from a published election view — so a wallet can mix without being +//! handed a coordinator URL. //! //! # Where the election comes from now //! @@ -12,14 +12,14 @@ //! //! What lives here is the half that makes asking safe enough to do: an //! election is *recomputed* before it is used, so a relayed view cannot lie -//! about who was seated, and the beacon is pinned to a real block hash rather +//! about who leads a tier, and the beacon is pinned to a real block hash rather //! than taken on the publisher's word (#697). //! //! ⚠ The roster remains a trusted input, and no amount of care here changes //! that — see `verified_election` for why the mesh node-list checkpoint does //! not close it and what would. -use wraith_protocol::election_doc::election_is_honest; +use wraith_protocol::election_doc::{election_is_honest, endpoints_for_tier}; /// Election-document verification lives in `wraith_protocol::election_doc`, not /// here. @@ -31,7 +31,6 @@ use wraith_protocol::election_doc::election_is_honest; /// Two implementations of that is how two parties end up disagreeing about who /// is honest, which is the failure this whole area exists to prevent. pub use wraith_protocol::election_doc::{beacon_anchor_expectation, beacon_matches_chain}; -use wraith_protocol::sortition::shard_for; /// What several nodes say about the roster they drew from. #[derive(Debug, Clone, PartialEq, Eq)] @@ -52,9 +51,9 @@ pub enum RosterAgreement { /// /// # What this catches, and what it does not /// -/// `verify_election` proves a seat list follows from the roster published -/// beside it. It cannot prove the roster is the real qualified set: a node that -/// omits honest candidates — to improve its own odds, or to seat itself — +/// `election_is_honest` proves the tier leaders follow from the roster published +/// beside it. It cannot prove the roster is the real eligible set: a node that +/// omits honest candidates — to improve its own odds, or to lead a tier itself — /// produces an election that verifies perfectly. "No self-nomination" holds /// against the draw, not against control of the input set. /// @@ -95,96 +94,30 @@ pub fn roster_agreement(views: &[serde_json::Value], epoch: u64) -> RosterAgreem } } -/// The seats to try for `shard_key`, in order: the owning seat first, then a -/// deterministic sequence of alternates. -/// -/// # Why the order is fixed rather than chosen +/// Resolve the coordinators to try for a mix of `tier_id` from a node's election +/// JSON. Returns `(endpoints, epoch)`. /// -/// A seat whose node has gone dark leaves the wallets sharded to it with -/// nowhere to go until the epoch flips. The obvious repair — let each wallet -/// try whatever it can reach — costs the exact property the sharding exists to -/// create: wallets that share a key converge on ONE seat, and a larger set is -/// the whole point. Wallets probing independently -/// notice a failure at different moments and scatter across different seats, -/// so the anonymity set fragments precisely when the network is already -/// degraded, and nothing tells the user it happened. +/// `endpoints` is the tier's leader first, then its failover path — empty when +/// the election is disabled, unverifiable, the epoch is missing, or nothing has +/// advertised, in which case the caller falls back to a manually configured +/// URL. Pure (no I/O) so the daemon handler is a thin fetch around it. /// -/// Removing the dead seat from the roster instead does not work either. The -/// roster is deliberately snapshotted a full epoch behind so that every node -/// answers the same question (`wraith_protocol::roster_snapshot`), and that -/// module says plainly that nothing there is consensus — it only gives nodes a -/// commitment to compare so divergence is *seen*. Liveness is a local -/// observation, so a liveness-driven roster is a divergent roster, which is a -/// split election rather than a failover. +/// # Why the order is fixed rather than chosen /// -/// So the fallback is derived from what is already agreed: walk forward from -/// the owning seat. Every wallet on a dead seat moves to the SAME next seat, and -/// it joins that seat's existing cohort rather than forming a new one — the set -/// moves as a body and gets larger, never smaller. During the window where some -/// wallets have noticed and others have not, there are two cohorts, not N. +/// A leader that has gone dark leaves its tier's wallets needing somewhere to +/// go. Letting each wallet try whatever it can reach costs the property the +/// draw exists to create: wallets mixing one denomination converge on ONE +/// coordinator, and a larger set is the whole point. Wallets probing +/// independently notice a failure at different moments and scatter, so the +/// anonymity set fragments precisely when the network is already degraded. /// -/// Needs no liveness consensus, no roster mutation and no new wire field: the -/// order is a function of the published seat count and the shard key. -pub fn seat_try_order(shard_key: &[u8; 32], seat_count: usize) -> Vec { - if seat_count == 0 { - return Vec::new(); - } - let first = shard_for(shard_key, seat_count) as usize; - (0..seat_count).map(|i| (first + i) % seat_count).collect() -} - -/// Coordinator endpoints to try for `shard_key`, in [`seat_try_order`]. -/// -/// The caller dials them in order and stops at the first that answers. Seats -/// that have advertised no endpoint are skipped rather than occupying a -/// position: a seat nobody can dial is not a fallback. -pub fn pick_seat_endpoints(status: &serde_json::Value, shard_key: &[u8; 32]) -> Vec { - if status.get("enabled").and_then(|v| v.as_bool()) != Some(true) { - return Vec::new(); - } - let Some(coords) = status.get("coordinators").and_then(|c| c.as_array()) else { - return Vec::new(); - }; - if coords.is_empty() { - return Vec::new(); - } - - seat_try_order(shard_key, coords.len()) - .into_iter() - .filter_map(|seat| { - coords - .iter() - .find(|c| c.get("seat").and_then(|s| s.as_u64()) == Some(seat as u64))? - .get("endpoint") - .and_then(|e| e.as_str()) - .filter(|e| !e.is_empty()) - .map(String::from) - }) - .collect() -} - -/// The shard key comes from `wraith_protocol`, not from here. -/// -/// It used to be defined in this file, which meant the value a wallet shards -/// on lived somewhere a node could not reach — and the library carried a -/// *different* scheme (by session id) documented as the one "a wallet and -/// every node agree" on, with no callers. Two schemes, different answers, and -/// the dead one inviting whoever wired it up next. -pub use wraith_protocol::shard_key_for_tier_epoch; - -/// Resolve the coordinators to try for a mix of `tier_id` from a node's election -/// JSON. Returns `(endpoints, epoch)`. -/// -/// `endpoints` is the owning seat first, then the deterministic fallbacks from -/// [`seat_try_order`] — empty when the election is disabled, unverifiable, the -/// epoch is missing, or nothing has advertised, in which case the caller falls -/// back to a manually configured URL. Pure (no I/O) so the daemon handler is a -/// thin fetch around it. +/// So every wallet walks the same published order, recomputed from the draw's +/// inputs: a dead leader's cohort moves as one body to the same next node. /// /// It returns the ORDER rather than a single answer on purpose. Returning one /// endpoint and adding a second function for the alternates would put two -/// schemes in this file, which is the mistake recorded above `shard_key_for_tier_epoch` -/// — the dead one sits there inviting whoever wires it up next. +/// schemes in this file, and the dead one sits there inviting whoever wires it +/// up next. pub fn resolve_from_election( election: &serde_json::Value, tier_id: &str, @@ -193,15 +126,19 @@ pub fn resolve_from_election( // Refuse a draw that does not follow from its own published inputs. The // caller falls back to a manually configured coordinator, which is a // worse answer than a verified election and a better one than obeying an - // unverifiable claim about who is in charge. - if election.get("enabled").and_then(|v| v.as_bool()) == Some(true) - && !election_is_honest(election) + // unverifiable claim about who is in charge. A disabled election names + // nobody either, and a document that does not say it is enabled is not + // trusted to be. + if election.get("enabled").and_then(|v| v.as_bool()) != Some(true) + || !election_is_honest(election) { return (Vec::new(), epoch); } - let endpoints = epoch - .map(|ep| pick_seat_endpoints(election, &shard_key_for_tier_epoch(tier_id, ep))) - .unwrap_or_default(); + let endpoints = if epoch.is_some() { + endpoints_for_tier(election, tier_id) + } else { + Vec::new() + }; (endpoints, epoch) } @@ -214,95 +151,129 @@ mod tests { use wraith_protocol::sortition::CoordinatorNodeId; use wraith_protocol::{derive_beacon, snapshot_height_for_epoch}; - /// Build an election document that verifies, by running the real draw. - fn honest_election(epoch: u64, roster_size: u8, seats: usize) -> serde_json::Value { - use wraith_protocol::sortition::elect_coordinators; + /// Build an election document that verifies, by running the real draw — + /// the same shape `ghost-pool` publishes. + fn honest_election(epoch: u64, roster_size: u8) -> serde_json::Value { + use wraith_protocol::EpochCoordinators; let beacon = [9u8; 32]; let roster: Vec = (0..roster_size).map(|i| [i; 32]).collect(); - let elected = elect_coordinators(&beacon, epoch, &roster, seats); + let schedule = EpochCoordinators::elect(epoch, &beacon, &roster); json!({ "enabled": true, "epoch": epoch, - "seats": seats, "beacon": hex::encode(beacon), "anchor_height": epoch * 144, "roster": roster.iter().map(hex::encode).collect::>(), - "coordinators": elected.iter().map(|c| json!({ - "node_id": hex::encode(c.node_id), - "seat": c.seat, - "rank": hex::encode(c.rank), - "endpoint": format!("http://seat{}:9100", c.seat), + "tiers": schedule.tiers.iter().map(|t| json!({ + "tier": t.tier_id, + "leader": t.leader().map(hex::encode), + "order": t.order.iter().map(hex::encode).collect::>(), + })).collect::>(), + "coordinators": schedule.roster.iter().map(|id| json!({ + "node_id": hex::encode(id), + "endpoint": format!("http://node{}:9100", id[0]), + "leads": schedule.tiers_led_by(id), })).collect::>(), }) } + fn tier_index(e: &serde_json::Value, tier: &str) -> usize { + e["tiers"] + .as_array() + .unwrap() + .iter() + .position(|t| t["tier"] == tier) + .unwrap() + } + #[test] fn an_honest_election_verifies_and_resolves() { - let e = honest_election(7, 6, 3); + let e = honest_election(7, 6); assert!(election_is_honest(&e)); let (endpoints, epoch) = resolve_from_election(&e, "100k_sats"); - let endpoint = endpoints.first().cloned(); assert_eq!(epoch, Some(7)); - assert!(endpoint.is_some(), "a verified election must resolve"); + assert_eq!(endpoints.len(), 6, "the leader, then every other node"); + let leader = e["tiers"][tier_index(&e, "100k_sats")]["leader"] + .as_str() + .unwrap(); + let leader_ep = e["coordinators"] + .as_array() + .unwrap() + .iter() + .find(|c| c["node_id"] == leader) + .unwrap()["endpoint"] + .as_str() + .unwrap(); + assert_eq!(endpoints[0], leader_ep, "the leader is tried first"); } /// The attack the verification exists for: whoever relays the election - /// names itself every seat. Before this check the wallet would have - /// dialled it (#697). + /// names itself leader. Before this check the wallet would have dialled it + /// (#697). #[test] - fn a_seat_list_that_does_not_follow_from_the_beacon_is_refused() { - let mut e = honest_election(7, 6, 3); - let usurper = hex::encode([0xEE; 32]); - for c in e["coordinators"].as_array_mut().unwrap() { - c["node_id"] = json!(usurper); - } + fn a_leader_that_does_not_follow_from_the_beacon_is_refused() { + let mut e = honest_election(7, 6); + let i = tier_index(&e, "100m_sats"); + let order = e["tiers"][i]["order"].as_array_mut().unwrap(); + order.swap(0, 1); assert!(!election_is_honest(&e)); - assert!(resolve_from_election(&e, "100k_sats").0.is_empty()); + assert!(resolve_from_election(&e, "100m_sats").0.is_empty()); } - /// Dropping a qualified node from the roster would change who wins, so - /// the published roster has to be the one the draw was made from. + /// Leaving `tiers` intact and forging the per-node summary is refused too: + /// it is what a reader looks at. #[test] - fn a_trimmed_roster_is_refused() { - let mut e = honest_election(7, 6, 3); - e["roster"].as_array_mut().unwrap().truncate(3); + fn a_forged_leads_list_is_refused() { + let mut e = honest_election(7, 6); + for c in e["coordinators"].as_array_mut().unwrap() { + c["leads"] = json!(["100k_sats", "1m_sats", "10m_sats", "100m_sats"]); + } assert!(!election_is_honest(&e)); } - /// A forged rank is refused even when the winner is right — the rank is - /// the evidence, not decoration. + /// Dropping a node from the roster would change who leads, so the + /// published roster has to be the one the draw was made from. #[test] - fn a_forged_rank_is_refused() { - let mut e = honest_election(7, 6, 3); - e["coordinators"][0]["rank"] = json!(hex::encode([0u8; 32])); + fn a_trimmed_roster_is_refused() { + let mut e = honest_election(7, 6); + e["roster"].as_array_mut().unwrap().truncate(3); assert!(!election_is_honest(&e)); } /// Swapping the beacon re-draws the whole election, so a substituted one - /// cannot match the published seats. + /// cannot match the published leaders. #[test] fn a_substituted_beacon_is_refused() { - let mut e = honest_election(7, 6, 3); + let mut e = honest_election(7, 6); e["beacon"] = json!(hex::encode([1u8; 32])); assert!(!election_is_honest(&e)); } - /// An election missing the inputs entirely — which is what every node - /// published before this commit — cannot be verified, so it is not used. + /// An election missing the inputs cannot be verified, so it is not used. #[test] fn an_election_without_its_inputs_is_refused() { - let mut e = honest_election(7, 6, 3); + let mut e = honest_election(7, 6); e.as_object_mut().unwrap().remove("beacon"); assert!(!election_is_honest(&e)); assert!(resolve_from_election(&e, "100k_sats").0.is_empty()); } + /// The old seat-shaped document names no tiers, so it cannot be verified + /// and a wallet falls back rather than guessing what it meant. + #[test] + fn a_seat_shaped_document_is_refused() { + let mut e = honest_election(7, 6); + e.as_object_mut().unwrap().remove("tiers"); + e["seats"] = json!(1); + assert!(!election_is_honest(&e)); + } + /// The anchor height is derived from the epoch, never read from the /// document — otherwise a publisher could name whichever block produced /// a beacon it liked and stay perfectly self-consistent. #[test] fn the_anchor_height_comes_from_the_epoch_not_the_document() { - let mut e = honest_election(7, 6, 3); + let mut e = honest_election(7, 6); e["anchor_height"] = json!(999_999); let (height, _) = beacon_anchor_expectation(&e).expect("has a beacon"); assert_eq!(height, snapshot_height_for_epoch(7)); @@ -313,18 +284,18 @@ mod tests { #[test] fn a_chain_derived_beacon_is_accepted() { let epoch = 11u64; - let mut e = honest_election(epoch, 6, 3); + let mut e = honest_election(epoch, 6); e["beacon"] = json!(hex::encode(derive_beacon(epoch, &[3u8; 32]))); assert!(beacon_matches_chain(&e, &hex::encode([3u8; 32]))); } /// **The attack this closes.** A publisher invents a beacon, then builds a - /// seat list that follows from it perfectly — so `election_is_honest` + /// schedule that follows from it perfectly — so `election_is_honest` /// passes. It cannot survive contact with the chain: the anchor block's /// hash is not something the publisher gets to state. #[test] fn a_fabricated_beacon_is_caught_by_the_chain_even_though_it_is_self_consistent() { - let e = honest_election(11, 6, 3); + let e = honest_election(11, 6); assert!(election_is_honest(&e), "internally consistent"); assert!(!beacon_matches_chain(&e, &hex::encode([3u8; 32]))); } @@ -332,7 +303,7 @@ mod tests { /// A malformed anchor hash is a refusal, not an accident that passes. #[test] fn a_malformed_anchor_hash_does_not_verify() { - let e = honest_election(11, 6, 3); + let e = honest_election(11, 6); assert!(!beacon_matches_chain(&e, "not-hex")); assert!(!beacon_matches_chain(&e, "")); } @@ -345,195 +316,51 @@ mod tests { assert_eq!(resolve_from_election(&e, "100k_sats"), (Vec::new(), None)); } - fn status(enabled: bool, coords: serde_json::Value) -> serde_json::Value { - json!({ "enabled": enabled, "coordinators": coords }) - } - - #[test] - fn picks_a_seated_endpoint_deterministically_for_a_key() { - let s = status( - true, - json!([ - {"node_id":"aa","seat":0,"endpoint":"http://a:9100"}, - {"node_id":"bb","seat":1,"endpoint":"http://b:9100"}, - ]), - ); - let key = [7u8; 32]; - // Same key → same owner (so wallets converge), and it's one of the seats. - let a = pick_seat_endpoints(&s, &key).first().cloned(); - assert_eq!(a, pick_seat_endpoints(&s, &key).first().cloned()); - assert!(matches!( - a.as_deref(), - Some("http://a:9100") | Some("http://b:9100") - )); - } - - #[test] - fn none_when_disabled_empty_or_unadvertised() { - // Disabled election. - assert_eq!( - pick_seat_endpoints(&status(false, json!([])), &[0u8; 32]) - .first() - .cloned(), - None - ); - // No coordinators seated. - assert_eq!( - pick_seat_endpoints(&status(true, json!([])), &[0u8; 32]) - .first() - .cloned(), - None - ); - // Single seat whose owner hasn't advertised an endpoint yet. - let s = status(true, json!([{"node_id":"aa","seat":0,"endpoint":null}])); - assert_eq!(pick_seat_endpoints(&s, &[0u8; 32]).first().cloned(), None); - // Empty-string endpoint is treated as unadvertised. - let s = status(true, json!([{"node_id":"aa","seat":0,"endpoint":""}])); - assert_eq!(pick_seat_endpoints(&s, &[0u8; 32]).first().cloned(), None); - } - - #[test] - fn shard_key_is_deterministic_and_separates_tier_and_epoch() { - // Stable for the same (tier, epoch) → wallets converge. - assert_eq!( - shard_key_for_tier_epoch("0.01btc", 100), - shard_key_for_tier_epoch("0.01btc", 100) - ); - // Different tier OR epoch → different key. - assert_ne!( - shard_key_for_tier_epoch("0.01btc", 100), - shard_key_for_tier_epoch("0.1btc", 100) - ); - assert_ne!( - shard_key_for_tier_epoch("0.01btc", 100), - shard_key_for_tier_epoch("0.01btc", 101) - ); - } - #[test] fn resolve_from_election_uses_epoch_and_falls_back() { - // A verified election resolves, and reports its epoch. - let s = honest_election(42, 6, 2); - let (eps, epoch) = resolve_from_election(&s, "0.01btc"); - let ep = eps.first().cloned(); + let s = honest_election(42, 6); + let (eps, epoch) = resolve_from_election(&s, "1m_sats"); assert_eq!(epoch, Some(42)); - assert!(ep.is_some()); - - // The same document without its inputs used to resolve too — the - // wallet took the seat list on trust. It now reports the epoch and - // refuses the endpoint, so the caller falls back (#697). - let unverifiable = json!({ - "enabled": true, - "epoch": 42, - "coordinators": [ - {"node_id":"aa","seat":0,"endpoint":"http://a:9100"}, - {"node_id":"bb","seat":1,"endpoint":"http://b:9100"}, - ] - }); - assert_eq!( - ( - resolve_from_election(&unverifiable, "0.01btc") - .0 - .first() - .cloned(), - resolve_from_election(&unverifiable, "0.01btc").1 - ), - (None, Some(42)) - ); + assert!(!eps.is_empty()); // No epoch (election pending) → no endpoint, caller falls back. let pending = json!({ "enabled": true, "epoch": null, "coordinators": [] }); assert_eq!( - resolve_from_election(&pending, "0.01btc"), + resolve_from_election(&pending, "1m_sats"), (Vec::new(), None) ); - // Disabled → nothing. - let off = json!({ "enabled": false }); - assert_eq!(resolve_from_election(&off, "0.01btc"), (Vec::new(), None)); - } - - /// The property the whole design rests on: every wallet sharded to a dead - /// seat moves to the SAME next seat. - /// - /// If wallets each probed for whatever they could reach, they would notice - /// the failure at different moments and scatter — the anonymity set - /// fragmenting exactly when the network is already degraded. Here the set - /// moves as a body, and joins the destination seat's existing cohort rather - /// than forming a new one. - #[test] - fn every_wallet_on_a_seat_falls_back_to_the_same_seat() { - // Two different keys that happen to shard to the same seat stand in for - // two wallets in one cohort: whatever their keys, the ORDER after the - // owning seat is a function of the seat count, so the cohort cannot split. - let n = 5; - for a in 0u8..40 { - for b in 0u8..40 { - let ka = [a; 32]; - let kb = [b; 32]; - let oa = seat_try_order(&ka, n); - let ob = seat_try_order(&kb, n); - if oa[0] == ob[0] { - assert_eq!( - oa, ob, - "two wallets on seat {} disagreed about where to go next", - oa[0] - ); - } - } - } - } - - /// The first seat tried must be the one that owns the key, or the fallback - /// order would quietly move every wallet off its own seat. - #[test] - fn the_order_starts_at_the_owning_seat() { - for i in 0u8..20 { - let key = [i; 32]; - for n in 1usize..8 { - assert_eq!(seat_try_order(&key, n)[0], shard_for(&key, n) as usize); - } - } - } - - /// Every seat appears exactly once: no seat is unreachable as a fallback, - /// and none is tried twice. - #[test] - fn the_order_is_a_permutation_of_the_seats() { - for n in 1usize..12 { - let order = seat_try_order(&[7u8; 32], n); - assert_eq!(order.len(), n); - let mut seen = order.clone(); - seen.sort_unstable(); - seen.dedup(); - assert_eq!(seen.len(), n, "n={n} produced a duplicate or a gap"); - } + // A tier the protocol does not have resolves to nothing. + assert!(resolve_from_election(&s, "0.01btc").0.is_empty()); } - /// A seat that has advertised no endpoint is skipped rather than occupying a - /// position — a seat nobody can dial is not a fallback. + /// Every tier resolves to a different leader, and every wallet mixing one + /// tier walks the same path — so a dead leader's cohort moves together. #[test] - fn seats_without_an_endpoint_are_skipped() { - let status = json!({ - "enabled": true, - "coordinators": [ - { "seat": 0, "endpoint": "" }, - { "seat": 1, "endpoint": "1.2.3.4:9100" }, - { "seat": 2, "endpoint": "5.6.7.8:9100" }, - ], - }); - let endpoints = pick_seat_endpoints(&status, &[3u8; 32]); - assert_eq!(endpoints.len(), 2, "the empty endpoint must not be offered"); - assert!(!endpoints.iter().any(|e| e.is_empty())); + fn tiers_resolve_to_different_leaders_on_one_shared_path_each() { + let e = honest_election(42, 8); + let firsts: std::collections::HashSet = + ["100k_sats", "1m_sats", "10m_sats", "100m_sats"] + .iter() + .map(|t| resolve_from_election(&e, t).0[0].clone()) + .collect(); + assert_eq!(firsts.len(), 4, "four tiers, four different coordinators"); + assert_eq!( + resolve_from_election(&e, "10m_sats"), + resolve_from_election(&e.clone(), "10m_sats"), + "two wallets on one tier get one path" + ); } - /// No seats, no panic — and no endpoints to pretend otherwise. + /// A node that has advertised no endpoint is skipped rather than holding a + /// place — a node nobody can dial is not a fallback. #[test] - fn an_empty_election_yields_nothing_to_try() { - assert!(seat_try_order(&[0u8; 32], 0).is_empty()); - let off = json!({ "enabled": false }); - assert!(pick_seat_endpoints(&off, &[0u8; 32]).is_empty()); - let empty = json!({ "enabled": true, "coordinators": [] }); - assert!(pick_seat_endpoints(&empty, &[0u8; 32]).is_empty()); + fn nodes_without_an_endpoint_are_skipped() { + let mut e = honest_election(42, 6); + e["coordinators"][2]["endpoint"] = json!(""); + e["coordinators"][3]["endpoint"] = serde_json::Value::Null; + let (eps, _) = resolve_from_election(&e, "100k_sats"); + assert_eq!(eps.len(), 4); + assert!(!eps.iter().any(|x| x.is_empty())); } fn view(epoch: u64, commitment: &str) -> serde_json::Value { diff --git a/apps/wraith-wallet/daemon/src/main.rs b/apps/wraith-wallet/daemon/src/main.rs index ff975fadb..72b0bf6c9 100644 --- a/apps/wraith-wallet/daemon/src/main.rs +++ b/apps/wraith-wallet/daemon/src/main.rs @@ -5905,8 +5905,8 @@ mod server { } None => (Vec::new(), None), }; - // Head is the seat that owns this tier; the tail is where every - // wallet on that seat goes if it stops answering — the same + // Head is the node leading this tier; the tail is where every + // wallet on the tier goes if it stops answering — the same // order for all of them, so the cohort moves together (#711). let mut it = endpoints.into_iter(); let endpoint = it.next(); diff --git a/bins/ghost-pool/src/coordinator_election.rs b/bins/ghost-pool/src/coordinator_election.rs index 0fda8dd30..2c082472e 100644 --- a/bins/ghost-pool/src/coordinator_election.rs +++ b/bins/ghost-pool/src/coordinator_election.rs @@ -55,9 +55,9 @@ use wraith_protocol::sortition::CoordinatorNodeId; pub const COORDINATOR_EPOCH_BLOCKS: u64 = wraith_protocol::EPOCH_BLOCKS; /// Below this many opted-in candidates, the election is reported as -/// `degraded`: the draw still runs and still seats someone, but with one or +/// `degraded`: the draw still runs and still names leaders, but with one or /// two candidates it cannot deliver rotation or resistance to -/// self-nomination, and saying so is better than publishing a seat list that +/// self-nomination, and saying so is better than publishing a schedule that /// looks like an election. pub const MIN_MEANINGFUL_ROSTER: usize = 3; @@ -68,11 +68,6 @@ pub fn roster_is_degraded(roster_size: usize) -> bool { roster_size < MIN_MEANINGFUL_ROSTER } -/// Target number of concurrent coordinator seats per epoch. Sessions are -/// sharded across these seats so no single coordinator owns every round. -/// (Demand-driven sizing replaces this fixed target in a later increment.) -pub const COORDINATOR_SEATS: usize = 5; - // REMOVED: `COORDINATOR_PEER_FRESHNESS_SECS` (300s). // // It required a peer to have pinged within five minutes to be electable. The @@ -86,31 +81,18 @@ pub const COORDINATOR_SEATS: usize = 5; // actually agree; an unreachable node costs one timeout as callers walk past // it, which is a latency cost rather than a correctness one. -/// Demand-driven seat sizing. Recent mixing sessions per seat before another -/// seat is added; minimum seats whenever any coordinator is eligible (so there -/// is always at least one, for liveness); and a hard ceiling. All tunable. -const TARGET_SESSIONS_PER_SEAT: u64 = 50; -const MIN_SEATS: usize = 1; -const MAX_SEATS: usize = 16; - -/// Size the coordinator seat count for an epoch from the frozen, mesh-summed -/// recent session `demand` and the number of `eligible` coordinators. -/// -/// `ceil(demand / TARGET_SESSIONS_PER_SEAT)`, floored at `MIN_SEATS` and capped -/// by both `MAX_SEATS` and the eligible set (can't seat more coordinators than -/// exist). Coarse buckets (one seat per `TARGET_SESSIONS_PER_SEAT`) make the -/// result robust to small per-node differences in the demand snapshot: nodes -/// only disagree on the count near a bucket edge, and even then the only cost is -/// a briefly-suboptimal session spread, never a safety issue (the CoinJoin is -/// atomic + blind-signed whichever seat runs it). Pure + deterministic so every -/// node computes the same seats from the same frozen inputs. -pub fn seats_for_demand(demand: u64, eligible: usize) -> usize { - if eligible == 0 { - return 0; - } - let by_demand = (demand.div_ceil(TARGET_SESSIONS_PER_SEAT) as usize).max(MIN_SEATS); - by_demand.min(MAX_SEATS).min(eligible) -} +// REMOVED: seats and `seats_for_demand`. +// +// The draw used to seat `ceil(demand / 50)` coordinators and shard tiers across +// them. Mainnet demand is zero, so that was one seat: one node carried every +// denomination for a day while every other opted-in node sat idle and earned +// nothing. And demand was read from each node's own snapshot of gossiped session +// counters, so two nodes with identical rosters could still disagree on the +// count and send a tier to different coordinators. +// +// Every tier now has its own leader (`wraith_protocol::sortition::tier_leaders`) +// and every opted-in node runs a coordinator, so there is no count to size or +// agree on. /// The coordinator epoch a chain height falls in. pub const fn epoch_for_height(height: u64) -> u64 { @@ -157,9 +139,9 @@ struct Cached { /// The beacon the draw was made with, and the roster it drew from. /// /// Published alongside the result so a wallet can recompute the election - /// and check it (`sortition::verify_election`) rather than believing the - /// seat list it is handed. Without these two the draw is unfalsifiable: - /// anyone relaying the view could seat whoever they liked (#697). + /// and check it (`election_doc::election_is_honest`) rather than believing + /// the leaders it is handed. Without these two the draw is unfalsifiable: + /// anyone relaying the view could name whoever they liked (#697). beacon: [u8; 32], roster: Vec, /// Height of the block whose hash the beacon is derived from, so the @@ -177,13 +159,6 @@ struct Cached { /// The endpoint map the view was built with, kept so a refresh can tell /// whether anything a wallet would dial has changed. endpoints: EndpointMap, - /// Session demand as first read this epoch. Frozen: the roster may still - /// change within an epoch, the seat target may not, or ordinary churn in - /// the session counters would reshuffle seats every few minutes. - demand: u64, - /// This node held a seat in some view of this epoch, not necessarily the - /// current one. See [`CoordinatorElection::should_serve`]. - seated_this_epoch: bool, } /// The view to cache next, or `None` to keep the current one. @@ -203,41 +178,40 @@ struct Cached { /// coordinator from everyone else. /// /// Recomputing the roster whenever it is asked for lets every node converge on -/// the gossip it now holds. The beacon and the seat target stay frozen for the -/// epoch; only the roster, and with it the draw, may move. +/// the gossip it now holds. The beacon stays frozen for the epoch; only the +/// roster, and with it the draw, may move. +/// +/// A tier's lead can therefore move mid-epoch. That aborts nothing: every +/// opted-in node runs its coordinator whether it leads or not, so the old leader +/// finishes the rounds it holds while new wallets follow the new view. fn next_view( prev: Option<&Cached>, - self_id: &CoordinatorNodeId, epoch: u64, fresh_beacon: Option<[u8; 32]>, roster: Vec, endpoints: EndpointMap, - demand: u64, ) -> Option { - let (beacon, demand, seated_before) = match prev.filter(|c| c.epoch == epoch) { + let beacon = match prev.filter(|c| c.epoch == epoch) { Some(c) => { if c.roster == roster && c.endpoints == endpoints { return None; } - (c.beacon, c.demand, c.seated_this_epoch) + c.beacon } // A new epoch needs its own beacon. Without one, keep the last good view // rather than cache a partial one. - None => (fresh_beacon?, demand, false), + None => fresh_beacon?, }; - let seats = seats_for_demand(demand, roster.len()); - let view = CoordinatorView::build(epoch, &beacon, &roster, endpoints.clone(), seats); + let view = CoordinatorView::build(epoch, &beacon, &roster, endpoints.clone()); let anchor_height = anchor_height_for_epoch(epoch); Some(Cached { epoch, - seated_this_epoch: seated_before || view.am_i_coordinator(self_id), view, beacon, roster_commitment: roster_commitment(epoch, anchor_height, &roster), roster, anchor_height, endpoints, - demand, }) } @@ -336,16 +310,14 @@ impl CoordinatorElection { /// `Cached::roster_commitment` stays regardless — it is how a split is /// *seen*, and it is the only field in the status response one node cannot /// self-check. - /// Returns the canonical roster, the endpoint map, and the summed recent - /// session `demand` across the eligible set (incl. self) — the frozen input - /// to [`seats_for_demand`]. - fn roster_with_endpoints(&self) -> (Vec, EndpointMap, u64) { + /// + /// Returns the canonical roster and the endpoint map. + fn roster_with_endpoints(&self) -> (Vec, EndpointMap) { let now = chrono::Utc::now().timestamp().max(0) as u64; let policy = EligibilityPolicy::default(); let mut endpoints = EndpointMap::new(); let mut facts: Vec = Vec::new(); - let mut demand: u64 = 0; // `all_peers`, not `get_connected_peers`. Eligibility must not depend on // whether THIS node currently holds a socket — that is what made two @@ -367,7 +339,6 @@ impl CoordinatorElection { endpoints.insert(p.node_id, ep); } } - demand = demand.saturating_add(p.coordinator_sessions as u64); facts.push(f); } @@ -385,13 +356,12 @@ impl CoordinatorElection { first_seen_secs: 0, last_seen_secs: now, }); - demand = demand.saturating_add(self.mesh.coordinator_sessions() as u64); } } let roster = eligible_roster(&facts, policy, now); endpoints.retain(|id, _| roster.contains(id)); - (canonical_roster(&roster), endpoints, demand) + (canonical_roster(&roster), endpoints) } /// Fetch the beacon for `epoch` by anchoring on the epoch-start block hash @@ -416,7 +386,7 @@ impl CoordinatorElection { /// poisons the cache with a partial view. pub async fn refresh_for_height(&self, current_height: u64) -> u64 { let epoch = epoch_for_height(current_height); - let (roster, endpoints, demand) = self.roster_with_endpoints(); + let (roster, endpoints) = self.roster_with_endpoints(); let same_epoch = self .cached @@ -437,15 +407,7 @@ impl CoordinatorElection { let guard = self.cached.read(); let prev = guard.as_ref(); let previous_size = prev.filter(|c| c.epoch == epoch).map(|c| c.roster.len()); - let next = next_view( - prev, - &self.self_id, - epoch, - fresh_beacon, - roster, - endpoints, - demand, - ); + let next = next_view(prev, epoch, fresh_beacon, roster, endpoints); (next, previous_size) }; let Some(next) = next else { @@ -458,7 +420,7 @@ impl CoordinatorElection { within_epoch = same_epoch, roster_size = next.roster.len(), previous_roster_size = ?previous_size, - seats = next.view.seats(), + leads = ?next.view.tiers_led_by(&self.self_id), roster_commitment = %hex::encode(next.roster_commitment), "Coordinator roster changed" ); @@ -466,101 +428,111 @@ impl CoordinatorElection { epoch } - /// Whether THIS node is an elected coordinator in the currently-cached - /// epoch. `false` before the first successful recompute. Read-only — this - /// does NOT activate any coordinator behaviour, it only reports the draw. - pub fn am_i_coordinator(&self) -> bool { - self.cached - .read() - .as_ref() - .map(|c| c.view.am_i_coordinator(&self.self_id)) - .unwrap_or(false) - } - - /// Whether THIS node should be running its coordinator: seated now, **or - /// seated at any point earlier in this epoch**. + /// Whether THIS node should be running its coordinator: whenever it is on + /// the roster, leading a tier or not. `false` before the first successful + /// recompute. /// - /// Distinct from [`Self::am_i_coordinator`] because the roster can now move - /// within an epoch. A node that loses its seat mid-epoch may be holding - /// rounds that participants have already committed inputs to, and stopping - /// the coordinator aborts them. So it keeps serving until the epoch turns: - /// new wallets follow the current view elsewhere, and the rounds it already - /// holds get to finish. The cost is an idle coordinator for the rest of a - /// day, which is cheap. + /// Not "whenever it leads". The nodes that do not lead a tier are every + /// tier's failover path, and a path is only a path if the nodes on it are + /// listening. It also means a lead that moves mid-epoch aborts nothing: the + /// old leader is still serving and finishes what it holds. pub fn should_serve(&self) -> bool { self.cached .read() .as_ref() - .is_some_and(|c| c.seated_this_epoch) + .is_some_and(|c| c.view.serves(&self.self_id)) } - /// A JSON snapshot of the cached election for the read-only HTTP endpoint: - /// `{enabled, roster_commitment, epoch, seats, my_seat, elected: [hex ids], - /// [{node_id, seat, endpoint}]}`. The `coordinators` array is what a wallet - /// reads to dial the seat that owns its session; `elected` is kept as the - /// flat hex list for existing consumers. Pre-serialised so + /// A JSON snapshot of the cached election for the read-only HTTP endpoint. + /// + /// `tiers` is each denomination's leader and failover order; `coordinators` + /// is every roster node with its endpoint and the tiers it leads, which is + /// what a wallet dials; `my_tiers` / `my_endpoint` are this node's own part. + /// The draw's inputs (`beacon`, `anchor_height`, `roster`) are published so a + /// consumer recomputes it rather than trusting it. Pre-serialised so /// `ghost-verification` needn't depend on `wraith-protocol`. pub fn status_json(&self) -> serde_json::Value { - let guard = self.cached.read(); - let Some(c) = guard.as_ref() else { - // Service is on but hasn't computed a view yet (e.g. anchor block - // not reachable). Report enabled-but-pending rather than failing. - return serde_json::json!({ - "enabled": true, - "epoch": serde_json::Value::Null, - "seats": 0, - "my_seat": serde_json::Value::Null, - "elected": [], - "coordinators": [], - "beacon": serde_json::Value::Null, - "anchor_height": serde_json::Value::Null, - "roster": [], - "roster_size": 0, - "roster_commitment": serde_json::Value::Null, - "degraded": true, - }); - }; + status_json_for(&self.cached, &self.self_id) + } +} - let seated = c.view.seated(); - let elected: Vec = seated.iter().map(|s| hex::encode(s.node_id)).collect(); - let coordinators: Vec = seated - .iter() - .map(|s| { - serde_json::json!({ - "node_id": hex::encode(s.node_id), - "seat": s.seat, - "rank": hex::encode(s.rank), - "endpoint": s.endpoint, - }) - }) - .collect(); - serde_json::json!({ +/// The body of [`CoordinatorElection::status_json`], free of the service so a +/// test can render exactly what the endpoint serves. +fn status_json_for( + cached: &RwLock>, + self_id: &CoordinatorNodeId, +) -> serde_json::Value { + let guard = cached.read(); + let Some(c) = guard.as_ref() else { + // Service is on but hasn't computed a view yet (e.g. anchor block + // not reachable). Report enabled-but-pending rather than failing. + return serde_json::json!({ "enabled": true, - // Compare this across nodes: equal means they drew from the same - // roster, unequal means the coordinator layer has split. It is the - // only field here that a single node cannot self-check. - "roster_commitment": hex::encode(c.roster_commitment), - "epoch": c.view.epoch(), - "seats": c.view.seats(), - "my_seat": c.view.my_seat(&self.self_id), - "elected": elected, - "coordinators": coordinators, - // The draw's inputs, so a consumer can recompute it rather than - // trust it (#697). `beacon` is SHA256(domain ‖ epoch ‖ anchor - // hash), and `anchor_height` names the block that anchor comes - // from — so the beacon is re-derivable straight from the chain - // and a publisher cannot invent one. - "beacon": hex::encode(c.beacon), - "anchor_height": c.anchor_height, - "roster": c.roster.iter().map(hex::encode).collect::>(), - // A draw over one candidate is not a draw. Reported so a reader - // cannot mistake a single opted-in node for an election that - // rotated, and so "no single party is the operator" is checkable - // rather than assumed (#708). - "roster_size": c.roster.len(), - "degraded": roster_is_degraded(c.roster.len()), + "epoch": serde_json::Value::Null, + "tiers": [], + "coordinators": [], + "my_tiers": [], + "my_endpoint": serde_json::Value::Null, + "beacon": serde_json::Value::Null, + "anchor_height": serde_json::Value::Null, + "roster": [], + "roster_size": 0, + "roster_commitment": serde_json::Value::Null, + "degraded": true, + }); + }; + + let tiers: Vec = c + .view + .tiers() + .iter() + .map(|t| { + serde_json::json!({ + "tier": t.tier_id, + "leader": t.leader().map(hex::encode), + "order": t.order.iter().map(hex::encode).collect::>(), + }) }) - } + .collect(); + let coordinators: Vec = c + .view + .serving() + .into_iter() + .map(|s| { + serde_json::json!({ + "node_id": hex::encode(s.node_id), + "endpoint": s.endpoint, + "leads": s.leads, + }) + }) + .collect(); + let serving = c.view.serves(self_id); + serde_json::json!({ + "enabled": true, + // Compare this across nodes: equal means they drew from the same + // roster, unequal means the coordinator layer has split. It is the + // only field here that a single node cannot self-check. + "roster_commitment": hex::encode(c.roster_commitment), + "epoch": c.view.epoch(), + "tiers": tiers, + "coordinators": coordinators, + "my_tiers": c.view.tiers_led_by(self_id), + "my_endpoint": if serving { c.endpoints.get(self_id).cloned() } else { None }, + // The draw's inputs, so a consumer can recompute it rather than + // trust it (#697). `beacon` is SHA256(domain ‖ epoch ‖ anchor + // hash), and `anchor_height` names the block that anchor comes + // from — so the beacon is re-derivable straight from the chain + // and a publisher cannot invent one. + "beacon": hex::encode(c.beacon), + "anchor_height": c.anchor_height, + "roster": c.roster.iter().map(hex::encode).collect::>(), + // A draw over one candidate is not a draw. Reported so a reader + // cannot mistake a single opted-in node for an election that + // rotated, and so "no single party is the operator" is checkable + // rather than assumed (#708). + "roster_size": c.roster.len(), + "degraded": roster_is_degraded(c.roster.len()), + }) } /// The JSON returned for the read-only endpoint when the feature is OFF (the @@ -651,103 +623,66 @@ mod tests { // ── election-through-the-view tests (the library + our reporting shape) ── + const TIERS: [&str; 4] = ["100k_sats", "1m_sats", "10m_sats", "100m_sats"]; + + fn leaders(v: &CoordinatorView) -> Vec { + v.tiers() + .iter() + .filter_map(|t| t.leader().copied()) + .collect() + } + #[test] fn election_is_deterministic_for_fixed_inputs() { let roster: Vec<_> = (0u8..12).map(node).collect(); let beacon = derive_beacon(3, &[1u8; 32]); - let a = CoordinatorView::build(3, &beacon, &roster, EndpointMap::new(), COORDINATOR_SEATS); - let b = CoordinatorView::build(3, &beacon, &roster, EndpointMap::new(), COORDINATOR_SEATS); - // Same inputs → identical seating. - assert_eq!(a.seats(), b.seats()); - assert_eq!(a.seats(), COORDINATOR_SEATS); - for id in &roster { - assert_eq!(a.my_seat(id), b.my_seat(id)); - assert_eq!(a.am_i_coordinator(id), b.am_i_coordinator(id)); - } + let a = CoordinatorView::build(3, &beacon, &roster, EndpointMap::new()); + let b = CoordinatorView::build(3, &beacon, &roster, EndpointMap::new()); + assert_eq!(a.tiers(), b.tiers()); } #[test] fn epoch_advancement_changes_the_view() { let roster: Vec<_> = (0u8..20).map(node).collect(); let anchor = [9u8; 32]; - let v_e3 = CoordinatorView::build( - 3, - &derive_beacon(3, &anchor), - &roster, - EndpointMap::new(), - COORDINATOR_SEATS, - ); - let v_e4 = CoordinatorView::build( - 4, - &derive_beacon(4, &anchor), - &roster, - EndpointMap::new(), - COORDINATOR_SEATS, - ); - let seated = |v: &CoordinatorView| -> Vec { - roster - .iter() - .copied() - .filter(|id| v.am_i_coordinator(id)) - .collect() - }; + let v3 = CoordinatorView::build(3, &derive_beacon(3, &anchor), &roster, EndpointMap::new()); + let v4 = CoordinatorView::build(4, &derive_beacon(4, &anchor), &roster, EndpointMap::new()); assert_ne!( - seated(&v_e3), - seated(&v_e4), - "a new epoch must reshuffle the coordinator set" + leaders(&v3), + leaders(&v4), + "a new epoch must reshuffle the leaders" ); } + /// The operator's requirement: opted-in nodes are called on. One seat used to + /// carry every tier while the rest of the roster idled. #[test] - fn self_as_coordinator_detection_matches_the_view() { - let roster: Vec<_> = (0u8..30).map(node).collect(); - let beacon = derive_beacon(2, &[5u8; 32]); - let view = - CoordinatorView::build(2, &beacon, &roster, EndpointMap::new(), COORDINATOR_SEATS); - // For every roster member, am_i_coordinator agrees with my_seat.is_some. - let mut seated_count = 0; + fn four_tiers_go_to_four_different_nodes_and_everyone_serves() { + let roster: Vec<_> = (1u8..=8).map(node).collect(); + let view = CoordinatorView::build( + 6711, + &derive_beacon(6711, &[3u8; 32]), + &roster, + EndpointMap::new(), + ); + let distinct: std::collections::HashSet<_> = leaders(&view).into_iter().collect(); + assert_eq!(distinct.len(), TIERS.len()); for id in &roster { - let is_coord = view.am_i_coordinator(id); - assert_eq!(is_coord, view.my_seat(id).is_some()); - if is_coord { - seated_count += 1; - } + assert!(view.serves(id), "every roster node runs a coordinator"); } - assert_eq!(seated_count, COORDINATOR_SEATS); - // A node not in the roster is never seated. - assert!(!view.am_i_coordinator(&node(200))); + assert!(!view.serves(&node(200))); } #[test] - fn empty_roster_seats_nobody() { + fn empty_roster_leads_and_serves_nothing() { let beacon = derive_beacon(1, &[0u8; 32]); - let view = CoordinatorView::build(1, &beacon, &[], EndpointMap::new(), COORDINATOR_SEATS); - assert_eq!(view.seats(), 0); - assert!(!view.am_i_coordinator(&node(0))); - } - - #[test] - fn seats_scale_with_demand_and_clamp() { - // No eligible coordinators → no seats, regardless of demand. - assert_eq!(seats_for_demand(1000, 0), 0); - // Any eligibility floors at MIN_SEATS even at zero demand. - assert_eq!(seats_for_demand(0, 5), MIN_SEATS); - // One seat per TARGET_SESSIONS_PER_SEAT, rounding up at the bucket edge. - assert_eq!(seats_for_demand(TARGET_SESSIONS_PER_SEAT, 10), 1); - assert_eq!(seats_for_demand(TARGET_SESSIONS_PER_SEAT + 1, 10), 2); - assert_eq!(seats_for_demand(TARGET_SESSIONS_PER_SEAT * 2, 10), 2); - // Capped by the eligible set … - assert_eq!(seats_for_demand(10_000, 3), 3); - // … and by MAX_SEATS when plenty are eligible. - assert_eq!(seats_for_demand(10_000_000, 100), MAX_SEATS); + let view = CoordinatorView::build(1, &beacon, &[], EndpointMap::new()); + assert!(leaders(&view).is_empty()); + assert!(!view.serves(&node(0))); } // ── refresh: the roster converges within an epoch ── - /// A node outside every test roster, for tests about the roster rather - /// than about this node's own seat. - const OBSERVER: CoordinatorNodeId = [0xEE; 32]; - fn endpoints_for(roster: &[CoordinatorNodeId]) -> EndpointMap { roster .iter() @@ -755,49 +690,28 @@ mod tests { .collect() } - fn first_draw_as( - me: &CoordinatorNodeId, - epoch: u64, - roster: &[CoordinatorNodeId], - demand: u64, - ) -> Cached { + fn first_draw(epoch: u64, roster: &[CoordinatorNodeId]) -> Cached { next_view( None, - me, epoch, Some(derive_beacon(epoch, &[7u8; 32])), roster.to_vec(), endpoints_for(roster), - demand, ) .expect("a first draw with a beacon always produces a view") } - fn first_draw(epoch: u64, roster: &[CoordinatorNodeId], demand: u64) -> Cached { - first_draw_as(&OBSERVER, epoch, roster, demand) - } - /// Same epoch, a new roster — what a refresh does once more gossip is in. - fn redraw_as( - me: &CoordinatorNodeId, - prev: &Cached, - roster: &[CoordinatorNodeId], - ) -> Option { + fn redraw(prev: &Cached, roster: &[CoordinatorNodeId]) -> Option { next_view( Some(prev), - me, prev.epoch, None, roster.to_vec(), endpoints_for(roster), - prev.demand, ) } - fn winner(c: &Cached) -> CoordinatorNodeId { - c.view.seated()[0].node_id - } - /// The mainnet failure, epoch 6711: a node that drew while three peers had /// not yet opted in kept that roster for the whole epoch, and elected a /// different coordinator from the nodes that drew later. @@ -806,37 +720,29 @@ mod tests { let full: Vec<_> = (1u8..=8).map(node).collect(); let partial: Vec<_> = full.iter().copied().filter(|id| id[0] > 3).collect(); - let late = first_draw(6711, &full, 0); - let early = first_draw(6711, &partial, 0); + let late = first_draw(6711, &full); + let early = first_draw(6711, &partial); assert_ne!( early.roster_commitment, late.roster_commitment, "precondition: the two nodes start out split" ); - let caught_up = redraw_as(&OBSERVER, &early, &full) + let caught_up = redraw(&early, &full) .expect("a roster that grew within the epoch must be redrawn, not held"); - - let seating = |c: &Cached| -> Vec<(u32, CoordinatorNodeId, Option)> { - c.view - .seated() - .into_iter() - .map(|s| (s.seat, s.node_id, s.endpoint)) - .collect() - }; assert_eq!(caught_up.roster_commitment, late.roster_commitment); assert_eq!(caught_up.beacon, late.beacon); assert_eq!( - seating(&caught_up), - seating(&late), - "same roster and beacon must seat the same coordinators" + caught_up.view.tiers(), + late.view.tiers(), + "same roster and beacon must name the same leaders and failover paths" ); } #[test] fn an_unchanged_roster_is_not_redrawn() { let roster: Vec<_> = (1u8..=8).map(node).collect(); - let cached = first_draw(10, &roster, 0); - assert!(redraw_as(&OBSERVER, &cached, &roster).is_none()); + let cached = first_draw(10, &roster); + assert!(redraw(&cached, &roster).is_none()); } #[test] @@ -844,154 +750,92 @@ mod tests { // A wallet dials the endpoint, so a stale one is as wrong as a stale // roster even when the draw itself is unchanged. let roster: Vec<_> = (1u8..=5).map(node).collect(); - let cached = first_draw(10, &roster, 0); + let cached = first_draw(10, &roster); let mut moved = endpoints_for(&roster); moved.insert(node(2), "10.9.9.9:9100".into()); - let next = next_view(Some(&cached), &OBSERVER, 10, None, roster, moved.clone(), 0) + let next = next_view(Some(&cached), 10, None, roster, moved.clone()) .expect("a changed endpoint must be republished"); assert_eq!(next.endpoints, moved); } #[test] - fn the_seat_target_is_frozen_for_the_epoch() { - // Session counters move constantly. If a roster change also re-read - // them, seats would be resized mid-epoch by ordinary traffic. - let roster: Vec<_> = (1u8..=8).map(node).collect(); - let cached = first_draw(10, &roster[..6], 0); - assert_eq!(cached.view.seats(), 1); - let busy = TARGET_SESSIONS_PER_SEAT * 4; - let next = next_view( - Some(&cached), - &OBSERVER, - 10, - None, - roster.clone(), - endpoints_for(&roster), - busy, - ) - .unwrap(); - assert_eq!( - next.view.seats(), - 1, - "demand read mid-epoch must not resize seats" - ); - assert_eq!(next.demand, 0); - } - - #[test] - fn a_new_epoch_is_drawn_with_its_own_beacon_and_demand() { + fn a_new_epoch_is_drawn_with_its_own_beacon() { let roster: Vec<_> = (1u8..=8).map(node).collect(); - let cached = first_draw(10, &roster, 0); - let busy = TARGET_SESSIONS_PER_SEAT * 3; + let cached = first_draw(10, &roster); let beacon = derive_beacon(11, &[8u8; 32]); let next = next_view( Some(&cached), - &OBSERVER, 11, Some(beacon), roster.clone(), endpoints_for(&roster), - busy, ) .expect("a new epoch is always redrawn"); assert_eq!(next.epoch, 11); assert_eq!(next.beacon, beacon); - assert_eq!(next.view.seats(), 3); } #[test] fn a_new_epoch_without_its_beacon_keeps_the_last_good_view() { let roster: Vec<_> = (1u8..=8).map(node).collect(); - let cached = first_draw(10, &roster, 0); + let cached = first_draw(10, &roster); assert!(next_view( Some(&cached), - &OBSERVER, 11, None, roster.clone(), - endpoints_for(&roster), - 0 + endpoints_for(&roster) ) .is_none()); } - /// An epoch in which the full roster seats a node the partial roster did - /// not — so the partial roster's winner loses its seat on catching up. - /// Searched for rather than hard-coded, so the test states the situation it - /// needs instead of depending on what one beacon happens to rank first. - fn epoch_where_catching_up_unseats( - full: &[CoordinatorNodeId], - partial: &[CoordinatorNodeId], - ) -> (u64, CoordinatorNodeId) { - (1..500u64) - .find_map(|epoch| { - let early = winner(&first_draw(epoch, partial, 0)); - let late = winner(&first_draw(epoch, full, 0)); - (early != late).then_some((epoch, early)) - }) - .expect("some epoch in 500 seats a node outside the partial roster") + // ── the published document ── + + /// Build the JSON the endpoint would serve for `c`, as `me`, without a + /// live mesh: the same code path as `status_json`, minus the service. + fn published(c: &Cached, me: &CoordinatorNodeId) -> serde_json::Value { + let svc = Cached::clone(c); + let lock = RwLock::new(Some(svc)); + status_json_for(&lock, me) } #[test] - fn a_seat_lost_within_the_epoch_is_served_until_the_epoch_turns() { - // Stopping the coordinator aborts the rounds it holds, and participants - // may already have committed inputs to them. Losing the seat to a - // roster that filled in is not a reason to do that. - let full: Vec<_> = (1u8..=8).map(node).collect(); - let partial: Vec<_> = full.iter().copied().filter(|id| id[0] > 3).collect(); - let (epoch, me) = epoch_where_catching_up_unseats(&full, &partial); - - let early = first_draw_as(&me, epoch, &partial, 0); - assert!( - early.view.am_i_coordinator(&me), - "precondition: seated early" - ); - assert!(early.seated_this_epoch); - - let caught_up = redraw_as(&me, &early, &full).unwrap(); - assert!( - !caught_up.view.am_i_coordinator(&me), - "the published view moves on — new wallets go to the new seat" - ); + fn the_published_election_passes_the_check_a_wallet_runs() { + // The wallet and the challenger both recompute the draw from the + // document's own inputs. A node publishing something they would refuse + // would send every wallet to its manual fallback. + let roster: Vec<_> = (1u8..=8).map(node).collect(); + let doc = published(&first_draw(6711, &roster), &roster[0]); assert!( - caught_up.seated_this_epoch, - "but the coordinator keeps serving what it already holds" + wraith_protocol::election_doc::election_is_honest(&doc), + "{doc:#}" ); + for tier in TIERS { + let eps = wraith_protocol::election_doc::endpoints_for_tier(&doc, tier); + assert_eq!(eps.len(), roster.len(), "{tier}: every node is on the path"); + } } #[test] - fn a_new_epoch_forgets_a_seat_held_in_the_last_one() { - let full: Vec<_> = (1u8..=8).map(node).collect(); - let partial: Vec<_> = full.iter().copied().filter(|id| id[0] > 3).collect(); - let (epoch, me) = epoch_where_catching_up_unseats(&full, &partial); - let held = redraw_as(&me, &first_draw_as(&me, epoch, &partial, 0), &full).unwrap(); - assert!(held.seated_this_epoch); - - // Walk forward to an epoch that does not seat `me`, so the only way it - // could still be serving is a seat carried over from before. - let next = (epoch + 1..epoch + 500) - .find_map(|e| { - let v = next_view( - Some(&held), - &me, - e, - Some(derive_beacon(e, &[7u8; 32])), - full.clone(), - endpoints_for(&full), - 0, - ) - .unwrap(); - (!v.view.am_i_coordinator(&me)).then_some(v) - }) - .expect("some later epoch leaves `me` unseated"); - assert!(!next.seated_this_epoch, "a seat does not outlive its epoch"); - } - - #[test] - fn seats_for_demand_is_deterministic() { - // Same frozen inputs → identical seats on every node (no path dependence). - for (d, e) in [(0u64, 1usize), (75, 8), (260, 4), (999, 50)] { - assert_eq!(seats_for_demand(d, e), seats_for_demand(d, e)); + fn a_node_publishes_its_own_part() { + let roster: Vec<_> = (1u8..=8).map(node).collect(); + let cached = first_draw(6711, &roster); + for me in &roster { + let doc = published(&cached, me); + let mine: Vec<&str> = doc["my_tiers"] + .as_array() + .unwrap() + .iter() + .map(|t| t.as_str().unwrap()) + .collect(); + assert_eq!(mine, cached.view.tiers_led_by(me)); + assert_eq!( + doc["my_endpoint"], + serde_json::json!(format!("10.0.0.{}:9100", me[0])) + ); } + let outsider = published(&cached, &node(200)); + assert!(outsider["my_tiers"].as_array().unwrap().is_empty()); + assert!(outsider["my_endpoint"].is_null()); } } diff --git a/bins/ghost-pool/src/coordinator_supervisor.rs b/bins/ghost-pool/src/coordinator_supervisor.rs index 6bab4f68d..ff672f18a 100644 --- a/bins/ghost-pool/src/coordinator_supervisor.rs +++ b/bins/ghost-pool/src/coordinator_supervisor.rs @@ -107,21 +107,21 @@ impl CoordinatorSupervisor { }))) } - /// Reconcile the running coordinator against the election: start when newly - /// elected, stop when the seat is lost, and refresh the advertised session - /// count while running. No-op when role activation is disabled. - pub async fn reconcile(&self, am_i_coordinator: bool) { + /// Reconcile the running coordinator against the election: start when this + /// node should serve, stop when it should not, and refresh the advertised + /// session count while running. No-op when role activation is disabled. + pub async fn reconcile(&self, should_serve: bool) { if !self.cfg.enabled { return; } let is_running = self.running.lock().is_some(); - match (am_i_coordinator, is_running) { + match (should_serve, is_running) { (true, false) => self.start().await, (false, true) => self.stop(), _ => {} } - // While serving, publish the live session count so the mesh can size - // seats by demand (the source for `seats_for_demand`). + // While serving, publish the live session count. Observability only: + // the election no longer sizes anything from it. if let Some(r) = self.running.lock().as_ref() { self.mesh .set_coordinator_sessions(r.state.sessions.len() as u32); @@ -184,12 +184,12 @@ impl CoordinatorSupervisor { fn stop(&self) { if let Some(r) = self.running.lock().take() { // Signal graceful shutdown; abort as a backstop so a stuck server - // can't keep the seat's port held into the next epoch. + // can't keep the port held. r.shutdown.notify_waiters(); r.handle.abort(); - info!("coordinator: seat lost — stopped"); + info!("coordinator: no longer serving — stopped"); } - // No longer coordinating → stop contributing to mesh demand. + // No longer coordinating → stop advertising sessions. self.mesh.set_coordinator_sessions(0); } } diff --git a/crates/ghost-verification/src/client.rs b/crates/ghost-verification/src/client.rs index b280c12b9..d85707adb 100644 --- a/crates/ghost-verification/src/client.rs +++ b/crates/ghost-verification/src/client.rs @@ -960,11 +960,11 @@ impl VerificationClient { // An UNSIGNED attestation is not evidence, whatever it says. Anything can // serve this JSON — that is #605's Stratum finding — so treat it as a // failed challenge rather than a claim worth probing. - let attested_seat = if signed { - payload.get("my_seat").and_then(|v| v.as_u64()) + let attested_serving = if signed { + payload.get("serving").and_then(|v| v.as_bool()) == Some(true) } else { debug!("coordinator attestation was unsigned; not evidence"); - None + false }; let endpoint = payload .get("advertised_endpoint") @@ -972,10 +972,10 @@ impl VerificationClient { .filter(|e| !e.is_empty()) .map(String::from); - // Probe what it named. A node that attests to a seat but answers nothing - // there has not shown it coordinates. - let endpoint_answered = match (attested_seat, endpoint.as_deref()) { - (Some(_), Some(ep)) => self.coordinator_answers_at(ep).await, + // Probe what it named. A node that attests to serving but answers + // nothing there has not shown it coordinates. + let endpoint_answered = match (attested_serving, endpoint.as_deref()) { + (true, Some(ep)) => self.coordinator_answers_at(ep).await, _ => false, }; @@ -991,7 +991,7 @@ impl VerificationClient { let raw_signed = signed.then(|| inner.to_string()); Ok(( CoordinatorProbe { - attested_seat, + attested_serving, endpoint, endpoint_answered, endpoint_is_theirs, @@ -1325,9 +1325,9 @@ impl VerificationClient { /// Result of full verification suite /// The coordinator endpoint an election view assigns to `node_id_hex`, if any. /// -/// Looked up by node id rather than by seat: the seat a node claims is the thing -/// under test, so trusting it to find the endpoint would let the answer be -/// chosen by the party being challenged. +/// Looked up by node id rather than by anything the target claims about its +/// role: that claim is the thing under test, so trusting it to find the +/// endpoint would let the answer be chosen by the party being challenged. fn endpoint_owned_by(view: &serde_json::Value, node_id_hex: &str) -> Option { let want = node_id_hex.trim().to_ascii_lowercase(); view.get("coordinators")? @@ -1353,8 +1353,9 @@ fn endpoint_owned_by(view: &serde_json::Value, node_id_hex: &str) -> Option, + /// The node signed, with its identity key and bound to our nonce, that it + /// runs a coordinator this epoch. + pub attested_serving: bool, /// The endpoint it named in that signed attestation. pub endpoint: Option, /// A coordinator actually answered there. @@ -1374,7 +1375,7 @@ impl CoordinatorProbe { /// /// * **signed** — an unsigned attestation is not evidence, whatever it says. /// Anything can serve that JSON (#605: `nc -l 3333` passes a bare connect). - /// * **answered** — a node that attests to a seat and answers nothing at the + /// * **answered** — a node that attests to serving and answers nothing at the /// address it named has not shown it coordinates. /// * **theirs** — and the address has to be ITS OWN. Without this a node /// passes by naming a neighbour's coordinator: its signature, a real @@ -1388,7 +1389,7 @@ impl CoordinatorProbe { /// target cannot improve its odds by lying, only a challenger's own node can /// be wrong about the fleet. pub fn passed(&self) -> bool { - self.attested_seat.is_some() && self.endpoint_answered && self.endpoint_is_theirs + self.attested_serving && self.endpoint_answered && self.endpoint_is_theirs } } @@ -1926,37 +1927,37 @@ mod tests { /// missing one would have let through. #[test] fn a_coordinator_passes_only_when_all_three_checks_hold() { - let probe = |seat: Option, answered: bool, theirs: bool| CoordinatorProbe { - attested_seat: seat, + let probe = |serving: bool, answered: bool, theirs: bool| CoordinatorProbe { + attested_serving: serving, endpoint: Some("10.0.0.2:9100".to_string()), endpoint_answered: answered, endpoint_is_theirs: theirs, }; assert!( - probe(Some(1), true, true).passed(), + probe(true, true, true).passed(), "signed, answering, and its own" ); assert!( - !probe(Some(1), false, true).passed(), - "attests to a seat and answers nothing there — has not shown it coordinates" + !probe(true, false, true).passed(), + "attests to serving and answers nothing there — has not shown it coordinates" ); assert!( - !probe(None, true, true).passed(), + !probe(false, true, true).passed(), "unsigned is not evidence whatever it says (#605: `nc -l 3333` passes a bare connect)" ); assert!( - !probe(Some(1), true, false).passed(), + !probe(true, true, false).passed(), "the neighbour attack: a real signature and a real coordinator answering, but the \ address belongs to somebody else" ); } - /// A node with no seat cannot pass on liveness alone. + /// A node that does not attest to serving cannot pass on liveness alone. #[test] - fn no_seat_means_no_capability_however_reachable() { + fn not_serving_means_no_capability_however_reachable() { let probe = CoordinatorProbe { - attested_seat: None, + attested_serving: false, endpoint: None, endpoint_answered: true, endpoint_is_theirs: true, @@ -1964,15 +1965,15 @@ mod tests { assert!(!probe.passed()); } - /// Ownership is resolved by node id, never by the seat the target claims — - /// the seat is the thing under test, so using it to find the endpoint would - /// let the answer be chosen by the party being challenged. + /// Ownership is resolved by node id, never by what the target claims about + /// its role — that claim is the thing under test, so using it to find the + /// endpoint would let the answer be chosen by the party being challenged. #[test] - fn ownership_is_looked_up_by_node_id_not_by_claimed_seat() { + fn ownership_is_looked_up_by_node_id_not_by_claimed_role() { let view = serde_json::json!({ "coordinators": [ - { "seat": 0, "node_id": "AA".repeat(32), "endpoint": "10.0.0.1:9100" }, - { "seat": 1, "node_id": "bb".repeat(32), "endpoint": "10.0.0.2:9100" }, + { "node_id": "AA".repeat(32), "endpoint": "10.0.0.1:9100", "leads": ["100k_sats"] }, + { "node_id": "bb".repeat(32), "endpoint": "10.0.0.2:9100", "leads": [] }, ], }); diff --git a/crates/ghost-verification/src/routes.rs b/crates/ghost-verification/src/routes.rs index 2b8f49e11..6bd40495a 100644 --- a/crates/ghost-verification/src/routes.rs +++ b/crates/ghost-verification/src/routes.rs @@ -1521,23 +1521,29 @@ pub struct CoordinatorVerifyQuery { /// answers there. A signature without a probe proves a key exists; a probe /// without a signature proves something listens. The capability needs both. /// -/// ⚠ Deliberately NOT included: the election document (beacon, roster, seat -/// list). It is derived from public inputs, so any node can compute a correct +/// ⚠ Deliberately NOT included: the election document (beacon, roster, tier +/// leaders). It is derived from public inputs, so any node can compute a correct /// one without coordinating anything — including it would invite exactly the /// mistake #605 records against Archive, where every field was derivable from /// the public header and a pruned node passed. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct CoordinatorAttestation { - /// Has this node opted into taking a seat. + /// Has this node turned the coordinator election on. pub enabled: bool, + /// Whether this node runs a coordinator this epoch — true for every node on + /// its own roster, leading a tier or not, since the rest are the failover + /// path. + pub serving: bool, /// Where this node says wallets should dial its coordinator. The address - /// the challenger must then probe — a claim, until it does. + /// the challenger must then probe — a claim, until it does. `None` when the + /// node is not serving. pub advertised_endpoint: Option, /// The epoch this node currently computes, so a node stuck on a stale view /// is visible rather than merely wrong. pub epoch: Option, - /// The seat this node currently holds, if any. - pub my_seat: Option, + /// The tiers this node leads this epoch. Informational: which node leads + /// what follows from public inputs, so it proves nothing about the node. + pub my_tiers: Vec, } /// `GET /verify/coordinator` — a node's signed attestation about its Wraith @@ -1555,20 +1561,29 @@ async fn coordinator_verify_handler( .get("enabled") .and_then(|v| v.as_bool()) .unwrap_or(false), + // This node's OWN endpoint, as its election service reports it. Never + // the first entry in `coordinators`, or any other node's: that would + // have the challenger probe somebody else's coordinator and credit + // this node for it. advertised_endpoint: status - .get("coordinators") - .and_then(|c| c.as_array()) - .and_then(|coords| { - let me = status.get("my_seat").and_then(|s| s.as_u64())?; - coords - .iter() - .find(|c| c.get("seat").and_then(|s| s.as_u64()) == Some(me))? - .get("endpoint") - .and_then(|e| e.as_str()) - .map(String::from) - }), + .get("my_endpoint") + .and_then(|e| e.as_str()) + .filter(|e| !e.is_empty()) + .map(String::from), + serving: status + .get("my_endpoint") + .and_then(|e| e.as_str()) + .is_some_and(|e| !e.is_empty()), epoch: status.get("epoch").and_then(|e| e.as_u64()), - my_seat: status.get("my_seat").and_then(|s| s.as_u64()), + my_tiers: status + .get("my_tiers") + .and_then(|t| t.as_array()) + .map(|t| { + t.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(), }; // Signed by default. An unsigned attestation is a diagnostic, never @@ -12463,8 +12478,9 @@ async fn api_self_check_handler(State(state): State>) -> /// Read-only decentralised-coordinator election view /// (`tasks/plan_decentralised_coordinators.md`). Returns -/// `{enabled, epoch, seats, my_seat, elected:[hex node ids]}` when the operator -/// has turned on `[coordinator] wraith_election_enabled`, else `{enabled:false}`. +/// `{enabled, epoch, tiers, coordinators, my_tiers, my_endpoint, beacon, roster, …}` +/// when the operator has turned on `[coordinator] wraith_election_enabled`, +/// else `{enabled:false}`. /// This endpoint activates nothing — it only reports the public election draw. async fn api_pool_coordinator_handler( State(state): State>, @@ -13042,8 +13058,10 @@ mod tests { secret } - /// A node that says it holds seat 1, alongside another seat it does not hold. - fn coordinator_state(my_seat: Option) -> Arc { + /// A node whose own endpoint is listed SECOND, after another node's. + fn coordinator_state( + my_endpoint: Option<&'static str>, + ) -> Arc { use ghost_common::types::NodeCapabilities; use ghost_policy::PolicyProfile; @@ -13057,24 +13075,26 @@ mod tests { serde_json::json!({ "enabled": true, "epoch": 6709, - "my_seat": my_seat, + "my_endpoint": my_endpoint, + "my_tiers": if my_endpoint.is_some() { vec!["100m_sats"] } else { vec![] }, "coordinators": [ - { "seat": 0, "endpoint": "10.0.0.1:9100" }, - { "seat": 1, "endpoint": "10.0.0.2:9100" }, + { "node_id": "aa", "endpoint": "10.0.0.1:9100", "leads": ["100k_sats"] }, + { "node_id": "bb", "endpoint": "10.0.0.2:9100", "leads": ["100m_sats"] }, ], }) }); Arc::new(state) } - /// A node must attest to the endpoint for ITS OWN seat. + /// A node must attest to ITS OWN endpoint. /// - /// Reporting the first seat in the list, or any other node's, would have the - /// challenger probe somebody else's coordinator and credit this node for it - /// — a capability that passes by pointing at a working neighbour. + /// Reporting the first coordinator in the list, or any other node's, would + /// have the challenger probe somebody else's coordinator and credit this + /// node for it — a capability that passes by pointing at a working + /// neighbour. #[tokio::test] - async fn the_attestation_names_this_nodes_own_seat_endpoint() { - let state = coordinator_state(Some(1)); + async fn the_attestation_names_this_nodes_own_endpoint() { + let state = coordinator_state(Some("10.0.0.2:9100")); let resp = coordinator_verify_handler( axum::extract::State(state), axum::extract::Query(CoordinatorVerifyQuery { @@ -13090,18 +13110,19 @@ mod tests { .expect("body"); let v: serde_json::Value = serde_json::from_slice(&body).expect("json"); assert_eq!(v["signed"], false, "unsigned was explicitly requested"); - assert_eq!(v["response"]["my_seat"], 1); + assert_eq!(v["response"]["serving"], true); assert_eq!( v["response"]["advertised_endpoint"], "10.0.0.2:9100", - "seat 1 is this node; 10.0.0.1 belongs to seat 0" + "10.0.0.1 is listed first and belongs to another node" ); + assert_eq!(v["response"]["my_tiers"], serde_json::json!(["100m_sats"])); assert_eq!(v["response"]["epoch"], 6709); } - /// A node holding no seat advertises no endpoint. Returning one anyway would - /// invite the challenger to probe a coordinator this node does not run. + /// A node that is not serving advertises no endpoint. Returning one anyway + /// would invite the challenger to probe a coordinator this node does not run. #[tokio::test] - async fn a_node_without_a_seat_advertises_nothing_to_probe() { + async fn a_node_not_serving_advertises_nothing_to_probe() { let state = coordinator_state(None); let resp = coordinator_verify_handler( axum::extract::State(state), @@ -13118,7 +13139,7 @@ mod tests { .expect("body"); let v: serde_json::Value = serde_json::from_slice(&body).expect("json"); assert!(v["response"]["advertised_endpoint"].is_null()); - assert!(v["response"]["my_seat"].is_null()); + assert_eq!(v["response"]["serving"], false); } /// With nothing wired the node says so, rather than defaulting to a claim. diff --git a/crates/ghost-verification/src/server.rs b/crates/ghost-verification/src/server.rs index 6236dddc8..ec1416ad3 100644 --- a/crates/ghost-verification/src/server.rs +++ b/crates/ghost-verification/src/server.rs @@ -1509,7 +1509,7 @@ pub struct VerificationState { get_self_check: Option serde_json::Value + Send + Sync>>, /// Coordinator-election snapshot callback (pre-serialized JSON to avoid a /// reverse-dependency on the `wraith-protocol`/`ghost-pool` election types). - /// Returns `{enabled, epoch, seats, my_seat, elected}` when the feature is + /// Returns `{enabled, epoch, tiers, coordinators, my_tiers, my_endpoint, …}` when the feature is /// on, or `{enabled:false}` when off / not wired. Read-only. get_coordinator_status: Option serde_json::Value + Send + Sync>>, /// Mesh-wide deduplicated active miner count callback. Returns the size diff --git a/crates/wraith-protocol/src/beacon.rs b/crates/wraith-protocol/src/beacon.rs index e38ad6398..39fd37942 100644 --- a/crates/wraith-protocol/src/beacon.rs +++ b/crates/wraith-protocol/src/beacon.rs @@ -20,7 +20,7 @@ //| FILE: beacon.rs | //|======================================================================================================================| -//! The randomness `elect_coordinators` consumes — and why it cannot be a block hash. +//! The randomness the coordinator draw consumes — and why it cannot be a block hash. //! //! `sortition::rank_of` is `H(domain ‖ beacon ‖ epoch ‖ node_id)`. Whoever //! controls the beacon controls the draw, so the beacon is the entire security @@ -369,20 +369,26 @@ mod tests { #[test] fn the_beacon_actually_moves_the_election() { - // End to end: this exists to feed `elect_coordinators`, and a different + // End to end: this exists to feed the coordinator draw, and a different // beacon must produce a different draw or none of the above matters. - use crate::sortition::{elect_coordinators, CoordinatorNodeId}; + use crate::epoch::EpochCoordinators; + use crate::sortition::CoordinatorNodeId; let roster: Vec = (1..=20u8).map(|i| [i; 32]).collect(); let a = round_of(5).finalise(&[9u8; 32], 1_001, 1_000, 5).unwrap(); let b = round_of(5).finalise(&[8u8; 32], 1_001, 1_000, 5).unwrap(); - let ea = elect_coordinators(&a, 7, &roster, 4); - let eb = elect_coordinators(&b, 7, &roster, 4); + let leaders = |beacon: &[u8; 32]| -> Vec { + EpochCoordinators::elect(7, beacon, &roster) + .tiers + .iter() + .filter_map(|t| t.leader().copied()) + .collect() + }; assert_ne!( - ea.iter().map(|c| c.node_id).collect::>(), - eb.iter().map(|c| c.node_id).collect::>(), - "a different beacon must draw a different set" + leaders(&a), + leaders(&b), + "a different beacon must draw different leaders" ); } } diff --git a/crates/wraith-protocol/src/election_doc.rs b/crates/wraith-protocol/src/election_doc.rs index 8b5297d67..bfcf9e6d0 100644 --- a/crates/wraith-protocol/src/election_doc.rs +++ b/crates/wraith-protocol/src/election_doc.rs @@ -29,21 +29,38 @@ //! and a second implementation of that answer is how two parties end up //! disagreeing about who is honest. //! -//! The functions were written for the wallet (#697) and moved unchanged. +//! The functions were written for the wallet (#697) and moved here when the +//! challenger needed them too. -use crate::epoch::{derive_beacon, snapshot_height_for_epoch}; -use crate::sortition::{verify_election, CoordinatorNodeId, ElectedCoordinator}; +use crate::epoch::{derive_beacon, snapshot_height_for_epoch, EpochCoordinators}; +use crate::sortition::CoordinatorNodeId; -/// The election's whole claim is public verifiability: rank is -/// `H(beacon ‖ epoch ‖ node_id)`, so nobody can nominate themselves. That -/// property belongs to the *draw*, not to a JSON document describing one — -/// and until this check existed the wallet believed the document. Anything -/// relaying it could have named itself every seat, and every wallet asking -/// for a coordinator would have been sent to it (#697). +/// Recompute the schedule from the inputs a document publishes beside it: +/// `epoch`, `beacon`, `roster`. `None` if any is missing or malformed. +fn recompute(election: &serde_json::Value) -> Option { + let epoch = election.get("epoch")?.as_u64()?; + let beacon = election.get("beacon")?.as_str().and_then(decode_32)?; + let roster = election + .get("roster")? + .as_array()? + .iter() + .map(|v| v.as_str().and_then(decode_32)) + .collect::>>()?; + Some(EpochCoordinators::elect(epoch, &beacon, &roster)) +} + +/// The election's whole claim is public verifiability: a node's rank for a +/// tier is `H(beacon ‖ epoch ‖ tier ‖ node_id)`, so nobody can nominate +/// themselves. That property belongs to the *draw*, not to a JSON document +/// describing one — and until this check existed the wallet believed the +/// document. Anything relaying it could have named itself leader of every +/// tier, and every wallet asking for a coordinator would have been sent to it +/// (#697). /// -/// What recomputing buys: the seat list must actually follow from the beacon -/// and roster published beside it. A relay that edits seats, drops a -/// qualified node, or forges a rank is refused. +/// What recomputing buys: the published `tiers` — each tier's leader and +/// failover order — and the `leads` beside each coordinator must actually follow +/// from the beacon and roster published with them. A relay that edits a leader, +/// reorders a failover path, or drops a node from the roster is refused. /// /// What it does not buy, and this matters: the beacon and roster arrive from /// the same place as the result. A node that lies about *both*, consistently, @@ -51,59 +68,91 @@ use crate::sortition::{verify_election, CoordinatorNodeId, ElectedCoordinator}; /// be pinned — it is `SHA256(domain ‖ epoch ‖ block_hash_at(anchor_height))`, /// so a wallet with chain access can re-derive it and refuse a fabricated /// one; `anchor_height` is published for exactly that. The roster is the -/// remaining trusted input, and closing it needs the qualified set to come -/// from consensus rather than from whoever answered. +/// remaining trusted input; comparing it across nodes (`roster_commitment`) +/// catches a unilateral liar, and closing it fully needs consensus. pub fn election_is_honest(election: &serde_json::Value) -> bool { - let Some(epoch) = election.get("epoch").and_then(|e| e.as_u64()) else { + let Some(expected) = recompute(election) else { return false; }; - let Some(beacon) = election - .get("beacon") - .and_then(|b| b.as_str()) - .and_then(decode_32) - else { + + // Every tier, in order, with exactly the recomputed failover path. + let Some(tiers) = election.get("tiers").and_then(|t| t.as_array()) else { return false; }; - let Some(roster) = election.get("roster").and_then(|r| r.as_array()).map(|r| { - r.iter() - .map(|v| v.as_str().and_then(decode_32)) - .collect::>>() - }) else { + if tiers.len() != expected.tiers.len() { return false; - }; - let Some(roster) = roster else { return false }; - let Some(seats) = election.get("seats").and_then(|s| s.as_u64()) else { + } + for (claimed, want) in tiers.iter().zip(&expected.tiers) { + if claimed.get("tier").and_then(|t| t.as_str()) != Some(want.tier_id.as_str()) { + return false; + } + let Some(order) = claimed.get("order").and_then(|o| o.as_array()).map(|o| { + o.iter() + .map(|v| v.as_str().and_then(decode_32)) + .collect::>>() + }) else { + return false; + }; + if order.as_deref() != Some(want.order.as_slice()) { + return false; + } + } + + // The per-node summary must agree too: it is what a reader looks at, and a + // relay could otherwise leave `tiers` intact and forge `leads`. + let Some(coordinators) = election.get("coordinators").and_then(|c| c.as_array()) else { return false; }; + coordinators.iter().all(|c| { + let Some(id) = c + .get("node_id") + .and_then(|n| n.as_str()) + .and_then(decode_32) + else { + return false; + }; + let Some(leads) = c.get("leads").and_then(|l| l.as_array()) else { + return false; + }; + let leads: Vec<&str> = leads.iter().filter_map(|t| t.as_str()).collect(); + expected.is_coordinator(&id) && leads == expected.tiers_led_by(&id) + }) +} - // The claimed draw, in seat order — `verify_election` compares against a - // freshly computed one, so the ordering has to match how it was built. - let Some(claimed) = election +/// The endpoints to dial for `tier_id`, leader first, in the tier's failover +/// order — recomputed from the document's inputs, never read from its claims. +/// +/// Endpoints come from `coordinators[].endpoint`; a node that advertises none +/// is skipped rather than holding a place. Empty if the inputs are missing. +/// Call [`election_is_honest`] first: this answers "who", not "should I". +pub fn endpoints_for_tier(election: &serde_json::Value, tier_id: &str) -> Vec { + let Some(schedule) = recompute(election) else { + return Vec::new(); + }; + let Some(tier) = schedule.for_tier(tier_id) else { + return Vec::new(); + }; + let published = election .get("coordinators") .and_then(|c| c.as_array()) - .map(|c| { - c.iter() - .map(|v| { - Some(ElectedCoordinator { - node_id: v - .get("node_id") - .and_then(|n| n.as_str()) - .and_then(decode_32)?, - rank: v.get("rank").and_then(|r| r.as_str()).and_then(decode_32)?, - seat: v.get("seat").and_then(|s| s.as_u64())? as u32, - }) - }) - .collect::>>() - }) - else { - return false; + .cloned() + .unwrap_or_default(); + let endpoint_of = |id: &CoordinatorNodeId| -> Option { + published + .iter() + .find(|c| { + c.get("node_id") + .and_then(|n| n.as_str()) + .and_then(decode_32) + .as_ref() + == Some(id) + })? + .get("endpoint")? + .as_str() + .filter(|e| !e.trim().is_empty()) + .map(String::from) }; - let Some(mut claimed) = claimed else { - return false; - }; - claimed.sort_by_key(|c| c.seat); - - verify_election(&beacon, epoch, &roster, seats as usize, &claimed) + tier.order.iter().filter_map(endpoint_of).collect() } /// The block height whose hash must anchor this election's beacon, and the diff --git a/crates/wraith-protocol/src/epoch.rs b/crates/wraith-protocol/src/epoch.rs index dd0692858..db4d0f684 100644 --- a/crates/wraith-protocol/src/epoch.rs +++ b/crates/wraith-protocol/src/epoch.rs @@ -28,7 +28,8 @@ use sha2::{Digest, Sha256}; -use crate::sortition::{elect_coordinators, shard_for, CoordinatorNodeId, ElectedCoordinator}; +use crate::sortition::{tier_leaders, CoordinatorNodeId, TierLeadership}; +use crate::tier::LiteTier; /// Blocks per coordinator epoch. ~1 day at 10-minute blocks. Coordinators are /// re-elected (and the draw reshuffled) every `EPOCH_BLOCKS`. @@ -48,23 +49,6 @@ pub const fn snapshot_height_for_epoch(epoch: u64) -> u64 { } } -/// Domain separator for the coordinator shard key. -const SHARD_KEY_DOMAIN: &[u8] = b"ghost/wraith/coordinator-shard/v1"; - -/// The key a wallet and a node both shard on to agree which seat serves a -/// tier this epoch: `SHA256(domain ‖ tier_id ‖ epoch_le)`. -/// -/// Lives here because both sides must derive byte-identical bytes or they -/// disagree about who is coordinating. It was previously defined only in the -/// wallet daemon, where a node could not reach it. -pub fn shard_key_for_tier_epoch(tier_id: &str, epoch: u64) -> [u8; 32] { - let mut h = Sha256::new(); - h.update(SHARD_KEY_DOMAIN); - h.update(tier_id.as_bytes()); - h.update(epoch.to_le_bytes()); - h.finalize().into() -} - /// Domain separator for the per-epoch beacon. const BEACON_DOMAIN: &[u8] = b"ghost/wraith/coordinator-beacon/v1"; @@ -150,74 +134,66 @@ pub fn canonical_roster(qualified: &[CoordinatorNodeId]) -> Vec, + /// The canonical roster the draw was made from. Every node on it runs a + /// coordinator: it may lead a tier, and it is on every tier's failover path. + pub roster: Vec, + /// One entry per tier, in [`LiteTier::all`] order. Empty orders when the + /// roster is empty. + pub tiers: Vec, } impl EpochCoordinators { - /// Elect up to `n` coordinators for `epoch` from `qualified` under `beacon`. - /// The membership is canonicalised first so the result is independent of the - /// order the caller collected it in. - pub fn elect(epoch: u64, beacon: &[u8; 32], qualified: &[CoordinatorNodeId], n: usize) -> Self { + /// Draw every protocol tier's leader for `epoch` from `qualified` under + /// `beacon`. The membership is canonicalised first so the result is + /// independent of the order the caller collected it in. + /// + /// The tier list is [`LiteTier::all`], not a parameter: a wallet and a node + /// passing different lists would name different leaders, which is the same + /// class of split this module exists to prevent. + pub fn elect(epoch: u64, beacon: &[u8; 32], qualified: &[CoordinatorNodeId]) -> Self { let roster = canonical_roster(qualified); + let tier_ids: Vec<&str> = LiteTier::all().iter().map(|t| t.id()).collect(); Self { epoch, - coordinators: elect_coordinators(beacon, epoch, &roster, n), + tiers: tier_leaders(beacon, epoch, &tier_ids, &roster), + roster, } } - /// How many coordinators are seated this epoch. - pub fn seats(&self) -> usize { - self.coordinators.len() + /// The leadership for `tier_id` this epoch, or `None` for a tier the + /// protocol does not have. + pub fn for_tier(&self, tier_id: &str) -> Option<&TierLeadership> { + self.tiers.iter().find(|t| t.tier_id == tier_id) } - /// The coordinator that owns `tier_id`'s sessions this epoch — the same - /// answer for a wallet and for every node, because both derive it from - /// [`shard_key_for_tier_epoch`]. `None` when no coordinators are seated. + /// The node that leads `tier_id` this epoch — the same answer for a wallet + /// and for every node, because both draw it from the same beacon and + /// roster. `None` for an empty roster or an unknown tier. /// - /// Shards on `(tier, epoch)` rather than on a session id. A session id - /// does not exist until a coordinator creates one, so a wallet choosing - /// *whom to ask* cannot use it — and sharding by tier makes every wallet - /// wanting the same denomination in the same epoch converge on the same - /// seat, which is a larger anonymity set rather than load spreading. - /// - /// Takes no epoch: it is always `self.epoch`. It used to be a parameter, - /// which let a caller pair one epoch's election with another epoch's shard - /// key and silently get a different seat — the same class of mistake as the - /// one below, and reachable by a plain typo. - /// - /// This replaced a `coordinator_for_session(session_id)` that documented - /// itself as the value "a wallet and every node agree" on, while the - /// wallet actually sharded by `(tier, epoch)` and nothing called the - /// library version. Two schemes selecting different seats, one of them - /// dead and inviting: whoever wired up its `owns_session` companion would - /// have had wallets dialling one seat while another believed it owned the - /// work. - pub fn coordinator_for_tier(&self, tier_id: &str) -> Option<&ElectedCoordinator> { - if self.coordinators.is_empty() { - return None; - } - let key = shard_key_for_tier_epoch(tier_id, self.epoch); - let seat = shard_for(&key, self.coordinators.len()); - // seats are exactly 0..len in seat order, so index directly. - self.coordinators.get(seat as usize) + /// Takes no epoch: it is always `self.epoch`. An epoch parameter once let a + /// caller pair one epoch's draw with another epoch's key and silently get a + /// different answer. + pub fn coordinator_for_tier(&self, tier_id: &str) -> Option<&CoordinatorNodeId> { + self.for_tier(tier_id)?.leader() } - /// Whether `node_id` is seated as a coordinator this epoch. + /// Whether `node_id` runs a coordinator this epoch. Every roster node does: + /// the ones not leading a tier are the failover path for the ones that are. pub fn is_coordinator(&self, node_id: &CoordinatorNodeId) -> bool { - self.coordinators.iter().any(|c| &c.node_id == node_id) + self.roster.contains(node_id) } - /// If `node_id` is seated, the seat (shard) it owns this epoch. - pub fn seat_of(&self, node_id: &CoordinatorNodeId) -> Option { - self.coordinators + /// The tiers `node_id` leads this epoch, in [`LiteTier::all`] order. + pub fn tiers_led_by(&self, node_id: &CoordinatorNodeId) -> Vec<&str> { + self.tiers .iter() - .find(|c| &c.node_id == node_id) - .map(|c| c.seat) + .filter(|t| t.leader() == Some(node_id)) + .map(|t| t.tier_id.as_str()) + .collect() } } @@ -271,104 +247,86 @@ mod tests { assert_eq!(canonical_roster(&a).len(), 6); } + const TIER_IDS: [&str; 4] = ["100k_sats", "1m_sats", "10m_sats", "100m_sats"]; + #[test] fn elect_is_deterministic_and_membership_order_independent() { let q = qualified(20); let mut shuffled = q.clone(); shuffled.reverse(); - let a = EpochCoordinators::elect(10, &beacon(1), &q, 4); - let b = EpochCoordinators::elect(10, &beacon(1), &shuffled, 4); + let a = EpochCoordinators::elect(10, &beacon(1), &q); + let b = EpochCoordinators::elect(10, &beacon(1), &shuffled); assert_eq!( a, b, "schedule must not depend on membership collection order" ); - assert_eq!(a.seats(), 4); } #[test] - fn every_tier_maps_to_a_seated_coordinator() { - let q = qualified(15); - let ec = EpochCoordinators::elect(3, &beacon(2), &q, 5); - for tier in ["100k_sats", "1m_sats", "10m_sats", "100m_sats"] { - let c = ec - .coordinator_for_tier(tier) - .expect("a coordinator owns every tier"); - assert!(ec.is_coordinator(&c.node_id)); - // Stable: every wallet asking for this tier in this epoch lands - // on the same seat, which is the point — a larger anonymity set, - // not load spreading. - assert_eq!(ec.coordinator_for_tier(tier).unwrap().node_id, c.node_id); - } + fn the_tier_list_is_the_protocols() { + // Pinned, because a wallet and a node drawing over different lists name + // different leaders. + let ec = EpochCoordinators::elect(3, &beacon(2), &qualified(8)); + let ids: Vec<&str> = ec.tiers.iter().map(|t| t.tier_id.as_str()).collect(); + assert_eq!(ids, TIER_IDS); + assert!( + ec.coordinator_for_tier("500k_sats").is_none(), + "no such tier" + ); } - /// The assignment rotates with the epoch, so one seat does not own a - /// denomination for ever. #[test] - fn a_tier_moves_between_seats_across_epochs() { + fn every_tier_has_its_own_leader_on_the_roster() { let q = qualified(15); - let mut hit = std::collections::HashSet::new(); - // Re-elect each epoch, as a node does. This used to hold one election - // and vary only the epoch argument, which exercised a pairing that - // cannot occur now the argument is gone. - for epoch in 0u64..200 { - let ec = EpochCoordinators::elect(epoch, &beacon(2), &q, 5); - hit.insert(ec.coordinator_for_tier("100k_sats").unwrap().seat); + let ec = EpochCoordinators::elect(3, &beacon(2), &q); + let mut seen = std::collections::HashSet::new(); + for tier in TIER_IDS { + let leader = *ec.coordinator_for_tier(tier).expect("every tier is led"); + assert!(ec.is_coordinator(&leader)); + assert!(seen.insert(leader), "{tier} shares a leader"); + assert_eq!(ec.tiers_led_by(&leader), vec![tier]); } - assert_eq!(hit.len(), 5, "every seat serves the tier in some epoch"); } - /// A wallet and a node derive the identical shard key, or they disagree - /// about who is coordinating. #[test] - fn the_shard_key_is_a_pure_function_of_tier_and_epoch() { - assert_eq!( - shard_key_for_tier_epoch("100k_sats", 7), - shard_key_for_tier_epoch("100k_sats", 7) - ); - assert_ne!( - shard_key_for_tier_epoch("100k_sats", 7), - shard_key_for_tier_epoch("1m_sats", 7) - ); - assert_ne!( - shard_key_for_tier_epoch("100k_sats", 7), - shard_key_for_tier_epoch("100k_sats", 8) - ); - } - - #[test] - fn is_coordinator_and_seat_of() { + fn every_roster_node_serves_whether_or_not_it_leads() { + // The failover path is only a path if the nodes on it are listening. let q = qualified(12); - let ec = EpochCoordinators::elect(1, &beacon(9), &q, 4); - let seated = ec.coordinators[2].node_id; - assert!(ec.is_coordinator(&seated)); - assert_eq!(ec.seat_of(&seated), Some(2)); - // a node not in the elected set - let absent = node(200); - assert!(!ec.is_coordinator(&absent)); - assert_eq!(ec.seat_of(&absent), None); + let ec = EpochCoordinators::elect(1, &beacon(9), &q); + for id in &q { + assert!(ec.is_coordinator(id)); + } + let idle = q.iter().filter(|id| ec.tiers_led_by(id).is_empty()).count(); + assert_eq!(idle, q.len() - TIER_IDS.len()); + assert!( + !ec.is_coordinator(&node(200)), + "off the roster, never serving" + ); } + /// The lead rotates with the epoch, so no node owns a denomination for ever. #[test] - fn rotation_reshuffles_between_epochs() { - let q = qualified(20); - let a: Vec<_> = EpochCoordinators::elect(100, &beacon(5), &q, 4) - .coordinators - .into_iter() - .map(|c| c.node_id) - .collect(); - let b: Vec<_> = EpochCoordinators::elect(101, &beacon(5), &q, 4) - .coordinators - .into_iter() - .map(|c| c.node_id) + fn a_tier_moves_between_nodes_across_epochs() { + let q = qualified(15); + let hit: std::collections::HashSet<_> = (0u64..200) + .map(|epoch| { + *EpochCoordinators::elect(epoch, &beacon(2), &q) + .coordinator_for_tier("100m_sats") + .unwrap() + }) .collect(); - assert_ne!(a, b, "a new epoch rotates the coordinator set"); + assert_eq!( + hit.len(), + q.len(), + "every node leads the tier in some epoch" + ); } #[test] - fn empty_roster_seats_nobody() { - let ec = EpochCoordinators::elect(1, &beacon(1), &[], 4); - assert_eq!(ec.seats(), 0); + fn empty_roster_leads_nothing() { + let ec = EpochCoordinators::elect(1, &beacon(1), &[]); assert!(ec.coordinator_for_tier("100k_sats").is_none()); + assert!(!ec.is_coordinator(&node(0))); } fn anchors(n: usize) -> Vec<[u8; 32]> { (0..n as u8).map(|i| [i.wrapping_add(1); 32]).collect() diff --git a/crates/wraith-protocol/src/epoch_handover.rs b/crates/wraith-protocol/src/epoch_handover.rs index 0b8a2a3e3..25809f904 100644 --- a/crates/wraith-protocol/src/epoch_handover.rs +++ b/crates/wraith-protocol/src/epoch_handover.rs @@ -22,12 +22,12 @@ //! What happens to a session that is still signing when the epoch rotates. //! -//! `shard_key_for_tier_epoch` mixes the epoch in, so at every rotation a tier's -//! sessions map to a *different* seat. Meanwhile `service::owns_tier` asks "is -//! this tier mine **this epoch**". Put those together and a coordinator holding -//! a `Locked` session — participants committed, partial signatures collected — -//! answers `false` the instant the epoch turns, while wallets computing the new -//! epoch's shard dial somebody else. +//! The draw mixes the epoch in, so at every rotation a tier's lead usually moves +//! to a *different* node. Meanwhile `service::owns_tier` asks "is this tier mine +//! **this epoch**". Put those together and a coordinator holding a `Locked` +//! session — participants committed, partial signatures collected — answers +//! `false` the instant the epoch turns, while wallets drawing the new epoch's +//! leaders dial somebody else. //! //! Nobody is wrong. Each side follows the rule it was given. The session simply //! stops having an owner, with participants' inputs already committed to it. @@ -135,7 +135,7 @@ pub fn owns_session( if coords.epoch != session.opened_in { return None; } - Some(coords.coordinator_for_tier(tier_id).map(|c| c.node_id) == Some(*self_id)) + Some(coords.coordinator_for_tier(tier_id) == Some(self_id)) } /// Whether to open a new session for `tier_id` at `current_height`. @@ -188,7 +188,7 @@ pub fn disposition( tier_id: &str, outstanding: usize, ) -> Disposition { - let ours = current.coordinator_for_tier(tier_id).map(|c| c.node_id) == Some(*self_id); + let ours = current.coordinator_for_tier(tier_id) == Some(self_id); match (ours, outstanding) { (true, _) => Disposition::Active, (false, 0) => Disposition::Idle, @@ -207,16 +207,14 @@ mod tests { (1..=8u8).map(node).collect() } fn elect(epoch: u64) -> EpochCoordinators { - EpochCoordinators::elect(epoch, &[9u8; 32], &roster(), 4) + EpochCoordinators::elect(epoch, &[9u8; 32], &roster()) } - /// Find a tier whose owning seat differs between two epochs. That is the + /// Find a tier whose leader differs between two epochs. That is the /// rotation case; if no tier moved, the fixture proves nothing. fn tier_that_moves(a: &EpochCoordinators, b: &EpochCoordinators) -> String { - for t in ["100k_sats", "1m_sats", "10k_sats", "500k_sats", "5m_sats"] { - if a.coordinator_for_tier(t).map(|c| c.node_id) - != b.coordinator_for_tier(t).map(|c| c.node_id) - { + for t in ["100k_sats", "1m_sats", "10m_sats", "100m_sats"] { + if a.coordinator_for_tier(t) != b.coordinator_for_tier(t) { return t.to_string(); } } @@ -237,12 +235,12 @@ mod tests { // holding partial signatures stops owning its own session at rotation. let (a, b) = (elect(3), elect(4)); let tier = tier_that_moves(&a, &b); - let owner = a.coordinator_for_tier(&tier).unwrap().node_id; + let owner = a.coordinator_for_tier(&tier).copied().unwrap(); let session = SessionEpoch::at_height(3 * EPOCH_BLOCKS + 100); // Under the old scheme the new epoch's election answers, and it says no. - assert_ne!(b.coordinator_for_tier(&tier).unwrap().node_id, owner); + assert_ne!(b.coordinator_for_tier(&tier).copied().unwrap(), owner); // Bound to its opening epoch, it is still ours. assert_eq!(owns_session(&a, &owner, &tier, &session), Some(true)); @@ -253,7 +251,7 @@ mod tests { // Silently answering from whichever election is to hand IS the bug. let (a, b) = (elect(3), elect(4)); let tier = tier_that_moves(&a, &b); - let owner = a.coordinator_for_tier(&tier).unwrap().node_id; + let owner = a.coordinator_for_tier(&tier).copied().unwrap(); let session = SessionEpoch::at_height(3 * EPOCH_BLOCKS + 100); assert_eq!(owns_session(&b, &owner, &tier, &session), None); } @@ -262,7 +260,7 @@ mod tests { fn a_session_is_not_opened_on_the_edge_of_a_rotation() { let coords = elect(3); let tier = "100k_sats"; - let owner = coords.coordinator_for_tier(tier).unwrap().node_id; + let owner = coords.coordinator_for_tier(tier).copied().unwrap(); let last = (3 + 1) * EPOCH_BLOCKS - 1; assert!(matches!( check_open(&coords, &owner, tier, last), @@ -277,7 +275,7 @@ mod tests { fn a_node_does_not_open_sessions_for_a_tier_it_does_not_own() { let coords = elect(3); let tier = "100k_sats"; - let owner = coords.coordinator_for_tier(tier).unwrap().node_id; + let owner = coords.coordinator_for_tier(tier).copied().unwrap(); let other = roster().into_iter().find(|n| *n != owner).unwrap(); assert!(matches!( check_open(&coords, &other, tier, 3 * EPOCH_BLOCKS + 10), @@ -291,7 +289,7 @@ mod tests { // competes with the new owner. Draining is neither. let (a, b) = (elect(3), elect(4)); let tier = tier_that_moves(&a, &b); - let old = a.coordinator_for_tier(&tier).unwrap().node_id; + let old = a.coordinator_for_tier(&tier).copied().unwrap(); assert_eq!(disposition(&a, &old, &tier, 2), Disposition::Active); assert_eq!(disposition(&b, &old, &tier, 2), Disposition::Draining); @@ -302,7 +300,7 @@ mod tests { fn the_new_owner_is_active_immediately() { let (a, b) = (elect(3), elect(4)); let tier = tier_that_moves(&a, &b); - let new = b.coordinator_for_tier(&tier).unwrap().node_id; + let new = b.coordinator_for_tier(&tier).copied().unwrap(); assert_eq!(disposition(&b, &new, &tier, 0), Disposition::Active); } } diff --git a/crates/wraith-protocol/src/lib.rs b/crates/wraith-protocol/src/lib.rs index 087048860..443b474b6 100644 --- a/crates/wraith-protocol/src/lib.rs +++ b/crates/wraith-protocol/src/lib.rs @@ -108,16 +108,17 @@ pub mod sortition; pub mod spend_log_file; pub mod standing_order; pub use epoch::{ - canonical_roster, derive_beacon, epoch_for_height, shard_key_for_tier_epoch, - snapshot_height_for_epoch, EpochCoordinators, EPOCH_BLOCKS, + canonical_roster, derive_beacon, epoch_for_height, snapshot_height_for_epoch, + EpochCoordinators, EPOCH_BLOCKS, }; pub use remix::{ RemixEnrolment, RemixError, RemixId, RemixQueue, RemixStatus, DEFAULT_QUEUE_TIMEOUT_SECS, DEFAULT_REMIX_COUNT, MAX_REMIX_COUNT, }; -pub use service::{CoordinatorView, EndpointMap}; +pub use service::{CoordinatorView, EndpointMap, ServingCoordinator}; pub use sortition::{ - elect_coordinators, rank_of, shard_for, verify_election, CoordinatorNodeId, ElectedCoordinator, + coordinator_order_for_tier, rank_of, tier_leaders, CoordinatorNodeId, ElectedCoordinator, + TierLeadership, }; /// Session type determines fee structure diff --git a/crates/wraith-protocol/src/roster_snapshot.rs b/crates/wraith-protocol/src/roster_snapshot.rs index a0e4a5866..236b21fd7 100644 --- a/crates/wraith-protocol/src/roster_snapshot.rs +++ b/crates/wraith-protocol/src/roster_snapshot.rs @@ -263,7 +263,7 @@ mod tests { // Why any of this matters: the two rosters produce different // coordinators from the same beacon, so the two nodes disagree about // who owns a session. - use crate::sortition::elect_coordinators; + use crate::epoch::EpochCoordinators; let beacon = [7u8; 32]; let ours = roster(&[1, 2, 3, 4, 5, 6, 7, 8]); let theirs = roster(&[1, 2, 3, 4, 5, 6, 7, 9]); @@ -272,11 +272,15 @@ mod tests { Agreement::Diverged { .. } )); - let a = elect_coordinators(&beacon, 3, &ours, 4); - let b = elect_coordinators(&beacon, 3, &theirs, 4); - assert_ne!( - a.iter().map(|c| c.node_id).collect::>(), - b.iter().map(|c| c.node_id).collect::>() - ); + // One node's difference does not move every epoch's leaders, but it + // moves some — and a single epoch of disagreement is a split. + let leaders = |r: &[CoordinatorNodeId], epoch: u64| -> Vec { + EpochCoordinators::elect(epoch, &beacon, r) + .tiers + .iter() + .filter_map(|t| t.leader().copied()) + .collect() + }; + assert!((0..50u64).any(|e| leaders(&ours, e) != leaders(&theirs, e))); } } diff --git a/crates/wraith-protocol/src/service.rs b/crates/wraith-protocol/src/service.rs index de7b356c0..df501dbc1 100644 --- a/crates/wraith-protocol/src/service.rs +++ b/crates/wraith-protocol/src/service.rs @@ -2,50 +2,41 @@ //! live use (increment 4b, layer 1). //! //! A Ghost node and a wallet both build a [`CoordinatorView`] for an epoch from -//! the *same* agreed inputs (the beacon, the frozen qualified roster, and the -//! node→endpoint map), and get consistent answers to the two operational -//! questions the wiring needs: +//! the *same* agreed inputs (the beacon, the roster, and the node→endpoint map), +//! and get consistent answers to the two operational questions the wiring needs: //! -//! - **node**: "am I elected this epoch, and for which seat (shard)?" — so a node -//! knows when to spin up its co-located coordinator and which sessions it owns. -//! - **wallet**: "which coordinator endpoint owns *my* session?" — so a wallet -//! connects to the right node without trusting anyone to tell it. +//! - **node**: "do I run a coordinator this epoch, and which tiers do I lead?" +//! - **wallet**: "which endpoints do I dial for my tier, and in what order?" — so +//! a wallet connects to the right node without trusting anyone to tell it. //! //! Because the underlying schedule is deterministic ([`EpochCoordinators`]), the -//! node and the wallet independently agree on who owns each session. This module -//! is still pure — the endpoint map and election inputs are passed in; populating -//! them from live consensus/discovery is the next layer. +//! node and the wallet independently agree on who leads each tier. This module +//! is still pure — the endpoint map and election inputs are passed in. use std::collections::BTreeMap; use crate::epoch::EpochCoordinators; -use crate::sortition::CoordinatorNodeId; +use crate::sortition::{CoordinatorNodeId, TierLeadership}; /// Maps a coordinator node id to the base URL of its coordinator endpoint /// (e.g. `https://node.example:9100`). Sourced from the node-discovery layer. pub type EndpointMap = BTreeMap; -/// One seated coordinator with its reachable endpoint — the unit the read-only -/// status endpoint publishes and the wallet resolves against. -#[derive(Debug, Clone)] -pub struct SeatedCoordinator { - /// The elected node's id. +/// One roster node as the status endpoint publishes it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServingCoordinator { + /// The node's id. pub node_id: CoordinatorNodeId, - /// Seat index `0..seats`; sessions shard onto seats via `shard_for`. - pub seat: u32, - /// The sortition hash that won the seat. Published so a consumer can - /// recompute the draw and see *why* this node holds it, rather than - /// taking the seat list on trust (#697). - pub rank: [u8; 32], - /// The endpoint a wallet dials for this seat, or `None` if the owner hasn't - /// advertised one yet (the wallet then waits or picks another epoch). + /// Where a wallet dials it, or `None` if it has not advertised yet. pub endpoint: Option, + /// The tiers it leads this epoch. Empty for a node that is only on the + /// failover path, which still runs its coordinator. + pub leads: Vec, } /// The resolved coordinator assignment for one epoch, with endpoints attached. #[derive(Debug, Clone)] pub struct CoordinatorView { - epoch: u64, coords: EpochCoordinators, endpoints: EndpointMap, } @@ -53,95 +44,103 @@ pub struct CoordinatorView { impl CoordinatorView { /// Wrap an already-computed schedule with an endpoint map. pub fn new(coords: EpochCoordinators, endpoints: EndpointMap) -> Self { - Self { - epoch: coords.epoch, - coords, - endpoints, - } + Self { coords, endpoints } } /// Build directly from election inputs + endpoints. pub fn build( epoch: u64, beacon: &[u8; 32], - qualified: &[CoordinatorNodeId], + roster: &[CoordinatorNodeId], endpoints: EndpointMap, - n: usize, ) -> Self { - Self::new( - EpochCoordinators::elect(epoch, beacon, qualified, n), - endpoints, - ) + Self::new(EpochCoordinators::elect(epoch, beacon, roster), endpoints) } /// The epoch this view is for. pub fn epoch(&self) -> u64 { - self.epoch + self.coords.epoch } - /// Number of coordinators seated this epoch. - pub fn seats(&self) -> usize { - self.coords.seats() + /// Every tier's leader and failover order, in `LiteTier::all` order. + pub fn tiers(&self) -> &[TierLeadership] { + &self.coords.tiers } - /// The seated coordinators in seat order, each with its advertised endpoint - /// (`None` when the owner hasn't advertised one). Drives the read-only status - /// endpoint and the wallet's "which endpoint owns my session" resolution. - pub fn seated(&self) -> Vec { - let mut out: Vec = self - .coords - .coordinators + /// Every roster node, in canonical order, with its endpoint and the tiers + /// it leads. Drives the read-only status endpoint. + pub fn serving(&self) -> Vec { + self.coords + .roster .iter() - .map(|c| SeatedCoordinator { - node_id: c.node_id, - seat: c.seat, - rank: c.rank, - endpoint: self.endpoints.get(&c.node_id).cloned(), + .map(|id| ServingCoordinator { + node_id: *id, + endpoint: self.endpoints.get(id).cloned(), + leads: self + .coords + .tiers_led_by(id) + .into_iter() + .map(String::from) + .collect(), }) - .collect(); - out.sort_unstable_by_key(|s| s.seat); - out + .collect() } // ── node-side ──────────────────────────────────────────────────────────── - /// Whether `self_id` is elected a coordinator this epoch. - pub fn am_i_coordinator(&self, self_id: &CoordinatorNodeId) -> bool { + /// Whether `self_id` should run a coordinator this epoch — true for every + /// roster node, leading or not, because the rest are the failover path. + pub fn serves(&self, self_id: &CoordinatorNodeId) -> bool { self.coords.is_coordinator(self_id) } - /// If `self_id` is elected, the seat (shard) it must serve this epoch. - pub fn my_seat(&self, self_id: &CoordinatorNodeId) -> Option { - self.coords.seat_of(self_id) + /// The tiers `self_id` leads this epoch. + pub fn tiers_led_by(&self, self_id: &CoordinatorNodeId) -> Vec<&str> { + self.coords.tiers_led_by(self_id) } - /// Whether `self_id` owns the sessions for `tier_id` this epoch — the - /// node's check for "are these sessions mine to coordinate?". + /// Whether `self_id` leads `tier_id` this epoch — the node's check for "are + /// these sessions mine to coordinate?". pub fn owns_tier(&self, self_id: &CoordinatorNodeId, tier_id: &str) -> bool { self.coordinator_node_for_tier(tier_id).as_ref() == Some(self_id) } // ── wallet-side ────────────────────────────────────────────────────────── - /// The coordinator *node* that owns `tier_id`'s sessions this epoch - /// (`None` when no coordinators are seated). + /// The node that leads `tier_id` this epoch (`None` for an empty roster). pub fn coordinator_node_for_tier(&self, tier_id: &str) -> Option { - self.coords.coordinator_for_tier(tier_id).map(|c| c.node_id) + self.coords.coordinator_for_tier(tier_id).copied() } - /// The endpoint a wallet should connect to for `tier_id`. `None` if no - /// coordinator is seated, or the owning coordinator has no known endpoint - /// (the wallet then waits for discovery to catch up, or picks another epoch). + /// The leader's endpoint for `tier_id`. `None` if nobody leads it, or the + /// leader has not advertised one. pub fn endpoint_for_tier(&self, tier_id: &str) -> Option<&str> { let node = self.coordinator_node_for_tier(tier_id)?; self.endpoints.get(&node).map(String::as_str) } + + /// The endpoints to try for `tier_id`, leader first, in the tier's failover + /// order. Nodes that have advertised no endpoint are skipped rather than + /// holding a place: a node nobody can dial is not a fallback. + pub fn endpoints_for_tier(&self, tier_id: &str) -> Vec<&str> { + self.coords + .for_tier(tier_id) + .map(|t| { + t.order + .iter() + .filter_map(|id| self.endpoints.get(id).map(String::as_str)) + .collect() + }) + .unwrap_or_default() + } } #[cfg(test)] mod tests { use super::*; + const TIERS: [&str; 4] = ["100k_sats", "1m_sats", "10m_sats", "100m_sats"]; + fn node(i: u8) -> CoordinatorNodeId { let mut id = [0u8; 32]; id[0] = i; @@ -161,50 +160,49 @@ mod tests { .collect() } - fn view(n: usize) -> (CoordinatorView, Vec) { + fn view() -> (CoordinatorView, Vec) { let q = qualified(12); - let v = CoordinatorView::build(3, &beacon(7), &q, endpoints(&q), n); + let v = CoordinatorView::build(3, &beacon(7), &q, endpoints(&q)); (v, q) } #[test] - fn node_knows_if_and_where_it_is_elected() { - let (v, _q) = view(4); - assert_eq!(v.seats(), 4); - // exactly the seated nodes report a seat - let seated: Vec<_> = (0..12u8) - .map(node) - .filter(|id| v.am_i_coordinator(id)) + fn a_node_knows_it_serves_and_which_tiers_it_leads() { + let (v, q) = view(); + let leaders: Vec<_> = q + .iter() + .filter(|id| !v.tiers_led_by(id).is_empty()) .collect(); - assert_eq!(seated.len(), 4); - for (expect_seat, id) in seated.iter().enumerate() { - // my_seat agrees with am_i_coordinator - assert!(v.my_seat(id).is_some()); - let _ = expect_seat; + assert_eq!(leaders.len(), TIERS.len(), "four different leaders"); + for id in &q { + assert!(v.serves(id), "every roster node serves"); } - // a non-roster node is never elected - assert!(!v.am_i_coordinator(&node(200))); - assert_eq!(v.my_seat(&node(200)), None); + assert!(!v.serves(&node(200))); + assert!(v.tiers_led_by(&node(200)).is_empty()); } #[test] fn wallet_resolves_a_real_endpoint_for_every_tier() { - let (v, _q) = view(5); - for tier in ["100k_sats", "1m_sats", "10m_sats", "100m_sats"] { + let (v, _q) = view(); + for tier in TIERS { let ep = v .endpoint_for_tier(tier) .expect("an endpoint for every tier"); assert!(ep.starts_with("https://node")); + assert_eq!( + v.endpoints_for_tier(tier)[0], + ep, + "the leader is tried first" + ); } } /// The node's "is this mine?" answer and the wallet's "who do I dial?" - /// answer are the same function, so they cannot disagree. They used to be - /// two different functions keyed on different things. + /// answer are the same function, so they cannot disagree. #[test] fn node_and_wallet_agree_on_the_owner() { - let (v, _q) = view(5); - for tier in ["100k_sats", "1m_sats", "10m_sats", "100m_sats"] { + let (v, _q) = view(); + for tier in TIERS { let owner = v.coordinator_node_for_tier(tier).unwrap(); assert!(v.owns_tier(&owner, tier)); for other in (0..12u8).map(node).filter(|x| *x != owner) { @@ -216,51 +214,38 @@ mod tests { #[test] fn missing_endpoint_yields_none_but_owner_still_known() { let q = qualified(12); - // endpoints for everyone EXCEPT whoever ends up owning the tier let mut eps = endpoints(&q); - let v_full = CoordinatorView::build(3, &beacon(7), &q, eps.clone(), 5); - let owner = v_full.coordinator_node_for_tier("100k_sats").unwrap(); + let full = CoordinatorView::build(3, &beacon(7), &q, eps.clone()); + let owner = full.coordinator_node_for_tier("100k_sats").unwrap(); eps.remove(&owner); - let v = CoordinatorView::build(3, &beacon(7), &q, eps, 5); - // owner still known… + let v = CoordinatorView::build(3, &beacon(7), &q, eps); assert_eq!(v.coordinator_node_for_tier("100k_sats"), Some(owner)); - // …but no endpoint to dial assert_eq!(v.endpoint_for_tier("100k_sats"), None); + // …and the wallet walks straight to the next node that can be dialled. + let tried = v.endpoints_for_tier("100k_sats"); + assert_eq!(tried.len(), q.len() - 1); + assert_eq!(tried, full.endpoints_for_tier("100k_sats")[1..]); } #[test] - fn empty_roster_seats_nobody() { - let v = CoordinatorView::build(1, &beacon(1), &[], EndpointMap::new(), 4); - assert_eq!(v.seats(), 0); - assert!(!v.am_i_coordinator(&node(0))); + fn empty_roster_serves_nobody() { + let v = CoordinatorView::build(1, &beacon(1), &[], EndpointMap::new()); + assert!(!v.serves(&node(0))); assert_eq!(v.endpoint_for_tier("100k_sats"), None); + assert!(v.endpoints_for_tier("100k_sats").is_empty()); + assert!(v.serving().is_empty()); } #[test] - fn seated_lists_every_seat_with_its_endpoint_in_order() { - let (v, _q) = view(5); - let seated = v.seated(); - assert_eq!(seated.len(), 5); - for (i, s) in seated.iter().enumerate() { - assert_eq!(s.seat as usize, i, "seats must be 0..n in order"); - assert!(s - .endpoint - .as_deref() - .expect("every seated coordinator has an endpoint here") - .starts_with("https://node")); + fn serving_lists_every_roster_node_with_what_it_leads() { + let (v, q) = view(); + let serving = v.serving(); + assert_eq!(serving.len(), q.len()); + let led: usize = serving.iter().map(|s| s.leads.len()).sum(); + assert_eq!(led, TIERS.len(), "each tier is led exactly once"); + for s in &serving { + assert!(s.endpoint.as_deref().unwrap().starts_with("https://node")); + assert_eq!(s.leads, v.tiers_led_by(&s.node_id)); } } - - #[test] - fn seated_endpoint_is_none_when_owner_has_not_advertised() { - let q = qualified(12); - let mut eps = endpoints(&q); - let full = CoordinatorView::build(3, &beacon(7), &q, eps.clone(), 5); - let owner = full.seated()[0].node_id; - eps.remove(&owner); - let v = CoordinatorView::build(3, &beacon(7), &q, eps, 5); - // Still seated (owner known) but no endpoint to dial. - assert_eq!(v.seated()[0].node_id, owner); - assert_eq!(v.seated()[0].endpoint, None); - } } diff --git a/crates/wraith-protocol/src/sortition.rs b/crates/wraith-protocol/src/sortition.rs index e68f62979..9a0cdae88 100644 --- a/crates/wraith-protocol/src/sortition.rs +++ b/crates/wraith-protocol/src/sortition.rs @@ -1,29 +1,38 @@ //! Deterministic, publicly-verifiable coordinator election (sortition). //! -//! This is increment 1 of the decentralised-coordinator design -//! (`tasks/plan_decentralised_coordinators.md`): the pure selection core, with -//! NO coupling to consensus, networking, or funds. Given three agreed inputs — -//! an unpredictable randomness `beacon`, the `epoch` number, and the -//! consensus-frozen `roster` of qualified node ids — it elects `n` coordinators -//! for that epoch. +//! The pure selection core of the decentralised-coordinator design +//! (`tasks/plan_decentralised_coordinators.md`), with NO coupling to consensus, +//! networking, or funds. Given three agreed inputs — an unpredictable `beacon`, +//! the `epoch` number, and the `roster` of eligible node ids — it names a +//! leader for every denomination, and the order a wallet falls back through. //! //! ## Properties (the whole point) //! -//! - **No self-nomination.** A node's rank is `H(beacon ‖ epoch ‖ node_id)`. The -//! node controls neither the beacon nor (cheaply) its own id, so it cannot -//! grind itself into a seat. The network doesn't *vote for* candidates; it -//! agrees on the beacon + roster, and the winners fall out deterministically. +//! - **No self-nomination.** A node's rank for a tier is +//! `H(beacon ‖ epoch ‖ tier ‖ node_id)`. The node controls neither the beacon +//! nor (cheaply) its own id, so it cannot grind itself into a lead. The +//! network doesn't *vote for* candidates; it agrees on the beacon + roster, +//! and the leaders fall out deterministically. //! - **Determinism.** Every node (and every wallet) computes the byte-identical //! result from the same inputs — so they agree on who coordinates without a //! second round of communication. -//! - **Public verifiability.** Anyone can recompute the election (`verify_election`) -//! and the session→coordinator mapping (`shard_for`); there is no trusted -//! tallier. -//! - **Fairness.** Ranks are uniform over the roster, so each qualified node is -//! elected with probability ≈ `n / roster_len` per epoch. +//! - **Public verifiability.** Anyone can recompute [`tier_leaders`]; there is +//! no trusted tallier. +//! - **Evenness.** Every tier has a *different* leader whenever the roster is at +//! least as large as the tier list, and every node is equally likely to lead +//! every tier. Opting in means being called on — see [`tier_leaders`]. //! - **Rotation.** A fresh `beacon`/`epoch` reshuffles the draw, so coordination //! rotates across the network over time. //! +//! ## Why there is no seat count +//! +//! This used to elect `n` coordinators into seats and send each tier to +//! `shard(tier, epoch) mod n`. With `n` sized from session demand — zero on +//! mainnet — that was one seat, so a single node carried every denomination for +//! a whole day while every other opted-in node sat idle. And `n` was one more +//! number nodes had to agree on: two nodes with identical rosters but different +//! demand snapshots sent the same tier to different coordinators. +//! //! The *security* of the whole scheme rests on the beacon being **ungrindable** //! (increment 2) — that is deliberately abstracted out here: this module treats //! the beacon as a given 32-byte value and is correct for any such value. @@ -36,21 +45,19 @@ use std::collections::HashSet; /// dependency. Callers pass the qualified-node roster as these ids. pub type CoordinatorNodeId = [u8; 32]; -/// Domain separators so a sortition hash can never collide with a shard hash or -/// any other hash in the system. Versioned for forward changes. +/// Domain separator so a sortition hash can never collide with any other hash +/// in the system. Versioned for forward changes. const DOMAIN_RANK: &[u8] = b"ghost/wraith/coordinator-sortition/rank/v1"; -const DOMAIN_SHARD: &[u8] = b"ghost/wraith/coordinator-sortition/shard/v1"; -/// One elected coordinator for an epoch. +/// One place in a tier's ranking. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ElectedCoordinator { - /// The elected node's id. + /// The node's id. pub node_id: CoordinatorNodeId, - /// The sortition hash that won it the seat (lower = higher priority). Carried - /// so verifiers and observers can see *why* this node was chosen. + /// The sortition hash that placed it (lower = higher priority). Carried so + /// verifiers and observers can see *why* this node is where it is. pub rank: [u8; 32], - /// Seat index `0..n`, in ascending-rank order. Sessions are sharded across - /// seats via [`shard_for`], so the seat is this coordinator's shard. + /// Position `0..n` in the tier's ranking, in ascending-rank order. pub seat: u32, } @@ -132,69 +139,83 @@ pub fn coordinator_order_for_tier( .collect() } -/// Elect `n` coordinators from `roster` for `epoch` under `beacon`. +/// Who coordinates one denomination for an epoch. /// -/// Ranks every (deduplicated) roster member, sorts ascending by `(rank, node_id)` -/// — node_id only as a tie-break, astronomically unlikely with 256-bit ranks — -/// and returns the lowest `n`, each tagged with its seat `0..n`. Returns fewer -/// than `n` (all of them) when the roster is smaller, and an empty vec for an -/// empty roster or `n == 0`. A duplicated id in `roster` is counted once (a node -/// cannot hold two seats). -pub fn elect_coordinators( - beacon: &[u8; 32], - epoch: u64, - roster: &[CoordinatorNodeId], - n: usize, -) -> Vec { - if n == 0 || roster.is_empty() { - return Vec::new(); - } - let mut seen: HashSet = HashSet::with_capacity(roster.len()); - let mut ranked: Vec<([u8; 32], CoordinatorNodeId)> = roster - .iter() - .filter(|id| seen.insert(**id)) - .map(|id| (rank_of(beacon, epoch, id), *id)) - .collect(); - // Total order: by rank, then node_id. Deterministic on every node. - ranked.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); - ranked - .into_iter() - .take(n) - .enumerate() - .map(|(seat, (rank, node_id))| ElectedCoordinator { - node_id, - rank, - seat: seat as u32, - }) - .collect() +/// `order[0]` leads it. The rest is where its wallets go if the leader does not +/// answer, walked in this order by every wallet, so a dead leader's cohort moves +/// as one body to the same next node instead of scattering. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TierLeadership { + /// The tier's stable id (`LiteTier::id`). + pub tier_id: String, + /// Leader first, then the failover path. Holds every roster node. + pub order: Vec, } -/// Deterministically map a session to one of `seats` elected coordinators, so a -/// wallet and every node independently agree which coordinator owns the session: -/// `SHA256(DOMAIN_SHARD ‖ session_id)[..8] mod seats`. Returns 0 when `seats <= 1`. -pub fn shard_for(session_id: &[u8; 32], seats: usize) -> u32 { - if seats <= 1 { - return 0; +impl TierLeadership { + /// The node that leads this tier. `None` only for an empty roster. + pub fn leader(&self) -> Option<&CoordinatorNodeId> { + self.order.first() } - let mut h = Sha256::new(); - h.update(DOMAIN_SHARD); - h.update(session_id); - let d = h.finalize(); - let v = u64::from_le_bytes(d[..8].try_into().expect("32-byte digest has 8 bytes")); - (v % seats as u64) as u32 } -/// Recompute the election and check a claimed result matches exactly. The audit -/// path: anyone with the same agreed inputs confirms a published coordinator set -/// was elected honestly. -pub fn verify_election( +/// Every tier's leader and failover order for `epoch`, one entry per tier in +/// the order `tiers` gives them. +/// +/// # Different leaders, not merely independent ones +/// +/// Each tier has its own ranking ([`coordinator_order_for_tier`]). Taking each +/// ranking's top node independently would let one node lead several tiers while +/// others lead none — with eight nodes and four tiers, all four leaders differ +/// in only ~41% of epochs. So tiers are filled in turn, each by the +/// best-ranked node **not already leading one**. +/// +/// That changes nothing about fairness. Ranks are independent and uniform and +/// the rule never looks at who a node is, so every node is equally likely to +/// lead every tier; what it removes is the doubling-up. With `n ≥ tiers`, every +/// epoch has `tiers` different leaders; with fewer nodes than tiers every node +/// leads, and some lead more than one. +/// +/// # The failover order +/// +/// The leader, then the rest of the tier's own ranking. The next node is often +/// another tier's leader; its load grows, which is the right trade against +/// sending wallets to a node nobody else would pick. +/// +/// Deduplicates the roster; an empty roster yields empty orders. +pub fn tier_leaders( beacon: &[u8; 32], epoch: u64, + tiers: &[&str], roster: &[CoordinatorNodeId], - n: usize, - claimed: &[ElectedCoordinator], -) -> bool { - elect_coordinators(beacon, epoch, roster, n) == claimed +) -> Vec { + let mut leading: HashSet = HashSet::with_capacity(tiers.len()); + tiers + .iter() + .map(|tier_id| { + let ranked: Vec = + coordinator_order_for_tier(beacon, epoch, tier_id, roster) + .into_iter() + .map(|c| c.node_id) + .collect(); + // Once every node leads something, doubling up is unavoidable; the + // tier then goes to its own top-ranked node. + let pick = ranked + .iter() + .position(|id| !leading.contains(id)) + .unwrap_or(0); + let mut order = ranked; + if !order.is_empty() { + let leader = order.remove(pick); + leading.insert(leader); + order.insert(0, leader); + } + TierLeadership { + tier_id: (*tier_id).to_string(), + order, + } + }) + .collect() } #[cfg(test)] @@ -217,175 +238,157 @@ mod tests { [seed; 32] } + const TIERS: [&str; 4] = ["100k_sats", "1m_sats", "10m_sats", "100m_sats"]; + + fn leaders(b: &[u8; 32], epoch: u64, r: &[CoordinatorNodeId]) -> Vec { + tier_leaders(b, epoch, &TIERS, r) + .iter() + .map(|t| *t.leader().expect("non-empty roster")) + .collect() + } + #[test] fn determinism_same_inputs_same_output() { let r = roster(20); - let a = elect_coordinators(&beacon(1), 42, &r, 4); - let b = elect_coordinators(&beacon(1), 42, &r, 4); - assert_eq!(a, b, "election must be byte-identical across calls"); - assert_eq!(a.len(), 4); - // seats are 0..n in order - for (i, e) in a.iter().enumerate() { - assert_eq!(e.seat, i as u32); + let a = tier_leaders(&beacon(1), 42, &TIERS, &r); + let b = tier_leaders(&beacon(1), 42, &TIERS, &r); + assert_eq!(a, b, "the draw must be byte-identical across calls"); + assert_eq!(a.len(), TIERS.len()); + for (t, want) in a.iter().zip(TIERS) { + assert_eq!(t.tier_id, want, "one entry per tier, in the order given"); } - // strictly ascending ranks - assert!(a.windows(2).all(|w| w[0].rank <= w[1].rank)); } #[test] - fn no_self_nomination_node_cannot_force_a_win() { - // A fixed node either wins or not depending purely on the beacon, which - // it does not control. Across many beacons its win-rate ≈ n/roster, and - // it is NOT always elected — so it cannot nominate itself in. - let r = roster(20); - let target = r[7]; - let mut wins = 0; - let trials = 4000u64; - for e in 0..trials { - // vary the beacon each trial (stand-in for the per-epoch beacon) - let elected = elect_coordinators(&beacon((e % 251) as u8 + 1), e, &r, 4); - if elected.iter().any(|c| c.node_id == target) { - wins += 1; - } - } - // Expected ≈ trials * 4/20 = 20%. Assert it's clearly bounded away from - // both 0% (could never win) and 100% (always wins / self-nominated). - let pct = (wins as f64) / (trials as f64); - assert!( - pct > 0.10 && pct < 0.32, - "win-rate {pct:.3} should sit near 0.20 — never guaranteed, never impossible" + fn two_nodes_with_the_roster_in_different_orders_agree() { + let r = roster(8); + let mut shuffled = r.clone(); + shuffled.reverse(); + assert_eq!( + tier_leaders(&beacon(3), 9, &TIERS, &r), + tier_leaders(&beacon(3), 9, &TIERS, &shuffled) ); } #[test] - fn fairness_uniform_over_epochs() { - // Over many epochs every roster member is elected ≈ equally. - let r = roster(20); - let n = 4usize; - let trials = 8000u64; - let mut counts = vec![0u64; r.len()]; - for e in 0..trials { - for c in elect_coordinators(&beacon(9), e, &r, n) { - let idx = r.iter().position(|x| *x == c.node_id).unwrap(); - counts[idx] += 1; - } - } - let expected = (trials as f64) * (n as f64) / (r.len() as f64); // = 1600 - for (i, &c) in counts.iter().enumerate() { - let dev = (c as f64 - expected).abs() / expected; - assert!( - dev < 0.15, - "node {i} elected {c} times, expected ~{expected:.0} (dev {dev:.3})" - ); + fn every_tier_has_a_different_leader_when_there_are_enough_nodes() { + // Independent top picks would double up in ~59% of epochs at 8 nodes. + for e in 0..2000u64 { + let l = leaders(&beacon((e % 251) as u8), e, &roster(8)); + let distinct: HashSet<_> = l.iter().collect(); + assert_eq!(distinct.len(), TIERS.len(), "epoch {e}: {l:?}"); } } + /// The property the operator asked for: unbiased, and even. Every node + /// leads every tier about equally often, so every opted-in node is called on + /// and has the same chance at the tier that pays most. #[test] - fn n_exceeds_roster_elects_all_no_panic() { - let r = roster(3); - let e = elect_coordinators(&beacon(2), 1, &r, 10); - assert_eq!(e.len(), 3, "cannot elect more than the roster"); - let ids: HashSet<_> = e.iter().map(|c| c.node_id).collect(); - assert_eq!(ids.len(), 3, "all distinct"); - } - - #[test] - fn empty_roster_and_zero_n() { - assert!(elect_coordinators(&beacon(1), 1, &[], 4).is_empty()); - assert!(elect_coordinators(&beacon(1), 1, &roster(5), 0).is_empty()); + fn every_node_leads_every_tier_equally_often() { + let r = roster(8); + let trials = 16_000u64; + let mut counts = vec![[0u64; 4]; r.len()]; + for e in 0..trials { + for (t, leader) in leaders(&beacon(9), e, &r).iter().enumerate() { + let idx = r.iter().position(|x| x == leader).unwrap(); + counts[idx][t] += 1; + } + } + let expected = trials as f64 / r.len() as f64; // 2000 per (node, tier) + for (i, per_tier) in counts.iter().enumerate() { + for (t, &c) in per_tier.iter().enumerate() { + let dev = (c as f64 - expected).abs() / expected; + assert!( + dev < 0.10, + "node {i} led {} {c} times, expected ~{expected:.0} (dev {dev:.3})", + TIERS[t] + ); + } + } } #[test] - fn duplicate_ids_counted_once() { - let mut r = roster(5); - r.push(r[2]); // duplicate - r.push(r[2]); - let e = elect_coordinators(&beacon(3), 7, &r, 5); - let ids: HashSet<_> = e.iter().map(|c| c.node_id).collect(); - assert_eq!(ids.len(), e.len(), "no node holds two seats"); - assert_eq!(e.len(), 5, "5 distinct after dedup"); + fn no_self_nomination_node_cannot_force_a_win() { + // A fixed node leads or not depending purely on the beacon, which it + // does not control: never guaranteed, never impossible. + let r = roster(20); + let target = r[7]; + let trials = 4000u64; + let wins = (0..trials) + .filter(|&e| leaders(&beacon((e % 251) as u8 + 1), e, &r).contains(&target)) + .count(); + // Expected ≈ 4/20 = 20%. + let pct = wins as f64 / trials as f64; + assert!( + pct > 0.10 && pct < 0.32, + "lead-rate {pct:.3} should sit near 0.20" + ); } #[test] fn rotation_changes_the_draw() { let r = roster(20); - let a: Vec<_> = elect_coordinators(&beacon(5), 100, &r, 4) - .into_iter() - .map(|c| c.node_id) - .collect(); - let b: Vec<_> = elect_coordinators(&beacon(5), 101, &r, 4) - .into_iter() - .map(|c| c.node_id) - .collect(); - assert_ne!(a, b, "a fresh epoch should reshuffle the coordinator set"); + assert_ne!( + leaders(&beacon(5), 100, &r), + leaders(&beacon(5), 101, &r), + "a fresh epoch should reshuffle the leaders" + ); } #[test] - fn shard_for_is_deterministic_and_bounded() { - let s = [0x5a; 32]; - assert_eq!(shard_for(&s, 4), shard_for(&s, 4), "deterministic"); - assert!(shard_for(&s, 4) < 4, "in range"); - assert_eq!(shard_for(&s, 1), 0); - assert_eq!(shard_for(&s, 0), 0); + fn fewer_nodes_than_tiers_still_covers_every_tier() { + let r = roster(2); + let l = leaders(&beacon(2), 1, &r); + assert_eq!(l.len(), TIERS.len(), "no tier is left without a leader"); + let distinct: HashSet<_> = l.iter().collect(); + assert_eq!(distinct.len(), 2, "both nodes are called on"); } #[test] - fn shard_for_distributes_evenly() { - let seats = 5usize; - let trials = 5000u32; - let mut buckets = vec![0u32; seats]; - for i in 0..trials { - let mut sid = [0u8; 32]; - sid[..4].copy_from_slice(&i.to_le_bytes()); - buckets[shard_for(&sid, seats) as usize] += 1; - } - let expected = (trials as f64) / (seats as f64); - for (k, &c) in buckets.iter().enumerate() { - let dev = (c as f64 - expected).abs() / expected; - assert!( - dev < 0.12, - "seat {k} got {c}, expected ~{expected:.0} (dev {dev:.3})" + fn the_order_is_the_leader_then_the_tiers_own_ranking() { + let r = roster(9); + let b = beacon(4); + for t in tier_leaders(&b, 55, &TIERS, &r) { + let ranking: Vec<_> = coordinator_order_for_tier(&b, 55, &t.tier_id, &r) + .into_iter() + .map(|c| c.node_id) + .collect(); + let leader = *t.leader().unwrap(); + let rest: Vec<_> = ranking.iter().copied().filter(|id| *id != leader).collect(); + assert_eq!( + t.order[1..], + rest[..], + "{}: failover keeps rank order", + t.tier_id + ); + assert_eq!( + t.order.len(), + r.len(), + "every node is somewhere in the path" ); } } #[test] - fn verify_election_accepts_valid_rejects_tampered() { - let r = roster(12); - let (b, e, n) = (beacon(4), 55u64, 3usize); - let valid = elect_coordinators(&b, e, &r, n); - assert!( - verify_election(&b, e, &r, n, &valid), - "honest result verifies" - ); - - // tamper: swap in a non-elected node - let mut t1 = valid.clone(); - t1[0].node_id = r[11]; - assert!( - !verify_election(&b, e, &r, n, &t1), - "substituted node rejected" - ); - - // tamper: reorder seats - let mut t2 = valid.clone(); - t2.swap(0, 1); - assert!( - !verify_election(&b, e, &r, n, &t2), - "reordered seats rejected" - ); - - // tamper: forged rank - let mut t3 = valid.clone(); - t3[1].rank = [0u8; 32]; - assert!(!verify_election(&b, e, &r, n, &t3), "forged rank rejected"); + fn empty_roster_leads_nothing() { + for t in tier_leaders(&beacon(1), 1, &TIERS, &[]) { + assert!(t.order.is_empty()); + assert!(t.leader().is_none()); + } + } - // wrong beacon → different election → claimed (old) no longer verifies - assert!( - !verify_election(&beacon(99), e, &r, n, &valid), - "beacon-bound" - ); + #[test] + fn duplicate_ids_counted_once() { + let mut r = roster(5); + r.push(r[2]); + r.push(r[2]); + for t in tier_leaders(&beacon(3), 7, &TIERS, &r) { + let ids: HashSet<_> = t.order.iter().collect(); + assert_eq!(ids.len(), t.order.len(), "no node appears twice"); + assert_eq!(t.order.len(), 5); + } } + #[test] fn tiers_of_the_same_length_still_get_different_orderings() { // Every other tier fixture here has a distinct name length, so the diff --git a/scripts/mutate-wraith.sh b/scripts/mutate-wraith.sh index 92bfabf39..f0c217761 100755 --- a/scripts/mutate-wraith.sh +++ b/scripts/mutate-wraith.sh @@ -133,6 +133,10 @@ mutate "every tier gets the same ordering" sortition.rs \ ' h.update(tier_id.as_bytes());' \ ' h.update(b"");' sortition +mutate "one node may lead several tiers while others idle" sortition.rs \ +' .position(|id| !leading.contains(id))' \ +' .position(|_| true)' sortition + mutate "eligibility ignores maturity" eligibility.rs \ ' if known < policy.maturity_secs {' \ ' if false {' eligibility