Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
455 changes: 141 additions & 314 deletions apps/wraith-wallet/daemon/src/coordinator_resolve.rs

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions apps/wraith-wallet/daemon/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5905,8 +5905,8 @@ mod server {
}
None => (Vec::new(), None),
};
// Head is the seat that owns this tier; the tail is where every
// wallet on that seat goes if it stops answering — the same
// Head is the node leading this tier; the tail is where every
// wallet on the tier goes if it stops answering — the same
// order for all of them, so the cohort moves together (#711).
let mut it = endpoints.into_iter();
let endpoint = it.next();
Expand Down
590 changes: 217 additions & 373 deletions bins/ghost-pool/src/coordinator_election.rs

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions bins/ghost-pool/src/coordinator_supervisor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,21 +107,21 @@ impl CoordinatorSupervisor {
})))
}

/// Reconcile the running coordinator against the election: start when newly
/// elected, stop when the seat is lost, and refresh the advertised session
/// count while running. No-op when role activation is disabled.
pub async fn reconcile(&self, am_i_coordinator: bool) {
/// Reconcile the running coordinator against the election: start when this
/// node should serve, stop when it should not, and refresh the advertised
/// session count while running. No-op when role activation is disabled.
pub async fn reconcile(&self, should_serve: bool) {
if !self.cfg.enabled {
return;
}
let is_running = self.running.lock().is_some();
match (am_i_coordinator, is_running) {
match (should_serve, is_running) {
(true, false) => self.start().await,
(false, true) => self.stop(),
_ => {}
}
// While serving, publish the live session count so the mesh can size
// seats by demand (the source for `seats_for_demand`).
// While serving, publish the live session count. Observability only:
// the election no longer sizes anything from it.
if let Some(r) = self.running.lock().as_ref() {
self.mesh
.set_coordinator_sessions(r.state.sessions.len() as u32);
Expand Down Expand Up @@ -184,12 +184,12 @@ impl CoordinatorSupervisor {
fn stop(&self) {
if let Some(r) = self.running.lock().take() {
// Signal graceful shutdown; abort as a backstop so a stuck server
// can't keep the seat's port held into the next epoch.
// can't keep the port held.
r.shutdown.notify_waiters();
r.handle.abort();
info!("coordinator: seat lost — stopped");
info!("coordinator: no longer serving — stopped");
}
// No longer coordinating → stop contributing to mesh demand.
// No longer coordinating → stop advertising sessions.
self.mesh.set_coordinator_sessions(0);
}
}
Expand Down
63 changes: 32 additions & 31 deletions crates/ghost-verification/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -960,22 +960,22 @@ impl VerificationClient {
// An UNSIGNED attestation is not evidence, whatever it says. Anything can
// serve this JSON — that is #605's Stratum finding — so treat it as a
// failed challenge rather than a claim worth probing.
let attested_seat = if signed {
payload.get("my_seat").and_then(|v| v.as_u64())
let attested_serving = if signed {
payload.get("serving").and_then(|v| v.as_bool()) == Some(true)
} else {
debug!("coordinator attestation was unsigned; not evidence");
None
false
};
let endpoint = payload
.get("advertised_endpoint")
.and_then(|v| v.as_str())
.filter(|e| !e.is_empty())
.map(String::from);

// Probe what it named. A node that attests to a seat but answers nothing
// there has not shown it coordinates.
let endpoint_answered = match (attested_seat, endpoint.as_deref()) {
(Some(_), Some(ep)) => self.coordinator_answers_at(ep).await,
// Probe what it named. A node that attests to serving but answers
// nothing there has not shown it coordinates.
let endpoint_answered = match (attested_serving, endpoint.as_deref()) {
(true, Some(ep)) => self.coordinator_answers_at(ep).await,
_ => false,
};

Expand All @@ -991,7 +991,7 @@ impl VerificationClient {
let raw_signed = signed.then(|| inner.to_string());
Ok((
CoordinatorProbe {
attested_seat,
attested_serving,
endpoint,
endpoint_answered,
endpoint_is_theirs,
Expand Down Expand Up @@ -1325,9 +1325,9 @@ impl VerificationClient {
/// Result of full verification suite
/// The coordinator endpoint an election view assigns to `node_id_hex`, if any.
///
/// Looked up by node id rather than by seat: the seat a node claims is the thing
/// under test, so trusting it to find the endpoint would let the answer be
/// chosen by the party being challenged.
/// Looked up by node id rather than by anything the target claims about its
/// role: that claim is the thing under test, so trusting it to find the
/// endpoint would let the answer be chosen by the party being challenged.
fn endpoint_owned_by(view: &serde_json::Value, node_id_hex: &str) -> Option<String> {
let want = node_id_hex.trim().to_ascii_lowercase();
view.get("coordinators")?
Expand All @@ -1353,8 +1353,9 @@ fn endpoint_owned_by(view: &serde_json::Value, node_id_hex: &str) -> Option<Stri
/// committed to*.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CoordinatorProbe {
/// The seat the node signed for, with its identity key, bound to our nonce.
pub attested_seat: Option<u64>,
/// The node signed, with its identity key and bound to our nonce, that it
/// runs a coordinator this epoch.
pub attested_serving: bool,
/// The endpoint it named in that signed attestation.
pub endpoint: Option<String>,
/// A coordinator actually answered there.
Expand All @@ -1374,7 +1375,7 @@ impl CoordinatorProbe {
///
/// * **signed** — an unsigned attestation is not evidence, whatever it says.
/// Anything can serve that JSON (#605: `nc -l 3333` passes a bare connect).
/// * **answered** — a node that attests to a seat and answers nothing at the
/// * **answered** — a node that attests to serving and answers nothing at the
/// address it named has not shown it coordinates.
/// * **theirs** — and the address has to be ITS OWN. Without this a node
/// passes by naming a neighbour's coordinator: its signature, a real
Expand All @@ -1388,7 +1389,7 @@ impl CoordinatorProbe {
/// target cannot improve its odds by lying, only a challenger's own node can
/// be wrong about the fleet.
pub fn passed(&self) -> bool {
self.attested_seat.is_some() && self.endpoint_answered && self.endpoint_is_theirs
self.attested_serving && self.endpoint_answered && self.endpoint_is_theirs
}
}

Expand Down Expand Up @@ -1926,53 +1927,53 @@ mod tests {
/// missing one would have let through.
#[test]
fn a_coordinator_passes_only_when_all_three_checks_hold() {
let probe = |seat: Option<u64>, answered: bool, theirs: bool| CoordinatorProbe {
attested_seat: seat,
let probe = |serving: bool, answered: bool, theirs: bool| CoordinatorProbe {
attested_serving: serving,
endpoint: Some("10.0.0.2:9100".to_string()),
endpoint_answered: answered,
endpoint_is_theirs: theirs,
};

assert!(
probe(Some(1), true, true).passed(),
probe(true, true, true).passed(),
"signed, answering, and its own"
);
assert!(
!probe(Some(1), false, true).passed(),
"attests to a seat and answers nothing there — has not shown it coordinates"
!probe(true, false, true).passed(),
"attests to serving and answers nothing there — has not shown it coordinates"
);
assert!(
!probe(None, true, true).passed(),
!probe(false, true, true).passed(),
"unsigned is not evidence whatever it says (#605: `nc -l 3333` passes a bare connect)"
);
assert!(
!probe(Some(1), true, false).passed(),
!probe(true, true, false).passed(),
"the neighbour attack: a real signature and a real coordinator answering, but the \
address belongs to somebody else"
);
}

/// A node with no seat cannot pass on liveness alone.
/// A node that does not attest to serving cannot pass on liveness alone.
#[test]
fn no_seat_means_no_capability_however_reachable() {
fn not_serving_means_no_capability_however_reachable() {
let probe = CoordinatorProbe {
attested_seat: None,
attested_serving: false,
endpoint: None,
endpoint_answered: true,
endpoint_is_theirs: true,
};
assert!(!probe.passed());
}

/// Ownership is resolved by node id, never by the seat the target claims
/// the seat is the thing under test, so using it to find the endpoint would
/// let the answer be chosen by the party being challenged.
/// Ownership is resolved by node id, never by what the target claims about
/// its role — that claim is the thing under test, so using it to find the
/// endpoint would let the answer be chosen by the party being challenged.
#[test]
fn ownership_is_looked_up_by_node_id_not_by_claimed_seat() {
fn ownership_is_looked_up_by_node_id_not_by_claimed_role() {
let view = serde_json::json!({
"coordinators": [
{ "seat": 0, "node_id": "AA".repeat(32), "endpoint": "10.0.0.1:9100" },
{ "seat": 1, "node_id": "bb".repeat(32), "endpoint": "10.0.0.2:9100" },
{ "node_id": "AA".repeat(32), "endpoint": "10.0.0.1:9100", "leads": ["100k_sats"] },
{ "node_id": "bb".repeat(32), "endpoint": "10.0.0.2:9100", "leads": [] },
],
});

Expand Down
95 changes: 58 additions & 37 deletions crates/ghost-verification/src/routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1521,23 +1521,29 @@ pub struct CoordinatorVerifyQuery {
/// answers there. A signature without a probe proves a key exists; a probe
/// without a signature proves something listens. The capability needs both.
///
/// ⚠ Deliberately NOT included: the election document (beacon, roster, seat
/// list). It is derived from public inputs, so any node can compute a correct
/// ⚠ Deliberately NOT included: the election document (beacon, roster, tier
/// leaders). It is derived from public inputs, so any node can compute a correct
/// one without coordinating anything — including it would invite exactly the
/// mistake #605 records against Archive, where every field was derivable from
/// the public header and a pruned node passed.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct CoordinatorAttestation {
/// Has this node opted into taking a seat.
/// Has this node turned the coordinator election on.
pub enabled: bool,
/// Whether this node runs a coordinator this epoch — true for every node on
/// its own roster, leading a tier or not, since the rest are the failover
/// path.
pub serving: bool,
/// Where this node says wallets should dial its coordinator. The address
/// the challenger must then probe — a claim, until it does.
/// the challenger must then probe — a claim, until it does. `None` when the
/// node is not serving.
pub advertised_endpoint: Option<String>,
/// The epoch this node currently computes, so a node stuck on a stale view
/// is visible rather than merely wrong.
pub epoch: Option<u64>,
/// The seat this node currently holds, if any.
pub my_seat: Option<u64>,
/// The tiers this node leads this epoch. Informational: which node leads
/// what follows from public inputs, so it proves nothing about the node.
pub my_tiers: Vec<String>,
}

/// `GET /verify/coordinator` — a node's signed attestation about its Wraith
Expand All @@ -1555,20 +1561,29 @@ async fn coordinator_verify_handler(
.get("enabled")
.and_then(|v| v.as_bool())
.unwrap_or(false),
// This node's OWN endpoint, as its election service reports it. Never
// the first entry in `coordinators`, or any other node's: that would
// have the challenger probe somebody else's coordinator and credit
// this node for it.
advertised_endpoint: status
.get("coordinators")
.and_then(|c| c.as_array())
.and_then(|coords| {
let me = status.get("my_seat").and_then(|s| s.as_u64())?;
coords
.iter()
.find(|c| c.get("seat").and_then(|s| s.as_u64()) == Some(me))?
.get("endpoint")
.and_then(|e| e.as_str())
.map(String::from)
}),
.get("my_endpoint")
.and_then(|e| e.as_str())
.filter(|e| !e.is_empty())
.map(String::from),
serving: status
.get("my_endpoint")
.and_then(|e| e.as_str())
.is_some_and(|e| !e.is_empty()),
epoch: status.get("epoch").and_then(|e| e.as_u64()),
my_seat: status.get("my_seat").and_then(|s| s.as_u64()),
my_tiers: status
.get("my_tiers")
.and_then(|t| t.as_array())
.map(|t| {
t.iter()
.filter_map(|v| v.as_str().map(String::from))
.collect()
})
.unwrap_or_default(),
};

// Signed by default. An unsigned attestation is a diagnostic, never
Expand Down Expand Up @@ -12463,8 +12478,9 @@ async fn api_self_check_handler(State(state): State<Arc<VerificationState>>) ->

/// Read-only decentralised-coordinator election view
/// (`tasks/plan_decentralised_coordinators.md`). Returns
/// `{enabled, epoch, seats, my_seat, elected:[hex node ids]}` when the operator
/// has turned on `[coordinator] wraith_election_enabled`, else `{enabled:false}`.
/// `{enabled, epoch, tiers, coordinators, my_tiers, my_endpoint, beacon, roster, …}`
/// when the operator has turned on `[coordinator] wraith_election_enabled`,
/// else `{enabled:false}`.
/// This endpoint activates nothing — it only reports the public election draw.
async fn api_pool_coordinator_handler(
State(state): State<Arc<VerificationState>>,
Expand Down Expand Up @@ -13042,8 +13058,10 @@ mod tests {
secret
}

/// A node that says it holds seat 1, alongside another seat it does not hold.
fn coordinator_state(my_seat: Option<u64>) -> Arc<crate::server::VerificationState> {
/// A node whose own endpoint is listed SECOND, after another node's.
fn coordinator_state(
my_endpoint: Option<&'static str>,
) -> Arc<crate::server::VerificationState> {
use ghost_common::types::NodeCapabilities;
use ghost_policy::PolicyProfile;

Expand All @@ -13057,24 +13075,26 @@ mod tests {
serde_json::json!({
"enabled": true,
"epoch": 6709,
"my_seat": my_seat,
"my_endpoint": my_endpoint,
"my_tiers": if my_endpoint.is_some() { vec!["100m_sats"] } else { vec![] },
"coordinators": [
{ "seat": 0, "endpoint": "10.0.0.1:9100" },
{ "seat": 1, "endpoint": "10.0.0.2:9100" },
{ "node_id": "aa", "endpoint": "10.0.0.1:9100", "leads": ["100k_sats"] },
{ "node_id": "bb", "endpoint": "10.0.0.2:9100", "leads": ["100m_sats"] },
],
})
});
Arc::new(state)
}

/// A node must attest to the endpoint for ITS OWN seat.
/// A node must attest to ITS OWN endpoint.
///
/// Reporting the first seat in the list, or any other node's, would have the
/// challenger probe somebody else's coordinator and credit this node for it
/// — a capability that passes by pointing at a working neighbour.
/// Reporting the first coordinator in the list, or any other node's, would
/// have the challenger probe somebody else's coordinator and credit this
/// node for it — a capability that passes by pointing at a working
/// neighbour.
#[tokio::test]
async fn the_attestation_names_this_nodes_own_seat_endpoint() {
let state = coordinator_state(Some(1));
async fn the_attestation_names_this_nodes_own_endpoint() {
let state = coordinator_state(Some("10.0.0.2:9100"));
let resp = coordinator_verify_handler(
axum::extract::State(state),
axum::extract::Query(CoordinatorVerifyQuery {
Expand All @@ -13090,18 +13110,19 @@ mod tests {
.expect("body");
let v: serde_json::Value = serde_json::from_slice(&body).expect("json");
assert_eq!(v["signed"], false, "unsigned was explicitly requested");
assert_eq!(v["response"]["my_seat"], 1);
assert_eq!(v["response"]["serving"], true);
assert_eq!(
v["response"]["advertised_endpoint"], "10.0.0.2:9100",
"seat 1 is this node; 10.0.0.1 belongs to seat 0"
"10.0.0.1 is listed first and belongs to another node"
);
assert_eq!(v["response"]["my_tiers"], serde_json::json!(["100m_sats"]));
assert_eq!(v["response"]["epoch"], 6709);
}

/// A node holding no seat advertises no endpoint. Returning one anyway would
/// invite the challenger to probe a coordinator this node does not run.
/// A node that is not serving advertises no endpoint. Returning one anyway
/// would invite the challenger to probe a coordinator this node does not run.
#[tokio::test]
async fn a_node_without_a_seat_advertises_nothing_to_probe() {
async fn a_node_not_serving_advertises_nothing_to_probe() {
let state = coordinator_state(None);
let resp = coordinator_verify_handler(
axum::extract::State(state),
Expand All @@ -13118,7 +13139,7 @@ mod tests {
.expect("body");
let v: serde_json::Value = serde_json::from_slice(&body).expect("json");
assert!(v["response"]["advertised_endpoint"].is_null());
assert!(v["response"]["my_seat"].is_null());
assert_eq!(v["response"]["serving"], false);
}

/// With nothing wired the node says so, rather than defaulting to a claim.
Expand Down
2 changes: 1 addition & 1 deletion crates/ghost-verification/src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1509,7 +1509,7 @@ pub struct VerificationState {
get_self_check: Option<Box<dyn Fn() -> serde_json::Value + Send + Sync>>,
/// Coordinator-election snapshot callback (pre-serialized JSON to avoid a
/// reverse-dependency on the `wraith-protocol`/`ghost-pool` election types).
/// Returns `{enabled, epoch, seats, my_seat, elected}` when the feature is
/// Returns `{enabled, epoch, tiers, coordinators, my_tiers, my_endpoint, …}` when the feature is
/// on, or `{enabled:false}` when off / not wired. Read-only.
get_coordinator_status: Option<Box<dyn Fn() -> serde_json::Value + Send + Sync>>,
/// Mesh-wide deduplicated active miner count callback. Returns the size
Expand Down
Loading
Loading