fix(auto): an at-limit escape lands on a throttled idle peer; reset lifts a reported limit

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- `auto` escapes an account at its limit onto a peer whose usage only a 429 from the usage endpoint hides, by the headroom its last good reading showed, when no measured peer has any room. That throttle belongs to the endpoint, not the account, and it can outlast the two-hour trust ceiling by many hours on accounts nobody is using: three idle accounts sat unreadable for up to 19 hours while the engine reported `no-qualifying-candidate` every minute on a refused account. Never on failover, where the active account is unreadable too.
+- `reset` forgets a limit reported with `limit-hit` on the account it just reset. The report carried the refusal's own deadline, so it overruled every reading until then: the account read spent for 17 hours after a successful reset, and `auto` reported `all-exhausted` with an account at 4%.
 - `auto` wakes the moment a verb changes what it decides on, whichever process ran the verb: `add`, `add-oauth`, `add-token`, `import`, `remove`, `compact`, `switch`, `rotate`, `hold`, `unhold`, `prefer`, `auto-ignite`, `reorder`, `alias`, `limit-hit`, `reset` and `config set`/`unset` each write a fresh nonce to `<home>/auto.wake` on their way out, and a sleeping daemon reads that file once a second and ticks when it changes. Until now the only wake was a line on a supervised daemon's stdin, and the daemon Infinitus's menu-bar helper installs runs under launchd with no supervisor at all — so an account signed in through the app while every other read spent waited out the ten-minute `all-exhausted` sleep (six accounts spent, a seventh added at 15:23, the switch made by hand at 15:26, the daemon's next look at 15:32), and a refusal the server reported with `limit-hit` was read a poll later rather than at once.
 
 ## 0.3.3 — 2026-09-25

src/cmd/reset.rs

@@ -19,6 +19,7 @@ use crate::ctx::Ctx;
 use crate::driver::{Driver, DriverError, Login, Usage};
 use crate::errors::{ErrorCode, Result, SwapdError};
 use crate::output;
+use crate::secrets::slot_key;
 
 pub fn run(ctx: &Ctx, driver: &dyn Driver, ident: &str) -> Result<ListPayload> {
     let id = driver.id();
@@ -89,6 +90,11 @@ pub fn run(ctx: &Ctx, driver: &dyn Driver, ident: &str) -> Result<ListPayload> {
             format!("the provider refused the reset: {}", outcome.result),
         ));
     }
+    // A limit someone reported on this account (`swapd limit-hit`) is the one
+    // the reset just ended; left in place it would overrule every reading
+    // until its own deadline and keep the account out of the rotation.
+    ctx.store
+        .clear_reported_limit(&slot_key(id, slot), &meta.email, &meta.organization_uuid)?;
 
     // The board after the reset: the windows it cleared are only visible in
     // a fetch made after it, so this one ignores the serve TTL.

src/core/auto.rs

@@ -230,6 +230,10 @@ struct Snapshot {
     fresh: BTreeMap<u32, bool>,
     /// Slot → when its trusted measurement was fetched.
     fetched_at: BTreeMap<u32, Option<f64>>,
+    /// Slot → the headroom its last good reading showed, for a slot whose
+    /// trust has lapsed under usage 429s alone. Not decision-grade: only an
+    /// at-limit escape with nothing measurable to land on reads it.
+    throttled: BTreeMap<u32, f64>,
 }
 
 impl Snapshot {
@@ -250,6 +254,10 @@ impl Snapshot {
     fn fetched_at(&self, slot: u32) -> Option<f64> {
         self.fetched_at.get(&slot).copied().flatten()
     }
+
+    fn throttled(&self, slot: u32) -> Option<f64> {
+        self.throttled.get(&slot).copied()
+    }
 }
 
 /// The engine. Owns its `Ctx` because it re-reads `settings.json` into it
@@ -1177,6 +1185,7 @@ impl<'a> AutoEngine<'a> {
             fetch_errors: BTreeMap::new(),
             fresh: BTreeMap::new(),
             fetched_at: BTreeMap::new(),
+            throttled: BTreeMap::new(),
         };
         for account in &view.accounts {
             let entry = entries.get(&self.key_of(account.slot));
@@ -1191,6 +1200,16 @@ impl<'a> AutoEngine<'a> {
                 if let Some(kind) = entry.and_then(|e| e.last_error.clone()) {
                     snap.fetch_errors.insert(account.slot, kind);
                 }
+                // A usage 429 throttles the endpoint, not the account: an
+                // account nobody uses can sit in it for hours, long past the
+                // trust ceiling, with all of its quota intact.
+                let last_good = entry
+                    .filter(|e| measurable && e.last_error.as_deref() == Some("http-429"))
+                    .and_then(|e| e.last_good.as_deref());
+                let models = &self.ctx.settings.models;
+                if let Some(pct) = last_good.and_then(|w| binding_pct(w, models)) {
+                    snap.throttled.insert(account.slot, 100.0 - pct);
+                }
             }
             snap.fresh
                 .insert(account.slot, entry.is_some_and(|entry| entry.fresh(now)));
@@ -1507,6 +1526,21 @@ impl<'a> AutoEngine<'a> {
             qualifying.push((key, index, slot));
         }
 
+        // Last resort, for an active account known to be refused: a peer
+        // only the usage endpoint's throttle hides, by the headroom it last
+        // showed. Landing on one that turns out spent costs a switch; staying
+        // costs the session. Never on failover — the active account is
+        // unreadable there too, and this would hop between blind targets.
+        if qualifying.is_empty() && fallback.is_empty() && trigger == "at-limit" {
+            for (index, slot) in candidates.iter().copied().enumerate() {
+                if Some(slot) == no_return {
+                    continue;
+                }
+                if let Some(h) = snap.throttled(slot).filter(|h| *h > 0.0) {
+                    fallback.push((Key::new(&[-h]), index, slot));
+                }
+            }
+        }
         let mut ranked = if qualifying.is_empty() {
             fallback
         } else {

src/core/auto/tests.rs

@@ -52,6 +52,8 @@ struct FakeDriver {
     /// email → the windows `usage()` reports for it once it has been ignited
     /// (the window the run opened). An email not listed stays as it was.
     warm: Mutex<BTreeMap<String, Vec<Window>>>,
+    /// Accounts whose usage endpoint answers 429.
+    throttled: Mutex<Vec<String>>,
 }
 
 impl FakeDriver {
@@ -65,6 +67,7 @@ impl FakeDriver {
             live_reads: Mutex::new(0),
             ignites: Mutex::new(Vec::new()),
             warm: Mutex::new(BTreeMap::new()),
+            throttled: Mutex::new(Vec::new()),
         }
     }
 
@@ -79,6 +82,12 @@ impl FakeDriver {
         self.usage.lock().unwrap().remove(email);
     }
 
+    /// Make this account's usage endpoint answer 429, the throttle that
+    /// leaves the account itself untouched.
+    fn throttle_usage(&self, email: &str) {
+        self.throttled.lock().unwrap().push(email.to_string());
+    }
+
     fn set_usage(&self, email: &str, windows: Vec<Window>) {
         self.usage
             .lock()
@@ -180,6 +189,9 @@ impl Driver for FakeDriver {
     fn usage(&self, login: &Login) -> std::result::Result<Usage, DriverError> {
         let email = Self::email_of(login);
         self.usage_calls.lock().unwrap().push(email.clone());
+        if self.throttled.lock().unwrap().contains(&email) {
+            return Err(DriverError::Throttled { retry_after: None });
+        }
         match self.usage.lock().unwrap().get(&email) {
             Some(windows) => Ok(Usage {
                 windows: windows.clone(),
@@ -1331,6 +1343,66 @@ fn a_switch_in_flight_is_reported_as_switch_in_progress() {
     );
 }
 
+/// An idle account whose usage endpoint has 429'd past the trust ceiling is
+/// still where an at-limit escape lands when every readable peer is spent: the
+/// throttle is the endpoint's, not the account's (a fleet sat on a refused
+/// account for hours beside three unused ones, 2026-10-09). An account
+/// unreadable for any other reason is not.
+#[test]
+fn an_at_limit_escape_lands_on_a_peer_only_the_usage_throttle_hides() {
+    for throttled in [true, false] {
+        let board = Board::new();
+        board.seed(3, "three@example.com", "rt-3", T0 + 86_400.0);
+        board
+            .driver
+            .set_usage("one@example.com", usage_at(50.0, T0, 3600.0));
+        board
+            .driver
+            .set_usage("two@example.com", usage_at(100.0, T0, 18_000.0));
+        board
+            .driver
+            .set_usage("three@example.com", usage_at(0.0, T0, 3600.0));
+        // Every account read once while the endpoint still answers.
+        for _ in 0..5 {
+            board.tick();
+            board.advance(60.0);
+        }
+        assert!(board.last("switch").is_none());
+
+        if throttled {
+            board.driver.throttle_usage("three@example.com");
+        } else {
+            board.driver.fail_usage("three@example.com");
+        }
+        board.advance(3.0 * 3600.0);
+        let now = board.now();
+        board
+            .driver
+            .set_usage("one@example.com", usage_at(100.0, now, 18_000.0));
+        board
+            .driver
+            .set_usage("two@example.com", usage_at(100.0, now, 18_000.0));
+        for _ in 0..10 {
+            board.tick();
+            if board.last("switch").is_some() {
+                break;
+            }
+            board.advance(60.0);
+        }
+
+        let switch = board.last("switch");
+        if throttled {
+            let switch = switch.expect("the throttled idle peer is the escape");
+            assert_eq!(switch["to"]["number"], 3);
+            assert_eq!(switch["trigger"], "at-limit");
+            assert_eq!(board.live_email(), "three@example.com");
+        } else {
+            assert!(switch.is_none(), "an account failing for real is no target");
+            assert_eq!(board.live_email(), "one@example.com");
+        }
+    }
+}
+
 /// Two ways the ranking can come back empty with the active account at its
 /// threshold: nothing readable to compare against, and a candidate that is
 /// readable but does not clear the hysteresis margin. Neither is

src/core/usage_store.rs

@@ -905,6 +905,21 @@ impl UsageStore {
         Ok(resets_at)
     }
 
+    /// Forget a refusal the provider has since lifted: a spent banked reset
+    /// (`swapd reset`) ends the limit the report was a deadline for, so the
+    /// next measurement must be read as it stands rather than overruled.
+    pub fn clear_reported_limit(&self, key: &str, email: &str, org: &str) -> Result<()> {
+        let _lock = self.lock()?;
+        let mut rows = self.read_rows()?;
+        let Some(row) = rows.get_mut(key).filter(|row| matches(row, email, org)) else {
+            return Ok(());
+        };
+        row.reported_limit_at = None;
+        row.reported_limit_resets_at = None;
+        row.next_poll_at = Some(self.now());
+        self.write_rows(rows)
+    }
+
     /// Merge a failed fetch, fenced by its lease (`usage_store.py:1116-1137`).
     /// Never touches `last_good`/`fetched_at` — stale-on-error — so a failing
     /// account keeps its last measurement while its trust ages out.
@@ -1777,6 +1792,27 @@ mod tests {
         assert!(!entry(&store).reported_limit_held);
     }
 
+    #[test]
+    fn a_spent_reset_lifts_a_reported_limit_before_its_deadline() {
+        let dir = tempfile::tempdir().unwrap();
+        let clock = TestClock::new();
+        let store = store(&dir, &clock);
+
+        fetch_ok(&store, five(10.0));
+        report_limit(&store, Some(T0 + 86_400.0));
+        assert!(entry(&store).reported_limit_held);
+
+        store
+            .clear_reported_limit("claude:1", "you@example.com", "org-0000")
+            .unwrap();
+        clock.advance(MIN_INTERVAL_S + 1.0);
+        fetch_ok(&store, five(4.0));
+        let e = entry(&store);
+        assert!(!e.reported_limit_held);
+        assert_eq!(e.reported_limit_at, None);
+        assert_eq!(e.decision_windows().unwrap()[0].pct, 4.0);
+    }
+
     #[test]
     fn a_legacy_or_future_schema_reads_as_empty() {
         let dir = tempfile::tempdir().unwrap();