From 61ac283d42c6aa08cebef4e1ceb8a88132ea96ef Mon Sep 17 00:00:00 2001 From: Ilia Baranov <90713890+iliabaranov@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:15:21 -0700 Subject: [PATCH 1/9] wifi: guard esp_wifi_init against OOM panic (defer instead of crash) Enabling WiFi at runtime while the USB-NCM tether stack is resident (and/or mbedTLS holds its dynamic buffers during a Tailscale bring-up burst) exhausts internal DMA-capable RAM, and a WiFi-driver allocation ABORTS internally rather than returning ESP_ERR_NO_MEM -> PANIC + reboot (repro: two PANIC resets when WiFi was toggled on at ~73 KB free internal on an eth+USB-resident PSTOP54). Add a pre-flight internal-heap floor (WIFI_MIN_INTERNAL_HEAP, 90 KB) checked before esp_wifi_init; on a shortfall, synthesize ESP_ERR_NO_MEM and fall into the existing cleanup + 4 s deferred-retry path. This converts the OOM crash into a safe, retried deferral (fail-safe: no WiFi -> eventual STOP if it's the only uplink, never a crash-loop). Validated on PSTOP54: enabling WiFi (the exact action that PANIC-crashed it) now leaves rst_hist unchanged (no new PANIC), uptime monotonic (no reboot), and the safety bond held (state=2) throughout; restore clean. Threshold is conservative + tunable: set above the ~73 KB panic point, reachable once mbedTLS frees ~50 KB when TLS is idle. Needs HIL tuning for the WiFi-actually-associates case; the deeper fix (fitting WiFi alongside a resident USB-NCM stack) is the Internal-RAM Phase-3 work. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01UJz4ZB7WcPErDqKLWDq3ht --- .../components/dcs_support/src/dcs_wifi.c | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/firmware/components/dcs_support/src/dcs_wifi.c b/firmware/components/dcs_support/src/dcs_wifi.c index a43ef1de..69fe89ad 100644 --- a/firmware/components/dcs_support/src/dcs_wifi.c +++ b/firmware/components/dcs_support/src/dcs_wifi.c @@ -46,6 +46,20 @@ static const char * TAG = "dcs_wifi"; #define WIFI_BACKOFF_MIN_MS 2000u #define WIFI_BACKOFF_MAX_MS 30000u +/* Pre-flight internal-RAM floor for WiFi bring-up. esp_wifi_init + the stock + * RX/TX buffers need ~50 KB of INTERNAL (DMA-capable) heap. When that isn't + * free — WiFi enabled while the USB-NCM tether stack is resident, and/or the + * mbedTLS dynamic buffers are held during a Tailscale bring-up burst — a WiFi + * driver allocation aborts INTERNALLY (not a clean ESP_ERR_NO_MEM return) and + * panics/reboots the unit (observed: two PANIC resets when WiFi was enabled at + * ~73 KB free internal on an eth+USB-resident unit). Refusing up front below + * this floor turns that OOM crash into the same safe, retried deferral the + * esp_wifi_init-error path already takes. Conservative + tunable: set above the + * ~73 KB panic point; reachable once mbedTLS frees its ~50 KB of dynamic + * buffers when TLS is idle. The deeper fix (fitting WiFi alongside a resident + * USB-NCM stack) is the Internal-RAM Phase-3 work. */ +#define WIFI_MIN_INTERNAL_HEAP (90u * 1024u) + static atomic_bool s_enabled = false; /* driver running (gates on_wifi_evt) */ static atomic_bool s_want_up = false; /* admin intent — survives a NO_MEM */ static bool s_inited = false; /* driver + netif created by us */ @@ -209,13 +223,20 @@ static esp_err_t wifi_set_enabled_locked(bool on) * Only nvs_enable is overridden (we use WIFI_STORAGE_RAM). */ wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); cfg.nvs_enable = 0; - esp_err_t r = esp_wifi_init(&cfg); + /* OOM-panic guard: check the internal-RAM floor BEFORE esp_wifi_init so + * a tight-RAM bring-up defers instead of aborting inside the driver. + * On a shortfall we synthesize ESP_ERR_NO_MEM and fall straight into the + * existing cleanup + deferred-retry path below (no separate handling). */ + size_t free_int = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); + esp_err_t r = (free_int < WIFI_MIN_INTERNAL_HEAP) ? ESP_ERR_NO_MEM : esp_wifi_init(&cfg); if (r != ESP_OK) { ESP_LOGE( TAG, - "esp_wifi_init: %s — internal RAM tight (likely the " - "Tailscale bring-up burst); will retry", - esp_err_to_name(r)); + "esp_wifi_init deferred: %s (free internal %u B < %u floor) — RAM " + "tight (USB-NCM resident / Tailscale bring-up burst); will retry", + esp_err_to_name(r), + (unsigned)free_int, + (unsigned)WIFI_MIN_INTERNAL_HEAP); if (s_sta != NULL) { esp_netif_destroy_default_wifi(s_sta); s_sta = NULL; From a8eff9ac52f5a5bc4219edd1a8e5bfe141dce41d Mon Sep 17 00:00:00 2001 From: Ilia Baranov <90713890+iliabaranov@users.noreply.github.com> Date: Sat, 8 Aug 2026 03:52:21 -0700 Subject: [PATCH 2/9] =?UTF-8?q?wifi:=20Internal-RAM=20Phase=203=20?= =?UTF-8?q?=E2=80=94=20trim=20static=20TX=20buffers=2016=20->=206=20(~16?= =?UTF-8?q?=20KB=20internal)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 16 WiFi static TX buffers pre-reserve ~25 KB of DMA-locked internal RAM at esp_wifi_init; 6 is ample for the low-rate (10 Hz, tiny-frame) pstop link and returns ~16 KB, so WiFi init leaves ~39 KB free instead of the ~23 KB that OOM-panicked before (the real fix behind the WIFI_MIN_INTERNAL_HEAP guard, which is re-tuned 90 -> 72 KB to match the smaller footprint). TX-side only, by design: TX buffers do not gate the WPA2 EAPOL handshake, so this avoids the RX-ring starvation that broke association in an earlier trim. Dynamic TX was not an option — ESP_WIFI_DYNAMIC_TX_BUFFER depends on !SPIRAM_TRY_ALLOCATE_WIFI_LWIP and we keep WiFi lwIP in PSRAM (a bigger win). Builds with STATIC_TX_BUFFER_NUM=6; the ~16 KB is realized at esp_wifi_init (static WiFi buffers aren't allocated until WiFi starts, so no idle delta on an eth-only unit). docs/INTERNAL_RAM_REDUCTION.md Phase 3 marked done (safe half). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01UJz4ZB7WcPErDqKLWDq3ht --- docs/INTERNAL_RAM_REDUCTION.md | 26 ++++++++++++++----- .../components/dcs_support/src/dcs_wifi.c | 9 ++++--- firmware/sdkconfig.defaults | 12 +++++++++ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/INTERNAL_RAM_REDUCTION.md b/docs/INTERNAL_RAM_REDUCTION.md index 51871ee1..e955fffd 100644 --- a/docs/INTERNAL_RAM_REDUCTION.md +++ b/docs/INTERNAL_RAM_REDUCTION.md @@ -61,13 +61,27 @@ worth risking the OTA/rollback safety chain. Revisit only if a future feature re-pressures internal RAM, and then prefer Phase 3 (WiFi) or a proper mark-valid decouple over PSRAM-stacking flash-op tasks. -### Phase 3 — biggest reservoir (design decision, deferred) +### Phase 3 — WiFi static TX trim (2026-08-08) ✅ done (safe half) WiFi driver static buffers ≈ **40–50 KB internal, DMA-locked (cannot go to PSRAM)**. -Options: don't start WiFi when USB-NCM/Ethernet is the live link (lazy-start only for -the AP config portal); or trim `ESP_WIFI_STATIC_TX_BUFFER_NUM` 16→4–6, -`STATIC_RX_BUFFER_NUM` 10→6, `RX_BA_WIN` 16→6, TX→dynamic. **Risk:** MOD — WiFi is a -live STA transport + AP-fallback config portal; this is a product decision, so it's -deferred out of this pass. +**Done:** trimmed `ESP_WIFI_STATIC_TX_BUFFER_NUM` **16 → 6** (`sdkconfig.defaults`), +returning ~16 KB of internal DMA RAM at WiFi-init time (each static TX buffer ≈ +1.6 KB). This is the direct fix for the WiFi-enable OOM panic (WiFi now needs ~16 KB +less internal, so at ~73 KB free it leaves ~39 KB instead of the ~23 KB that +crashed); the `dcs_wifi.c WIFI_MIN_INTERNAL_HEAP` guard was re-tuned 90 → 72 KB to +match. TX buffers do **not** gate the WPA2 EAPOL 4-way handshake, so this is the +low-risk half. +- **TX static→dynamic was NOT possible:** `ESP_WIFI_DYNAMIC_TX_BUFFER` + `depends on !SPIRAM_TRY_ALLOCATE_WIFI_LWIP`, and we keep WiFi lwIP in PSRAM + (a bigger internal win); IDF forces static TX in that mode, so count-trim is the + lever. +- **NOT done (deliberate):** the RX side (`STATIC_RX_BUFFER_NUM`, `RX_BA_WIN`) is + left at stock — an earlier over-trim starved the RX ring and broke the EAPOL + handshake even at good signal. Revisit only with a WiFi-AP bench to validate + association after each RX reduction. +- **Validation:** builds with `STATIC_TX_BUFFER_NUM=6`; the ~16 KB is realized at + `esp_wifi_init` (not at idle, since static WiFi buffers aren't allocated until + WiFi starts). Runtime WiFi-fits demo needs a scenario with ≥72 KB free at + WiFi-enable (e.g. TLS idle) — pending a WiFi-AP bench. ## Explicitly NOT touched (safety / hardware constraints) - **ml_wg_mgr stack (16 KB)** — THE pstop heartbeat datapath (full WG diff --git a/firmware/components/dcs_support/src/dcs_wifi.c b/firmware/components/dcs_support/src/dcs_wifi.c index 69fe89ad..c2c76593 100644 --- a/firmware/components/dcs_support/src/dcs_wifi.c +++ b/firmware/components/dcs_support/src/dcs_wifi.c @@ -56,9 +56,12 @@ static const char * TAG = "dcs_wifi"; * this floor turns that OOM crash into the same safe, retried deferral the * esp_wifi_init-error path already takes. Conservative + tunable: set above the * ~73 KB panic point; reachable once mbedTLS frees its ~50 KB of dynamic - * buffers when TLS is idle. The deeper fix (fitting WiFi alongside a resident - * USB-NCM stack) is the Internal-RAM Phase-3 work. */ -#define WIFI_MIN_INTERNAL_HEAP (90u * 1024u) + * buffers when TLS is idle. Re-tuned 90 KB -> 72 KB after Internal-RAM Phase 3 + * (2026-08-08) trimmed the WiFi static TX buffers 16 -> 6 (~16 KB less internal + * footprint), so WiFi now fits at a lower free-internal level; 72 KB still + * leaves a safe post-init margin (init needs ~34 KB after Phase 3) and stays + * above the reduced danger zone. */ +#define WIFI_MIN_INTERNAL_HEAP (72u * 1024u) static atomic_bool s_enabled = false; /* driver running (gates on_wifi_evt) */ static atomic_bool s_want_up = false; /* admin intent — survives a NO_MEM */ diff --git a/firmware/sdkconfig.defaults b/firmware/sdkconfig.defaults index 09156a9a..f88e3b60 100644 --- a/firmware/sdkconfig.defaults +++ b/firmware/sdkconfig.defaults @@ -85,6 +85,18 @@ CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=y CONFIG_FREERTOS_VTASKLIST_INCLUDE_COREID=y # WiFi +# Internal-RAM Phase 3 (2026-08-08): trim WiFi static TX buffers 16 -> 6. Each +# static TX buffer is ~1.6 KB of internal DMA RAM (DMA-locked, cannot go to +# PSRAM), so 16 pre-reserved ~25 KB; 6 frees ~16 KB back to the general internal +# heap so WiFi fits alongside a resident USB-NCM tether stack — the condition +# that OOM-panicked esp_wifi_init (see dcs_wifi.c WIFI_MIN_INTERNAL_HEAP guard). +# TX-side only: 6 buffers are ample for the low-rate (10 Hz, tiny-frame) pstop +# link, and TX buffers do NOT gate the WPA2 EAPOL 4-way handshake. The RX ring +# is deliberately LEFT at stock (STATIC_RX=10, DYNAMIC_RX=32) because an earlier +# over-trim starved the RX ring and broke EAPOL even at good signal. +# (Dynamic TX is unavailable here: ESP_WIFI_DYNAMIC_TX_BUFFER depends on +# !SPIRAM_TRY_ALLOCATE_WIFI_LWIP, and we keep WiFi lwIP buffers in PSRAM.) +CONFIG_ESP_WIFI_STATIC_TX_BUFFER_NUM=6 CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=10 CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=32 CONFIG_ESP_WIFI_AMPDU_TX_ENABLED=y From 571f8f96ff499e3ed39a2c1fdbcb0b52b9d636e9 Mon Sep 17 00:00:00 2001 From: Ilia Baranov <90713890+iliabaranov@users.noreply.github.com> Date: Sat, 8 Aug 2026 03:58:56 -0700 Subject: [PATCH 3/9] =?UTF-8?q?wifi(machn):=20Internal-RAM=20Phase=203=20?= =?UTF-8?q?=E2=80=94=20static=20TX=20buffers=2016=20->=206=20(parity=20wit?= =?UTF-8?q?h=20remote)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror firmware/sdkconfig.defaults on the machine app so machn gets the same ~16 KB internal-DMA reduction at esp_wifi_init (it shares the dcs_support dcs_wifi OOM guard). TX-side only; RX ring left at stock. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01UJz4ZB7WcPErDqKLWDq3ht --- machn/sdkconfig.defaults | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/machn/sdkconfig.defaults b/machn/sdkconfig.defaults index 79c977e3..1fbeadf8 100644 --- a/machn/sdkconfig.defaults +++ b/machn/sdkconfig.defaults @@ -85,6 +85,12 @@ CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=y CONFIG_FREERTOS_VTASKLIST_INCLUDE_COREID=y # WiFi +# Internal-RAM Phase 3 (2026-08-08): WiFi static TX buffers 16 -> 6, matching the +# remote (firmware/sdkconfig.defaults). Returns ~16 KB internal DMA at +# esp_wifi_init so WiFi fits alongside the resident USB-NCM stack (OOM-panic fix; +# paired with the dcs_wifi.c WIFI_MIN_INTERNAL_HEAP guard, a shared component). +# TX-side only; RX ring left at stock to protect the WPA2 EAPOL handshake. +CONFIG_ESP_WIFI_STATIC_TX_BUFFER_NUM=6 CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=10 CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=32 CONFIG_ESP_WIFI_AMPDU_TX_ENABLED=y From e37cff58c572c20d0ebe368eee51e9e66dacf67b Mon Sep 17 00:00:00 2001 From: Ilia Baranov <90713890+iliabaranov@users.noreply.github.com> Date: Sat, 8 Aug 2026 15:44:17 -0700 Subject: [PATCH 4/9] mem(ncm): move tether RX copy to PSRAM to relieve internal SRAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The on_usb_rx per-frame copy handed to lwIP was malloc'd from internal SRAM on the device-side USB-NCM path — the scarcest heap on the S3, and the pool that dipped to an ~8 KB low-watermark on the tethered device. The buffer is filled by a plain CPU memcpy on the TinyUSB task (never DMA, never ISR) and consumed on the TCPIP thread, so the flash-cache- disable hazard does not apply; move it to PSRAM via heap_caps_malloc. This removes per-frame internal-heap alloc/free churn (fragmentation of the last few internal KB) from the tether RX hot path. netif_l2_free()'s free() handles a PSRAM pointer unchanged. The dominant USB-NCM consumer (the ~19.3 KB NTB DMA block) is pinned to internal SRAM by the S3 USB-OTG DMA engine and cannot move; reclaiming it via NTB count/size trim is tracked separately, gated on the USB-NCM stability regression. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01UJz4ZB7WcPErDqKLWDq3ht --- components/ml_dev_tether/src/ml_dev_tether.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/components/ml_dev_tether/src/ml_dev_tether.c b/components/ml_dev_tether/src/ml_dev_tether.c index 9ba2d4a7..5dbcdb57 100644 --- a/components/ml_dev_tether/src/ml_dev_tether.c +++ b/components/ml_dev_tether/src/ml_dev_tether.c @@ -25,6 +25,7 @@ #include #include "esp_event.h" +#include "esp_heap_caps.h" #include "esp_log.h" #include "esp_mac.h" #include "esp_netif.h" @@ -82,8 +83,14 @@ static esp_err_t on_usb_rx(void * buffer, uint16_t len, void * ctx) esp_err_t r = ESP_OK; if (n) { /* lwIP takes ownership; we hand it its own buffer so tinyusb can - * recycle the original. */ - void * copy = malloc(len); + * recycle the original. Allocate from PSRAM: this copy is filled by a + * plain CPU memcpy on the TinyUSB task (never DMA, never ISR — see the + * comment above) and consumed on the TCPIP thread, so the flash-cache- + * disable hazard does not apply. Keeping it off internal SRAM removes + * per-frame internal-heap alloc/free churn from the tether RX hot path + * (the device-side USB-NCM path, where internal RAM is scarcest). + * netif_l2_free()'s free() handles a PSRAM pointer unchanged. */ + void * copy = heap_caps_malloc(len, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); if (!copy) { r = ESP_ERR_NO_MEM; } else { From ff78e738f5e1a51e0ccd63b393918f164720a0d2 Mon Sep 17 00:00:00 2001 From: Ilia Baranov <90713890+iliabaranov@users.noreply.github.com> Date: Sat, 8 Aug 2026 07:50:52 -0700 Subject: [PATCH 5/9] safety(net): raise REBOND_AFTER_MS 1500 -> 2500ms (above the 2.0s machine timeout) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-session reply-loss watchdog was set to exceed the machine's heartbeat timeout so the remote only re-bonds AFTER machn would itself have dropped the bond — never on a sub-timeout reply blip. That invariant broke when max_missed was raised 3 -> 5 (2026-08-04): the machine timeout moved 1000 -> 2000ms but REBOND_AFTER_MS stayed 1500ms, dropping it BELOW the timeout. Consequence (root-caused from a soak disconnect): when a second remote joins the tailnet, the peer-join X25519/DISCO crypto burst time-shares the wg_mgr task that also decrypts the safety-heartbeat replies, lagging them ~1.6s. At 1500ms that lag forced a nuisance rebond even though machn still held the bond (gap 49->57, sf_txdrv=0, mismatch=0 — replies merely late, not lost). 2500ms (= 2000 + 500 jitter) restores "rebond only after a real machine-side drop." Safety unaffected: machn's 2.0s timeout remains the STOP authority; this constant governs only connectivity re-sync. Immediate mitigation for the cross-remote disruption; the deeper fix (priority-isolate the safety decrypt from control-plane crypto) is designed separately for review. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01UJz4ZB7WcPErDqKLWDq3ht --- firmware/main/main.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/firmware/main/main.c b/firmware/main/main.c index d88943b2..62722d9e 100644 --- a/firmware/main/main.c +++ b/firmware/main/main.c @@ -101,10 +101,19 @@ /* Reply-loss watchdog, per session. If a machine stops replying for longer * than this, that link has desynced and won't recover on its own — re-bond - * that session to re-sync counters. Must exceed the machine heartbeat - * timeout (1000 ms) plus normal jitter so healthy single-reply drops never - * trigger it. Other sessions are untouched. */ -#define REBOND_AFTER_MS 1500u + * that session to re-sync counters. Other sessions are untouched. + * + * MUST exceed the machine's heartbeat timeout + jitter, so the remote only + * tears the session AFTER machn would itself have dropped the bond — never on + * a sub-timeout reply blip. That timeout is now `heartbeat_ms × max_missed` + * = 400 × 5 = 2000 ms (raised 3→5 on 2026-08-04). The old value (1500 ms) was + * set against the legacy 1000 ms timeout and was NOT updated with max_missed, + * so it dropped BELOW the machine timeout: a ~1.6 s reply lag (e.g. a peer-join + * X25519/DISCO crypto burst time-sharing the wg_mgr decrypt task) forced a + * nuisance rebond even though machn still held the bond. 2500 ms = 2000 + 500 + * jitter margin restores the invariant. Safety is unaffected — machn's 2.0 s + * timeout remains the STOP authority; this only governs connectivity re-sync. */ +#define REBOND_AFTER_MS 2500u /* Bond handshake, per session: one BOND in flight at a time; retry after * this long without a reply. Long spacing means we never have more than one @@ -626,7 +635,8 @@ static uint32_t sess_send_period_ms(const pstop_sess_t * s) /* Reply-loss watchdog threshold: replies arrive once per transmit, so the * threshold scales with the adopted send period (4 missed replies), floored - * at the legacy 1500 ms that the full-rate link was validated with. */ + * at REBOND_AFTER_MS (2500 ms — kept above the machine's 2.0 s heartbeat + * timeout so a sub-timeout blip never forces a rebond). */ static uint64_t sess_rebond_after_ms(const pstop_sess_t * s) { uint64_t t = (uint64_t)sess_send_period_ms(s) * 4u; From cc1afdc0b3932017e355400bcd481e83a0742f3d Mon Sep 17 00:00:00 2001 From: Ilia Baranov <90713890+iliabaranov@users.noreply.github.com> Date: Sat, 8 Aug 2026 07:55:09 -0700 Subject: [PATCH 6/9] net(usb-ncm): widen pstop TX retry budget 3 -> 6 with backoff (sf_txdrv under load) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 3-device soak at ~70% core-1 load showed a remote on USB-NCM hitting pstop_sf_txdrv=9 (unrecovered TX drops) and one self-rebond. Root cause: chip TX over USB-NCM refuses (ERR_IF, can_xmit=false) when all IN NTBs are in-flight awaiting the host's bulk-IN poll; the only transient recovery is the sess_sendto retry loop, and at high load the prio-5 CPU1 TinyUSB task is slow to drain NTBs, so bursts outran the old ~3 ms (3×1 ms) window. Raise PSTOP_TX_RETRY_MAX 3 -> 6 with a 1 ms→2 ms backoff (~9 ms worst case). ERR_IF-gated only — a dead link (route/ENOMEM) still fails on attempt 0, so dead-link STOP latency is unchanged; ~9 ms is still << the send period and << the machine's 2.0 s timeout. Zero internal-RAM cost (unlike raising the IN NTB count, the secondary lever held in reserve since it re-pressures the Phase-3 internal-RAM budget). Also corrects the stale "~1.2 s timeout" comment to the current 2.0 s (400 ms × max_missed 5). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01UJz4ZB7WcPErDqKLWDq3ht --- firmware/main/main.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/firmware/main/main.c b/firmware/main/main.c index 62722d9e..4b44a142 100644 --- a/firmware/main/main.c +++ b/firmware/main/main.c @@ -779,14 +779,18 @@ static void sess_drain(pstop_sess_t * s) static uint32_t g_sf_txdrv_recovered; /* Max immediate retries on a TRANSIENT uplink-driver refusal (ERR_IF, errno -1 - * — e.g. USB-NCM all IN NTBs momentarily in-flight). Each retry yields 1 ms - * (FreeRTOS HZ=1000) so the lower-priority TinyUSB task can complete a bulk-IN - * transfer and return an NTB to the free list; the resend then succeeds within - * the SAME 200 ms send tick, so the machine never sees a heartbeat gap. Bounded - * at 3 (<= 3 ms << the 200 ms period and << the machine's ~1.2 s timeout), and - * gated on ERR_IF ONLY — a genuinely dead link (route/ENOMEM) still fails on the - * first attempt, so dead-link detection latency is UNCHANGED. */ -#define PSTOP_TX_RETRY_MAX 3 + * — e.g. USB-NCM all IN NTBs momentarily in-flight). Each retry yields so the + * lower-priority (prio 5, core 1) TinyUSB task can complete a bulk-IN transfer + * and return an NTB to the free list; the resend then succeeds within the SAME + * send tick, so the machine never sees a heartbeat gap. Raised 3 -> 6 (with a + * 1 ms→2 ms backoff, ~9 ms worst case) after a soak at ~70 % core-1 load showed + * bursts outrunning the old ~3 ms window (pstop_sf_txdrv=9): at high load the + * prio-5 USB task is slow to drain, so a wider window is needed to absorb the + * transient. Still ≪ the 200 ms period and ≪ the machine's ~2.0 s timeout + * (400 ms × max_missed 5). Gated on ERR_IF ONLY — a genuinely dead link + * (route/ENOMEM) still fails on the first attempt, so dead-link detection + * latency is UNCHANGED. */ +#define PSTOP_TX_RETRY_MAX 6 static bool sess_sendto(pstop_sess_t * s, const uint8_t * bytes) { @@ -802,7 +806,7 @@ static bool sess_sendto(pstop_sess_t * s, const uint8_t * bytes) if (errno != -1) { return false; /* not ERR_IF (route / ENOMEM / other): fail fast, never spin */ } - vTaskDelay(1); /* yield 1 ms so the prio-5 USB task can free an IN NTB */ + vTaskDelay((attempt < 3) ? 1 : 2); /* yield so the prio-5 USB task frees an IN NTB; widen as a burst persists */ } return false; /* still ERR_IF after the bounded retries -> counted as g_sf_txdrv */ } From 65abb3d3a2c11021dd3a7ae3d33e0bd69b3b4771 Mon Sep 17 00:00:00 2001 From: Ilia Baranov <90713890+iliabaranov@users.noreply.github.com> Date: Sat, 8 Aug 2026 08:23:48 -0700 Subject: [PATCH 7/9] net(wg): pace wireguardif_periodic handshake initiations (Option B safety-decrypt isolation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of a both-remotes-simultaneous rebond under tailnet churn: wireguardif_periodic() loops all WIREGUARD_MAX_PEERS (128) and runs a full X25519 handshake initiation (~30 ms) for EVERY peer that needs one, in a single uninterruptible call. When a netmap resync makes many peers re-handshake at once on a full peer table, that one call runs multiple seconds — and ml_wg_mgr drains the safety-heartbeat reply queue only BETWEEN calls, so both bonded remotes' replies starve for >2.5 s and both rebond (machn itself never rebooted). Fix (Option B, user-approved): budget the expensive handshake INITIATIONS to 4 per call, round-robin across calls (static resume index) so every peer is still serviced, deferring the rest to the next call. Cheap per-peer work (reset / keepalive / link_up) still runs for all peers each call. An established bond has no initiation pending, so the pinned safety peer is never delayed by the budget. Caps each call to ~120 ms of X25519 << the 2.0 s heartbeat timeout / 2.5 s reply-loss threshold, so ml_wg_mgr's between-call drain keeps the safety decrypt fresh. No API change; no dropped handshakes (deferred, retried next call). Shared wireguard_lwip -> applies to both remotes and machn (the both-remote stall was machine-side). Complements REBOND_AFTER_MS 2500 (which alone couldn't cover a >2.5 s stall). Broader mitigation: deploy the Tailscale ACLs to cut the churn that drives the handshake storm (fleet-side, tracked separately). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01UJz4ZB7WcPErDqKLWDq3ht --- .../wireguard_lwip/src/wireguardif.c | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/components/microlink/components/wireguard_lwip/src/wireguardif.c b/components/microlink/components/wireguard_lwip/src/wireguardif.c index 0e20d57a..0a91b64a 100644 --- a/components/microlink/components/wireguard_lwip/src/wireguardif.c +++ b/components/microlink/components/wireguard_lwip/src/wireguardif.c @@ -1230,7 +1230,21 @@ void wireguardif_periodic(struct netif *netif) { // Perform the same work as wireguardif_tmr but from the caller's task context, // avoiding heavy crypto (X25519, ChaCha20-Poly1305) on the lwIP TCPIP thread. bool link_up = false; - for (x = 0; x < WIREGUARD_MAX_PEERS; x++) { + /* Pace the expensive X25519 handshake INITIATIONS so a burst (all peers + * re-handshaking after a netmap resync on a full 128-peer table) cannot + * monopolize this single call for seconds and starve the caller's + * safety-heartbeat reply decrypt — root cause of a both-remotes rebond + * under tailnet churn (ml_wg_mgr drains wg_rx only between calls). Cheap + * per-peer work (reset / keepalive / link_up) still runs for every peer; + * only the ~30 ms initiation is budgeted, round-robin across calls so every + * peer is eventually serviced. An established bond has no initiation + * pending, so the pinned safety peer is never itself delayed by the budget. + * Budget 4 => <= ~120 ms of X25519 per call, well under the 2.0 s heartbeat + * timeout and 2.5 s reply-loss rebond threshold. */ + static uint16_t s_hs_resume = 0; + int hs_budget = 4; + for (int i = 0; i < WIREGUARD_MAX_PEERS; i++) { + x = (int)((s_hs_resume + (uint16_t)i) % (uint16_t)WIREGUARD_MAX_PEERS); peer = &device->peers[x]; if (peer->valid) { if (should_reset_peer(peer)) { @@ -1246,7 +1260,7 @@ void wireguardif_periodic(struct netif *netif) { if (should_send_keepalive(peer)) { wireguardif_send_keepalive(device, peer); } - if (should_send_initiation(peer)) { + if (should_send_initiation(peer) && (hs_budget > 0)) { WG_PRINT("[WG_PERIODIC] Handshake retry wg_idx=%d key=%02x%02x%02x%02x " "ip=%s:%u connect_ip=%s:%u active=%d send_hs=%d\n", x, @@ -1258,6 +1272,8 @@ void wireguardif_periodic(struct netif *netif) { peer->connect_port, peer->active, peer->send_handshake); wireguard_start_handshake(device->netif, peer); + hs_budget--; + s_hs_resume = (uint16_t)((x + 1) % WIREGUARD_MAX_PEERS); } if ((peer->curr_keypair.valid) || (peer->prev_keypair.valid)) { link_up = true; From 012ee88280bcf3beb36dce287be20bd0b6009be1 Mon Sep 17 00:00:00 2001 From: Ilia Baranov <90713890+iliabaranov@users.noreply.github.com> Date: Sun, 9 Aug 2026 16:36:08 -0700 Subject: [PATCH 8/9] safety(net): derive rebond threshold from the adopted heartbeat (review) The remote's self-rebond threshold was a hard-coded 2500 ms that only exceeded the machine's bond-drop timeout for the hb=400/max_missed=5 default. Derive it per session instead: max(2500, req_hb_ms*5 + 500), using the heartbeat the machine advertises in every reply. max_missed is NOT on the wire, so the x5 is a documented hard-coded coupling to machn's MACHN_MAX_MISSED_HEARTBEATS=5; long-term fix is advertising the absolute timeout. Also per review: the TX-retry comment claimed ~9 ms worst case but the inclusive loop bound does 7 attempts and delayed after the final failure (~11 ms). Skip the pointless trailing delay (making ~9 ms accurate) and correct the comment to 7 attempts (1 + 6 retries). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UJz4ZB7WcPErDqKLWDq3ht --- firmware/main/main.c | 56 +++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/firmware/main/main.c b/firmware/main/main.c index 4b44a142..e4bfeb47 100644 --- a/firmware/main/main.c +++ b/firmware/main/main.c @@ -105,16 +105,32 @@ * * MUST exceed the machine's heartbeat timeout + jitter, so the remote only * tears the session AFTER machn would itself have dropped the bond — never on - * a sub-timeout reply blip. That timeout is now `heartbeat_ms × max_missed` - * = 400 × 5 = 2000 ms (raised 3→5 on 2026-08-04). The old value (1500 ms) was - * set against the legacy 1000 ms timeout and was NOT updated with max_missed, - * so it dropped BELOW the machine timeout: a ~1.6 s reply lag (e.g. a peer-join - * X25519/DISCO crypto burst time-sharing the wg_mgr decrypt task) forced a - * nuisance rebond even though machn still held the bond. 2500 ms = 2000 + 500 - * jitter margin restores the invariant. Safety is unaffected — machn's 2.0 s - * timeout remains the STOP authority; this only governs connectivity re-sync. */ + * a sub-timeout reply blip. That timeout is `heartbeat_ms × max_missed` + * = 400 × 5 = 2000 ms with today's defaults (max_missed raised 3→5 on + * 2026-08-04). The old value (1500 ms) was set against the legacy 1000 ms + * timeout and was NOT updated with max_missed, so it dropped BELOW the machine + * timeout: a ~1.6 s reply lag (e.g. a peer-join X25519/DISCO crypto burst + * time-sharing the wg_mgr decrypt task) forced a nuisance rebond even though + * machn still held the bond. + * + * This constant is only the FLOOR. A hard-coded 2500 ms held the invariant + * for the 400 ms default alone; the actual threshold is derived per session + * from the machine-adopted heartbeat in sess_rebond_after_ms() so any + * configured heartbeat keeps the remote's watchdog above the machine's. + * The floor also covers the pre-first-reply state (heartbeat not yet + * learned). Safety is unaffected — machn's heartbeat timeout remains the + * STOP authority; this only governs connectivity re-sync. */ #define REBOND_AFTER_MS 2500u +/* Missed-heartbeat multiplier in the machine's bond-drop timeout. max_missed + * is NOT advertised on the wire (replies carry only heartbeat_timeout), so + * this is a hard-coded coupling to machn's MACHN_MAX_MISSED_HEARTBEATS = 5 + * (machn/main/main.c) — if that constant changes, this one MUST change with + * it. Long-term fix: advertise the machine's absolute timeout in the reply + * instead of making the remote reconstruct it. */ +#define REBOND_MACHINE_MAX_MISSED 5u +#define REBOND_JITTER_MARGIN_MS 500u + /* Bond handshake, per session: one BOND in flight at a time; retry after * this long without a reply. Long spacing means we never have more than one * BOND outstanding — critical because the pstop protocol rejects duplicate @@ -633,13 +649,18 @@ static uint32_t sess_send_period_ms(const pstop_sess_t * s) return p; } -/* Reply-loss watchdog threshold: replies arrive once per transmit, so the - * threshold scales with the adopted send period (4 missed replies), floored - * at REBOND_AFTER_MS (2500 ms — kept above the machine's 2.0 s heartbeat - * timeout so a sub-timeout blip never forces a rebond). */ +/* Reply-loss watchdog threshold: the machine's bond-drop timeout is + * req_hb_ms × max_missed, so derive from the per-session adopted heartbeat + * plus jitter margin. This keeps the invariant (the remote tears a session + * only AFTER the machine would have dropped the bond) for ANY configured + * heartbeat — the previous hard-coded 2500 ms only exceeded the machine + * timeout for the 400 ms / max_missed=5 default. The ×5 multiplier is a + * hard-coded mirror of machn's MACHN_MAX_MISSED_HEARTBEATS because + * max_missed is not on the wire (see REBOND_MACHINE_MAX_MISSED). Floored at + * REBOND_AFTER_MS, which also covers req_hb_ms == 0 (no reply adopted yet). */ static uint64_t sess_rebond_after_ms(const pstop_sess_t * s) { - uint64_t t = (uint64_t)sess_send_period_ms(s) * 4u; + uint64_t t = ((uint64_t)s->req_hb_ms * (uint64_t)REBOND_MACHINE_MAX_MISSED) + REBOND_JITTER_MARGIN_MS; return (t < (uint64_t)REBOND_AFTER_MS) ? (uint64_t)REBOND_AFTER_MS : t; } @@ -782,8 +803,9 @@ static uint32_t g_sf_txdrv_recovered; * — e.g. USB-NCM all IN NTBs momentarily in-flight). Each retry yields so the * lower-priority (prio 5, core 1) TinyUSB task can complete a bulk-IN transfer * and return an NTB to the free list; the resend then succeeds within the SAME - * send tick, so the machine never sees a heartbeat gap. Raised 3 -> 6 (with a - * 1 ms→2 ms backoff, ~9 ms worst case) after a soak at ~70 % core-1 load showed + * send tick, so the machine never sees a heartbeat gap. Raised 3 -> 6 retries + * (7 attempts total; 1 ms→2 ms backoff between attempts, ~9 ms worst case — + * no delay after the final failure) after a soak at ~70 % core-1 load showed * bursts outrunning the old ~3 ms window (pstop_sf_txdrv=9): at high load the * prio-5 USB task is slow to drain, so a wider window is needed to absorb the * transient. Still ≪ the 200 ms period and ≪ the machine's ~2.0 s timeout @@ -806,7 +828,9 @@ static bool sess_sendto(pstop_sess_t * s, const uint8_t * bytes) if (errno != -1) { return false; /* not ERR_IF (route / ENOMEM / other): fail fast, never spin */ } - vTaskDelay((attempt < 3) ? 1 : 2); /* yield so the prio-5 USB task frees an IN NTB; widen as a burst persists */ + if (attempt < PSTOP_TX_RETRY_MAX) { /* no send follows the last failure — don't burn a pointless delay */ + vTaskDelay((attempt < 3) ? 1 : 2); /* yield so the prio-5 USB task frees an IN NTB; widen as a burst persists */ + } } return false; /* still ERR_IF after the bounded retries -> counted as g_sf_txdrv */ } From bda129ed513c15cd54cdfa02a5744f2262ecddb5 Mon Sep 17 00:00:00 2001 From: Ilia Baranov <90713890+iliabaranov@users.noreply.github.com> Date: Sun, 9 Aug 2026 16:36:08 -0700 Subject: [PATCH 9/9] wifi(mem): add largest-free-block floor to the bring-up pre-flight (review) The pre-flight only checked total free internal heap, but the live fleet runs fragmented (observed 111 KB free with only a 31 KB largest block), so the 72 KB floor can pass while a single driver allocation still aborts internally. Add a second floor on heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL): 24 KB, sized ~50% above the biggest single internal allocation esp_wifi_init makes with this config (~16 KB static RX-buffer region) while staying below the fleet's observed 31 KB steady state so healthy-but-fragmented units still come up. Either floor failing defers via the existing retry path; the log says which floor tripped. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UJz4ZB7WcPErDqKLWDq3ht --- .../components/dcs_support/src/dcs_wifi.c | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/firmware/components/dcs_support/src/dcs_wifi.c b/firmware/components/dcs_support/src/dcs_wifi.c index c2c76593..05eb4432 100644 --- a/firmware/components/dcs_support/src/dcs_wifi.c +++ b/firmware/components/dcs_support/src/dcs_wifi.c @@ -63,6 +63,20 @@ static const char * TAG = "dcs_wifi"; * above the reduced danger zone. */ #define WIFI_MIN_INTERNAL_HEAP (72u * 1024u) +/* Second floor: LARGEST contiguous internal block. Total-free alone is not + * sufficient — the live fleet runs fragmented (observed: 111 KB free internal + * but only a 31 KB largest block), so the 72 KB floor can pass while no single + * allocation of even modest size can succeed, and the driver aborts internally + * exactly as in the total-free shortfall case. The driver blob is closed + * source, so the requirement is bounded empirically: with this project's + * config (static RX 10 × ~1.6 KB, static TX 6 × ~1.6 KB, ~6.5 KB driver task + * stack, DMA descriptor arrays) the largest single internal allocation + * esp_wifi_init makes is the ~16 KB static RX-buffer region; 24 KB gives it + * ~50 % headroom while staying BELOW the fleet's observed steady-state 31 KB + * largest block, so healthy-but-fragmented units still bring WiFi up instead + * of deferring forever. Both floors funnel into the same deferred-retry path. */ +#define WIFI_MIN_INTERNAL_BLOCK (24u * 1024u) + static atomic_bool s_enabled = false; /* driver running (gates on_wifi_evt) */ static atomic_bool s_want_up = false; /* admin intent — survives a NO_MEM */ static bool s_inited = false; /* driver + netif created by us */ @@ -226,20 +240,30 @@ static esp_err_t wifi_set_enabled_locked(bool on) * Only nvs_enable is overridden (we use WIFI_STORAGE_RAM). */ wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); cfg.nvs_enable = 0; - /* OOM-panic guard: check the internal-RAM floor BEFORE esp_wifi_init so - * a tight-RAM bring-up defers instead of aborting inside the driver. - * On a shortfall we synthesize ESP_ERR_NO_MEM and fall straight into the + /* OOM-panic guard: check BOTH internal-RAM floors (total free AND + * largest contiguous block — fragmentation can starve the driver even + * with plenty of total free) BEFORE esp_wifi_init so a tight-RAM + * bring-up defers instead of aborting inside the driver. On a + * shortfall we synthesize ESP_ERR_NO_MEM and fall straight into the * existing cleanup + deferred-retry path below (no separate handling). */ size_t free_int = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); - esp_err_t r = (free_int < WIFI_MIN_INTERNAL_HEAP) ? ESP_ERR_NO_MEM : esp_wifi_init(&cfg); + size_t block_int = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL); + bool floors_ok = (free_int >= WIFI_MIN_INTERNAL_HEAP) && (block_int >= WIFI_MIN_INTERNAL_BLOCK); + esp_err_t r = (!floors_ok) ? ESP_ERR_NO_MEM : esp_wifi_init(&cfg); if (r != ESP_OK) { ESP_LOGE( TAG, - "esp_wifi_init deferred: %s (free internal %u B < %u floor) — RAM " - "tight (USB-NCM resident / Tailscale bring-up burst); will retry", + "esp_wifi_init deferred: %s (%s floor: free internal %u B / min %u, " + "largest block %u B / min %u) — RAM tight/fragmented (USB-NCM " + "resident / Tailscale bring-up burst); will retry", esp_err_to_name(r), + (free_int < WIFI_MIN_INTERNAL_HEAP) ? "total-free" + : (block_int < WIFI_MIN_INTERNAL_BLOCK) ? "largest-block" + : "driver", /* floors passed; esp_wifi_init itself failed */ (unsigned)free_int, - (unsigned)WIFI_MIN_INTERNAL_HEAP); + (unsigned)WIFI_MIN_INTERNAL_HEAP, + (unsigned)block_int, + (unsigned)WIFI_MIN_INTERNAL_BLOCK); if (s_sta != NULL) { esp_netif_destroy_default_wifi(s_sta); s_sta = NULL;