hardening: connectivity robustness + internal-SRAM relief

components/microlink/components/wireguard_lwip/src/wireguardif.c

@@ -1230,7 +1230,21 @@ void wireguardif_periodic(struct netif *netif) {
 	// Perform the same work as wireguardif_tmr but from the caller's task context,
 	// avoiding heavy crypto (X25519, ChaCha20-Poly1305) on the lwIP TCPIP thread.
 	bool link_up = false;
-	for (x = 0; x < WIREGUARD_MAX_PEERS; x++) {
+	/* Pace the expensive X25519 handshake INITIATIONS so a burst (all peers
+	 * re-handshaking after a netmap resync on a full 128-peer table) cannot
+	 * monopolize this single call for seconds and starve the caller's
+	 * safety-heartbeat reply decrypt — root cause of a both-remotes rebond
+	 * under tailnet churn (ml_wg_mgr drains wg_rx only between calls). Cheap
+	 * per-peer work (reset / keepalive / link_up) still runs for every peer;
+	 * only the ~30 ms initiation is budgeted, round-robin across calls so every
+	 * peer is eventually serviced. An established bond has no initiation
+	 * pending, so the pinned safety peer is never itself delayed by the budget.
+	 * Budget 4 => <= ~120 ms of X25519 per call, well under the 2.0 s heartbeat
+	 * timeout and 2.5 s reply-loss rebond threshold. */
+	static uint16_t s_hs_resume = 0;
+	int hs_budget = 4;
+	for (int i = 0; i < WIREGUARD_MAX_PEERS; i++) {
+		x = (int)((s_hs_resume + (uint16_t)i) % (uint16_t)WIREGUARD_MAX_PEERS);
 		peer = &device->peers[x];
 		if (peer->valid) {
 			if (should_reset_peer(peer)) {
@@ -1246,7 +1260,7 @@ void wireguardif_periodic(struct netif *netif) {
 			if (should_send_keepalive(peer)) {
 				wireguardif_send_keepalive(device, peer);
 			}
-			if (should_send_initiation(peer)) {
+			if (should_send_initiation(peer) && (hs_budget > 0)) {
 				WG_PRINT("[WG_PERIODIC] Handshake retry wg_idx=%d key=%02x%02x%02x%02x "
 				       "ip=%s:%u connect_ip=%s:%u active=%d send_hs=%d\n",
 				       x,
@@ -1258,6 +1272,8 @@ void wireguardif_periodic(struct netif *netif) {
 				       peer->connect_port,
 				       peer->active, peer->send_handshake);
 				wireguard_start_handshake(device->netif, peer);
+				hs_budget--;
+				s_hs_resume = (uint16_t)((x + 1) % WIREGUARD_MAX_PEERS);
 			}
 			if ((peer->curr_keypair.valid) || (peer->prev_keypair.valid)) {
 				link_up = true;

components/ml_dev_tether/src/ml_dev_tether.c

@@ -25,6 +25,7 @@
 #include <string.h>
 
 #include "esp_event.h"
+#include "esp_heap_caps.h"
 #include "esp_log.h"
 #include "esp_mac.h"
 #include "esp_netif.h"
@@ -82,8 +83,14 @@ static esp_err_t on_usb_rx(void * buffer, uint16_t len, void * ctx)
   esp_err_t r = ESP_OK;
   if (n) {
     /* lwIP takes ownership; we hand it its own buffer so tinyusb can
-         * recycle the original. */
-    void * copy = malloc(len);
+         * recycle the original. Allocate from PSRAM: this copy is filled by a
+         * plain CPU memcpy on the TinyUSB task (never DMA, never ISR — see the
+         * comment above) and consumed on the TCPIP thread, so the flash-cache-
+         * disable hazard does not apply. Keeping it off internal SRAM removes
+         * per-frame internal-heap alloc/free churn from the tether RX hot path
+         * (the device-side USB-NCM path, where internal RAM is scarcest).
+         * netif_l2_free()'s free() handles a PSRAM pointer unchanged. */
+    void * copy = heap_caps_malloc(len, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
     if (!copy) {
       r = ESP_ERR_NO_MEM;
     } else {

docs/INTERNAL_RAM_REDUCTION.md

@@ -61,13 +61,27 @@ worth risking the OTA/rollback safety chain. Revisit only if a future feature
 re-pressures internal RAM, and then prefer Phase 3 (WiFi) or a proper mark-valid
 decouple over PSRAM-stacking flash-op tasks.
 
-### Phase 3 — biggest reservoir (design decision, deferred)
+### Phase 3 — WiFi static TX trim (2026-08-08) ✅ done (safe half)
 WiFi driver static buffers ≈ **40–50 KB internal, DMA-locked (cannot go to PSRAM)**.
-Options: don't start WiFi when USB-NCM/Ethernet is the live link (lazy-start only for
-the AP config portal); or trim `ESP_WIFI_STATIC_TX_BUFFER_NUM` 16→4–6,
-`STATIC_RX_BUFFER_NUM` 10→6, `RX_BA_WIN` 16→6, TX→dynamic. **Risk:** MOD — WiFi is a
-live STA transport + AP-fallback config portal; this is a product decision, so it's
-deferred out of this pass.
+**Done:** trimmed `ESP_WIFI_STATIC_TX_BUFFER_NUM` **16 → 6** (`sdkconfig.defaults`),
+returning ~16 KB of internal DMA RAM at WiFi-init time (each static TX buffer ≈
+1.6 KB). This is the direct fix for the WiFi-enable OOM panic (WiFi now needs ~16 KB
+less internal, so at ~73 KB free it leaves ~39 KB instead of the ~23 KB that
+crashed); the `dcs_wifi.c WIFI_MIN_INTERNAL_HEAP` guard was re-tuned 90 → 72 KB to
+match. TX buffers do **not** gate the WPA2 EAPOL 4-way handshake, so this is the
+low-risk half.
+- **TX static→dynamic was NOT possible:** `ESP_WIFI_DYNAMIC_TX_BUFFER`
+  `depends on !SPIRAM_TRY_ALLOCATE_WIFI_LWIP`, and we keep WiFi lwIP in PSRAM
+  (a bigger internal win); IDF forces static TX in that mode, so count-trim is the
+  lever.
+- **NOT done (deliberate):** the RX side (`STATIC_RX_BUFFER_NUM`, `RX_BA_WIN`) is
+  left at stock — an earlier over-trim starved the RX ring and broke the EAPOL
+  handshake even at good signal. Revisit only with a WiFi-AP bench to validate
+  association after each RX reduction.
+- **Validation:** builds with `STATIC_TX_BUFFER_NUM=6`; the ~16 KB is realized at
+  `esp_wifi_init` (not at idle, since static WiFi buffers aren't allocated until
+  WiFi starts). Runtime WiFi-fits demo needs a scenario with ≥72 KB free at
+  WiFi-enable (e.g. TLS idle) — pending a WiFi-AP bench.
 
 ## Explicitly NOT touched (safety / hardware constraints)
 - **ml_wg_mgr stack (16 KB)** — THE pstop heartbeat datapath (full WG

firmware/components/dcs_support/src/dcs_wifi.c

@@ -46,6 +46,37 @@ static const char * TAG = "dcs_wifi";
 #define WIFI_BACKOFF_MIN_MS 2000u
 #define WIFI_BACKOFF_MAX_MS 30000u
 
+/* Pre-flight internal-RAM floor for WiFi bring-up. esp_wifi_init + the stock
+ * RX/TX buffers need ~50 KB of INTERNAL (DMA-capable) heap. When that isn't
+ * free — WiFi enabled while the USB-NCM tether stack is resident, and/or the
+ * mbedTLS dynamic buffers are held during a Tailscale bring-up burst — a WiFi
+ * driver allocation aborts INTERNALLY (not a clean ESP_ERR_NO_MEM return) and
+ * panics/reboots the unit (observed: two PANIC resets when WiFi was enabled at
+ * ~73 KB free internal on an eth+USB-resident unit). Refusing up front below
+ * this floor turns that OOM crash into the same safe, retried deferral the
+ * esp_wifi_init-error path already takes. Conservative + tunable: set above the
+ * ~73 KB panic point; reachable once mbedTLS frees its ~50 KB of dynamic
+ * buffers when TLS is idle. Re-tuned 90 KB -> 72 KB after Internal-RAM Phase 3
+ * (2026-08-08) trimmed the WiFi static TX buffers 16 -> 6 (~16 KB less internal
+ * footprint), so WiFi now fits at a lower free-internal level; 72 KB still
+ * leaves a safe post-init margin (init needs ~34 KB after Phase 3) and stays
+ * above the reduced danger zone. */
+#define WIFI_MIN_INTERNAL_HEAP (72u * 1024u)
+
+/* Second floor: LARGEST contiguous internal block. Total-free alone is not
+ * sufficient — the live fleet runs fragmented (observed: 111 KB free internal
+ * but only a 31 KB largest block), so the 72 KB floor can pass while no single
+ * allocation of even modest size can succeed, and the driver aborts internally
+ * exactly as in the total-free shortfall case. The driver blob is closed
+ * source, so the requirement is bounded empirically: with this project's
+ * config (static RX 10 × ~1.6 KB, static TX 6 × ~1.6 KB, ~6.5 KB driver task
+ * stack, DMA descriptor arrays) the largest single internal allocation
+ * esp_wifi_init makes is the ~16 KB static RX-buffer region; 24 KB gives it
+ * ~50 % headroom while staying BELOW the fleet's observed steady-state 31 KB
+ * largest block, so healthy-but-fragmented units still bring WiFi up instead
+ * of deferring forever. Both floors funnel into the same deferred-retry path. */
+#define WIFI_MIN_INTERNAL_BLOCK (24u * 1024u)
+
 static atomic_bool s_enabled = false; /* driver running (gates on_wifi_evt) */
 static atomic_bool s_want_up = false; /* admin intent — survives a NO_MEM   */
 static bool s_inited = false; /* driver + netif created by us */
@@ -209,13 +240,30 @@ static esp_err_t wifi_set_enabled_locked(bool on)
                  * Only nvs_enable is overridden (we use WIFI_STORAGE_RAM). */
         wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
         cfg.nvs_enable = 0;
-        esp_err_t r = esp_wifi_init(&cfg);
+        /* OOM-panic guard: check BOTH internal-RAM floors (total free AND
+         * largest contiguous block — fragmentation can starve the driver even
+         * with plenty of total free) BEFORE esp_wifi_init so a tight-RAM
+         * bring-up defers instead of aborting inside the driver. On a
+         * shortfall we synthesize ESP_ERR_NO_MEM and fall straight into the
+         * existing cleanup + deferred-retry path below (no separate handling). */
+        size_t free_int = heap_caps_get_free_size(MALLOC_CAP_INTERNAL);
+        size_t block_int = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL);
+        bool floors_ok = (free_int >= WIFI_MIN_INTERNAL_HEAP) && (block_int >= WIFI_MIN_INTERNAL_BLOCK);
+        esp_err_t r = (!floors_ok) ? ESP_ERR_NO_MEM : esp_wifi_init(&cfg);
         if (r != ESP_OK) {
           ESP_LOGE(
             TAG,
-            "esp_wifi_init: %s — internal RAM tight (likely the "
-            "Tailscale bring-up burst); will retry",
-            esp_err_to_name(r));
+            "esp_wifi_init deferred: %s (%s floor: free internal %u B / min %u, "
+            "largest block %u B / min %u) — RAM tight/fragmented (USB-NCM "
+            "resident / Tailscale bring-up burst); will retry",
+            esp_err_to_name(r),
+            (free_int < WIFI_MIN_INTERNAL_HEAP)     ? "total-free"
+            : (block_int < WIFI_MIN_INTERNAL_BLOCK) ? "largest-block"
+                                                    : "driver", /* floors passed; esp_wifi_init itself failed */
+            (unsigned)free_int,
+            (unsigned)WIFI_MIN_INTERNAL_HEAP,
+            (unsigned)block_int,
+            (unsigned)WIFI_MIN_INTERNAL_BLOCK);
           if (s_sta != NULL) {
             esp_netif_destroy_default_wifi(s_sta);
             s_sta = NULL;

firmware/main/main.c

@@ -101,10 +101,35 @@
 
 /* Reply-loss watchdog, per session. If a machine stops replying for longer
  * than this, that link has desynced and won't recover on its own — re-bond
- * that session to re-sync counters. Must exceed the machine heartbeat
- * timeout (1000 ms) plus normal jitter so healthy single-reply drops never
- * trigger it. Other sessions are untouched. */
-#define REBOND_AFTER_MS 1500u
+ * that session to re-sync counters. Other sessions are untouched.
+ *
+ * MUST exceed the machine's heartbeat timeout + jitter, so the remote only
+ * tears the session AFTER machn would itself have dropped the bond — never on
+ * a sub-timeout reply blip. That timeout is `heartbeat_ms × max_missed`
+ * = 400 × 5 = 2000 ms with today's defaults (max_missed raised 3→5 on
+ * 2026-08-04). The old value (1500 ms) was set against the legacy 1000 ms
+ * timeout and was NOT updated with max_missed, so it dropped BELOW the machine
+ * timeout: a ~1.6 s reply lag (e.g. a peer-join X25519/DISCO crypto burst
+ * time-sharing the wg_mgr decrypt task) forced a nuisance rebond even though
+ * machn still held the bond.
+ *
+ * This constant is only the FLOOR. A hard-coded 2500 ms held the invariant
+ * for the 400 ms default alone; the actual threshold is derived per session
+ * from the machine-adopted heartbeat in sess_rebond_after_ms() so any
+ * configured heartbeat keeps the remote's watchdog above the machine's.
+ * The floor also covers the pre-first-reply state (heartbeat not yet
+ * learned). Safety is unaffected — machn's heartbeat timeout remains the
+ * STOP authority; this only governs connectivity re-sync. */
+#define REBOND_AFTER_MS 2500u
+
+/* Missed-heartbeat multiplier in the machine's bond-drop timeout. max_missed
+ * is NOT advertised on the wire (replies carry only heartbeat_timeout), so
+ * this is a hard-coded coupling to machn's MACHN_MAX_MISSED_HEARTBEATS = 5
+ * (machn/main/main.c) — if that constant changes, this one MUST change with
+ * it. Long-term fix: advertise the machine's absolute timeout in the reply
+ * instead of making the remote reconstruct it. */
+#define REBOND_MACHINE_MAX_MISSED 5u
+#define REBOND_JITTER_MARGIN_MS 500u
 
 /* Bond handshake, per session: one BOND in flight at a time; retry after
  * this long without a reply. Long spacing means we never have more than one
@@ -624,12 +649,18 @@ static uint32_t sess_send_period_ms(const pstop_sess_t * s)
   return p;
 }
 
-/* Reply-loss watchdog threshold: replies arrive once per transmit, so the
- * threshold scales with the adopted send period (4 missed replies), floored
- * at the legacy 1500 ms that the full-rate link was validated with. */
+/* Reply-loss watchdog threshold: the machine's bond-drop timeout is
+ * req_hb_ms × max_missed, so derive from the per-session adopted heartbeat
+ * plus jitter margin. This keeps the invariant (the remote tears a session
+ * only AFTER the machine would have dropped the bond) for ANY configured
+ * heartbeat — the previous hard-coded 2500 ms only exceeded the machine
+ * timeout for the 400 ms / max_missed=5 default. The ×5 multiplier is a
+ * hard-coded mirror of machn's MACHN_MAX_MISSED_HEARTBEATS because
+ * max_missed is not on the wire (see REBOND_MACHINE_MAX_MISSED). Floored at
+ * REBOND_AFTER_MS, which also covers req_hb_ms == 0 (no reply adopted yet). */
 static uint64_t sess_rebond_after_ms(const pstop_sess_t * s)
 {
-  uint64_t t = (uint64_t)sess_send_period_ms(s) * 4u;
+  uint64_t t = ((uint64_t)s->req_hb_ms * (uint64_t)REBOND_MACHINE_MAX_MISSED) + REBOND_JITTER_MARGIN_MS;
   return (t < (uint64_t)REBOND_AFTER_MS) ? (uint64_t)REBOND_AFTER_MS : t;
 }
 
@@ -769,14 +800,19 @@ static void sess_drain(pstop_sess_t * s)
 static uint32_t g_sf_txdrv_recovered;
 
 /* Max immediate retries on a TRANSIENT uplink-driver refusal (ERR_IF, errno -1
- * — e.g. USB-NCM all IN NTBs momentarily in-flight). Each retry yields 1 ms
- * (FreeRTOS HZ=1000) so the lower-priority TinyUSB task can complete a bulk-IN
- * transfer and return an NTB to the free list; the resend then succeeds within
- * the SAME 200 ms send tick, so the machine never sees a heartbeat gap. Bounded
- * at 3 (<= 3 ms << the 200 ms period and << the machine's ~1.2 s timeout), and
- * gated on ERR_IF ONLY — a genuinely dead link (route/ENOMEM) still fails on the
- * first attempt, so dead-link detection latency is UNCHANGED. */
-#define PSTOP_TX_RETRY_MAX 3
+ * — e.g. USB-NCM all IN NTBs momentarily in-flight). Each retry yields so the
+ * lower-priority (prio 5, core 1) TinyUSB task can complete a bulk-IN transfer
+ * and return an NTB to the free list; the resend then succeeds within the SAME
+ * send tick, so the machine never sees a heartbeat gap. Raised 3 -> 6 retries
+ * (7 attempts total; 1 ms→2 ms backoff between attempts, ~9 ms worst case —
+ * no delay after the final failure) after a soak at ~70 % core-1 load showed
+ * bursts outrunning the old ~3 ms window (pstop_sf_txdrv=9): at high load the
+ * prio-5 USB task is slow to drain, so a wider window is needed to absorb the
+ * transient. Still ≪ the 200 ms period and ≪ the machine's ~2.0 s timeout
+ * (400 ms × max_missed 5). Gated on ERR_IF ONLY — a genuinely dead link
+ * (route/ENOMEM) still fails on the first attempt, so dead-link detection
+ * latency is UNCHANGED. */
+#define PSTOP_TX_RETRY_MAX 6
 
 static bool sess_sendto(pstop_sess_t * s, const uint8_t * bytes)
 {
@@ -792,7 +828,9 @@ static bool sess_sendto(pstop_sess_t * s, const uint8_t * bytes)
     if (errno != -1) {
       return false; /* not ERR_IF (route / ENOMEM / other): fail fast, never spin */
     }
-    vTaskDelay(1); /* yield 1 ms so the prio-5 USB task can free an IN NTB */
+    if (attempt < PSTOP_TX_RETRY_MAX) { /* no send follows the last failure — don't burn a pointless delay */
+      vTaskDelay((attempt < 3) ? 1 : 2); /* yield so the prio-5 USB task frees an IN NTB; widen as a burst persists */
+    }
   }
   return false; /* still ERR_IF after the bounded retries -> counted as g_sf_txdrv */
 }

firmware/sdkconfig.defaults

@@ -85,6 +85,18 @@ CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=y
 CONFIG_FREERTOS_VTASKLIST_INCLUDE_COREID=y
 
 # WiFi
+# Internal-RAM Phase 3 (2026-08-08): trim WiFi static TX buffers 16 -> 6. Each
+# static TX buffer is ~1.6 KB of internal DMA RAM (DMA-locked, cannot go to
+# PSRAM), so 16 pre-reserved ~25 KB; 6 frees ~16 KB back to the general internal
+# heap so WiFi fits alongside a resident USB-NCM tether stack — the condition
+# that OOM-panicked esp_wifi_init (see dcs_wifi.c WIFI_MIN_INTERNAL_HEAP guard).
+# TX-side only: 6 buffers are ample for the low-rate (10 Hz, tiny-frame) pstop
+# link, and TX buffers do NOT gate the WPA2 EAPOL 4-way handshake. The RX ring
+# is deliberately LEFT at stock (STATIC_RX=10, DYNAMIC_RX=32) because an earlier
+# over-trim starved the RX ring and broke EAPOL even at good signal.
+# (Dynamic TX is unavailable here: ESP_WIFI_DYNAMIC_TX_BUFFER depends on
+# !SPIRAM_TRY_ALLOCATE_WIFI_LWIP, and we keep WiFi lwIP buffers in PSRAM.)
+CONFIG_ESP_WIFI_STATIC_TX_BUFFER_NUM=6
 CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=10
 CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=32
 CONFIG_ESP_WIFI_AMPDU_TX_ENABLED=y

machn/sdkconfig.defaults

@@ -85,6 +85,12 @@ CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=y
 CONFIG_FREERTOS_VTASKLIST_INCLUDE_COREID=y
 
 # WiFi
+# Internal-RAM Phase 3 (2026-08-08): WiFi static TX buffers 16 -> 6, matching the
+# remote (firmware/sdkconfig.defaults). Returns ~16 KB internal DMA at
+# esp_wifi_init so WiFi fits alongside the resident USB-NCM stack (OOM-panic fix;
+# paired with the dcs_wifi.c WIFI_MIN_INTERNAL_HEAP guard, a shared component).
+# TX-side only; RX ring left at stock to protect the WPA2 EAPOL handshake.
+CONFIG_ESP_WIFI_STATIC_TX_BUFFER_NUM=6
 CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=10
 CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=32
 CONFIG_ESP_WIFI_AMPDU_TX_ENABLED=y