From 8e43c5920bced39c047120bc7f5c251f556a0fbd Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 03:10:41 -0500 Subject: [PATCH 1/4] Use smaller packets or the PC's other address when no video arrives When the PC says it sent video and the iPad still has no picture, the iPad asks for smaller media datagrams (1,200, then 576 bytes, each with a fresh keyframe). If that doesn't help, it asks the PC to sign its other home network addresses with the pairing, confirms the same PC at one of them, and reconnects there. The address that worked is tried first next time. The size request is a field appended to KEYFRAME_REQUEST, and the frame count is appended to HEARTBEAT. Both are gated on FEATURE_MEDIA_SIZE and HOSTCAP_MEDIA_SIZE, so older PCs and iPads see the same bytes as before. A PC session that sent video and ended with the iPad's error goodbye without a counted frame now summarizes as no_video or no_frame instead of other. --- host/src/diagnostics.rs | 28 ++ host/src/transport/mod.rs | 8 +- host/src/transport/session.rs | 232 ++++++++++++++++- host/tests/e2e_synthetic.rs | 122 +++++++++ .../App/ConnectionManager.swift | 149 ++++++++++- ios/EternalMonitor/App/MovedPC.swift | 35 ++- ios/EternalMonitor/App/NoVideoFallback.swift | 97 +++++++ ios/EternalMonitor/App/SessionSummary.swift | 14 +- .../Network/ControlChannel.swift | 11 +- ios/EternalMonitor/Network/WireProtocol.swift | 28 +- ios/EternalMonitorTests/FakeUDPHost.swift | 72 +++++ .../NoVideoFallbackTests.swift | 246 ++++++++++++++++++ .../WireProtocolGoldenTests.swift | 39 ++- proto/src/v2/control.rs | 98 ++++++- proto/testdata/v2_vectors.txt | 3 + proto/tests/golden.rs | 70 ++++- 16 files changed, 1220 insertions(+), 32 deletions(-) create mode 100644 ios/EternalMonitor/App/NoVideoFallback.swift create mode 100644 ios/EternalMonitorTests/NoVideoFallbackTests.swift diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index ab3e4d47..87ce175f 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -418,6 +418,15 @@ pub fn session_summary( EndReason::LinkLost => "other", EndReason::Liveness | EndReason::Goodbye(ByeReason::Error) if !streamed => "no_video", EndReason::Liveness => "signal_lost", + // The iPad gave up on video that went out: none of it arrived, or + // no frame of it completed. + EndReason::Goodbye(ByeReason::Error) if !end.frames_received => { + if end.frags_received { + "no_frame" + } else { + "no_video" + } + } EndReason::Goodbye(ByeReason::Error) => "other", EndReason::Failed(StageFailure::Capture) => "capture_failed", EndReason::Failed(StageFailure::Encoder) => "encoder_failed", @@ -705,6 +714,7 @@ pub(crate) mod tests { reason, first_video: first_video_ms.map(Duration::from_millis), frames_received: first_video_ms.is_some(), + frags_received: first_video_ms.is_some(), duration: Duration::from_millis(duration_ms), } } @@ -777,6 +787,24 @@ pub(crate) mod tests { end(Goodbye(ByeReason::Error), Some(500), 30_000), ["steady", "streamed", "other", "goodbye"], ), + // The iPad gave up on video that went out: none of it arrived, + // or no frame of it completed. + ( + SessionEnd { + frames_received: false, + frags_received: false, + ..end(Goodbye(ByeReason::Error), Some(300), 10_000) + }, + ["video_sent", "streamed", "no_video", "goodbye"], + ), + ( + SessionEnd { + frames_received: false, + frags_received: true, + ..end(Goodbye(ByeReason::Error), Some(300), 10_000) + }, + ["video_sent", "streamed", "no_frame", "goodbye"], + ), ( end(Liveness, None, 3_000), ["accepted", "no_picture", "no_video", "liveness"], diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index c2f7b1da..0ea8dd42 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -582,7 +582,13 @@ pub async fn start_sender( } let payload = &nal.data; - let payload_size = media_payload_size(shared.max_dgram.load(Ordering::SeqCst) as usize).unwrap_or(MAX_MEDIA_PAYLOAD); + // The PC's packet size, or the smaller one this session's + // client asked for when none of the larger reached it. + let max_dgram = shared.session.lock().max_media_dgram(session_id).map_or( + shared.max_dgram.load(Ordering::SeqCst), + |asked| shared.max_dgram.load(Ordering::SeqCst).min(u32::from(asked)), + ); + let payload_size = media_payload_size(max_dgram as usize).unwrap_or(MAX_MEDIA_PAYLOAD); let frag_count_usize = payload.len().div_ceil(payload_size).max(1); // Held back until the client answers; the frames after it // wait for the keyframe its answer asks for. diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index db3ade05..9ad0a41b 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -14,8 +14,9 @@ use std::time::{Duration, Instant}; use eternal_wire::v2::control::{ ByeReason, ControlHeader, ControlMessage, FailedReason, HelloAck, HelloStatus, InputEvent, - KeyframeRequest, Nack, ReceiverReport, StreamConfig, FEATURE_STANDBY_STATUS, - FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_NACK, + KeyframeRequest, Nack, ReceiverReport, StreamConfig, FEATURE_MEDIA_SIZE, + FEATURE_STANDBY_STATUS, FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_MEDIA_SIZE, + HOSTCAP_NACK, MIN_MEDIA_DGRAM, }; use tracing::{info, warn}; @@ -216,6 +217,8 @@ pub struct SessionEnd { pub first_video: Option, /// The client's receiver reports counted complete frames. pub frames_received: bool, + /// The client's receiver reports counted any video fragment. + pub frags_received: bool, pub duration: Duration, } @@ -278,6 +281,13 @@ struct ActiveSession { first_video_at: Option, /// A receiver report counted complete frames. frames_received: bool, + /// A receiver report counted any video fragment. + frags_received: bool, + /// Video frames let through to this session ([`Session::may_send_media`]). + frames_sent: u32, + /// The largest media datagram the client asked for in this session + /// ([`KeyframeRequest::max_media_dgram`]). + max_media_dgram: Option, /// A Wi-Fi goodbye while the cable takes over. bye: Option, /// The msg_seq of input packets taken in this session. @@ -540,10 +550,14 @@ impl Session { let Some(session) = self .active .as_mut() - .filter(|session| session.session_id == session_id && !session.answered) + .filter(|session| session.session_id == session_id) else { return true; }; + if session.answered { + session.frames_sent = session.frames_sent.saturating_add(1); + return true; + } if session.withheld || session.unanswered_sent + bytes > UNANSWERED_MEDIA_BUDGET || host_spent + bytes > UNANSWERED_MEDIA_BUDGET @@ -552,10 +566,20 @@ impl Session { return false; } session.unanswered_sent += bytes; + session.frames_sent = session.frames_sent.saturating_add(1); self.unanswered_media = (host_spent + bytes, now); true } + /// The largest media datagram `session_id`'s client asked for, if it + /// asked for one. + pub fn max_media_dgram(&self, session_id: u32) -> Option { + self.active + .as_ref() + .filter(|session| session.session_id == session_id) + .and_then(|session| session.max_media_dgram) + } + /// The host-wide media to clients not yet heard that still counts at /// `now`. fn unanswered_media_at(&self, now: Instant) -> usize { @@ -1094,6 +1118,9 @@ impl Session { msg_seq_out: 0, first_video_at: None, frames_received: false, + frags_received: false, + frames_sent: 0, + max_media_dgram: None, bye: None, input_seqs: SeqWindow::default(), answered, @@ -1163,6 +1190,24 @@ impl Session { } actions.force_idr = session.heard(Heard::Keyframe, now); info!(reason = ?request.reason, "Keyframe request received"); + // Only a client that was told the host takes it asks; anything else + // is ignored, as before. Each smaller size restarts the picture at + // once: the frames sent so far gave the client no picture. + let asked = request.max_media_dgram; + if asked != 0 && session.info.feature_caps & FEATURE_MEDIA_SIZE != 0 { + let size = asked.clamp(MIN_MEDIA_DGRAM, eternal_wire::v2::MAX_DGRAM_SIZE as u16); + if session.max_media_dgram.is_none_or(|current| size < current) { + session.max_media_dgram = Some(size); + session.last_keyframe_grant = Some(now); + info!( + max_dgram = size, + frames_sent = session.frames_sent, + "Media packet size lowered for this session" + ); + actions.force_idr = true; + return actions; + } + } if session.grant_keyframe(now) { info!(reason = ?request.reason, "Keyframe requested by client — forcing IDR"); @@ -1178,6 +1223,7 @@ impl Session { actions.force_idr = session.heard(Heard::Report, now); session.last_report = Some(report); session.frames_received |= report.frames_complete > 0; + session.frags_received |= report.frags_received > 0; actions.report = Some(report); } } @@ -1356,11 +1402,14 @@ impl Session { if send_heartbeat { let peer = session.peer; let session_id = session.session_id; + let frames_sent = (session.info.feature_caps & FEATURE_MEDIA_SIZE != 0) + .then_some(session.frames_sent); let msg_seq = self.next_msg_seq(); let heartbeat = ControlMessage::Heartbeat(eternal_wire::v2::control::Heartbeat { host_time_us: crate::clock::host_now_us(), stream_config: self.negotiated_config(config), notice: config.notice(), + frames_sent, }); actions.replies.push(( peer, @@ -1563,6 +1612,7 @@ impl Session { reason: EndReason::Refused(status), first_video: None, frames_received: false, + frags_received: false, duration: Duration::ZERO, }), ..Actions::default() @@ -1638,6 +1688,13 @@ impl Session { auth_token: [u8; 16], reason: &str, ) -> HelloAck { + // Only the client that asks hears that the host takes smaller media + // datagrams; older iPads get the reply they always did. + let media_size = status == HelloStatus::Ok + && self.active.as_ref().is_some_and(|session| { + session.session_id == session_id + && session.info.feature_caps & FEATURE_MEDIA_SIZE != 0 + }); HelloAck { status, accepted_version: 2, @@ -1649,7 +1706,9 @@ impl Session { stream_config: self.negotiated_config(config), host_name: config.host_name(), auth_token, - host_caps: HOSTCAP_NACK | config.host_caps(), + host_caps: HOSTCAP_NACK + | config.host_caps() + | if media_size { HOSTCAP_MEDIA_SIZE } else { 0 }, reason: reason.to_string(), } } @@ -1805,6 +1864,7 @@ impl ActiveSession { .first_video_at .map(|at| at.saturating_duration_since(start)), frames_received: self.frames_received, + frags_received: self.frags_received, duration: now.saturating_duration_since(start), } } @@ -2418,6 +2478,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 10, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, }); let first = session.handle_control_authed(peer, request.clone(), &TestConfig, now); @@ -2458,6 +2519,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 10, reason, + max_media_dgram: 0, }) }; @@ -2758,6 +2820,164 @@ mod tests { assert!(matches!(message, ControlMessage::Heartbeat(_))); } + fn media_size_hello(nonce: u32, feature_caps: u16) -> ControlMessage { + let ControlMessage::Hello2(mut hello) = hello(nonce, 50000) else { + unreachable!() + }; + hello.feature_caps = feature_caps; + ControlMessage::Hello2(hello) + } + + fn sized_request(max_media_dgram: u16) -> ControlMessage { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 7, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram, + }) + } + + fn heartbeat_of(actions: &Actions) -> eternal_wire::v2::control::Heartbeat { + match eternal_wire::v2::control::parse_control(&actions.replies[0].1) + .unwrap() + .1 + { + ControlMessage::Heartbeat(heartbeat) => heartbeat, + other => panic!("{other:?}"), + } + } + + /// Only an iPad that says it may ask for smaller media datagrams hears + /// that the host takes them, and gets the frame count in heartbeats. + /// Any other iPad gets the reply and heartbeats it always did. + #[test] + fn media_size_is_offered_only_to_an_ipad_that_asks() { + let peer = addr([10, 0, 0, 5], 50000); + let now = Instant::now(); + for (feature_caps, offered) in [(FEATURE_MEDIA_SIZE, true), (0, false)] { + let mut session = Session::new(1234); + let accepted = session.handle_control( + peer, + 0, + media_size_hello(1, feature_caps), + &TestConfig, + now, + ); + let ack = parse_ack(&accepted.replies[0].1); + assert_eq!(ack.host_caps & HOSTCAP_MEDIA_SIZE != 0, offered); + assert_eq!(ack.host_caps & !HOSTCAP_MEDIA_SIZE, HOSTCAP_NACK); + let id = session.session_id().unwrap(); + for _ in 0..3 { + assert!(session.may_send_media(id, 1000, now)); + } + let ticked = session.tick(&TestConfig, true, now + Duration::from_millis(100)); + assert_eq!( + heartbeat_of(&ticked).frames_sent, + offered.then_some(3), + "{feature_caps}" + ); + // The heartbeat an older iPad gets is the plain one. + if !offered { + assert_eq!( + ticked.replies[0].1.len(), + eternal_wire::v2::control::CONTROL_HEADER_SIZE + + 8 + + eternal_wire::v2::control::STREAM_CONFIG_SIZE + ); + } + } + // A refusal never offers it. + let mut busy = Session::new(1234); + busy.handle_control( + peer, + 0, + media_size_hello(1, FEATURE_MEDIA_SIZE), + &TestConfig, + now, + ); + let refused = busy.handle_control( + addr([10, 0, 0, 6], 50000), + 0, + media_size_hello(2, FEATURE_MEDIA_SIZE), + &TestConfig, + now, + ); + let ack = parse_ack(&refused.replies[0].1); + assert_eq!(ack.status, HelloStatus::Busy); + assert_eq!(ack.host_caps & HOSTCAP_MEDIA_SIZE, 0); + } + + /// A smaller size restarts the picture at once, inside the keyframe + /// rate limit, and holds for the rest of the session: never larger, + /// never under the minimum. A new session starts at the PC's size. + #[test] + fn a_smaller_media_size_lowers_only_this_session_and_forces_an_idr() { + let peer = addr([10, 0, 0, 5], 50000); + let now = Instant::now(); + let mut session = Session::new(1234); + session.handle_control( + peer, + 0, + media_size_hello(1, FEATURE_MEDIA_SIZE), + &TestConfig, + now, + ); + let id = session.session_id().unwrap(); + assert_eq!(session.max_media_dgram(id), None); + + let plain = session.handle_control_authed(peer, sized_request(0), &TestConfig, now); + assert!(plain.force_idr); + assert_eq!(session.max_media_dgram(id), None); + + let at = |millis| now + Duration::from_millis(millis); + for (millis, asked, size, idr) in [ + (100, 1200, 1200, true), + (200, 1200, 1200, false), + (300, 1300, 1200, false), + (400, 576, 576, true), + (450, 100, 576, false), + ] { + let actions = + session.handle_control_authed(peer, sized_request(asked), &TestConfig, at(millis)); + assert_eq!(session.max_media_dgram(id), Some(size), "{asked}"); + assert_eq!(actions.force_idr, idr, "{asked}"); + assert!(actions.replies.is_empty() && actions.new_target.is_none()); + } + assert_eq!(session.max_media_dgram(id.wrapping_add(1)), None); + + session.handle_control( + peer, + 0, + media_size_hello(2, FEATURE_MEDIA_SIZE), + &TestConfig, + at(500), + ); + let next = session.session_id().unwrap(); + assert_ne!(next, id); + assert_eq!(session.max_media_dgram(next), None); + } + + /// An iPad that didn't say it may ask has its request read as the + /// keyframe request it always was. + #[test] + fn a_media_size_from_an_ipad_that_did_not_offer_is_ignored() { + let peer = addr([10, 0, 0, 5], 50000); + let now = Instant::now(); + let mut session = Session::new(1234); + session.handle_control(peer, 0, media_size_hello(1, 0), &TestConfig, now); + let id = session.session_id().unwrap(); + let first = session.handle_control_authed(peer, sized_request(576), &TestConfig, now); + assert!(first.force_idr); + let second = session.handle_control_authed( + peer, + sized_request(576), + &TestConfig, + now + Duration::from_millis(100), + ); + assert!(!second.force_idr, "rate limited as any keyframe request"); + assert_eq!(session.max_media_dgram(id), None); + } + struct PairConfig(parking_lot::Mutex); impl ConfigSource for PairConfig { fn stream_config(&self) -> StreamConfig { @@ -3418,6 +3638,7 @@ mod tests { reason: EndReason::Goodbye(ByeReason::UserDisconnect), first_video: Some(Duration::from_millis(40)), frames_received: true, + frags_received: false, duration: Duration::from_secs(2), } ); @@ -3668,6 +3889,7 @@ mod tests { reason: EndReason::Refused(HelloStatus::VersionUnsupported), first_video: None, frames_received: false, + frags_received: false, duration: Duration::ZERO, } ); @@ -3890,6 +4112,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 10, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, }); let ping = ControlMessage::Ping(eternal_wire::v2::control::Ping { t1_us: 1 }); let nack = ControlMessage::Nack(Nack { @@ -4368,6 +4591,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 0, reason: KeyframeReason::Startup, + max_media_dgram: 0, }), ControlMessage::Nack(Nack { stream_epoch: 7, diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index d9bc5f80..48e3caf5 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -707,6 +707,7 @@ fn run_slow_usb(bytes_per_sec: f64, run_for: Duration) -> SlowUsbRun { stream_epoch: epoch, last_complete_seq: seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, }); write_datagram(&mut stream, &encode_control(session_id, msg_seq, &request)) .await @@ -913,6 +914,7 @@ fn lossy_wifi_recovery_measurement() { stream_epoch: c.stream_epoch, last_complete_seq: assembler.latest_completed_seq(), reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } } @@ -1584,6 +1586,123 @@ fn audio_stream_end_to_end() { } } +/// What one fake iPad saw for `window`: the largest media datagram, whether +/// a keyframe came, and the latest heartbeat frame count. +fn media_seen(receiver: &mut FakeReceiver, window: Duration) -> (usize, bool, Option) { + let started = Instant::now(); + let mut buffer = [0; 2048]; + let (mut largest, mut keyframe, mut frames_sent) = (0, false, None); + while started.elapsed() < window { + receiver.maybe_report(0, 0); + let Ok((len, _)) = receiver.socket.recv_from(&mut buffer) else { + continue; + }; + match classify(&buffer[..len]) { + Classified::Media { flags } => { + largest = largest.max(len); + keyframe |= flags & 1 != 0; + } + Classified::Control(_) => { + if let Ok((_, ControlMessage::Heartbeat(heartbeat))) = parse_control(&buffer[..len]) + { + frames_sent = heartbeat.frames_sent; + } + } + _ => {} + } + } + (largest, keyframe, frames_sent) +} + +/// An iPad that said it may ask for smaller media datagrams gets them for +/// the rest of its session, starting with a keyframe, and heartbeats that +/// count the frames sent. An iPad that didn't say so gets neither, and its +/// request is the keyframe request it always was. +#[test] +fn an_ipad_gets_the_smaller_media_datagrams_it_asks_for() { + use eternal_wire::v2::control::{FEATURE_MEDIA_SIZE, HOSTCAP_MEDIA_SIZE}; + let _env = test_env(); + init_test_tracing(); + ffmpeg_next::init().unwrap(); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + for name in ["ETERNAL_DROP", "ETERNAL_REORDER", "ETERNAL_JITTER_MS"] { + std::env::remove_var(name); + } + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".into()); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor_shared = shared.clone(); + let supervisor_sender = supervisor_tx.clone(); + let supervisor = std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + supervisor_shared, + GpuInfo::software_fallback(), + supervisor_sender, + supervisor_rx, + ) + }); + + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let sized = |max_media_dgram| { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 0, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram, + }) + }; + for offered in [true, false] { + let features = if offered { FEATURE_MEDIA_SIZE } else { 0 }; + let mut receiver = FakeReceiver::connect_full(listen_port, CAP_DECODE_H264, features); + assert_eq!(receiver.host_caps & HOSTCAP_MEDIA_SIZE != 0, offered); + let (largest, _, frames_sent) = media_seen(&mut receiver, Duration::from_secs(2)); + assert!( + largest > 1000, + "{offered}: the keyframe goes in full-size datagrams ({largest})" + ); + assert_eq!( + frames_sent.is_some_and(|frames| frames > 0), + offered, + "{frames_sent:?}" + ); + if !offered { + assert_eq!(frames_sent, None); + } + + receiver.send(&sized(576)); + // Frames already queued may still be large; what follows isn't. + let (_, queued_keyframe, _) = media_seen(&mut receiver, Duration::from_millis(500)); + let (largest, keyframe, _) = media_seen(&mut receiver, Duration::from_secs(2)); + if offered { + assert!( + queued_keyframe || keyframe, + "the smaller size starts with a keyframe" + ); + assert!(largest > 0 && largest <= 576, "{largest}"); + } else { + assert!(largest > 576, "{largest}"); + } + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + std::thread::sleep(Duration::from_millis(200)); + } + // The size was this session's: the next one starts at the PC's. + let mut receiver = + FakeReceiver::connect_full(listen_port, CAP_DECODE_H264, FEATURE_MEDIA_SIZE); + let (largest, _, _) = media_seen(&mut receiver, Duration::from_secs(2)); + assert!(largest > 1000, "{largest}"); + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + })); + let _ = supervisor_tx.send(SupervisorCommand::Shutdown); + supervisor.join().unwrap(); + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } +} + /// Native first-second audio measurement across repeated connects with the /// PC's real WASAPI loopback (inaudible: it only reads the mix). #[cfg(windows)] @@ -1789,6 +1908,7 @@ fn synthetic_stream_end_to_end_v2() { stream_epoch: header.stream_epoch, last_complete_seq: header.frame_seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } if let Some(at) = keyframe_requested_at { @@ -2210,6 +2330,7 @@ fn lossy_stream_recovers_and_adapts() { stream_epoch: epoch, last_complete_seq: seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } for frame in assembler.take_ready() { @@ -2379,6 +2500,7 @@ fn lossy_legacy_stream_uses_keyframe_recovery() { stream_epoch: current_epoch, last_complete_seq: highest_seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index a8381af5..e94613be 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -99,7 +99,8 @@ final class ConnectionManager: ObservableObject { /// understands why a PC isn't streaming. nonisolated static func featureCaps(wantsInput: Bool, wantsAudio: Bool, isUSB: Bool) -> UInt16 { (wantsInput ? Hello2.featureWantsInput : 0) | (wantsAudio ? Hello2.featureWantsAudio : 0) - | (isUSB ? 0 : Hello2.featureSupportsNack) | Hello2.featureChecksUpdates | Hello2.featureStandbyStatus + | (isUSB ? 0 : Hello2.featureSupportsNack | Hello2.featureMediaSize) | Hello2.featureChecksUpdates + | Hello2.featureStandbyStatus } private var awaitingPairing = false /// The PC answered that it isn't streaming; the attempt waits for it. @@ -239,6 +240,9 @@ final class ConnectionManager: ObservableObject { var reconnectGraceSeconds = 2.0 var reconnectBackoffSeconds: (Int) -> Double = { attempt in Double(min(1 << (attempt - 1), 15)) } var appleDevicesNoteSeconds = AppleDevicesSighting.lifetime + /// How long video the PC says it sent may go missing before the + /// next no-video step (see `NoVideoFallback`). + var noVideoStepSeconds = 1.5 } var timing: Timing /// The Wi-Fi link to a host's port. Tests pass a fake one, or replace it @@ -252,6 +256,9 @@ final class ConnectionManager: ObservableObject { var homeAddress: (String) -> Bool = MovedPC.isHomeAddress /// How this iPad reaches an address. Tests replace it. var localRoute: (String, UInt16) async -> MovedPC.Route? = { await MovedPC.route(to: $0, port: $1) } + /// Where the iPad sends to reach a PC address it was given. Tests on + /// loopback point the addresses a fake signs at other fakes. + var dialAddress: (String, UInt16) -> (host: String, port: UInt16) = { ($0, $1) } /// "host:port" of each address picked from Bonjour's list in this /// launch. Saved entries keep the mark once they show a picture. private var foundAddresses = Set() @@ -262,10 +269,21 @@ final class ConnectionManager: ObservableObject { let name: String /// Its pairing token, which the PC proved it holds. let token: Data + /// The PC's video didn't arrive at `from`; it still answers there. + var noVideo = false } /// The move the next attempt makes, and the one the current attempt made. private var pendingMove: Move? private var attemptMove: Move? + /// The PC's video didn't arrive at the address asked for before, and did + /// at another: the next attempt goes there first, and back to the address + /// asked for (kept here) if the PC doesn't answer there. + private var pendingRedirect: (host: String, port: UInt16)? + private var attemptRedirect: (host: String, port: UInt16)? + /// The current session's no-video steps, with a PC that takes them, and + /// the step whose result isn't known yet, as its log line names it. + private var noVideoFallback: NoVideoFallback? + private var noVideoStep: String? init(timing: Timing = Timing(), udpReceiver: @escaping (UInt16) -> any MediaLink = { UDPReceiver(port: $0) }, pairingStore: PairingStore? = nil) { @@ -360,10 +378,26 @@ final class ConnectionManager: ObservableObject { if let token { do { try pairingStore.save(token: token, hostName: "", address: PairingStore.address(host: host, port: port)) } catch { record(.warning, "pairing", "Could not save the scanned pairing on this iPad.") } + } else if let better = betterAddress(host: host, port: port) { + pendingRedirect = (host.trimmingCharacters(in: .whitespacesAndNewlines), port) + connect(target: .udp(host: better.host, port: better.port)) + pendingRedirect = nil + return } connect(target: .udp(host: host, port: port)) } + /// Where the same PC showed video after none arrived at `host:port`: an + /// address this iPad holds the same pairing for. + private func betterAddress(host: String, port: UInt16) -> NoVideoPaths.Address? { + let host = host.trimmingCharacters(in: .whitespacesAndNewlines) + guard let better = NoVideoPaths.better(than: host, port: port), + let token = (try? pairingStore.token(address: PairingStore.address(host: host, port: port))) ?? nil, + let there = (try? pairingStore.token(address: PairingStore.address(host: better.host, port: better.port))) ?? nil, + token == there else { return nil } + return better + } + func submitPairing() { guard awaitingPairing, let code = pairingPrompt?.submit() else { return } // The code goes in a new HELLO2. The refused one ends here, as on the @@ -380,6 +414,7 @@ final class ConnectionManager: ObservableObject { func forgetPairedHosts() { do { try pairingStore.forgetAll() } catch { connectionError = "Could not remove saved pairings from this iPad." } + NoVideoPaths.forgetAll() } /// A connection the user or a cable starts. It replaces any automatic @@ -433,6 +468,10 @@ final class ConnectionManager: ObservableObject { explicitPairingToken = nil attemptMove = isUSB ? nil : pendingMove pendingMove = nil + attemptRedirect = isUSB ? nil : pendingRedirect + pendingRedirect = nil + noVideoFallback = nil + noVideoStep = nil let connectionID = UUID() currentAttempt.withLock { $0 = connectionID } transportMode = isUSB ? "USB" : "WiFi" @@ -465,7 +504,12 @@ final class ConnectionManager: ObservableObject { audioStats = AudioStats() debugState = ConnectionDebugState(host: normalizedHost, port: port) record(.info, "connection", "Starting connection attempt=\(connectionID) transport=\(transportMode) to \(normalizedHost):\(port) drawing=\(drawing.enabled) input=\(wantsInput) audio=\(wantsAudio)") - if attemptMove != nil { record(.info, "connection", "Trying the saved PC at the new address Bonjour gave") } + if attemptMove != nil { + record(.info, "connection", attemptMove?.noVideo == true + ? "Trying the PC at another of its addresses: its video didn't arrive at the first" + : "Trying the saved PC at the new address Bonjour gave") + } + if attemptRedirect != nil { record(.info, "connection", "no_video_fallback step=remembered result=tried") } let decoder = VideoDecoder() let metrics = ReceiverMetrics() @@ -515,6 +559,8 @@ final class ConnectionManager: ObservableObject { if self.debugState.decodedFrames == 1 { self.firstFrameAt = ProcessInfo.processInfo.systemUptime self.record(.info, "decode", "First frame decoded successfully" + self.attemptTimings) + self.endNoVideoStep("video") + self.noVideoFallback = nil self.timeoutTask?.cancel() self.noPictureStreak = NoPictureStreak() self.state = .connected @@ -678,6 +724,8 @@ final class ConnectionManager: ObservableObject { self.lastTarget = (normalizedHost, port) } self.sessionAcceptedAt = acceptedAt + self.noVideoFallback = !isUSB && info.hostCaps & HelloAck.hostCapMediaSize != 0 + ? NoVideoFallback(wait: self.timing.noVideoStepSeconds) : nil self.acceptedAddresses.insert("\(normalizedHost):\(port)") self.noPictureStreak.answered(by: info.hostName, pairingToken: info.authToken, at: isUSB ? nil : "\(normalizedHost):\(port)") @@ -786,7 +834,14 @@ final class ConnectionManager: ObservableObject { self.disconnect(.legacyPC) } else { self.record(.warning, "ctrl", "HELLO2 handshake got no reply yet") - if !isUSB { self.lookForMovedPC(connectionID, host: normalizedHost, port: port) } + if let from = self.attemptRedirect { + // Not on this network now: back to the address asked for. + self.record(.info, "connection", "no_video_fallback step=remembered result=no_reply") + self.disconnect(.pcMoved) + self.startAttempt(.udp(host: from.host, port: from.port)) + } else if !isUSB { + self.lookForMovedPC(connectionID, host: normalizedHost, port: port) + } } } } @@ -830,6 +885,9 @@ final class ConnectionManager: ObservableObject { Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } self.hostStreamConfig = heartbeat.streamConfig + if let frames = heartbeat.framesSent { + self.noVideoFallback?.heard(framesSent: frames, at: ProcessInfo.processInfo.systemUptime) + } let locked = heartbeat.notice == Heartbeat.noticePCLocked if locked != self.pcLocked { self.pcLocked = locked @@ -992,6 +1050,7 @@ final class ConnectionManager: ObservableObject { } } + self.advanceNoVideoFallback() // Runs on EVERY tick, including silent ones: a dead host sends // neither media nor heartbeats, so skipping this when nothing // arrived is exactly when the liveness watchdog is needed. @@ -1200,13 +1259,91 @@ final class ConnectionManager: ObservableObject { return false } RecentConnectionStore.shared.moved(from: move.from, to: new, name: hostInfo?.hostName ?? move.name, - replacesOld: homeAddress(move.from.host)) - if homeAddress(move.from.host) { + replacesOld: move.noVideo || homeAddress(move.from.host)) + if move.noVideo { + // The PC still answers at the old address: its pairing stays, + // and connecting there goes here first. + NoVideoPaths.remember(from: .init(host: move.from.host, port: move.from.port), + to: .init(host: new.host, port: new.port)) + } else if homeAddress(move.from.host) { try? pairingStore.remove(address: PairingStore.address(host: move.from.host, port: move.from.port)) } return true } + /// Takes the next no-video step once it is due: the PC said it sent + /// video and no picture came of it (see `NoVideoFallback`). A path that + /// drops large datagrams often lets a frame's small last fragment by. + private func advanceNoVideoFallback() { + guard noVideoFallback != nil, state == .connecting, hostInfo != nil, !awaitingPairing, !waitingForStream, + let step = noVideoFallback?.next(at: ProcessInfo.processInfo.systemUptime) else { return } + endNoVideoStep("no_video") + switch step { + case .smallerPackets(let size): + // The PC is sending and the iPad is working on it: one fresh + // window, as when data starts to arrive. + if !didExtendTimeout { + didExtendTimeout = true + armConnectTimeout(seconds: timing.connectTimeoutSeconds) + } + noVideoStep = "step=smaller_packets size=\(size)" + controlChannel?.sendKeyframeRequest(streamEpoch: 0, lastCompleteSeq: 0, reason: .startup, maxMediaDgram: size) + case .otherAddress: + tryOtherAddress() + } + } + + /// Logs the pending no-video step's `result`. + private func endNoVideoStep(_ result: String) { + guard let step = noVideoStep else { return } + noVideoStep = nil + record(.info, "connection", "no_video_fallback \(step) result=\(result)") + } + + /// No picture came of the PC's video at this address, even in small + /// datagrams. Asked here, the PC signs its home network addresses with + /// this iPad's pairing; the first other one this iPad reaches over + /// Wi-Fi or Ethernet, where the PC shows again that it holds the pairing + /// and receives there, gets a new attempt. A forwarded or spoofed + /// address can't make either answer. + private func tryOtherAddress() { + let host = debugState.host, port = debugState.port + let done = { [weak self] (result: String) -> Void in + self?.record(.info, "connection", "no_video_fallback step=other_address result=\(result)") + } + guard attemptMove == nil else { return done("moved_before") } + guard let connectionID = currentAttempt.withLock({ $0 }), let name = hostInfo?.hostName, + let token = (try? pairingStore.token(address: PairingStore.address(host: host, port: port))) ?? nil + else { return done("unpaired") } + let current = { [weak self] in + guard let self else { return false } + return self.currentAttempt.withLock { $0 == connectionID } && self.state == .connecting + } + Task { @MainActor [weak self] in + guard let self else { return } + let identity = await MovedPC.identify(host: host, port: port, token: token, link: self.makeUDPLink(port)) + guard current() else { return } + guard let identity else { return done("no_answer") } + for other in MovedPC.otherAddresses(identity, asked: host, homeAddress: self.homeAddress) { + let target = self.dialAddress(other, identity.port) + let route = await self.localRoute(target.host, target.port) + guard current() else { return } + guard let route, MovedPC.isDirect(route) else { continue } + let same = await MovedPC.confirm(host: target.host, port: target.port, token: token, + link: self.makeUDPLink(target.port)) + guard current() else { return } + guard same else { continue } + done("moved") + self.disconnect(.noVideoPath) + self.pendingMove = Move(from: (host, port), name: name, token: token, noVideo: true) + self.explicitPairingToken = token + self.startAttempt(.udp(host: target.host, port: target.port)) + return + } + done("none") + } + } + /// Retires the automatic reconnect cycle, if one is running: its task /// stops at its next check and can no longer write state. private func endReconnectCycle() { @@ -1453,12 +1590,14 @@ final class ConnectionManager: ObservableObject { /// `.userDisconnect`. func disconnect(_ end: AttemptEnd = .userDisconnect) { if state != .disconnected { + endNoVideoStep("no_video") recordStreamSummary(reason: "disconnect=\(end.name)") record(.info, "connection", "Disconnect state=\(state) reason=\(end.name)") recordSummary(end) } awaitingPairing = false waitingForStream = false + noVideoFallback = nil pairingPrompt = nil currentAttempt.withLock { $0 = nil } timeoutTask?.cancel() diff --git a/ios/EternalMonitor/App/MovedPC.swift b/ios/EternalMonitor/App/MovedPC.swift index 6e53aa92..6c181fb1 100644 --- a/ios/EternalMonitor/App/MovedPC.swift +++ b/ios/EternalMonitor/App/MovedPC.swift @@ -85,12 +85,24 @@ enum MovedPC { guard text.split(separator: ".").count == 4, let bytes = IPv4Address(text)?.rawValue, bytes.count == 4 else { return nil } return bytes.reduce(0) { $0 << 8 | UInt32($1) } } - guard route.direct, !tunnels.contains(where: route.interface.hasPrefix), - let address = ipv4(route.address), let mask = ipv4(route.netmask), let candidate = ipv4(host), + guard isDirect(route), let address = ipv4(route.address), let mask = ipv4(route.netmask), let candidate = ipv4(host), mask != 0, mask != .max else { return false } return address & mask == candidate & mask } + /// Whether `route` is a direct Wi-Fi or Ethernet path, not a tunnel. + static func isDirect(_ route: Route) -> Bool { + route.direct && !tunnels.contains(where: route.interface.hasPrefix) + } + + /// The other home network addresses `identity` signed, in the PC's + /// order (those with a default gateway first), without `asked`. + static func otherAddresses(_ identity: WireIdentity, asked: String, + homeAddress: (String) -> Bool = isHomeAddress) -> [String] { + identity.addresses.map { $0.map(String.init).joined(separator: ".") } + .filter { $0 != asked && homeAddress($0) } + } + /// How this iPad reaches `host:port` now, from a UDP path to it that /// sends nothing; nil if no path is ready within 2 s. static func route(to host: String, port: UInt16) async -> Route? { @@ -161,6 +173,13 @@ enum MovedPC { /// wrong proof. The token itself is never sent. static func confirm(host: String, port: UInt16, token: Data, link: any MediaLink, attempts: Int = 8, interval: Double = 0.25) async -> Bool { + await identify(host: host, port: port, token: token, link: link, attempts: attempts, interval: interval) != nil + } + + /// As `confirm`, with the PC's signed answer: the port it receives the + /// stream on and its home network addresses, one of them `host`. + static func identify(host: String, port: UInt16, token: Data, link: any MediaLink, attempts: Int = 8, + interval: Double = 0.25) async -> WireIdentity? { var generator = SystemRandomNumberGenerator() let nonce = Data((0..<16).map { _ in UInt8.random(in: .min ... .max, using: &generator) }) let request = Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identify(WireIdentify(nonce: nonce))) @@ -169,7 +188,7 @@ enum MovedPC { let box = LinkBox(link) return await withCheckedContinuation { continuation in let done = OSAllocatedUnfairLock(initialState: false) - let finish: @Sendable (Bool) -> Void = { result in + let finish: @Sendable (WireIdentity?) -> Void = { result in guard done.withLock({ finished in defer { finished = true }; return !finished }) else { return } box.link.controlQueue.async { box.link.onControlDatagram = nil @@ -184,13 +203,13 @@ enum MovedPC { identity.version == WireIdentify.version, identity.nonce == nonce else { return } let signed = WireIdentity.message(nonce: nonce, port: identity.port, addresses: identity.addresses) guard HMAC.isValidAuthenticationCode(identity.proof, authenticating: signed, using: key) else { - return finish(false) + return finish(nil) } // The PC signs where it receives the stream. An answer for // another address isn't the saved PC at this one. - if identity.port == port, let asked, identity.addresses.contains(asked) { finish(true) } + if identity.port == port, let asked, identity.addresses.contains(asked) { finish(identity) } } - link.onError = { _ in finish(false) } + link.onError = { _ in finish(nil) } link.onListenerReady = { _ in for attempt in 0.. framesBeforeStep, sendingSince == nil { sendingSince = now } + } + + /// The step to take at `now`, while there is no picture; nil while the + /// PC sends nothing, or within `wait` of its sending. + mutating func next(at now: TimeInterval) -> Step? { + guard !finished, let since = sendingSince, now - since >= wait else { return nil } + sendingSince = nil + framesBeforeStep = framesSent + guard sizesAsked < Self.sizes.count else { + finished = true + return .otherAddress + } + sizesAsked += 1 + return .smallerPackets(Self.sizes[sizesAsked - 1]) + } +} + +/// Where a PC's video didn't arrive, and the address of the same PC where it +/// did, so the next connect to the first goes to the second. Kept in +/// UserDefaults as "host:port" pairs. +enum NoVideoPaths { + private static let key = "noVideoPaths" + private static let kept = 16 + + struct Address: Codable, Equatable { + var host: String + var port: UInt16 + } + + private struct Entry: Codable { + var from: Address + var to: Address + } + + /// The address where the PC at `host:port` showed video instead. + static func better(than host: String, port: UInt16) -> Address? { + load().last { $0.from == Address(host: host, port: port) }?.to + } + + static func remember(from: Address, to: Address) { + var entries = load().filter { $0.from != from && $0.from != to } + entries.append(Entry(from: from, to: to)) + save(Array(entries.suffix(kept))) + } + + static func forgetAll() { + UserDefaults.standard.removeObject(forKey: key) + } + + private static func load() -> [Entry] { + guard let data = UserDefaults.standard.data(forKey: key) else { return [] } + return (try? JSONDecoder().decode([Entry].self, from: data)) ?? [] + } + + private static func save(_ entries: [Entry]) { + guard let data = try? JSONEncoder().encode(entries) else { return } + UserDefaults.standard.set(data, forKey: key) + } +} diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index 4ae3449b..702d1e44 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -30,9 +30,14 @@ enum AttemptEnd: Equatable { /// iPadOS found no usable network for the path: Wi-Fi is off or /// joined to none (`ConnectionProblem.wifiOff`). case noNetwork - /// No answer at the saved address, and Bonjour found the same PC at - /// another: a new attempt goes there. + /// No answer at the address tried, and the same PC is known at another: + /// Bonjour found it there, or its video arrived there before. A new + /// attempt goes there. case pcMoved + /// The PC sent video that gave no picture at this address, even in + /// small datagrams, and showed it is the same PC at another: a new + /// attempt goes there. + case noVideoPath /// The PC found at a saved PC's new address turned out to be another PC. case otherPC case receiverFailed @@ -102,7 +107,7 @@ struct SessionSummary: Equatable { case .appBackground, .superseded, .pcGoodbye: return "app_ended" case .connectTimeout: return sessionId == nil ? "no_reply" : "no_picture" case .pcMoved: return "no_reply" - case .closedBeforePicture: return "no_picture" + case .closedBeforePicture, .noVideoPath: return "no_picture" default: return "error" } } @@ -111,6 +116,7 @@ struct SessionSummary: Equatable { switch end { case .connectTimeout where sessionId == nil, .pcMoved: return "no_reply" case .connectTimeout: return datagramsReceived == 0 ? "no_video" : "no_frame" + case .noVideoPath: return "no_video" case .pcStoppedResponding: return "signal_lost" case .cableUnplugged, .usbLinkFailed, .closedBeforePicture: return "usb_lost" case .noNetwork: return "no_network" @@ -123,7 +129,7 @@ struct SessionSummary: Equatable { switch end { case .userDisconnect, .usbTurnedOff: return "user" case .appBackground: return "background" - case .superseded, .retriedWithCode, .pcMoved: return "superseded" + case .superseded, .retriedWithCode, .pcMoved, .noVideoPath: return "superseded" case .pcGoodbye: return "goodbye" case .connectTimeout: return "timeout" case .pcStoppedResponding: return "liveness" diff --git a/ios/EternalMonitor/Network/ControlChannel.swift b/ios/EternalMonitor/Network/ControlChannel.swift index 5f540e64..18fbfd48 100644 --- a/ios/EternalMonitor/Network/ControlChannel.swift +++ b/ios/EternalMonitor/Network/ControlChannel.swift @@ -324,7 +324,10 @@ final class ControlChannel { timer.resume() } - func sendKeyframeRequest(streamEpoch: UInt32, lastCompleteSeq: UInt32, reason: KeyframeReason) { + /// `maxMediaDgram` asks the PC for media datagrams no larger, for the + /// rest of the session; only a PC with `HelloAck.hostCapMediaSize` gets it. + func sendKeyframeRequest(streamEpoch: UInt32, lastCompleteSeq: UInt32, reason: KeyframeReason, + maxMediaDgram: UInt16 = 0) { queue.async { [self] in guard sessionId != 0 else { return } let now = nowUs() @@ -332,12 +335,14 @@ final class ControlChannel { // dropped frames' own requests then keep the spacing busy, and // would always hide it from the PC's log. if reason != .startup, let last = lastKeyframeRequestUs, now &- last < Self.keyframeRequestSpacingUs { return } + let maxMediaDgram = hostCaps & HelloAck.hostCapMediaSize != 0 ? maxMediaDgram : 0 lastKeyframeRequestUs = now - onDiagnostic?("Requesting keyframe (\(reason))") + onDiagnostic?("Requesting keyframe (\(reason))" + (maxMediaDgram == 0 ? "" : " with media datagrams up to \(maxMediaDgram) bytes")) sendMessage(.keyframeRequest(KeyframeRequest( streamEpoch: streamEpoch, lastCompleteSeq: lastCompleteSeq, - reason: reason + reason: reason, + maxMediaDgram: maxMediaDgram ))) } } diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index 95342fcb..295fd1b3 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -264,6 +264,10 @@ struct Hello2: Equatable { static let standbyLength = 164 /// This app checks for its own updates, so the PC need not tell the user. 0.3.1 and older never set it. static let featureChecksUpdates: UInt16 = 1 << 4 + /// This iPad may ask for smaller media datagrams in a keyframe request, + /// and wants heartbeats to count the frames the PC sent. Bit 5 is taken + /// by another change in flight. + static let featureMediaSize: UInt16 = 1 << 6 static let maxNameLength = 64 var protoMin: UInt8 = 2 @@ -313,6 +317,10 @@ struct HelloAck: Equatable { /// Every 0.3.4 and later PC sets it in replies that accept a session; /// see `OldPC`. static let hostCapContiguousFrameSeq: UInt16 = 1 << 5 + /// The PC takes `KeyframeRequest.maxMediaDgram` for the rest of the + /// session, and its heartbeats carry `framesSent`. Only an iPad that + /// sets `Hello2.featureMediaSize` hears it. + static let hostCapMediaSize: UInt16 = 1 << 7 var status: HelloStatus var acceptedVersion: UInt8 var clientNonce: UInt32 @@ -340,6 +348,10 @@ struct Heartbeat: Equatable { static let noticeNone: UInt8 = 0 /// The PC is locked or shows a permission prompt, so it can't capture. static let noticePCLocked: UInt8 = 1 + /// The video frames the PC sent this session, from a PC that offered + /// `HelloAck.hostCapMediaSize`; nil from others. Appended after + /// `notice`, which is then always written. + var framesSent: UInt32? } enum ByeReason: UInt8 { @@ -361,6 +373,11 @@ struct KeyframeRequest: Equatable { var streamEpoch: UInt32 var lastCompleteSeq: UInt32 var reason: KeyframeReason + /// The largest media datagram to send from now on in this session; 0 + /// asks nothing. Appended: only for a PC with `HelloAck.hostCapMediaSize`. + var maxMediaDgram: UInt16 = 0 + /// The smallest a request may ask for, as the PC's own setting. + static let minMediaDgram: UInt16 = 576 } struct Nack: Equatable { @@ -549,13 +566,15 @@ extension Wire { case .heartbeat(let hb): body.appendLE(hb.hostTimeUs) hb.streamConfig.encode(into: &body) - if hb.notice != Heartbeat.noticeNone { body.append(hb.notice) } + if hb.notice != Heartbeat.noticeNone || hb.framesSent != nil { body.append(hb.notice) } + if let frames = hb.framesSent { body.appendLE(frames) } case .bye(let reason): body.append(reason.rawValue) case .keyframeRequest(let k): body.appendLE(k.streamEpoch) body.appendLE(k.lastCompleteSeq) body.append(k.reason.rawValue) + if k.maxMediaDgram != 0 { body.appendLE(k.maxMediaDgram) } case .nack(let n): precondition(n.isValid) body.appendLE(n.streamEpoch) @@ -671,7 +690,8 @@ extension Wire { guard let hostTimeUs = r.readU64(), let cfg = StreamConfig.decode(&r) else { return nil } message = .heartbeat(Heartbeat(hostTimeUs: hostTimeUs, streamConfig: cfg, - notice: r.readU8() ?? Heartbeat.noticeNone)) + notice: r.readU8() ?? Heartbeat.noticeNone, + framesSent: r.remaining >= 4 ? r.readU32() : nil)) case .bye: guard let raw = r.readU8(), let reason = ByeReason(rawValue: raw) else { return nil } message = .bye(reason) @@ -680,8 +700,8 @@ extension Wire { let last: UInt32 = r.readU32(), let raw = r.readU8(), let reason = KeyframeReason(rawValue: raw) else { return nil } - message = .keyframeRequest( - KeyframeRequest(streamEpoch: epoch, lastCompleteSeq: last, reason: reason)) + message = .keyframeRequest(KeyframeRequest(streamEpoch: epoch, lastCompleteSeq: last, reason: reason, + maxMediaDgram: r.remaining >= 2 ? r.readU16() ?? 0 : 0)) case .receiverReport: message = parseReceiverReport(&r) case .nack: diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index fdb71a7a..8bc148e9 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -285,6 +285,78 @@ final class FakeUDPHost: @unchecked Sendable { return requests } + /// What `servePath` saw the iPad do. + struct PathRun { + /// The media datagram sizes the iPad asked for, in turn. + var sizesAsked: [UInt16] = [] + /// Keyframe requests that asked for no size. + var plainKeyframeRequests = 0 + var identifyAnswers = 0 + } + + /// Plays a PC whose media reaches the iPad only in datagrams of at most + /// `limit` bytes (0: none at all), as over a path that drops larger ones. + /// Accepts the next HELLO2 with `hostCaps`, then until `stop` says so, + /// about every 100 ms, sends a heartbeat (counting the frames sent when + /// `countFrames`) and `frame` as the next keyframe, cut at the size the + /// iPad asked for. Answers identify requests as the PC holding `token` + /// receiving on `addresses`, to whichever socket asks. + func servePath(limit: Int, frame: Data, hostCaps: UInt16, countFrames: Bool = true, token: Data = Data(17...32), + addresses: [Data] = [Data([127, 0, 0, 1])], until stop: () -> Bool) -> PathRun { + var run = PathRun() + var size = Wire.maxDatagramSize + var framesSent: UInt32 = 0 + var lastSent = Date.distantPast + let deadline = Date().addingTimeInterval(60) + while !stop(), Date() < deadline { + if let (data, from, length) = receive() { + switch Wire.parseControl(data)?.message { + case .hello2(let hello)? where client == nil: + client = (from, length) + lastNonce = hello.clientNonce + send(Self.ack(nonce: hello.clientNonce, authToken: token, hostCaps: hostCaps)) + case .keyframeRequest(let request)?: + if request.maxMediaDgram == 0 { + run.plainKeyframeRequests += 1 + } else { + run.sizesAsked.append(request.maxMediaDgram) + size = min(size, Int(request.maxMediaDgram)) + } + case .identify(let request)?: + run.identifyAnswers += 1 + let answer = Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identity(WireIdentity( + nonce: request.nonce, port: port, addresses: addresses, + proof: MovedPC.proof(token: token, nonce: request.nonce, port: port, addresses: addresses)))) + var to = from + _ = answer.withUnsafeBytes { bytes in + withUnsafePointer(to: &to) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { + sendto(fd, bytes.baseAddress, bytes.count, 0, $0, length) + } + } + } + default: + break + } + } + guard client != nil, Date().timeIntervalSince(lastSent) >= 0.1 else { continue } + lastSent = Date() + framesSent += 1 + let payload = size - MediaHeader.size + let count = (frame.count + payload - 1) / payload + for index in 0.. [PairingRecord] { values.map { PairingRecord(account: $0.key, token: $0.value) } } + func save(account: String, token: Data) throws { values[account] = token } + func remove(account: String?) throws { + if let account { values.removeValue(forKey: account) } else { values.removeAll() } + } +} + +/// A PC that accepts the session and sends video, over a path that drops +/// large datagrams or all of them: the iPad asks for smaller ones, then +/// tries the PC's other address. +@MainActor +final class NoVideoFallbackTests: XCTestCase { + nonisolated private static let token = Data(17...32) + /// The other address the first fake signs; the manager dials it at the + /// second fake. + nonisolated private static let otherAddress = Data([192, 0, 2, 7]) + nonisolated private static let bothAddresses = [Data([127, 0, 0, 1]), otherAddress] + + override func setUp() async throws { + let store = RecentConnectionStore.shared + let saved = store.connections + let defaults = ["recentConnections", "lastHost", "lastPort", "noVideoPaths"] + .map { ($0, UserDefaults.standard.object(forKey: $0)) } + addTeardownBlock { @MainActor in + store.connections = saved + for (key, value) in defaults { UserDefaults.standard.set(value, forKey: key) } + } + UserDefaults.standard.removeObject(forKey: "noVideoPaths") + store.connections = [] + } + + private func until(_ description: String, seconds: Double = 60, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + /// A manager whose steps come 0.3 s apart, paired with "Test PC" at + /// `pc`, reaching every address directly, and dialling the other + /// address the PC signs at `other`. + private func manager(pairedAt pc: FakeUDPHost, other: FakeUDPHost? = nil, + connectTimeout: Double = 60) throws -> (ConnectionManager, PairingStore) { + let pairings = PairingStore(storage: MemoryPairings()) + try pairings.save(token: Self.token, hostName: "Test PC", address: PairingStore.address(host: "127.0.0.1", port: pc.port)) + var timing = ConnectionManager.Timing(connectTimeoutSeconds: connectTimeout, reconnectGraceSeconds: 0.1, + reconnectBackoffSeconds: { _ in 0.05 }) + timing.noVideoStepSeconds = 0.3 + let manager = ConnectionManager(timing: timing, pairingStore: pairings) + manager.homeAddress = { _ in true } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "127.0.0.1", netmask: "255.0.0.0") } + if let other { + manager.dialAddress = { host, port in host == "192.0.2.7" ? ("127.0.0.1", other.port) : (host, port) } + } + return (manager, pairings) + } + + /// Every line the manager logs, across attempts (each clears the list). + private func lines(_ manager: ConnectionManager) -> (() -> [String], AnyCancellable) { + var lines: [String] = [] + let sink = manager.$diagnostics.sink { if let line = $0.last?.message { lines.append(line) } } + return ({ lines }, sink) + } + + // MARK: - The steps + + /// Each step waits for frames the PC says it sent after the last one, + /// and none comes while the PC sends nothing. + func testStepsWaitForVideoThePCSaysItSent() { + var steps = NoVideoFallback(wait: 1.5) + XCTAssertNil(steps.next(at: 100)) + steps.heard(framesSent: 0, at: 0) + XCTAssertNil(steps.next(at: 100), "a PC that sent nothing yet") + steps.heard(framesSent: 3, at: 100) + XCTAssertNil(steps.next(at: 101)) + XCTAssertEqual(steps.next(at: 101.5), .smallerPackets(1200)) + XCTAssertNil(steps.next(at: 110)) + steps.heard(framesSent: 3, at: 110) + XCTAssertNil(steps.next(at: 120), "nothing sent since the step") + steps.heard(framesSent: 4, at: 120) + steps.heard(framesSent: 9, at: 121) + XCTAssertNil(steps.next(at: 121.4)) + XCTAssertEqual(steps.next(at: 121.5), .smallerPackets(576)) + steps.heard(framesSent: 12, at: 122) + XCTAssertEqual(steps.next(at: 123.5), .otherAddress) + steps.heard(framesSent: 20, at: 124) + XCTAssertNil(steps.next(at: 200), "then no more") + } + + // MARK: - Connecting + + /// Datagrams over 1,000 bytes never arrive; only each frame's short last + /// fragment does. The iPad asks for 1,200 bytes, then 576, and streams. + func testAPathThatDropsLargeDatagramsStreamsWithSmallerOnes() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let streaming = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: 1000, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("streaming") { manager.state == .connected } + streaming.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, [1200, 576]) + XCTAssertEqual(run.identifyAnswers, 0, "smaller datagrams were enough") + let steps = logged().filter { $0.hasPrefix("no_video_fallback ") } + XCTAssertEqual(steps, ["no_video_fallback step=smaller_packets size=1200 result=no_video", + "no_video_fallback step=smaller_packets size=576 result=video"]) + } + + /// The first address carries no video at all. Asked there, the PC signs + /// another of its addresses; at that one it shows again that it holds + /// the pairing before the token goes there, and the iPad streams from it + /// with no tap. Connecting to the first address later goes there first. + func testAnAddressThatDropsAllVideoMovesToTheVerifiedOtherAddress() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, pairings) = try manager(pairedAt: first, other: other) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let moved = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: first.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + first.servePath(limit: 0, frame: frame, hostCaps: HelloAck.hostCapMediaSize, + addresses: Self.bothAddresses) { moved.withLock { $0 } } + } + let accepted = await Task.detached { + other.answerIdentify(token: Self.token) && other.acceptSession(authToken: Self.token) + }.value + XCTAssertTrue(accepted, "the iPad never checked and tried the other address") + moved.withLock { $0 = true } + let run = await served.value + try await until("handshake there") { manager.hostInfo != nil && manager.debugState.port == other.port } + other.sendKeyframe(frame) + try await until("streaming") { manager.state == .connected } + + XCTAssertEqual(run.sizesAsked, [1200, 576]) + XCTAssertGreaterThan(run.identifyAnswers, 0) + XCTAssertFalse(other.helloBeforeIdentify, "nothing with the token before the PC showed it is the same there") + XCTAssertEqual(other.lastAuthToken, Self.token) + let all = logged() + let steps = all.filter { $0.hasPrefix("no_video_fallback ") } + XCTAssertEqual(steps, ["no_video_fallback step=smaller_packets size=1200 result=no_video", + "no_video_fallback step=smaller_packets size=576 result=no_video", + "no_video_fallback step=other_address result=moved"]) + XCTAssertTrue(all.contains("Disconnect state=connecting reason=noVideoPath"), all.joined(separator: "\n")) + let summary = all.first { $0.hasPrefix("session_summary ") } ?? "" + XCTAssertTrue(summary.contains(" stage=accepted outcome=no_picture failure=no_video fallback=none end=superseded "), summary) + + // The working address is the saved one now; the first keeps its + // pairing, as the PC still answers there. + XCTAssertEqual(RecentConnectionStore.shared.connections.first?.port, other.port) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: first.port)) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: other.port)), Self.token) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) + XCTAssertEqual(UserDefaults.standard.integer(forKey: "lastPort"), Int(other.port)) + + // Connecting to the first address later goes to the other first. + manager.cancel() + manager.connect(host: "127.0.0.1", port: first.port) + XCTAssertTrue(manager.diagnostics.contains { $0.message == "no_video_fallback step=remembered result=tried" }) + async let atFirst = Task.detached { first.received(within: 1) }.value + let answered = await Task.detached { other.acceptSession(authToken: Self.token) }.value + XCTAssertTrue(answered, "the address that showed video goes first") + let firstTypes = await atFirst + XCTAssertFalse(firstTypes.contains(.hello2), "\(firstTypes)") + + // Where the other address doesn't answer, the attempt goes back to + // the first. + manager.cancel() + manager.connect(host: "127.0.0.1", port: first.port) + let backAtFirst = await Task.detached { first.helloAttempts(within: 8) }.value + XCTAssertGreaterThan(backAtFirst, 0) + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=no_reply"), + logged().joined(separator: "\n")) + } + + /// The other address the PC signs answers without the pairing's proof, + /// as a forwarder or another device would: no HELLO2 and no token go + /// there, and the attempt ends as before. + func testAnOtherAddressThatDoesNotShowItIsTheSamePCIsLeftAlone() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, _) = try manager(pairedAt: first, other: other, connectTimeout: 8) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let ended = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: first.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + first.servePath(limit: 0, frame: frame, hostCaps: HelloAck.hostCapMediaSize, + addresses: Self.bothAddresses) { ended.withLock { $0 } } + } + let types = await Task.detached { () -> [Wire.PacketType] in + guard other.answerIdentify(token: Data(2...17)) else { return [] } + return other.received(within: 3) + }.value + XCTAssertFalse(types.contains(.hello2), "\(types)") + XCTAssertFalse(other.helloBeforeIdentify) + try await until("timed out", seconds: 20) { manager.state == .disconnected } + ended.withLock { $0 = true } + _ = await served.value + XCTAssertEqual(manager.problem, .noPicture) + XCTAssertTrue(logged().contains("no_video_fallback step=other_address result=none"), logged().joined(separator: "\n")) + XCTAssertNil(UserDefaults.standard.data(forKey: "noVideoPaths")) + } + + /// A PC that doesn't offer smaller datagrams, as every PC before this + /// version: the iPad asks for nothing new and asks no identity, and the + /// attempt times out as it always did. + func testAnOlderPCIsLeftAsBefore() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc, connectTimeout: 3) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let ended = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: 0, frame: frame, hostCaps: 0, countFrames: false) { ended.withLock { $0 } } + } + try await until("timed out", seconds: 20) { manager.state == .disconnected } + ended.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, []) + XCTAssertEqual(run.identifyAnswers, 0) + XCTAssertEqual(manager.problem, .noPicture) + let all = logged() + XCTAssertFalse(all.contains { $0.hasPrefix("no_video_fallback ") }, all.joined(separator: "\n")) + XCTAssertTrue(all.contains { $0.hasPrefix("Session accepted by 127.0.0.1:") && $0.contains("no video datagrams arrived") }) + } +} diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index b7d2273e..b33977cf 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,7 +35,7 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 37, + Self.vectors.count, 40, "golden vector count drifted — update both test suites together" ) } @@ -408,6 +408,43 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(old.notice, Heartbeat.noticeNone) } + /// A PC that takes smaller media datagrams says so, to an iPad that + /// asked; the request and the frame count are appended to the bodies + /// earlier versions read, so those read them as before. + func testMediaSizeVectors() throws { + XCTAssertEqual(HelloAck.hostCapMediaSize, 0x0080) + XCTAssertEqual(Hello2.featureMediaSize, 0x0040) + XCTAssertEqual(KeyframeRequest.minMediaDgram, 576) + + let ackData = try vector("hello_ack_media_size") + let (ackHeader, ackMessage) = try XCTUnwrap(Wire.parseControl(ackData)) + guard case .helloAck(let ack) = ackMessage else { return XCTFail("wrong type") } + XCTAssertEqual(ack.hostCaps, HelloAck.hostCapNack | HelloAck.hostCapContiguousFrameSeq | HelloAck.hostCapMediaSize) + XCTAssertEqual(Wire.encodeControl(sessionId: ackHeader.sessionId, msgSeq: ackHeader.msgSeq, message: ackMessage), ackData) + + let request = try vector("keyframe_request_media_size") + let (requestHeader, requestMessage) = try XCTUnwrap(Wire.parseControl(request)) + XCTAssertEqual(requestHeader.msgSeq, 51) + XCTAssertEqual(requestMessage, .keyframeRequest(KeyframeRequest(streamEpoch: 3, lastCompleteSeq: 0, reason: .startup, + maxMediaDgram: 1200))) + XCTAssertEqual(Wire.encodeControl(sessionId: 0x1234_5678, msgSeq: 51, message: requestMessage), request) + XCTAssertEqual(request.count, try vector("keyframe_request").count + 2) + + let counted = try vector("heartbeat_frames_sent") + let (heartbeatHeader, heartbeatMessage) = try XCTUnwrap(Wire.parseControl(counted)) + guard case .heartbeat(let hb) = heartbeatMessage else { return XCTFail("wrong type") } + XCTAssertEqual(hb.framesSent, 1234) + XCTAssertEqual(hb.notice, Heartbeat.noticeNone) + XCTAssertEqual(hb.streamConfig, Self.sampleConfig) + XCTAssertEqual(Wire.encodeControl(sessionId: 0x1234_5678, msgSeq: heartbeatHeader.msgSeq, message: heartbeatMessage), + counted) + let plain = try vector("heartbeat") + XCTAssertEqual(counted.count, plain.count + 5) + XCTAssertEqual(counted.dropFirst(16).prefix(plain.count - 16), plain.dropFirst(16)) + guard case .heartbeat(let old) = try XCTUnwrap(Wire.parseControl(plain)).message else { return XCTFail("wrong type") } + XCTAssertNil(old.framesSent) + } + func testByeVector() throws { let data = try vector("bye_background") let (_, message) = try XCTUnwrap(Wire.parseControl(data)) diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index e3df226f..e383fa28 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -44,6 +44,11 @@ pub const FEATURE_STANDBY_STATUS: u16 = 1 << 3; /// The iPad app checks for its own updates (0.3.2 and later), so a HELLO2 /// without it comes from 0.3.1 or older. Older hosts ignore it. pub const FEATURE_CHECKS_UPDATES: u16 = 1 << 4; +/// The iPad may ask for smaller media datagrams in a +/// [`KeyframeRequest::max_media_dgram`], and wants [`Heartbeat::frames_sent`] +/// to tell a PC that sends nothing from video lost on the way (0.3.5 and +/// later). Bit 5 is taken by another change in flight. +pub const FEATURE_MEDIA_SIZE: u16 = 1 << 6; pub const HOSTCAP_NACK: u16 = 1 << 0; pub const HOSTCAP_AUDIO: u16 = 1 << 1; pub const HOSTCAP_USB: u16 = 1 << 2; @@ -57,6 +62,14 @@ pub const HOSTCAP_PEN: u16 = 1 << 4; /// session, and iPads read an accepted reply without it as a PC to update, /// so it stays set there. pub const HOSTCAP_CONTIGUOUS_FRAME_SEQ: u16 = 1 << 5; +/// The host honours [`KeyframeRequest::max_media_dgram`] for the rest of the +/// session and puts [`Heartbeat::frames_sent`] in its heartbeats. Set only +/// for an iPad whose HELLO2 has [`FEATURE_MEDIA_SIZE`], so an older iPad +/// gets the same bytes as before. Bit 6 is taken by another change in flight. +pub const HOSTCAP_MEDIA_SIZE: u16 = 1 << 7; +/// The smallest media datagram an iPad may ask for: the IPv4 minimum +/// reassembly size, also the smallest packet size the PC's own setting takes. +pub const MIN_MEDIA_DGRAM: u16 = 576; /// [`StreamConfig::flags`] bits. pub const STREAM_FLAG_SOFTWARE_ENCODER: u8 = 1 << 0; @@ -271,6 +284,10 @@ pub struct Heartbeat { pub stream_config: StreamConfig, /// What the iPad should say over the picture, for as long as it lasts. pub notice: u8, + /// Video frames sent to this session so far, for an iPad that set + /// [`FEATURE_MEDIA_SIZE`]; `None` for others. Appended after `notice`, + /// which is then always written. + pub frames_sent: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -323,6 +340,11 @@ pub struct KeyframeRequest { pub stream_epoch: u32, pub last_complete_seq: u32, pub reason: KeyframeReason, + /// The largest media datagram the client asks for from now on in this + /// session, at least [`MIN_MEDIA_DGRAM`]; 0 asks nothing. Appended + /// (0.3.5): sent only to a host with [`HOSTCAP_MEDIA_SIZE`], and older + /// hosts ignore it as any trailing body bytes. + pub max_media_dgram: u16, } /// Requests missing fragments from one frame. Indices are unique and sorted. @@ -566,15 +588,21 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - let mut cfg = [0u8; STREAM_CONFIG_SIZE]; hb.stream_config.encode_into(&mut cfg); body.extend_from_slice(&cfg); - if hb.notice != NOTICE_NONE { + if hb.notice != NOTICE_NONE || hb.frames_sent.is_some() { body.push(hb.notice); } + if let Some(frames) = hb.frames_sent { + body.extend_from_slice(&frames.to_le_bytes()); + } } ControlMessage::Bye(reason) => body.push(*reason as u8), ControlMessage::KeyframeRequest(k) => { body.extend_from_slice(&k.stream_epoch.to_le_bytes()); body.extend_from_slice(&k.last_complete_seq.to_le_bytes()); body.push(k.reason as u8); + if k.max_media_dgram != 0 { + body.extend_from_slice(&k.max_media_dgram.to_le_bytes()); + } } ControlMessage::Nack(n) => { n.validate().expect("invalid outgoing NACK"); @@ -785,6 +813,11 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), } else { NOTICE_NONE }, + frames_sent: if r.remaining() >= 4 { + Some(r.u32()?) + } else { + None + }, }), PacketType::Bye => ControlMessage::Bye( ByeReason::from_u8(r.u8()?).ok_or(WireError::InvalidField("bye_reason"))?, @@ -794,6 +827,7 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), last_complete_seq: r.u32()?, reason: KeyframeReason::from_u8(r.u8()?) .ok_or(WireError::InvalidField("keyframe_reason"))?, + max_media_dgram: if r.remaining() >= 2 { r.u16()? } else { 0 }, }), PacketType::Nack => { let stream_epoch = r.u32()?; @@ -1134,12 +1168,26 @@ mod tests { host_time_us: 987_654_321, stream_config: sample_config(), notice: NOTICE_NONE, + frames_sent: None, + }), + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: sample_config(), + notice: NOTICE_NONE, + frames_sent: Some(0), }), ControlMessage::Bye(ByeReason::AppBackground), ControlMessage::KeyframeRequest(KeyframeRequest { stream_epoch: 3, last_complete_seq: 4111, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, + }), + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 3, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram: 1200, }), ControlMessage::ReceiverReport(ReceiverReport { stream_epoch: 3, @@ -1232,6 +1280,7 @@ mod tests { host_time_us: 987_654_321, stream_config: sample_config(), notice, + frames_sent: None, }) }; // No notice: the bytes every earlier iPad already reads. @@ -1252,6 +1301,53 @@ mod tests { assert_eq!(parse_control(&plain).unwrap().1, heartbeat(NOTICE_NONE)); } + /// The media size fields are appended: without them the bytes are the + /// ones every earlier PC and iPad reads, and a body cut back to the old + /// length, as an older parser reads it, is the request without them. + #[test] + fn the_media_size_fields_are_appended_only_when_set() { + let request = |max_media_dgram| { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 3, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram, + }) + }; + let plain = encode_control(7, 1, &request(0)); + assert_eq!(plain.len(), CONTROL_HEADER_SIZE + 9); + let sized = encode_control(7, 1, &request(576)); + assert_eq!(sized.len(), plain.len() + 2); + assert_eq!(sized[8..plain.len()], plain[8..]); + assert_eq!(&sized[plain.len()..], &576u16.to_le_bytes()); + assert_eq!(parse_control(&sized).unwrap().1, request(576)); + assert_eq!(parse_control(&plain).unwrap().1, request(0)); + + let heartbeat = |notice, frames_sent| { + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: sample_config(), + notice, + frames_sent, + }) + }; + let plain = encode_control(7, 1, &heartbeat(NOTICE_NONE, None)); + // The count needs the notice byte before it, even with no notice. + let counted = encode_control(7, 1, &heartbeat(NOTICE_NONE, Some(0x0102_0304))); + assert_eq!(counted.len(), plain.len() + 5); + assert_eq!(counted[8..plain.len()], plain[8..]); + assert_eq!(&counted[plain.len()..], &[NOTICE_NONE, 4, 3, 2, 1]); + assert_eq!( + parse_control(&counted).unwrap().1, + heartbeat(NOTICE_NONE, Some(0x0102_0304)) + ); + let locked = encode_control(7, 1, &heartbeat(NOTICE_PC_LOCKED, Some(9))); + assert_eq!( + parse_control(&locked).unwrap().1, + heartbeat(NOTICE_PC_LOCKED, Some(9)) + ); + } + #[test] fn trailing_bytes_are_ignored_for_forward_compat() { let mut datagram = encode_control(1, 1, &ControlMessage::Ping(Ping { t1_us: 5 })); diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 82419b11..98386fcd 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -23,6 +23,8 @@ heartbeat 454d021200001800785634122a000000b168de3a0000000003000000000aa0053c0000 heartbeat_pc_locked 454d021200001900785634122a000000b168de3a0000000003000000000aa0053c000000c0e1e40001 bye_background 454d021300000100785634122b00000001 keyframe_request 454d021400000900785634122c000000030000000f10000000 +keyframe_request_media_size 454d021400000b007856341233000000030000000000000002b004 +heartbeat_frames_sent 454d021200001d007856341234000000b168de3a0000000003000000000aa0053c000000c0e1e40000d2040000 receiver_report 454d021500002400785634122d00000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00 ping 454d021600000800785634122e0000000807060504030201 pong 454d021700001800785634122e000000080706050403020110070605040302012007060504030201 @@ -32,6 +34,7 @@ nack 454d021900001100785634123100000003000000891300002c0103000011002b01 receiver_report_v030 454d021500003200785634123200000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00d204000037020000590000003c00 hello_ack_paired 454d02110000390000000000030000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f100100 hello_ack_contiguous_seq 454d02110000390000000000050000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f102100 +hello_ack_media_size 454d02110000390000000000060000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f10a100 hello2_v030 454d02100000410000000000040000000202efbeadde94260300010074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e20100 hello2_standby_padded 454d02100000940000000000040000000202efbeadde94260300090074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e201000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 identify 454d021a00004400000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index 968c6cc1..2a383437 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -249,6 +249,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { host_time_us: 987_654_321, stream_config: stream_config(), notice: NOTICE_NONE, + frames_sent: None, }), ), ( @@ -259,6 +260,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { host_time_us: 987_654_321, stream_config: stream_config(), notice: NOTICE_PC_LOCKED, + frames_sent: None, }), ), ( @@ -275,6 +277,29 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { stream_epoch: 3, last_complete_seq: 4111, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, + }), + ), + ( + "keyframe_request_media_size", + 0x1234_5678, + 51, + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 3, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram: 1200, + }), + ), + ( + "heartbeat_frames_sent", + 0x1234_5678, + 52, + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: stream_config(), + notice: NOTICE_NONE, + frames_sent: Some(1234), }), ), ( @@ -398,7 +423,12 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { ack.host_caps = HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ; vectors.push(( "hello_ack_contiguous_seq", - encode_control(0, 5, &ControlMessage::HelloAck(ack)), + encode_control(0, 5, &ControlMessage::HelloAck(ack.clone())), + )); + ack.host_caps = HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ | HOSTCAP_MEDIA_SIZE; + vectors.push(( + "hello_ack_media_size", + encode_control(0, 6, &ControlMessage::HelloAck(ack)), )); let (_, ControlMessage::Hello2(mut hello)) = parse_control( &vectors @@ -704,3 +734,41 @@ fn contiguous_frame_seq_capability_round_trips() { assert_eq!(caps("hello_ack_paired") & HOSTCAP_CONTIGUOUS_FRAME_SEQ, 0); assert_eq!(HOSTCAP_CONTIGUOUS_FRAME_SEQ, 0x0020); } + +/// A host that takes smaller media datagrams says so; the request and the +/// heartbeat count are appended to the bodies earlier versions read. +#[test] +fn media_size_vectors_round_trip_and_extend_the_old_bodies() { + let vectors: BTreeMap<_, _> = canonical_vectors().into_iter().collect(); + let message = |name: &str| parse_control(&vectors[name]).unwrap().1; + let ControlMessage::HelloAck(ack) = message("hello_ack_media_size") else { + panic!("hello_ack_media_size") + }; + assert_eq!( + ack.host_caps, + HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ | HOSTCAP_MEDIA_SIZE + ); + assert_eq!(HOSTCAP_MEDIA_SIZE, 0x0080); + assert_eq!(FEATURE_MEDIA_SIZE, 0x0040); + assert_eq!(MIN_MEDIA_DGRAM, 576); + + let ControlMessage::KeyframeRequest(request) = message("keyframe_request_media_size") else { + panic!("keyframe_request_media_size") + }; + assert_eq!(request.max_media_dgram, 1200); + assert_eq!(request.reason, KeyframeReason::Startup); + let sized = &vectors["keyframe_request_media_size"]; + let plain = &vectors["keyframe_request"]; + assert_eq!(sized.len(), plain.len() + 2); + + let ControlMessage::Heartbeat(heartbeat) = message("heartbeat_frames_sent") else { + panic!("heartbeat_frames_sent") + }; + assert_eq!(heartbeat.frames_sent, Some(1234)); + assert_eq!(heartbeat.notice, NOTICE_NONE); + let counted = &vectors["heartbeat_frames_sent"]; + let plain = &vectors["heartbeat"]; + assert_eq!(counted.len(), plain.len() + 5); + // Past the msg_seq, the old heartbeat is a prefix of the new one. + assert_eq!(counted[16..plain.len()], plain[16..]); +} From 900097192486c2be3ea369d201dafda072590114 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 03:32:46 -0500 Subject: [PATCH 2/4] Check the remembered address's proof and cap the media-size keyframes - A remembered address gets HELLO2 and the token only after the PC there proves it holds the pairing, as a move with the same-PC checks. Without the proof, the entry is forgotten and the attempt goes to the address tapped. Entries are bound to a digest of the pairing token. - The other-address step only tries addresses on the iPad's own subnet, as a PC that Bonjour moved does. - Only the first two smaller sizes in a session skip the keyframe rate limit; later ones are ordinary keyframe requests. - The iPad's summary for a move after no picture says no_frame when some video arrived, as the PC's does. --- host/src/transport/session.rs | 53 ++++++++++-- .../App/ConnectionManager.swift | 80 ++++++++++++++---- ios/EternalMonitor/App/MovedPC.swift | 8 +- ios/EternalMonitor/App/NoVideoFallback.swift | 28 +++++-- ios/EternalMonitor/App/SessionSummary.swift | 2 +- .../NoVideoFallbackTests.swift | 83 +++++++++++++++++-- .../SessionSummaryTests.swift | 6 ++ 7 files changed, 217 insertions(+), 43 deletions(-) diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 9ad0a41b..9355dc00 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -109,6 +109,10 @@ struct RefusalMinute { } /// Honor at most one keyframe request per this window (PLI storm guard). pub const KEYFRAME_REQUEST_MIN_INTERVAL: Duration = Duration::from_millis(500); +/// Smaller media sizes per session whose keyframe skips that window: the +/// iPad asks for two (1,200 then 576 bytes). Any later one is an ordinary, +/// rate-limited keyframe request. +const MEDIA_SIZE_IDRS: u8 = 2; /// Media bytes, headers included, a Wi-Fi client's address gets before /// anything of its session comes back from it: room for the first picture, @@ -286,8 +290,10 @@ struct ActiveSession { /// Video frames let through to this session ([`Session::may_send_media`]). frames_sent: u32, /// The largest media datagram the client asked for in this session - /// ([`KeyframeRequest::max_media_dgram`]). + /// ([`KeyframeRequest::max_media_dgram`]), and how many times it went + /// down. max_media_dgram: Option, + media_size_lowered: u8, /// A Wi-Fi goodbye while the cable takes over. bye: Option, /// The msg_seq of input packets taken in this session. @@ -1121,6 +1127,7 @@ impl Session { frags_received: false, frames_sent: 0, max_media_dgram: None, + media_size_lowered: 0, bye: None, input_seqs: SeqWindow::default(), answered, @@ -1191,21 +1198,24 @@ impl Session { actions.force_idr = session.heard(Heard::Keyframe, now); info!(reason = ?request.reason, "Keyframe request received"); // Only a client that was told the host takes it asks; anything else - // is ignored, as before. Each smaller size restarts the picture at - // once: the frames sent so far gave the client no picture. + // is ignored, as before. The first smaller sizes restart the picture + // at once: the frames sent so far gave the client no picture. let asked = request.max_media_dgram; if asked != 0 && session.info.feature_caps & FEATURE_MEDIA_SIZE != 0 { let size = asked.clamp(MIN_MEDIA_DGRAM, eternal_wire::v2::MAX_DGRAM_SIZE as u16); if session.max_media_dgram.is_none_or(|current| size < current) { session.max_media_dgram = Some(size); - session.last_keyframe_grant = Some(now); + session.media_size_lowered = session.media_size_lowered.saturating_add(1); info!( max_dgram = size, frames_sent = session.frames_sent, "Media packet size lowered for this session" ); - actions.force_idr = true; - return actions; + if session.media_size_lowered <= MEDIA_SIZE_IDRS { + session.last_keyframe_grant = Some(now); + actions.force_idr = true; + return actions; + } } } @@ -2945,6 +2955,37 @@ mod tests { } assert_eq!(session.max_media_dgram(id.wrapping_add(1)), None); + // A client that keeps asking for less, a byte at a time, gets two + // keyframes past the rate limit, then no more than any other client. + session.handle_control( + peer, + 0, + media_size_hello(3, FEATURE_MEDIA_SIZE), + &TestConfig, + at(1_000), + ); + let id = session.session_id().unwrap(); + let mut forced = Vec::new(); + for step in 0..825u64 { + let actions = session.handle_control_authed( + peer, + sized_request(1400 - step as u16), + &TestConfig, + at(1_000 + step * 10), + ); + if actions.force_idr { + forced.push(step * 10); + } + } + assert_eq!(session.max_media_dgram(id), Some(MIN_MEDIA_DGRAM)); + // Two at once, then one per rate-limit window from the second. + assert_eq!(forced[..2], [0, 10]); + assert!(forced[2..] + .windows(2) + .all(|pair| pair[1] - pair[0] >= KEYFRAME_REQUEST_MIN_INTERVAL.as_millis() as u64)); + assert_eq!(forced[2], 510); + assert!(forced.len() <= 2 + 8240 / 500, "{forced:?}"); + session.handle_control( peer, 0, diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index e94613be..9b87e04e 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -280,6 +280,9 @@ final class ConnectionManager: ObservableObject { /// asked for (kept here) if the PC doesn't answer there. private var pendingRedirect: (host: String, port: UInt16)? private var attemptRedirect: (host: String, port: UInt16)? + /// The proof the PC at a remembered address is checked for, before any + /// attempt starts (see `connectViaRemembered`). + private var redirectCheck: UUID? /// The current session's no-video steps, with a PC that takes them, and /// the step whose result isn't known yet, as its log line names it. private var noVideoFallback: NoVideoFallback? @@ -378,24 +381,59 @@ final class ConnectionManager: ObservableObject { if let token { do { try pairingStore.save(token: token, hostName: "", address: PairingStore.address(host: host, port: port)) } catch { record(.warning, "pairing", "Could not save the scanned pairing on this iPad.") } - } else if let better = betterAddress(host: host, port: port) { - pendingRedirect = (host.trimmingCharacters(in: .whitespacesAndNewlines), port) - connect(target: .udp(host: better.host, port: better.port)) - pendingRedirect = nil + } else if let remembered = rememberedAddress(host: host, port: port) { + connectViaRemembered(from: (host.trimmingCharacters(in: .whitespacesAndNewlines), port), remembered) return } connect(target: .udp(host: host, port: port)) } - /// Where the same PC showed video after none arrived at `host:port`: an - /// address this iPad holds the same pairing for. - private func betterAddress(host: String, port: UInt16) -> NoVideoPaths.Address? { + /// Where the PC paired at `host:port` showed video after none arrived + /// there, kept for this pairing, with the pairing and the PC's name. + private func rememberedAddress(host: String, port: UInt16) -> (to: NoVideoPaths.Address, token: Data, name: String)? { let host = host.trimmingCharacters(in: .whitespacesAndNewlines) - guard let better = NoVideoPaths.better(than: host, port: port), - let token = (try? pairingStore.token(address: PairingStore.address(host: host, port: port))) ?? nil, - let there = (try? pairingStore.token(address: PairingStore.address(host: better.host, port: better.port))) ?? nil, - token == there else { return nil } - return better + let address = PairingStore.address(host: host, port: port) + guard let token = (try? pairingStore.token(address: address)) ?? nil, + let to = NoVideoPaths.better(than: host, port: port, pairing: token), + let name = pairingStore.pcName(address: address) ?? RecentConnectionStore.shared.entry(host: host, port: port)?.name + else { return nil } + return (to, token, name) + } + + /// Connecting to `from`, where the PC's video didn't arrive before, goes + /// to the address where it did, once the PC there shows it holds this + /// pairing and receives there: the address may belong to another device + /// now. Only then does a HELLO2 with the token go there, as a move, with + /// the same-PC checks of one. Without the proof the entry is forgotten + /// and the attempt goes to `from`. + private func connectViaRemembered(from: (host: String, port: UInt16), + _ remembered: (to: NoVideoPaths.Address, token: Data, name: String)) { + endReconnectCycle() + let check = UUID() + redirectCheck = check + connectionError = nil + problem = nil + state = .connecting + Task { @MainActor [weak self] in + guard let self else { return } + let to = remembered.to + let same = await MovedPC.confirm(host: to.host, port: to.port, token: remembered.token, + link: self.makeUDPLink(to.port)) + guard self.redirectCheck == check else { return } + self.redirectCheck = nil + self.state = .disconnected + guard same else { + NoVideoPaths.forget(from: .init(host: from.host, port: from.port)) + self.startAttempt(.udp(host: from.host, port: from.port)) + self.record(.info, "connection", "no_video_fallback step=remembered result=not_confirmed") + return + } + self.pendingMove = Move(from: from, name: remembered.name, token: remembered.token, noVideo: true) + self.pendingRedirect = from + self.explicitPairingToken = remembered.token + self.startAttempt(.udp(host: to.host, port: to.port)) + self.pendingRedirect = nil + } } func submitPairing() { @@ -1264,7 +1302,7 @@ final class ConnectionManager: ObservableObject { // The PC still answers at the old address: its pairing stays, // and connecting there goes here first. NoVideoPaths.remember(from: .init(host: move.from.host, port: move.from.port), - to: .init(host: new.host, port: new.port)) + to: .init(host: new.host, port: new.port), pairing: move.token) } else if homeAddress(move.from.host) { try? pairingStore.remove(address: PairingStore.address(host: move.from.host, port: move.from.port)) } @@ -1302,9 +1340,9 @@ final class ConnectionManager: ObservableObject { /// No picture came of the PC's video at this address, even in small /// datagrams. Asked here, the PC signs its home network addresses with - /// this iPad's pairing; the first other one this iPad reaches over - /// Wi-Fi or Ethernet, where the PC shows again that it holds the pairing - /// and receives there, gets a new attempt. A forwarded or spoofed + /// this iPad's pairing; the first other one on this iPad's own network, + /// where the PC shows again that it holds the pairing and receives + /// there, gets a new attempt. A forwarded or spoofed /// address can't make either answer. private func tryOtherAddress() { let host = debugState.host, port = debugState.port @@ -1328,7 +1366,9 @@ final class ConnectionManager: ObservableObject { let target = self.dialAddress(other, identity.port) let route = await self.localRoute(target.host, target.port) guard current() else { return } - guard let route, MovedPC.isDirect(route) else { continue } + // As for a PC Bonjour moved: only an address on this iPad's + // own network, so nothing relays the stream elsewhere. + guard let route, MovedPC.isLocal(target.host, over: route) else { continue } let same = await MovedPC.confirm(host: target.host, port: target.port, token: token, link: self.makeUDPLink(target.port)) guard current() else { return } @@ -1589,6 +1629,12 @@ final class ConnectionManager: ObservableObject { /// Ends the attempt. `end` says why; only the user's own action is /// `.userDisconnect`. func disconnect(_ end: AttemptEnd = .userDisconnect) { + // Still checking the remembered address: nothing went out yet. + if redirectCheck != nil { + redirectCheck = nil + state = .disconnected + return + } if state != .disconnected { endNoVideoStep("no_video") recordStreamSummary(reason: "disconnect=\(end.name)") diff --git a/ios/EternalMonitor/App/MovedPC.swift b/ios/EternalMonitor/App/MovedPC.swift index 6c181fb1..5e0e0723 100644 --- a/ios/EternalMonitor/App/MovedPC.swift +++ b/ios/EternalMonitor/App/MovedPC.swift @@ -85,16 +85,12 @@ enum MovedPC { guard text.split(separator: ".").count == 4, let bytes = IPv4Address(text)?.rawValue, bytes.count == 4 else { return nil } return bytes.reduce(0) { $0 << 8 | UInt32($1) } } - guard isDirect(route), let address = ipv4(route.address), let mask = ipv4(route.netmask), let candidate = ipv4(host), + guard route.direct, !tunnels.contains(where: route.interface.hasPrefix), + let address = ipv4(route.address), let mask = ipv4(route.netmask), let candidate = ipv4(host), mask != 0, mask != .max else { return false } return address & mask == candidate & mask } - /// Whether `route` is a direct Wi-Fi or Ethernet path, not a tunnel. - static func isDirect(_ route: Route) -> Bool { - route.direct && !tunnels.contains(where: route.interface.hasPrefix) - } - /// The other home network addresses `identity` signed, in the PC's /// order (those with a default gateway first), without `asked`. static func otherAddresses(_ identity: WireIdentity, asked: String, diff --git a/ios/EternalMonitor/App/NoVideoFallback.swift b/ios/EternalMonitor/App/NoVideoFallback.swift index b2270e5f..b7828db9 100644 --- a/ios/EternalMonitor/App/NoVideoFallback.swift +++ b/ios/EternalMonitor/App/NoVideoFallback.swift @@ -1,3 +1,4 @@ +import CryptoKit import Foundation /// A PC accepted the session and sends video, and no picture comes of it: @@ -54,8 +55,9 @@ struct NoVideoFallback: Equatable { } /// Where a PC's video didn't arrive, and the address of the same PC where it -/// did, so the next connect to the first goes to the second. Kept in -/// UserDefaults as "host:port" pairs. +/// did, so the next connect to the first goes to the second once the PC +/// there proves it holds the pairing. Each entry is bound to that pairing by +/// a digest of its token. Kept in UserDefaults. enum NoVideoPaths { private static let key = "noVideoPaths" private static let kept = 16 @@ -68,19 +70,31 @@ enum NoVideoPaths { private struct Entry: Codable { var from: Address var to: Address + /// SHA-256 of the pairing token, in hex. + var pairing: String } - /// The address where the PC at `host:port` showed video instead. - static func better(than host: String, port: UInt16) -> Address? { - load().last { $0.from == Address(host: host, port: port) }?.to + /// The address where the PC paired with `pairing` at `host:port` showed + /// video instead. + static func better(than host: String, port: UInt16, pairing: Data) -> Address? { + let digest = digest(pairing) + return load().last { $0.from == Address(host: host, port: port) && $0.pairing == digest }?.to } - static func remember(from: Address, to: Address) { + static func remember(from: Address, to: Address, pairing: Data) { var entries = load().filter { $0.from != from && $0.from != to } - entries.append(Entry(from: from, to: to)) + entries.append(Entry(from: from, to: to, pairing: digest(pairing))) save(Array(entries.suffix(kept))) } + static func forget(from: Address) { + save(load().filter { $0.from != from }) + } + + private static func digest(_ token: Data) -> String { + SHA256.hash(data: token).map { String(format: "%02x", $0) }.joined() + } + static func forgetAll() { UserDefaults.standard.removeObject(forKey: key) } diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index 702d1e44..e2af7146 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -116,7 +116,7 @@ struct SessionSummary: Equatable { switch end { case .connectTimeout where sessionId == nil, .pcMoved: return "no_reply" case .connectTimeout: return datagramsReceived == 0 ? "no_video" : "no_frame" - case .noVideoPath: return "no_video" + case .noVideoPath: return datagramsReceived == 0 ? "no_video" : "no_frame" case .pcStoppedResponding: return "signal_lost" case .cableUnplugged, .usbLinkFailed, .closedBeforePicture: return "usb_lost" case .noNetwork: return "no_network" diff --git a/ios/EternalMonitorTests/NoVideoFallbackTests.swift b/ios/EternalMonitorTests/NoVideoFallbackTests.swift index 8070989b..9b0d4d25 100644 --- a/ios/EternalMonitorTests/NoVideoFallbackTests.swift +++ b/ios/EternalMonitorTests/NoVideoFallbackTests.swift @@ -168,24 +168,95 @@ final class NoVideoFallbackTests: XCTestCase { XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) XCTAssertEqual(UserDefaults.standard.integer(forKey: "lastPort"), Int(other.port)) - // Connecting to the first address later goes to the other first. + // Connecting to the first address later goes to the other first, + // once the PC there shows again that it holds the pairing. manager.cancel() manager.connect(host: "127.0.0.1", port: first.port) - XCTAssertTrue(manager.diagnostics.contains { $0.message == "no_video_fallback step=remembered result=tried" }) async let atFirst = Task.detached { first.received(within: 1) }.value - let answered = await Task.detached { other.acceptSession(authToken: Self.token) }.value + let answered = await Task.detached { + other.answerIdentify(token: Self.token) && other.acceptSession(authToken: Self.token) + }.value XCTAssertTrue(answered, "the address that showed video goes first") + XCTAssertFalse(other.helloBeforeIdentify, "nothing with the token before the proof") let firstTypes = await atFirst XCTAssertFalse(firstTypes.contains(.hello2), "\(firstTypes)") + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=tried"), logged().joined(separator: "\n")) - // Where the other address doesn't answer, the attempt goes back to - // the first. + // Where nothing at the other address shows the proof, the attempt + // goes to the first, and the other is forgotten. manager.cancel() manager.connect(host: "127.0.0.1", port: first.port) let backAtFirst = await Task.detached { first.helloAttempts(within: 8) }.value XCTAssertGreaterThan(backAtFirst, 0) - XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=no_reply"), + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=not_confirmed"), logged().joined(separator: "\n")) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token)) + } + + /// The remembered address belongs to another PC now: one with pairing + /// on, which can't make this pairing's proof, or one with pairing off, + /// which would take any HELLO2 and doesn't answer the identity check. + /// The token never goes there, the pairing stays, and the attempt goes + /// to the address tapped. + func testARememberedAddressThatIsNowAnotherPCNeverGetsTheToken() async throws { + for pairingOn in [true, false] { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, pairings) = try manager(pairedAt: first) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: first.port), + to: .init(host: "127.0.0.1", port: other.port), pairing: Self.token) + manager.connect(host: "127.0.0.1", port: first.port) + async let atOther = Task.detached { () -> [Wire.PacketType] in + if pairingOn, !other.answerIdentify(token: Data(2...17)) { return [] } + return other.received(within: 4) + }.value + let accepted = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(accepted, "\(pairingOn): the tapped address got no HELLO2") + XCTAssertEqual(first.lastAuthToken, Self.token, "\(pairingOn)") + let types = await atOther + XCTAssertFalse(types.contains(.hello2), "\(pairingOn): \(types)") + XCTAssertFalse(other.helloBeforeIdentify, "\(pairingOn)") + if !pairingOn { XCTAssertEqual(Set(types), [.identify]) } + try await until("handshake at the tapped address") { manager.hostInfo != nil } + XCTAssertEqual(manager.debugState.port, first.port) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token)) + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=not_confirmed"), "\(pairingOn)") + } + // An entry kept for another pairing is never used for this one. + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: 1), to: .init(host: "127.0.0.1", port: 2), + pairing: Data(2...17)) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: 1, pairing: Self.token)) + XCTAssertEqual(NoVideoPaths.better(than: "127.0.0.1", port: 1, pairing: Data(2...17))?.port, 2) + } + + /// The other address the PC signs is outside this iPad's own network: + /// it is never asked, as for a PC Bonjour moved there. + func testAnOtherAddressOffThisNetworkIsNeverAsked() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, _) = try manager(pairedAt: first, other: other, connectTimeout: 8) + defer { manager.cancel() } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "10.0.0.5", netmask: "255.255.255.0") } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let ended = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: first.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + first.servePath(limit: 0, frame: frame, hostCaps: HelloAck.hostCapMediaSize, + addresses: Self.bothAddresses) { ended.withLock { $0 } } + } + try await until("asked the PC for its addresses", seconds: 20) { + logged().contains("no_video_fallback step=other_address result=none") + } + let types = await Task.detached { other.received(within: 1) }.value + XCTAssertEqual(types, []) + try await until("timed out", seconds: 20) { manager.state == .disconnected } + ended.withLock { $0 = true } + let run = await served.value + XCTAssertGreaterThan(run.identifyAnswers, 0) } /// The other address the PC signs answers without the pairing's proof, diff --git a/ios/EternalMonitorTests/SessionSummaryTests.swift b/ios/EternalMonitorTests/SessionSummaryTests.swift index 83f2fc5a..0ae21b58 100644 --- a/ios/EternalMonitorTests/SessionSummaryTests.swift +++ b/ios/EternalMonitorTests/SessionSummaryTests.swift @@ -63,6 +63,8 @@ final class SessionSummaryTests: XCTestCase { func testEachWayAnAttemptEndsHasItsSummary() { var noFrame = summary(.connectTimeout, duration: 10) noFrame.datagramsReceived = 12 + var movedNoFrame = summary(.noVideoPath, duration: 7) + movedNoFrame.datagramsReceived = 3 var pairing = summary(.userDisconnect, session: nil, duration: 30) pairing.awaitingPairing = true var retried = summary(.retriedWithCode, session: nil, duration: 4) @@ -82,6 +84,10 @@ final class SessionSummaryTests: XCTestCase { (summary(.otherPC), ["accepted", "error", "other", "error"]), (summary(.connectTimeout, duration: 10), ["accepted", "no_picture", "no_video", "timeout"]), (noFrame, ["accepted", "no_picture", "no_frame", "timeout"]), + // Moved to the PC's other address: as the PC says, nothing came, + // or no frame of what came completed. + (summary(.noVideoPath, duration: 7), ["accepted", "no_picture", "no_video", "superseded"]), + (movedNoFrame, ["accepted", "no_picture", "no_frame", "superseded"]), (summary(.pcStoppedResponding, firstFrame: 0.3, duration: 90), ["steady", "streamed", "signal_lost", "liveness"]), (summary(.pictureStalled, firstFrame: 0.3, duration: 90), ["steady", "streamed", "other", "error"]), (summary(.cableUnplugged, usb: true, firstFrame: 0.3), ["first_frame", "streamed", "usb_lost", "error"]), From e909a94095f6f8b877d32ffa88c110b7526fda4b Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 03:50:43 -0500 Subject: [PATCH 3/4] Retire a remembered address that fails, and check it is on this network - Before the identity check, a remembered address must be on this iPad's own subnet, as the other-address step requires; otherwise the entry is forgotten and the attempt goes to the address tapped. - When the attempt at a remembered address fails after its proof (no HELLO2 answer, no picture before the timeout, or another PC by name or pairing), the entry is forgotten and a new attempt goes to the address tapped. - Try again and a resume go to the address tapped while the remembered one is checked. - A golden vector covers a heartbeat that carries both the no-display notice and the frame count. --- .../App/ConnectionManager.swift | 54 ++++++++++----- .../NoVideoFallbackTests.swift | 65 +++++++++++++++++++ .../WireProtocolGoldenTests.swift | 11 +++- proto/testdata/v2_vectors.txt | 1 + proto/tests/golden.rs | 20 ++++++ 5 files changed, 134 insertions(+), 17 deletions(-) diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index b1e0357c..307eafb8 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -404,14 +404,17 @@ final class ConnectionManager: ObservableObject { } /// Connecting to `from`, where the PC's video didn't arrive before, goes - /// to the address where it did, once the PC there shows it holds this - /// pairing and receives there: the address may belong to another device - /// now. Only then does a HELLO2 with the token go there, as a move, with - /// the same-PC checks of one. Without the proof the entry is forgotten - /// and the attempt goes to `from`. + /// to the address where it did, once that address is on this iPad's own + /// network and the PC there shows it holds this pairing and receives + /// there: the address may belong to another device now. Only then does a + /// HELLO2 with the token go there, as a move, with the same-PC checks of + /// one. Otherwise the entry is forgotten and the attempt goes to `from`, + /// as it does if that move fails (see `leaveRememberedAddress`). private func connectViaRemembered(from: (host: String, port: UInt16), _ remembered: (to: NoVideoPaths.Address, token: Data, name: String)) { endReconnectCycle() + // Try again and a resume go to the address tapped. + lastTarget = from let check = UUID() redirectCheck = check connectionError = nil @@ -420,17 +423,23 @@ final class ConnectionManager: ObservableObject { Task { @MainActor [weak self] in guard let self else { return } let to = remembered.to + let toFrom = { [weak self] (result: String) in + guard let self else { return } + self.redirectCheck = nil + self.state = .disconnected + NoVideoPaths.forget(from: .init(host: from.host, port: from.port)) + self.startAttempt(.udp(host: from.host, port: from.port)) + self.record(.info, "connection", "no_video_fallback step=remembered result=\(result)") + } + let route = await self.localRoute(to.host, to.port) + guard self.redirectCheck == check else { return } + guard let route, MovedPC.isLocal(to.host, over: route) else { return toFrom("not_local") } let same = await MovedPC.confirm(host: to.host, port: to.port, token: remembered.token, link: self.makeUDPLink(to.port)) guard self.redirectCheck == check else { return } + guard same else { return toFrom("not_confirmed") } self.redirectCheck = nil self.state = .disconnected - guard same else { - NoVideoPaths.forget(from: .init(host: from.host, port: from.port)) - self.startAttempt(.udp(host: from.host, port: from.port)) - self.record(.info, "connection", "no_video_fallback step=remembered result=not_confirmed") - return - } self.pendingMove = Move(from: from, name: remembered.name, token: remembered.token, noVideo: true) self.pendingRedirect = from self.explicitPairingToken = remembered.token @@ -439,6 +448,19 @@ final class ConnectionManager: ObservableObject { } } + /// The attempt at a remembered address (see `connectViaRemembered`) + /// passed its proof and failed after it: it ends with `end`, the entry is + /// forgotten, and a new attempt goes to the address tapped. False when + /// this attempt didn't go to a remembered address. + private func leaveRememberedAddress(_ end: AttemptEnd, result: String) -> Bool { + guard let from = attemptRedirect else { return false } + NoVideoPaths.forget(from: .init(host: from.host, port: from.port)) + record(.info, "connection", "no_video_fallback step=remembered result=\(result)") + disconnect(end) + startAttempt(.udp(host: from.host, port: from.port)) + return true + } + func submitPairing() { guard awaitingPairing, let code = pairingPrompt?.submit() else { return } // The code goes in a new HELLO2. The refused one ends here, as on the @@ -759,6 +781,7 @@ final class ConnectionManager: ObservableObject { if let move = self.attemptMove { guard Self.isSamePC(move, info) else { self.record(.warning, "connection", "The PC at the new address isn't the saved one") + if self.leaveRememberedAddress(.otherPC, result: "other_pc") { return } self.disconnect(.otherPC) self.problem = .noReply(usb: false) return @@ -811,6 +834,7 @@ final class ConnectionManager: ObservableObject { // it is another PC, and the saved pairing stays. if self.attemptMove != nil, status == .unauthorized || status == .rateLimited { self.record(.warning, "connection", "The PC at the new address isn't the saved one: it asked for pairing") + if self.leaveRememberedAddress(.otherPC, result: "other_pc") { return } self.disconnect(.otherPC) self.problem = .noReply(usb: false) return @@ -876,11 +900,8 @@ final class ConnectionManager: ObservableObject { self.disconnect(.legacyPC) } else { self.record(.warning, "ctrl", "HELLO2 handshake got no reply yet") - if let from = self.attemptRedirect { - // Not on this network now: back to the address asked for. - self.record(.info, "connection", "no_video_fallback step=remembered result=no_reply") - self.disconnect(.pcMoved) - self.startAttempt(.udp(host: from.host, port: from.port)) + if self.leaveRememberedAddress(.pcMoved, result: "no_reply") { + // Back to the address tapped. } else if !isUSB { self.lookForMovedPC(connectionID, host: normalizedHost, port: port) } @@ -1427,6 +1448,7 @@ final class ConnectionManager: ObservableObject { let level = debug.timeoutLevel(handshakeDone: handshakeDone, answeredHereBefore: self.acceptedAddresses.contains("\(debug.host):\(debug.port)")) self.record(level, "timeout", debug.timeoutSummary(handshakeDone: handshakeDone) + self.attemptTimings) + if self.leaveRememberedAddress(.connectTimeout, result: "no_picture") { return } // Only an attempt the PC answered counts. if handshakeDone { self.noPictureStreak.endedWithoutPicture() } // A PC that said it isn't streaming and then went quiet (it diff --git a/ios/EternalMonitorTests/NoVideoFallbackTests.swift b/ios/EternalMonitorTests/NoVideoFallbackTests.swift index 9b0d4d25..3a676c61 100644 --- a/ios/EternalMonitorTests/NoVideoFallbackTests.swift +++ b/ios/EternalMonitorTests/NoVideoFallbackTests.swift @@ -232,6 +232,71 @@ final class NoVideoFallbackTests: XCTestCase { XCTAssertEqual(NoVideoPaths.better(than: "127.0.0.1", port: 1, pairing: Data(2...17))?.port, 2) } + /// A remembered address that is outside this iPad's own network now + /// gets nothing: no identity check, no HELLO2. The attempt goes to the + /// address tapped, and Try again goes there too. + func testARememberedAddressOffThisNetworkIsNeverAsked() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, pairings) = try manager(pairedAt: first) + defer { manager.cancel() } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "10.0.0.5", netmask: "255.255.255.0") } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: first.port), + to: .init(host: "127.0.0.1", port: other.port), pairing: Self.token) + manager.connect(host: "127.0.0.1", port: first.port) + let accepted = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(accepted) + XCTAssertEqual(first.lastAuthToken, Self.token) + let types = await Task.detached { other.received(within: 1) }.value + XCTAssertEqual(types, []) + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=not_local"), logged().joined(separator: "\n")) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token)) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) + } + + /// The PC at the remembered address passed its proof, then the attempt + /// there failed: HELLO2 went unanswered, the session showed no frame, + /// or the PC that answered had another name. The entry is forgotten, + /// the attempt goes to the address tapped at once, and so does the + /// next tap, with no identity check at the remembered address. + func testARememberedAddressThatFailsAfterItsProofIsRetired() async throws { + for failure in ["no_reply", "no_picture", "other_pc"] { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, _) = try manager(pairedAt: first, connectTimeout: 3) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: first.port), + to: .init(host: "127.0.0.1", port: other.port), pairing: Self.token) + manager.connect(host: "127.0.0.1", port: first.port) + let atOther = await Task.detached { () -> Bool in + guard other.answerIdentify(token: Self.token) else { return false } + switch failure { + case "no_picture": return other.acceptSession(authToken: Self.token) + case "other_pc": return other.acceptSession(hostName: "Other PC", authToken: Data(repeating: 0, count: 16)) + default: return other.helloAttempts(within: 1) > 0 + } + }.value + XCTAssertTrue(atOther, "\(failure): the proof passed and HELLO2 went there") + let atFirst = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(atFirst, "\(failure): then to the address tapped") + try await until("\(failure): handshake at the address tapped", seconds: 20) { + manager.hostInfo != nil && manager.debugState.port == first.port + } + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=\(failure)"), logged().joined(separator: "\n")) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token), failure) + _ = await Task.detached { other.received(within: 0.5) }.value + + manager.cancel() + manager.connect(host: "127.0.0.1", port: first.port) + let again = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(again, "\(failure): the next tap goes to the address tapped") + let types = await Task.detached { other.received(within: 0.5) }.value + XCTAssertFalse(types.contains(.identify) || types.contains(.hello2), "\(failure): \(types)") + } + } + /// The other address the PC signs is outside this iPad's own network: /// it is never asked, as for a PC Bonjour moved there. func testAnOtherAddressOffThisNetworkIsNeverAsked() async throws { diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index b33977cf..02f33b20 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,7 +35,7 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 40, + Self.vectors.count, 41, "golden vector count drifted — update both test suites together" ) } @@ -443,6 +443,15 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(counted.dropFirst(16).prefix(plain.count - 16), plain.dropFirst(16)) guard case .heartbeat(let old) = try XCTUnwrap(Wire.parseControl(plain)).message else { return XCTFail("wrong type") } XCTAssertNil(old.framesSent) + + // A notice and the count together: the notice byte, then the count. + let both = try vector("heartbeat_no_display_frames_sent") + let (bothHeader, bothMessage) = try XCTUnwrap(Wire.parseControl(both)) + guard case .heartbeat(let noDisplay) = bothMessage else { return XCTFail("wrong type") } + XCTAssertEqual(noDisplay.notice, Heartbeat.noticeNoDisplay) + XCTAssertEqual(noDisplay.framesSent, 77) + XCTAssertEqual(Array(both.dropFirst(plain.count)), [Heartbeat.noticeNoDisplay, 77, 0, 0, 0]) + XCTAssertEqual(Wire.encodeControl(sessionId: 0x1234_5678, msgSeq: bothHeader.msgSeq, message: bothMessage), both) } func testByeVector() throws { diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 98386fcd..622c0037 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -25,6 +25,7 @@ bye_background 454d021300000100785634122b00000001 keyframe_request 454d021400000900785634122c000000030000000f10000000 keyframe_request_media_size 454d021400000b007856341233000000030000000000000002b004 heartbeat_frames_sent 454d021200001d007856341234000000b168de3a0000000003000000000aa0053c000000c0e1e40000d2040000 +heartbeat_no_display_frames_sent 454d021200001d007856341235000000b168de3a0000000003000000000aa0053c000000c0e1e400024d000000 receiver_report 454d021500002400785634122d00000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00 ping 454d021600000800785634122e0000000807060504030201 pong 454d021700001800785634122e000000080706050403020110070605040302012007060504030201 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index 2a383437..3294c423 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -302,6 +302,17 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { frames_sent: Some(1234), }), ), + ( + "heartbeat_no_display_frames_sent", + 0x1234_5678, + 53, + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: stream_config(), + notice: NOTICE_NO_DISPLAY, + frames_sent: Some(77), + }), + ), ( "receiver_report", 0x1234_5678, @@ -771,4 +782,13 @@ fn media_size_vectors_round_trip_and_extend_the_old_bodies() { assert_eq!(counted.len(), plain.len() + 5); // Past the msg_seq, the old heartbeat is a prefix of the new one. assert_eq!(counted[16..plain.len()], plain[16..]); + + // A notice and the count together: the notice byte, then the count. + let ControlMessage::Heartbeat(both) = message("heartbeat_no_display_frames_sent") else { + panic!("heartbeat_no_display_frames_sent") + }; + assert_eq!(both.notice, NOTICE_NO_DISPLAY); + assert_eq!(both.frames_sent, Some(77)); + let bytes = &vectors["heartbeat_no_display_frames_sent"]; + assert_eq!(&bytes[plain.len()..], &[NOTICE_NO_DISPLAY, 77, 0, 0, 0]); } From 8ef2b6126d1d21a39950ea0e9f251e8d8033b19a Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 04:03:48 -0500 Subject: [PATCH 4/4] Give an unanswered HELLO2 time to give up before the connect timeout in the remembered-address test --- ios/EternalMonitorTests/NoVideoFallbackTests.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ios/EternalMonitorTests/NoVideoFallbackTests.swift b/ios/EternalMonitorTests/NoVideoFallbackTests.swift index 3a676c61..17785f0a 100644 --- a/ios/EternalMonitorTests/NoVideoFallbackTests.swift +++ b/ios/EternalMonitorTests/NoVideoFallbackTests.swift @@ -263,7 +263,9 @@ final class NoVideoFallbackTests: XCTestCase { func testARememberedAddressThatFailsAfterItsProofIsRetired() async throws { for failure in ["no_reply", "no_picture", "other_pc"] { let first = FakeUDPHost(), other = FakeUDPHost() - let (manager, _) = try manager(pairedAt: first, connectTimeout: 3) + // Well past HELLO2's 2 s of retries, so an unanswered HELLO2 always + // gives up before the connect timeout, even on a slow runner. + let (manager, _) = try manager(pairedAt: first, connectTimeout: 6) defer { manager.cancel() } let (logged, sink) = lines(manager) defer { sink.cancel() }