From a759b3d39ea113fc0ae35346ac58f63ad50657a1 Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 03:30:59 +0200 Subject: [PATCH 01/13] refactor(workspace): one output clock type for every restart path --- crates/irl-core/src/lib.rs | 1 + crates/irl-core/src/timing.rs | 98 ++++++++++++++++++++++ crates/irl-source/src/audio/hold.rs | 6 +- crates/irl-source/src/audio/mod.rs | 13 +-- crates/irl-source/src/audio/pump.rs | 28 +++---- crates/irl-source/src/receiver/audio_in.rs | 4 +- crates/irl-source/src/receiver/stream.rs | 6 +- crates/irl-source/src/shared.rs | 8 +- crates/irl-source/src/source.rs | 4 +- crates/irl-source/tests/audio_core.rs | 20 ++--- 10 files changed, 133 insertions(+), 55 deletions(-) diff --git a/crates/irl-core/src/lib.rs b/crates/irl-core/src/lib.rs index 6ce25e3..724b1a1 100644 --- a/crates/irl-core/src/lib.rs +++ b/crates/irl-core/src/lib.rs @@ -37,5 +37,6 @@ pub use speed::{ DrainWatch, SpeedCarry, SpeedController, SpeedInputs, SpeedTrim, StuckReport, catchup_speed_max, }; pub use stats::{StatKind, StatValue, StatsSnapshot}; +pub use timing::OutputClock; pub use url_opts::awaits_caller; pub use video_delay::{DelayRaise, DelayRelax, RampStep, VideoDelay}; diff --git a/crates/irl-core/src/timing.rs b/crates/irl-core/src/timing.rs index d26856b..a1794a9 100644 --- a/crates/irl-core/src/timing.rs +++ b/crates/irl-core/src/timing.rs @@ -17,6 +17,55 @@ pub fn output_next_ts(anchor_ns: u64, samples: u64, rate: u32) -> u64 { anchor_ns.wrapping_add(offset as u64) } +/// The audio output clock: [`output_next_ts`] over an anchor set when playback +/// primes and a count of the samples claimed since. +/// +/// The anchor only moves through [`Self::restart`] and [`Self::stand_down`], +/// which are the declared restarts; every other timestamp is a claim on the +/// counter, so consecutive submissions stay contiguous. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct OutputClock { + primed: bool, + anchor_ns: u64, + samples: u64, +} + +impl OutputClock { + /// Whether a clock line is running. + pub fn is_primed(&self) -> bool { + self.primed + } + + /// Samples claimed since the clock line started. + pub fn samples(&self) -> u64 { + self.samples + } + + /// Start a new clock line whose first sample plays at `at_ns`. + pub fn restart(&mut self, at_ns: u64) { + self.primed = true; + self.anchor_ns = at_ns; + self.samples = 0; + } + + /// Stop the clock line until the next [`Self::restart`]. + pub fn stand_down(&mut self) { + *self = Self::default(); + } + + /// The timestamp the next claimed sample will carry. + pub fn next_ts(&self, rate: u32) -> u64 { + output_next_ts(self.anchor_ns, self.samples, rate) + } + + /// Reserve `frames` and return the timestamp of the first of them. + pub fn claim(&mut self, frames: u32, rate: u32) -> u64 { + let ts = self.next_ts(rate); + self.samples += u64::from(frames); + ts + } +} + /// Lead kept ahead of wall clock: `max(AUDIO_OUT_LEAD_MS, 3 chunks)`, or one /// chunk in low-latency mode. /// @@ -340,6 +389,55 @@ mod tests { assert_eq!(prev, anchor + total_ns); } + #[test] + fn a_clock_starts_unprimed_and_a_restart_primes_it() { + let mut clock = OutputClock::default(); + assert!(!clock.is_primed()); + clock.restart(1_000); + assert!(clock.is_primed()); + assert_eq!(clock.samples(), 0); + assert_eq!(clock.next_ts(RATE as u32), 1_000); + } + + #[test] + fn claims_are_contiguous_on_the_counter() { + let anchor = 5_000_000_000; + let mut clock = OutputClock::default(); + clock.restart(anchor); + let mut samples = 0u64; + for frames in [1024u32, 1000, 1031, 1024] { + let ts = clock.claim(frames, RATE as u32); + assert_eq!(ts, output_next_ts(anchor, samples, RATE as u32)); + samples += u64::from(frames); + } + assert_eq!(clock.samples(), samples); + assert_eq!( + clock.next_ts(RATE as u32), + output_next_ts(anchor, samples, RATE as u32) + ); + } + + #[test] + fn a_restart_drops_the_claimed_samples_and_moves_the_anchor() { + let mut clock = OutputClock::default(); + clock.restart(1_000_000_000); + clock.claim(4800, RATE as u32); + clock.restart(3_000_000_000); + assert!(clock.is_primed()); + assert_eq!(clock.samples(), 0); + assert_eq!(clock.claim(1024, RATE as u32), 3_000_000_000); + } + + #[test] + fn standing_down_unprimes_and_forgets_the_line() { + let mut clock = OutputClock::default(); + clock.restart(1_000_000_000); + clock.claim(1024, RATE as u32); + clock.stand_down(); + assert_eq!(clock, OutputClock::default()); + assert!(!clock.is_primed()); + } + #[test] fn frames_to_ns_truncates() { assert_eq!(frames_to_ns(48_000, 48_000), 1_000_000_000); diff --git a/crates/irl-source/src/audio/hold.rs b/crates/irl-source/src/audio/hold.rs index c084056..c0d3709 100644 --- a/crates/irl-source/src/audio/hold.rs +++ b/crates/irl-source/src/audio/hold.rs @@ -39,7 +39,7 @@ pub fn observe_video_packet(shared: &Shared, received_ns: u64, dts_ns: i64) { state.hold.observe(received_ns, audio_pts_ns - dts_ns); let covered_ms = covered_ms(shared, &state); - let change = if !state.primed { + let change = if !state.clock.is_primed() { state.hold.before_prime(state.hold_ms, covered_ms) } else if regulates(shared) { state.hold.regulate(received_ns, state.hold_ms, covered_ms) @@ -190,7 +190,7 @@ fn apply(shared: &Shared, state: &mut AudioState, change: HoldChange) { } } - if state.primed { + if state.clock.is_primed() { let moved_ns = u64::from(to_ms.abs_diff(from_ms)) * 1_000_000; state.hold_unbuilt_ns = if to_ms > from_ms { state.hold_unbuilt_ns + moved_ns @@ -215,7 +215,7 @@ fn apply(shared: &Shared, state: &mut AudioState, change: HoldChange) { "Video has arrived at most {skew_ms}ms behind its audio for {}s; releasing the audio hold from {from_ms}ms to {to_ms}ms{in_effect}", consts::AUDIO_HOLD_RELAX_WINDOW_MS / 1000 ); - } else if state.primed { + } else if state.clock.is_primed() { irl_info!( "Video has arrived {skew_ms}ms behind its audio for {}s; holding audio back {to_ms}ms instead of {from_ms}ms to keep lip sync, built up by playing up to 2% slow{in_effect}", consts::AUDIO_HOLD_RAISE_WINDOW_MS / 1000 diff --git a/crates/irl-source/src/audio/mod.rs b/crates/irl-source/src/audio/mod.rs index 9caf445..0ad7789 100644 --- a/crates/irl-source/src/audio/mod.rs +++ b/crates/irl-source/src/audio/mod.rs @@ -8,7 +8,6 @@ use std::sync::Arc; use std::sync::atomic::Ordering::Relaxed; use irl_core::consts; -use irl_core::timing; use crate::shared::{AudioState, Shared}; @@ -62,20 +61,10 @@ impl AudioSink for obs::SourceHandle { } } -/// Reserve `frames` on the sample-counter clock and return the OBS timestamp -/// for them. Caller holds `audio_state`. -pub fn output_claim(state: &mut AudioState, frames: u32, rate: u32) -> u64 { - let ts = timing::output_next_ts(state.anchor_ns, state.samples, rate); - state.samples += frames as u64; - ts -} - /// Output clock, playout mapping, fades and concealment back to the /// not-yet-primed state. Caller holds `audio_state`. pub fn reset_audio_timing_state(state: &mut AudioState) { - state.primed = false; - state.anchor_ns = 0; - state.samples = 0; + state.clock.stand_down(); state.conceal_fade_pending = false; state.out_last = irl_core::LastSample::default(); state.offset_baseline_ns = 0; diff --git a/crates/irl-source/src/audio/pump.rs b/crates/irl-source/src/audio/pump.rs index a85323a..48b1f04 100644 --- a/crates/irl-source/src/audio/pump.rs +++ b/crates/irl-source/src/audio/pump.rs @@ -146,7 +146,7 @@ impl AudioPump { let lead_ns = timing::output_lead_ns(base_samples, fmt.rate, low_latency); let now = (self.now_ns)(); - if !state.primed { + if !state.clock.is_primed() { // The speed is only computed after priming, so the controller // re-arms from 1.0 for as long as playback is unprimed, which is // the window every connection prep and runtime reset opens. @@ -177,7 +177,7 @@ impl AudioPump { return true; } - if state.primed { + if state.clock.is_primed() { // Cap runaway concealment latency before it desyncs A/V. Runs // even on a healthy-lead cycle: the offset inflates from past // outages, not from the current queue depth. @@ -185,7 +185,7 @@ impl AudioPump { maybe_reanchor_offset(shared, state, now, chunk_ns, fmt.target_ms); } - let next_ts = timing::output_next_ts(state.anchor_ns, state.samples, fmt.rate as u32); + let next_ts = state.clock.next_ts(fmt.rate as u32); // Enough queued ahead of wall clock — nothing to do until the // lead runs down, and the output clock says exactly when that is. @@ -215,13 +215,12 @@ impl AudioPump { "Audio output stalled {}ms; restarting output clock", (now - next_ts) / 1_000_000 ); - state.anchor_ns = now + chunk_ns; - state.samples = 0; + state.clock.restart(now + chunk_ns); state.conceal_fade_pending = true; } } - if !state.primed { + if !state.clock.is_primed() { let mut prime_ms = timing::prime_threshold_ms(fmt.target_ms, lead_ns, low_latency); // Elsewhere the hold is part of the target already; low-latency // mode has none, and keeps the hold queued instead. @@ -238,10 +237,8 @@ impl AudioPump { return false; } - state.primed = true; state.hold_unbuilt_ns = 0; - state.anchor_ns = now + chunk_ns; - state.samples = 0; + state.clock.restart(now + chunk_ns); // Reads and writes are both whole decoded chunks, so the residual // can only ever be a multiple of one: a 120ms target is not a // level the buffer can hold, and the loop straddles it at 106 or @@ -421,7 +418,7 @@ Video stays in sync with it; check the sender's frame rate and clock", } } - let timestamp = super::output_claim(state, frames_out, fmt.rate as u32); + let timestamp = state.clock.claim(frames_out, fmt.rate as u32); let emitted: &[u8] = if use_speed_buf { &self.speed_scratch[..emit_bytes] } else { @@ -484,7 +481,7 @@ Video stays in sync with it; check the sender's frame rate and clock", // subsequent ones are pure silence. state.out_last.forget(); - let timestamp = super::output_claim(state, frames as u32, fmt.rate as u32); + let timestamp = state.clock.claim(frames as u32, fmt.rate as u32); self.sink.output_audio(&obs::AudioFrame::interleaved( &self.pump_scratch[..silence_bytes], frames as u32, @@ -599,9 +596,7 @@ impl BufferFormat { /// never gets here: its concealment keeps the counter moving, so a late clock /// there really is an output-side stall. fn suspend_low_latency_clock(shared: &Shared, state: &mut AudioState, lag_ns: u64, now_us: u64) { - state.primed = false; - state.anchor_ns = 0; - state.samples = 0; + state.clock.stand_down(); state.latest_obs_end_ts_ns = 0; state.latest_buffered_end_pts_ns = 0; state.offset_baseline_set = false; @@ -676,8 +671,7 @@ fn maybe_reanchor_offset( return; } - state.anchor_ns = now + chunk_ns; - state.samples = 0; + state.clock.restart(now + chunk_ns); state.latest_obs_end_ts_ns = 0; state.latest_buffered_end_pts_ns = 0; state.offset_baseline_set = false; @@ -706,7 +700,7 @@ fn maybe_trim_hidden_backlog( if !shared.hot.adaptive_speed.load(Relaxed) { return false; } - if low_latency || state.primed { + if low_latency || state.clock.is_primed() { return false; } if chunk_count <= 1 { diff --git a/crates/irl-source/src/receiver/audio_in.rs b/crates/irl-source/src/receiver/audio_in.rs index bfc5845..4f0f41d 100644 --- a/crates/irl-source/src/receiver/audio_in.rs +++ b/crates/irl-source/src/receiver/audio_in.rs @@ -244,9 +244,7 @@ impl AudioIntake { } }; - state.primed = false; - state.anchor_ns = 0; - state.samples = 0; + state.clock.stand_down(); state.out_last.forget(); state.latest_buffered_end_pts_ns = 0; state.latest_audio_stream_pts_ns = 0; diff --git a/crates/irl-source/src/receiver/stream.rs b/crates/irl-source/src/receiver/stream.rs index 50fb647..8c9e293 100644 --- a/crates/irl-source/src/receiver/stream.rs +++ b/crates/irl-source/src/receiver/stream.rs @@ -147,7 +147,7 @@ pub fn fade_out_buffered_audio( let Some(buf) = guard.as_mut() else { return }; let buffered_ms = buf.fill_ms(); - if buffered_ms <= 0 || !state.primed { + if buffered_ms <= 0 || !state.clock.is_primed() { return; } @@ -179,10 +179,10 @@ pub fn fade_out_buffered_audio( // after a slow teardown or a stalled worker: OBS would compensate for // the late audio by growing its global buffering. A disconnect already // discards the remaining input, so discard an expired fade as well. - if irl_core::timing::output_next_ts(state.anchor_ns, state.samples, sample_rate) < now_ns() { + if state.clock.next_ts(sample_rate) < now_ns() { return; } - let timestamp = audio::output_claim(state, frames, sample_rate); + let timestamp = state.clock.claim(frames, sample_rate); sink.output_audio(&obs::AudioFrame::interleaved( &fade_buf[..got], frames, diff --git a/crates/irl-source/src/shared.rs b/crates/irl-source/src/shared.rs index c41bb7b..1ccfb2d 100644 --- a/crates/irl-source/src/shared.rs +++ b/crates/irl-source/src/shared.rs @@ -29,7 +29,8 @@ use parking_lot::{Condvar, Mutex, MutexGuard}; use irl_core::arrival::ArrivalFloor; use irl_core::{ - AudioBuffer, AudioHold, DrainWatch, HwDecode, LastSample, SpeedCarry, SpeedTrim, Watermarks, + AudioBuffer, AudioHold, DrainWatch, HwDecode, LastSample, OutputClock, SpeedCarry, SpeedTrim, + Watermarks, }; /// Settings latched when the stream opens; changing any of them forces a @@ -110,10 +111,7 @@ pub struct RunFlags { /// Audio timing state that is not a counter, under `Shared::audio_state`. #[derive(Debug, Default)] pub struct AudioState { - // Output clock: `ts = anchor + samples / rate`, anchored once at prime. - pub primed: bool, - pub anchor_ns: u64, - pub samples: u64, + pub clock: OutputClock, // Playout mapping (audio → OBS clock), the lip-sync source for video. pub latest_obs_end_ts_ns: u64, diff --git a/crates/irl-source/src/source.rs b/crates/irl-source/src/source.rs index 220cc1a..e5dffc3 100644 --- a/crates/irl-source/src/source.rs +++ b/crates/irl-source/src/source.rs @@ -292,8 +292,8 @@ impl Source for IrlSource { // Connected, but nothing on screen yet: the first connection attempt // is still in avformat_open_input, or the keyframe gate has not // opened. - let playing = - running.shared.conn.video_ts_init.load(Relaxed) || running.shared.audio_state().primed; + let playing = running.shared.conn.video_ts_init.load(Relaxed) + || running.shared.audio_state().clock.is_primed(); if playing { MediaState::Playing } else { diff --git a/crates/irl-source/tests/audio_core.rs b/crates/irl-source/tests/audio_core.rs index 25052d6..3484a3a 100644 --- a/crates/irl-source/tests/audio_core.rs +++ b/crates/irl-source/tests/audio_core.rs @@ -197,12 +197,12 @@ fn a_pump_burst_stops_when_disconnect_pauses_playback() { } assert!(pump.pump_once()); let count = recorder.len(); - let samples = shared.audio_state().samples; + let samples = shared.audio_state().clock.samples(); shared.flags.reconnecting.store(true, Relaxed); clock.fetch_add(1_000_000_000, Relaxed); assert!(!pump.pump_once()); assert_eq!(recorder.len(), count); - assert_eq!(shared.audio_state().samples, samples); + assert_eq!(shared.audio_state().clock.samples(), samples); } /// A slow transport close must not submit a fade from the stopped clock: @@ -222,8 +222,7 @@ fn disconnect_fade_does_not_submit_stale_audio() { .unwrap() .write_pts(&data, 1_000_000_000); let mut state = shared.audio_state(); - state.primed = true; - state.anchor_ns = now + 80_000_000; + state.clock.restart(now + 80_000_000); obs_irl_source::receiver::stream::fade_out_buffered_audio( &shared, &mut state, @@ -241,7 +240,8 @@ fn disconnect_fade_does_not_submit_stale_audio() { "{rate}Hz disconnect submitted audio 920ms late" ); assert_eq!( - state.samples, 0, + state.clock.samples(), + 0, "a discarded fade must not claim output samples" ); } @@ -459,7 +459,7 @@ fn an_idle_pump_reports_when_it_next_has_work() { // Primed and queued ahead: it now knows when the lead runs down, and that // is further away than a 1ms poll. - assert!(shared.audio_state().primed); + assert!(shared.audio_state().clock.is_primed()); let hint = pump.idle_sleep_ms(); assert!( hint > consts::AUDIO_PUMP_SLEEP_MS && hint <= consts::AUDIO_PUMP_MAX_SLEEP_MS, @@ -491,7 +491,7 @@ fn a_quiet_low_latency_input_suspends_the_clock_instead_of_restart_looping() { let mut pts_ns = 0i64; write_chunks(&shared, &mut pts_ns, 6, 0.1); while pump.pump_once() {} - assert!(shared.audio_state().primed, "never primed"); + assert!(shared.audio_state().clock.is_primed(), "never primed"); // The input goes quiet. Wall clock runs well past the stall threshold. for _ in 0..40 { @@ -505,7 +505,7 @@ fn a_quiet_low_latency_input_suspends_the_clock_instead_of_restart_looping() { "a quiet low-latency source must not restart the output clock" ); assert!( - !shared.audio_state().primed, + !shared.audio_state().clock.is_primed(), "the clock should be stood down, not left running" ); @@ -513,7 +513,7 @@ fn a_quiet_low_latency_input_suspends_the_clock_instead_of_restart_looping() { write_chunks(&shared, &mut pts_ns, 6, 0.2); while pump.pump_once() {} assert!( - shared.audio_state().primed, + shared.audio_state().clock.is_primed(), "did not re-prime on real audio" ); assert_eq!(shared.conn.audio_output_restarts.load(Relaxed), 0); @@ -577,7 +577,7 @@ fn the_buffer_settles_on_the_configured_target_with_aac_chunks() { clock.fetch_add(AAC_NS, Relaxed); } - assert!(shared.audio_state().primed, "never primed"); + assert!(shared.audio_state().clock.is_primed(), "never primed"); let tail = &levels[levels.len() - 1000..]; let mean = tail.iter().map(|&f| f as f64).sum::() / tail.len() as f64; let chunk_ms = (AAC_NS / 1_000_000) as f64; From 61de0ddb53c4c06e700a51bb3731766308f91cdd Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 03:34:27 +0200 Subject: [PATCH 02/13] refactor(workspace): playout mapping as one type that owns its drift baseline --- crates/irl-core/src/lib.rs | 2 + crates/irl-core/src/playout.rs | 192 +++++++++++++++++++++ crates/irl-core/src/video_time.rs | 50 +----- crates/irl-source/src/audio/hold.rs | 6 - crates/irl-source/src/audio/mod.rs | 5 +- crates/irl-source/src/audio/pump.rs | 55 ++---- crates/irl-source/src/receiver/audio_in.rs | 3 +- crates/irl-source/src/receiver/stream.rs | 16 +- crates/irl-source/src/shared.rs | 24 +-- crates/irl-source/src/video/output.rs | 23 +-- crates/irl-source/tests/audio_core.rs | 10 +- crates/irl-source/tests/network_sim.rs | 8 +- crates/irl-source/tests/video_pipeline.rs | 13 +- 13 files changed, 250 insertions(+), 157 deletions(-) create mode 100644 crates/irl-core/src/playout.rs diff --git a/crates/irl-core/src/lib.rs b/crates/irl-core/src/lib.rs index 724b1a1..675ef09 100644 --- a/crates/irl-core/src/lib.rs +++ b/crates/irl-core/src/lib.rs @@ -18,6 +18,7 @@ pub mod config; pub mod consts; pub mod dsp; pub mod pacing; +pub mod playout; pub mod pts_repair; mod rescale; pub mod speed; @@ -32,6 +33,7 @@ pub use audio_hold::{AudioHold, HoldChange}; pub use config::{HwDecode, Watermarks}; pub use dsp::LastSample; pub use pacing::{DueVerdict, PacingQueue}; +pub use playout::PlayoutMapping; pub use pts_repair::{PtsAction, PtsRepair, Verdict}; pub use speed::{ DrainWatch, SpeedCarry, SpeedController, SpeedInputs, SpeedTrim, StuckReport, catchup_speed_max, diff --git a/crates/irl-core/src/playout.rs b/crates/irl-core/src/playout.rs new file mode 100644 index 0000000..971023c --- /dev/null +++ b/crates/irl-core/src/playout.rs @@ -0,0 +1,192 @@ +//! The audio playout mapping: which stream PTS is playing at which OBS time. +//! Video is scheduled through it, so it is the lip-sync reference. + +/// The latest chunk the audio pump handed to OBS, as the OBS time it ends at +/// and the stream PTS its content ends at, plus the baseline the offset +/// between the two is watched against. +/// +/// The offset's absolute value carries the stream's PTS epoch and means +/// nothing on its own; only its movement does. Concealment advances the OBS +/// side while the stream side stands still, so an outage grows the offset by +/// its length, and that growth is what [`Self::drift_ns`] reports. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct PlayoutMapping { + obs_end_ns: u64, + pts_end_ns: i64, + baseline: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Baseline { + offset_ns: i64, + /// The audio hold when the baseline was taken. The hold moves the offset + /// on purpose, so drift is measured net of how far it moved since. + hold_ms: i32, +} + +impl PlayoutMapping { + /// Record the chunk just handed to OBS: it ends at `obs_end_ns` on the OBS + /// clock and at `pts_end_ns` in stream time. + pub fn publish(&mut self, obs_end_ns: u64, pts_end_ns: i64) { + self.obs_end_ns = obs_end_ns; + self.pts_end_ns = pts_end_ns; + } + + /// Whether a chunk has been handed to OBS since the mapping was last + /// cleared, whether or not its stream PTS makes the mapping usable. + pub fn has_output(&self) -> bool { + self.obs_end_ns != 0 + } + + /// The stream PTS the latest chunk's content ends at. + pub fn pts_end_ns(&self) -> i64 { + self.pts_end_ns + } + + /// Whether there is a mapping to schedule video through. + pub fn is_published(&self) -> bool { + self.obs_end_ns != 0 && self.pts_end_ns > 0 + } + + /// The stream PTS to OBS clock offset, once a mapping is published. + pub fn offset_ns(&self) -> Option { + self.is_published() + .then(|| self.obs_end_ns as i64 - self.pts_end_ns) + } + + /// The OBS time at which stream PTS `pts_ns` plays, once a mapping is + /// published. A time before the OBS clock's epoch clamps to zero, which + /// OBS takes as "now". + pub fn map(&self, pts_ns: i64) -> Option { + let mapped = pts_ns + self.offset_ns()?; + Some(if mapped < 0 { 0 } else { mapped as u64 }) + } + + /// Take the current offset as the baseline drift is measured from, with + /// `hold_ms` the audio hold in force. Only when a mapping is published and + /// no baseline is held; returns whether one was taken. + pub fn take_baseline(&mut self, hold_ms: i32) -> bool { + if self.baseline.is_some() { + return false; + } + let Some(offset_ns) = self.offset_ns() else { + return false; + }; + self.baseline = Some(Baseline { offset_ns, hold_ms }); + true + } + + /// How far the offset has grown past its baseline, net of what the audio + /// hold (now `hold_ms`) moved it by since. `None` without a published + /// mapping or a baseline. + pub fn drift_ns(&self, hold_ms: i32) -> Option { + let offset_ns = self.offset_ns()?; + let baseline = self.baseline?; + let hold_moved_ns = i64::from(hold_ms - baseline.hold_ms) * 1_000_000; + Some(offset_ns - baseline.offset_ns - hold_moved_ns) + } + + /// Forget the latest chunk and keep the baseline. + pub fn unpublish(&mut self) { + self.obs_end_ns = 0; + self.pts_end_ns = 0; + } + + /// Forget the latest chunk and the baseline. + pub fn clear(&mut self) { + *self = Self::default(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn published(obs_end_ns: u64, pts_end_ns: i64) -> PlayoutMapping { + let mut mapping = PlayoutMapping::default(); + mapping.publish(obs_end_ns, pts_end_ns); + mapping + } + + #[test] + fn mapping_shifts_the_pts_onto_the_obs_clock() { + // Audio whose stream PTS ends at 10 s plays out at 3 s on the OBS + // clock: everything is shifted back by 7 s. + let mapping = published(3_000_000_000, 10_000_000_000); + assert_eq!(mapping.map(10_000_000_000), Some(3_000_000_000)); + assert_eq!(mapping.map(10_016_000_000), Some(3_016_000_000)); + assert_eq!(mapping.offset_ns(), Some(-7_000_000_000)); + } + + #[test] + fn negative_mapped_pts_clamps_to_zero() { + // A frame from before the audio epoch would map before the OBS clock + // started; OBS takes 0 as "now". + assert_eq!(published(1_000_000_000, 10_000_000_000).map(0), Some(0)); + assert_eq!(published(1, 1).map(-5), Some(0)); + } + + #[test] + fn nothing_maps_until_both_ends_are_usable() { + assert!(!PlayoutMapping::default().has_output()); + assert_eq!(PlayoutMapping::default().map(0), None); + // A chunk went out, but its content carried no usable stream PTS. + let no_pts = published(5_000_000_000, 0); + assert!(no_pts.has_output()); + assert!(!no_pts.is_published()); + assert_eq!(no_pts.offset_ns(), None); + assert_eq!(no_pts.map(1_000), None); + assert!(published(5_000_000_000, 1).is_published()); + } + + #[test] + fn drift_is_measured_from_the_first_baseline() { + let mut mapping = published(3_000_000_000, 10_000_000_000); + assert_eq!(mapping.drift_ns(0), None, "no baseline yet"); + assert!(mapping.take_baseline(0)); + assert!(!mapping.take_baseline(0), "a held baseline stays"); + assert_eq!(mapping.drift_ns(0), Some(0)); + + // A 300 ms outage concealed: the OBS side ran on, the stream side + // stood still. + mapping.publish(3_300_000_000, 10_000_000_000); + assert_eq!(mapping.drift_ns(0), Some(300_000_000)); + } + + #[test] + fn drift_is_net_of_the_hold_moving_the_offset() { + let mut mapping = published(3_000_000_000, 10_000_000_000); + assert!(mapping.take_baseline(100)); + // The hold rose by 200 ms and the offset followed it. + mapping.publish(3_200_000_000, 10_000_000_000); + assert_eq!(mapping.drift_ns(300), Some(0)); + // A hold released below the baseline's reads the other way. + assert_eq!(mapping.drift_ns(0), Some(300_000_000)); + } + + #[test] + fn no_baseline_is_taken_without_a_mapping() { + let mut mapping = PlayoutMapping::default(); + assert!(!mapping.take_baseline(0)); + mapping.publish(5_000_000_000, 0); + assert!(!mapping.take_baseline(0)); + assert_eq!(mapping.drift_ns(0), None); + } + + #[test] + fn unpublishing_keeps_the_baseline_and_clearing_does_not() { + let mut mapping = published(3_000_000_000, 10_000_000_000); + assert!(mapping.take_baseline(0)); + + mapping.unpublish(); + assert!(!mapping.has_output()); + assert_eq!(mapping.drift_ns(0), None); + mapping.publish(3_050_000_000, 10_000_000_000); + assert_eq!(mapping.drift_ns(0), Some(50_000_000)); + + mapping.clear(); + assert_eq!(mapping, PlayoutMapping::default()); + mapping.publish(3_050_000_000, 10_000_000_000); + assert_eq!(mapping.drift_ns(0), None); + } +} diff --git a/crates/irl-core/src/video_time.rs b/crates/irl-core/src/video_time.rs index 4743198..80fb652 100644 --- a/crates/irl-core/src/video_time.rs +++ b/crates/irl-core/src/video_time.rs @@ -1,27 +1,10 @@ -//! Video timestamp mapping: the arithmetic behind video due times and the -//! lead libobs can absorb. Locking and logging stay in the plugin crate. +//! Video timestamp arithmetic outside the audio playout mapping +//! ([`crate::PlayoutMapping`]): the video-only fallback, the lead libobs can +//! absorb and the frame interval. Locking and logging stay in the plugin +//! crate. use crate::consts; -/// Map a stream PTS through the audio playout mapping: the OBS timestamp at -/// which audio with `buffered_end_pts_ns` will play is `obs_end_ts_ns`, so -/// `pts` lands at `obs_end + (pts − buffered_end)`. Negative results clamp to -/// zero. -/// -/// The caller decides whether a mapping exists at all (`obs_end_ts_ns != 0 -/// && buffered_end_pts_ns > 0`), and holds the last known offset for -/// `VIDEO_OFFSET_HOLD_NS` after that stops being true. -pub fn map_through_playout(pts_ns: i64, obs_end_ts_ns: u64, buffered_end_pts_ns: i64) -> u64 { - let mapped = pts_ns + (obs_end_ts_ns as i64 - buffered_end_pts_ns); - if mapped < 0 { 0 } else { mapped as u64 } -} - -/// The stream-PTS → OBS-clock offset the mapping implies, for re-deriving -/// queued frames' due times. -pub fn playout_offset_ns(obs_end_ts_ns: u64, buffered_end_pts_ns: i64) -> i64 { - obs_end_ts_ns as i64 - buffered_end_pts_ns -} - /// Video-only fallback anchored at the first frame: drift below −500 ms shows /// now, drift above +500 ms caps at now + 200 ms. /// @@ -82,31 +65,6 @@ pub fn interval_ema(prev_ns: i64, delta_ns: i64) -> i64 { mod tests { use super::*; - #[test] - fn mapping_shifts_the_pts_onto_the_obs_clock() { - // Audio whose stream PTS ends at 10 s plays out at 3 s on the OBS - // clock: everything is shifted back by 7 s. - let obs_end = 3_000_000_000u64; - let buffered_end = 10_000_000_000i64; - assert_eq!( - map_through_playout(10_000_000_000, obs_end, buffered_end), - 3_000_000_000 - ); - assert_eq!( - map_through_playout(10_016_000_000, obs_end, buffered_end), - 3_016_000_000 - ); - assert_eq!(playout_offset_ns(obs_end, buffered_end), -7_000_000_000); - } - - #[test] - fn negative_mapped_pts_clamps_to_zero() { - // A frame from before the audio epoch would map before the OBS clock - // started; OBS takes 0 as "now". - assert_eq!(map_through_playout(0, 1_000_000_000, 10_000_000_000), 0); - assert_eq!(map_through_playout(-5, 1, 1), 0); - } - #[test] fn fallback_passes_a_frame_inside_the_drift_window() { let now = 10_000_000_000u64; diff --git a/crates/irl-source/src/audio/hold.rs b/crates/irl-source/src/audio/hold.rs index c0d3709..8c8e81b 100644 --- a/crates/irl-source/src/audio/hold.rs +++ b/crates/irl-source/src/audio/hold.rs @@ -99,12 +99,6 @@ pub fn credit_build(state: &mut AudioState, played_ns: u64, consumed_ns: u64) { state.hold_built_ns += built_ns; } -/// How far the hold has moved since the playout offset baseline was taken, -/// which the offset is expected to follow rather than read as drift. -pub fn moved_since_baseline_ns(state: &AudioState) -> i64 { - i64::from(state.hold_ms - state.offset_baseline_hold_ms) * 1_000_000 -} - /// Forget the readings, keeping the hold in force: the audio timeline broke, /// so readings from before it no longer compare with readings after it, but /// the sender is the same one. diff --git a/crates/irl-source/src/audio/mod.rs b/crates/irl-source/src/audio/mod.rs index 0ad7789..2d6ebb9 100644 --- a/crates/irl-source/src/audio/mod.rs +++ b/crates/irl-source/src/audio/mod.rs @@ -67,8 +67,7 @@ pub fn reset_audio_timing_state(state: &mut AudioState) { state.clock.stand_down(); state.conceal_fade_pending = false; state.out_last = irl_core::LastSample::default(); - state.offset_baseline_ns = 0; - state.offset_baseline_set = false; + state.mapping.clear(); state.recovery_until_us = 0; state.speed_carry.reset(); state.align_read_pending = false; @@ -76,8 +75,6 @@ pub fn reset_audio_timing_state(state: &mut AudioState) { // to build once it re-primes. state.hold_unbuilt_ns = 0; state.latest_audio_stream_pts_ns = 0; - state.latest_buffered_end_pts_ns = 0; - state.latest_obs_end_ts_ns = 0; state.decoded_frame_samples = 0; state.startup_warmup_remaining_ms = 0; state.drain = irl_core::DrainWatch::default(); diff --git a/crates/irl-source/src/audio/pump.rs b/crates/irl-source/src/audio/pump.rs index 48b1f04..e0b2475 100644 --- a/crates/irl-source/src/audio/pump.rs +++ b/crates/irl-source/src/audio/pump.rs @@ -439,15 +439,14 @@ Video stays in sync with it; check the sender's frame rate and clock", timing::frames_to_ns(u64::from(frames_out), fmt.rate as u32), stream_duration_ns, ); - let first_mapping = state.latest_obs_end_ts_ns == 0; + let first_mapping = !state.mapping.has_output(); finalize_audio_output( shared, state, timestamp, frames_out, fmt.rate as u32, - chunk_pts_ns, - stream_duration_ns, + chunk_pts_ns + stream_duration_ns as i64, ); if first_mapping { // Video holds its first frame until this mapping exists and then @@ -494,14 +493,14 @@ Video stays in sync with it; check the sender's frame rate and clock", // Stream PTS does not advance during concealment: the video mapping // offset grows by the outage length, which matches the real playout // delay until the hidden trim pulls it back. + let pts_end_ns = state.mapping.pts_end_ns(); finalize_audio_output( shared, state, timestamp, frames as u32, fmt.rate as u32, - state.latest_buffered_end_pts_ns, - 0, + pts_end_ns, ); true } @@ -597,9 +596,7 @@ impl BufferFormat { /// there really is an output-side stall. fn suspend_low_latency_clock(shared: &Shared, state: &mut AudioState, lag_ns: u64, now_us: u64) { state.clock.stand_down(); - state.latest_obs_end_ts_ns = 0; - state.latest_buffered_end_pts_ns = 0; - state.offset_baseline_set = false; + state.mapping.clear(); state.conceal_fade_pending = true; shared.conn.audio_underruns.fetch_add(1, Relaxed); @@ -631,28 +628,16 @@ fn maybe_reanchor_offset( chunk_ns: u64, buffer_target_ms: i32, ) { - if state.latest_obs_end_ts_ns == 0 || state.latest_buffered_end_pts_ns <= 0 { - return; - } - - let offset_ns = state.latest_obs_end_ts_ns as i64 - state.latest_buffered_end_pts_ns; - - // The offset's absolute value is arbitrary (it carries the stream's PTS - // epoch); only its drift from the primed baseline is meaningful, so - // anchor the comparison the first time a valid offset is seen after - // priming. - if !state.offset_baseline_set { - state.offset_baseline_ns = offset_ns; - state.offset_baseline_hold_ms = state.hold_ms; - state.offset_baseline_set = true; + // The baseline is the first offset seen after priming. + if state.mapping.take_baseline(state.hold_ms) { return; } - // A raised audio hold grows the offset on purpose, by playing slower // until the buffer holds it; that is latency asked for, not drift. + let Some(excess_ns) = state.mapping.drift_ns(state.hold_ms) else { + return; + }; let margin_ns = consts::AUDIO_OFFSET_REANCHOR_MARGIN_MS * 1_000_000; - let excess_ns = - offset_ns - state.offset_baseline_ns - super::hold::moved_since_baseline_ns(state); if excess_ns <= margin_ns { return; } @@ -672,9 +657,7 @@ fn maybe_reanchor_offset( } state.clock.restart(now + chunk_ns); - state.latest_obs_end_ts_ns = 0; - state.latest_buffered_end_pts_ns = 0; - state.offset_baseline_set = false; + state.mapping.clear(); state.conceal_fade_pending = true; shared.lifetime.audio_offset_reanchors.fetch_add(1, Relaxed); @@ -754,22 +737,18 @@ fn maybe_trim_hidden_backlog( true } -/// Publish the playout mapping and count the chunk. +/// Publish the playout mapping for a chunk of `frames` submitted at +/// `timestamp` whose content ends at stream PTS `pts_end_ns`, and count it. fn finalize_audio_output( shared: &Shared, state: &mut AudioState, timestamp: u64, frames: u32, - samples_per_sec: u32, - chunk_pts_ns: i64, - stream_duration_ns: u64, + rate: u32, + pts_end_ns: i64, ) { - state.latest_buffered_end_pts_ns = chunk_pts_ns + stream_duration_ns as i64; - state.latest_obs_end_ts_ns = if samples_per_sec > 0 { - timestamp + frames as u64 * 1_000_000_000 / samples_per_sec as u64 - } else { - timestamp - }; + let obs_end_ns = timestamp + timing::frames_to_ns(u64::from(frames), rate); + state.mapping.publish(obs_end_ns, pts_end_ns); shared.conn.total_audio_frames.fetch_add(1, Relaxed); } diff --git a/crates/irl-source/src/receiver/audio_in.rs b/crates/irl-source/src/receiver/audio_in.rs index 4f0f41d..036e6f4 100644 --- a/crates/irl-source/src/receiver/audio_in.rs +++ b/crates/irl-source/src/receiver/audio_in.rs @@ -246,9 +246,8 @@ impl AudioIntake { state.clock.stand_down(); state.out_last.forget(); - state.latest_buffered_end_pts_ns = 0; + state.mapping.unpublish(); state.latest_audio_stream_pts_ns = 0; - state.latest_obs_end_ts_ns = 0; state.startup_warmup_remaining_ms = consts::STARTUP_AUDIO_WARMUP_MS; reconfigured } diff --git a/crates/irl-source/src/receiver/stream.rs b/crates/irl-source/src/receiver/stream.rs index 8c9e293..f12faf9 100644 --- a/crates/irl-source/src/receiver/stream.rs +++ b/crates/irl-source/src/receiver/stream.rs @@ -527,17 +527,11 @@ impl Receiver { let (av_drift_ms, av_skew_ms) = { let state = shared.audio_state(); let skew = crate::source::av_skew_ms(shared, &state); - let drift = match state.playout_mapping() { - // Net of the audio hold, which moves the offset on purpose. - Some((obs_end, buffered_end)) if state.offset_baseline_set => { - (obs_end as i64 - - buffered_end - - state.offset_baseline_ns - - crate::audio::hold::moved_since_baseline_ns(&state)) - / 1_000_000 - } - _ => 0, - }; + // Net of the audio hold, which moves the offset on purpose. + let drift = state + .mapping + .drift_ns(state.hold_ms) + .map_or(0, |drift_ns| drift_ns / 1_000_000); (drift, skew) }; diff --git a/crates/irl-source/src/shared.rs b/crates/irl-source/src/shared.rs index 1ccfb2d..7103ebf 100644 --- a/crates/irl-source/src/shared.rs +++ b/crates/irl-source/src/shared.rs @@ -29,8 +29,8 @@ use parking_lot::{Condvar, Mutex, MutexGuard}; use irl_core::arrival::ArrivalFloor; use irl_core::{ - AudioBuffer, AudioHold, DrainWatch, HwDecode, LastSample, OutputClock, SpeedCarry, SpeedTrim, - Watermarks, + AudioBuffer, AudioHold, DrainWatch, HwDecode, LastSample, OutputClock, PlayoutMapping, + SpeedCarry, SpeedTrim, Watermarks, }; /// Settings latched when the stream opens; changing any of them forces a @@ -113,11 +113,9 @@ pub struct RunFlags { pub struct AudioState { pub clock: OutputClock, - // Playout mapping (audio → OBS clock), the lip-sync source for video. - pub latest_obs_end_ts_ns: u64, - pub latest_buffered_end_pts_ns: i64, - pub offset_baseline_ns: i64, - pub offset_baseline_set: bool, + /// Where the audio the pump hands OBS plays, which video is scheduled + /// through. Other threads read it through [`Self::playout_mapping`]. + pub mapping: PlayoutMapping, // Concealment. pub out_last: LastSample, @@ -167,9 +165,6 @@ pub struct AudioState { /// offset grew for the hold. The video thread takes the growth out of its /// standing delay. pub hold_built_ns: u64, - /// The hold when `offset_baseline_ns` was taken, so a re-anchor does not - /// read the hold building as concealment drift. - pub offset_baseline_hold_ms: i32, /// When the pump first waited on a skew reading to prime; zero while it /// has not. pub hold_wait_since_ns: u64, @@ -179,11 +174,10 @@ pub struct AudioState { } impl AudioState { - /// The audio → OBS playout mapping, `(latest_obs_end_ts_ns, - /// latest_buffered_end_pts_ns)`, once the pump has published one. - pub fn playout_mapping(&self) -> Option<(u64, i64)> { - (self.latest_obs_end_ts_ns != 0 && self.latest_buffered_end_pts_ns > 0) - .then_some((self.latest_obs_end_ts_ns, self.latest_buffered_end_pts_ns)) + /// A copy of the playout mapping, so a reader on another thread holds + /// `audio_state` only for the copy. + pub fn playout_mapping(&self) -> PlayoutMapping { + self.mapping } } diff --git a/crates/irl-source/src/video/output.rs b/crates/irl-source/src/video/output.rs index 1688ba1..48c416b 100644 --- a/crates/irl-source/src/video/output.rs +++ b/crates/irl-source/src/video/output.rs @@ -179,13 +179,9 @@ impl VideoThread { let pts_ns = frame.pts(); let now = obs::time::gettime_ns(); - let (obs_end, buffered_end, startup_warmup_ms) = { + let (mapping, startup_warmup_ms) = { let state = self.shared.audio_state(); - ( - state.latest_obs_end_ts_ns, - state.latest_buffered_end_pts_ns, - state.startup_warmup_remaining_ms, - ) + (state.playout_mapping(), state.startup_warmup_remaining_ms) }; let frame_interval_ns = self.shared.conn.video_frame_interval_ns.load(Relaxed); @@ -195,9 +191,8 @@ impl VideoThread { // `VideoThread::settle_anchor_candidate`. let delay_ns = self.delay.delay_ns(); - if obs_end != 0 && buffered_end > 0 { - let mapped = video_time::map_through_playout(pts_ns, obs_end, buffered_end) - .saturating_add(delay_ns); + if let Some(mapped) = mapping.map(pts_ns) { + let mapped = mapped.saturating_add(delay_ns); self.record_lead(mapped as i64, now, frame_interval_ns); return mapped; } @@ -223,7 +218,7 @@ impl VideoThread { // flag the receiver publishes. if self.shared.flags.audio_present.load(Relaxed) { let mut audio_lead_ns = 0; - if obs_end == 0 { + if !mapping.has_output() { audio_lead_ns = startup_warmup_ms as i64 * 1_000_000; if !self.shared.cfg.low_latency_audio { audio_lead_ns += self.shared.hot.watermarks().target_ms as i64 * 1_000_000; @@ -244,7 +239,7 @@ impl VideoThread { /// a held offset belongs to audio that has stopped, and anchoring on it /// would be anchoring on a guess. pub fn mapping_published(&self) -> bool { - self.shared.audio_state().playout_mapping().is_some() + self.shared.audio_state().playout_mapping().is_published() } /// The current stream-PTS → OBS-clock offset, for re-deriving the due time @@ -258,11 +253,11 @@ impl VideoThread { /// keeps the due times the frames arrived with. The standing video delay /// rides on the offset, so a reschedule keeps it. pub fn playout_offset(&mut self) -> Option { - let mapping = self.shared.audio_state().playout_mapping(); + let offset_ns = self.shared.audio_state().playout_mapping().offset_ns(); let now = obs::time::gettime_ns(); - if let Some((obs_end, buffered_end)) = mapping { - self.playout_offset_ns = video_time::playout_offset_ns(obs_end, buffered_end); + if let Some(offset_ns) = offset_ns { + self.playout_offset_ns = offset_ns; self.playout_offset_time_ns = now; } else if self.playout_offset_time_ns == 0 || now.saturating_sub(self.playout_offset_time_ns) > consts::VIDEO_OFFSET_HOLD_NS diff --git a/crates/irl-source/tests/audio_core.rs b/crates/irl-source/tests/audio_core.rs index 3484a3a..0601d7f 100644 --- a/crates/irl-source/tests/audio_core.rs +++ b/crates/irl-source/tests/audio_core.rs @@ -85,7 +85,7 @@ fn expected_ts(anchor: u64, samples: u64, rate: u64) -> u64 { /// Lip sync, measured on the audio content rather than on the bookkeeping: /// a ramp encodes each sample's media time, so where a sample actually lands -/// on the OBS clock can be compared with where [`map_through_playout`] would +/// on the OBS clock can be compared with where [`PlayoutMapping::map`] would /// put the video frame carrying the same media time. /// /// 44.1 kHz is the interesting rate. 1024 samples is 2089.795 ticks of the @@ -94,7 +94,7 @@ fn expected_ts(anchor: u64, samples: u64, rate: u64) -> u64 { /// rate every phone encoder sends, and the only one where the repaired /// timeline can slip against the frames it labels. /// -/// [`map_through_playout`]: irl_core::video_time::map_through_playout +/// [`PlayoutMapping::map`]: irl_core::PlayoutMapping::map #[test] fn planar_aac_samples_follow_the_video_playout_mapping() { /// Two chunks of PI settling, comfortably inside the ~45 ms at which @@ -139,11 +139,7 @@ fn planar_aac_samples_follow_the_video_playout_mapping() { let content_pts = (out.samples[i * CHANNELS as usize] as f64 * 1e9) as i64; let audio_due = out.timestamp + i as u64 * 1_000_000_000 / rate as u64; let state = shared.audio_state(); - let video_due = irl_core::video_time::map_through_playout( - content_pts, - state.latest_obs_end_ts_ns, - state.latest_buffered_end_pts_ns, - ); + let video_due = state.mapping.map(content_pts).expect("a published mapping"); worst_ns = worst_ns.max((audio_due as i64 - video_due as i64).abs()); checked += 1; } diff --git a/crates/irl-source/tests/network_sim.rs b/crates/irl-source/tests/network_sim.rs index 1dc03d6..7864d29 100644 --- a/crates/irl-source/tests/network_sim.rs +++ b/crates/irl-source/tests/network_sim.rs @@ -175,12 +175,8 @@ impl Sim { let skew = self.video_skew_ns.expect("a sender with video"); let pts = self.delivered.last().expect("audio delivered") - skew; let state = self.shared.audio_state(); - assert!(state.latest_obs_end_ts_ns != 0, "audio has not primed"); - let due = irl_core::video_time::map_through_playout( - pts, - state.latest_obs_end_ts_ns, - state.latest_buffered_end_pts_ns, - ); + assert!(state.mapping.has_output(), "audio has not primed"); + let due = state.mapping.map(pts).expect("a published mapping"); (due as i64 - self.now_ns() as i64) / 1_000_000 } diff --git a/crates/irl-source/tests/video_pipeline.rs b/crates/irl-source/tests/video_pipeline.rs index f382d20..db33fc7 100644 --- a/crates/irl-source/tests/video_pipeline.rs +++ b/crates/irl-source/tests/video_pipeline.rs @@ -643,8 +643,7 @@ fn queued_frames_reschedule_onto_the_audio_playout_offset() { let now = obs::time::gettime_ns(); { let mut state = shared.audio_state(); - state.latest_obs_end_ts_ns = now + 1_000_000_000; - state.latest_buffered_end_pts_ns = 10_000_000_000; + state.mapping.publish(now + 1_000_000_000, 10_000_000_000); } feed(&mut thread, 10_000_000_000, now); @@ -658,7 +657,7 @@ fn queued_frames_reschedule_onto_the_audio_playout_offset() { // with it rather than trailing by the depth of the queue. { let mut state = shared.audio_state(); - state.latest_obs_end_ts_ns = now + 500_000_000; + state.mapping.publish(now + 500_000_000, 10_000_000_000); } thread.run_once(now); assert_eq!(thread.pacing().next_due(), Some(now + 500_000_000)); @@ -668,7 +667,7 @@ fn queued_frames_reschedule_onto_the_audio_playout_offset() { // end of it. { let mut state = shared.audio_state(); - state.latest_obs_end_ts_ns = now; + state.mapping.publish(now, 10_000_000_000); } thread.run_once(now); assert!(recorder.emitted().is_empty()); @@ -716,8 +715,7 @@ fn a_frame_is_due_where_the_audio_mapping_puts_it() { { let mut state = shared.audio_state(); // A frame at stream PTS 0 maps two seconds into the future. - state.latest_obs_end_ts_ns = now + 2_000_000_000; - state.latest_buffered_end_pts_ns = 1; + state.mapping.publish(now + 2_000_000_000, 1); } let mut frame = sw_frame(Pix::AV_PIX_FMT_YUV420P, 64, 32); @@ -758,8 +756,7 @@ fn a_disabled_keyframe_gate_passes_non_key_frames() { /// `buffered_end_pts_ns` plays out at OBS time `obs_end_ts_ns`. fn publish_mapping(shared: &Shared, obs_end_ts_ns: u64, buffered_end_pts_ns: i64) { let mut state = shared.audio_state(); - state.latest_obs_end_ts_ns = obs_end_ts_ns; - state.latest_buffered_end_pts_ns = buffered_end_pts_ns; + state.mapping.publish(obs_end_ts_ns, buffered_end_pts_ns); } /// Anchor the play head on a frame at stream PTS 10 s that arrives at the From 75f1312601b43dda86c554cc241483679c5eb77f Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 03:36:02 +0200 Subject: [PATCH 03/13] refactor(irl-source): hidden backlog trim sizes its keep from the pump's chunk and lead --- crates/irl-source/src/audio/pump.rs | 36 +++++++++++++---------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/crates/irl-source/src/audio/pump.rs b/crates/irl-source/src/audio/pump.rs index e0b2475..408fcc3 100644 --- a/crates/irl-source/src/audio/pump.rs +++ b/crates/irl-source/src/audio/pump.rs @@ -172,7 +172,16 @@ impl AudioPump { .audio_fill_peak_ms .fetch_max(fill_ms, Relaxed); - if has_audio && maybe_trim_hidden_backlog(shared, state, fill_ms, chunk_count, low_latency) + if has_audio + && maybe_trim_hidden_backlog( + shared, + state, + fill_ms, + chunk_count, + low_latency, + chunk_ns, + lead_ns, + ) { return true; } @@ -673,12 +682,17 @@ fn maybe_reanchor_offset( /// live, content is never skipped: the read loop stops ingesting above a fill /// ceiling (transport backpressure) and playback bleeds the backlog off at up /// to the Catch-Up Speed. +/// +/// `chunk_ns` and `lead_ns` are the pump's for this cycle: one output chunk +/// and the lead kept ahead of wall clock. fn maybe_trim_hidden_backlog( shared: &Shared, state: &AudioState, fill_ms: i32, chunk_count: usize, low_latency: bool, + chunk_ns: u64, + lead_ns: u64, ) -> bool { if !shared.hot.adaptive_speed.load(Relaxed) { return false; @@ -690,28 +704,10 @@ fn maybe_trim_hidden_backlog( return false; } - let out_rate = shared.audio_buf().as_ref().map_or(0, |b| b.sample_rate()); - let mut chunk_ms = 0; - if out_rate > 0 && state.decoded_frame_samples > 0 { - chunk_ms = (state.decoded_frame_samples as i64 * 1000 / out_rate as i64) as i32; - } - if chunk_ms <= 0 { - chunk_ms = 21; - } - // Keep enough to satisfy the prime threshold, which includes the OBS-side // lead. - let chunk_samples = if state.decoded_frame_samples > 0 { - state.decoded_frame_samples - } else { - consts::AUDIO_DEFAULT_FRAME_SAMPLES - }; let target_ms = shared.hot.watermarks().target_ms; - let mut keep_ms = target_ms + chunk_ms; - if out_rate > 0 { - keep_ms += - (timing::output_lead_ns(chunk_samples, out_rate, low_latency) / 1_000_000) as i32; - } + let keep_ms = target_ms + (chunk_ns / 1_000_000) as i32 + (lead_ns / 1_000_000) as i32; if fill_ms <= keep_ms + consts::AUDIO_TRIM_TRIGGER_MS { return false; } From b213ddf16a582f0bf97aeec0a89103c548cf3e07 Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 03:44:07 +0200 Subject: [PATCH 04/13] fix(irl-source): video thread reads one injected clock --- crates/irl-source/src/video/output.rs | 4 +- crates/irl-source/src/video/thread.rs | 27 ++- crates/irl-source/tests/video_pipeline.rs | 252 +++++++++++----------- 3 files changed, 156 insertions(+), 127 deletions(-) diff --git a/crates/irl-source/src/video/output.rs b/crates/irl-source/src/video/output.rs index 48c416b..584d1c9 100644 --- a/crates/irl-source/src/video/output.rs +++ b/crates/irl-source/src/video/output.rs @@ -177,7 +177,7 @@ impl VideoThread { // time base captured at stream open, because the video thread must not // touch the format context (it can be freed mid-reconnect). let pts_ns = frame.pts(); - let now = obs::time::gettime_ns(); + let now = self.now_ns(); let (mapping, startup_warmup_ms) = { let state = self.shared.audio_state(); @@ -254,7 +254,7 @@ impl VideoThread { /// rides on the offset, so a reschedule keeps it. pub fn playout_offset(&mut self) -> Option { let offset_ns = self.shared.audio_state().playout_mapping().offset_ns(); - let now = obs::time::gettime_ns(); + let now = self.now_ns(); if let Some(offset_ns) = offset_ns { self.playout_offset_ns = offset_ns; diff --git a/crates/irl-source/src/video/thread.rs b/crates/irl-source/src/video/thread.rs index 3815b27..56217c7 100644 --- a/crates/irl-source/src/video/thread.rs +++ b/crates/irl-source/src/video/thread.rs @@ -111,6 +111,9 @@ pub struct VideoThread { /// hold_built_ns`) as of the last cycle; `None` until the first read after /// a reset. See [`Self::absorb_audio_hold`]. hold_built_seen_ns: Option, + /// The OBS clock, read for every timing decision this thread makes. + /// Injectable so tests can step it; production reads `os_gettime_ns`. + now_ns: Box u64 + Send>, /// The OBS canvas tick. Injectable so tests can drive pacing without a /// running libobs — `obs_get_frame_interval_ns` reads libobs's global /// video state and faults when `obs_startup` never ran. @@ -155,6 +158,7 @@ impl VideoThread { delay: VideoDelay::default(), arrival: ArrivalFloor::default(), hold_built_seen_ns: None, + now_ns: Box::new(obs::time::gettime_ns), canvas_tick_ns: Box::new(obs::time::canvas_frame_interval_ns), sink, } @@ -162,7 +166,7 @@ impl VideoThread { pub fn run(&mut self) { while self.shared.is_active() { - let wait = self.run_once(obs::time::gettime_ns()); + let wait = self.run_once(); if wait.is_zero() { continue; } @@ -180,9 +184,10 @@ impl VideoThread { self.finish(); } - /// One pass of the loop body, at OBS clock `now_ns`. Returns how long to - /// sleep before the next pass; zero means "go round again immediately". - pub fn run_once(&mut self, now_ns: u64) -> Duration { + /// One pass of the loop body. Returns how long to sleep before the next + /// pass; zero means "go round again immediately". + pub fn run_once(&mut self) -> Duration { + let now_ns = self.now_ns(); if self.shared.video.take_clear() { // The receiver already dropped `video_queue`; the paced frames // behind it must go too, or the blank would be repainted a lead @@ -227,7 +232,7 @@ impl VideoThread { self.publish_counters(); // Fresh clock for the sleep: the emit above may have taken long // enough to make the next frame due already. - self.sleep_hint(obs::time::gettime_ns(), slack_ns) + self.sleep_hint(self.now_ns(), slack_ns) } /// Exit path: drop everything, including the decoder this thread owns. @@ -668,6 +673,10 @@ impl VideoThread { self.shared.conn.video_delay_ns.store(0, Relaxed); } + pub(crate) fn now_ns(&self) -> u64 { + (self.now_ns)() + } + /// The canvas tick, or the default when libobs has not reported one. fn canvas_tick_ns(&self) -> u64 { (self.canvas_tick_ns)().unwrap_or(consts::VIDEO_CANVAS_TICK_DEFAULT_NS) @@ -732,6 +741,14 @@ impl VideoThread { /* ── Test seams ───────────────────────────────────────── */ + /// Replace the OBS clock (tests only: `os_gettime_ns` is libobs's + /// monotonic clock and cannot be stepped from here). + #[must_use] + pub fn with_clock(mut self, now_ns: Box u64 + Send>) -> Self { + self.now_ns = now_ns; + self + } + /// Replace the canvas-tick source (tests only). #[must_use] pub fn with_canvas_tick(mut self, tick: Box Option + Send>) -> Self { diff --git a/crates/irl-source/tests/video_pipeline.rs b/crates/irl-source/tests/video_pipeline.rs index db33fc7..7fcfffb 100644 --- a/crates/irl-source/tests/video_pipeline.rs +++ b/crates/irl-source/tests/video_pipeline.rs @@ -16,7 +16,8 @@ mod common; use obs_irl_source::{shared, video}; use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering::Relaxed}; +use std::time::Duration; use ffmpeg::AVPixelFormat as Pix; use ffmpeg::sys::{AVColorRange, AVColorSpace, AVColorTransferCharacteristic}; @@ -97,13 +98,38 @@ fn shared() -> Arc { ) } -fn thread_with(shared: Arc) -> (VideoThread, Arc) { +/// The OBS clock the video thread reads, stepped by the test. +#[derive(Clone)] +struct Clock(Arc); + +impl Clock { + fn now(&self) -> u64 { + self.0.load(Relaxed) + } + + fn set(&self, now_ns: u64) { + self.0.store(now_ns, Relaxed); + } +} + +fn thread_with(shared: Arc) -> (VideoThread, Arc, Clock) { let recorder = Arc::new(Recorder::default()); + // Far enough from zero that the tests can stamp arrivals seconds in the + // past. + let clock = Clock(Arc::new(AtomicU64::new(100_000_000_000))); + let reads = clock.clone(); // No libobs is running, so the real canvas tick would fault; 60fps is // what a default OBS install reports. let thread = VideoThread::with_sink(shared, Box::new(RecorderSink(recorder.clone()))) - .with_canvas_tick(Box::new(|| Some(16_666_667))); - (thread, recorder) + .with_canvas_tick(Box::new(|| Some(16_666_667))) + .with_clock(Box::new(move || reads.now())); + (thread, recorder, clock) +} + +/// One pass of the video thread's loop with the clock at `now_ns`. +fn run_at(thread: &mut VideoThread, clock: &Clock, now_ns: u64) -> Duration { + clock.set(now_ns); + thread.run_once() } /// A software frame with real buffers. @@ -208,7 +234,7 @@ fn colour_spaces_map_the_way_the_c_did() { #[test] fn sysmem_frames_are_referenced_not_copied() { - let (mut thread, _recorder) = thread_with(shared()); + let (mut thread, _recorder, _) = thread_with(shared()); let frame = sw_frame(Pix::AV_PIX_FMT_YUV420P, 64, 32); let out = thread.to_sysmem(&frame).expect("system-memory passthrough"); @@ -229,7 +255,7 @@ fn sysmem_frames_are_referenced_not_copied() { #[test] fn a_mappable_frame_lends_its_planes_to_libobs() { - let (mut thread, recorder) = thread_with(shared()); + let (mut thread, recorder, _) = thread_with(shared()); let frame = sw_frame(Pix::AV_PIX_FMT_YUV420P, 64, 32); thread.output_frame(&frame, 1_234_567_890); @@ -252,7 +278,7 @@ fn a_mappable_frame_lends_its_planes_to_libobs() { #[test] fn an_unmappable_frame_arrives_as_nv12_from_the_scaler() { - let (mut thread, recorder) = thread_with(shared()); + let (mut thread, recorder, _) = thread_with(shared()); // 10-bit 4:4:4 is not in the OBS table, so it has to be converted. let frame = sw_frame(Pix::AV_PIX_FMT_YUV444P10LE, 64, 32); assert_eq!(output::avpixfmt_to_obs(frame.pix_fmt()), VideoFormat::None); @@ -456,16 +482,16 @@ fn a_resolution_change_re_anchors_the_video_clock() { #[test] fn a_frame_is_handed_over_a_lead_before_it_is_due() { let shared = shared(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); // Anchor the play head first: the very first frame out deliberately gets // no lead, so a second frame is needed to see one. The video-only fallback // schedules it at its arrival, so it is delayed by a tick first (see // `video_without_audio_is_delayed_by_a_tick_and_anchors_on_the_fallback`). - let t0 = obs::time::gettime_ns(); + let t0 = clock.now(); feed(&mut thread, t0 as i64, t0); let now = t0 + TICK; - thread.run_once(now); + run_at(&mut thread, &clock, now); assert_eq!(recorder.emitted().len(), 1, "anchor frame emitted"); // A frame due 25ms out: inside two 60fps ticks (33.3ms) of the lead, so @@ -473,7 +499,7 @@ fn a_frame_is_handed_over_a_lead_before_it_is_due() { // time is its PTS on the fallback epoch plus the delay: `t0 + 25 ms + // 1 tick`, 25 ms from `now`.) feed(&mut thread, t0 as i64 + 25_000_000, t0); - thread.run_once(now); + run_at(&mut thread, &clock, now); assert_eq!( recorder.emitted().len(), 2, @@ -482,7 +508,7 @@ fn a_frame_is_handed_over_a_lead_before_it_is_due() { // One due far out still waits. feed(&mut thread, t0 as i64 + 500_000_000, now); - let wait = thread.run_once(now); + let wait = run_at(&mut thread, &clock, now); assert_eq!(recorder.emitted().len(), 2, "not due, not delivered"); assert!(!wait.is_zero(), "should sleep toward the lead"); } @@ -498,7 +524,7 @@ fn a_frame_is_handed_over_a_lead_before_it_is_due() { #[test] fn a_full_pacing_queue_does_not_keep_the_video_thread_awake() { let shared = shared(); - let (mut thread, _recorder) = thread_with(shared.clone()); + let (mut thread, _recorder, clock) = thread_with(shared.clone()); // A backlog of packets, as any target above the decode lead produces. for i in 0..200 { @@ -507,7 +533,7 @@ fn a_full_pacing_queue_does_not_keep_the_video_thread_awake() { assert!(!shared.video.is_empty()); // Fill the pacing queue past its decode lead with frames due far ahead. - let now = obs::time::gettime_ns(); + let now = clock.now(); let mut pts = 0i64; while thread.pacing().has_room() { feed(&mut thread, now as i64 + 60_000_000_000 + pts, now); @@ -547,19 +573,19 @@ fn a_decoder_handover_is_taken_even_with_no_pacing_room() { #[test] fn a_queued_frame_is_transferred_paced_and_emitted() { let shared = shared(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); let mut queued = sw_frame(Pix::AV_PIX_FMT_YUV420P, 64, 32); // The receiver hands the queue nanosecond PTS. queued.set_pts(5_000_000_000); let source_plane = queued.plane(0).unwrap().as_ptr() as usize; - let now = obs::time::gettime_ns(); + let now = clock.now(); thread.pace_decoded(queued, now); // With no audio mapping the video-only anchor puts the first frame at its // arrival, and a tick's allowance is added as the standing delay: one // cycle at that due time takes it all the way out. - let wait = thread.run_once(now + TICK); + let wait = run_at(&mut thread, &clock, now + TICK); let emitted = recorder.only(); assert_eq!(emitted.planes[0].0, source_plane, "still zero-copy"); @@ -576,20 +602,20 @@ fn a_queued_frame_is_transferred_paced_and_emitted() { #[test] fn a_future_frame_waits_instead_of_being_emitted() { let shared = shared(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); - let now = obs::time::gettime_ns(); + let now = clock.now(); let mut queued = sw_frame(Pix::AV_PIX_FMT_YUV420P, 64, 32); // 200 ms into the future on the video-only anchor: the fallback anchors on // the first frame, so pace the *second* one forward. queued.set_pts(0); thread.pace_decoded(queued, now); let now = now + TICK; - thread.run_once(now); + run_at(&mut thread, &clock, now); assert_eq!(recorder.emitted().len(), 1, "the anchor frame goes out"); feed(&mut thread, 200_000_000, now); - let wait = thread.run_once(now); + let wait = run_at(&mut thread, &clock, now); assert_eq!(recorder.emitted().len(), 1, "not due yet"); assert_eq!(thread.pacing().len(), 1); @@ -604,20 +630,20 @@ fn a_future_frame_waits_instead_of_being_emitted() { #[test] fn a_clear_request_drops_the_queue_and_blanks_the_source() { let shared = shared(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); - let now = obs::time::gettime_ns(); + let now = clock.now(); feed(&mut thread, 0, now); let now = now + TICK; - thread.run_once(now); + run_at(&mut thread, &clock, now); assert_eq!(recorder.emitted().len(), 1); feed(&mut thread, 10_000_000_000, now); - thread.run_once(now); + run_at(&mut thread, &clock, now); assert_eq!(thread.pacing().len(), 1, "parked until its due time"); shared.video.request_clear(); - let wait = thread.run_once(now); + let wait = run_at(&mut thread, &clock, now); assert_eq!(recorder.cleared.load(Relaxed), 1); assert!( @@ -636,18 +662,18 @@ fn a_clear_request_drops_the_queue_and_blanks_the_source() { #[test] fn queued_frames_reschedule_onto_the_audio_playout_offset() { let shared = shared(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); // Audio that ends at stream PTS 10 s plays out at OBS time `now + 1 s`: // everything maps 1 s into the future minus 10 s of stream time. - let now = obs::time::gettime_ns(); + let now = clock.now(); { let mut state = shared.audio_state(); state.mapping.publish(now + 1_000_000_000, 10_000_000_000); } feed(&mut thread, 10_000_000_000, now); - thread.run_once(now); + run_at(&mut thread, &clock, now); assert!(recorder.emitted().is_empty(), "due a second from now"); let due = thread.pacing().next_due().expect("paced"); @@ -659,7 +685,7 @@ fn queued_frames_reschedule_onto_the_audio_playout_offset() { let mut state = shared.audio_state(); state.mapping.publish(now + 500_000_000, 10_000_000_000); } - thread.run_once(now); + run_at(&mut thread, &clock, now); assert_eq!(thread.pacing().next_due(), Some(now + 500_000_000)); // Once the offset says "now", the frame has no margin left at all: a @@ -669,17 +695,17 @@ fn queued_frames_reschedule_onto_the_audio_playout_offset() { let mut state = shared.audio_state(); state.mapping.publish(now, 10_000_000_000); } - thread.run_once(now); + run_at(&mut thread, &clock, now); assert!(recorder.emitted().is_empty()); assert_eq!(thread.pacing().next_due(), Some(now + TICK)); - thread.run_once(now + TICK); + run_at(&mut thread, &clock, now + TICK); assert_eq!(recorder.only().timestamp, now + TICK); } #[test] fn clearing_the_ts_init_mirror_re_anchors_the_fallback_clock() { let shared = shared(); - let (mut thread, _recorder) = thread_with(shared.clone()); + let (mut thread, _recorder, _) = thread_with(shared.clone()); // No audio mapping, so the video-only anchor runs: the first frame anchors // the epoch at `now`. @@ -709,9 +735,9 @@ fn clearing_the_ts_init_mirror_re_anchors_the_fallback_clock() { #[test] fn a_frame_is_due_where_the_audio_mapping_puts_it() { let shared = shared(); - let (mut thread, _recorder) = thread_with(shared.clone()); + let (mut thread, _recorder, clock) = thread_with(shared.clone()); - let now = obs::time::gettime_ns(); + let now = clock.now(); { let mut state = shared.audio_state(); // A frame at stream PTS 0 maps two seconds into the future. @@ -761,11 +787,11 @@ fn publish_mapping(shared: &Shared, obs_end_ts_ns: u64, buffered_end_pts_ns: i64 /// Anchor the play head on a frame at stream PTS 10 s that arrives at the /// returned OBS time and maps 100 ms out: a comfortable margin. -fn anchor_with_margin(shared: &Shared, thread: &mut VideoThread) -> u64 { - let t0 = obs::time::gettime_ns(); +fn anchor_with_margin(shared: &Shared, thread: &mut VideoThread, clock: &Clock) -> u64 { + let t0 = clock.now(); publish_mapping(shared, t0 + 100_000_000, 10_000_000_000); feed(thread, 10_000_000_000, t0); - thread.run_once(t0 + 100_000_000); + run_at(thread, clock, t0 + 100_000_000); t0 } @@ -786,13 +812,13 @@ fn shared_with_audio() -> Arc { #[test] fn video_waits_for_the_audio_mapping_before_anchoring_the_play_head() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); - let now = obs::time::gettime_ns(); + let now = clock.now(); feed(&mut thread, 10_000_000_000, now); // Well past the fallback due time (+120 ms): still held. - let wait = thread.run_once(now + 300_000_000); + let wait = run_at(&mut thread, &clock, now + 300_000_000); assert!( recorder.emitted().is_empty(), "no mapping yet, nothing may anchor" @@ -801,9 +827,9 @@ fn video_waits_for_the_audio_mapping_before_anchoring_the_play_head() { // Audio primes: the chunk ending at stream PTS 10 s plays at +400 ms. publish_mapping(&shared, now + 400_000_000, 10_000_000_000); - thread.run_once(now + 350_000_000); + run_at(&mut thread, &clock, now + 350_000_000); assert!(recorder.emitted().is_empty(), "due at +400 ms, not before"); - thread.run_once(now + 400_000_000); + run_at(&mut thread, &clock, now + 400_000_000); assert_eq!(recorder.only().timestamp, now + 400_000_000); } @@ -816,11 +842,11 @@ fn video_waits_for_the_audio_mapping_before_anchoring_the_play_head() { #[test] fn frames_already_past_due_when_the_mapping_arrives_do_not_anchor_the_play_head() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); // 25 fps: 40 ms apart, so no two frames fall inside one 60fps tick, // arriving in real time over the 160 ms before the mapping. - let now = obs::time::gettime_ns(); + let now = clock.now(); for i in 0..4 { feed( &mut thread, @@ -828,14 +854,14 @@ fn frames_already_past_due_when_the_mapping_arrives_do_not_anchor_the_play_head( now - 160_000_000 + i as u64 * 40_000_000, ); } - thread.run_once(now); + run_at(&mut thread, &clock, now); assert!(recorder.emitted().is_empty()); assert_eq!(thread.pacing().len(), 4); // Audio primes so that the first three frames map 100, 60 and 20 ms into // the past — all past a canvas tick — and the fourth lands 20 ms out. publish_mapping(&shared, now - 100_000_000, 10_000_000_000); - thread.run_once(now); + run_at(&mut thread, &clock, now); assert!( recorder.emitted().is_empty(), "the stale frames must not go out in place of the on-time one" @@ -843,7 +869,7 @@ fn frames_already_past_due_when_the_mapping_arrives_do_not_anchor_the_play_head( assert_eq!(thread.pacing().len(), 1, "three stale frames dropped"); assert_eq!(shared.conn.video_delay_ns.load(Relaxed), 0); - thread.run_once(now + 20_000_000); + run_at(&mut thread, &clock, now + 20_000_000); assert_eq!(recorder.only().timestamp, now + 20_000_000); } @@ -852,13 +878,13 @@ fn frames_already_past_due_when_the_mapping_arrives_do_not_anchor_the_play_head( #[test] fn a_frame_late_by_under_a_canvas_tick_still_anchors() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); - let now = obs::time::gettime_ns(); + let now = clock.now(); feed(&mut thread, 10_000_000_000, now - 50_000_000); publish_mapping(&shared, now, 10_000_000_000); - thread.run_once(now + 10_000_000); + run_at(&mut thread, &clock, now + 10_000_000); assert_eq!( recorder.only().timestamp, now, @@ -871,22 +897,22 @@ fn a_frame_late_by_under_a_canvas_tick_still_anchors() { #[test] fn video_stops_waiting_for_audio_that_never_primes() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); - let now = obs::time::gettime_ns(); + let now = clock.now(); feed(&mut thread, 10_000_000_000, now); - thread.run_once(now); + run_at(&mut thread, &clock, now); assert!(recorder.emitted().is_empty()); // warm-up 150 + target 120 + lead 80 + the audio hold's wait for a skew // reading 2000 + margin 1000. let give_up = now + 3_350_000_000; - thread.run_once(give_up - 1_000_000); + run_at(&mut thread, &clock, give_up - 1_000_000); assert!(recorder.emitted().is_empty(), "still inside the wait"); // Past it: the held frame is stale by more than a tick and is dropped, // and a fresh frame goes out on the fallback at its own due time. - thread.run_once(give_up + 1_000_000); + run_at(&mut thread, &clock, give_up + 1_000_000); assert!(recorder.emitted().is_empty()); assert_eq!( thread.pacing().len(), @@ -894,18 +920,9 @@ fn video_stops_waiting_for_audio_that_never_primes() { "the stale frame was dropped, not anchored" ); - // The fallback schedules against the real clock (`due_time` reads it), - // so the frame's arrival has to be on that clock too: stamped with the - // test's 1.35 s of pretend time it would look a second late, which in - // production cannot happen (a packet never arrives after a due time the - // fallback capped at now + 200 ms). - feed( - &mut thread, - 10_000_000_000 + 1_400_000_000, - obs::time::gettime_ns(), - ); + feed(&mut thread, 10_000_000_000 + 1_400_000_000, clock.now()); let due = thread.pacing().next_due().expect("paced on the fallback"); - thread.run_once(due); + run_at(&mut thread, &clock, due); assert_eq!(recorder.only().timestamp, due); } @@ -917,25 +934,20 @@ fn video_stops_waiting_for_audio_that_never_primes() { #[test] fn video_without_audio_is_delayed_by_a_tick_and_anchors_on_the_fallback() { let shared = shared(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); - let t0 = obs::time::gettime_ns(); + let t0 = clock.now(); feed(&mut thread, t0 as i64, t0); - thread.run_once(t0); + run_at(&mut thread, &clock, t0); assert!( recorder.emitted().is_empty(), "no audio to wait for, but not decodable in time either" ); assert_eq!(shared.conn.video_delay_ns.load(Relaxed), TICK); - // The fallback anchors on the clock as `due_time` read it, a hair after - // `t0`, so the due time is a tick past that rather than past `t0`. let due = thread.pacing().next_due().expect("paced"); - assert!( - due >= t0 + TICK && due < t0 + TICK + 1_000_000, - "due {due} vs t0 {t0}" - ); + assert_eq!(due, t0 + TICK); - thread.run_once(due); + run_at(&mut thread, &clock, due); assert_eq!(recorder.only().timestamp, due); } @@ -975,19 +987,19 @@ fn video_that_trails_its_audio_is_delayed_into_pacing_not_dropped() { for canvas_fps in [30u64, 60] { let tick = 1_000_000_000 / canvas_fps; let shared = shared_with_audio(); - let (thread, recorder) = thread_with(shared.clone()); + let (thread, recorder, clock) = thread_with(shared.clone()); let mut thread = thread.with_canvas_tick(Box::new(move || Some(tick))); // Audio has primed such that a video frame arriving now maps 150 ms // into the past, and every later frame likewise. - let t0 = obs::time::gettime_ns(); + let t0 = clock.now(); let late = 150_000_000; publish_mapping(&shared, t0 - late, 10_000_000_000); for i in 0..30u64 { let arrival = t0 + i * FRAME; feed(&mut thread, 10_000_000_000 + (i * FRAME) as i64, arrival); - thread.run_once(arrival); + run_at(&mut thread, &clock, arrival); } // 150 ms of lateness plus a tick of decode allowance, in whole ticks: @@ -1029,18 +1041,18 @@ fn video_that_trails_its_audio_is_delayed_into_pacing_not_dropped() { #[test] fn a_stale_startup_burst_does_not_raise_the_delay() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); // Twenty 30fps frames from the probe, all pushed at `t0`, mapping from // 593 ms in the past up to one 40 ms out, with the one before it 6.7 ms // out: on time, but only just. - let t0 = obs::time::gettime_ns(); + let t0 = clock.now(); let last_due = t0 + 40_000_000; publish_mapping(&shared, last_due, 10_000_000_000 + 19 * FRAME as i64); for i in 0..20i64 { feed(&mut thread, 10_000_000_000 + i * FRAME as i64, t0); } - thread.run_once(t0); + run_at(&mut thread, &clock, t0); assert_eq!(thread.pacing().len(), 2, "eighteen stale frames dropped"); assert_eq!( @@ -1048,7 +1060,7 @@ fn a_stale_startup_burst_does_not_raise_the_delay() { 0, "buffered frames are not evidence of a late sender" ); - thread.run_once(last_due - FRAME); + run_at(&mut thread, &clock, last_due - FRAME); let emitted = recorder.emitted(); assert_eq!( emitted[0].timestamp, @@ -1064,23 +1076,23 @@ fn a_stale_startup_burst_does_not_raise_the_delay() { #[test] fn a_late_stream_is_measured_on_its_newest_frame() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); // Frames that arrived in real time over the last lookback, each mapping // 150 ms before its arrival, the newest one now. - let t0 = obs::time::gettime_ns(); + let t0 = clock.now(); let late = 150_000_000; publish_mapping(&shared, t0 - late, 10_000_000_000); live_history(&mut thread, t0, 10_000_000_000); feed(&mut thread, 10_000_000_000, t0); - thread.run_once(t0); + run_at(&mut thread, &clock, t0); // 150 ms plus a tick, in ticks, from the newest frame; the older ones // were stale even after that and went. let delay = 10 * TICK; assert_eq!(shared.conn.video_delay_ns.load(Relaxed), delay); assert_eq!(thread.pacing().len(), 1); - thread.run_once(t0 - late + delay); + run_at(&mut thread, &clock, t0 - late + delay); assert_eq!(recorder.only().timestamp, t0 - late + delay); } @@ -1093,10 +1105,10 @@ fn a_late_stream_is_measured_on_its_newest_frame() { #[test] fn recurring_lateness_after_the_anchor_raises_the_delay_once() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); // A frame arriving now maps 100 ms out: a comfortable margin to anchor on. - let t0 = anchor_with_margin(&shared, &mut thread); + let t0 = anchor_with_margin(&shared, &mut thread, &clock); assert_eq!(recorder.emitted().len(), 1, "anchored with a full margin"); assert_eq!(shared.conn.video_delay_ns.load(Relaxed), 0); @@ -1110,7 +1122,7 @@ fn recurring_lateness_after_the_anchor_raises_the_delay_once() { 10_000_000_000 + (i * FRAME) as i64, arrival_of(i), ); - thread.run_once(arrival_of(i)); + run_at(&mut thread, &clock, arrival_of(i)); delay_seen.push(shared.conn.video_delay_ns.load(Relaxed)); } @@ -1138,9 +1150,9 @@ fn recurring_lateness_after_the_anchor_raises_the_delay_once() { #[test] fn frames_handed_over_short_of_the_lead_but_before_due_do_not_raise_the_delay() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); - let t0 = anchor_with_margin(&shared, &mut thread); + let t0 = anchor_with_margin(&shared, &mut thread, &clock); assert_eq!(recorder.emitted().len(), 1); // Two seconds of frames with 3 ms in hand: a tenth of the lead. @@ -1151,7 +1163,7 @@ fn frames_handed_over_short_of_the_lead_but_before_due_do_not_raise_the_delay() 10_000_000_000 + (i * FRAME) as i64, arrival_of(i), ); - thread.run_once(arrival_of(i)); + run_at(&mut thread, &clock, arrival_of(i)); } assert_eq!(shared.conn.video_delay_ns.load(Relaxed), 0); @@ -1163,9 +1175,9 @@ fn frames_handed_over_short_of_the_lead_but_before_due_do_not_raise_the_delay() #[test] fn a_single_late_burst_after_the_anchor_does_not_raise_the_delay() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); - let t0 = anchor_with_margin(&shared, &mut thread); + let t0 = anchor_with_margin(&shared, &mut thread, &clock); assert_eq!(recorder.emitted().len(), 1); let due_of = |i: u64| t0 + 100_000_000 + i * FRAME; @@ -1176,7 +1188,7 @@ fn a_single_late_burst_after_the_anchor_does_not_raise_the_delay() { 10_000_000_000 + (i * FRAME) as i64, due_of(i) + 20_000_000, ); - thread.run_once(due_of(i) + 20_000_000); + run_at(&mut thread, &clock, due_of(i) + 20_000_000); } // ... then a second and a half of frames with their full margin. for i in 7..=50u64 { @@ -1185,9 +1197,9 @@ fn a_single_late_burst_after_the_anchor_does_not_raise_the_delay() { 10_000_000_000 + (i * FRAME) as i64, due_of(i) - 100_000_000, ); - thread.run_once(due_of(i) - 100_000_000); + run_at(&mut thread, &clock, due_of(i) - 100_000_000); } - thread.run_once(due_of(50)); + run_at(&mut thread, &clock, due_of(50)); assert_eq!(shared.conn.video_delay_ns.load(Relaxed), 0); assert_eq!( @@ -1203,24 +1215,24 @@ fn a_single_late_burst_after_the_anchor_does_not_raise_the_delay() { #[test] fn a_clear_forgets_the_video_delay() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); - let t0 = obs::time::gettime_ns(); + let t0 = clock.now(); publish_mapping(&shared, t0 - 150_000_000, 10_000_000_000); live_history(&mut thread, t0, 10_000_000_000); feed(&mut thread, 10_000_000_000, t0); - thread.run_once(t0); + run_at(&mut thread, &clock, t0); assert_eq!(shared.conn.video_delay_ns.load(Relaxed), 10 * TICK); shared.video.request_clear(); let t1 = t0 + 5_000_000_000; - thread.run_once(t1); + run_at(&mut thread, &clock, t1); assert_eq!(shared.conn.video_delay_ns.load(Relaxed), 0); // The next connection's audio primes with room to spare: no delay. publish_mapping(&shared, t1 + 200_000_000, 20_000_000_000); feed(&mut thread, 20_000_000_000, t1); - thread.run_once(t1 + 200_000_000); + run_at(&mut thread, &clock, t1 + 200_000_000); assert_eq!( recorder.emitted().last().map(|f| f.timestamp), Some(t1 + 200_000_000) @@ -1239,19 +1251,19 @@ fn a_clear_forgets_the_video_delay() { #[test] fn video_later_than_the_delay_ceiling_anchors_and_plays_unpaced() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); let max = irl_core::consts::VIDEO_DELAY_MAX_MS * 1_000_000; // A frame arriving now maps six seconds into the past, past the ceiling, // and every later one too. - let t0 = obs::time::gettime_ns(); + let t0 = clock.now(); let late = 6_000_000_000; publish_mapping(&shared, t0 - late, 10_000_000_000); for i in 0..60u64 { let arrival = t0 + i * FRAME; feed(&mut thread, 10_000_000_000 + (i * FRAME) as i64, arrival); - thread.run_once(arrival); + run_at(&mut thread, &clock, arrival); assert_eq!( thread.pacing().len(), 0, @@ -1289,14 +1301,14 @@ fn video_later_than_the_delay_ceiling_anchors_and_plays_unpaced() { #[test] fn a_delay_set_by_a_bad_first_frame_ramps_back_without_a_jump() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); // The first frame in hand maps 1.3 s into the past. - let t0 = obs::time::gettime_ns(); + let t0 = clock.now(); publish_mapping(&shared, t0 - 1_300_000_000, 10_000_000_000); live_history(&mut thread, t0, 10_000_000_000); feed(&mut thread, 10_000_000_000, t0); - thread.run_once(t0); + run_at(&mut thread, &clock, t0); let set = shared.conn.video_delay_ns.load(Relaxed); assert_eq!(set, (1_300_000_000 + TICK).div_ceil(TICK) * TICK); @@ -1308,11 +1320,11 @@ fn a_delay_set_by_a_bad_first_frame_ramps_back_without_a_jump() { for i in 1..=frames { let arrival = t0 + i * FRAME; feed(&mut thread, pts_of(i), arrival); - thread.run_once(arrival); + run_at(&mut thread, &clock, arrival); delays.push(shared.conn.video_delay_ns.load(Relaxed)); } for k in 1..=60u64 { - thread.run_once(t0 + (frames + k) * FRAME); + run_at(&mut thread, &clock, t0 + (frames + k) * FRAME); } // Untouched for the ten-second window, then down to a tick of headroom, @@ -1353,15 +1365,15 @@ fn a_delay_set_by_a_bad_first_frame_ramps_back_without_a_jump() { #[test] fn the_video_delay_hands_itself_back_as_the_audio_hold_builds() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); // Every frame maps 300 ms into the past. - let t0 = obs::time::gettime_ns(); + let t0 = clock.now(); let late = 300_000_000; publish_mapping(&shared, t0 - late, 10_000_000_000); live_history(&mut thread, t0, 10_000_000_000); feed(&mut thread, 10_000_000_000, t0); - thread.run_once(t0); + run_at(&mut thread, &clock, t0); let set = shared.conn.video_delay_ns.load(Relaxed); assert_eq!(set, (late + TICK).div_ceil(TICK) * TICK); @@ -1376,12 +1388,12 @@ fn the_video_delay_hands_itself_back_as_the_audio_hold_builds() { shared.audio_state().hold_built_ns = grown; let arrival = t0 + i * FRAME; feed(&mut thread, 10_000_000_000 + (i * FRAME) as i64, arrival); - thread.run_once(arrival); + run_at(&mut thread, &clock, arrival); delays.push(shared.conn.video_delay_ns.load(Relaxed)); } // By now video is in hand early; let what is queued go out. for k in 1..=30u64 { - thread.run_once(t0 + (frames + k) * FRAME); + run_at(&mut thread, &clock, t0 + (frames + k) * FRAME); } assert!( @@ -1423,11 +1435,11 @@ fn the_video_delay_hands_itself_back_as_the_audio_hold_builds() { #[test] fn video_catching_up_to_live_at_connection_start_sets_no_delay() { let shared = shared_with_audio(); - let (mut thread, recorder) = thread_with(shared.clone()); + let (mut thread, recorder, clock) = thread_with(shared.clone()); // 39 frames, 1.3 s of video, arrive at four times real time; from then // on frames arrive live, 100 ms before they are due. - let t0 = obs::time::gettime_ns(); + let t0 = clock.now(); let caught_up = 39u64; let live_from = t0 + (caught_up - 1) * FRAME / 4; let arrival_of = |i: u64| { @@ -1447,7 +1459,7 @@ fn video_catching_up_to_live_at_connection_start_sets_no_delay() { for i in 0..caught_up + 90 { feed(&mut thread, pts_of(i), arrival_of(i)); - thread.run_once(arrival_of(i)); + run_at(&mut thread, &clock, arrival_of(i)); assert_eq!( shared.conn.video_delay_ns.load(Relaxed), 0, From 65da6d9ec3c5d4e9d418ca1518c92731111c29bd Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 04:08:27 +0200 Subject: [PATCH 05/13] test(irl-source): end-to-end a/v sync simulation across pump and video thread --- crates/irl-source/src/receiver/decode.rs | 24 +- crates/irl-source/src/source.rs | 2 +- crates/irl-source/src/video/thread.rs | 30 +- crates/irl-source/tests/av_sync_sim.rs | 1246 ++++++++++++++++++++++ crates/irl-source/tests/common/mod.rs | 36 + crates/irl-source/tests/network_sim.rs | 33 +- scripts/test-with-libobs-shim.sh | 2 +- 7 files changed, 1330 insertions(+), 43 deletions(-) create mode 100644 crates/irl-source/tests/av_sync_sim.rs diff --git a/crates/irl-source/src/receiver/decode.rs b/crates/irl-source/src/receiver/decode.rs index d8e3c95..5389634 100644 --- a/crates/irl-source/src/receiver/decode.rs +++ b/crates/irl-source/src/receiver/decode.rs @@ -151,15 +151,12 @@ impl Receiver { ffmpeg::rescale_q(pts, self.video_tb, ffmpeg::NS_TIME_BASE) }); let bytes = self.pkt.size().max(0) as usize; - self.shared.conn.video_arrival_pts_ns.store(pts_ns, Relaxed); let received_ns = obs::time::gettime_ns(); - - // Where the audio hold reads the sender's skew: this packet against - // the newest audio decoded before it, in mux order. - if let Some(dts) = self.pkt.dts_or_pts() { - let dts_ns = ffmpeg::rescale_q(dts, self.video_tb, ffmpeg::NS_TIME_BASE); - crate::audio::hold::observe_video_packet(&self.shared, received_ns, dts_ns); - } + let dts_ns = self + .pkt + .dts_or_pts() + .map(|dts| ffmpeg::rescale_q(dts, self.video_tb, ffmpeg::NS_TIME_BASE)); + note_video_arrival(&self.shared, received_ns, pts_ns, dts_ns); match self.pkt.new_ref() { Ok(packet) => self.shared.video.push_packet( @@ -178,3 +175,14 @@ impl Receiver { } } } + +/// What the receiver reads off a video packet as it arrives, before queueing +/// it: its PTS for the skew stat, and its decode timestamp for the audio +/// hold, which reads the sender's skew against the newest audio decoded +/// before it, in mux order. +pub fn note_video_arrival(shared: &Shared, received_ns: u64, pts_ns: i64, dts_ns: Option) { + shared.conn.video_arrival_pts_ns.store(pts_ns, Relaxed); + if let Some(dts_ns) = dts_ns { + audio::hold::observe_video_packet(shared, received_ns, dts_ns); + } +} diff --git a/crates/irl-source/src/source.rs b/crates/irl-source/src/source.rs index e5dffc3..b7ddc4c 100644 --- a/crates/irl-source/src/source.rs +++ b/crates/irl-source/src/source.rs @@ -461,7 +461,7 @@ fn fit_to_canvas(source: SourceHandle) { /// the sender's skew before any buffering here. The decoded-frame PTS would /// trail arrival by the queue depth and read the video delay as skew. Zero /// until both streams have delivered something. -pub(crate) fn av_skew_ms(shared: &Shared, audio: &AudioState) -> i64 { +pub fn av_skew_ms(shared: &Shared, audio: &AudioState) -> i64 { let video_pts_ns = shared.conn.video_arrival_pts_ns.load(Relaxed); if video_pts_ns == 0 || audio.latest_audio_stream_pts_ns == 0 { return 0; diff --git a/crates/irl-source/src/video/thread.rs b/crates/irl-source/src/video/thread.rs index 56217c7..281042e 100644 --- a/crates/irl-source/src/video/thread.rs +++ b/crates/irl-source/src/video/thread.rs @@ -20,10 +20,10 @@ use irl_core::consts; use irl_core::pacing::{DueVerdict, PacingQueue}; use irl_core::video_delay::{DelayRaise, DelayRelax, VideoDelay}; -use crate::shared::{Shared, VideoDecoder, VideoMsg}; +use crate::shared::{Shared, TimedPacket, VideoDecoder, VideoMsg}; use crate::video::VideoSink; use crate::video::decode; -use crate::video::intake::DecodeState; +use crate::video::intake::{self, DecodeState}; /// A frame waiting for its due time. `received_ns` is when the packet it was /// decoded from reached this thread, which is what its arrival margin is @@ -44,6 +44,11 @@ impl Paced { } } +/// Turns a queued packet into the frame it decodes to, in place of a decoder: +/// stamped in nanoseconds, it then goes through the same intake a decoded +/// H.264 frame does. +pub type DecodeStub = Box Option + Send>; + /// The video thread's wait for the audio playout mapping before it anchors /// libobs's play head; see [`VideoThread::awaiting_audio_mapping`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -66,6 +71,8 @@ pub struct VideoThread { /// The video decoder, handed over by the receiver when a connection opens. /// Owned here because *when* a packet is decoded is this thread's decision. decoder: Option, + /// Stands in for the decoder in tests; see [`Self::with_decode_stub`]. + decode_stub: Option, /// Reusable destination for `receive_frame`. scratch: Frame, /// Reusable output list for one packet's frames. @@ -133,6 +140,7 @@ impl VideoThread { Self { shared, decoder: None, + decode_stub: None, scratch: Frame::new().expect("frame allocation"), decoded: Vec::new(), state: DecodeState::default(), @@ -284,6 +292,16 @@ impl VideoThread { &packet.packet, &mut produced, ); + } else if let Some(frame) = + self.decode_stub.as_mut().and_then(|stub| stub(&packet)) + { + produced.extend(intake::handle_frame( + &shared, + &mut self.state, + &frame, + ffmpeg::NS_TIME_BASE, + ffmpeg::AVCodecID::AV_CODEC_ID_H264, + )); } for frame in produced.drain(..) { self.pace_decoded(frame, received_ns); @@ -756,6 +774,14 @@ impl VideoThread { self } + /// Decode packets with `stub` while no decoder is installed (tests only: + /// the bundled FFmpeg carries no decoder a synthetic packet could feed). + #[must_use] + pub fn with_decode_stub(mut self, stub: DecodeStub) -> Self { + self.decode_stub = Some(stub); + self + } + /// The pacing queue, read-only (test seam). #[doc(hidden)] pub fn pacing(&self) -> &PacingQueue { diff --git a/crates/irl-source/tests/av_sync_sim.rs b/crates/irl-source/tests/av_sync_sim.rs new file mode 100644 index 0000000..25342c5 --- /dev/null +++ b/crates/irl-source/tests/av_sync_sim.rs @@ -0,0 +1,1246 @@ +//! End-to-end A/V sync simulation: the receiver's audio intake and video +//! arrival bookkeeping, the audio pump and the video thread, all driven on one +//! virtual clock against a synthetic sender. +//! +//! What it measures that nothing else in the suite does is the offset between +//! picture and sound as libobs is handed them. Both ends carry their own +//! truth, so neither is read through the playout mapping the video thread +//! schedules by: +//! +//! - every audio sample carries its own index on the sender's timeline +//! ([`encode_sample`]), so the OBS time at which content PTS `p` plays is +//! read off the submitted PCM, through speed changes, concealment and +//! re-anchors; +//! - every video frame carries its PTS in its first luma bytes, so the frame +//! libobs received is known whatever timestamp it was given. +//! +//! The offset of a frame is when libobs shows it (its timestamp, or its +//! hand-over if that was later) minus when the audio of the same instant +//! plays. Positive is picture behind sound. +//! +//! **What it does not cover.** The demuxer and the decoder, as in +//! `network_sim.rs`. Video packets go through the real channel and the video +//! thread's real intake loop, and a stub turns each packet into its frame. + +mod common; + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; + +use parking_lot::Mutex; + +use irl_core::consts; +use obs_irl_source::audio::{AudioPump, AudioSink}; +use obs_irl_source::config::Config; +use obs_irl_source::receiver::ReceiverFlags; +use obs_irl_source::receiver::audio_in::AudioIntake; +use obs_irl_source::receiver::decode::note_video_arrival; +use obs_irl_source::shared::{Shared, TimedPacket}; +use obs_irl_source::source::av_skew_ms; +use obs_irl_source::video::VideoSink; +use obs_irl_source::video::thread::VideoThread; + +use common::{CHANNELS, RATE}; + +const MS: u64 = 1_000_000; +const SEC: u64 = 1_000_000_000; + +/// OBS clock when the connection opens. Far enough from zero for a relay's +/// backlog, captured before it, to have a capture time. +const START_NS: u64 = 10 * SEC; +/// The sender's first audio PTS. Not zero: the playout mapping treats a zero +/// PTS as "no PTS". +const FIRST_PTS_NS: i64 = 10_000_000_000; +/// An AAC frame. +const AUDIO_FRAMES: i64 = 1024; +/// Network latency both streams share; it moves nothing relative to the +/// other, but keeps arrival after capture. +const NET_NS: u64 = 20 * MS; + +/// Lip sync people do not notice: ITU-R BT.1359 puts the detectability +/// threshold at 45 ms of sound ahead of picture, which is the direction every +/// error here runs in. The bar for "in sync" wherever the design promises it. +const SYNC_TOLERANCE_NS: i64 = 45 * MS as i64; + +/// Exact PTS of sample `n` on the sender's timeline. +fn sample_pts_ns(n: i64) -> i64 { + n * 1_000_000_000 / i64::from(RATE) +} + +fn secs(ns: u64) -> f64 { + ns as f64 / SEC as f64 +} + +/// A video lag that moves linearly between `(seconds, ms)` points and holds +/// its ends. +fn lag_profile(points: &'static [(f64, f64)]) -> impl Fn(u64) -> u64 { + move |capture_ns| { + let t = secs(capture_ns); + let ms = match points.iter().position(|&(at, _)| at > t) { + Some(0) => points[0].1, + Some(i) => { + let ((t0, a), (t1, b)) = (points[i - 1], points[i]); + a + (b - a) * (t - t0) / (t1 - t0) + } + None => points.last().map_or(0.0, |p| p.1), + }; + (ms * MS as f64) as u64 + } +} + +// ── The sender ──────────────────────────────────────────────── + +/// Which streams an outage touches. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Streams { + Both, + Video, +} + +/// An outage of the link. A stall holds everything that would have arrived +/// inside the window and hands it over at the end, as a burst; a loss drops +/// it. +#[derive(Clone, Copy, Debug)] +struct Stall { + at_ns: u64, + len_ns: u64, + streams: Streams, + lost: bool, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Media { + /// One decoded audio frame starting at sample index `n`. + Audio(i64), + /// One video packet stamped `pts_ns`. + Video(i64), +} + +#[derive(Clone, Copy, Debug)] +struct Arrival { + at_ns: u64, + capture_ns: u64, + media: Media, +} + +/// A change made to the running source. +enum Event { + /// The user edits Target Buffer. + TargetBuffer(i32), +} + +/// One scenario: what the sender does, and the source's settings. +struct Scenario { + secs: u64, + target_ms: i32, + low_latency: bool, + audio: bool, + fps: u64, + canvas_fps: u64, + /// How far behind its audio the sender's video reaches the plugin, by + /// capture time since the connection opened. + video_lag: Box u64>, + stalls: Vec, + /// A relay replaying video from its last keyframe: how far behind live + /// the replay starts, and how many times faster than real time it runs. + relay_backlog: Option<(u64, u64)>, + events: Vec<(u64, Event)>, +} + +impl Scenario { + /// A clean link at the default Target Buffer: 30 fps on a 60 fps canvas. + fn new(secs: u64) -> Self { + Self { + secs, + target_ms: consts::DEFAULT_BUFFER_TARGET_MS as i32, + low_latency: false, + audio: true, + fps: 30, + canvas_fps: 60, + video_lag: Box::new(|_| 0), + stalls: Vec::new(), + relay_backlog: None, + events: Vec::new(), + } + } + + fn fps(mut self, fps: u64) -> Self { + self.fps = fps; + self + } + + fn low_latency(mut self) -> Self { + self.low_latency = true; + self + } + + fn video_only(mut self) -> Self { + self.audio = false; + self + } + + fn video_lag(mut self, lag: impl Fn(u64) -> u64 + 'static) -> Self { + self.video_lag = Box::new(lag); + self + } + + fn stall(mut self, at_s: f64, len_ms: u64, streams: Streams) -> Self { + self.stalls.push(Stall { + at_ns: (at_s * SEC as f64) as u64, + len_ns: len_ms * MS, + streams, + lost: false, + }); + self + } + + fn loss(mut self, at_s: f64, len_ms: u64) -> Self { + self.stalls.push(Stall { + at_ns: (at_s * SEC as f64) as u64, + len_ns: len_ms * MS, + streams: Streams::Both, + lost: true, + }); + self + } + + fn relay_backlog(mut self, backlog_ms: u64, speedup: u64) -> Self { + self.relay_backlog = Some((backlog_ms * MS, speedup)); + self + } + + fn at(mut self, at_s: f64, event: Event) -> Self { + self.events.push(((at_s * SEC as f64) as u64, event)); + self + } + + fn frame_ns(&self) -> u64 { + SEC / self.fps + } + + fn canvas_tick_ns(&self) -> u64 { + SEC / self.canvas_fps + } + + /// When a packet that would reach the plugin at `at_ns` actually does + /// once the outages have had it, or `None` if one dropped it. + fn through_stalls(&self, at_ns: u64, video: bool) -> Option { + let mut at = at_ns; + for stall in &self.stalls { + let applies = video || stall.streams == Streams::Both; + let start = START_NS + stall.at_ns; + if applies && (start..start + stall.len_ns).contains(&at) { + if stall.lost { + return None; + } + at = start + stall.len_ns; + } + } + Some(at) + } + + /// Every packet the sender's link delivers, in arrival order: mux order, + /// so a burst lands in capture order. + fn arrivals(&self) -> Vec { + let end_ns = self.secs * SEC; + let mut out = Vec::new(); + + if self.audio { + let first = FIRST_PTS_NS * i64::from(RATE) / 1_000_000_000; + let mut last_at = 0; + for n in (first..).step_by(AUDIO_FRAMES as usize) { + let capture = (sample_pts_ns(n) - FIRST_PTS_NS) as u64; + if capture >= end_ns { + break; + } + let Some(at) = self.through_stalls(START_NS + capture + NET_NS, false) else { + continue; + }; + last_at = at.max(last_at); + out.push(Arrival { + at_ns: last_at, + capture_ns: START_NS + capture, + media: Media::Audio(n), + }); + } + } + + let frame = self.frame_ns(); + let backlog = self.relay_backlog.map_or(0, |(ns, _)| ns.div_ceil(frame)); + let mut last_at = 0; + for k in 0.. { + let j = k as i64 - backlog as i64; + let offset = j * frame as i64; + if offset >= end_ns as i64 { + break; + } + let capture = (START_NS as i64 + offset) as u64; + let mut at = capture + NET_NS + (self.video_lag)(capture.saturating_sub(START_NS)); + if let Some((_, speedup)) = self.relay_backlog { + at = at.max(START_NS + NET_NS + k * frame / speedup); + } + let Some(at) = self.through_stalls(at, true) else { + continue; + }; + last_at = at.max(last_at); + out.push(Arrival { + at_ns: last_at, + capture_ns: capture, + media: Media::Video(FIRST_PTS_NS + offset), + }); + } + + out.sort_by_key(|a| (a.at_ns, a.capture_ns)); + out + } + + fn run(self) -> Report { + Sim::new(&self).run(&self) + } +} + +// ── What libobs is handed ───────────────────────────────────── + +/// The span of the fine half of the audio's timeline code; see +/// [`encode_sample`]. +const FINE_SPAN: i64 = 65_536; + +/// The timeline code every audio sample carries: which sample of the +/// sender's timeline it is, in two channels. Not as one number, because the +/// speed resampler's gain strays from 1 by about 1e-5 from one filter phase +/// to the next, which on a sample index in the millions moves it by tens of +/// samples. So channel 1 carries the index modulo [`FINE_SPAN`], centred, +/// where that error is under a sample, and channel 0 the index in units of +/// the span, which says which span the fine one is in. +fn encode_sample(n: i64) -> [f32; 2] { + [ + (n as f64 / FINE_SPAN as f64) as f32, + (n.rem_euclid(FINE_SPAN) - FINE_SPAN / 2) as f32, + ] +} + +/// The sample index [`encode_sample`] wrote, from what came out of the pump. +fn decode_sample(coarse: f32, fine: f32) -> f64 { + let fine = f64::from(fine) + (FINE_SPAN / 2) as f64; + let spans = ((f64::from(coarse) * FINE_SPAN as f64 - fine) / FINE_SPAN as f64).round(); + spans * FINE_SPAN as f64 + fine +} + +/// Points per audio submission at which its content is read. +const PROBE_POINTS: usize = 9; + +/// One audio submission: when it plays and which content it carries. +#[derive(Clone, Copy, Debug)] +struct Submission { + timestamp: u64, + frames: u32, + rate: u32, + /// The timeline code at evenly spaced output samples, `(index, sample)`: + /// sample indices on the sender's timeline wherever the chunk carries + /// real, unfaded audio. + points: [(u32, f64); PROBE_POINTS], +} + +impl Submission { + fn first(&self) -> f64 { + self.points[0].1 + } + + fn last(&self) -> f64 { + self.points[PROBE_POINTS - 1].1 + } + + /// Whether the chunk is real audio played straight: no fade, no + /// concealment. At any speed the controller uses, every stretch of it + /// advances by roughly one content sample per output sample; faded or + /// concealed audio is nowhere near that, and neither is a stretch across + /// the fine code's wrap. + fn is_straight(&self) -> bool { + self.frames >= PROBE_POINTS as u32 + && self.points.windows(2).all(|w| { + let slope = (w[1].1 - w[0].1) / f64::from(w[1].0 - w[0].0); + (0.9..1.1).contains(&slope) + }) + } + + /// OBS time at which content sample `sample` plays, if this chunk carries + /// it, interpolated between the probe points around it. + fn plays_at(&self, sample: f64) -> Option { + // Between two chunks the content line steps by about a sample. + if sample < self.first() - 2.0 || sample > self.last() { + return None; + } + let i = self + .points + .partition_point(|p| p.1 < sample) + .clamp(1, PROBE_POINTS - 1); + let ((i0, v0), (i1, v1)) = (self.points[i - 1], self.points[i]); + let at = f64::from(i0) + (sample - v0).max(0.0) / (v1 - v0) * f64::from(i1 - i0); + Some(self.timestamp as f64 + at * 1e9 / f64::from(self.rate)) + } +} + +#[derive(Clone, Default)] +struct AudioProbe(Arc>>); + +impl AudioSink for AudioProbe { + fn output_audio(&self, audio: &obs::AudioFrame<'_>) { + let sys = audio.as_sys(); + let frames = sys.frames as usize; + // SAFETY: the frame borrows a live interleaved-float buffer of + // `frames * CHANNELS` samples, which is what the pump built. + let pcm = unsafe { + std::slice::from_raw_parts(sys.data[0].cast::(), frames * CHANNELS as usize) + }; + let mut points = [(0, 0.0); PROBE_POINTS]; + for (k, point) in points.iter_mut().enumerate() { + let i = k * frames.saturating_sub(1) / (PROBE_POINTS - 1); + let at = i * CHANNELS as usize; + *point = (i as u32, decode_sample(pcm[at], pcm[at + 1])); + } + self.0.lock().push(Submission { + timestamp: sys.timestamp, + frames: sys.frames, + rate: sys.samples_per_sec, + points, + }); + } +} + +/// One frame libobs received. +#[derive(Clone, Copy, Debug)] +struct Shown { + pts_ns: i64, + timestamp: u64, + handed_ns: u64, + /// How late the frame that anchored libobs's play head was handed over + /// against its own timestamp (negative: early). libobs shows that frame + /// on arrival and advances its play head by wall clock from there, so + /// every frame after it is shown that much off its timestamp. + anchor_error_ns: i64, + /// Whether this is the frame that anchored the play head. + anchor: bool, +} + +impl Shown { + /// When libobs shows it: at its timestamp as the anchored play head + /// reaches it, or on arrival if it came later than that. + fn displayed_ns(&self) -> u64 { + (self.timestamp as i64 + self.anchor_error_ns).max(self.handed_ns as i64) as u64 + } + + fn late_ns(&self) -> i64 { + self.handed_ns as i64 - self.timestamp as i64 + } +} + +#[derive(Default)] +struct VideoOut { + shown: Vec, + /// `None` while libobs's play head is unanchored: at the start, and after + /// a clear. + anchor_error_ns: Option, +} + +#[derive(Clone)] +struct VideoProbe { + out: Arc>, + clock: Arc, +} + +impl VideoSink for VideoProbe { + fn output_video(&self, frame: &obs::VideoFrame<'_>) { + let raw = frame.as_sys(); + // SAFETY: the plane is the stub's I420 luma, lent as it is and at + // least 64 bytes wide; the first eight carry the frame's PTS. + let pts = unsafe { std::ptr::read_unaligned(raw.data[0].cast::()) }; + let handed_ns = self.clock.load(Relaxed); + let mut out = self.out.lock(); + let anchor = out.anchor_error_ns.is_none(); + let anchor_error_ns = *out + .anchor_error_ns + .get_or_insert(handed_ns as i64 - raw.timestamp as i64); + out.shown.push(Shown { + pts_ns: pts, + timestamp: raw.timestamp, + handed_ns, + anchor_error_ns, + anchor, + }); + } + + fn output_video_none(&self) { + self.out.lock().anchor_error_ns = None; + } +} + +/// The decoder stand-in: a small I420 keyframe carrying the packet's PTS in +/// its first luma bytes. +fn stub_decode(packet: &TimedPacket) -> Option { + let mut frame = ffmpeg::Frame::alloc_video(ffmpeg::AVPixelFormat::AV_PIX_FMT_YUV420P, 64, 32) + .expect("alloc"); + frame.set_pts(packet.pts_ns); + // SAFETY: a freshly allocated 64x32 I420 frame owns a luma plane of at + // least 64 * 32 bytes, and its flags are a plain field. + unsafe { + let raw = frame.as_mut_ptr(); + (*raw).flags |= ffmpeg::sys::AV_FRAME_FLAG_KEY; + std::ptr::write_unaligned((*raw).data[0].cast::(), packet.pts_ns); + } + Some(frame) +} + +// ── The simulated plugin ────────────────────────────────────── + +/// The stats as a stats proc would read them. +#[derive(Clone, Copy, Debug)] +struct StatSample { + at_ns: u64, + video_delay_ms: u64, + audio_hold_ms: i32, + av_skew_ms: i64, + reanchors: u64, + fill_ms: i32, + target_ms: i32, +} + +struct Sim { + clock: Arc, + shared: Arc, + intake: AudioIntake, + flags: ReceiverFlags, + pump: AudioPump, + video: VideoThread, + audio_out: AudioProbe, + video_out: Arc>, +} + +impl Sim { + fn new(scenario: &Scenario) -> Self { + let shared = common::shared( + common::stream_config(scenario.low_latency), + common::hot_values(scenario.target_ms), + ); + shared.flags.audio_present.store(scenario.audio, Relaxed); + shared.flags.video_present.store(true, Relaxed); + + let clock = Arc::new(AtomicU64::new(START_NS)); + let audio_out = AudioProbe::default(); + let pump = { + let ns = Arc::clone(&clock); + let us = Arc::clone(&clock); + AudioPump::with_sink(Arc::clone(&shared), Box::new(audio_out.clone())) + .with_clock(Box::new(move || ns.load(Relaxed))) + .with_us_clock(Box::new(move || us.load(Relaxed) / 1000)) + }; + + let video_out = Arc::new(Mutex::new(VideoOut::default())); + let sink = VideoProbe { + out: Arc::clone(&video_out), + clock: Arc::clone(&clock), + }; + let tick = scenario.canvas_tick_ns(); + let video = { + let ns = Arc::clone(&clock); + VideoThread::with_sink(Arc::clone(&shared), Box::new(sink)) + .with_clock(Box::new(move || ns.load(Relaxed))) + .with_canvas_tick(Box::new(move || Some(tick))) + .with_decode_stub(Box::new(stub_decode)) + }; + + let mut intake = AudioIntake::default(); + intake.init_pts_repair(ffmpeg::NS_TIME_BASE); + + Self { + clock, + shared, + intake, + flags: ReceiverFlags { + has_audio_stream: scenario.audio, + has_video_stream: true, + ..Default::default() + }, + pump, + video, + audio_out, + video_out, + } + } + + fn now(&self) -> u64 { + self.clock.load(Relaxed) + } + + /// The receiver thread reading one packet off the link. + fn receive(&mut self, media: Media) { + let now = self.now(); + match media { + Media::Audio(n) => { + let pts = sample_pts_ns(n); + let duration = sample_pts_ns(n + AUDIO_FRAMES) - pts; + let frame = common::decoded_audio(pts, duration, AUDIO_FRAMES as i32, |i, c| { + encode_sample(n + i as i64)[c] + }); + self.intake.handle_frame( + &self.shared, + &mut self.flags, + &frame, + ffmpeg::NS_TIME_BASE, + ); + } + Media::Video(pts) => { + note_video_arrival(&self.shared, now, pts, Some(pts)); + self.shared.video.push_packet( + TimedPacket { + packet: ffmpeg::Packet::new().unwrap(), + pts_ns: pts, + bytes: 4096, + received_ns: now, + }, + &self.shared.lifetime, + ); + } + } + } + + fn apply(&mut self, event: &Event) { + match *event { + Event::TargetBuffer(target_ms) => { + let edit = Config { + stream: self.shared.cfg.clone(), + hot: common::hot_values(target_ms), + close_when_inactive: false, + }; + edit.apply_hot(&self.shared); + } + } + } + + fn stats(&self) -> StatSample { + let state = self.shared.audio_state(); + StatSample { + at_ns: self.now(), + video_delay_ms: self.shared.conn.video_delay_ns.load(Relaxed) / MS, + audio_hold_ms: self.shared.conn.audio_hold_ms.load(Relaxed), + av_skew_ms: av_skew_ms(&self.shared, &state), + reanchors: self.shared.lifetime.audio_offset_reanchors.load(Relaxed), + fill_ms: self + .shared + .audio_buf() + .as_ref() + .map_or(0, irl_core::AudioBuffer::fill_ms), + target_ms: self.shared.hot.watermarks().target_ms, + } + } + + /// Run the three threads against the sender, each woken when it would + /// be: the receiver as packets arrive, the audio thread when its pump + /// says, the video thread when its pacing sleep ends, a packet lands with + /// room to decode it, or audio publishes its first mapping. + fn run(mut self, scenario: &Scenario) -> Report { + let arrivals = scenario.arrivals(); + let end_ns = START_NS + scenario.secs * SEC; + let mut next_arrival = 0; + let mut next_event = 0; + let mut pump_wake = START_NS; + let mut video_wake = START_NS; + let mut next_sample = START_NS; + let mut mapped = false; + let mut stats = Vec::new(); + + while self.now() < end_ns { + let now = self.now(); + + while let Some((at, event)) = scenario.events.get(next_event) { + if START_NS + at > now { + break; + } + self.apply(event); + next_event += 1; + } + + while let Some(arrival) = arrivals.get(next_arrival) { + if arrival.at_ns > now { + break; + } + self.receive(arrival.media); + next_arrival += 1; + } + + if now >= pump_wake { + while self.pump.pump_once() {} + pump_wake = now + u64::from(self.pump.idle_sleep_ms()) * MS; + } + + let published = self.shared.audio_state().mapping.is_published(); + let woken = published && !mapped; + mapped = published; + let has_work = |sim: &Self| sim.shared.video.has_work(sim.video.pacing().has_room()); + if now >= video_wake || woken || has_work(&self) { + let mut spins = 0; + loop { + let wait = self.video.run_once(); + if !wait.is_zero() && !has_work(&self) { + video_wake = now + wait.as_nanos() as u64; + break; + } + spins += 1; + assert!(spins < 1000, "the video thread spins at {}s", secs(now)); + } + } + + if now >= next_sample { + stats.push(self.stats()); + next_sample = now + 100 * MS; + } + + let mut next = pump_wake.min(video_wake).min(next_sample).min(end_ns); + if let Some(arrival) = arrivals.get(next_arrival) { + next = next.min(arrival.at_ns); + } + if let Some((at, _)) = scenario.events.get(next_event) { + next = next.min(START_NS + at); + } + self.clock.store(next.max(now + 1), Relaxed); + } + + let video_sent = arrivals + .iter() + .filter(|a| matches!(a.media, Media::Video(_))) + .count(); + // Only real audio played straight carries a readable timeline; fades + // and concealment are left out, so what remains is in content order. + let played = self + .audio_out + .0 + .lock() + .iter() + .filter(|s| s.is_straight()) + .copied() + .collect(); + Report { + submissions: self.audio_out.0.lock().clone(), + played, + shown: self.video_out.lock().shown.clone(), + stats, + video_sent, + tick_ns: scenario.canvas_tick_ns(), + frame_ns: scenario.frame_ns(), + restarts: self.shared.conn.audio_output_restarts.load(Relaxed), + reanchors: self.shared.lifetime.audio_offset_reanchors.load(Relaxed), + audible_skipped: self.shared.conn.audible_skipped_chunks.load(Relaxed), + } + } +} + +// ── What the run shows ──────────────────────────────────────── + +struct Report { + submissions: Vec, + /// The submissions of real audio played straight. + played: Vec, + shown: Vec, + stats: Vec, + video_sent: usize, + tick_ns: u64, + frame_ns: u64, + restarts: u64, + reanchors: u64, + audible_skipped: u64, +} + +/// What the frames shown inside one stretch of the run measured. +#[derive(Debug)] +struct Window { + from_s: f64, + to_s: f64, + frames: usize, + /// Frames whose audio could not be located (it played faded, concealed, + /// or not yet). + unmeasured: usize, + min_offset_ms: f64, + max_offset_ms: f64, + mean_offset_ms: f64, + late: usize, + max_late_ms: f64, +} + +impl Report { + /// When the audio of content PTS `pts_ns` plays, read off the PCM. + fn audio_plays_at(&self, pts_ns: i64) -> Option { + let sample = pts_ns as f64 * f64::from(RATE) / 1e9; + let i = self.played.partition_point(|s| s.last() < sample); + self.played.get(i)?.plays_at(sample) + } + + /// Picture minus sound for one frame, in ms; positive is picture behind. + fn offset_ms(&self, frame: &Shown) -> Option { + let audio = self.audio_plays_at(frame.pts_ns)?; + Some((frame.displayed_ns() as f64 - audio) / 1e6) + } + + /// The frames handed over between `from_s` and `to_s` after the open. + fn window(&self, from_s: f64, to_s: f64) -> Window { + let from = START_NS + (from_s * SEC as f64) as u64; + let to = START_NS + (to_s * SEC as f64) as u64; + let frames: Vec<&Shown> = self + .shown + .iter() + .filter(|f| (from..to).contains(&f.handed_ns)) + .collect(); + // A frame whose audio had not played by the end has nothing to be + // measured against yet. + let played_to = self.played.last().map_or(0.0, Submission::last); + let frames: Vec<&Shown> = frames + .into_iter() + .filter(|f| (f.pts_ns as f64 * f64::from(RATE) / 1e9) < played_to) + .collect(); + let offsets: Vec = frames.iter().filter_map(|f| self.offset_ms(f)).collect(); + // The anchoring frame may go up to a canvas tick late by design (see + // `VideoThread::settle_anchor_candidate`); what that costs shows up + // in every later frame's offset instead. + let late: Vec = frames + .iter() + .filter(|f| !f.anchor) + .map(|f| f.late_ns()) + .filter(|&l| l > consts::VIDEO_PACING_SLACK_NS) + .collect(); + Window { + from_s, + to_s, + frames: frames.len(), + unmeasured: frames.len() - offsets.len(), + min_offset_ms: offsets.iter().copied().fold(f64::INFINITY, f64::min), + max_offset_ms: offsets.iter().copied().fold(f64::NEG_INFINITY, f64::max), + mean_offset_ms: offsets.iter().sum::() / offsets.len().max(1) as f64, + late: late.len(), + max_late_ms: late.iter().copied().max().unwrap_or(0) as f64 / 1e6, + } + } + + fn last_stats(&self) -> StatSample { + *self.stats.last().expect("sampled") + } + + /// One line per scenario for the record, with `--nocapture`. + fn print(&self, name: &str, steady: &Window) { + let s = self.last_stats(); + println!( + "{name}: offset {:.1}..{:.1}ms (mean {:.1}) late {} (max {:.1}ms) over {:.0}-{:.0}s; \ + anchor error {:.1}ms, shown {}/{} video_delay={}ms audio_hold={}ms av_skew={}ms \ + reanchors={} restarts={}", + steady.min_offset_ms, + steady.max_offset_ms, + steady.mean_offset_ms, + steady.late, + steady.max_late_ms, + steady.from_s, + steady.to_s, + self.shown.first().map_or(0, |f| f.anchor_error_ns) as f64 / 1e6, + self.shown.len(), + self.video_sent, + s.video_delay_ms, + s.audio_hold_ms, + s.av_skew_ms, + self.reanchors, + self.restarts, + ); + } + + // ── Promises ── + + /// Picture and sound agree within `tolerance_ns` for every frame shown in + /// the window, and every one of them could be measured. + fn assert_in_sync(&self, w: &Window, tolerance_ns: i64) { + let tol = tolerance_ns as f64 / 1e6; + assert!(w.frames > 0, "no frames shown in {w:?}"); + assert!( + w.unmeasured * 20 <= w.frames, + "the audio of most frames could not be located: {w:?}" + ); + assert!( + w.min_offset_ms >= -tol && w.max_offset_ms <= tol, + "out of sync by more than {tol:.1}ms: {w:?}" + ); + } + + /// No frame in the window reached libobs after its own timestamp. + fn assert_paced(&self, w: &Window) { + assert_eq!(w.late, 0, "frames handed over late: {w:?}"); + } + + /// The libobs audio contract: timestamps contiguous except across the + /// restarts and re-anchors the plugin declares and counts. + fn assert_clock_only_jumps_where_declared(&self) { + let mut jumps = 0; + for pair in self.submissions.windows(2) { + let (a, b) = (&pair[0], &pair[1]); + assert_eq!(a.rate, RATE as u32, "the submitted rate must never change"); + let expected = a.timestamp + u64::from(a.frames) * SEC / u64::from(a.rate); + if (b.timestamp as i64 - expected as i64).abs() > 1 { + jumps += 1; + } + } + let declared = self.restarts + self.reanchors; + assert!( + jumps <= declared, + "{jumps} discontinuities against {declared} declared" + ); + } + + /// What every scenario owes whatever the link did. + fn assert_healthy(&self) { + self.assert_clock_only_jumps_where_declared(); + assert_eq!(self.audible_skipped, 0, "audible audio was skipped"); + } + + /// Every frame sent from `from_s` on was shown: nothing dropped once the + /// play head anchored. + fn assert_nothing_dropped_after(&self, from_s: f64) { + let from = START_NS + (from_s * SEC as f64) as u64; + let first = self + .shown + .iter() + .find(|f| f.handed_ns >= from) + .expect("frames shown") + .pts_ns; + let shown: Vec = self + .shown + .iter() + .filter(|f| f.pts_ns >= first) + .map(|f| f.pts_ns) + .collect(); + let gaps = shown + .windows(2) + .filter(|p| p[1] - p[0] > self.frame_ns as i64 + 1) + .count(); + assert_eq!(gaps, 0, "frames dropped after {from_s}s"); + } + + /// Picture and sound settled into sync between `from_s` and `to_s`: + /// every frame within the tolerance and handed over in time. + fn assert_settled(&self, from_s: f64, to_s: f64) -> Window { + let w = self.window(from_s, to_s); + self.assert_in_sync(&w, SYNC_TOLERANCE_NS); + self.assert_paced(&w); + w + } + + /// Sync in `later` is where it was in `earlier`: whatever happened in + /// between left no lasting offset. + fn assert_no_drift(&self, earlier: &Window, later: &Window) { + let drift = later.mean_offset_ms - earlier.mean_offset_ms; + assert!( + drift.abs() <= 2.0, + "sync drifted {drift:+.1}ms: {earlier:?} then {later:?}" + ); + } + + /// The audio latency came back to what the settings and the hold ask + /// for: the buffer at its target. + fn assert_latency_settled(&self) { + let s = self.last_stats(); + assert!( + (s.fill_ms - s.target_ms).abs() <= 30, + "the buffer holds {}ms against a {}ms target", + s.fill_ms, + s.target_ms + ); + } + + /// The video delay the stats reported, over the samples from `from_s` on. + fn video_delay_ms(&self, from_s: f64) -> std::ops::RangeInclusive { + let from = START_NS + (from_s * SEC as f64) as u64; + let delays = self + .stats + .iter() + .filter(|s| s.at_ns >= from) + .map(|s| s.video_delay_ms); + let (lo, hi) = delays.fold((u64::MAX, 0), |(lo, hi), d| (lo.min(d), hi.max(d))); + lo..=hi + } + + /// The stats sample just before the first re-anchor. + fn before_first_reanchor(&self) -> Option { + let i = self.stats.iter().position(|s| s.reanchors > 0)?; + Some(self.stats[i.checked_sub(1)?]) + } +} + +// ── Scenarios ───────────────────────────────────────────────── +// +// A sender's video can trail its audio in the mux by its encoder's habits +// (pocketSRT queues audio early, a stabiliser holds video back). Each +// scenario states what the plugin promises for it, and how soon. + +/// Nothing wrong with the link or the sender: picture and sound agree within +/// a canvas tick from the first frame shown, every frame is handed over ahead +/// of its timestamp, and no delay or hold is ever needed. The tick is what +/// the anchoring frame may be late by (see `VideoThread:: +/// settle_anchor_candidate`), and libobs keeps that error for the connection. +#[test] +fn a_clean_link_is_in_sync_from_the_first_frame() { + for fps in [30, 60] { + let report = Scenario::new(30).fps(fps).run(); + let all = report.window(0.0, 30.0); + report.print(&format!("clean {fps}fps"), &all); + + report.assert_in_sync(&all, report.tick_ns as i64); + report.assert_paced(&all); + report.assert_nothing_dropped_after(0.0); + report.assert_healthy(); + assert_eq!(report.video_delay_ms(0.0), 0..=0, "{fps}fps"); + assert_eq!(report.last_stats().audio_hold_ms, 0); + } +} + +/// pocketSRT queues its audio 300 ms and more ahead of its video, and its +/// video also stalls on its own for a couple of hundred milliseconds every +/// few seconds (#34). The hold covers the steady skew from the first frame. +/// The frames a stall holds back cannot be shown in time and go out late; +/// the picture is back in sync once the burst is through, and it must be in +/// the same sync after the last stall as after the first. +#[test] +fn audio_ahead_of_its_video_with_recurring_video_stalls_stays_in_sync() { + for lag_ms in [350, 700] { + let mut scenario = Scenario::new(70).video_lag(move |_| lag_ms * MS); + for k in 0..15 { + scenario = scenario.stall(10.0 + 4.0 * f64::from(k), 200, Streams::Video); + } + let report = scenario.run(); + let between_stalls = |k: u32| { + let at = 10.0 + 4.0 * f64::from(k); + report.assert_settled(at + 1.0, at + 4.0) + }; + + let before = report.assert_settled(0.0, 10.0); + let first = between_stalls(0); + for k in 1..13 { + between_stalls(k); + } + let last = between_stalls(13); + report.print(&format!("pocketSRT {lag_ms}ms"), &last); + + report.assert_no_drift(&before, &last); + report.assert_no_drift(&first, &last); + let hold = report.last_stats().audio_hold_ms; + let expected = lag_ms as i32 + consts::AUDIO_HOLD_MARGIN_MS + - consts::DEFAULT_BUFFER_TARGET_MS as i32 + - consts::AUDIO_OUT_LEAD_MS; + assert!( + (hold - expected).abs() <= 3, + "{lag_ms}ms: hold {hold}ms against {expected}ms" + ); + assert!( + report.stats.iter().all(|s| s.audio_hold_ms <= hold), + "{lag_ms}ms: the stalls ratcheted the hold" + ); + assert_eq!(report.video_delay_ms(0.0), 0..=0, "{lag_ms}ms"); + report.assert_healthy(); + } +} + +/// The stabiliser stream from #33: video 1.65 s behind its audio, for good. +/// The hold is sized before audio starts, so the picture is in sync from the +/// first frame and never needs a delay. +#[test] +fn video_seconds_behind_its_audio_is_in_sync_from_the_first_frame() { + let report = Scenario::new(40).video_lag(|_| 1650 * MS).run(); + let all = report.assert_settled(0.0, 40.0); + report.print("stabiliser", &all); + + report.assert_nothing_dropped_after(0.0); + assert_eq!(report.video_delay_ms(0.0), 0..=0); + let hold = report.last_stats().audio_hold_ms; + assert!( + (hold - (1650 + consts::AUDIO_HOLD_MARGIN_MS - 120 - consts::AUDIO_OUT_LEAD_MS)).abs() <= 3, + "hold {hold}ms" + ); + report.assert_latency_settled(); + report.assert_healthy(); +} + +const SKEW_RISES_TO_600_MS: &[(f64, f64)] = &[(20.0, 0.0), (23.0, 600.0)]; + +/// Video falls 600 ms behind its audio over three seconds, stays there, and +/// recovers. While it falls the delay covers what it can and frames go out +/// late; after the raise window the hold rises and is built at -2 %, which +/// for 600 ms is half a minute, and the delay hands itself back as it builds. +/// After the recovery the hold is released and the latency drained. In sync +/// before, once built, and after. +#[test] +fn a_skew_that_rises_and_recovers_is_followed_both_ways() { + let report = Scenario::new(120) + .video_lag(lag_profile(&[ + (20.0, 0.0), + (23.0, 600.0), + (60.0, 600.0), + (63.0, 0.0), + ])) + .run(); + + let before = report.assert_settled(0.0, 20.0); + report.assert_settled(55.0, 60.0); + let after = report.assert_settled(80.0, 120.0); + report.print("skew 0-600-0", &after); + + assert_eq!(report.window(28.0, 120.0).late, 0, "late once sized"); + report.assert_no_drift(&before, &after); + // The release keeps anything within its hysteresis. + assert!(report.last_stats().audio_hold_ms <= consts::AUDIO_HOLD_RELAX_MIN_MS); + assert_eq!(report.video_delay_ms(80.0), 0..=0); + report.assert_latency_settled(); + report.assert_nothing_dropped_after(28.0); + report.assert_healthy(); +} + +/// Both streams stall for three seconds while the hold for a new 600 ms skew +/// is being built. Audio conceals, the backlog lands at once and drains at +/// the Catch-Up Speed, the inflated latency is re-anchored away, and the +/// standing delay relaxes: in sync within about twenty seconds of the stall, +/// and back where the connection started. +#[test] +fn a_full_stall_during_a_hold_build_recovers_sync() { + let report = Scenario::new(120) + .video_lag(lag_profile(SKEW_RISES_TO_600_MS)) + .stall(30.0, 3000, Streams::Both) + .run(); + + let before = report.assert_settled(0.0, 20.0); + report.assert_settled(55.0, 120.0); + let after = report.window(100.0, 120.0); + report.print("stall during build", &after); + + report.assert_no_drift(&before, &after); + assert_eq!(report.video_delay_ms(100.0), 0..=0); + report.assert_latency_settled(); + report.assert_healthy(); +} + +/// A loss while the delay stands (a hold build in progress) leaves audio +/// latency inflated by its concealment, which the pump re-anchors away while +/// the video delay is still in force. Video follows the new playout and the +/// delay relaxes: in sync within half a minute, with no lasting offset. +#[test] +fn a_reanchor_while_the_video_delay_stands_recovers_sync() { + let report = Scenario::new(120) + .video_lag(lag_profile(SKEW_RISES_TO_600_MS)) + .loss(26.0, 1500) + .run(); + + let at_reanchor = report.before_first_reanchor().expect("a re-anchor"); + assert!( + at_reanchor.video_delay_ms > 0, + "the delay had gone by the re-anchor: {at_reanchor:?}" + ); + assert_eq!(report.reanchors, 1); + + let before = report.assert_settled(0.0, 20.0); + let after = report.assert_settled(60.0, 120.0); + report.print("re-anchor under delay", &after); + + report.assert_no_drift(&before, &after); + assert_eq!(report.video_delay_ms(70.0), 0..=0); + report.assert_latency_settled(); + report.assert_healthy(); +} + +/// The user raises Target Buffer to 500 ms while the hold for a 600 ms skew is +/// being built. The hold gives back what the deeper buffer now covers, and +/// the effective target is the same skew plus margin as before. +#[test] +fn a_target_buffer_edit_during_a_hold_build_recomposes_the_hold() { + let report = target_edit_during_build(); + + let s = report.last_stats(); + assert!( + (s.audio_hold_ms - (600 + consts::AUDIO_HOLD_MARGIN_MS - 500 - consts::AUDIO_OUT_LEAD_MS)) + .abs() + <= 25, + "hold {}ms", + s.audio_hold_ms + ); + assert_eq!(s.target_ms, 500 + s.audio_hold_ms); + assert_eq!(report.window(30.0, 120.0).late, 0); + report.assert_latency_settled(); + report.assert_healthy(); +} + +/// The same edit must also end in sync. It does not: once the delay has +/// relaxed part of the way, a remainder within `VIDEO_DELAY_RELAX_MIN_MS` plus +/// a tick of what frames need never relaxes, and the hold credit, which only +/// counts the hold's own growth, never absorbs it. Measured: the delay stands +/// at 66 ms and the picture trails the sound by 83 ms for good. +#[test] +#[ignore = "known gap: a video delay left within the relax hysteresis after a Target Buffer edit never drains"] +fn a_target_buffer_edit_during_a_hold_build_ends_in_sync() { + let report = target_edit_during_build(); + let after = report.assert_settled(90.0, 120.0); + report.print("target edit during build", &after); + assert_eq!(report.video_delay_ms(90.0), 0..=0); +} + +fn target_edit_during_build() -> Report { + Scenario::new(120) + .video_lag(lag_profile(SKEW_RISES_TO_600_MS)) + .at(28.0, Event::TargetBuffer(500)) + .run() +} + +/// Low Latency Audio has no target and no speed control: the hold is sized +/// before audio starts and kept queued on top of the single chunk of lead. +#[test] +fn low_latency_audio_with_trailing_video_is_in_sync_from_the_first_frame() { + let report = Scenario::new(40) + .low_latency() + .video_lag(|_| 350 * MS) + .run(); + let all = report.assert_settled(0.0, 40.0); + report.print("low latency", &all); + + let hold = report.last_stats().audio_hold_ms; + let lead_ms = (sample_pts_ns(AUDIO_FRAMES) / 1_000_000) as i32; + assert!( + (hold - (350 + consts::AUDIO_HOLD_MARGIN_MS - lead_ms)).abs() <= 3, + "hold {hold}ms" + ); + assert_eq!(report.video_delay_ms(0.0), 0..=0); + report.assert_nothing_dropped_after(0.0); + report.assert_healthy(); +} + +/// Without audio there is nothing to be in sync with: the video-only clock +/// schedules each frame at its arrival, a tick of delay leaves room to +/// decode it, and every frame goes out paced, a frame apart. +#[test] +fn video_without_audio_is_paced_a_tick_behind_its_arrival() { + let report = Scenario::new(20).video_only().run(); + let all = report.window(0.0, 20.0); + report.print("video only", &all); + + let tick_ms = report.tick_ns / MS; + assert_eq!(report.video_delay_ms(0.5), tick_ms..=tick_ms); + report.assert_paced(&all); + report.assert_nothing_dropped_after(0.0); + for pair in report.shown.windows(2) { + let step = pair[1].timestamp - pair[0].timestamp; + assert!(step.abs_diff(report.frame_ns) <= 1, "frames {step}ns apart"); + } +} + +/// A relay hands a new subscriber video from its last keyframe, a second +/// behind live audio, and catches up at four times real time. That is not a +/// late sender: no delay, no hold, and the picture is in sync from the first +/// frame shown. +#[test] +fn a_relay_catching_up_at_connect_sets_no_delay_and_ends_in_sync() { + let report = Scenario::new(40).relay_backlog(1000, 4).run(); + let all = report.assert_settled(0.0, 40.0); + report.print("relay catch-up", &all); + + assert_eq!(report.video_delay_ms(0.0), 0..=0); + assert_eq!(report.last_stats().audio_hold_ms, 0); + report.assert_nothing_dropped_after(0.0); + report.assert_latency_settled(); + report.assert_healthy(); +} diff --git a/crates/irl-source/tests/common/mod.rs b/crates/irl-source/tests/common/mod.rs index 1f6289d..e025948 100644 --- a/crates/irl-source/tests/common/mod.rs +++ b/crates/irl-source/tests/common/mod.rs @@ -81,6 +81,42 @@ pub fn push_packet(shared: &Shared, pts_ns: i64, bytes: usize) { ); } +/// A decoded audio frame of `frames` interleaved float samples per channel at +/// [`RATE`], stamped `pts_ns` in a nanosecond time base, as the decoder would +/// hand it to the intake. Channel `c` of sample `i` holds `value(i, c)`. +pub fn decoded_audio( + pts_ns: i64, + duration_ns: i64, + frames: i32, + value: impl Fn(usize, usize) -> f32, +) -> ffmpeg::Frame { + let mut frame = ffmpeg::Frame::new().unwrap(); + // SAFETY: setting the audio parameters before av_frame_get_buffer is the + // documented allocation sequence; the buffer is then written through its + // own data pointer for exactly frames * CHANNELS samples. + unsafe { + let raw = frame.as_mut_ptr(); + (*raw).format = ffmpeg::AVSampleFormat::AV_SAMPLE_FMT_FLT as core::ffi::c_int; + (*raw).nb_samples = frames; + (*raw).sample_rate = RATE; + ffmpeg::sys::av_channel_layout_default(&raw mut (*raw).ch_layout, CHANNELS); + assert_eq!(ffmpeg::sys::av_frame_get_buffer(raw, 0), 0); + (*raw).pts = pts_ns; + // In the frame's time base, which here is nanoseconds, not a sample + // count. PTS repair sizes its expected gap from this. + (*raw).duration = duration_ns; + + let dst = (*raw).data[0].cast::(); + for i in 0..frames as usize { + for c in 0..CHANNELS as usize { + dst.add(i * CHANNELS as usize + c) + .write_unaligned(value(i, c)); + } + } + } + frame +} + /// One audio submission, flattened. pub struct Emitted { pub timestamp: u64, diff --git a/crates/irl-source/tests/network_sim.rs b/crates/irl-source/tests/network_sim.rs index 7864d29..9ab2152 100644 --- a/crates/irl-source/tests/network_sim.rs +++ b/crates/irl-source/tests/network_sim.rs @@ -31,7 +31,7 @@ use obs_irl_source::receiver::ReceiverFlags; use obs_irl_source::receiver::audio_in::AudioIntake; use obs_irl_source::shared::Shared; -use common::{CHANNELS, CHUNK_FRAMES, CHUNK_NS, RATE, Recorder}; +use common::{CHUNK_FRAMES, CHUNK_NS, RATE, Recorder}; // ── The simulated plugin ────────────────────────────────────── @@ -191,35 +191,6 @@ impl Sim { .map_or(0, irl_core::AudioBuffer::fill_ms) } - /// One decoded audio chunk of constant-valued PCM, as the decoder would - /// hand it to the intake. - fn decoded_chunk(pts_ns: i64, value: f32) -> ffmpeg::Frame { - let mut frame = ffmpeg::Frame::new().unwrap(); - // SAFETY: setting the audio parameters before av_frame_get_buffer is - // the documented allocation sequence; the buffer is then written - // through its own data pointer for exactly nb_samples * channels - // samples. - unsafe { - let raw = frame.as_mut_ptr(); - (*raw).format = ffmpeg::AVSampleFormat::AV_SAMPLE_FMT_FLT as core::ffi::c_int; - (*raw).nb_samples = CHUNK_FRAMES; - (*raw).sample_rate = RATE; - ffmpeg::sys::av_channel_layout_default(&raw mut (*raw).ch_layout, CHANNELS); - assert_eq!(ffmpeg::sys::av_frame_get_buffer(raw, 0), 0); - (*raw).pts = pts_ns; - // In the frame's time base, which here is nanoseconds — not a - // sample count. PTS repair sizes its expected gap from this. - (*raw).duration = CHUNK_NS as i64; - - let dst = (*raw).data[0]; - for i in 0..(CHUNK_FRAMES * CHANNELS) as usize { - let bytes = value.to_le_bytes(); - std::ptr::copy_nonoverlapping(bytes.as_ptr(), dst.add(i * 4), 4); - } - } - frame - } - /// Advance one chunk of wall clock. `link` says whether the sender's output /// reaches us; while it is `Down` the receiver is blocked, so nothing is /// ingested at all. @@ -236,7 +207,7 @@ impl Sim { if link == Link::Up && self.since_delivery >= self.burst_ticks { self.since_delivery = 0; for pts in std::mem::take(&mut self.pending) { - let frame = Self::decoded_chunk(pts, 0.25); + let frame = common::decoded_audio(pts, CHUNK_NS as i64, CHUNK_FRAMES, |_, _| 0.25); self.intake.handle_frame( &self.shared, &mut self.flags, diff --git a/scripts/test-with-libobs-shim.sh b/scripts/test-with-libobs-shim.sh index 2c288fa..18a6be4 100755 --- a/scripts/test-with-libobs-shim.sh +++ b/scripts/test-with-libobs-shim.sh @@ -21,7 +21,7 @@ fi # The tests whose libobs imports the shim covers. shell_stats is not one of # them: calldata is real libobs bookkeeping and needs the real library. -TESTS=(audio_core network_sim video_pipeline) +TESTS=(audio_core av_sync_sim network_sim video_pipeline) SHIM_SRC=scripts/libobs-shim/libobs_shim.c SHIM_DIR=target/libobs-shim From 1402c5be272ce3e375da7816bffb954a60985b85 Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 04:35:15 +0200 Subject: [PATCH 06/13] refactor(irl-core): sliding window extreme in its own module The audio hold's monotonic min/max queue moves to irl_core::window so the video delay can judge its relax window the same way. --- crates/irl-core/src/audio_hold.rs | 57 +---------------------------- crates/irl-core/src/lib.rs | 1 + crates/irl-core/src/window.rs | 59 +++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 56 deletions(-) create mode 100644 crates/irl-core/src/window.rs diff --git a/crates/irl-core/src/audio_hold.rs b/crates/irl-core/src/audio_hold.rs index 247ef1e..4d05a68 100644 --- a/crates/irl-core/src/audio_hold.rs +++ b/crates/irl-core/src/audio_hold.rs @@ -22,9 +22,8 @@ //! takes the *worst* reading across the longer `AUDIO_HOLD_RELAX_WINDOW_MS`, //! so the hold does not pump up and down. -use std::collections::VecDeque; - use crate::consts; +use crate::window::{Extreme, WindowExtreme}; /// How much longer audio must be held so that video `skew_ms` behind it is in /// hand `margin_ms` before it is due: the part of the skew that `covered_ms` @@ -149,60 +148,6 @@ impl Default for AudioHold { } } -#[derive(Debug, Clone, Copy)] -enum Extreme { - Lowest, - Highest, -} - -/// The lowest or highest value pushed within the last `window_ns`, kept as a -/// monotonic queue: a value that a newer, more extreme one has beaten can -/// never be the answer again, so it is dropped on the spot and every push is -/// amortised O(1). -#[derive(Debug)] -struct WindowExtreme { - window_ns: u64, - extreme: Extreme, - /// `(when, value)`, oldest first, strictly less extreme from front to back. - entries: VecDeque<(u64, i64)>, -} - -impl WindowExtreme { - fn new(window_ns: u64, extreme: Extreme) -> Self { - Self { - window_ns, - extreme, - entries: VecDeque::new(), - } - } - - fn clear(&mut self) { - self.entries.clear(); - } - - fn push(&mut self, now_ns: u64, value: i64) { - while let Some(&(_, last)) = self.entries.back() { - let beaten = match self.extreme { - Extreme::Lowest => last >= value, - Extreme::Highest => last <= value, - }; - if !beaten { - break; - } - self.entries.pop_back(); - } - self.entries.push_back((now_ns, value)); - let horizon_ns = now_ns.saturating_sub(self.window_ns); - while self.entries.front().is_some_and(|&(at, _)| at < horizon_ns) { - self.entries.pop_front(); - } - } - - fn value(&self) -> Option { - self.entries.front().map(|&(_, value)| value) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/irl-core/src/lib.rs b/crates/irl-core/src/lib.rs index 675ef09..99da21a 100644 --- a/crates/irl-core/src/lib.rs +++ b/crates/irl-core/src/lib.rs @@ -27,6 +27,7 @@ pub mod timing; pub mod url_opts; pub mod video_delay; pub mod video_time; +mod window; pub use audio_buffer::{AudioBuffer, BufferState}; pub use audio_hold::{AudioHold, HoldChange}; diff --git a/crates/irl-core/src/window.rs b/crates/irl-core/src/window.rs new file mode 100644 index 0000000..67a22dd --- /dev/null +++ b/crates/irl-core/src/window.rs @@ -0,0 +1,59 @@ +//! The extreme of a value over a sliding time window, shared by the audio +//! hold and the video delay. + +use std::collections::VecDeque; + +/// Which end of the values a [`WindowExtreme`] keeps. +#[derive(Debug, Clone, Copy)] +pub(crate) enum Extreme { + Lowest, + Highest, +} + +/// The lowest or highest value pushed within the last `window_ns`, kept as a +/// monotonic queue: a value that a newer, more extreme one has beaten can +/// never be the answer again, so it is dropped on the spot and every push is +/// amortised O(1). +#[derive(Debug)] +pub(crate) struct WindowExtreme { + window_ns: u64, + extreme: Extreme, + /// `(when, value)`, oldest first, strictly less extreme from front to back. + entries: VecDeque<(u64, i64)>, +} + +impl WindowExtreme { + pub(crate) fn new(window_ns: u64, extreme: Extreme) -> Self { + Self { + window_ns, + extreme, + entries: VecDeque::new(), + } + } + + pub(crate) fn clear(&mut self) { + self.entries.clear(); + } + + pub(crate) fn push(&mut self, now_ns: u64, value: i64) { + while let Some(&(_, last)) = self.entries.back() { + let beaten = match self.extreme { + Extreme::Lowest => last >= value, + Extreme::Highest => last <= value, + }; + if !beaten { + break; + } + self.entries.pop_back(); + } + self.entries.push_back((now_ns, value)); + let horizon_ns = now_ns.saturating_sub(self.window_ns); + while self.entries.front().is_some_and(|&(at, _)| at < horizon_ns) { + self.entries.pop_front(); + } + } + + pub(crate) fn value(&self) -> Option { + self.entries.front().map(|&(_, value)| value) + } +} From fbd03e633dc264591a462235cfa2445884951b12 Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 04:37:06 +0200 Subject: [PATCH 07/13] refactor(irl-source): drop the video lead warning The "Video lead is beyond what OBS can queue" line assumed frames reach libobs as early as the audio mapping places them. Pacing hands each frame over a couple of canvas ticks before it is due, so libobs never holds more than that and the warning described a queue that does not exist. Its frame budget, throttle and default frame interval go with it; the frame-interval EMA stays for the stats line's vfps. --- crates/irl-core/src/consts.rs | 9 ----- crates/irl-core/src/video_time.rs | 40 ++------------------- crates/irl-source/src/video/intake.rs | 10 +++--- crates/irl-source/src/video/output.rs | 50 +++------------------------ crates/irl-source/src/video/thread.rs | 3 -- 5 files changed, 12 insertions(+), 100 deletions(-) diff --git a/crates/irl-core/src/consts.rs b/crates/irl-core/src/consts.rs index b1eb2c3..398a6d4 100644 --- a/crates/irl-core/src/consts.rs +++ b/crates/irl-core/src/consts.rs @@ -239,16 +239,10 @@ pub const VIDEO_PACKET_QUEUE_MAX_BYTES: usize = 64 * 1024 * 1024; // ── Video timing / pacing ── -/// Reporting budget for frames parked in libobs's async queue. -pub const OBS_ASYNC_FRAME_BUDGET: i64 = 24; -/// Throttle for the video lead warning. -pub const VIDEO_LEAD_WARN_INTERVAL_NS: u64 = 10_000_000_000; /// Bounds on the measured frame interval (250fps..10fps). pub const VIDEO_INTERVAL_MIN_NS: i64 = 4_000_000; /// Bounds on the measured frame interval (250fps..10fps). pub const VIDEO_INTERVAL_MAX_NS: i64 = 100_000_000; -/// Interval estimate before enough frames have arrived. -pub const VIDEO_INTERVAL_DEFAULT_NS: i64 = 33_333_333; /// Pacing queue frame ceiling. /// /// It has to carry the largest Target Buffer at the highest frame rate anyone @@ -475,11 +469,8 @@ mod tests { assert_eq!(VIDEO_PACKET_QUEUE_MAX_BYTES, 67_108_864); // ── video timing / pacing ── - assert_eq!(OBS_ASYNC_FRAME_BUDGET, 24); - assert_eq!(VIDEO_LEAD_WARN_INTERVAL_NS, 10_000_000_000); assert_eq!(VIDEO_INTERVAL_MIN_NS, 4_000_000); assert_eq!(VIDEO_INTERVAL_MAX_NS, 100_000_000); - assert_eq!(VIDEO_INTERVAL_DEFAULT_NS, 33_333_333); assert_eq!(VIDEO_PACING_MAX_FRAMES, 1024); assert_eq!(VIDEO_PACING_MAX_BYTES, 1_073_741_824); assert_eq!(VIDEO_PACING_SLACK_NS, 1_000_000); diff --git a/crates/irl-core/src/video_time.rs b/crates/irl-core/src/video_time.rs index 80fb652..ecbaebe 100644 --- a/crates/irl-core/src/video_time.rs +++ b/crates/irl-core/src/video_time.rs @@ -1,7 +1,6 @@ //! Video timestamp arithmetic outside the audio playout mapping -//! ([`crate::PlayoutMapping`]): the video-only fallback, the lead libobs can -//! absorb and the frame interval. Locking and logging stay in the plugin -//! crate. +//! ([`crate::PlayoutMapping`]): the video-only fallback and the frame +//! interval. Locking and logging stay in the plugin crate. use crate::consts; @@ -23,28 +22,6 @@ pub fn fallback_anchor(pts_ns: i64, pts_base_ns: i64, sys_base_ns: u64, now_ns: } } -/// Lead libobs can absorb: `OBS_ASYNC_FRAME_BUDGET × frame_interval`, floored -/// at the audio re-anchor margin. -/// -/// The caller adds the jitter buffer's own contribution to the lead -/// (`buffer_target_ms × 1e6`). The budget is expressed in frames because that -/// is what libobs counts: the same 400 ms is 12 frames at 30 fps and 48 at -/// 120 fps. -pub fn queue_safe_ns(frame_interval_ns: i64) -> i64 { - let interval = if frame_interval_ns <= 0 { - consts::VIDEO_INTERVAL_DEFAULT_NS - } else { - frame_interval_ns - }; - let budget_ns = consts::OBS_ASYNC_FRAME_BUDGET * interval; - let floor_ns = consts::AUDIO_OFFSET_REANCHOR_MARGIN_MS * 1_000_000; - if budget_ns < floor_ns { - floor_ns - } else { - budget_ns - } -} - /// EMA (1/8 step) of PTS deltas, ignoring deltas outside the 4–100 ms window. /// /// Measured rather than taken from `avg_frame_rate`, which live SRT/RTMP @@ -100,19 +77,6 @@ mod tests { assert_eq!(fallback_anchor(500_000_000, 0, now, now), now + 500_000_000); } - #[test] - fn queue_safe_floors_at_the_reanchor_margin() { - // 24 frames at 120 fps is 200 ms, below the 400 ms floor. - assert_eq!(queue_safe_ns(8_333_333), 400_000_000); - // At 60 fps, 24 frames is a hair under 400 ms, so the floor still wins. - assert_eq!(queue_safe_ns(16_666_666), 400_000_000); - // At 30 fps it is 800 ms, and the budget wins. - assert_eq!(queue_safe_ns(33_333_333), 24 * 33_333_333); - // No measurement yet: the 30 fps default stands in. - assert_eq!(queue_safe_ns(0), 24 * consts::VIDEO_INTERVAL_DEFAULT_NS); - assert_eq!(queue_safe_ns(-1), 24 * consts::VIDEO_INTERVAL_DEFAULT_NS); - } - #[test] fn interval_ema_takes_the_first_usable_delta_whole() { assert_eq!(interval_ema(0, 16_666_666), 16_666_666); diff --git a/crates/irl-source/src/video/intake.rs b/crates/irl-source/src/video/intake.rs index 4754941..261f3e4 100644 --- a/crates/irl-source/src/video/intake.rs +++ b/crates/irl-source/src/video/intake.rs @@ -181,11 +181,11 @@ pub fn handle_frame( // this thread does not own. let pts_ns = ffmpeg::rescale_q(pts, tb, ffmpeg::NS_TIME_BASE); - // Frame interval EMA, for the frame rate the stats line reports and the - // lead warning's frame budget. Measured rather than taken from - // avg_frame_rate, which live SRT/RTMP demuxers routinely leave unset or - // wrong. Out-of-range deltas (PTS repair, discontinuities, reordering) are - // skipped rather than smoothed in. + // Frame interval EMA, for the frame rate the stats line reports. + // Measured rather than taken from avg_frame_rate, which live SRT/RTMP + // demuxers routinely leave unset or wrong. Out-of-range deltas (PTS + // repair, discontinuities, reordering) are skipped rather than smoothed + // in. let delta = pts_ns - state.prev_pts_ns; let measurable = state.prev_pts_ns != 0; state.prev_pts_ns = pts_ns; diff --git a/crates/irl-source/src/video/output.rs b/crates/irl-source/src/video/output.rs index 584d1c9..5640363 100644 --- a/crates/irl-source/src/video/output.rs +++ b/crates/irl-source/src/video/output.rs @@ -9,7 +9,7 @@ use std::sync::atomic::Ordering::Relaxed; use ffmpeg::sys::{AVColorRange, AVColorSpace, AVColorTransferCharacteristic}; use ffmpeg::{AVPixelFormat, Frame, FramePool, Scaler}; -use irl_core::{consts, timing, video_time}; +use irl_core::{consts, video_time}; use obs::{ColorRange, ColorSpace, VideoFormat, VideoFrame}; use crate::video::thread::VideoThread; @@ -183,7 +183,6 @@ impl VideoThread { let state = self.shared.audio_state(); (state.playout_mapping(), state.startup_warmup_remaining_ms) }; - let frame_interval_ns = self.shared.conn.video_frame_interval_ns.load(Relaxed); // No audio-stream test: a published mapping already implies the pump // handed OBS a real chunk, so it implies the audio stream. @@ -192,9 +191,7 @@ impl VideoThread { let delay_ns = self.delay.delay_ns(); if let Some(mapped) = mapping.map(pts_ns) { - let mapped = mapped.saturating_add(delay_ns); - self.record_lead(mapped as i64, now, frame_interval_ns); - return mapped; + return mapped.saturating_add(delay_ns); } // `conn.video_ts_init` is the authority here: it is cleared on a @@ -229,7 +226,6 @@ impl VideoThread { } } - self.record_lead(computed as i64, now, frame_interval_ns); computed } @@ -245,9 +241,9 @@ impl VideoThread { /// The current stream-PTS → OBS-clock offset, for re-deriving the due time /// of frames already queued. /// - /// Free of the side effects in [`Self::due_time`]: the lead warning and the - /// fallback anchor belong to a frame arriving, and running them for every - /// queued frame would report the queue rather than the stream. + /// Free of the side effects in [`Self::due_time`]: the fallback anchor + /// belongs to a frame arriving, and running it for every queued frame + /// would report the queue rather than the stream. /// `None` when there is no audio to slave to, or when the mapping has been /// gone long enough that holding it would be a guess; the caller then /// keeps the due times the frames arrived with. The standing video delay @@ -268,42 +264,6 @@ impl VideoThread { Some(self.playout_offset_ns + self.delay.delay_ns() as i64) } - /// Warn when the mapping placed this frame further ahead of wall clock - /// than libobs can queue. - /// - /// The lead is never clamped. What libobs queues is the lead's *growth* - /// since its play head last anchored, not its size, so a large but steady - /// lead queues nothing and clamping it would only shift video ahead of - /// audio. - pub fn record_lead(&mut self, ts: i64, now: u64, frame_interval_ns: i64) { - let lead_ns = ts - now as i64; - let frame_interval_ns = if frame_interval_ns <= 0 { - consts::VIDEO_INTERVAL_DEFAULT_NS - } else { - frame_interval_ns - }; - let queue_safe_ns = self.shared.hot.watermarks().target_ms as i64 * 1_000_000 - + video_time::queue_safe_ns(frame_interval_ns); - if lead_ns <= queue_safe_ns { - return; - } - - // Only a risk while the lead is still climbing — a steady lead of any - // size is free — so this is a "watch this" line, not a fault. - if timing::throttle( - &mut self.lead_warn_time_ns, - now, - consts::VIDEO_LEAD_WARN_INTERVAL_NS, - ) { - irl_info!( - "Video lead {}ms is beyond what OBS can queue ({}ms at {:.0}fps); harmless while it holds steady, but a rise of that size would make OBS drop queued video", - lead_ns / 1_000_000, - queue_safe_ns / 1_000_000, - 1_000_000_000.0 / frame_interval_ns as f64 - ); - } - } - /* ── Output ───────────────────────────────────────────── */ /// Hand a system-memory frame to OBS with the timestamp pacing scheduled diff --git a/crates/irl-source/src/video/thread.rs b/crates/irl-source/src/video/thread.rs index 281042e..142898c 100644 --- a/crates/irl-source/src/video/thread.rs +++ b/crates/irl-source/src/video/thread.rs @@ -98,8 +98,6 @@ pub struct VideoThread { pub(crate) ts_init: bool, pub(crate) sys_base: u64, pub(crate) pts_base: i64, - /// Throttle for the "Video lead" line. - pub(crate) lead_warn_time_ns: u64, /// Set while libobs's async play head is unanchored, so the next frame out /// goes at its due time rather than a lead early. See /// [`Self::emit_slack_ns`] and [`Self::settle_anchor_candidate`]. @@ -159,7 +157,6 @@ impl VideoThread { ts_init: false, sys_base: 0, pts_base: 0, - lead_warn_time_ns: 0, // A fresh source has libobs's `last_frame_ts` at 0 too. anchor_pending: true, anchor_wait: AnchorWait::Idle, From 94d82b421736c96088d2cbd05a57feb3cfa1fb89 Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 04:42:00 +0200 Subject: [PATCH 08/13] feat(irl-source): video delay follows the audio playout offset instead of tracking it by hand The video delay was a stored offset on top of the audio playout offset, kept in step by hand-offs: the pump credited hold growth to AudioState::hold_built_ns, the video thread absorbed it, and raises and ramps shifted the queue. Those missed cases. After a Target Buffer edit during a hold build the delay stood at 66 ms for good, and concealment growth during an outage was never absorbed, so the picture stayed some 300 ms behind the sound for 17 to 24 s after the link came back. irl_core::video_delay now keeps an absolute floor F under the schedule: the worst arrival minus PTS seen, plus a tick for the decode. A frame is due at pts + max(O, min(F, O + max)), so the delay is clamp(F - O, 0, max) and is derived wherever it is needed. The raise, window, relax and ceiling rules are unchanged but act on F, and a raise still sets a whole number of ticks of delay against the offset in force. The liveness check moves into the same type, and a PTS timeline reset now forgets it with the floor. The relax window slides instead of running in fixed blocks, and the part of the floor below the offset, which is not on screen, drops at once instead of slewing. Otherwise a hold released for a recovered sender leaves a stale floor showing for up to ten seconds. Removed: VideoDelay::absorb, hold::credit_build, AudioState::hold_built_ns and hold_unbuilt_ns, VideoThread::absorb_audio_hold, PacingQueue::shift and shift_earlier. Queued frames keep their undelayed due time, so reschedule re-derives every due time, fallback-scheduled ones included. The audio hold itself is unchanged. video_delay_ms and vdelay= report the derived delay, republished every video cycle. When the playout alone takes it to or from zero, one info line says so. The fallback adds the startup audio lead whenever no mapping is published, matching the test that picks the mapping, so a chunk whose PTS ends at or below zero no longer leaves video with neither. Tests whose numbers moved: - a_delay_set_by_a_bad_first_frame_ramps_back_without_a_jump settles at 0, not one tick: the relax target is what frames need plus a tick of headroom in floor terms, and 100 ms in hand covers both. - the hand-back test drives only the mapping; nothing credits the build. - av_sync_sim: a Target Buffer edit during a build ends in sync (82.8 ms to 16.1 ms, the anchoring tick) and is no longer ignored. A stall during a build is in sync 0.1 s after it (was 17.4 s), a loss under the delay 1.6 s after it (was 24.4 s); both assert that now, and their settled windows start at 34 s and 30 s instead of 55 s and 60 s. The loss scenario checks that the delay stood before the loss rather than at the re-anchor, since concealment now takes it away first. - network_sim checks the built hold by the buffer level and the video margin instead of the removed credit. - video_delay unit tests are restated against the floor; the absorb ones go, and tests for a raise against a moved offset, the derived delay, slew visibility and reset are added. --- crates/irl-core/src/consts.rs | 13 +- crates/irl-core/src/pacing.rs | 54 +- crates/irl-core/src/video_delay.rs | 815 +++++++++++++--------- crates/irl-source/src/audio/hold.rs | 31 +- crates/irl-source/src/audio/mod.rs | 3 - crates/irl-source/src/audio/pump.rs | 6 - crates/irl-source/src/config.rs | 2 - crates/irl-source/src/shared.rs | 9 +- crates/irl-source/src/video/output.rs | 60 +- crates/irl-source/src/video/thread.rs | 274 ++++---- crates/irl-source/tests/av_sync_sim.rs | 90 ++- crates/irl-source/tests/network_sim.rs | 17 +- crates/irl-source/tests/video_pipeline.rs | 21 +- 13 files changed, 767 insertions(+), 628 deletions(-) diff --git a/crates/irl-core/src/consts.rs b/crates/irl-core/src/consts.rs index 398a6d4..3938738 100644 --- a/crates/irl-core/src/consts.rs +++ b/crates/irl-core/src/consts.rs @@ -287,7 +287,8 @@ pub const VIDEO_PACING_MAX_WAIT_MS: u64 = 50; /// fallback anyway. pub const VIDEO_ANCHOR_WAIT_MARGIN_MS: i64 = 1000; -/// Ceiling on the standing video delay (`irl_core::video_delay`). +/// Ceiling on the video delay (`irl_core::video_delay`): the video floor less +/// the audio playout offset. /// /// A late frame is shown on arrival with or without the delay, which only /// lets it be paced, so the ceiling is not a latency bound but a stop on a @@ -315,12 +316,12 @@ pub const VIDEO_LIVE_TOLERANCE_MS: u64 = 16; /// A stream that never stops catching up is measured anyway after this. pub const VIDEO_LIVE_MAX_WAIT_MS: u64 = 2000; /// How long every frame handed over must have needed less than the video -/// delay in force before it ramps back down. Long enough that one quiet -/// stretch on a link that really is late does not start a ramp a raise then -/// has to undo. +/// floor before it slews back down; the window slides. Long enough that one +/// quiet stretch on a link that really is late does not start a slew a raise +/// then has to undo. pub const VIDEO_DELAY_RELAX_WINDOW_MS: u64 = 10_000; -/// Surplus delay below this is left alone: a ramp is not worth starting for -/// a couple of canvas ticks. +/// A floor higher than frames need by less than this is left alone: a slew +/// is not worth starting for a couple of canvas ticks. pub const VIDEO_DELAY_RELAX_MIN_MS: u64 = 50; /// How early, against its due time, video stamped exactly the measured skew /// behind its audio should be in hand once the audio hold diff --git a/crates/irl-core/src/pacing.rs b/crates/irl-core/src/pacing.rs index 48e272f..1c05a73 100644 --- a/crates/irl-core/src/pacing.rs +++ b/crates/irl-core/src/pacing.rs @@ -91,15 +91,16 @@ impl PacingQueue { } } - /// Re-derive every due time: `due = map(pts)`. + /// Re-derive every due time: `due = map(pts, frame)`. The frame is + /// passed so the caller can keep its own scheduling state on it. /// /// Rescheduling against one offset per cycle preserves the spacing /// between frames and moves the whole queue with the audio it is mapped /// to, so video follows a latency change instead of trailing it for the /// depth of the queue. - pub fn reschedule(&mut self, map: impl Fn(i64) -> u64) { + pub fn reschedule(&mut self, mut map: impl FnMut(i64, &mut F) -> u64) { for entry in &mut self.entries { - entry.due_ns = map(entry.pts_ns); + entry.due_ns = map(entry.pts_ns, &mut entry.frame); } } @@ -144,23 +145,6 @@ impl PacingQueue { self.entries.front().map(|e| &e.frame) } - /// Move every due time later by `delta_ns`: the video delay was raised. - /// Unlike [`Self::reschedule`] this needs no mapping, so it also serves a - /// queue scheduled on the video-only fallback. - pub fn shift(&mut self, delta_ns: u64) { - for entry in &mut self.entries { - entry.due_ns = entry.due_ns.saturating_add(delta_ns); - } - } - - /// Move every due time earlier by `delta_ns`: the video delay is ramping - /// back down. - pub fn shift_earlier(&mut self, delta_ns: u64) { - for entry in &mut self.entries { - entry.due_ns = entry.due_ns.saturating_sub(delta_ns); - } - } - /// Pop the head. pub fn pop(&mut self) -> Option { let entry = self.entries.pop_front()?; @@ -281,7 +265,7 @@ mod tests { } // The offset the audio side publishes moves by +250 ms. - q.reschedule(|pts| (pts + 250_000_000) as u64); + q.reschedule(|pts, _| (pts + 250_000_000) as u64); let dues: Vec = q.entries.iter().map(|e| e.due_ns).collect(); assert_eq!(dues[0], 250_000_000); for w in dues.windows(2) { @@ -289,36 +273,10 @@ mod tests { } // And back the other way: every due time is re-derived, not adjusted. - q.reschedule(|pts| (pts + 100_000_000) as u64); + q.reschedule(|pts, _| (pts + 100_000_000) as u64); assert_eq!(q.next_due(), Some(100_000_000)); } - #[test] - fn shift_moves_every_due_time_and_keeps_the_head() { - let mut q = queue(); - push( - &mut q, - TestFrame { - pts_ns: 0, - bytes: 1, - }, - 1_000, - ); - push( - &mut q, - TestFrame { - pts_ns: 40, - bytes: 1, - }, - 1_040, - ); - q.shift(500); - assert_eq!(q.next_due(), Some(1_500)); - assert_eq!(q.head().map(|f| f.pts_ns), Some(0)); - q.pop(); - assert_eq!(q.next_due(), Some(1_540)); - } - #[test] fn empty_queue_has_no_verdict() { let mut q = queue(); diff --git a/crates/irl-core/src/video_delay.rs b/crates/irl-core/src/video_delay.rs index 1e2682d..3fc2353 100644 --- a/crates/irl-core/src/video_delay.rs +++ b/crates/irl-core/src/video_delay.rs @@ -1,28 +1,43 @@ -//! The standing delay on the video schedule. +//! The floor under the video schedule. //! //! A frame handed to libobs after it is due is shown a tick late, and when two //! late frames land inside one tick libobs drops the older one ("low fps"). -//! The plugin cannot make a late frame early, so it moves the whole video -//! schedule later by this delay, sized from the worst shortfall seen. It -//! trades a known lip-sync error for a smooth picture. +//! The plugin cannot make a late frame early, so it keeps every due time at +//! or above a floor sized from how late frames reach it: `F`, the worst +//! `arrival - pts` seen plus an allowance, in stream-to-OBS-clock terms like +//! the audio playout offset `O` itself. A frame is due at +//! `pts + max(O, min(F, O + max))`, so the delay against the audio, +//! `clamp(F - O, 0, max)`, is derived and never stored. When the audio playout +//! grows (an audio hold building, concealment) the delay shrinks by the same +//! amount and nothing on screen moves; when it shrinks (a re-anchor, a hold +//! released) the floor keeps video where its frames can still be shown. The +//! delay is a lip-sync error traded for a smooth picture. +//! +//! The floor carries the stream's PTS epoch, so it is forgotten whenever the +//! timeline breaks. //! //! Before libobs's play head is anchored nothing has been shown, so a -//! shortfall raises the delay at once. Afterwards a raise moves the picture, -//! so it takes a shortfall that recurs across a window. +//! shortfall raises the floor at once, measured only on video that is live +//! rather than a relay's catch-up ([`ArrivalFloor`]). Afterwards a raise moves +//! the picture, so it takes a shortfall that recurs across a window. //! //! The pre-anchor reading is one frame and can be a startup burst, so the -//! delay also comes back down: when a whole window of frames needed less by -//! more than a threshold, it ramps (never steps) to what the window needed -//! plus a tick, at the Catch-Up Speed the caller passes. A raise cancels a -//! ramp. +//! floor also comes back down: when a whole window of frames needed less by +//! more than a threshold, it slews to what the window needed plus a tick, at +//! the Catch-Up Speed the caller passes. Only the part of the floor above the +//! offset is on screen, so only that part slews; the rest goes at once. A +//! raise cancels a slew. //! //! The bar is "not late", not "a full delivery lead early": the lead covers //! the pacing timer oversleeping, and a frame handed over on arrival never //! sleeps. +use crate::arrival::ArrivalFloor; use crate::consts; +use crate::window::{Extreme, WindowExtreme}; -/// One change of the delay, for the caller to log and mirror into the stats. +/// One raise of the floor, for the caller to log, as the delay against the +/// audio offset in force. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DelayRaise { /// The delay before. @@ -36,275 +51,211 @@ pub struct DelayRaise { pub capped: bool, } -/// The delay turned out larger than a whole window of frames needed, and -/// starts ramping down. +/// The floor turned out higher than a whole window of frames needed, and +/// starts slewing down. As delays against the audio offset in force. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DelayRelax { /// The delay in force. pub from_ns: u64, - /// Where the ramp ends. + /// Where the slew ends. pub to_ns: u64, } -/// One step of the ramp. +/// One step of the slew. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RampStep { /// The delay now. pub delay_ns: u64, - /// How much earlier every due time moved this step. - pub moved_ns: u64, - /// The ramp reached its target. + /// The slew reached its target. pub done: bool, } -/// What the frames handed over since `since_ns` needed, at most. -#[derive(Debug, Clone, Copy)] -struct Surplus { - since_ns: u64, - frames: u32, - worst_need_ns: u64, -} - /// Late frames seen since the play head was anchored, awaiting a verdict. #[derive(Debug, Clone, Copy)] struct Window { since_ns: u64, last_ns: u64, frames: u32, - /// The largest delay any frame in the window needed. - worst_ns: u64, + /// The highest floor any frame in the window needed. + worst_ns: i64, } -/// The standing delay and the observation window behind it. +/// The floor, the observation windows behind it and the liveness check that +/// gates the pre-anchor reading. +/// +/// Every `offset_ns` parameter is the frame's undelayed due time minus its +/// PTS: the audio playout offset when the frame is mapped through it, or the +/// video-only fallback's equivalent. #[derive(Debug)] pub struct VideoDelay { - delay_ns: u64, + floor_ns: Option, max_ns: u64, window_ns: u64, min_frames: u32, window: Option, /// Relaxing: the observation window, the threshold a surplus must exceed, - /// what has been observed, and the ramp in progress. A zero window turns - /// relaxing off. + /// the highest floor frames needed across the window, since when frames + /// have arrived without a gap, and the slew in progress. A zero window + /// turns relaxing off. relax_window_ns: u64, relax_min_ns: u64, - surplus: Option, - ramp_to_ns: Option, + needs: WindowExtreme, + needs_since_ns: Option, + last_arrival_ns: u64, + ramp_to_ns: Option, ramp_at_ns: u64, + live: ArrivalFloor, } impl VideoDelay { - /// A zero delay that may grow to `max_ns`, and after the anchor only when - /// at least `min_frames` frames fall short within `window_ns`, spread over - /// at least half of it. + /// No floor yet; the delay it implies may grow to `max_ns`, and after the + /// anchor only when at least `min_frames` frames fall short within + /// `window_ns`, spread over at least half of it. pub fn new(max_ns: u64, window_ns: u64, min_frames: u32) -> Self { Self { - delay_ns: 0, + floor_ns: None, max_ns, window_ns, min_frames, window: None, relax_window_ns: 0, relax_min_ns: 0, - surplus: None, + needs: WindowExtreme::new(0, Extreme::Highest), + needs_since_ns: None, + last_arrival_ns: 0, ramp_to_ns: None, ramp_at_ns: 0, + live: ArrivalFloor::default(), } } - /// Let the delay ramp back down when every frame across `window_ns` + /// Let the floor slew back down when every frame across `window_ns` /// needed less than it by more than `min_ns`. #[cfg(test)] #[must_use] fn with_relax(mut self, window_ns: u64, min_ns: u64) -> Self { self.relax_window_ns = window_ns; self.relax_min_ns = min_ns; + self.needs = WindowExtreme::new(window_ns, Extreme::Highest); self } - /// The delay every due time carries. - pub fn delay_ns(&self) -> u64 { - self.delay_ns + /// The evidence for a lower floor is void. + fn forget_needs(&mut self) { + self.needs.clear(); + self.needs_since_ns = None; + self.ramp_to_ns = None; } - /// Whether the delay has reached its ceiling: a shortfall past this cannot - /// be scheduled away, and the frames stay late. - pub fn at_ceiling(&self) -> bool { - self.delay_ns >= self.max_ns + /// The delay a frame scheduled at `offset_ns` carries. + pub fn delay_ns(&self, offset_ns: i64) -> u64 { + self.floor_ns.map_or(0, |floor_ns| { + floor_ns + .saturating_sub(offset_ns) + .clamp(0, self.max_ns as i64) as u64 + }) } - /// Back to zero, for a new connection or a cleared source. + /// The due time of a frame at `pts_ns` whose undelayed due time is + /// `base_ns`. + pub fn due_ns(&self, pts_ns: i64, base_ns: u64) -> u64 { + base_ns.saturating_add(self.delay_ns(base_ns as i64 - pts_ns)) + } + + /// Whether the delay at `offset_ns` has reached its ceiling: a shortfall + /// past this cannot be scheduled away, and the frames stay late. + pub fn at_ceiling(&self, offset_ns: i64) -> bool { + self.delay_ns(offset_ns) >= self.max_ns + } + + /// Forget the floor and everything measured: a new connection, a cleared + /// source, or a timeline that broke, since the floor carries the PTS + /// epoch. pub fn reset(&mut self) { - self.delay_ns = 0; + self.floor_ns = None; self.window = None; - self.surplus = None; - self.ramp_to_ns = None; + self.forget_needs(); + self.live.reset(); } - /// The whole delay a frame needs so that it is in hand `want_ns` before it - /// is due, in whole canvas ticks. `margin_ns` is how early the frame was in - /// hand against its due time *as the pacing queue holds it*, delay - /// included, so the delay is taken back out before the shortfall is - /// measured. - fn required_ns(&self, margin_ns: i64, want_ns: u64, tick_ns: u64) -> u64 { - let raw_margin_ns = margin_ns - self.delay_ns as i64; - let shortfall_ns = want_ns as i64 - raw_margin_ns; - if shortfall_ns <= 0 { - return 0; - } - round_up_to_tick(shortfall_ns as u64, tick_ns) + /// A video packet with `pts_ns` reached the video thread at + /// `received_ns`, for the liveness check. + pub fn note_packet(&mut self, received_ns: u64, pts_ns: i64) { + self.live.note(received_ns, pts_ns); } - /// Raise to `need_ns` if that is more than the delay already is. - fn raise_to(&mut self, need_ns: u64, frames: u32) -> Option { - let to_ns = need_ns.min(self.max_ns); - if to_ns <= self.delay_ns { - return None; - } - let raise = DelayRaise { - from_ns: self.delay_ns, - to_ns, - frames, - capped: need_ns > self.max_ns, - }; - self.delay_ns = to_ns; - // The evidence for a smaller delay is void. - self.surplus = None; - self.ramp_to_ns = None; - Some(raise) + /// Whether video arriving now is live rather than a relay's replay still + /// catching up, which says nothing about the sender. + pub fn live(&self, now_ns: u64) -> bool { + self.live.live(now_ns) } - /// A frame handed over after the anchor, `arrival_margin_ns` being its - /// due time minus its packet's arrival, delay included. Records what it - /// needed; once a whole relax window needed less than the delay in force - /// by more than the threshold, starts a ramp down to that plus a tick. - pub fn note_arrival( + /// Raise the floor so that a frame scheduled at `offset_ns` is due no + /// earlier than `need_ns` past its PTS, in whole canvas ticks of delay, + /// if that is more than the delay already is. + fn raise_to( &mut self, - now_ns: u64, - arrival_margin_ns: i64, + need_ns: i64, + offset_ns: i64, tick_ns: u64, - ) -> Option { - if self.relax_window_ns == 0 { - return None; - } - let need_ns = self.required_ns(arrival_margin_ns, tick_ns, tick_ns); - // A ramp must not run past what a frame in hand needs. - if let Some(to_ns) = self.ramp_to_ns { - let floor_ns = need_ns + tick_ns; - if floor_ns >= self.delay_ns { - self.ramp_to_ns = None; - } else if floor_ns > to_ns { - self.ramp_to_ns = Some(floor_ns); - } - } - let surplus = self.surplus.get_or_insert(Surplus { - since_ns: now_ns, - frames: 0, - worst_need_ns: 0, - }); - surplus.frames += 1; - surplus.worst_need_ns = surplus.worst_need_ns.max(need_ns); - if now_ns.saturating_sub(surplus.since_ns) < self.relax_window_ns { - return None; - } - let seen = *surplus; - self.surplus = None; - if seen.frames < self.min_frames || self.ramp_to_ns.is_some() { + frames: u32, + ) -> Option { + let shortfall_ns = need_ns.saturating_sub(offset_ns); + if shortfall_ns <= 0 { return None; } - let to_ns = seen.worst_need_ns + tick_ns; - if self.delay_ns <= to_ns + self.relax_min_ns { + let wanted_ns = round_up_to_tick(shortfall_ns as u64, tick_ns); + let to_ns = wanted_ns.min(self.max_ns); + let from_ns = self.delay_ns(offset_ns); + if to_ns <= from_ns { return None; } - self.ramp_to_ns = Some(to_ns); - self.ramp_at_ns = now_ns; - Some(DelayRelax { - from_ns: self.delay_ns, + self.floor_ns = Some(offset_ns + to_ns as i64); + self.forget_needs(); + Some(DelayRaise { + from_ns, to_ns, + frames, + capped: wanted_ns > self.max_ns, }) } - /// Advance a ramp in progress to `now_ns`, moving the delay down by - /// `rate` of the time since the last step (0.05 plays video 5% fast). - pub fn ramp(&mut self, now_ns: u64, rate: f64) -> Option { - let to_ns = self.ramp_to_ns?; - let elapsed_ns = now_ns.saturating_sub(self.ramp_at_ns); - self.ramp_at_ns = now_ns; - let want_ns = (elapsed_ns as f64 * rate.max(0.0)) as u64; - let moved_ns = want_ns.min(self.delay_ns.saturating_sub(to_ns)); - self.delay_ns -= moved_ns; - let done = self.delay_ns <= to_ns; - if done { - self.ramp_to_ns = None; - } - (moved_ns > 0 || done).then_some(RampStep { - delay_ns: self.delay_ns, - moved_ns, - done, - }) - } - - /// The audio playout moved every due time `grown_ns` later because the - /// audio hold ([`crate::audio_hold`]) is building its cushion. Take as - /// much of that back out of the delay as there is, so due times stay - /// where they were and the lip-sync error the delay stands for shrinks by - /// the same amount; nothing on screen moves. Returns how much the delay - /// came down. - /// - /// Readings already taken were measured against the playout as it was, - /// so what they called for is `grown_ns` less now. - pub fn absorb(&mut self, grown_ns: u64) -> u64 { - let moved_ns = grown_ns.min(self.delay_ns); - self.delay_ns -= moved_ns; - if let Some(window) = self.window.as_mut() { - window.worst_ns = window.worst_ns.saturating_sub(grown_ns); - } - if let Some(surplus) = self.surplus.as_mut() { - surplus.worst_need_ns = surplus.worst_need_ns.saturating_sub(grown_ns); - } - if let Some(to_ns) = self.ramp_to_ns { - let to_ns = to_ns.saturating_sub(grown_ns); - self.ramp_to_ns = (to_ns < self.delay_ns).then_some(to_ns); - } - moved_ns - } - - /// A frame considered for anchoring the play head, `margin_ns` being its - /// due time minus its packet's arrival. Nothing has been shown yet, so a - /// shortfall against `want_ns` raises the delay at once. + /// A frame considered for anchoring the play head: its packet reached + /// the video thread at `received_ns`. Nothing has been shown yet, so a + /// frame not in hand a canvas tick before it is due (the decode still to + /// come) raises the floor at once. pub fn before_anchor( &mut self, - margin_ns: i64, - want_ns: u64, + received_ns: u64, + pts_ns: i64, + offset_ns: i64, tick_ns: u64, ) -> Option { self.window = None; - let need_ns = self.required_ns(margin_ns, want_ns, tick_ns); - self.raise_to(need_ns, 0) + let need_ns = received_ns as i64 - pts_ns + tick_ns as i64; + self.raise_to(need_ns, offset_ns, tick_ns, 0) } - /// A frame handed over after the anchor, `margin_ns` being its due time - /// minus the hand-over time. A shortfall against `want_ns` is recorded; - /// the delay is raised only once the window says it recurs. + /// A frame handed over at `now_ns`, after the anchor. A frame handed over + /// past its due time is recorded; the floor is raised only once the + /// window says that recurs. pub fn note( &mut self, now_ns: u64, - margin_ns: i64, - want_ns: u64, + pts_ns: i64, + offset_ns: i64, tick_ns: u64, ) -> Option { - // Against the delay its due time carries: a raise below must not make - // this frame look short by the amount just added. - let need_ns = self.required_ns(margin_ns, want_ns, tick_ns); + let need_ns = now_ns as i64 - pts_ns; // An elapsed window is judged on what it holds, before this frame // starts the next one. - let raise = self.expire(now_ns); - if need_ns > self.delay_ns { + let raise = self.expire(now_ns, offset_ns, tick_ns); + if need_ns > offset_ns + self.delay_ns(offset_ns) as i64 { let window = self.window.get_or_insert(Window { since_ns: now_ns, last_ns: now_ns, frames: 0, - worst_ns: 0, + worst_ns: i64::MIN, }); window.last_ns = now_ns; window.frames += 1; @@ -316,7 +267,7 @@ impl VideoDelay { /// Judge a window that has run its course. Called once per pacing cycle as /// well as from [`Self::note`], so a window whose late frames simply /// stopped is closed rather than kept open for the next one. - pub fn expire(&mut self, now_ns: u64) -> Option { + pub fn expire(&mut self, now_ns: u64, offset_ns: i64, tick_ns: u64) -> Option { let window = self.window?; if now_ns.saturating_sub(window.since_ns) < self.window_ns { return None; @@ -327,16 +278,109 @@ impl VideoDelay { if !recurring { return None; } - self.raise_to(window.worst_ns, window.frames) + self.raise_to(window.worst_ns, offset_ns, tick_ns, window.frames) + } + + /// A frame handed over at `now_ns` after the anchor, whose packet reached + /// the video thread at `received_ns`. Records the floor it needed (a + /// canvas tick in hand); once every frame across a whole relax window, + /// arriving without a gap, needed less than the floor by more than the + /// threshold, starts a slew down to what they needed plus a tick. + /// + /// The window slides, so a sender that recovers is noticed one window + /// after it did. The audio hold releases on the same kind of window, and + /// the floor must have come down by then: a hold released over a stale + /// floor would put the picture behind the sound for no reason. + pub fn note_arrival( + &mut self, + now_ns: u64, + received_ns: u64, + pts_ns: i64, + offset_ns: i64, + tick_ns: u64, + ) -> Option { + if self.relax_window_ns == 0 { + return None; + } + let tick = tick_ns as i64; + let need_ns = received_ns as i64 - pts_ns + tick; + // A slew must not run past what a frame in hand needs. + if let (Some(to_ns), Some(floor_ns)) = (self.ramp_to_ns, self.floor_ns) { + let keep_ns = need_ns + tick; + if keep_ns >= floor_ns { + self.ramp_to_ns = None; + } else if keep_ns > to_ns { + self.ramp_to_ns = Some(keep_ns); + } + } + // Video that stopped for longer than a raise window is not a window + // of frames that needed less. + if now_ns.saturating_sub(self.last_arrival_ns) > self.window_ns { + self.needs.clear(); + self.needs_since_ns = None; + } + self.last_arrival_ns = now_ns; + let since_ns = *self.needs_since_ns.get_or_insert(now_ns); + self.needs.push(now_ns, need_ns); + if now_ns.saturating_sub(since_ns) < self.relax_window_ns || self.ramp_to_ns.is_some() { + return None; + } + let floor_ns = self.floor_ns?; + let to_ns = self.needs.value()? + tick; + if floor_ns <= to_ns.saturating_add(self.relax_min_ns as i64) { + return None; + } + self.ramp_to_ns = Some(to_ns); + self.ramp_at_ns = now_ns; + let to_delay_ns = to_ns.saturating_sub(offset_ns).clamp(0, self.max_ns as i64) as u64; + Some(DelayRelax { + from_ns: self.delay_ns(offset_ns), + to_ns: to_delay_ns, + }) + } + + /// Advance a slew in progress to `now_ns`, moving the delay a frame at + /// `offset_ns` carries down by `rate` of the time since the last step + /// (0.05 plays video 5% fast). + pub fn ramp(&mut self, now_ns: u64, rate: f64, offset_ns: i64) -> Option { + let to_ns = self.ramp_to_ns?; + let Some(floor_ns) = self.floor_ns else { + self.ramp_to_ns = None; + return None; + }; + let elapsed_ns = now_ns.saturating_sub(self.ramp_at_ns); + self.ramp_at_ns = now_ns; + let before_ns = self.delay_ns(offset_ns); + // Above the ceiling and below the offset the floor is not on screen, + // so only the part between them slews. + let start_ns = floor_ns.min(offset_ns.saturating_add(self.max_ns as i64)); + let mut next_ns = if to_ns >= start_ns { + to_ns + } else { + let step_ns = (elapsed_ns as f64 * rate.max(0.0)) as i64; + (start_ns - step_ns).max(to_ns) + }; + if next_ns <= offset_ns { + next_ns = to_ns; + } + self.floor_ns = Some(next_ns); + let done = next_ns <= to_ns; + if done { + self.ramp_to_ns = None; + } + let delay_ns = self.delay_ns(offset_ns); + (delay_ns < before_ns || done).then_some(RampStep { delay_ns, done }) } } impl Default for VideoDelay { - /// The plugin's delay, from the `VIDEO_DELAY_*` constants, relaxing on. + /// The plugin's floor, from the `VIDEO_DELAY_*` constants, relaxing on. fn default() -> Self { + let relax_window_ns = consts::VIDEO_DELAY_RELAX_WINDOW_MS * 1_000_000; Self { - relax_window_ns: consts::VIDEO_DELAY_RELAX_WINDOW_MS * 1_000_000, + relax_window_ns, relax_min_ns: consts::VIDEO_DELAY_RELAX_MIN_MS * 1_000_000, + needs: WindowExtreme::new(relax_window_ns, Extreme::Highest), ..Self::new( consts::VIDEO_DELAY_MAX_MS * 1_000_000, consts::VIDEO_DELAY_WINDOW_MS * 1_000_000, @@ -362,184 +406,265 @@ mod tests { const TICK: u64 = 16_666_667; const WINDOW: u64 = 1_000_000_000; const MAX: u64 = 1_000_000_000; + /// A playout offset: stream PTS plus this is the OBS time the audio of it + /// plays at. + const O: i64 = -7_000_000_000; + /// A frame's PTS, and the OBS time it is due at with no delay. + const PTS: i64 = 10_000_000_000; + const DUE: u64 = (PTS + O) as u64; fn delay() -> VideoDelay { VideoDelay::new(MAX, WINDOW, 3) } + /// A frame at `PTS` reaching the thread `margin_ns` before its undelayed + /// due time, considered for the anchor at offset `O`. + fn before_anchor(d: &mut VideoDelay, margin_ns: i64) -> Option { + d.before_anchor((DUE as i64 - margin_ns) as u64, PTS, O, TICK) + } + + /// The frame `i` frames after `PTS`, handed over at `now_ns` against + /// offset `O`. + fn note(d: &mut VideoDelay, now_ns: u64, i: u64) -> Option { + d.note(now_ns, PTS + (i * 33_333_333) as i64, O, TICK) + } + + /// The frame `i` frames after `PTS`, handed over `late_ns` after it is + /// due (negative: early). Returns the raise and the hand-over time. + fn note_late(d: &mut VideoDelay, i: u64, late_ns: i64) -> (Option, u64) { + let at = (due_of(d, i) as i64 + late_ns) as u64; + (note(d, at, i), at) + } + + /// When the frame `i` frames after `PTS` is due at offset `O`, delay + /// included. + fn due_of(d: &VideoDelay, i: u64) -> u64 { + let pts = PTS + (i * 33_333_333) as i64; + d.due_ns(pts, (pts + O) as u64) + } + #[test] fn a_frame_in_hand_early_enough_needs_no_delay() { let mut d = delay(); - assert_eq!(d.before_anchor(TICK as i64, TICK, TICK), None); - assert_eq!(d.before_anchor(500_000_000, TICK, TICK), None); - assert_eq!(d.delay_ns(), 0); + assert_eq!(before_anchor(&mut d, TICK as i64), None); + assert_eq!(before_anchor(&mut d, 500_000_000), None); + assert_eq!(d.delay_ns(O), 0); + assert_eq!(due_of(&d, 0), DUE); } #[test] fn a_frame_with_no_margin_gets_the_wanted_allowance_before_the_anchor() { let mut d = delay(); - let raise = d.before_anchor(0, TICK, TICK).expect("raised"); + let raise = before_anchor(&mut d, 0).expect("raised"); assert_eq!(raise.from_ns, 0); assert_eq!(raise.to_ns, TICK); assert!(!raise.capped); - assert_eq!(d.delay_ns(), TICK); + assert_eq!(d.delay_ns(O), TICK); + assert_eq!(due_of(&d, 0), DUE + TICK); } #[test] fn a_late_frame_gets_its_lateness_plus_the_allowance_in_whole_ticks() { let mut d = delay(); // 150 ms late plus a 16.7 ms allowance is 166.7 ms: exactly 10 ticks. - let raise = d.before_anchor(-150_000_000, TICK, TICK).expect("raised"); + let raise = before_anchor(&mut d, -150_000_000).expect("raised"); assert_eq!(raise.to_ns, 10 * TICK); // 151 ms late needs an eleventh. let mut d = delay(); - let raise = d.before_anchor(-151_000_000, TICK, TICK).expect("raised"); + let raise = before_anchor(&mut d, -151_000_000).expect("raised"); assert_eq!(raise.to_ns, 11 * TICK); } #[test] - fn the_delay_only_ever_grows_before_the_anchor() { + fn the_floor_only_ever_rises_before_the_anchor() { let mut d = delay(); - d.before_anchor(-100_000_000, TICK, TICK).expect("raised"); - let first = d.delay_ns(); - // A frame with more margin (its due already carries the delay). + before_anchor(&mut d, -100_000_000).expect("raised"); + let first = d.delay_ns(O); assert_eq!( - d.before_anchor(first as i64 + 100_000_000, TICK, TICK), + before_anchor(&mut d, 100_000_000), None, - "a smaller shortfall must not lower the delay" + "a smaller shortfall must not lower the floor" ); - assert_eq!(d.delay_ns(), first); - // A frame with less raises it further. - let more = d.before_anchor(-200_000_000, TICK, TICK).expect("raised"); + assert_eq!(d.delay_ns(O), first); + let more = before_anchor(&mut d, -200_000_000).expect("raised"); assert_eq!(more.from_ns, first); assert!(more.to_ns > first); } #[test] - fn the_existing_delay_counts_toward_the_margin() { + fn a_frame_the_floor_already_covers_raises_nothing() { let mut d = delay(); - d.before_anchor(0, TICK, TICK).expect("raised to a tick"); - // Due times now carry the delay: a frame due a tick after it arrived - // has, net of the delay, no margin — exactly what the delay covers. - assert_eq!(d.before_anchor(TICK as i64, TICK, TICK), None); + before_anchor(&mut d, 0).expect("raised to a tick"); + // The same frame again is in hand a tick before its new due time. + assert_eq!(before_anchor(&mut d, 0), None); + assert_eq!(d.delay_ns(O), TICK); } #[test] fn the_ceiling_caps_the_delay_and_says_so() { let mut d = delay(); - assert!(!d.at_ceiling()); - let raise = d.before_anchor(-2_000_000_000, TICK, TICK).expect("raised"); + assert!(!d.at_ceiling(O)); + let raise = before_anchor(&mut d, -2_000_000_000).expect("raised"); assert_eq!(raise.to_ns, MAX); assert!(raise.capped); - assert!(d.at_ceiling()); + assert!(d.at_ceiling(O)); // Still short at the ceiling: nothing more to do, nothing to report. - assert_eq!(d.before_anchor(-3_000_000_000, TICK, TICK), None); - assert!(d.at_ceiling()); + assert_eq!(before_anchor(&mut d, -3_000_000_000), None); + assert!(d.at_ceiling(O)); + } + + #[test] + fn the_delay_is_derived_from_the_offset_in_force() { + let mut d = delay(); + before_anchor(&mut d, -300_000_000).expect("raised"); + let set = d.delay_ns(O); + assert_eq!(set, 19 * TICK); + // The audio playout grows by 100 ms (the audio hold building): the + // delay shrinks by as much, and the due time stays where it was. + let grown = O + 100_000_000; + assert_eq!(d.delay_ns(grown), set - 100_000_000); + assert_eq!( + d.due_ns(PTS, (PTS + grown) as u64), + d.due_ns(PTS, (PTS + O) as u64) + ); + // Grown past the floor: no delay, and video follows the audio. + let past = O + set as i64 + 50_000_000; + assert_eq!(d.delay_ns(past), 0); + assert_eq!(d.due_ns(PTS, (PTS + past) as u64), (PTS + past) as u64); + // And back below it (a re-anchor): the floor holds video back again. + assert_eq!(d.delay_ns(O), set); + // The ceiling applies to the derived delay, wherever the offset is. + assert_eq!(d.delay_ns(O - 5_000_000_000), MAX); + assert!(d.at_ceiling(O - 5_000_000_000)); + assert!(!d.at_ceiling(O)); } #[test] - fn a_delay_below_the_ceiling_is_not_at_it() { + fn a_raise_is_sized_against_the_offset_in_force() { let mut d = delay(); - d.before_anchor(-150_000_000, TICK, TICK).expect("raised"); - assert!(d.delay_ns() > 0); - assert!(!d.at_ceiling()); + before_anchor(&mut d, -300_000_000).expect("raised"); + // The playout has since grown by 250 ms, so a frame as late as the + // first only needs what is left, and raises nothing. + let grown = O + 250_000_000; + let received = (DUE as i64 + 300_000_000) as u64; + assert_eq!(d.before_anchor(received, PTS, grown, TICK), None); + // One 100 ms later than that does, to a whole number of ticks + // against the grown offset. + let raise = d + .before_anchor(received + 100_000_000, PTS, grown, TICK) + .expect("raised"); + assert_eq!(raise.to_ns, (150_000_000 + TICK).div_ceil(TICK) * TICK); + assert_eq!(d.delay_ns(grown), raise.to_ns); + } + + #[test] + fn reset_forgets_the_floor_and_the_liveness_history() { + let mut d = delay(); + for i in 0..20u64 { + d.note_packet(i * 33_333_333, (i * 33_333_333) as i64); + } + assert!(d.live(20 * 33_333_333)); + before_anchor(&mut d, -150_000_000).expect("raised"); d.reset(); - assert!(!d.at_ceiling()); + assert_eq!(d.delay_ns(O), 0); + assert_eq!(d.delay_ns(i64::MIN), 0); + assert!(!d.at_ceiling(O)); + assert!(!d.live(20 * 33_333_333)); + assert_eq!(d.expire(u64::MAX, O, TICK), None); } #[test] fn after_the_anchor_a_frame_handed_over_before_it_is_due_is_not_short() { let mut d = delay(); - let t0 = 10_000_000_000; // In hand with anything from a nanosecond to a full lead to spare. - for (i, margin) in [1, 5_000_000, 20_000_000, 33_333_334] + for (i, early) in [1, 5_000_000, 20_000_000, 33_333_334] .into_iter() .enumerate() { - assert_eq!(d.note(t0 + i as u64 * 300_000_000, margin, 0, TICK), None); + let i = i as u64 * 9; + assert_eq!(note_late(&mut d, i, -early).0, None); } - assert_eq!(d.expire(t0 + 2 * WINDOW), None); - assert_eq!(d.delay_ns(), 0); + assert_eq!(d.expire(u64::MAX, O, TICK), None); + assert_eq!(d.delay_ns(O), 0); } #[test] - fn after_the_anchor_one_late_frame_does_not_raise_the_delay() { + fn after_the_anchor_one_late_frame_does_not_raise_the_floor() { let mut d = delay(); - let t0 = 10_000_000_000; - assert_eq!(d.note(t0, -5_000_000, 0, TICK), None); - assert_eq!(d.expire(t0 + WINDOW), None); - assert_eq!(d.delay_ns(), 0); + let (raise, at) = note_late(&mut d, 0, 5_000_000); + assert_eq!(raise, None); + assert_eq!(d.expire(at + WINDOW, O, TICK), None); + assert_eq!(d.delay_ns(O), 0); } #[test] - fn after_the_anchor_a_burst_of_late_frames_does_not_raise_the_delay() { + fn after_the_anchor_a_burst_of_late_frames_does_not_raise_the_floor() { let mut d = delay(); - let t0 = 10_000_000_000; - // Six frames late within 100 ms: one hiccup, not a standing shortfall. + // Six frames late within 200 ms: one hiccup, not a standing shortfall. + let mut at = 0; for i in 0..6 { - assert_eq!(d.note(t0 + i * TICK, -5_000_000, 0, TICK), None); + let (raise, t) = note_late(&mut d, i, 5_000_000); + assert_eq!(raise, None); + at = t; } - assert_eq!(d.expire(t0 + WINDOW), None); - assert_eq!(d.delay_ns(), 0); + assert_eq!(d.expire(at + WINDOW, O, TICK), None); + assert_eq!(d.delay_ns(O), 0); } #[test] - fn after_the_anchor_a_recurring_shortfall_raises_the_delay_to_its_worst() { + fn after_the_anchor_a_recurring_shortfall_raises_the_floor_to_its_worst() { let mut d = delay(); - let t0 = 10_000_000_000; // Three frames spread across the window, 5, 20 and 1 ms late: the // worst needs 20 ms, which is two ticks. - assert_eq!(d.note(t0, -5_000_000, 0, TICK), None); - assert_eq!(d.note(t0 + 600_000_000, -20_000_000, 0, TICK), None); + let (raise, t0) = note_late(&mut d, 0, 5_000_000); + assert_eq!(raise, None); + assert_eq!(note_late(&mut d, 18, 20_000_000).0, None); assert_eq!( - d.note(t0 + 900_000_000, -1_000_000, 0, TICK), + note_late(&mut d, 27, 1_000_000).0, None, "the window has not run its course" ); - let raise = d.expire(t0 + WINDOW).expect("raised at the window's end"); + let raise = d + .expire(t0 + WINDOW, O, TICK) + .expect("raised at the window's end"); assert_eq!(raise.to_ns, 2 * TICK); assert_eq!(raise.frames, 3); - assert_eq!(d.delay_ns(), 2 * TICK); + assert_eq!(d.delay_ns(O), 2 * TICK); } #[test] fn a_late_frame_after_the_window_judges_it_and_starts_the_next() { let mut d = delay(); - let t0 = 10_000_000_000; + let t0 = due_of(&d, 0) + 5_000_000; for i in 0..3 { - assert_eq!(d.note(t0 + i * 400_000_000, -5_000_000, 0, TICK), None); + let k = i * 12; + assert_eq!(note_late(&mut d, k, 5_000_000).0, None); } - // The fourth frame arrives after the window elapsed: the raise comes - // out of this call, judged on the three before it. + // The fourth frame is handed over after the window elapsed: the raise + // comes out of this call, judged on the three before it. let t = t0 + WINDOW + 1; - let raise = d.note(t, -5_000_000, 0, TICK).expect("raised"); + let k = 30; + let late_by = t as i64 - due_of(&d, k) as i64; + assert!((0..TICK as i64).contains(&late_by)); + let raise = note(&mut d, t, k).expect("raised"); assert_eq!(raise.frames, 3); assert_eq!(raise.to_ns, TICK); - // The fourth frame needed a tick too, and the raise just provided it: - // it does not start a new window. - assert_eq!(d.expire(t + WINDOW), None); + // The fourth frame needed under a tick too, and the raise just + // provided it: it does not start a new window. + assert_eq!(d.expire(t + WINDOW, O, TICK), None); } #[test] - fn frames_covered_by_the_delay_are_not_counted() { + fn frames_the_floor_covers_are_not_counted() { let mut d = delay(); - d.before_anchor(0, TICK, TICK).expect("raised to a tick"); - let t0 = 10_000_000_000; + before_anchor(&mut d, 0).expect("raised to a tick"); for i in 0..5 { - // Due carries the delay; net of it the frame is exactly on time. - assert_eq!(d.note(t0 + i * 300_000_000, TICK as i64, 0, TICK), None); + let k = i * 9; + // Handed over exactly at the delayed due time: on time. + assert_eq!(note_late(&mut d, k, 0).0, None); } - assert_eq!(d.expire(t0 + 2 * WINDOW), None); - assert_eq!(d.delay_ns(), TICK); - } - - #[test] - fn reset_returns_to_zero() { - let mut d = delay(); - d.before_anchor(0, TICK, TICK).expect("raised"); - d.reset(); - assert_eq!(d.delay_ns(), 0); - assert_eq!(d.expire(u64::MAX), None); + assert_eq!(d.expire(u64::MAX, O, TICK), None); + assert_eq!(d.delay_ns(O), TICK); } #[test] @@ -556,131 +681,145 @@ mod tests { delay().with_relax(RELAX, 50_000_000) } - /// Feed `secs` of 30fps frames whose raw margin (delay excluded) is - /// `raw_margin_ns`, stepping the ramp at 5% as the caller does. - fn run(d: &mut VideoDelay, t0: u64, secs: u64, raw_margin_ns: i64) -> Vec { + /// Feed `secs` of 30fps frames from `t0` that each reach the thread + /// `margin_ns` before they would be due at offset `O`, with the playout + /// offset at `offset(now)`, stepping the slew at 5% as the caller does. + /// Returns the relax decisions and the delay after every frame. + fn run_with( + d: &mut VideoDelay, + t0: u64, + secs: u64, + margin_ns: i64, + offset: impl Fn(u64) -> i64, + ) -> (Vec, Vec) { let mut relaxes = Vec::new(); + let mut delays = Vec::new(); for i in 0..secs * 30 { let now = t0 + i * 33_333_333; - d.ramp(now, 0.05); - let margin = raw_margin_ns + d.delay_ns() as i64; - relaxes.extend(d.note_arrival(now, margin, TICK)); + let offset_ns = offset(now); + d.ramp(now, 0.05, offset_ns); + let pts = now as i64 + margin_ns - O; + relaxes.extend(d.note_arrival(now, now, pts, offset_ns, TICK)); + delays.push(d.delay_ns(offset_ns)); } - relaxes + (relaxes, delays) + } + + fn run(d: &mut VideoDelay, t0: u64, secs: u64, margin_ns: i64) -> Vec { + run_with(d, t0, secs, margin_ns, |_| O).0 } #[test] - fn a_delay_set_by_a_startup_transient_ramps_back_down() { + fn a_floor_set_by_a_startup_transient_slews_back_down() { let mut d = relaxing(); // One frame 1.3 s late at the anchor, on a stream whose frames then // arrive 100 ms early for good. - d.before_anchor(-1_300_000_000, TICK, TICK).expect("raised"); - let set = d.delay_ns(); - let relaxes = run(&mut d, 1_000_000_000, 60, 100_000_000); + before_anchor(&mut d, -1_300_000_000).expect("raised"); + let set = d.delay_ns(O); + let (relaxes, delays) = run_with(&mut d, 1_000_000_000, 60, 100_000_000, |_| O); assert_eq!(relaxes.len(), 1, "one decision, not one per window"); assert_eq!(relaxes[0].from_ns, set); assert_eq!( - relaxes[0].to_ns, TICK, - "nothing needed, so one tick of headroom" + relaxes[0].to_ns, 0, + "a tick of allowance and a tick of headroom fit in 100 ms" ); - assert_eq!(d.delay_ns(), TICK); + assert_eq!(d.delay_ns(O), 0); + assert!(delays.windows(2).all(|w| w[1] <= w[0]), "only ever down"); } #[test] - fn the_ramp_is_gradual() { + fn the_slew_is_gradual() { let mut d = relaxing(); - d.before_anchor(-1_300_000_000, TICK, TICK).expect("raised"); - let set = d.delay_ns(); - // Eleven seconds: the window closes at ten, then one second of ramp + before_anchor(&mut d, -1_300_000_000).expect("raised"); + let set = d.delay_ns(O); + // Eleven seconds: the window closes at ten, then one second of slew // at 5% takes back about 50 ms, not the whole delay. run(&mut d, 1_000_000_000, 11, 100_000_000); - let taken = set - d.delay_ns(); + let taken = set - d.delay_ns(O); assert!((30_000_000..70_000_000).contains(&taken), "took {taken}ns"); } #[test] - fn a_delay_the_sender_really_needs_stays() { + fn a_floor_the_sender_really_needs_stays() { let mut d = relaxing(); - d.before_anchor(-150_000_000, TICK, TICK).expect("raised"); - let set = d.delay_ns(); + before_anchor(&mut d, -150_000_000).expect("raised"); + let set = d.delay_ns(O); // Every frame keeps arriving 150 ms late. assert!(run(&mut d, 1_000_000_000, 60, -150_000_000).is_empty()); - assert_eq!(d.delay_ns(), set); + assert_eq!(d.delay_ns(O), set); } #[test] - fn one_late_frame_in_the_window_keeps_the_delay_it_needed() { + fn one_late_frame_in_the_window_keeps_the_floor_it_needed() { let mut d = relaxing(); - d.before_anchor(-1_300_000_000, TICK, TICK).expect("raised"); + before_anchor(&mut d, -1_300_000_000).expect("raised"); let t0 = 1_000_000_000; run(&mut d, t0, 5, 100_000_000); // Mid-window, one frame 400 ms late. - let margin = -400_000_000 + d.delay_ns() as i64; - d.note_arrival(t0 + 5_000_000_000, margin, TICK); + let now = t0 + 5_000_000_000; + let pts = now as i64 - 400_000_000 - O; + d.note_arrival(now, now, pts, O, TICK); let relaxes = run(&mut d, t0 + 5_033_333_333, 60, 100_000_000); - // 400 ms plus the tick allowance, in ticks, plus a tick of headroom. - assert_eq!(relaxes[0].to_ns, 26 * TICK); + // 400 ms plus the tick allowance, plus a tick of headroom. + assert_eq!(relaxes[0].to_ns, 400_000_000 + 2 * TICK); } #[test] - fn a_raise_cancels_the_ramp() { + fn a_raise_cancels_the_slew() { let mut d = relaxing(); - d.before_anchor(-1_300_000_000, TICK, TICK).expect("raised"); + before_anchor(&mut d, -1_300_000_000).expect("raised"); run(&mut d, 1_000_000_000, 12, 100_000_000); - let mid = d.delay_ns(); - let raise = d.before_anchor(-2_000_000_000 + mid as i64, TICK, TICK); + let raise = before_anchor(&mut d, -2_000_000_000 + MAX as i64 / 2); assert!(raise.is_some()); - let after = d.delay_ns(); - assert_eq!(d.ramp(20_000_000_000, 0.05), None); - assert_eq!(d.delay_ns(), after); - } - - #[test] - fn audio_hold_growth_is_taken_out_of_the_delay() { - let mut d = delay(); - d.before_anchor(-300_000_000, TICK, TICK).expect("raised"); - let set = d.delay_ns(); - assert_eq!(d.absorb(100_000_000), 100_000_000); - assert_eq!(d.delay_ns(), set - 100_000_000); - // Never below zero, and only what there was is reported. - assert_eq!(d.absorb(set), set - 100_000_000); - assert_eq!(d.delay_ns(), 0); - assert_eq!(d.absorb(TICK), 0); + let after = d.delay_ns(O); + assert_eq!(d.ramp(20_000_000_000, 0.05, O), None); + assert_eq!(d.delay_ns(O), after); } #[test] - fn late_frames_seen_before_the_growth_need_that_much_less() { - let mut d = delay(); - // Three frames 400 ms late across the window... - for i in 0..3u64 { - d.note(i * 400_000_000, -400_000_000, 0, TICK); - } - // ...then the audio playout grows by 300 ms before the verdict. - d.absorb(300_000_000); - let raise = d.expire(WINDOW + 1).expect("still short"); - // What is left of the 400 ms the frames needed. - assert_eq!(raise.to_ns, 24 * TICK - 300_000_000); + fn a_floor_below_the_offset_relaxes_at_once() { + let mut d = relaxing(); + before_anchor(&mut d, -300_000_000).expect("raised"); + // The audio hold has grown the playout past the floor, so nothing + // is delayed; the sender then recovers to 100 ms late against `O`. + let grown = O + 500_000_000; + assert_eq!(d.delay_ns(grown), 0); + let (relaxes, delays) = run_with(&mut d, 1_000_000_000, 11, -100_000_000, |_| grown); + assert_eq!(relaxes.len(), 1); + assert_eq!(relaxes[0].from_ns, 0, "nothing was on screen"); + assert!(delays.iter().all(|&d| d == 0)); + // The floor went straight to what frames need: should the hold be + // released, the delay is what these frames need, not what the first + // one did. + assert_eq!(d.delay_ns(O), 100_000_000 + 2 * TICK); } #[test] - fn growth_moves_a_ramp_target_with_it() { + fn only_the_part_of_the_floor_above_the_offset_slews() { let mut d = relaxing(); - d.before_anchor(-1_300_000_000, TICK, TICK).expect("raised"); - run(&mut d, 1_000_000_000, 11, 100_000_000); - let ramping = d.delay_ns(); - assert!(ramping > TICK, "a ramp is under way"); - // Growth past the ramp's target ends it where the growth left it. - d.absorb(ramping); - assert_eq!(d.delay_ns(), 0); - assert_eq!(d.ramp(30_000_000_000, 0.05), None); + before_anchor(&mut d, -1_300_000_000).expect("raised"); + // Halfway through the slew the playout grows past what is left. + let t0 = 1_000_000_000; + let ramp_from = t0 + RELAX; + let (_, delays) = run_with(&mut d, t0, 20, 100_000_000, |now| { + if now < ramp_from + 2_000_000_000 { + O + } else { + O + 1_400_000_000 + } + }); + assert_eq!(*delays.last().unwrap(), 0); + // Back at the old offset nothing reappears: the floor went to what + // frames need as soon as it was out of sight. + assert_eq!(d.delay_ns(O), 0); } #[test] - fn without_with_relax_the_delay_never_shrinks() { + fn without_with_relax_the_floor_never_drops() { let mut d = delay(); - d.before_anchor(-1_300_000_000, TICK, TICK).expect("raised"); - let set = d.delay_ns(); + before_anchor(&mut d, -1_300_000_000).expect("raised"); + let set = d.delay_ns(O); assert!(run(&mut d, 1_000_000_000, 60, 100_000_000).is_empty()); - assert_eq!(d.delay_ns(), set); + assert_eq!(d.delay_ns(O), set); } } diff --git a/crates/irl-source/src/audio/hold.rs b/crates/irl-source/src/audio/hold.rs index 8c8e81b..6fb8e2e 100644 --- a/crates/irl-source/src/audio/hold.rs +++ b/crates/irl-source/src/audio/hold.rs @@ -2,9 +2,10 @@ //! //! The receiver reads the skew as it pushes each video packet and moves the //! hold ([`observe_video_packet`]). The pump waits a bounded time for the -//! first reading before it primes ([`prime_may_wait`]) and credits what it -//! builds afterwards ([`credit_build`]), which the video thread takes out of -//! its standing delay. A new connection forgets the hold +//! first reading before it primes ([`prime_may_wait`]). A hold raised after +//! that is built by the speed controller, and the audio playout offset grows +//! with it; the video delay is derived from that offset, so it shrinks as the +//! hold builds without being told. A new connection forgets the hold //! ([`reset_connection`]). //! //! Outside low-latency mode the hold is folded into the published watermarks, @@ -84,21 +85,6 @@ pub fn prime_may_wait(shared: &Shared, state: &mut AudioState, now_ns: u64) -> b false } -/// The pump played `played_ns` of OBS time for `consumed_ns` of stream: the -/// playout offset grew by the difference. While a raise made after priming is -/// still being built, that growth is the hold building, and is credited to -/// [`AudioState::hold_built_ns`] for the video thread. -/// -/// Caller holds `audio_state`. -pub fn credit_build(state: &mut AudioState, played_ns: u64, consumed_ns: u64) { - if state.hold_unbuilt_ns == 0 || played_ns <= consumed_ns { - return; - } - let built_ns = (played_ns - consumed_ns).min(state.hold_unbuilt_ns); - state.hold_unbuilt_ns -= built_ns; - state.hold_built_ns += built_ns; -} - /// Forget the readings, keeping the hold in force: the audio timeline broke, /// so readings from before it no longer compare with readings after it, but /// the sender is the same one. @@ -107,7 +93,6 @@ pub fn credit_build(state: &mut AudioState, played_ns: u64, consumed_ns: u64) { pub fn forget_readings(state: &mut AudioState) { state.hold.reset(); state.hold_live.reset(); - state.hold_unbuilt_ns = 0; } /// Back to no hold: a new connection measures its own. Restores Target @@ -184,14 +169,6 @@ fn apply(shared: &Shared, state: &mut AudioState, change: HoldChange) { } } - if state.clock.is_primed() { - let moved_ns = u64::from(to_ms.abs_diff(from_ms)) * 1_000_000; - state.hold_unbuilt_ns = if to_ms > from_ms { - state.hold_unbuilt_ns + moved_ns - } else { - state.hold_unbuilt_ns.saturating_sub(moved_ns) - }; - } state.hold_ms = to_ms; shared.conn.audio_hold_ms.store(to_ms, Relaxed); diff --git a/crates/irl-source/src/audio/mod.rs b/crates/irl-source/src/audio/mod.rs index 2d6ebb9..ab4ccdf 100644 --- a/crates/irl-source/src/audio/mod.rs +++ b/crates/irl-source/src/audio/mod.rs @@ -71,9 +71,6 @@ pub fn reset_audio_timing_state(state: &mut AudioState) { state.recovery_until_us = 0; state.speed_carry.reset(); state.align_read_pending = false; - // Priming waits for the whole target, hold included, so nothing is left - // to build once it re-primes. - state.hold_unbuilt_ns = 0; state.latest_audio_stream_pts_ns = 0; state.decoded_frame_samples = 0; state.startup_warmup_remaining_ms = 0; diff --git a/crates/irl-source/src/audio/pump.rs b/crates/irl-source/src/audio/pump.rs index 408fcc3..a1ce2bd 100644 --- a/crates/irl-source/src/audio/pump.rs +++ b/crates/irl-source/src/audio/pump.rs @@ -246,7 +246,6 @@ impl AudioPump { return false; } - state.hold_unbuilt_ns = 0; state.clock.restart(now + chunk_ns); // Reads and writes are both whole decoded chunks, so the residual // can only ever be a multiple of one: a 120ms target is not a @@ -443,11 +442,6 @@ Video stays in sync with it; check the sender's frame rate and clock", )); state.out_last.remember_bytes(emitted, channels); - super::hold::credit_build( - state, - timing::frames_to_ns(u64::from(frames_out), fmt.rate as u32), - stream_duration_ns, - ); let first_mapping = !state.mapping.has_output(); finalize_audio_output( shared, diff --git a/crates/irl-source/src/config.rs b/crates/irl-source/src/config.rs index 448dcda..2fffe00 100644 --- a/crates/irl-source/src/config.rs +++ b/crates/irl-source/src/config.rs @@ -117,8 +117,6 @@ impl Config { // Target Buffer can leave less room under for the hold. let hold_ms = next.target_ms - user_next.target_ms; if hold_ms < folded_ms { - let cut_ns = u64::from(folded_ms.abs_diff(hold_ms)) * 1_000_000; - state.hold_unbuilt_ns = state.hold_unbuilt_ns.saturating_sub(cut_ns); state.hold_ms = hold_ms; shared.conn.audio_hold_ms.store(hold_ms, Relaxed); } diff --git a/crates/irl-source/src/shared.rs b/crates/irl-source/src/shared.rs index 7103ebf..042fc8e 100644 --- a/crates/irl-source/src/shared.rs +++ b/crates/irl-source/src/shared.rs @@ -159,12 +159,6 @@ pub struct AudioState { /// The hold in force, in ms. Outside low-latency mode it is folded into /// the published watermarks: their target is Target Buffer plus this. pub hold_ms: i32, - /// How much of the raises made since priming the buffer has yet to build. - pub hold_unbuilt_ns: u64, - /// Running total of what it has built, which is how far the playout - /// offset grew for the hold. The video thread takes the growth out of its - /// standing delay. - pub hold_built_ns: u64, /// When the pump first waited on a skew reading to prime; zero while it /// has not. pub hold_wait_since_ns: u64, @@ -198,7 +192,8 @@ pub struct ConnStats { pub video_corrupt_held: AtomicU64, /// `f32::to_bits` of the smoothed playback speed. pub current_speed_bits: AtomicU32, - /// Mirror of the video thread's standing delay, for the stats. + /// The video delay the video thread's schedule carries (its floor less + /// the audio playout offset), republished every cycle for the stats. pub video_delay_ns: AtomicU64, /// Mirror of `AudioState::hold_ms`, for the stats and the video thread. pub audio_hold_ms: AtomicI32, diff --git a/crates/irl-source/src/video/output.rs b/crates/irl-source/src/video/output.rs index 5640363..1eb60fb 100644 --- a/crates/irl-source/src/video/output.rs +++ b/crates/irl-source/src/video/output.rs @@ -165,14 +165,21 @@ impl VideoThread { /* ── Timestamp mapping ────────────────────────────────── */ - /// The OBS timestamp a freshly transferred frame is scheduled for. + /// The OBS timestamp a freshly transferred frame is scheduled for: its + /// [`Self::base_due`] plus the video delay that implies. + pub fn due_time(&mut self, frame: &Frame) -> u64 { + let base_ns = self.base_due(frame); + self.delay.due_ns(frame.pts(), base_ns) + } + + /// The OBS timestamp a frame is due at before the video delay. /// /// While audio is playing, queued audio is the master playout clock: video /// PTS maps through the same stream-PTS → OBS-clock offset as the latest /// chunk handed to OBS, which keeps lip sync stable across buffering and /// speed changes. Without that mapping it falls back to the video-only /// wall-clock anchor. - pub fn due_time(&mut self, frame: &Frame) -> u64 { + pub fn base_due(&mut self, frame: &Frame) -> u64 { // `frame.pts()` is already in nanoseconds: intake rescales it with the // time base captured at stream open, because the video thread must not // touch the format context (it can be freed mid-reconnect). @@ -186,12 +193,8 @@ impl VideoThread { // No audio-stream test: a published mapping already implies the pump // handed OBS a real chunk, so it implies the audio stream. - // Both schedules carry the standing video delay; see - // `VideoThread::settle_anchor_candidate`. - let delay_ns = self.delay.delay_ns(); - if let Some(mapped) = mapping.map(pts_ns) { - return mapped.saturating_add(delay_ns); + return mapped; } // `conn.video_ts_init` is the authority here: it is cleared on a @@ -207,26 +210,20 @@ impl VideoThread { self.shared.conn.video_ts_init.store(true, Relaxed); } - let mut computed = video_time::fallback_anchor(pts_ns, self.pts_base, self.sys_base, now) - .saturating_add(delay_ns); - - // Startup fallback before the audio playout mapping exists. The - // mapping cannot stand in for "there is audio" here, so this reads the - // flag the receiver publishes. - if self.shared.flags.audio_present.load(Relaxed) { - let mut audio_lead_ns = 0; - if !mapping.has_output() { - audio_lead_ns = startup_warmup_ms as i64 * 1_000_000; - if !self.shared.cfg.low_latency_audio { - audio_lead_ns += self.shared.hot.watermarks().target_ms as i64 * 1_000_000; - } - } - if audio_lead_ns > 0 { - computed += audio_lead_ns as u64; - } - } + let computed = video_time::fallback_anchor(pts_ns, self.pts_base, self.sys_base, now); - computed + // Audio that is coming but not mapped yet will play its Target + // Buffer (and whatever is left of the warm-up) after it arrives. The + // mapping cannot stand in for "there is audio" here, so this reads + // the flag the receiver publishes. + if !self.shared.flags.audio_present.load(Relaxed) { + return computed; + } + let mut audio_lead_ms = i64::from(startup_warmup_ms); + if !self.shared.cfg.low_latency_audio { + audio_lead_ms += i64::from(self.shared.hot.watermarks().target_ms); + } + computed + audio_lead_ms.max(0) as u64 * 1_000_000 } /// Whether audio has published a playout mapping at all — the same test @@ -243,11 +240,10 @@ impl VideoThread { /// /// Free of the side effects in [`Self::due_time`]: the fallback anchor /// belongs to a frame arriving, and running it for every queued frame - /// would report the queue rather than the stream. - /// `None` when there is no audio to slave to, or when the mapping has been - /// gone long enough that holding it would be a guess; the caller then - /// keeps the due times the frames arrived with. The standing video delay - /// rides on the offset, so a reschedule keeps it. + /// would report the queue rather than the stream. `None` when there is no + /// audio to slave to, or when the mapping has been gone long enough that + /// holding it would be a guess; the caller then keeps the due times the + /// frames arrived with. pub fn playout_offset(&mut self) -> Option { let offset_ns = self.shared.audio_state().playout_mapping().offset_ns(); let now = self.now_ns(); @@ -261,7 +257,7 @@ impl VideoThread { return None; } - Some(self.playout_offset_ns + self.delay.delay_ns() as i64) + Some(self.playout_offset_ns) } /* ── Output ───────────────────────────────────────────── */ diff --git a/crates/irl-source/src/video/thread.rs b/crates/irl-source/src/video/thread.rs index 142898c..80b6c22 100644 --- a/crates/irl-source/src/video/thread.rs +++ b/crates/irl-source/src/video/thread.rs @@ -15,7 +15,6 @@ use std::sync::atomic::Ordering::Relaxed; use std::time::Duration; use ffmpeg::{AVPixelFormat, Frame, FramePool, Scaler}; -use irl_core::arrival::ArrivalFloor; use irl_core::consts; use irl_core::pacing::{DueVerdict, PacingQueue}; use irl_core::video_delay::{DelayRaise, DelayRelax, VideoDelay}; @@ -27,10 +26,14 @@ use crate::video::intake::{self, DecodeState}; /// A frame waiting for its due time. `received_ns` is when the packet it was /// decoded from reached this thread, which is what its arrival margin is -/// measured from. +/// measured from. `base_ns` is its due time before the video delay: mapped +/// through the audio playout, or on the video-only fallback when there is no +/// mapping to re-derive it from. pub struct Paced { frame: Frame, + pts_ns: i64, received_ns: u64, + base_ns: u64, } impl Paced { @@ -42,6 +45,12 @@ impl Paced { pub fn received_ns(&self) -> u64 { self.received_ns } + + /// The frame's undelayed due time minus its PTS: the offset the video + /// delay is measured against. + fn offset_ns(&self) -> i64 { + self.base_ns as i64 - self.pts_ns + } } /// Turns a queued packet into the frame it decodes to, in place of a decoder: @@ -105,17 +114,19 @@ pub struct VideoThread { /// Where the wait for the audio playout mapping stands. See /// [`Self::awaiting_audio_mapping`]. anchor_wait: AnchorWait, - /// The standing delay on the video schedule, so that video which reaches - /// this thread too late to be paced against the audio playout still can - /// be. See [`Self::settle_anchor_candidate`]. + /// The floor under the video schedule, so that video which reaches this + /// thread too late to be paced against the audio playout still can be. + /// See [`Self::settle_anchor_candidate`]. pub(crate) delay: VideoDelay, - /// Whether video arriving now is live or still catching up to live, which - /// decides whether a frame may size the delay before the anchor. - arrival: ArrivalFloor, - /// The audio pump's running total of hold built (`AudioState:: - /// hold_built_ns`) as of the last cycle; `None` until the first read after - /// a reset. See [`Self::absorb_audio_hold`]. - hold_built_seen_ns: Option, + /// The offset the schedule runs at: the audio playout offset this cycle, + /// or the newest frame's on the fallback. The delay it implies is what + /// the stats report. + offset_ns: Option, + /// The floor moved this cycle, and its own log line said so. + floor_moved: bool, + /// Whether the last published delay was above zero, so the line that + /// says the audio playout moved past the floor is edge-triggered. + delay_shown: bool, /// The OBS clock, read for every timing decision this thread makes. /// Injectable so tests can step it; production reads `os_gettime_ns`. now_ns: Box u64 + Send>, @@ -161,8 +172,9 @@ impl VideoThread { anchor_pending: true, anchor_wait: AnchorWait::Idle, delay: VideoDelay::default(), - arrival: ArrivalFloor::default(), - hold_built_seen_ns: None, + offset_ns: None, + floor_moved: false, + delay_shown: false, now_ns: Box::new(obs::time::gettime_ns), canvas_tick_ns: Box::new(obs::time::canvas_frame_interval_ns), sink, @@ -210,8 +222,9 @@ impl VideoThread { // to 0, so the next frame re-anchors the play head. self.anchor_pending = true; self.anchor_wait = AnchorWait::Idle; - // The delay was sized for the connection that just ended. + // The floor was sized for the connection that just ended. self.reset_delay(); + self.publish_delay(); self.sink.output_video_none(); return Duration::ZERO; } @@ -221,20 +234,24 @@ impl VideoThread { let slack_ns = self.emit_slack_ns(); self.decode_intake(); - self.absorb_audio_hold(); // Before both the emit and the sleep below, so each cycle schedules // against the offset as it is now rather than as it was when the // frames were decoded. self.pacing_reschedule(); + if !self.anchor_pending { + self.tend_floor(now_ns); + } if self.anchor_pending && self.awaiting_audio_mapping(now_ns) { // Nothing goes out until audio has published where video belongs. // The audio pump wakes this thread the moment it does; the sleep // is only the backstop. self.publish_counters(); + self.publish_delay(); return Duration::from_millis(consts::VIDEO_PACING_MAX_WAIT_MS); } self.pacing_emit_due(now_ns, slack_ns); self.publish_counters(); + self.publish_delay(); // Fresh clock for the sleep: the emit above may have taken long // enough to make the next frame due already. self.sleep_hint(self.now_ns(), slack_ns) @@ -262,6 +279,8 @@ impl VideoThread { let shared = self.shared.clone(); if shared.video_flags.timeline_reset.swap(false, Relaxed) { self.state.reset_timeline(); + // The floor and the liveness history carry the old PTS epoch. + self.reset_delay(); } // A decoder handover is taken even with no room: it produces nothing // by itself, and leaving it behind the queue would strand a reconnect @@ -325,26 +344,30 @@ impl VideoThread { /// `decode_intake`. pub fn pace_decoded(&mut self, frame: Frame, received_ns: u64) { let sysmem = self.to_sysmem(&frame); - let due_ns = match &sysmem { - Some(f) => self.due_time(f), + let base_ns = match &sysmem { + Some(f) => self.base_due(f), None => 0, }; // Releases the decoder's surface before the next packet is sent. drop(frame); if let Some(f) = sysmem { let pts_ns = f.pts(); - self.arrival.note(received_ns, pts_ns); + self.delay.note_packet(received_ns, pts_ns); + self.offset_ns = Some(base_ns as i64 - pts_ns); + let due_ns = self.delay.due_ns(pts_ns, base_ns); let bytes = f.image_buffer_size().unwrap_or(0); let paced = Paced { frame: f, + pts_ns, received_ns, + base_ns, }; self.pacing.push(paced, pts_ns, bytes, due_ns); } } /// Re-derive every queued frame's due time from the offset as it stands - /// now. + /// now and the floor as it stands now. /// /// The audio side reclaims playout latency two ways, and both would leave /// paced video behind for the depth of this queue. The speed controller @@ -352,18 +375,44 @@ impl VideoThread { /// its residence late for the whole drain; a re-anchor steps the offset /// outright. Rescheduling against one offset per cycle preserves the /// spacing between frames and moves the whole queue with the audio it is - /// mapped to. + /// mapped to. Without a mapping each frame keeps the fallback due time it + /// was given at intake, and only the floor is re-applied. fn pacing_reschedule(&mut self) { - if self.pacing.is_empty() { - return; + let offset_ns = self.playout_offset(); + if offset_ns.is_some() { + self.offset_ns = offset_ns; } - let Some(offset_ns) = self.playout_offset() else { + let delay = &self.delay; + self.pacing.reschedule(|pts_ns, paced| { + if let Some(offset_ns) = offset_ns { + paced.base_ns = (pts_ns + offset_ns).max(0) as u64; + } + delay.due_ns(pts_ns, paced.base_ns) + }); + } + + /// The floor's per-cycle work once the play head is anchored: a slew in + /// progress steps on, and a window of late frames that ran its course is + /// judged. + fn tend_floor(&mut self, now_ns: u64) { + let Some(offset_ns) = self.offset_ns else { return; }; - self.pacing.reschedule(|pts_ns| { - let due = pts_ns + offset_ns; - if due > 0 { due as u64 } else { 0 } - }); + let mut moved = false; + if let Some(step) = self.delay.ramp(now_ns, self.delay_ramp_rate(), offset_ns) { + moved = true; + if step.done { + irl_info!("Video delay settled at {}ms", step.delay_ns / 1_000_000); + } + } + if let Some(raise) = self.delay.expire(now_ns, offset_ns, self.canvas_tick_ns()) { + moved = true; + self.announce_delay_raise(raise, false); + } + if moved { + self.floor_moved = true; + self.pacing_reschedule(); + } } /// Emit every frame whose moment has arrived. Over the ceilings the head @@ -373,11 +422,6 @@ impl VideoThread { let tick_ns = self.canvas_tick_ns(); if self.anchor_pending { self.settle_anchor_candidate(now_ns, tick_ns); - } else { - self.step_delay_ramp(now_ns); - if let Some(raise) = self.delay.expire(now_ns) { - self.apply_delay_raise(raise, false); - } } loop { // While the play head needs anchoring, a hard ceiling must not @@ -396,18 +440,25 @@ impl VideoThread { return; }; if !self.anchor_pending { + let offset_ns = paced.offset_ns(); // Measured at hand-over: a frame that reaches libobs before its // due time is shown on time, whether or not it had the full // delivery lead (that lead is an oversleep allowance for frames // the queue holds, and this one may never have waited). - let handover_margin_ns = due_ns as i64 - now_ns as i64; - if let Some(raise) = self.delay.note(now_ns, handover_margin_ns, 0, tick_ns) { - self.apply_delay_raise(raise, false); + if let Some(raise) = self.delay.note(now_ns, paced.pts_ns, offset_ns, tick_ns) { + self.floor_moved = true; + self.pacing_reschedule(); + self.announce_delay_raise(raise, false); } - // And at arrival, which is what says whether the delay in - // force is more than this stream needs. - let arrival_margin_ns = due_ns as i64 - paced.received_ns() as i64; - if let Some(relax) = self.delay.note_arrival(now_ns, arrival_margin_ns, tick_ns) { + // And at arrival, which is what says whether the floor is + // higher than this stream needs. + if let Some(relax) = self.delay.note_arrival( + now_ns, + paced.received_ns, + paced.pts_ns, + offset_ns, + tick_ns, + ) { self.announce_delay_relax(relax); } } @@ -480,13 +531,13 @@ impl VideoThread { /// /// First its arrival margin. Nothing has been shown yet, so a frame whose /// packet reached this thread less than a canvas tick before it is due - /// raises the standing video delay on the spot - /// ([`VideoDelay::before_anchor`]) and the queue moves with it. The - /// allowance is a tick for the decode still to come, not the delivery - /// lead: the lead covers the pacing timer oversleeping, and a frame handed - /// over on arrival never sleeps. This keeps a sender whose video trails its - /// audio by more than Target Buffer covers from playing unpaced, with - /// libobs dropping a frame whenever two arrive inside one canvas tick. + /// raises the video floor on the spot ([`VideoDelay::before_anchor`]) and + /// the queue is rescheduled onto it. The allowance is a tick for the + /// decode still to come, not the delivery lead: the lead covers the pacing + /// timer oversleeping, and a frame handed over on arrival never sleeps. + /// This keeps a sender whose video trails its audio by more than Target + /// Buffer covers from playing unpaced, with libobs dropping a frame + /// whenever two arrive inside one canvas tick. /// /// Only the newest frame in hand is measured (nothing queued behind it and /// the channel empty, so its arrival is a live one). The receiver pushes @@ -514,25 +565,25 @@ impl VideoThread { let audio_present = self.shared.flags.audio_present.load(Relaxed); let mut dropped = 0u32; let mut raised: Option = None; - while let (Some(due_ns), Some(received_ns)) = ( - self.pacing.next_due(), - self.pacing.head().map(Paced::received_ns), - ) { + while let (Some(due_ns), Some(head)) = (self.pacing.next_due(), self.pacing.head()) { + let (received_ns, pts_ns, offset_ns) = + (head.received_ns, head.pts_ns, head.offset_ns()); let on_time = now_ns as i64 - due_ns as i64 <= tick_ns as i64; let newest_in_hand = self.pacing.len() == 1 && self.shared.video.is_empty(); // With audio to be in sync with, only live video says anything // about the sender: video still catching up to live (a relay // replaying from its last keyframe next to live audio) is behind // its audio for now, not for good. - let measurable = newest_in_hand && (!audio_present || self.arrival.live(now_ns)); + let measurable = newest_in_hand && (!audio_present || self.delay.live(now_ns)); let raise = if measurable { self.delay - .before_anchor(due_ns as i64 - received_ns as i64, tick_ns, tick_ns) + .before_anchor(received_ns, pts_ns, offset_ns, tick_ns) } else { None }; if let Some(raise) = raise { - self.pacing.shift(raise.to_ns - raise.from_ns); + self.floor_moved = true; + self.pacing_reschedule(); // One line for the whole settlement, from the first delay to // the last. raised = Some(match raised { @@ -550,14 +601,14 @@ impl VideoThread { // Late past the delay ceiling on the newest frame: no delay makes // this stream on time, so it anchors as it is rather than dropping // every frame while waiting for one that is (#33). - if newest_in_hand && self.delay.at_ceiling() { + if newest_in_hand && self.delay.at_ceiling(offset_ns) { break; } self.pacing.pop(); dropped += 1; } if let Some(raise) = raised { - self.apply_delay_raise(raise, true); + self.announce_delay_raise(raise, true); } if dropped > 0 { irl_info!( @@ -566,16 +617,9 @@ impl VideoThread { } } - /// Publish a raised delay: mirror it for the stats and say why, with what - /// the user can do about it. The queue itself is moved by the caller - /// (before the anchor, inside the settling loop) or is already scheduled - /// on the new delay (after it, the next reschedule carries it; the frames - /// queued now are shifted here). - fn apply_delay_raise(&mut self, raise: DelayRaise, at_anchor: bool) { - if !at_anchor { - self.pacing.shift(raise.to_ns - raise.from_ns); - } - self.shared.conn.video_delay_ns.store(raise.to_ns, Relaxed); + /// Say why the floor rose, with what the user can do about it. The queue + /// has already been rescheduled onto it. + fn announce_delay_raise(&self, raise: DelayRaise, at_anchor: bool) { let to_ms = raise.to_ns / 1_000_000; let by_ms = (raise.to_ns - raise.from_ns) / 1_000_000; let audio_present = self.shared.flags.audio_present.load(Relaxed); @@ -586,23 +630,24 @@ impl VideoThread { // cannot move. let hold_follows = !self.shared.cfg.low_latency_audio && self.shared.hot.adaptive_speed.load(Relaxed); + let remedy = if hold_follows { + format!( + "Audio is held back to match if the sender keeps its video this far behind; if the video delay stays, raise Target Buffer by at least {to_ms}ms" + ) + } else { + format!( + "If the sound runs ahead of the picture, set Sync Offset to +{to_ms}ms in Advanced Audio Properties, or raise Target Buffer by at least {to_ms}ms" + ) + }; match (at_anchor, audio_present) { - (true, true) if hold_follows => irl_warn!( - "Video reaches the plugin too late to be shown in time with its audio; delaying video by {to_ms}ms. Audio is held back to match if the sender keeps its video this far behind; if the video delay stays, raise Target Buffer by at least {to_ms}ms" - ), (true, true) => irl_warn!( - "Video reaches the plugin too late to be shown in time with its audio; delaying video by {to_ms}ms. If the sound runs ahead of the picture, set Sync Offset to +{to_ms}ms in Advanced Audio Properties, or raise Target Buffer by at least {to_ms}ms" + "Video reaches the plugin too late to be shown in time with its audio; delaying video by {to_ms}ms. {remedy}" ), (true, false) => { irl_info!("Video reaches the plugin as it falls due; delaying video by {to_ms}ms") } - (false, true) if hold_follows => irl_warn!( - "Video ran late on {} frames in the last {}ms; delaying video by {by_ms}ms more, {to_ms}ms in all. Audio is held back to match if the sender keeps its video this far behind; if the video delay stays, raise Target Buffer by at least {to_ms}ms", - raise.frames, - consts::VIDEO_DELAY_WINDOW_MS - ), (false, true) => irl_warn!( - "Video ran late on {} frames in the last {}ms; delaying video by {by_ms}ms more, {to_ms}ms in all. If the sound runs ahead of the picture, set Sync Offset to +{to_ms}ms in Advanced Audio Properties, or raise Target Buffer by at least {to_ms}ms", + "Video ran late on {} frames in the last {}ms; delaying video by {by_ms}ms more, {to_ms}ms in all. {remedy}", raise.frames, consts::VIDEO_DELAY_WINDOW_MS ), @@ -620,9 +665,13 @@ impl VideoThread { } } - /// A whole window of frames needed less delay than is in force: say so. - /// The ramp itself runs from [`Self::step_delay_ramp`]. - fn announce_delay_relax(&self, relax: DelayRelax) { + /// A whole window of frames needed less than the floor: say so when that + /// is on screen. The slew itself runs from [`Self::tend_floor`]. + fn announce_delay_relax(&mut self, relax: DelayRelax) { + if relax.to_ns >= relax.from_ns { + return; + } + self.floor_moved = true; irl_info!( "No frame in the last {}s needed more than {}ms of the {}ms video delay; playing video {:.0}% fast until it is back in step with its audio", consts::VIDEO_DELAY_RELAX_WINDOW_MS / 1000, @@ -632,60 +681,49 @@ impl VideoThread { ); } - /// How fast a ramp takes the delay back: the Catch-Up Speed, the same + /// How fast a slew takes the delay back: the Catch-Up Speed, the same /// promise the audio side makes about a backlog. Video has no pitch, so /// the rate is far less noticeable here than it is there. fn delay_ramp_rate(&self) -> f64 { f64::from(self.shared.hot.max_speed() - 1.0).max(0.0) } - /// Advance a delay ramp in progress: every due time, queued frames - /// included, moves earlier by this cycle's share, so the picture plays - /// slightly fast instead of jumping. - fn step_delay_ramp(&mut self, now_ns: u64) { - let Some(step) = self.delay.ramp(now_ns, self.delay_ramp_rate()) else { + /// Mirror the delay the schedule carries for the stats, and say so once + /// when the audio playout moving (not the floor) takes it to or from + /// zero: an audio hold that finished building, or a re-anchor that put + /// the playout back below what video needs. + fn publish_delay(&mut self) { + let delay_ns = self + .offset_ns + .map_or(0, |offset_ns| self.delay.delay_ns(offset_ns)); + self.shared.conn.video_delay_ns.store(delay_ns, Relaxed); + let floor_moved = std::mem::take(&mut self.floor_moved); + let shown = delay_ns >= 1_000_000; + if shown == self.delay_shown { return; - }; - self.pacing.shift_earlier(step.moved_ns); - self.shared - .conn - .video_delay_ns - .store(step.delay_ns, Relaxed); - if step.done { - irl_info!("Video delay settled at {}ms", step.delay_ns / 1_000_000); } - } - - /// Take what the audio hold built since the last cycle out of the - /// standing delay. - /// - /// The hold grows the audio playout offset, and with it every video due - /// time. Video late enough to need a delay was late against the old - /// offset, so each millisecond the offset grows is a millisecond of delay - /// it no longer needs. Taking it back keeps due times where they were: the - /// picture neither jumps nor changes speed while the lip-sync error drains. - fn absorb_audio_hold(&mut self) { - let built_ns = self.shared.audio_state().hold_built_ns; - let Some(seen_ns) = self.hold_built_seen_ns.replace(built_ns) else { - return; - }; - let grown_ns = built_ns.saturating_sub(seen_ns); - if grown_ns == 0 || self.delay.absorb(grown_ns) == 0 { + self.delay_shown = shown; + if floor_moved { return; } - let delay_ns = self.delay.delay_ns(); - self.shared.conn.video_delay_ns.store(delay_ns, Relaxed); - if delay_ns == 0 { - irl_info!("Audio is held back far enough for its video; video delay back to 0ms"); + if shown { + irl_info!( + "The audio playout moved earlier than video can be shown; video now runs {}ms behind its audio", + delay_ns / 1_000_000 + ); + } else { + irl_info!( + "The audio playout now waits long enough for video; video back in step with its audio" + ); } } - /// Forget the delay: a new connection or a cleared source sizes its own. + /// Forget the floor: a new connection, a cleared source or a broken + /// timeline sizes its own. fn reset_delay(&mut self) { - self.arrival.reset(); self.delay.reset(); - self.hold_built_seen_ns = None; - self.shared.conn.video_delay_ns.store(0, Relaxed); + self.offset_ns = None; + self.floor_moved = true; } pub(crate) fn now_ns(&self) -> u64 { diff --git a/crates/irl-source/tests/av_sync_sim.rs b/crates/irl-source/tests/av_sync_sim.rs index 25342c5..adbdbd7 100644 --- a/crates/irl-source/tests/av_sync_sim.rs +++ b/crates/irl-source/tests/av_sync_sim.rs @@ -499,7 +499,6 @@ struct StatSample { video_delay_ms: u64, audio_hold_ms: i32, av_skew_ms: i64, - reanchors: u64, fill_ms: i32, target_ms: i32, } @@ -623,7 +622,6 @@ impl Sim { video_delay_ms: self.shared.conn.video_delay_ns.load(Relaxed) / MS, audio_hold_ms: self.shared.conn.audio_hold_ms.load(Relaxed), av_skew_ms: av_skew_ms(&self.shared, &state), - reanchors: self.shared.lifetime.audio_offset_reanchors.load(Relaxed), fill_ms: self .shared .audio_buf() @@ -960,10 +958,35 @@ impl Report { lo..=hi } - /// The stats sample just before the first re-anchor. - fn before_first_reanchor(&self) -> Option { - let i = self.stats.iter().position(|s| s.reanchors > 0)?; - Some(self.stats[i.checked_sub(1)?]) + /// The stats sample taken at or just before `at_s` after the open. + fn stats_at(&self, at_s: f64) -> StatSample { + let at = START_NS + (at_s * SEC as f64) as u64; + let i = self.stats.partition_point(|s| s.at_ns <= at); + self.stats[i.saturating_sub(1)] + } + + /// Seconds after `from_s` until picture and sound are back in sync for + /// good: the hand-over time of the first frame after which every frame + /// shown up to `to_s` is measurable, within [`SYNC_TOLERANCE_NS`] and on + /// time. + fn resynced_after(&self, from_s: f64, to_s: f64) -> f64 { + let from = START_NS + (from_s * SEC as f64) as u64; + let to = START_NS + (to_s * SEC as f64) as u64; + let tol = SYNC_TOLERANCE_NS as f64 / 1e6; + let frames: Vec<&Shown> = self + .shown + .iter() + .filter(|f| (from..to).contains(&f.handed_ns)) + .collect(); + let last_bad = frames.iter().rposition(|f| { + let off = self.offset_ms(f); + off.is_none_or(|o| o.abs() > tol) || f.late_ns() > consts::VIDEO_PACING_SLACK_NS + }); + let synced_ns = match last_bad { + None => from, + Some(i) => frames.get(i + 1).map_or(to, |f| f.handed_ns), + }; + (synced_ns - from) as f64 / SEC as f64 } } @@ -1095,10 +1118,11 @@ fn a_skew_that_rises_and_recovers_is_followed_both_ways() { } /// Both streams stall for three seconds while the hold for a new 600 ms skew -/// is being built. Audio conceals, the backlog lands at once and drains at -/// the Catch-Up Speed, the inflated latency is re-anchored away, and the -/// standing delay relaxes: in sync within about twenty seconds of the stall, -/// and back where the connection started. +/// is being built, with the picture still delayed behind the sound. Audio +/// conceals, which grows the playout by the stall, and the backlog lands at +/// once and drains at the Catch-Up Speed. The delay is the video floor less +/// the playout, so the stall itself takes it away: in sync within a second +/// of the stall ending, and back where the connection started. #[test] fn a_full_stall_during_a_hold_build_recovers_sync() { let report = Scenario::new(120) @@ -1106,21 +1130,33 @@ fn a_full_stall_during_a_hold_build_recovers_sync() { .stall(30.0, 3000, Streams::Both) .run(); + let at_stall = report.stats_at(29.9); + assert!( + at_stall.video_delay_ms > 0, + "the delay had gone before the stall: {at_stall:?}" + ); let before = report.assert_settled(0.0, 20.0); - report.assert_settled(55.0, 120.0); + let resynced_s = report.resynced_after(33.0, 120.0); + assert!( + resynced_s <= 1.0, + "in sync {resynced_s:.2}s after the stall" + ); + report.assert_settled(34.0, 120.0); let after = report.window(100.0, 120.0); report.print("stall during build", &after); + println!("stall during build: in sync {resynced_s:.2}s after the stall"); report.assert_no_drift(&before, &after); - assert_eq!(report.video_delay_ms(100.0), 0..=0); + assert_eq!(report.video_delay_ms(34.0), 0..=0); report.assert_latency_settled(); report.assert_healthy(); } /// A loss while the delay stands (a hold build in progress) leaves audio -/// latency inflated by its concealment, which the pump re-anchors away while -/// the video delay is still in force. Video follows the new playout and the -/// delay relaxes: in sync within half a minute, with no lasting offset. +/// latency inflated by its concealment, which the pump re-anchors away once +/// the buffer is back at its target, under a video floor sized for the skew. Video +/// follows the playout down only as far as the floor: in sync within a few +/// seconds of the loss, with no lasting offset. #[test] fn a_reanchor_while_the_video_delay_stands_recovers_sync() { let report = Scenario::new(120) @@ -1128,19 +1164,22 @@ fn a_reanchor_while_the_video_delay_stands_recovers_sync() { .loss(26.0, 1500) .run(); - let at_reanchor = report.before_first_reanchor().expect("a re-anchor"); + let at_loss = report.stats_at(25.9); assert!( - at_reanchor.video_delay_ms > 0, - "the delay had gone by the re-anchor: {at_reanchor:?}" + at_loss.video_delay_ms > 0, + "the delay had gone before the loss: {at_loss:?}" ); assert_eq!(report.reanchors, 1); let before = report.assert_settled(0.0, 20.0); - let after = report.assert_settled(60.0, 120.0); + let resynced_s = report.resynced_after(27.5, 120.0); + assert!(resynced_s <= 3.0, "in sync {resynced_s:.2}s after the loss"); + let after = report.assert_settled(30.0, 120.0); report.print("re-anchor under delay", &after); + println!("re-anchor under delay: in sync {resynced_s:.2}s after the loss"); report.assert_no_drift(&before, &after); - assert_eq!(report.video_delay_ms(70.0), 0..=0); + assert_eq!(report.video_delay_ms(28.0), 0..=0); report.assert_latency_settled(); report.assert_healthy(); } @@ -1166,17 +1205,16 @@ fn a_target_buffer_edit_during_a_hold_build_recomposes_the_hold() { report.assert_healthy(); } -/// The same edit must also end in sync. It does not: once the delay has -/// relaxed part of the way, a remainder within `VIDEO_DELAY_RELAX_MIN_MS` plus -/// a tick of what frames need never relaxes, and the hold credit, which only -/// counts the hold's own growth, never absorbs it. Measured: the delay stands -/// at 66 ms and the picture trails the sound by 83 ms for good. +/// The same edit must also end in sync: the effective target is unchanged, +/// so the playout still grows past the video floor, and the delay, which is +/// the floor less the playout, goes with it. Within the anchoring frame's +/// tick of where the connection started. #[test] -#[ignore = "known gap: a video delay left within the relax hysteresis after a Target Buffer edit never drains"] fn a_target_buffer_edit_during_a_hold_build_ends_in_sync() { let report = target_edit_during_build(); let after = report.assert_settled(90.0, 120.0); report.print("target edit during build", &after); + report.assert_in_sync(&after, (report.tick_ns + 2 * MS) as i64); assert_eq!(report.video_delay_ms(90.0), 0..=0); } diff --git a/crates/irl-source/tests/network_sim.rs b/crates/irl-source/tests/network_sim.rs index 9ab2152..261b5ab 100644 --- a/crates/irl-source/tests/network_sim.rs +++ b/crates/irl-source/tests/network_sim.rs @@ -642,7 +642,10 @@ fn audio_starts_without_a_skew_reading_once_the_wait_runs_out() { /// covers it; after the raise window the hold rises to match, and the speed /// controller builds it by playing slower until video is back on time. The /// growth is the hold the user did not have to ask for, so it must not be -/// mistaken for concealment drift and thrown away by a re-anchor. +/// mistaken for concealment drift and thrown away by a re-anchor. Built, the +/// playout leaves video a canvas tick in hand on top of the tick it needs to +/// decode: the floor any delay was sized to sits under the offset, and the +/// delay it implies is zero. #[test] fn video_that_falls_behind_mid_stream_is_caught_up_by_slowing_audio() { let mut sim = Sim::new(120).with_video_trailing_by(50); @@ -661,10 +664,16 @@ fn video_that_falls_behind_mid_stream_is_caught_up_by_slowing_audio() { // target: about a minute for 600 ms. sim.run(70.0, Link::Up); sim.assert_in_lip_sync(); - let built_ms = sim.shared.audio_state().hold_built_ns / 1_000_000; + let built_ms = sim.mean_fill_ms(10.0); assert!( - (590..=600).contains(&built_ms), - "credited {built_ms}ms of a 600ms raise to the video thread" + (built_ms - f64::from(120 + 600)).abs() <= 25.0, + "the buffer holds {built_ms:.1}ms against a 720ms target" + ); + let tick_ms = (consts::VIDEO_CANVAS_TICK_DEFAULT_NS / 1_000_000) as i64; + assert!( + sim.video_margin_now_ms() >= 2 * tick_ms, + "video arriving now is {}ms early: a floor sized from this skew would still delay it", + sim.video_margin_now_ms() ); assert_eq!(sim.reanchors(), 0, "the hold building was read as drift"); sim.assert_healthy(); diff --git a/crates/irl-source/tests/video_pipeline.rs b/crates/irl-source/tests/video_pipeline.rs index 7fcfffb..368bdec 100644 --- a/crates/irl-source/tests/video_pipeline.rs +++ b/crates/irl-source/tests/video_pipeline.rs @@ -1327,10 +1327,11 @@ fn a_delay_set_by_a_bad_first_frame_ramps_back_without_a_jump() { run_at(&mut thread, &clock, t0 + (frames + k) * FRAME); } - // Untouched for the ten-second window, then down to a tick of headroom, - // gradually: 5% takes back 1.3 s in about 26 s. + // Untouched for the ten-second window, then down to what frames need, + // gradually: 5% takes back 1.3 s in about 26 s. 100 ms in hand covers the + // tick of decode and the tick of headroom, so nothing is left. assert_eq!(delays[8 * 30], set, "inside the window the delay stands"); - assert_eq!(*delays.last().unwrap(), TICK, "settled on what frames need"); + assert_eq!(*delays.last().unwrap(), 0, "settled on what frames need"); let halfway = delays[(10 + 13) * 30]; assert!( halfway > set / 3 && halfway < set * 2 / 3, @@ -1357,11 +1358,11 @@ fn a_delay_set_by_a_bad_first_frame_ramps_back_without_a_jump() { /// Once the audio hold rises for video that trails its audio, the speed /// controller builds it by playing slower, and the audio playout offset (with /// every due time mapped through it) grows until video is on time again. The -/// standing delay covered that same lateness in the meantime, so each -/// millisecond of growth is one it no longer needs: it hands it back as the -/// hold builds. Due times stay exactly where they were, so the picture -/// neither jumps nor changes speed while the lip-sync error the delay stood -/// for drains to nothing. +/// delay covered that same lateness in the meantime, and it is the floor less +/// the offset, so each millisecond of growth is one it no longer carries. Due +/// times stay exactly where they were, so the picture neither jumps nor +/// changes speed while the lip-sync error the delay stood for drains to +/// nothing. #[test] fn the_video_delay_hands_itself_back_as_the_audio_hold_builds() { let shared = shared_with_audio(); @@ -1377,15 +1378,13 @@ fn the_video_delay_hands_itself_back_as_the_audio_hold_builds() { let set = shared.conn.video_delay_ns.load(Relaxed); assert_eq!(set, (late + TICK).div_ceil(TICK) * TICK); - // The hold builds at -2 %: the offset grows by 2 % of each frame, and the - // pump credits the same growth as built. + // The hold builds at -2 %: the offset grows by 2 % of each frame. let growth_per_frame = FRAME / 50; let frames = 30 * 30u64; let mut delays = Vec::new(); for i in 1..=frames { let grown = i * growth_per_frame; publish_mapping(&shared, t0 - late + grown, 10_000_000_000); - shared.audio_state().hold_built_ns = grown; let arrival = t0 + i * FRAME; feed(&mut thread, 10_000_000_000 + (i * FRAME) as i64, arrival); run_at(&mut thread, &clock, arrival); From 8608e33d743561da96f21204291f286085bd70e7 Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 04:42:34 +0200 Subject: [PATCH 09/13] docs(workspace): describe the video delay as a floor under the schedule The Video delay and Audio hold sections describe the floor the delay is derived from, why the relax window slides, and the hold building without any hand-off to the video thread. --- docs/architecture.md | 16 ++++++++++------ docs/audio-pipeline.md | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 7c18a17..19028c6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -76,13 +76,17 @@ While an audio stream is present, the anchor frame's due time must come from the A frame can only get the lead if it is in hand a lead before it is due. Its arrival margin (due time minus the OBS time its packet reached the video thread) is set by the sender. Video that leaves the encoder later than its audio has less margin, and once the skew exceeds what Target Buffer and the audio output lead cover, every frame is late. A late frame goes out on arrival, unpaced, and libobs drops one whenever two arrive inside a canvas tick: low fps. -`irl_core::video_delay` closes that gap with a standing delay on every video due time, sized from the worst shortfall seen. The bar is "not late", not "a full lead early": the lead covers the pacing timer oversleeping on a queued frame, and a frame handed over on arrival never sleeps. +`irl_core::video_delay` closes that gap with a floor under the video schedule. A frame maps through the audio playout offset `O` (its PTS plus `O` is when the audio of the same instant plays), and the floor `F` is the lowest such offset at which frames are still in hand when due: the worst `arrival - PTS` seen, plus a tick of allowance for the decode. Every frame is due at `pts + max(O, min(F, O + VIDEO_DELAY_MAX_MS))`. The video delay, `F - O` clamped to between zero and `VIDEO_DELAY_MAX_MS`, is never stored. It follows from the floor and the offset in force, and `video_delay_ms` reports it. It is the lip-sync error, traded for a smooth picture. -- **Before the anchor**, a shortfall raises the delay at once, measured at packet arrival with one canvas tick of allowance for the decode still to come (`VideoThread::settle_anchor_candidate`). It measures only once video is live (`irl_core::arrival`): a relay hands a new subscriber video from its last keyframe next to live audio, so a connection can open with video a second behind and catching up faster than real time. Both streams share a mux, so the network cannot cause that reading; the test is whether the floor of arrival minus PTS has stopped falling across `VIDEO_LIVE_LOOKBACK_MS`, bounded by `VIDEO_LIVE_MAX_WAIT_MS`. It also measures only the newest frame in hand, because the probe backlog arrives as one burst with one arrival time and its older frames look late when they were only buffered. -- **After the anchor**, the measurement is the hand-over itself, and a raise moves the picture. It takes `VIDEO_DELAY_MIN_FRAMES` frames past due, spread across `VIDEO_DELAY_WINDOW_MS`, and the delay is capped at `VIDEO_DELAY_MAX_MS`. -- **Relax.** Every frame handed over also reports its arrival margin. When a whole `VIDEO_DELAY_RELAX_WINDOW_MS` of them needed less than the delay in force by more than `VIDEO_DELAY_RELAX_MIN_MS`, the delay ramps down to what the window needed plus a tick, at the Catch-Up Speed, so video plays a few percent fast and nothing jumps. A raise cancels the ramp. This matters because the pre-anchor measurement is one frame at connection start, and on a poor link it can read a second late on a stream with no skew. +Because the delay is derived, nothing has to keep it in step with the audio. When the playout grows (an audio hold building, concealment during an outage) the delay shrinks by the same amount and due times stay where they were. When the playout shrinks (a re-anchor, a hold released, a smaller Target Buffer) video follows it down only as far as the floor. The bar is "not late", not "a full lead early": the lead covers the pacing timer oversleeping on a queued frame, and a frame handed over on arrival never sleeps. -A clear and a decoder handover reset the delay. The video-only fallback schedules its first frame at arrival, so a stream without audio always carries a delay of exactly one tick. `video_delay_ms` reports it. +- **Before the anchor**, a shortfall raises the floor at once, measured at packet arrival with one canvas tick of allowance for the decode still to come (`VideoThread::settle_anchor_candidate`), and rounded so that the delay it sets is a whole number of ticks. It measures only once video is live (`irl_core::arrival`): a relay hands a new subscriber video from its last keyframe next to live audio, so a connection can open with video a second behind and catching up faster than real time. Both streams share a mux, so the network cannot cause that reading; the test is whether the floor of arrival minus PTS has stopped falling across `VIDEO_LIVE_LOOKBACK_MS`, bounded by `VIDEO_LIVE_MAX_WAIT_MS`. It also measures only the newest frame in hand, because the probe backlog arrives as one burst with one arrival time and its older frames look late when they were only buffered. +- **After the anchor**, the measurement is the hand-over itself, and a raise moves the picture. It takes `VIDEO_DELAY_MIN_FRAMES` frames past due, spread across `VIDEO_DELAY_WINDOW_MS`. The delay is capped at `VIDEO_DELAY_MAX_MS`; a sender later than that anchors on its newest frame and plays unpaced. +- **Relax.** Every frame handed over also reports the floor it needed at arrival. When every frame across the last `VIDEO_DELAY_RELAX_WINDOW_MS` (a sliding window, with no gap in the video longer than a raise window) needed less than the floor by more than `VIDEO_DELAY_RELAX_MIN_MS`, the floor comes down to what they needed plus a tick. Only the part of the floor above the offset is on screen. That part slews at the Catch-Up Speed, so video plays a few percent fast and nothing jumps, and the rest goes at once. A raise cancels the slew. This matters because the pre-anchor measurement is one frame at connection start, and on a poor link it can read a second late on a stream with no skew. The window slides so that a sender that recovers is noticed when the audio hold notices it: a hold released over a stale floor would put the picture behind the sound for nothing. + +The floor carries the stream's PTS epoch, so a clear, a decoder handover and a timeline reset (PTS repair starting a new timeline) forget it, together with the liveness history. Without a mapping, each frame keeps the fallback due time it was given at intake and the floor applies to that in the same way. The video-only fallback schedules its first frame at arrival, so a stream without audio always carries a delay of exactly one tick. + +The log warns when the floor rises, says when it starts to come down, and says once when the playout moving takes the delay to or from zero. ## Audio hold @@ -91,7 +95,7 @@ The video delay is a bridge, not the answer to a sender whose video trails its a The receiver reads the skew as it pushes each video packet: its decode timestamp against the newest audio PTS decoded before it. Mux order makes the reading trustworthy, since loss and stalls delay both streams together. A relay's catch-up replay is kept out by an `ArrivalFloor` of its own. The part of the skew that Target Buffer and the output lead do not cover, plus `AUDIO_HOLD_MARGIN_MS`, is folded into the published watermarks, so their target is Target Buffer plus the hold and every reader of the target follows it. `Config::apply_hot` composes a Target Buffer edit with it. - Before priming, the pump waits up to `AUDIO_HOLD_PRIME_WAIT_MS` for the first reading, so a sender that is always late starts in sync. -- After priming, the hold moves only while the speed controller regulates the buffer (not in low-latency mode, not with Adaptive Latency Control off). A skew sustained across `AUDIO_HOLD_RAISE_WINDOW_MS` raises it, and the controller builds it at -2 %. The pump credits that growth of the playout offset to `AudioState::hold_built_ns`, and `VideoThread::absorb_audio_hold` takes the same amount out of the video delay, so due times stay put while the lip-sync error drains. The offset re-anchor nets the hold out of its drift for the same reason. +- After priming, the hold moves only while the speed controller regulates the buffer (not in low-latency mode, not with Adaptive Latency Control off). A skew sustained across `AUDIO_HOLD_RAISE_WINDOW_MS` raises it, and the controller builds it at -2 %. The playout offset grows as it builds, and the video delay is the video floor less that offset, so the delay drains with it: due times stay put while the lip-sync error goes. The offset re-anchor nets the hold out of its drift, since that growth is latency asked for, not concealment. - Low-latency mode has no target to fold the hold into: the pump primes on the hold's worth of audio and keeps that much queued. Shorter bursts stay the video delay's job, and the delay's warnings stop suggesting a Sync Offset where the hold can move, since both would correct the same skew. `audio_hold_ms` reports the hold. diff --git a/docs/audio-pipeline.md b/docs/audio-pipeline.md index 6e8c13d..74fb146 100644 --- a/docs/audio-pipeline.md +++ b/docs/audio-pipeline.md @@ -134,7 +134,7 @@ Instead of a fixed audio offset, video is delayed by the current buffered-audio That mapping assumes the sender delivers the audio and the video of an instant together. Many do not. A hardware encoder runs a few frames behind the microphone, pocketSRT queues its audio about 300ms ahead of its deadline, and a phone with video stabilization on sends each frame a second or more after its sound. Mapped through the audio playout, such video is past due when it arrives, and no video schedule can fix that, because a frame cannot be shown before it arrives. So the audio waits for the picture. The receiver reads the skew as each video packet arrives: the packet's timestamp against the newest audio decoded before it, in mux order, so a network stall (which delays both streams together) does not move the reading. The part of the skew that Target Buffer and the output lead do not cover, plus a 100ms margin, becomes the audio hold, and the jitter buffer's target is raised by it. -The hold is sized before playback primes, so a sender that always sends its video late starts in sync. After that it rises only for a skew that lasts two seconds, and the speed controller builds it by playing at -2%. The standing video delay covers the gap meanwhile and hands itself back as the audio playout grows, so the picture neither jumps nor changes speed. A shorter burst of late video stays the video delay's to cover, since paying for a hiccup in latency for the rest of the stream is the wrong trade. The hold falls back once ten seconds of video needed less. The hold is reported as `audio_hold_ms`, the raw stamp difference as `av_skew_ms`. +The hold is sized before playback primes, so a sender that always sends its video late starts in sync. After that it rises only for a skew that lasts two seconds, and the speed controller builds it by playing at -2%. The video delay covers the gap meanwhile. It is the video floor less the audio playout, so it shrinks as the playout grows, and the picture neither jumps nor changes speed. A shorter burst of late video stays the video delay's to cover, since paying for a hiccup in latency for the rest of the stream is the wrong trade. The hold falls back once ten seconds of video needed less. The hold is reported as `audio_hold_ms`, the raw stamp difference as `av_skew_ms`. When there is no audio playout mapping yet (audio-less start), video falls back to the rebased timestamp, and if that drifts too far from wall clock (over 500ms) it is clamped rather than fully re-anchored. That avoids visible jumps while still preventing long freezes if the stream sends a bad future timestamp. From 30faa1fe8e9f5bb3e423020a9bb54ca08fbff94f Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 04:44:30 +0200 Subject: [PATCH 10/13] fix(irl-source): an audio format change forgets the drift baseline with the mapping the output clock restarts on a reconfigure, so the offset it primes on is not comparable with the baseline of the old format, and the re-anchor check measured drift against it. --- crates/irl-core/src/playout.rs | 14 +------------- crates/irl-source/src/receiver/audio_in.rs | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/crates/irl-core/src/playout.rs b/crates/irl-core/src/playout.rs index 971023c..db1a70a 100644 --- a/crates/irl-core/src/playout.rs +++ b/crates/irl-core/src/playout.rs @@ -86,12 +86,6 @@ impl PlayoutMapping { Some(offset_ns - baseline.offset_ns - hold_moved_ns) } - /// Forget the latest chunk and keep the baseline. - pub fn unpublish(&mut self) { - self.obs_end_ns = 0; - self.pts_end_ns = 0; - } - /// Forget the latest chunk and the baseline. pub fn clear(&mut self) { *self = Self::default(); @@ -174,16 +168,10 @@ mod tests { } #[test] - fn unpublishing_keeps_the_baseline_and_clearing_does_not() { + fn clearing_forgets_the_baseline() { let mut mapping = published(3_000_000_000, 10_000_000_000); assert!(mapping.take_baseline(0)); - mapping.unpublish(); - assert!(!mapping.has_output()); - assert_eq!(mapping.drift_ns(0), None); - mapping.publish(3_050_000_000, 10_000_000_000); - assert_eq!(mapping.drift_ns(0), Some(50_000_000)); - mapping.clear(); assert_eq!(mapping, PlayoutMapping::default()); mapping.publish(3_050_000_000, 10_000_000_000); diff --git a/crates/irl-source/src/receiver/audio_in.rs b/crates/irl-source/src/receiver/audio_in.rs index 036e6f4..6894b3b 100644 --- a/crates/irl-source/src/receiver/audio_in.rs +++ b/crates/irl-source/src/receiver/audio_in.rs @@ -246,7 +246,7 @@ impl AudioIntake { state.clock.stand_down(); state.out_last.forget(); - state.mapping.unpublish(); + state.mapping.clear(); state.latest_audio_stream_pts_ns = 0; state.startup_warmup_remaining_ms = consts::STARTUP_AUDIO_WARMUP_MS; reconfigured From d92f22583be73a344352b4b920b142cdf0667e8f Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 04:54:03 +0200 Subject: [PATCH 11/13] chore(irl-core): udp relay that delays the video of an mpeg-ts stream --- crates/irl-core/examples/skew-relay.rs | 508 +++++++++++++++++++++++++ 1 file changed, 508 insertions(+) create mode 100644 crates/irl-core/examples/skew-relay.rs diff --git a/crates/irl-core/examples/skew-relay.rs b/crates/irl-core/examples/skew-relay.rs new file mode 100644 index 0000000..6ddd967 --- /dev/null +++ b/crates/irl-core/examples/skew-relay.rs @@ -0,0 +1,508 @@ +//! UDP MPEG-TS relay that delays the video of a stream and passes everything +//! else through, to reproduce a sender whose video reaches the plugin later +//! than its audio. +//! +//! ```shell +//! cargo run -p irl-core --example skew-relay -- \ +//! --listen 127.0.0.1:9000 --to 127.0.0.1:9001 --delay-ms 500 +//! ``` +//! +//! Shifting timestamps (`ffmpeg -itsoffset`) does not reproduce that: the +//! plugin plays what the sender stamped. What a phone with stabilisation on, +//! or pocketSRT queueing its audio early, does is keep the stamps aligned and +//! deliver the video late, so this holds the TS packets of the video PID and +//! leaves the stamps alone. `docs/skew-testing.md` is the recipe around it. +//! +//! The clock of `--ramp` and `--stall-every` starts at the first datagram. +//! Run the tests with `cargo test -p irl-core --example skew-relay`. + +#![forbid(unsafe_code)] + +use std::collections::{BTreeMap, VecDeque}; +use std::io::ErrorKind; +use std::net::{SocketAddr, UdpSocket}; +use std::process::ExitCode; +use std::time::{Duration, Instant}; + +const USAGE: &str = "usage: skew-relay --listen ADDR --to ADDR [--video-pid PID] \ + [--delay-ms N | --ramp FROM:TO:SECONDS] [--stall-every SECONDS --stall-ms N]"; + +const TS_LEN: usize = 188; +const TS_SYNC: u8 = 0x47; +/// 7 x 188 = 1316 bytes, the usual MPEG-TS over UDP payload, under a 1500 MTU. +const PACKETS_PER_DATAGRAM: usize = 7; +const STREAM_TYPE_H264: u8 = 0x1B; +const STREAM_TYPE_HEVC: u8 = 0x24; +const STATUS_EVERY: Duration = Duration::from_secs(5); +const IDLE_WAIT: Duration = Duration::from_millis(20); +const MIN_WAIT: Duration = Duration::from_millis(1); + +type Packet = [u8; TS_LEN]; + +#[derive(Debug, Clone, Copy, PartialEq)] +enum Base { + Constant { ms: f64 }, + Ramp { from_ms: f64, to_ms: f64, secs: f64 }, +} + +/// Video is held for `hold_ms` up to every multiple of `every_s`, then the +/// backlog goes out as one burst. +#[derive(Debug, Clone, Copy, PartialEq)] +struct Stall { + every_s: f64, + hold_ms: f64, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +struct Schedule { + base: Base, + stall: Option, +} + +impl Schedule { + /// The delay of a video packet that arrives `t` seconds after the first + /// datagram. + fn delay_ms(&self, t: f64) -> f64 { + let base = match self.base { + Base::Constant { ms } => ms, + Base::Ramp { + from_ms, + to_ms, + secs, + } => from_ms + (to_ms - from_ms) * (t / secs).clamp(0.0, 1.0), + }; + let stall = self.stall.map_or(0.0, |s| { + let to_release_ms = (s.every_s - t.rem_euclid(s.every_s)) * 1000.0; + if to_release_ms <= s.hold_ms { + to_release_ms + } else { + 0.0 + } + }); + base + stall + } +} + +#[derive(Debug, PartialEq)] +struct Config { + listen: SocketAddr, + to: SocketAddr, + video_pid: Option, + schedule: Schedule, +} + +fn parse_args(mut args: impl Iterator) -> Result { + let (mut listen, mut to, mut video_pid) = (None, None, None); + let (mut delay, mut ramp, mut stall_every, mut stall_ms) = (None, None, None, None); + while let Some(flag) = args.next() { + let value = args.next().ok_or_else(|| format!("{flag} needs a value"))?; + let bad = |what: &str| format!("{flag}: {what}, got {value:?}"); + match flag.as_str() { + "--listen" => listen = Some(value.parse().map_err(|_| bad("not an address"))?), + "--to" => to = Some(value.parse().map_err(|_| bad("not an address"))?), + "--video-pid" => video_pid = Some(parse_pid(&value).ok_or_else(|| bad("not a PID"))?), + "--delay-ms" => delay = Some(parse_non_negative(&value).ok_or_else(|| bad("not ms"))?), + "--stall-ms" => { + stall_ms = Some(parse_non_negative(&value).ok_or_else(|| bad("not ms"))?) + } + "--stall-every" => { + let secs = parse_non_negative(&value).filter(|s| *s > 0.0); + stall_every = Some(secs.ok_or_else(|| bad("not seconds"))?); + } + "--ramp" => { + let parts: Option> = value.split(':').map(parse_non_negative).collect(); + match parts.as_deref() { + Some(&[from_ms, to_ms, secs]) if secs > 0.0 => { + ramp = Some(Base::Ramp { + from_ms, + to_ms, + secs, + }); + } + _ => return Err(bad("expected FROM:TO:SECONDS")), + } + } + _ => return Err(format!("unknown flag {flag}")), + } + } + let base = match (delay, ramp) { + (Some(_), Some(_)) => return Err("--delay-ms and --ramp exclude each other".into()), + (_, Some(ramp)) => ramp, + (ms, None) => Base::Constant { + ms: ms.unwrap_or(0.0), + }, + }; + let stall = match (stall_every, stall_ms) { + (None, None) => None, + (Some(every_s), Some(hold_ms)) if hold_ms < every_s * 1000.0 => { + Some(Stall { every_s, hold_ms }) + } + (Some(_), Some(_)) => return Err("--stall-ms must be shorter than --stall-every".into()), + _ => return Err("--stall-every and --stall-ms go together".into()), + }; + Ok(Config { + listen: listen.ok_or("--listen is required")?, + to: to.ok_or("--to is required")?, + video_pid, + schedule: Schedule { base, stall }, + }) +} + +fn parse_non_negative(s: &str) -> Option { + s.parse::().ok().filter(|v| v.is_finite() && *v >= 0.0) +} + +fn parse_pid(s: &str) -> Option { + let pid = match s.strip_prefix("0x").or_else(|| s.strip_prefix("0X")) { + Some(hex) => u16::from_str_radix(hex, 16).ok()?, + None => s.parse().ok()?, + }; + (pid < 0x1FFF).then_some(pid) +} + +fn pid_of(pkt: &[u8]) -> u16 { + (u16::from(pkt[1] & 0x1F) << 8) | u16::from(pkt[2]) +} + +/// The PSI section that starts in `pkt`, from its table id up to its CRC. +/// Sections that continue into the next packet are ignored: PAT and PMT fit +/// in one for every stream this is meant for. +fn section(pkt: &[u8]) -> Option<&[u8]> { + let starts_unit = pkt[1] & 0x40 != 0; + let control = (pkt[3] >> 4) & 0x3; + if !starts_unit || control & 0x1 == 0 { + return None; + } + let mut at = 4; + if control & 0x2 != 0 { + at += 1 + usize::from(pkt[4]); + } + let s = pkt.get(at + 1 + usize::from(*pkt.get(at)?)..)?; + let len = (usize::from(*s.get(1)? & 0x0F) << 8) | usize::from(*s.get(2)?); + s.get(..3 + len)?.get(..(3 + len).checked_sub(4)?) +} + +fn pmt_pids(pat: &[u8]) -> Vec { + if pat.first() != Some(&0x00) { + return Vec::new(); + } + let programs = pat.get(8..).unwrap_or_default().chunks_exact(4); + // Program 0 points at the network PID, not a PMT. + programs + .filter(|p| p[0] != 0 || p[1] != 0) + .map(|p| pid_of(&p[1..])) + .collect() +} + +fn pmt_video_pid(pmt: &[u8]) -> Option { + if pmt.first() != Some(&0x02) { + return None; + } + let mut at = 12 + ((usize::from(*pmt.get(10)? & 0x0F) << 8) | usize::from(*pmt.get(11)?)); + while let Some(es) = pmt.get(at..at + 5) { + if matches!(es[0], STREAM_TYPE_H264 | STREAM_TYPE_HEVC) { + return Some(pid_of(es)); + } + at += 5 + ((usize::from(es[3] & 0x0F) << 8) | usize::from(es[4])); + } + None +} + +struct Relay { + out: UdpSocket, + to: SocketAddr, + schedule: Schedule, + video_pid: Option, + pmt_pids: Vec, + origin: Option, + /// Video in arrival order, each with its release time. Released strictly + /// from the front, so a delay that falls faster than real time bunches + /// packets up instead of reordering them. + held: VecDeque<(Instant, Packet)>, + batch: Vec, + /// Packets in and out per PID. + counts: BTreeMap, + raw_datagrams: u64, + send_errors: u64, +} + +impl Relay { + fn elapsed_s(&self, now: Instant) -> f64 { + self.origin + .map_or(0.0, |o| now.duration_since(o).as_secs_f64()) + } + + fn ingest(&mut self, datagram: &[u8], now: Instant) { + let origin = *self.origin.get_or_insert(now); + let aligned = !datagram.is_empty() && datagram.len().is_multiple_of(TS_LEN); + if !aligned || datagram.chunks(TS_LEN).any(|p| p[0] != TS_SYNC) { + // Not plain TS (RTP, or a stray sender): forward it untouched. + self.raw_datagrams += 1; + self.flush(); + self.send_errors += u64::from(self.out.send_to(datagram, self.to).is_err()); + return; + } + let delay_ms = self + .schedule + .delay_ms(now.duration_since(origin).as_secs_f64()); + let release = now + Duration::from_secs_f64(delay_ms / 1000.0); + for chunk in datagram.chunks_exact(TS_LEN) { + let pkt: Packet = chunk.try_into().expect("chunks_exact yields TS_LEN bytes"); + let pid = pid_of(&pkt); + self.counts.entry(pid).or_default().0 += 1; + self.learn(pid, &pkt); + if Some(pid) != self.video_pid || (release <= now && self.held.is_empty()) { + self.emit(&pkt); + } else { + self.held.push_back((release, pkt)); + } + } + } + + /// Follows PAT and PMT until the video PID is known, unless it was given. + fn learn(&mut self, pid: u16, pkt: &[u8]) { + if self.video_pid.is_some() { + return; + } + let Some(sec) = section(pkt) else { return }; + if pid == 0 { + self.pmt_pids = pmt_pids(sec); + } else if self.pmt_pids.contains(&pid) { + self.video_pid = pmt_video_pid(sec); + } + } + + fn next_release(&self) -> Option { + self.held.front().map(|(at, _)| *at) + } + + fn release_due(&mut self, now: Instant) { + while let Some((at, _)) = self.held.front() + && *at <= now + { + let (_, pkt) = self.held.pop_front().expect("front was Some"); + self.emit(&pkt); + } + } + + fn emit(&mut self, pkt: &Packet) { + self.counts.entry(pid_of(pkt)).or_default().1 += 1; + self.batch.extend_from_slice(pkt); + if self.batch.len() == PACKETS_PER_DATAGRAM * TS_LEN { + self.flush(); + } + } + + fn flush(&mut self) { + if !self.batch.is_empty() { + self.send_errors += u64::from(self.out.send_to(&self.batch, self.to).is_err()); + self.batch.clear(); + } + } + + fn status(&self, now: Instant) -> String { + let t = self.elapsed_s(now); + let mut line = format!( + "[{t:7.1}s] delay={:.0}ms held={} raw={} send_err={}", + self.schedule.delay_ms(t), + self.held.len(), + self.raw_datagrams, + self.send_errors + ); + for (pid, (inn, out)) in &self.counts { + let tag = if Some(*pid) == self.video_pid { + " video" + } else { + "" + }; + line.push_str(&format!(" | {pid:#06x}{tag} {inn}/{out}")); + } + line + } +} + +fn run(cfg: Config) -> std::io::Result<()> { + let listen = UdpSocket::bind(cfg.listen)?; + let any: SocketAddr = if cfg.to.is_ipv4() { + "0.0.0.0:0" + } else { + "[::]:0" + } + .parse() + .expect("literal address"); + let mut relay = Relay { + out: UdpSocket::bind(any)?, + to: cfg.to, + schedule: cfg.schedule, + video_pid: cfg.video_pid, + pmt_pids: Vec::new(), + origin: None, + held: VecDeque::new(), + batch: Vec::with_capacity(PACKETS_PER_DATAGRAM * TS_LEN), + counts: BTreeMap::new(), + raw_datagrams: 0, + send_errors: 0, + }; + eprintln!("relaying {} -> {}, {:?}", cfg.listen, cfg.to, cfg.schedule); + let mut buf = vec![0u8; 65536]; + let mut next_status = Instant::now() + STATUS_EVERY; + loop { + let now = Instant::now(); + let wait = relay + .next_release() + .map_or(IDLE_WAIT, |at| at.saturating_duration_since(now)) + .clamp(MIN_WAIT, IDLE_WAIT); + listen.set_read_timeout(Some(wait))?; + match listen.recv(&mut buf) { + Ok(n) => relay.ingest(&buf[..n], Instant::now()), + Err(e) if matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => {} + Err(e) => return Err(e), + } + let now = Instant::now(); + relay.release_due(now); + relay.flush(); + if now >= next_status { + eprintln!("{}", relay.status(now)); + next_status += STATUS_EVERY; + } + } +} + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + if args.is_empty() || args.iter().any(|a| a == "-h" || a == "--help") { + eprintln!("{USAGE}"); + return if args.is_empty() { + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + }; + } + let cfg = match parse_args(args.into_iter()) { + Ok(cfg) => cfg, + Err(e) => { + eprintln!("{e}\n{USAGE}"); + return ExitCode::FAILURE; + } + }; + match run(cfg) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("skew-relay: {e}"); + ExitCode::FAILURE + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A packet on `pid` carrying one PSI section: `table_id`, then `body` + /// after the 5 bytes of the extended section header, then a dummy CRC. + fn psi_packet(pid: u16, table_id: u8, body: &[u8]) -> Packet { + let len = 5 + body.len() + 4; + let mut pkt = [0xFF; TS_LEN]; + pkt[..5].copy_from_slice(&[TS_SYNC, 0x40 | (pid >> 8) as u8, pid as u8, 0x10, 0x00]); + let mut sec = vec![ + table_id, + 0xB0 | (len >> 8) as u8, + len as u8, + 0, + 1, + 0xC1, + 0, + 0, + ]; + sec.extend_from_slice(body); + sec.extend_from_slice(&[0; 4]); + pkt[5..5 + sec.len()].copy_from_slice(&sec); + pkt + } + + #[test] + fn pat_lists_pmt_pids_and_skips_the_network_pid() { + let pkt = psi_packet(0, 0x00, &[0, 0, 0xE0, 0x10, 0, 1, 0xF0, 0x00]); + assert_eq!(pmt_pids(section(&pkt).unwrap()), vec![0x1000]); + } + + #[test] + fn pmt_finds_the_video_pid_past_audio_and_descriptors() { + let body = [ + 0xE1, 0x00, 0xF0, 0x00, // PCR PID, no program info + 0x0F, 0xE1, 0x01, 0xF0, 0x02, 0x0A, 0x00, // AAC on 0x101, 2 bytes of descriptor + 0x24, 0xE1, 0x00, 0xF0, 0x00, // HEVC on 0x100 + ]; + let pkt = psi_packet(0x1000, 0x02, &body); + assert_eq!(pmt_video_pid(section(&pkt).unwrap()), Some(0x100)); + } + + #[test] + fn pmt_without_video_has_no_video_pid() { + let pkt = psi_packet( + 0x1000, + 0x02, + &[0xE1, 0x01, 0xF0, 0x00, 0x0F, 0xE1, 0x01, 0xF0, 0], + ); + assert_eq!(pmt_video_pid(section(&pkt).unwrap()), None); + } + + #[test] + fn ramp_is_linear_then_holds() { + let s = Schedule { + base: Base::Ramp { + from_ms: 0.0, + to_ms: 1000.0, + secs: 10.0, + }, + stall: None, + }; + assert_eq!(s.delay_ms(5.0), 500.0); + assert_eq!(s.delay_ms(20.0), 1000.0); + } + + #[test] + fn stall_holds_up_to_each_period_boundary() { + let stall = Some(Stall { + every_s: 4.0, + hold_ms: 200.0, + }); + let s = Schedule { + base: Base::Constant { ms: 500.0 }, + stall, + }; + assert_eq!(s.delay_ms(1.0), 500.0); + assert!((s.delay_ms(3.85) - 650.0).abs() < 1e-6); + assert_eq!(s.delay_ms(4.0), 500.0); + } + + #[test] + fn args_parse_and_reject_conflicts() { + let args = |s: &str| { + s.split(' ') + .map(String::from) + .collect::>() + .into_iter() + }; + let cfg = parse_args(args( + "--listen 127.0.0.1:9000 --to 127.0.0.1:9001 --video-pid 0x100 \ + --ramp 0:1000:30 --stall-every 4 --stall-ms 200", + )) + .unwrap(); + assert_eq!(cfg.video_pid, Some(0x100)); + assert_eq!( + cfg.schedule.stall, + Some(Stall { + every_s: 4.0, + hold_ms: 200.0 + }) + ); + assert!( + parse_args(args( + "--listen 127.0.0.1:1 --to 127.0.0.1:2 --delay-ms 5 --ramp 0:1:1" + )) + .is_err() + ); + assert!(parse_args(args("--listen 127.0.0.1:1 --to 127.0.0.1:2 --stall-ms 5")).is_err()); + } +} From b21cffa2cc21c84fd959ee32268131becd9f0a49 Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 04:54:04 +0200 Subject: [PATCH 12/13] docs(workspace): how to reproduce late video against a real obs --- CLAUDE.md | 1 + docs/architecture.md | 2 +- docs/skew-testing.md | 102 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 docs/skew-testing.md diff --git a/CLAUDE.md b/CLAUDE.md index 080f728..8f59a0e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,6 +69,7 @@ scripts/package.sh linux target/release dist # the release archive, locall - `irl-core` and `ffmpeg` have unit tests in each module. Everywhere else, tests live in `tests/`. The link arguments that resolve libobs only reach integration-test targets, so `crates/irl-source` sets `test = false` on its lib and `crates/obs` keeps its lib free of `#[cfg(test)]`. Shared fixtures for irl-source live in `crates/irl-source/tests/common/`. - Tests that touch libobs run on Linux. CI skips them on Windows and macOS. `calldata_*` is safe anywhere. On a Mac, `make test-shim` runs the pacing, audio-core and network tests against a four-function stand-in; plain `cargo test -p irl-source` there dies with SIGSEGV before its first assertion. - `tests/network_sim.rs` drives the real jitter buffer, PTS repair, speed controller, output clock and packet queue against a synthetic sender on a virtual clock (stall, burst, dropouts, a sender off wall clock, video behind audio). It asserts the design's promises: the OBS clock never jumps except across a declared restart, audio is never skipped once primed, latency does not ratchet, decoded memory does not grow with Target Buffer. +- `tests/av_sync_sim.rs` is the end-to-end A/V sync sim: it measures the real offset between picture and sound as libobs is handed them. `docs/skew-testing.md` reproduces the same cases on a real OBS with `crates/irl-core/examples/skew-relay.rs`. - Where you read the buffer fill decides the number. It oscillates by one chunk within every cycle: before the pump's read it averages the target, after it a chunk lower. The stats line's `buf=` is a random sample of that oscillation. - `tests/video_pipeline.rs` reads the real clock and can flake under parallel load; rerun with `-- --test-threads=1` before suspecting the code. - Real-stream validation is manual: run the same feed through this build and a known-good one and compare the 30-second stats line field by field. diff --git a/docs/architecture.md b/docs/architecture.md index 19028c6..e45d7c2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -This page explains why the plugin is shaped the way it is: the threads, what each one owns, and how audio and video are kept in step. `CLAUDE.md` lists the rules that follow from it. `docs/audio-pipeline.md` covers the audio path in depth, and `docs/audio-timing-pitfalls.md` covers the speed controller. +This page explains why the plugin is shaped the way it is: the threads, what each one owns, and how audio and video are kept in step. `CLAUDE.md` lists the rules that follow from it. `docs/audio-pipeline.md` covers the audio path in depth, `docs/audio-timing-pitfalls.md` covers the speed controller, and `docs/skew-testing.md` reproduces late video on a real OBS. ## Crates diff --git a/docs/skew-testing.md b/docs/skew-testing.md new file mode 100644 index 0000000..d3cd80a --- /dev/null +++ b/docs/skew-testing.md @@ -0,0 +1,102 @@ +# Reproducing late video + +This page reproduces a sender whose video reaches the plugin later than its audio (issues #33 and #34) on a real OBS with the plugin loaded, and measures the lip sync that comes out. + +The bug class is arrival skew. The sender stamps audio and video correctly at capture, but the video packets of an instant arrive after the audio packets of the same instant. A phone with video stabilisation on delivers its video a second or more late. pocketSRT queues its audio 300 to 700 ms early. Shifting timestamps with `ffmpeg -itsoffset` does not reproduce this: it changes what the sender stamped, and the plugin plays what the sender stamped. The relay in `crates/irl-core/examples/skew-relay.rs` keeps the stamps and delays the delivery instead. It forwards MPEG-TS over UDP, holds every TS packet of the video PID for a configurable time, and passes audio and tables through at once. + +## What you need + +- OBS with the plugin build under test. +- ffmpeg with libx264 and the `lavfi` device. +- The relay, built with `cargo build --release -p irl-core --example skew-relay`. It lands in `target/release/examples/skew-relay`. + +## Set up the rig + +1. In OBS, add an IRL Source with the URL `udp://127.0.0.1:9001`. A new source fits itself to the canvas; keep it that way, because the flash detection below reads the whole frame. If OBS runs on another machine, use `udp://0.0.0.0:9001` and point the relay's `--to` at that machine. +2. Mute every other audio source in the scene. +3. Start the relay with no delay: + + ```shell + target/release/examples/skew-relay --listen 127.0.0.1:9000 --to 127.0.0.1:9001 + ``` + +4. Start the test source: + + ```shell + ffmpeg -re \ + -f lavfi -i "testsrc2=size=1280x720:rate=30,drawbox=x=0:y=0:w=iw:h=ih:color=white:t=fill:enable='lt(mod(t,1),0.04)'" \ + -f lavfi -i "aevalsrc='0.05*sin(2*PI*440*t)+0.5*sin(2*PI*1000*t)*lt(mod(t,1),0.04)':s=48000" \ + -c:v libx264 -preset veryfast -tune zerolatency -g 60 -b:v 4M \ + -c:a aac -b:a 128k \ + -f mpegts -pes_payload_size 0 "udp://127.0.0.1:9000?pkt_size=1316" + ``` + + The picture turns white for 40 ms at the start of every second, and a 1 kHz beep plays for the same 40 ms. Both come from the same timeline, so they coincide at capture. + +Two details in that command keep the rig honest. The quiet 440 Hz bed under the beep keeps the AAC frames full sized, and `-pes_payload_size 0` sends one audio frame per PES packet. Without them the MPEG-TS muxer packs up to 15 audio frames into one PES, the audio then arrives in bursts, and the bursts read as a skew of a few hundred milliseconds on their own. + +## Take a baseline + +1. Let the stream run for a minute with the relay at no delay. +2. Read the stats line (see below). Expect `av_skew` around -50 ms, which is the encoder's own mux order, `hold=0ms` and `vdelay` near zero. +3. Measure the lip sync from a recording (see below). + +This is the rig's own offset. Subtract it from every later measurement. + +## Delay profiles + +Stop the relay with Ctrl+C and start it again with the flags of a profile. ffmpeg keeps sending through the restart. The relay starts its clock at the first datagram it receives, so a restart also restarts a ramp or a stall cycle. + +| case | relay flags | +| --- | --- | +| pocketSRT: audio 500 ms early, and every 4 s the video stalls for 200 ms and then arrives as a burst | `--delay-ms 500 --stall-every 4 --stall-ms 200` | +| a phone with stabilisation on | `--delay-ms 1650` | +| skew that builds up | `--ramp 0:1000:30` | +| skew that recovers | `--ramp 1000:0:30` | + +`--ramp FROM:TO:SECONDS` moves the delay linearly from `FROM` to `TO` milliseconds over `SECONDS`, then holds `TO`. `--stall-every` and `--stall-ms` add to either a constant delay or a ramp. Video packets leave the relay in the order they came, so a ramp down bunches them up rather than reordering them. + +The relay finds the video PID from the PAT and the PMT (stream types H.264 and HEVC). Pass `--video-pid` (decimal or `0x` hex) for a stream that declares its video otherwise. Every 5 s it prints the current delay, how many video packets it holds, and the packets in and out per PID. + +## Read the stats line + +The plugin logs a stats line every 30 seconds to the OBS log (Help, Log Files). The three fields that matter here: + +```shell +grep -o "av_skew=[^ ]* hold=[^ ]* vdelay=[^ ]*" +``` + +- `av_skew` is the video PTS minus the audio PTS of what last reached the plugin. With the relay it reads about minus the relay's delay, plus the baseline. A stall makes it dip further while it lasts. +- `hold` is how much longer than Target Buffer the audio waits. It covers the part of the skew that Target Buffer and the output lead do not, plus a 100 ms margin. A sender that is late from the first packet is covered before audio starts. A skew that appears later raises the hold once it lasts 2 s, and the hold falls again after 10 s of video that needed less. In Low Latency Audio mode, or with Adaptive Latency Control off, the hold is sized only when the stream starts. +- `vdelay` is the standing delay on the video schedule. It bridges a skew until the hold is built, then returns to zero. A `vdelay` that stays above zero while the hold stands means video is late for a reason on this machine, such as decoding. + +## Measure lip sync from a recording + +1. Set the OBS recording format to MKV. +2. Wait until the stats line shows the hold settled, then record for 30 s. +3. List the flash frames: + + ```shell + ffmpeg -hide_banner -i recording.mkv -an \ + -vf "signalstats,metadata=mode=select:key=lavfi.signalstats.YAVG:value=200:function=greater,metadata=mode=print" \ + -f null - 2>&1 | grep -o "pts_time:[0-9.]*" + ``` + + Each flash shows up as one to three consecutive frames. The first of each run is the flash. + +4. List the beep onsets: + + ```shell + ffmpeg -hide_banner -i recording.mkv -vn -af "silencedetect=noise=-20dB:duration=0.5" \ + -f null - 2>&1 | grep -o "silence_end: [0-9.]*" + ``` + + The quiet bed stays under the -20 dB threshold, so the gaps between beeps count as silence and each `silence_end` is a beep onset. Ignore the last one, which is the end of the file. + +5. For each second, subtract the beep onset from the flash time. A positive result is the picture behind the sound. Subtract the baseline. + +The rig resolves one video frame (33 ms at 30 fps) and one AAC frame (21 ms). Treat differences below that as noise. The same check works by eye in a video editor: step to the first white frame and compare it with the start of the beep in the waveform. + +## The automated counterpart + +`crates/irl-source/tests/av_sync_sim.rs` runs the same cases (audio ahead with recurring video stalls, video seconds behind its audio, a skew that rises and recovers) through the receiver's intake, the audio pump and the video thread on a virtual clock, and measures the offset libobs is handed. Use the relay to confirm a sim result on a real OBS, and turn anything the relay finds into a case in the sim. From 89bf510a9269f2c6988a6aa6f6813b2e322fd5c1 Mon Sep 17 00:00:00 2001 From: datagutt Date: Fri, 2 Oct 2026 04:54:26 +0200 Subject: [PATCH 13/13] docs(workspace): video pipeline tests run on a virtual clock and no longer flake --- CLAUDE.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8f59a0e..d0861c3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,9 +69,8 @@ scripts/package.sh linux target/release dist # the release archive, locall - `irl-core` and `ffmpeg` have unit tests in each module. Everywhere else, tests live in `tests/`. The link arguments that resolve libobs only reach integration-test targets, so `crates/irl-source` sets `test = false` on its lib and `crates/obs` keeps its lib free of `#[cfg(test)]`. Shared fixtures for irl-source live in `crates/irl-source/tests/common/`. - Tests that touch libobs run on Linux. CI skips them on Windows and macOS. `calldata_*` is safe anywhere. On a Mac, `make test-shim` runs the pacing, audio-core and network tests against a four-function stand-in; plain `cargo test -p irl-source` there dies with SIGSEGV before its first assertion. - `tests/network_sim.rs` drives the real jitter buffer, PTS repair, speed controller, output clock and packet queue against a synthetic sender on a virtual clock (stall, burst, dropouts, a sender off wall clock, video behind audio). It asserts the design's promises: the OBS clock never jumps except across a declared restart, audio is never skipped once primed, latency does not ratchet, decoded memory does not grow with Target Buffer. -- `tests/av_sync_sim.rs` is the end-to-end A/V sync sim: it measures the real offset between picture and sound as libobs is handed them. `docs/skew-testing.md` reproduces the same cases on a real OBS with `crates/irl-core/examples/skew-relay.rs`. +- `tests/av_sync_sim.rs` is the end-to-end A/V sync sim: it measures the real offset between picture and sound as libobs is handed them. Run it whenever you touch the video delay, the audio hold or the playout mapping. `docs/skew-testing.md` reproduces the same cases on a real OBS with `crates/irl-core/examples/skew-relay.rs`. - Where you read the buffer fill decides the number. It oscillates by one chunk within every cycle: before the pump's read it averages the target, after it a chunk lower. The stats line's `buf=` is a random sample of that oscillation. -- `tests/video_pipeline.rs` reads the real clock and can flake under parallel load; rerun with `-- --test-threads=1` before suspecting the code. - Real-stream validation is manual: run the same feed through this build and a known-good one and compare the 30-second stats line field by field. ## CI and releases