From 6ff05196239986b8b2c4b12f192b422b2ad2d55b Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Sun, 2 Aug 2026 11:40:02 -0700 Subject: [PATCH] webrtc: restore ticker-paced audio; pace video against a schedule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-sample-duration rewrite paced both tracks by sleeping each sample's duration after writing it. Each sleep overshoots by scheduler latency, and chaining sleeps accumulates the overshoot into real drift — hundreds of ms per segment at audio packet rates — so the sender fell behind real time, the receiver's jitter buffer starved, and concealment made the audio garbled. Audio never needed per-sample durations anyway: Opus packets are a constant 20ms, so the original uniform-duration ticker (which self-corrects, and was working flawlessly in production) is simply restored. Video keeps its real per-sample durations — that's the fix for non-uniform frame spacing — but now paces against a wall-clock schedule instead of chained sleeps, absorbing overshoot in the next iteration the way a ticker does. Co-Authored-By: Claude Fable 5 --- pkg/media/webrtc_playback2.go | 39 +++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/pkg/media/webrtc_playback2.go b/pkg/media/webrtc_playback2.go index e287eb6e2..e4e728d8a 100644 --- a/pkg/media/webrtc_playback2.go +++ b/pkg/media/webrtc_playback2.go @@ -172,10 +172,35 @@ func (mm *MediaManager) WebRTCPlayback2(ctx context.Context, user string, rendit } else if !audioOnly { log.Warn(ctx, "no video samples to write") } + // The audio path deliberately keeps the original + // uniform-duration ticker instead of writeSamples: Opus + // packets are a constant 20ms, so the uniform split is + // exact, and this path is proven in production — audio + // timing regressions are immediately audible as garbling. + // Video is the track that needs per-sample durations (its + // spacing goes non-uniform when an encoder sheds frames). + var audioDur time.Duration if len(packet.Audio) > 0 { + audioDur = packet.Duration / time.Duration(len(packet.Audio)) + } + if audioDur > 0 { wroteAny = true g.Go(func() error { - return writeSamples(ctx, audioTrack, packet.Audio, scalar) + ticker := time.NewTicker(time.Duration(float64(audioDur) * (1 / scalar))) + defer ticker.Stop() + for _, audio := range packet.Audio { + err := audioTrack.WriteSample(media.Sample{Data: audio.Data, Duration: audioDur}) + if err != nil { + return fmt.Errorf("failed to write audio sample: %w", err) + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + continue + } + } + return nil }) } else { log.Warn(ctx, "no audio samples to write") @@ -253,12 +278,22 @@ func (mm *MediaManager) WebRTCPlayback2(ctx context.Context, user string, rendit // stays real even while catch-up (scalar > 1) speeds the pacing: the // receiver's clock is authoritative for playout, sending faster just refills // its buffer. +// +// Pacing is against a wall-clock schedule, NOT a per-sample sleep: each sleep +// overshoots by scheduler latency, and chaining sleeps accumulates that +// overshoot into real drift (hundreds of ms per segment at high sample rates +// — enough to starve the receiver's jitter buffer). Sleeping until the +// scheduled deadline instead absorbs overshoot in the next iteration, like a +// ticker does. func writeSamples(ctx context.Context, track *webrtc.TrackLocalStaticSample, samples []bus.PacketizedSample, scalar float64) error { + start := time.Now() + var scheduled time.Duration for _, s := range samples { if err := track.WriteSample(media.Sample{Data: s.Data, Duration: s.Duration}); err != nil { return fmt.Errorf("failed to write sample: %w", err) } - wait := time.Duration(float64(s.Duration) / scalar) + scheduled += time.Duration(float64(s.Duration) / scalar) + wait := scheduled - time.Since(start) if wait <= 0 { continue } -- 2.51.2