From ec2da91460b4a0e1fbd7eaad44f5b12cd24ffa2e Mon Sep 17 00:00:00 2001
From: Eli Mallon <eli@iame.li>
Date: Tue, 13 Oct 2026 09:25:37 -0700
Subject: [PATCH] Stems follow the video's play/pause; list only stems WebRTC
 sends

Stem audio elements now pause and resume with the video. The menu lists
at most two stem groups that have an Opus encode, matching the tracks the
node sends, so levels apply to the right stem. The e2e checks the stem
element actually plays and follows pause.
---
 .../src/components/mobile-player/video.tsx    | 50 ++++++++++++++++---
 js/core/src/lib/audio-stems.test.ts           | 41 +++++++++++++++
 js/core/src/lib/audio-stems.ts                | 14 ++++--
 js/e2e-web/flows/audio-stems.spec.ts          | 19 +++++++
 4 files changed, 111 insertions(+), 13 deletions(-)

diff --git a/js/components/src/components/mobile-player/video.tsx b/js/components/src/components/mobile-player/video.tsx
index a5c38c9fa..d209e2b19 100644
--- a/js/components/src/components/mobile-player/video.tsx
+++ b/js/components/src/components/mobile-player/video.tsx
@@ -642,6 +642,14 @@ export function WebRTCPlayerInner({
 
   const [mediaStream, stuck, audioTracks] = useWebRTC(src, selectedRendition);
   const levels = useAudioTrackLevels();
+  const playbackVideoRef = useRef<HTMLVideoElement | null>(null);
+  const setPlaybackVideoRef = useCallback(
+    (element: HTMLVideoElement | null) => {
+      playbackVideoRef.current = element;
+      assignVideoRef(videoRef, element);
+    },
+    [videoRef],
+  );
   const [fullMix, ...stems] = audioTracks;
 
   // The video element gets the video and the full mix; each stem the viewer
@@ -717,10 +725,9 @@ export function WebRTCPlayerInner({
   }, [mediaStream]);
 
   useEffect(() => {
-    if (!videoRef || !videoRef.current) {
-      return;
+    if (playbackVideoRef.current) {
+      playbackVideoRef.current.srcObject = videoStream;
     }
-    videoRef.current.srcObject = videoStream;
   }, [videoStream]);
 
   if (!mediaStream) {
@@ -754,12 +761,17 @@ export function WebRTCPlayerInner({
   }
   return (
     <>
-      <VideoElement url={url} ref={videoRef} />
+      <VideoElement url={url} ref={setPlaybackVideoRef} />
       {stems.map(
         (track, i) =>
           track &&
           levels[i + 1] > 0 && (
-            <StemAudio key={track.id} track={track} level={levels[i + 1]} />
+            <StemAudio
+              key={track.id}
+              track={track}
+              level={levels[i + 1]}
+              videoRef={playbackVideoRef}
+            />
           ),
       )}
     </>
@@ -770,9 +782,11 @@ export function WebRTCPlayerInner({
 function StemAudio({
   track,
   level,
+  videoRef,
 }: {
   track: MediaStreamTrack;
   level: number;
+  videoRef: React.RefObject<HTMLVideoElement | null>;
 }) {
   const ref = useRef<HTMLAudioElement | null>(null);
   const volume = useVolume();
@@ -790,9 +804,29 @@ function StemAudio({
     }
   }, [volume, level]);
 
-  return (
-    <audio ref={ref} autoPlay muted={muted} data-testid="player-stem-audio" />
-  );
+  // The video is the player's playback clock. In particular, pausing a live
+  // video must pause its separately rendered stem audio too.
+  useEffect(() => {
+    const video = videoRef.current;
+    const audio = ref.current;
+    if (!video || !audio) return;
+    const syncPlayback = () => {
+      if (video.paused) {
+        audio.pause();
+      } else {
+        audio.play().catch(console.error);
+      }
+    };
+    video.addEventListener("play", syncPlayback);
+    video.addEventListener("pause", syncPlayback);
+    syncPlayback();
+    return () => {
+      video.removeEventListener("play", syncPlayback);
+      video.removeEventListener("pause", syncPlayback);
+    };
+  }, [videoRef]);
+
+  return <audio ref={ref} muted={muted} data-testid="player-stem-audio" />;
 }
 
 export function WebcamIngestPlayer(props: VideoProps) {
diff --git a/js/core/src/lib/audio-stems.test.ts b/js/core/src/lib/audio-stems.test.ts
index a675448ff..d4642d725 100644
--- a/js/core/src/lib/audio-stems.test.ts
+++ b/js/core/src/lib/audio-stems.test.ts
@@ -58,6 +58,47 @@ describe("audioStems", () => {
       ),
     ).toEqual([{ alternateGroup: 2, label: "Track 2" }]);
   });
+
+  it("skips AAC-only groups before taking the first two playable stems", () => {
+    expect(
+      audioStems(
+        segment([
+          { codec: "opus", rate: 48000, channels: 2, alternateGroup: 1 },
+          {
+            codec: "aac",
+            rate: 48000,
+            channels: 2,
+            alternateGroup: 2,
+            label: "AAC only",
+          },
+          {
+            codec: "opus",
+            rate: 48000,
+            channels: 2,
+            alternateGroup: 5,
+            label: "Third",
+          },
+          {
+            codec: "opus",
+            rate: 48000,
+            channels: 2,
+            alternateGroup: 4,
+            label: "Second",
+          },
+          {
+            codec: "opus",
+            rate: 48000,
+            channels: 2,
+            alternateGroup: 3,
+            label: "First",
+          },
+        ]),
+      ),
+    ).toEqual([
+      { alternateGroup: 3, label: "First" },
+      { alternateGroup: 4, label: "Second" },
+    ]);
+  });
 });
 
 describe("audioTrackLevels", () => {
diff --git a/js/core/src/lib/audio-stems.ts b/js/core/src/lib/audio-stems.ts
index f5c5d7775..9cd7a7a5e 100644
--- a/js/core/src/lib/audio-stems.ts
+++ b/js/core/src/lib/audio-stems.ts
@@ -3,26 +3,30 @@ import { place } from "streamplace";
 export type AudioStem = { alternateGroup: number; label: string };
 
 /**
- * The stems in a segment's audio: tracks of the same alternateGroup carry the
- * same sound in different codecs, the lowest group (or an entry without one)
- * is the full mix and every higher group is a stem. In ascending group order,
- * which is also the order the node sends their WebRTC tracks in.
+ * The playable stems in a segment's audio: tracks of the same alternateGroup
+ * carry the same sound in different codecs, the lowest group (or an entry
+ * without one) is the full mix. WebRTC sends up to two higher groups with an
+ * Opus encode, in ascending order.
  */
 export function audioStems(
   segment: place.stream.segment.Main | null | undefined,
 ): AudioStem[] {
   const audio = segment?.audio ?? [];
   const main = Math.min(...audio.map((a) => a.alternateGroup ?? 0));
+  const opusGroups = new Set(
+    audio.filter((a) => a.codec === "opus").map((a) => a.alternateGroup ?? 0),
+  );
   // Each group's label, from whichever of its entries has one.
   const stems = new Map<number, string>();
   for (const a of audio) {
     const group = a.alternateGroup ?? 0;
-    if (group > main && !stems.get(group)) {
+    if (group > main && opusGroups.has(group) && !stems.get(group)) {
       stems.set(group, a.label ?? "");
     }
   }
   return [...stems]
     .sort(([a], [b]) => a - b)
+    .slice(0, 2)
     .map(([alternateGroup, label]) => ({
       alternateGroup,
       label: label || `Track ${alternateGroup}`,
diff --git a/js/e2e-web/flows/audio-stems.spec.ts b/js/e2e-web/flows/audio-stems.spec.ts
index 545db762f..fd4d7b101 100644
--- a/js/e2e-web/flows/audio-stems.spec.ts
+++ b/js/e2e-web/flows/audio-stems.spec.ts
@@ -54,6 +54,25 @@ test("audio stems: switch from the full mix to the stems and back", async ({
         (a.srcObject as MediaStream).getAudioTracks()[0].readyState,
     ),
   ).toBe("live");
+  await expect
+    .poll(() => stemAudio.evaluate((a: HTMLAudioElement) => a.paused))
+    .toBe(false);
+  const startTime = await stemAudio.evaluate(
+    (a: HTMLAudioElement) => a.currentTime,
+  );
+  await expect
+    .poll(() => stemAudio.evaluate((a: HTMLAudioElement) => a.currentTime))
+    .toBeGreaterThan(startTime + 0.1);
+
+  // Pausing the video must also pause the separately rendered stem.
+  await video.evaluate((v: HTMLVideoElement) => v.pause());
+  await expect
+    .poll(() => stemAudio.evaluate((a: HTMLAudioElement) => a.paused))
+    .toBe(true);
+  await video.evaluate((v: HTMLVideoElement) => v.play());
+  await expect
+    .poll(() => stemAudio.evaluate((a: HTMLAudioElement) => a.paused))
+    .toBe(false);
   await expect(music).toContainText("50%");
 
   await page.getByTestId("player-audio-full-mix").click();
-- 
2.51.2