From ddad5b91bbd96bbb40eb72742dd4e97c5f65d910 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Thu, 1 Oct 2026 09:01:28 -0700 Subject: [PATCH] Wire captions end to end through MUXL, distribution, and clients Origin nodes master canonical captions (speech recognition, CEA-608/708, pushCaptions) into signed MUXL WebVTT text tracks via muxl's per-GoP text hook. Every node extracts canonical cues from validated segments into the caption hub; nodes caption for accessibility when policy allows and share sidecars with relays over the segment websocket. Transcript records follow the live session, VOD captions merge MUXL text tracks with records, and the web and app players, settings, dashboard, VOD tools, docs, Playwright and Maestro flows cover the feature. muxl is pinned to the opus55/text-tracks commit until it is released. --- .maestro/08-captions.yaml | 67 +++ .maestro/README.md | 26 ++ .maestro/config.yaml | 1 + .maestro/push-caption.js | 45 ++ docs/browser-captions.md | 18 +- docs/captions-whisper.md | 124 ++++++ go.mod | 2 +- go.sum | 8 + hack/e2e-local.sh | 22 +- hack/lib/e2e-harness.sh | 12 +- .../settings/captions-category-settings.tsx | 5 +- js/app/src/screens/upload.tsx | 27 +- .../components/captions/caption-settings.tsx | 7 +- .../components/captions/captions-button.tsx | 133 +++--- .../src/components/captions/use-captions.tsx | 37 +- .../captions/video-captions-manager.tsx | 20 +- .../caption-policy-fields.tsx | 58 ++- .../content-metadata-form.tsx | 7 + .../src/components/mobile-player/video.tsx | 27 +- .../src/components/ui/dropdown.native.tsx | 85 ++-- .../tests/video-captions-manager.test.tsx | 125 ++++++ js/core/src/captions/api.ts | 5 +- js/core/src/captions/captions.test.ts | 29 +- js/core/src/captions/live-cues.ts | 55 ++- .../src/captions/review-regressions.test.ts | 128 ++++++ js/core/src/captions/text-tracks.ts | 36 +- js/core/src/livestream-store/state.ts | 3 +- js/core/src/livestream-store/store.ts | 1 + .../websocket-consumer.test.ts | 72 +++- .../livestream-store/websocket-consumer.ts | 21 +- js/dev-env/https-proxy.mjs | 11 + js/dev-env/package.json | 3 +- js/dev-env/run.mjs | 1 + .../src/content/docs/features-dev/captions.md | 153 +++++++ .../content/docs/features/live-dashboard.md | 5 + .../docs/guides/installing/captions.md | 143 +++++++ .../installing/downloading-streamplace.md | 3 + .../docs/guides/start-streaming/captions.md | 175 ++++++++ .../docs/guides/start-streaming/obs.md | 1 + .../place-stream-caption-pushcaptions.md | 4 +- .../content/docs/lex-reference/openapi.json | 2 +- js/e2e-web/README.md | 28 ++ js/e2e-web/flows/09-captions.spec.ts | 250 +++++++++++ js/e2e-web/flows/login.ts | 9 +- js/i18n/locales/en-US/common.ftl | 35 ++ js/i18n/locales/en-US/settings.ftl | 1 + js/i18n/public/locales/en-US/common.json | 35 ++ js/i18n/public/locales/en-US/settings.json | 1 + js/streamplace/src/captioner/index.ts | 30 +- js/web/package.json | 1 + js/web/public/locales/en-US/common.json | 35 ++ js/web/public/locales/en-US/settings.json | 1 + .../captions/caption-policy-fields.tsx | 91 ++++ .../components/captions/caption-settings.tsx | 3 +- .../captions/video-captions-manager.tsx | 114 +++++ .../components/dashboard/stream-monitor.tsx | 1 + .../src/components/player/caption-overlay.tsx | 31 +- .../src/components/player/captions-button.tsx | 93 ++++ js/web/src/components/player/hls-player.tsx | 1 + .../src/components/player/player-controls.tsx | 31 +- js/web/src/components/player/player.tsx | 20 +- .../components/player/use-player-captions.ts | 38 +- .../src/components/stream/player-offline.tsx | 3 + .../src/components/stream/video-section.tsx | 5 + js/web/src/hooks/actor-lookup.test.ts | 56 +-- js/web/src/hooks/actor-lookup.ts | 27 +- js/web/src/hooks/use-avatars.test.tsx | 47 ++ js/web/src/hooks/use-avatars.tsx | 14 +- js/web/src/lib/captioner-api.test.ts | 27 ++ js/web/src/lib/store/slices/blueskySlice.ts | 1 + .../store/slices/contentMetadataSlice.test.ts | 78 ++++ .../lib/store/slices/contentMetadataSlice.ts | 110 ++--- js/web/src/routeTree.gen.ts | 21 + js/web/src/routes/captioner.tsx | 99 ++--- js/web/src/routes/dashboard/stream/index.tsx | 34 +- js/web/src/routes/dashboard/videos.tsx | 63 ++- js/web/src/routes/embed/$user/index.tsx | 28 +- js/web/src/routes/embed/$user/video/$tid.tsx | 26 +- js/web/src/routes/embed/captions/$user.tsx | 10 +- js/web/src/routes/login.tsx | 2 + js/web/src/routes/settings.tsx | 7 + js/web/src/routes/settings/advanced.tsx | 8 +- js/web/src/routes/settings/captions.tsx | 16 + .../place/stream/caption/pushCaptions.json | 2 +- pkg/api/captions_records.go | 8 +- pkg/api/stream_key.go | 69 +-- pkg/aqhttp/transport.go | 1 + pkg/captions/hub.go | 81 +++- pkg/captions/hub_distribution_test.go | 51 +++ pkg/captions/hub_test.go | 14 + pkg/captions/muxl.go | 125 ++++++ pkg/captions/muxl_test.go | 107 +++++ pkg/captions/recognizer.go | 28 +- pkg/captions/recognizer_test.go | 52 +++ pkg/captions/records/distribution_test.go | 103 +++++ pkg/captions/records/importer.go | 2 +- pkg/captions/records/outbox.go | 183 ++++++++ pkg/captions/records/provider.go | 15 +- pkg/captions/records/provider_test.go | 14 +- pkg/captions/records/publisher.go | 13 +- pkg/captions/records/writer.go | 193 ++++++++- pkg/captions/records/writer_review_test.go | 121 ++++++ pkg/captions/records/writer_test.go | 7 +- pkg/captions/syndication.go | 25 ++ pkg/captions/transcript/authored.go | 104 +++++ pkg/captions/transcript/authored_test.go | 31 ++ pkg/captions/transcript/chunk.go | 11 +- pkg/captions/transcript/cues.go | 31 -- pkg/captions/transcript/cues_test.go | 85 +--- pkg/captions/transcript/transcript.go | 19 +- pkg/cmd/e2e.go | 122 +++++- pkg/cmd/e2e_https.go | 17 +- pkg/cmd/e2e_https_test.go | 69 +++ pkg/cmd/streamplace.go | 18 +- pkg/director/stream_session.go | 1 + pkg/media/auth_stream_key.go | 76 ++++ pkg/media/auth_stream_key_test.go | 74 ++++ pkg/media/captions_audio.go | 2 +- pkg/media/captions_buffer.go | 129 ++++++ pkg/media/captions_buffer_test.go | 43 ++ pkg/media/captions_clock_test.go | 128 ++++++ pkg/media/captions_distribution.go | 405 ++++++++++++++++++ .../captions_distribution_review_test.go | 105 +++++ pkg/media/captions_distribution_test.go | 237 ++++++++++ pkg/media/captions_engine.go | 24 ++ pkg/media/captions_engine_test.go | 41 ++ pkg/media/captions_language_test.go | 31 ++ pkg/media/captions_master.go | 365 ++++++++++++++++ pkg/media/captions_master_control_test.go | 59 +++ pkg/media/captions_master_feed.go | 232 ++++++++++ pkg/media/captions_master_socket.go | 94 ++++ pkg/media/captions_master_test.go | 309 +++++++++++++ pkg/media/captions_order_test.go | 36 ++ pkg/media/captions_transcode_test.go | 94 ++++ pkg/media/ingest_daemon.go | 5 + pkg/media/ingest_supervisor.go | 21 +- pkg/media/ingest_worker.go | 37 +- pkg/media/media.go | 15 +- pkg/media/media_signer.go | 29 +- pkg/media/muxl_segment.go | 31 +- pkg/media/segmenter.go | 1 + pkg/media/transcode_stream.go | 13 +- pkg/media/validate.go | 1 + pkg/multitest/captions_test.go | 403 +++++++++++++++++ pkg/muxl/muxl.go | 41 +- pkg/placestream/captionpushcaptions.go | 2 +- pkg/replication/websocketrep/captions_test.go | 163 +++++++ .../websocketrep/websocket_replicator.go | 17 +- pkg/spxrpc/caption_body_limit_test.go | 30 ++ .../place_stream_caption_authored_test.go | 116 +++++ pkg/spxrpc/place_stream_caption_push.go | 102 ++++- pkg/spxrpc/place_stream_caption_push_test.go | 191 +++++++++ pkg/spxrpc/place_stream_live.go | 27 ++ pkg/spxrpc/spxrpc.go | 1 + pkg/stt/proxy.go | 287 +++++++++++++ pkg/stt/proxy_test.go | 119 +++++ pkg/vod/captions.go | 283 ++++++++++++ pkg/vod/captions_review_test.go | 140 ++++++ pkg/vod/captions_test.go | 109 +++++ pkg/vod/captions_tracks.go | 87 ++++ pkg/vod/finalize_livestream.go | 1 + pkg/vod/flat_vod.go | 4 + pkg/vod/metafile.go | 138 +++++- pkg/vod/publish.go | 25 +- pkg/vod/publish_draft.go | 15 +- pkg/vod/publish_video.go | 24 +- pnpm-lock.yaml | 6 + 167 files changed, 9138 insertions(+), 790 deletions(-) create mode 100644 .maestro/08-captions.yaml create mode 100644 .maestro/push-caption.js create mode 100644 js/components/tests/video-captions-manager.test.tsx create mode 100644 js/core/src/captions/review-regressions.test.ts create mode 100644 js/dev-env/https-proxy.mjs create mode 100644 js/docs/src/content/docs/features-dev/captions.md create mode 100644 js/docs/src/content/docs/guides/installing/captions.md create mode 100644 js/docs/src/content/docs/guides/start-streaming/captions.md create mode 100644 js/e2e-web/flows/09-captions.spec.ts create mode 100644 js/web/src/components/captions/caption-policy-fields.tsx create mode 100644 js/web/src/components/captions/video-captions-manager.tsx create mode 100644 js/web/src/components/player/captions-button.tsx create mode 100644 js/web/src/hooks/use-avatars.test.tsx create mode 100644 js/web/src/lib/store/slices/contentMetadataSlice.test.ts create mode 100644 js/web/src/routes/settings/captions.tsx create mode 100644 pkg/captions/hub_distribution_test.go create mode 100644 pkg/captions/muxl.go create mode 100644 pkg/captions/muxl_test.go create mode 100644 pkg/captions/records/distribution_test.go create mode 100644 pkg/captions/records/outbox.go create mode 100644 pkg/captions/records/writer_review_test.go create mode 100644 pkg/captions/syndication.go create mode 100644 pkg/captions/transcript/authored.go create mode 100644 pkg/captions/transcript/authored_test.go create mode 100644 pkg/cmd/e2e_https_test.go create mode 100644 pkg/media/auth_stream_key.go create mode 100644 pkg/media/auth_stream_key_test.go create mode 100644 pkg/media/captions_buffer.go create mode 100644 pkg/media/captions_buffer_test.go create mode 100644 pkg/media/captions_clock_test.go create mode 100644 pkg/media/captions_distribution.go create mode 100644 pkg/media/captions_distribution_review_test.go create mode 100644 pkg/media/captions_distribution_test.go create mode 100644 pkg/media/captions_engine.go create mode 100644 pkg/media/captions_engine_test.go create mode 100644 pkg/media/captions_language_test.go create mode 100644 pkg/media/captions_master.go create mode 100644 pkg/media/captions_master_control_test.go create mode 100644 pkg/media/captions_master_feed.go create mode 100644 pkg/media/captions_master_socket.go create mode 100644 pkg/media/captions_master_test.go create mode 100644 pkg/media/captions_order_test.go create mode 100644 pkg/media/captions_transcode_test.go create mode 100644 pkg/multitest/captions_test.go create mode 100644 pkg/replication/websocketrep/captions_test.go create mode 100644 pkg/spxrpc/caption_body_limit_test.go create mode 100644 pkg/spxrpc/place_stream_caption_authored_test.go create mode 100644 pkg/spxrpc/place_stream_caption_push_test.go create mode 100644 pkg/stt/proxy.go create mode 100644 pkg/stt/proxy_test.go create mode 100644 pkg/vod/captions.go create mode 100644 pkg/vod/captions_review_test.go create mode 100644 pkg/vod/captions_test.go create mode 100644 pkg/vod/captions_tracks.go diff --git a/.maestro/08-captions.yaml b/.maestro/08-captions.yaml new file mode 100644 index 000000000..2baf9735f --- /dev/null +++ b/.maestro/08-captions.yaml @@ -0,0 +1,67 @@ +appId: ${APP_ID} +--- +- launchApp +- extendedWaitUntil: + visible: + id: "home-stream-card" + timeout: 30000 +- tapOn: + id: "home-stream-card" +- extendedWaitUntil: + visible: + text: ".*e2e test stream.*" + timeout: 30000 +- tapOn: + point: 50%,20% +- tapOn: + id: "player-cc-menu-button" +- tapOn: + id: "player-cc-track-off" +- tapOn: + point: 50%,20% +- tapOn: + id: "player-cc-button" +- tapOn: + point: 50%,20% +- tapOn: + id: "player-cc-menu-button" +- retry: + maxRetries: 3 + commands: + - runScript: + file: push-caption.js + env: + CAPTION_API_URL: ${CAPTION_API_URL} + STREAM_KEY: ${STREAM_KEY} + ACCOUNT_DID: ${ACCOUNT_DID} + - extendedWaitUntil: + visible: + id: "player-cc-track-${output.captionTrackID}" + timeout: 10000 +- tapOn: + id: "player-cc-track-${output.captionTrackID}" +- assertNotVisible: + id: "player-cc-menu" +- retry: + maxRetries: 5 + commands: + - runScript: + file: push-caption.js + env: + CAPTION_API_URL: ${CAPTION_API_URL} + STREAM_KEY: ${STREAM_KEY} + ACCOUNT_DID: ${ACCOUNT_DID} + - extendedWaitUntil: + visible: + text: ".*Maestro live caption proof.*" + timeout: 15000 +- assertVisible: + id: "caption-overlay-text" +- takeScreenshot: 08-caption-visible +- tapOn: + point: 50%,20% +- tapOn: + id: "player-cc-button" +- assertNotVisible: + id: "caption-overlay-text" +- takeScreenshot: 08-caption-off diff --git a/.maestro/README.md b/.maestro/README.md index e193f1695..7abbc546f 100644 --- a/.maestro/README.md +++ b/.maestro/README.md @@ -37,6 +37,20 @@ These focused tests use native presentation adapters; the Maestro flow exercises the actual native sheet and portal. The component tests also run in `pnpm run check`. +`08-captions` runs after login and chat flows. It toggles the shared +`player-cc-button`, selects Off through `player-cc-menu-button`, pushes known +live text with `push-caption.js`, verifies `caption-overlay-text`, then turns +captions off. The default low-latency player renders websocket cues through +the overlay; native HLS subtitle renditions are rendered by expo-video with +the OS caption style. The harness supplies `SERVER_URL`, `ACCOUNT_DID` and +`STREAM_KEY`; the stream key authenticates the caption procedure, not a PDS JWT. +The host-side `runScript` HTTP calls use `CAPTION_API_URL`, the harness's plain +loopback API, while the installed app continues to use HTTPS `SERVER_URL`. +The proof cue lasts four seconds and is pushed again after track selection, +so the overlay assertion exercises its actual media-timed interval. Readiness +retries wait in the open track menu for the mastered human track to appear +before selecting it; unexpected API errors still fail immediately. + ## HTTPS, and logging in The app reaches the harness over HTTPS only: release builds refuse cleartext, @@ -74,6 +88,18 @@ reach the harness. ### Port 443 +For a local run without host sudo or a privileged-port redirect, use an +unprivileged HTTPS listener and explicitly select your rooted emulator: + +```bash +E2E_HTTPS_PORT=14443 ANDROID_SERIAL=emulator-5554 hack/e2e-local.sh android +``` + +The runner redirects only that emulator's traffic from `10.0.2.2:443` to the +chosen host port, retaining portless HTTPS URLs and the real harness CA. +It removes the guest rule when the run exits. With the default port 443, +including CI's `android-e2e` job, no guest redirection is installed. + The harness binds 127.0.0.1:443, which an unprivileged host process can't do by default; if it fails, it says how to allow that until the next reboot. To set a development machine up once instead, redirect loopback 443 to a high diff --git a/.maestro/config.yaml b/.maestro/config.yaml index defbe44bc..40cb1ecb9 100644 --- a/.maestro/config.yaml +++ b/.maestro/config.yaml @@ -13,3 +13,4 @@ executionOrder: - 05-oauth-login - 06-chat-reply - 07-chat-profile + - 08-captions diff --git a/.maestro/push-caption.js b/.maestro/push-caption.js new file mode 100644 index 000000000..26aa94660 --- /dev/null +++ b/.maestro/push-caption.js @@ -0,0 +1,45 @@ +const now = Date.now(); +const response = http.post( + `${CAPTION_API_URL}/xrpc/place.stream.caption.pushCaptions`, + { + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${STREAM_KEY}`, + }, + body: JSON.stringify({ + streamer: ACCOUNT_DID, + language: "en", + source: "human", + cues: [ + { + id: `maestro-${now}`, + startTime: new Date(now).toISOString(), + endTime: new Date(now + 4000).toISOString(), + text: "Maestro live caption proof", + final: true, + }, + ], + }), + }, +); +output.captionReady = response.status === 200; +output.captionTrackID = ""; +if (!output.captionReady) { + if ( + response.status !== 400 || + JSON.parse(response.body).message !== "StreamNotLive" + ) + throw new Error( + `pushCaptions returned ${response.status}: ${response.body}`, + ); +} else { + const listed = http.get( + `${CAPTION_API_URL}/xrpc/place.stream.caption.listTracks?streamer=${encodeURIComponent(ACCOUNT_DID)}`, + ); + if (listed.status !== 200) + throw new Error(`listTracks returned ${listed.status}: ${listed.body}`); + const tracks = JSON.parse(listed.body).tracks; + const human = tracks.find((track) => track.source === "human"); + output.captionReady = !!human; + if (human) output.captionTrackID = human.id; +} diff --git a/docs/browser-captions.md b/docs/browser-captions.md index 515b55666..20cdd6015 100644 --- a/docs/browser-captions.md +++ b/docs/browser-captions.md @@ -8,11 +8,14 @@ text, not microphone audio, to `place.stream.caption.pushCaptions` about once a second. Final captions commit after 600 ms of silence or a 12-second window. A positive calibration offset delays caption timestamps to match encoder latency. -For canonical device captions, explicitly set the stream's caption policy to -**ingest** in stream settings. The captioner and browser go-live checkbox never -change this policy. Browser go-live's “Caption my stream on this device” option -uses the outgoing audio track and tiny model. Enabling it can reload the page to -turn on browser isolation; restart Go Live after that reload. +Pushed device captions are canonical under **auto** or **ingest**. Choose +**ingest** in stream settings to use only your supplied captions and avoid node +speech recognition; under auto, supplied captions take over when they arrive. +With canonical captions off, pushes are sidecar or local according to +`allowNodeCaptions`. The captioner and browser go-live checkbox never change this +policy. Browser go-live's “Caption my stream on this device” option uses the +outgoing audio track and tiny model. Enabling it can reload the page to turn on +browser isolation; restart Go Live after that reload. ## OBS display source (recommended) @@ -26,6 +29,11 @@ This display source needs **neither login nor microphone permissions**. Its canvas is transparent; `background` controls only the caption box. It receives interim and final live cues from the livestream websocket, regardless of whether they came from node recognition, ingest captions, or the captioner page. +The display shows the newest cue on arrival for its duration (at least five +seconds), or until a replacement arrives. Viewer-player overlays instead use +the latest livestream segment's start time plus elapsed time since that segment +arrived, and show cues only inside their start/end interval. Native HLS +TextTracks continue to use the browser's media timeline. Options: `fontSize` (pixels), `color` and `background` (CSS colors, URL-encoded where necessary), `position` (`top`, `center`, `bottom`), `maxLines` (1–10 wrapped diff --git a/docs/captions-whisper.md b/docs/captions-whisper.md index bc8e12f66..3d9f57a5d 100644 --- a/docs/captions-whisper.md +++ b/docs/captions-whisper.md @@ -89,3 +89,127 @@ country”, checks word offsets/probabilities and language, and verifies silence and cancellation. Scheduler tests use deterministic fake costs and check budget, pinned models, hysteresis, batch waiting, and shutdown. The benchmark smoke prints measured factors for all three bundled models. +Proxy regressions share admission with direct parent leases, release leases on +worker disconnect/engine shutdown, propagate inference cancellation, and reject +oversized frames. Real-model proxy smoke also recognizes that JFK excerpt +through the parent engine; workers do not need a second model load. + +## Origin mastering + +The node initializes one `MediaManager.STT` engine shared by in-process ingest, +sidecars, VOD jobs, and isolated workers. Workers lease and transcribe over a +private mode-0600 Unix socket; they never load a separate engine. Disconnecting +a worker releases its lease and cancels inference. Framed requests are bounded +to 16 MiB and recognition windows to 30 seconds. Origin recognition does not +depend on `SP_CAPTIONS`, which controls optional node sidecars. +Canonical sources publish into a private per-session hub, never directly into +the public caption hub. The same per-GoP metadata snapshot supplies both the +signed manifest and caption policy. Sources wait for the first snapshot before +decoding or leasing recognition, so an ingest-only or off policy never briefly +starts automatic recognition during startup. + +The streaming MUXL signer requests text for each AV GoP, through the actual +next-keyframe boundary. It waits until finalized recognition covers that GoP's +end, or until GoP closure plus `SP_CAPTIONS_MASTER_DELAY`, whichever comes first. +Lossless byte queues block producers at 32 MiB instead of dropping media or +growing indefinitely; aborts discard retained bytes and unblock writers. +Late final words move into the next unsigned GoP; cross-boundary cues retain +their session-qualified ID and are clipped into each overlapping GoP. Text +tracks are declared lazily per source/language, starting at reserved ID 100, +above node-added AV renditions. Ingest takeover declares a new immutable track +and the old automatic track continues as gaps; pushed language tags are not +replaced by recognition hints. +Continuous audio completion feeds only AV tracks to its native decoder, whose +initial track configuration cannot change. Lazily declared caption tracks stay +in the original signed source bytes and in the completed archival segment. + +All origin policies stamp GoPs with media start time, anchored to the first +fragment's arrival rather than time spent signing. An arrival/prediction drift +over one second reanchors the next GoP. Pushed wall-clock cues use the inverse +of that same mapping. Optional generic MUXL `SegmentTimeFn` leaves signing-time +stamps byte-identical when nil. + +`place.stream.caption.pushCaptions` accepts the streamer's OAuth session or +`Authorization: Bearer `. Stream-key pushes require an active +registered `place.stream.key`, apply the same stream allowance and ban checks +as WHIP, and cannot name a different streamer. Non-live streams return +`StreamNotLive`. Canonical pushes travel into the origin master (including +private worker control sockets), using its ingest-arrival/media clock to +translate wall-clock cue times. With canonical captions off, pushes instead +follow the signed policy's sidecar/local routing. +Push requests are limited to 2 MiB before binding. Each batch is validated +atomically against the lexicon limits (100 cues, 64-character IDs, 2000-character +text, BCP 47 language and ordered timestamps) before any cue is published. + +## Live distribution and archival captions + +Canonical captions reach every node, including the ingest node, only by reading +the text tracks in validated MUXL segments. `TextTrack.Language` is the BCP 47 +tag (`und` when unknown); `TextTrack.Label` is `auto`, `ingest`, or `human` +(empty or unknown labels are treated as ingest). Empty text segments publish no +cues. Tracks can appear during the +session, and pieces of one cue crossing GoPs extend one hub/HLS cue. Cue times +use the signed segment start plus the difference between the cue's media time +and the reference AV decode time, never the receiving node's clock. + +When canonical captions are off at the origin, or absent on a relay, the signed +`allowNodeCaptions` preference and `SP_CAPTIONS` gate sidecar recognition. Sidecar +recognition uses the same engine budget and continuous audio decoder as origin +mastering. A missing engine or refused lease leaves media playback unaffected. +An incoming canonical track or upstream sidecar stops competing recognition. +Upstream replay arriving before its first media segment stays private until the +signed policy is validated; canonical tracks take precedence over that replay. +Canonical-track presence is classified before accepting following sidecar frames; +discovered canonical tracks also remove competing sidecars from the live hub. + +Sidecars travel on the existing segment-replication websocket, opt-in with +`captions=1` on `place.stream.live.subscribeSegments`: binary frames remain media, +and JSON text frames carry `place.stream.caption.sidecar#event` with segment-clock +cue times. Receivers derive the sidecar track ID from origin/source/language and +bind its author to the connected upstream's server DID, not its claimed author. +This keeps media and captions on the +same upstream connection; old peers do not request the capability and continue +receiving only binary frames. Websocket replication is the implemented origin +pull path; the Iroh replicator currently has no segment transport implementation. +Both locally authored and forwarded sidecars honor `allowNodeCaptions=false`. +The hub rejects cross-origin ID collisions, does not store interim cues, and +bounds track count and final history. Viewer subscriptions remain nonblocking. + +Transcript writers begin with the first published segment. Only the origin +writes canonical transcripts through the streamer's stored OAuth session; a +node writes only its own sidecars to its server repo, never an upstream node's +records. Session end (including a return to preview) clears live captions and +starts an asynchronous final record flush; PDS retries cannot block media. +The newest canonical cue remains mutable across a periodic flush until another +cue follows, its end is older than one flush interval, or the session ends, so a +GoP boundary cannot truncate the recorded cue. Sidecar finals need no such hold. +Writers reconcile retained hub finals before every flush and before session +teardown, so a full viewer event buffer does not silently lose transcript words. +Their deduplication state is limited to the retained replay window. +Encoded records persist in `DataDir/captions/transcripts` until delivery succeeds. +PDS rate-limit reset times apply to final flushes and recovered records too; +records that outlive the shutdown deadline resume with their original record +keys on the next node startup. The outbox contains no OAuth credentials. +Node shutdown stops caption admission, drains accepted canonical segments, and +waits for workers and all current or already-finishing transcript sessions +before closing the speech engine. + +Live-derived VOD metafiles retain text-track metadata and byte ranges, including +each segment's language/source configuration and containing-GoP reference clock. +Reused numeric IDs after a reconnect cannot relabel earlier captions. New indexes +store an explicit containing-GoP reference, rather than infer it from text-byte +placement. Archives must obey MUXL's ascending numeric track-ID order within each +GoP; the indexer rejects noncanonical ordering without rewriting signed bytes. +`publishDraft` publishes their `place.stream.media.track` records alongside AV +tracks, rather than exposing tracks while the video is still a draft. VOD +caption lookup places text segments on the video's AV timeline, including late +track declarations and reconnects, and merges them with indexed transcripts. +A matching record copy of the streamer's mastered text is omitted; imports, +sidecars, and different authored text remain available. + +Imported SRT/WebVTT and human transcript tracks reconstruct cues only at explicit +silence gaps, preserving long cues, adjacent cue boundaries, and authored line +breaks. Imports encode a 1 ms boundary gap between otherwise adjacent cues by +shortening the preceding final word (or moving the next start 1 ms when that word +is already only 1 ms). Automatic and ingest transcripts retain broadcast display +layout. VTT, SRT, JSON, VOD HLS and overlays all use this same record provider. diff --git a/go.mod b/go.mod index 6f7045e00..db55a488e 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( github.com/streamplace/atmoq/go v0.0.4-0.20260701223355-13757de4ae08 github.com/streamplace/atproto-oauth-golang v0.0.0-20260413212710-98956064d06c github.com/streamplace/glex v0.0.0-20260820164827-814f46540f22 - github.com/streamplace/muxl/go v0.3.5 + github.com/streamplace/muxl/go v0.3.6-0.20261001091251-c4e9d84c5131 github.com/streamplace/oatproxy v0.0.0-20260710202406-60d97b9d780b github.com/stretchr/testify v1.11.1 github.com/tdewolff/canvas v0.0.0-20250728095813-50d4cb1eee71 diff --git a/go.sum b/go.sum index fa86cf1c4..0655b5425 100644 --- a/go.sum +++ b/go.sum @@ -1399,6 +1399,14 @@ github.com/streamplace/indigo v0.0.0-20260218231908-939cdaf0c507 h1:e8M3qPLr37Nx github.com/streamplace/indigo v0.0.0-20260218231908-939cdaf0c507/go.mod h1:Pm2I1+iDXn/hLbF7XCg/DsZi6uDCiOo7hZGWprSM7k0= github.com/streamplace/muxl/go v0.3.5 h1:sp33xrcblpl1nfKVXy03sDfQd7rM2aizbIaYajUH2zg= github.com/streamplace/muxl/go v0.3.5/go.mod h1:aCyYTW3o6c1Kush9UJ/Yv6EYMUbj8l8GTD7cHKcSxw8= +github.com/streamplace/muxl/go v0.3.6-0.20261001020556-d90d7005d415 h1:qwin+Lut7nWFmPSMOYB6Oq/AMMdKyY0OssiA6lygOmU= +github.com/streamplace/muxl/go v0.3.6-0.20261001020556-d90d7005d415/go.mod h1:aCyYTW3o6c1Kush9UJ/Yv6EYMUbj8l8GTD7cHKcSxw8= +github.com/streamplace/muxl/go v0.3.6-0.20261001070318-9c5ca1cc55df h1:Z+tWCUJHP0aQYBXFx2WRWN2YlpGc9BLkDcefTN+njO0= +github.com/streamplace/muxl/go v0.3.6-0.20261001070318-9c5ca1cc55df/go.mod h1:aCyYTW3o6c1Kush9UJ/Yv6EYMUbj8l8GTD7cHKcSxw8= +github.com/streamplace/muxl/go v0.3.6-0.20261001082214-e33616814339 h1:x/h+gs9H7W96mlY33BMCvNJ/aM29OXvYUa64ebRxU1c= +github.com/streamplace/muxl/go v0.3.6-0.20261001082214-e33616814339/go.mod h1:aCyYTW3o6c1Kush9UJ/Yv6EYMUbj8l8GTD7cHKcSxw8= +github.com/streamplace/muxl/go v0.3.6-0.20261001091251-c4e9d84c5131 h1:kZu4SqfJVTV7dcXCGIK6ke3K5uMZm6GNsNoMQMRjaqs= +github.com/streamplace/muxl/go v0.3.6-0.20261001091251-c4e9d84c5131/go.mod h1:aCyYTW3o6c1Kush9UJ/Yv6EYMUbj8l8GTD7cHKcSxw8= github.com/streamplace/oatproxy v0.0.0-20260710202406-60d97b9d780b h1:eWbwCtBbMyrDTHLYIold07OR2hmvzXsbAUxi57ElMLk= github.com/streamplace/oatproxy v0.0.0-20260710202406-60d97b9d780b/go.mod h1:wpY+T/wE00jrUhgh2dKXbbE91D36u86KGlENK/hWFkE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/hack/e2e-local.sh b/hack/e2e-local.sh index d3119707e..4fac247b4 100755 --- a/hack/e2e-local.sh +++ b/hack/e2e-local.sh @@ -113,6 +113,24 @@ printf '%s\n' "$hosts" | adb_ shell 'while grep -q " /system/etc/hosts " /proc/m chcon u:object_r:system_file:s0 /data/local/tmp/sp-e2e-hosts mount -o bind /data/local/tmp/sp-e2e-hosts /system/etc/hosts' +# Keep OAuth URLs portless without changing the host's privileged-port policy. +if [ -n "${E2E_HTTPS_PORT:-}" ] && [ "$E2E_HTTPS_PORT" != 443 ]; then + case "$DEVICE" in + emulator-*) ;; + *) echo "high-port HTTPS redirection requires an emulator serial"; exit 1 ;; + esac + stop_with_https_redirect() { + adb_ shell iptables -t nat -D OUTPUT -p tcp -d 10.0.2.2 --dport 443 \ + -m comment --comment streamplace-e2e -j DNAT \ + --to-destination "10.0.2.2:$E2E_HTTPS_PORT" >/dev/null 2>&1 || true + e2e_harness_stop + } + trap stop_with_https_redirect EXIT INT TERM + adb_ shell iptables -t nat -A OUTPUT -p tcp -d 10.0.2.2 --dport 443 \ + -m comment --comment streamplace-e2e -j DNAT \ + --to-destination "10.0.2.2:$E2E_HTTPS_PORT" +fi + # The user trust store keys certificates by OpenSSL's old subject hash. # Each run mints a new CA, so drop the one an earlier run left behind. ca_file="$(openssl x509 -subject_hash_old -noout -in "$E2E_TLS_CA").0" @@ -138,7 +156,9 @@ adb_ shell pm grant "$APP_ID" android.permission.POST_NOTIFICATIONS 2>/dev/null adb_ shell am force-stop com.android.chrome || true MAESTRO_ARGS=(-e APP_ID="$APP_ID" -e SERVER_URL="$SERVER_HTTPS_URL" - -e ACCOUNT_HANDLE="$ACCOUNT_HANDLE" -e ACCOUNT_PASSWORD="$ACCOUNT_PASSWORD") + -e ACCOUNT_HANDLE="$ACCOUNT_HANDLE" -e ACCOUNT_PASSWORD="$ACCOUNT_PASSWORD" + -e STREAM_KEY="$STREAM_KEY" -e ACCOUNT_DID="$ACCOUNT_DID" + -e CAPTION_API_URL="$SERVER_URL") # --- run the flows --------------------------------------------------------- # takeScreenshot paths are relative to maestro's cwd, so run from the diff --git a/hack/lib/e2e-harness.sh b/hack/lib/e2e-harness.sh index 42c57a618..fde31d82a 100644 --- a/hack/lib/e2e-harness.sh +++ b/hack/lib/e2e-harness.sh @@ -7,8 +7,8 @@ # # The caller's cwd must be the repo root. On return the harness is up and its # variables are exported: SERVER_URL, ACCOUNT_HANDLE, ACCOUNT_DID, -# ACCOUNT_PASSWORD, VIDEO_URI (the account's test VOD), and in HTTPS mode also -# SERVER_HTTPS_URL, PDS_HTTPS_URL, +# ACCOUNT_PASSWORD, STREAM_KEY, VIDEO_URI (the account's processed, playable +# fixture VOD), and in HTTPS mode SERVER_HTTPS_URL, PDS_HTTPS_URL, # E2E_PROXY_URL, E2E_TLS_SPKI and E2E_TLS_CA. An EXIT trap stops it; a caller # with its own EXIT work calls e2e_harness_stop from its trap instead. # @@ -18,8 +18,10 @@ # $E2E_HTTPS_STATION_HOSTNAME, public DNS names for 127.0.0.1 on different # registrable domains (the PDS name needs a wildcard too, for handles; see # pkg/cmd/e2e_https.go). Set E2E_HTTPS_PDS_HOSTNAME empty to skip HTTPS; the -# OAuth flows then skip themselves. A machine that redirects loopback 443 -# elsewhere (an iptables REDIRECT rule) sets E2E_HTTPS_PORT to the target port. +# OAuth flows then skip themselves. Set E2E_HTTPS_PORT to use an unprivileged +# listener: server-side Go/Node clients and proxied browsers reach it automatically. +# Direct device clients still need port-443 redirection (the Android runner +# installs it in the emulator only). # # E2E_HARNESS_LOG, if set, is where the harness log goes; it is kept after the # run (CI uploads it). Otherwise the log is a temp file, shown on a failed @@ -78,7 +80,7 @@ e2e_harness_start() { fi # shellcheck disable=SC1090 . "$E2E_ENVFILE" - export SERVER_URL ACCOUNT_HANDLE ACCOUNT_DID ACCOUNT_PASSWORD VIDEO_URI + export SERVER_URL ACCOUNT_HANDLE ACCOUNT_DID ACCOUNT_PASSWORD VIDEO_URI STREAM_KEY # only set in HTTPS mode export SERVER_HTTPS_URL PDS_HTTPS_URL E2E_PROXY_URL E2E_TLS_SPKI E2E_TLS_CA echo "harness up: SERVER_URL=$SERVER_URL ACCOUNT_HANDLE=$ACCOUNT_HANDLE${SERVER_HTTPS_URL:+ SERVER_HTTPS_URL=$SERVER_HTTPS_URL}" diff --git a/js/app/components/settings/captions-category-settings.tsx b/js/app/components/settings/captions-category-settings.tsx index 1dc383862..26c939738 100644 --- a/js/app/components/settings/captions-category-settings.tsx +++ b/js/app/components/settings/captions-category-settings.tsx @@ -1,11 +1,12 @@ -import { CaptionSettings, View, zero } from "@streamplace/components"; +import { CaptionSettings, useTheme, View, zero } from "@streamplace/components"; import { ScrollView } from "react-native"; export function CaptionsCategorySettings() { + const { theme } = useTheme(); return ( - + diff --git a/js/app/src/screens/upload.tsx b/js/app/src/screens/upload.tsx index aebab3d40..72275a7e4 100644 --- a/js/app/src/screens/upload.tsx +++ b/js/app/src/screens/upload.tsx @@ -32,6 +32,8 @@ import AQLink from "components/aqlink"; import { EmptyState, EmptyStateTile } from "components/empty-state"; import Loading from "components/loading/loading"; import BetaAccessGate from "components/upload/beta-access-gate"; +import * as DocumentPicker from "expo-document-picker"; +import { File as NativeFile } from "expo-file-system"; import { Image } from "expo-image"; import { AlertCircle, @@ -1741,7 +1743,30 @@ export function UploadVideoScreen({ route }: { route: any }) { {mode === "video" && videoUri_ && ( - + { + const result = + await DocumentPicker.getDocumentAsync({ + type: [ + "text/vtt", + "application/x-subrip", + "text/plain", + ], + copyToCacheDirectory: true, + }); + if (result.canceled) return null; + const asset = result.assets[0]; + return { + name: asset.name, + text: () => new NativeFile(asset.uri).text(), + }; + } + } + /> )} diff --git a/js/components/src/components/captions/caption-settings.tsx b/js/components/src/components/captions/caption-settings.tsx index e13e22ac7..f3046dfa6 100644 --- a/js/components/src/components/captions/caption-settings.tsx +++ b/js/components/src/components/captions/caption-settings.tsx @@ -75,7 +75,7 @@ function ChoiceGroup({ testID={`${testID}-${choice.value}`} accessibilityRole="radio" accessibilityLabel={choice.label} - accessibilityState={{ checked: selected }} + aria-checked={selected} onPress={() => onChange(choice.value)} style={{ flexDirection: "row", @@ -165,6 +165,11 @@ export function CaptionSettings() { onValueChange={setEnabled} /> + {Platform.OS !== "web" && ( + + {t("captions-native-style-description")} + + )} {!onOpenSettings && ( void { - const { track, enabled } = useCaptionSelection(); - const setTrack = useSetCaptionTrack(); - return useCallback( - () => setTrack(enabled ? null : track), - [enabled, track, setTrack], - ); + const enabled = useCaptionsEnabled(); + const setEnabled = useSetCaptionsEnabled(); + return useCallback(() => setEnabled(!enabled), [enabled, setEnabled]); } /** Menu label for a track: its language, marked when auto-generated. */ @@ -121,6 +119,11 @@ export function useLoadCaptionTracks() { const src = usePlayerStore((x) => x.src); const setServerTracks = usePlayerStore((x) => x.setCaptionServerTracks); const url = useStreamplaceStore((x) => x.url); + const session = useLivestreamStoreOptional((x) => + x.livestream + ? `${x.livestream.uri}:${x.livestream.record.endedAt ?? ""}` + : null, + ); useEffect(() => { setServerTracks(NO_TRACKS); if (!src) return; @@ -137,25 +140,29 @@ export function useLoadCaptionTracks() { } }); return () => controller.abort(); - }, [url, mode, src, setServerTracks]); + }, [url, mode, src, session, setServerTracks]); } -// Live cues of a track. While lines are on screen, re-evaluate every -// second so they leave once stale even when no new cue arrives. +// Live cues follow the segment presentation clock, including queued future cues. function useLiveCaptionLines(trackId: string | null): string[] { const cues = useLivestreamStoreOptional((x) => x.liveCaptions); + const clock = useLivestreamStoreOptional((x) => x.captionClock); const [tick, setTick] = useState(0); const active = useMemo( - () => (trackId ? activeLiveCaptions(cues, trackId, Date.now()) : []), + () => { + const presented = presentedCaptionTime(clock, Date.now()); + return trackId && presented !== null + ? activeLiveCaptions(cues, trackId, presented) + : []; + }, // tick only forces a re-evaluation against the current time. - [cues, trackId, tick], + [cues, clock, trackId, tick], ); - const showing = active.length > 0; useEffect(() => { - if (!showing) return; - const timer = setInterval(() => setTick((n) => n + 1), 1000); + if (!trackId || !clock) return; + const timer = setInterval(() => setTick((n) => n + 1), 250); return () => clearInterval(timer); - }, [showing]); + }, [trackId, clock]); return useMemo(() => active.map((c) => c.text), [active]); } diff --git a/js/components/src/components/captions/video-captions-manager.tsx b/js/components/src/components/captions/video-captions-manager.tsx index 39e9bdb92..b9b1840e8 100644 --- a/js/components/src/components/captions/video-captions-manager.tsx +++ b/js/components/src/components/captions/video-captions-manager.tsx @@ -33,10 +33,18 @@ function pickCaptionFile(): Promise { /** * Caption tracks of one of the viewer's videos: lists them with VTT/SRT * downloads, and imports a .vtt or .srt file per language - * (place.stream.caption.importCaptions). Uploading needs a file picker, - * so it is offered on the web only. + * (place.stream.caption.importCaptions). Native hosts supply their platform file picker. */ -export function VideoCaptionsManager({ video }: { video: string }) { +export function VideoCaptionsManager({ + video, + pickFile, +}: { + video: string; + pickFile?: () => Promise<{ + name: string; + text: () => Promise; + } | null>; +}) { const { t, i18n } = useTranslation(); const { theme } = useTheme(); const url = useStreamplaceStore((x) => x.url); @@ -65,11 +73,11 @@ export function VideoCaptionsManager({ video }: { video: string }) { return; } if (!agent || !isValidAtUri(video)) return; - const file = await pickCaptionFile(); - if (!file) return; setUploading(true); setMessage(null); try { + const file = await (pickFile ?? pickCaptionFile)(); + if (!file) return; await agent.client.call(place.stream.caption.importCaptions, { video, language: language.trim(), @@ -139,7 +147,7 @@ export function VideoCaptionsManager({ video }: { video: string }) { ); }) )} - {Platform.OS === "web" && ( + {(Platform.OS === "web" || pickFile) && ( {t("vod-captions-upload-description")} diff --git a/js/components/src/components/content-metadata/caption-policy-fields.tsx b/js/components/src/components/content-metadata/caption-policy-fields.tsx index b9660cbdd..ab55762c1 100644 --- a/js/components/src/components/content-metadata/caption-policy-fields.tsx +++ b/js/components/src/components/content-metadata/caption-policy-fields.tsx @@ -1,8 +1,9 @@ import { CaptionPolicySettings } from "@streamplace/core"; -import { ReactNode } from "react"; +import { ReactNode, useState } from "react"; import { useTranslation } from "react-i18next"; import { View } from "react-native"; import { useTheme } from "../../lib/theme/theme"; +import { Button } from "../ui/button"; import { Checkbox } from "../ui/checkbox"; import { Switch } from "../ui/switch"; import { Text } from "../ui/text"; @@ -69,38 +70,49 @@ export function CaptionPolicyFields({ }) { const { t } = useTranslation(); const { theme } = useTheme(); + const [advanced, setAdvanced] = useState(false); return ( {t("dashboard-captions")} onChange({ ...value, mode: on ? "auto" : "off" }) } /> - - - onChange({ ...value, mode: checked ? "ingest" : "auto" }) - } - label={t("dashboard-captions-ingest")} - description={t("dashboard-captions-ingest-description")} - /> - - - onChange({ ...value, allowNodeCaptions }) - } - /> + + {advanced && ( + + + + onChange({ ...value, mode: checked ? "ingest" : "auto" }) + } + label={t("dashboard-captions-ingest")} + description={t("dashboard-captions-ingest-description")} + /> + + + onChange({ ...value, allowNodeCaptions }) + } + /> + + )} {renderLanguagePicker && ( ( )} + + {/* Save Button - Always visible */} + + + + + + + captions.select( + captions.tracks.find((track) => track.id === id) ?? null, + ) + } + > + + {t("player-captions-off")} + + {captions.tracks.map((track) => ( + + {track.source === "auto" + ? t("player-captions-track-auto", { + language: captionLanguageName( + track.language, + i18n.language, + ), + }) + : captionLanguageName(track.language, i18n.language)} + + ))} + + + {t("player-captions-style")} + + + + + ); +} diff --git a/js/web/src/components/player/hls-player.tsx b/js/web/src/components/player/hls-player.tsx index 30c017e45..2e09f0b0f 100644 --- a/js/web/src/components/player/hls-player.tsx +++ b/js/web/src/components/player/hls-player.tsx @@ -112,6 +112,7 @@ export function HLSPlayer({ ? LIVE_LOWLATENCY_HLS_SETTINGS : LIVE_HLS_SETTINGS; const hls = new Hls(settings); + hls.subtitleDisplay = false; hlsRef.current = hls; hls.on(Hls.Events.MANIFEST_PARSED, (_e, data) => { diff --git a/js/web/src/components/player/player-controls.tsx b/js/web/src/components/player/player-controls.tsx index cafea65e5..5aee9270a 100644 --- a/js/web/src/components/player/player-controls.tsx +++ b/js/web/src/components/player/player-controls.tsx @@ -30,10 +30,13 @@ import { DropdownMenuTrigger, } from "../ui/dropdown-menu"; import { Slider } from "../ui/slider"; +import { CaptionsButton } from "./captions-button"; import type { QualityOption } from "./player"; +import type { PlayerCaptions } from "./use-player-captions"; export type PlayerControlsProps = { videoRef: RefObject; + captions?: PlayerCaptions; /** Element to send into browser fullscreen. Defaults to the parent of the video. */ containerRef: RefObject; /** Live streams hide the scrubber and show a "LIVE" badge in its place. */ @@ -70,6 +73,7 @@ export function shouldShowUnmutePrompt(playing: boolean, muted: boolean) { export function PlayerControls({ videoRef, + captions, containerRef, isLive, showControls, @@ -94,6 +98,7 @@ export function PlayerControls({ const [isPiP, setIsPiP] = useState(false); const [settingsOpen, setSettingsOpen] = useState(false); + const [captionsOpen, setCaptionsOpen] = useState(false); const { theatre, setTheatre } = useFullscreen(); const { t } = useTranslation(); @@ -248,14 +253,30 @@ export function PlayerControls({ } else if (e.key === "t" || e.key === "T") { e.preventDefault(); setTheatre(!theatre); + } else if (e.key.toLowerCase() === "c") { + e.preventDefault(); + captions?.toggle(); } }; window.addEventListener("keydown", onKey); return () => window.removeEventListener("keydown", onKey); - }, [togglePlay, toggleMute, toggleFullscreen, theatre, setTheatre]); + }, [ + togglePlay, + toggleMute, + toggleFullscreen, + theatre, + setTheatre, + captions?.toggle, + ]); const showUnmutePrompt = shouldShowUnmutePrompt(playing, muted); - const visible = forceVisible || showControls || bigPlay || showUnmutePrompt; + const visible = + forceVisible || + showControls || + bigPlay || + showUnmutePrompt || + captionsOpen || + settingsOpen; return (
+ {captions && ( + + )} {pipSupported && ( + ) : ( -

Signed in as {did}

+

{t("captioner-signed-in", { did })}

)} {!mediaAvailable && ( -

- Microphone access is unavailable. Use HTTPS (or localhost), allow - microphone permissions, or launch OBS with{" "} - --enable-media-stream. Open Browser Source → Interact to - sign in and start. -

+

{t("captioner-microphone-unavailable")}

)} {!globalThis.crossOriginIsolated && ( -

- SharedArrayBuffer is unavailable. Serve this page over HTTPS with - COOP/COEP headers; your proxy must preserve the node’s headers. -

+

{t("captioner-isolation-unavailable")}

)}
@@ -333,9 +324,9 @@ export function CaptionerPage() { disabled={(busy && !running) || !did || !mediaAvailable} onClick={() => void (running ? stop() : start())} > - {running ? "Stop captions" : "Start captions"} + {t(running ? "captioner-stop" : "captioner-start")} - {status} + {t(status, { model: statusModel })}
{error && (

@@ -343,7 +334,7 @@ export function CaptionerPage() {

)}
@@ -355,7 +346,7 @@ export function CaptionerPage() {
{did && (

- OBS display URL (no sign-in or microphone required):{" "} + {t("captioner-display-url")}{" "} )}

- Run directly inside OBS -

- Launch OBS with --enable-media-stream, add this HTTPS URL - as a Browser Source, then use Interact to sign in, select your mic, - and start captions. Do not add{" "} - --use-fake-ui-for-media-stream unless you accept - automatic mic access by every browser source. Keep the source active - (disable “Shutdown source when not visible”). For a clean transparent - output, use the separate display URL. -

+ {t("captioner-obs-title")} +

{t("captioner-obs-description")}

); diff --git a/js/web/src/routes/dashboard/stream/index.tsx b/js/web/src/routes/dashboard/stream/index.tsx index 22b2eef1b..658e7fea1 100644 --- a/js/web/src/routes/dashboard/stream/index.tsx +++ b/js/web/src/routes/dashboard/stream/index.tsx @@ -1,3 +1,4 @@ +import { CaptionPolicyFields } from "@/components/captions/caption-policy-fields"; import { useDashboardStore } from "@/components/dashboard/dashboard-store-context"; import { ModeratorsManager } from "@/components/dashboard/moderators"; import { Admonition } from "@/components/ui/admonition"; @@ -20,6 +21,7 @@ import { useSession } from "@/lib/session"; import { useStore } from "@/lib/store"; import { useKeyRecords } from "@/lib/store/hooks"; import { cn } from "@/lib/utils"; +import { buildCaptionPolicy, readCaptionPolicy } from "@streamplace/core"; import { createFileRoute } from "@tanstack/react-router"; import { Clipboard, Key, Loader2, Shield, Tags, Trash2, X } from "lucide-react"; import { useCallback, useEffect, useMemo, useState } from "react"; @@ -124,7 +126,6 @@ export function StreamSettingsPage() { function MetadataSection() { const { t } = useTranslation("common"); const createContentMetadata = useStore((s) => s.createContentMetadata); - const updateContentMetadata = useStore((s) => s.updateContentMetadata); const getContentMetadata = useStore((s) => s.getContentMetadata); const { did: userDid } = useSession(); const liveStore = useDashboardStore(); @@ -147,13 +148,14 @@ function MetadataSection() { const [deleteAfter, setDeleteAfter] = useState("300"); const [initialized, setInitialized] = useState(false); const [saving, setSaving] = useState(false); + const [captionPolicy, setCaptionPolicy] = useState(() => + readCaptionPolicy(undefined), + ); useEffect(() => { - if (initialized) return; + if (initialized || !userDid) return; setInitialized(true); - if (userDid) { - void getContentMetadata({ userDid }); - } + void getContentMetadata({ userDid }); }, [initialized, getContentMetadata, userDid]); // Hydrate from existing metadata once we have data @@ -162,6 +164,7 @@ function MetadataSection() { if (!lastRecord) return; const record = lastRecord.record; if (!record) return; + setCaptionPolicy(readCaptionPolicy(record.captionPolicy)); if (record.contentWarnings?.warnings) { setSelectedWarnings(new Set(record.contentWarnings.warnings)); } @@ -274,23 +277,14 @@ function MetadataSection() { distPolicy.allowedBroadcasters = broadcasters; } - const rkey = livestream?.uri.split("/").pop(); - const livestreamRef = - rkey && livestream - ? { uri: livestream.uri, cid: (livestream.cid as string) ?? "" } - : undefined; - const params = { contentWarnings: Array.from(selectedWarnings), contentRights: filteredRights, distributionPolicy: distPolicy, + captionPolicy: buildCaptionPolicy(captionPolicy), }; - if (livestreamRef) { - await updateContentMetadata({ rkey, livestreamRef, ...params }); - } else { - await createContentMetadata(params); - } + await createContentMetadata(params); toast.success(t("metadata-saved", { defaultValue: "Metadata saved" })); } catch (error) { console.error("Error saving metadata:", error); @@ -304,7 +298,6 @@ function MetadataSection() { } }, [ userDid, - livestream, selectedWarnings, contentRights, licenseSelect, @@ -313,8 +306,8 @@ function MetadataSection() { allowedBroadcasters, archiveIndefinite, deleteAfter, + captionPolicy, createContentMetadata, - updateContentMetadata, t, ]); @@ -334,6 +327,10 @@ function MetadataSection() {

) : ( <> + ): { video?: string } => ({ + video: typeof search.video === "string" ? search.video : undefined, + }), component: DashboardVideosPage, }); function DashboardVideosPage() { + const { t } = useTranslation("common"); + const { video: requestedVideo } = Route.useSearch(); const agent = usePDSAgent(); const isReady = useIsReady(); const userProfile = useUserProfile(); const { state: session, did } = useSession(); const upload = useUpload(); + const { + setTitle, + setDescription, + setTagsDirectly, + setWarningsDirectly, + setLicense, + } = upload; // Cursor-aware infinite list scoped to the logged-in user's repo. const { @@ -45,21 +60,54 @@ function DashboardVideosPage() { (video: any) => { const rec = video.record?.value || video.record || {}; setEditingVideoUri(video.uri); - upload.setTitle(rec.title || ""); - upload.setDescription(rec.description || ""); - upload.setTagsDirectly(rec.tags || []); + setTitle(rec.title || ""); + setDescription(rec.description || ""); + setTagsDirectly(rec.tags || []); setExistingThumb(rec.thumb || null); const cw = rec.contentWarnings?.warnings || []; - upload.setWarningsDirectly(new Set(cw)); + setWarningsDirectly(new Set(cw)); const rights = rec.contentRights || {}; - upload.setLicense( + setLicense( rights.license?.$type || "place.stream.metadata.contentRights#all-rights-reserved", ); }, - [upload], + [ + setTitle, + setDescription, + setTagsDirectly, + setWarningsDirectly, + setLicense, + ], ); + useEffect(() => { + if ( + !agent?.did || + !requestedVideo?.startsWith(`at://${agent.did}/place.stream.video/`) + ) + return; + let cancelled = false; + void agent.com.atproto.repo + .getRecord({ + repo: agent.did, + collection: "place.stream.video", + rkey: requestedVideo.split("/").pop()!, + }) + .then((result) => { + if (!cancelled) + handleSelectVideo({ uri: requestedVideo, record: result.data.value }); + }) + .catch((error) => { + if (cancelled) return; + console.error("Failed to load video for editing", error); + toast.error(t("could-not-load-video")); + }); + return () => { + cancelled = true; + }; + }, [agent, requestedVideo, handleSelectVideo, t]); + const handleUpdate = useCallback( async (u: ReturnType) => { if (!agent?.did || !editingVideoUri) return; @@ -196,6 +244,7 @@ function DashboardVideosPage() { isUpdating={updating} isDeleting={deleting} /> +
); } diff --git a/js/web/src/routes/embed/$user/index.tsx b/js/web/src/routes/embed/$user/index.tsx index d9e51eeb9..f384d2fce 100644 --- a/js/web/src/routes/embed/$user/index.tsx +++ b/js/web/src/routes/embed/$user/index.tsx @@ -1,4 +1,6 @@ import { Player } from "@/components/player/player"; +import { FullscreenProvider } from "@/contexts/fullscreen-context"; +import { useLivestreamStore } from "@/hooks/use-livestream-store"; import { captureError } from "@/lib/log"; import { getStreamplaceUrl } from "@/lib/streamplace-url"; import { createFileRoute } from "@tanstack/react-router"; @@ -10,6 +12,7 @@ export const Route = createFileRoute("/embed/$user/")({ function EmbedLive() { const { user } = Route.useParams(); + const { store } = useLivestreamStore(user); const { playlistUrl, thumbnailUrl } = useMemo(() => { const base = getStreamplaceUrl(); @@ -20,16 +23,19 @@ function EmbedLive() { }, [user]); return ( -
- - captureError(message, { user, source: "embed-live" }) - } - /> -
+ +
+ + captureError(message, { user, source: "embed-live" }) + } + /> +
+ ); } diff --git a/js/web/src/routes/embed/$user/video/$tid.tsx b/js/web/src/routes/embed/$user/video/$tid.tsx index e65ded00f..1bb5b733e 100644 --- a/js/web/src/routes/embed/$user/video/$tid.tsx +++ b/js/web/src/routes/embed/$user/video/$tid.tsx @@ -1,4 +1,5 @@ import { Player } from "@/components/player/player"; +import { FullscreenProvider } from "@/contexts/fullscreen-context"; import { captureError } from "@/lib/log"; import { getStreamplaceUrl } from "@/lib/streamplace-url"; import { createFileRoute } from "@tanstack/react-router"; @@ -21,16 +22,19 @@ function EmbedVideo() { }, [user, tid]); return ( -
- - captureError(message, { user, tid, source: "embed-vod" }) - } - /> -
+ +
+ + captureError(message, { user, tid, source: "embed-vod" }) + } + /> +
+ ); } diff --git a/js/web/src/routes/embed/captions/$user.tsx b/js/web/src/routes/embed/captions/$user.tsx index aa43c515d..83578e0eb 100644 --- a/js/web/src/routes/embed/captions/$user.tsx +++ b/js/web/src/routes/embed/captions/$user.tsx @@ -1,15 +1,16 @@ import { useLivestreamStore } from "@/hooks/use-livestream-store"; import type { LivestreamStore } from "@streamplace/core"; import { - activeLiveCaptions, CAPTION_WEB_FONTS, captionColor, captionFontSize, DEFAULT_CAPTION_PREFS, + displayLiveCaptions, parseCaptionPrefs, } from "@streamplace/core"; import { createFileRoute } from "@tanstack/react-router"; import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; import { useStore } from "zustand"; export const Route = createFileRoute("/embed/captions/$user")({ @@ -54,6 +55,7 @@ function CaptionOverlay() { function OverlayBody({ store }: { store: LivestreamStore }) { const search = Route.useSearch(); + const { t } = useTranslation("common"); const tracks = useStore(store, (state) => state.captionTracks); const captions = useStore(store, (state) => state.liveCaptions); const [now, setNow] = useState(Date.now()); @@ -71,14 +73,14 @@ function OverlayBody({ store }: { store: LivestreamStore }) { search.track ?? tracks.find((item) => item.origin === "canonical")?.id ?? tracks[0]?.id; - const lines = track ? activeLiveCaptions(captions, track, now) : []; + const lines = track ? displayLiveCaptions(captions, track, now) : []; const prefs = parseCaptionPrefs( JSON.stringify({ ...DEFAULT_CAPTION_PREFS, font: search.font }), ); const fontSize = Number.isFinite(search.fontSize) && (search.fontSize ?? 0) > 0 ? Math.min(200, search.fontSize!) - : captionFontSize(prefs, height, 22); + : `max(var(--text-lg), ${captionFontSize(prefs, height, 0)}px)`; const color = search.color && CSS.supports("color", search.color) ? search.color @@ -101,7 +103,7 @@ function OverlayBody({ store }: { store: LivestreamStore }) { >
setHandle(e.target.value)} @@ -147,6 +148,7 @@ function LoginPage() {
- +
@@ -111,6 +115,7 @@ function AdvancedSettings() {
setNewUrl(e.target.value)} @@ -125,6 +130,7 @@ function AdvancedSettings() { type="button" size="lg" onClick={onSubmitUrl} + data-testid="settings-save-node" disabled={!newUrl.trim()} > {t("save-button")} diff --git a/js/web/src/routes/settings/captions.tsx b/js/web/src/routes/settings/captions.tsx new file mode 100644 index 000000000..a25c5c09e --- /dev/null +++ b/js/web/src/routes/settings/captions.tsx @@ -0,0 +1,16 @@ +import { CaptionSettings } from "@/components/captions/caption-settings"; +import { createFileRoute } from "@tanstack/react-router"; +import { useTranslation } from "react-i18next"; + +export const Route = createFileRoute("/settings/captions")({ + component: CaptionsPage, +}); +function CaptionsPage() { + const { t } = useTranslation("settings"); + return ( +
+

{t("captions")}

+ +
+ ); +} diff --git a/lexicons/place/stream/caption/pushCaptions.json b/lexicons/place/stream/caption/pushCaptions.json index 8d02fc998..eea8b5296 100644 --- a/lexicons/place/stream/caption/pushCaptions.json +++ b/lexicons/place/stream/caption/pushCaptions.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "procedure", - "description": "Push live captions into a stream from a captioner: the Streamplace caption overlay, a CART stenographer, or an external tool. Requires the streamer's authorization. When the streamer's caption policy is `ingest`, pushed captions become the canonical caption track.", + "description": "Push live captions into a stream from a captioner, a CART stenographer, or an external tool. Requires the streamer's OAuth authorization or active registered stream key. Under canonical policy auto or ingest, pushed captions become streamer-signed MUXL text tracks. Under canonical policy off, pushed captions remain sidecar or local according to the streamer's node-caption policy.", "input": { "encoding": "application/json", "schema": { diff --git a/pkg/api/captions_records.go b/pkg/api/captions_records.go index 6f53cca0c..33f56bdd9 100644 --- a/pkg/api/captions_records.go +++ b/pkg/api/captions_records.go @@ -3,15 +3,15 @@ package api import ( "stream.place/streamplace/pkg/captions" "stream.place/streamplace/pkg/captions/records" + "stream.place/streamplace/pkg/vod" ) -// recordCaptions is the node's VideoCaptions: the place.stream.caption.transcript -// records in the index, with live captions placed on VODs through the -// recording objects this node keeps. +// recordCaptions combines the archival MUXL text tracks with indexed transcript +// records. Canonical records are a copy, not a second rendition. func (a *StreamplaceAPI) recordCaptions() captions.VideoCaptions { p := &records.Provider{Store: a.Model, NodeDID: a.CLI.ServerDID()} if a.StatefulDB != nil { p.Recording = records.StatedbRecording{State: a.StatefulDB} } - return p + return &vod.VideoCaptions{Model: a.Model, Store: a.PlaybackStore, Records: p} } diff --git a/pkg/api/stream_key.go b/pkg/api/stream_key.go index b686a73e7..454cf2714 100644 --- a/pkg/api/stream_key.go +++ b/pkg/api/stream_key.go @@ -2,82 +2,15 @@ package api import ( "context" - "crypto" "fmt" - "github.com/bluesky-social/indigo/atproto/atcrypto" - "github.com/decred/dcrd/dcrec/secp256k1" - "github.com/mr-tron/base58" "stream.place/streamplace/pkg/atproto" "stream.place/streamplace/pkg/log" "stream.place/streamplace/pkg/media" ) func (a *StreamplaceAPI) MakeMediaSigner(ctx context.Context, keyStr string) (media.MediaSigner, error) { - if len(keyStr) < 2 || keyStr[0] != 'z' { - return nil, fmt.Errorf("invalid authorization key (not a multibase base58btc string)") - } - - var addrBytes []byte - var didBytes []byte - priv, err := atcrypto.ParsePrivateMultibase(keyStr) - if err == nil { - addrBytes = priv.Bytes() - } else { - decoded, err := base58.Decode(keyStr[1:]) - if err != nil { - return nil, fmt.Errorf("invalid authorization key (not a base58btc string)") - } - addrBytes = decoded[:32] - didBytes = decoded[32:] - priv, err = atcrypto.ParsePrivateBytesK256(addrBytes) - if err != nil { - return nil, fmt.Errorf("invalid authorization key (not valid atproto): %w", err) - } - } - - key, _ := secp256k1.PrivKeyFromBytes(addrBytes) - if key == nil { - return nil, fmt.Errorf("invalid authorization key (not valid secp256k1)") - } - var signer crypto.Signer = key.ToECDSA() - pub, err := priv.PublicKey() - if err != nil { - return nil, fmt.Errorf("invalid authorization key (could not parse as atproto): %w", err) - } - - did := string(didBytes) - - if did != "" { - repo, err := a.ATSync.SyncBlueskyRepo(ctx, did, a.Model) - if err != nil { - return nil, fmt.Errorf("could not resolve streamplace key: %w", err) - } - err = a.CLI.StreamIsAllowed(repo.DID) - if err != nil { - return nil, fmt.Errorf("user is not allowed to stream: %w", err) - } - signingKey, err := a.Model.GetSigningKey(ctx, pub.DIDKey(), repo.DID) - if err != nil { - return nil, fmt.Errorf("signing key not found: %w", err) - } - if signingKey == nil { - return nil, fmt.Errorf("signing key not found") - } - } else { - atkey, err := atproto.ParsePubKey(signer.Public()) - if err != nil { - return nil, fmt.Errorf("invalid authorization key (not valid secp256k1): %w", err) - } - did = atkey.DIDKey() - err = a.CLI.StreamIsAllowed(did) - if err != nil { - return nil, fmt.Errorf("user is not allowed to stream: %w", err) - } - } - - ctx = log.WithLogValues(ctx, "did", did) - err = a.checkBanned(ctx, did) + did, signer, err := media.AuthenticateStreamKey(ctx, a.CLI, a.Model, a.ATSync, keyStr, false) if err != nil { return nil, err } diff --git a/pkg/aqhttp/transport.go b/pkg/aqhttp/transport.go index 676d7962b..f8c9e51bf 100644 --- a/pkg/aqhttp/transport.go +++ b/pkg/aqhttp/transport.go @@ -23,6 +23,7 @@ func (t *TrustedTransport) RoundTrip(req *http.Request) (*http.Response, error) func NewTrustedTransport() *TrustedTransport { return &TrustedTransport{ Base: &http.Transport{ + Proxy: http.ProxyFromEnvironment, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, diff --git a/pkg/captions/hub.go b/pkg/captions/hub.go index ccc11352d..1de9ee2b9 100644 --- a/pkg/captions/hub.go +++ b/pkg/captions/hub.go @@ -3,6 +3,7 @@ package captions import ( "context" "sort" + "strings" "sync" "time" ) @@ -11,7 +12,11 @@ import ( // It must cover the live HLS window plus the slowest player's lag behind it. const DefaultRetention = 15 * time.Minute -const subscriberBuffer = 256 +const ( + subscriberBuffer = 256 + maxTracksPerStream = 64 + maxFinalCuesPerTrack = 8192 +) // Hub is the per-node registry of live caption tracks. Sources publish cues // into it; outputs subscribe to cue events or read windows of final cues. @@ -31,7 +36,6 @@ type trackCaptions struct { track Track final []Cue // sorted by Start finalIDs map[string]struct{} - interim map[string]Cue } func NewHub(retention time.Duration) *Hub { @@ -55,28 +59,60 @@ func (h *Hub) stream(streamer string) *streamCaptions { // a final cue's ID is ignored. A slow subscriber misses events rather than // stalling sources; final cues stay readable through Cues. func (h *Hub) Publish(streamer string, track Track, cue Cue) { + h.publish(streamer, track, cue, false) +} + +// PublishCanonical joins a MUXL cue clipped across consecutive GoPs. The text +// and start of a final cue never change; only its discovered end can extend. +// This produces one display cue in HLS rather than duplicate clipped pieces. +func (h *Hub) PublishCanonical(streamer string, track Track, cue Cue) { + h.publish(streamer, track, cue, true) +} + +func (h *Hub) publish(streamer string, track Track, cue Cue, continuation bool) { h.mu.Lock() defer h.mu.Unlock() s := h.stream(streamer) t, ok := s.tracks[track.ID] if !ok { - t = &trackCaptions{finalIDs: map[string]struct{}{}, interim: map[string]Cue{}} + if len(s.tracks) >= maxTracksPerStream { + return + } + t = &trackCaptions{finalIDs: map[string]struct{}{}} s.tracks[track.ID] = t + } else if t.track.Origin != track.Origin { + return } t.track = track + if continuation && len(t.final) > 0 { + last := &t.final[len(t.final)-1] + sameCue := last.ID == cue.ID || (strings.HasPrefix(cue.ID, "muxl-") && !cue.Start.After(last.End)) + if strings.HasPrefix(cue.ID, "muxl-") && last.Text == cue.Text && !cue.Start.Before(last.Start) && !cue.End.After(last.End) { + return + } + if last.Text == cue.Text && !cue.Start.Before(last.Start) && cue.End.After(last.End) && sameCue { + last.End = cue.End + cue = *last + h.prune(t) + for ch := range s.subs { + select { + case ch <- Event{Streamer: streamer, Track: track, Cue: cue}: + default: + } + } + return + } + } if _, done := t.finalIDs[cue.ID]; done { return } if cue.Final { - delete(t.interim, cue.ID) t.finalIDs[cue.ID] = struct{}{} i := sort.Search(len(t.final), func(i int) bool { return t.final[i].Start.After(cue.Start) }) t.final = append(t.final, Cue{}) copy(t.final[i+1:], t.final[i:]) t.final[i] = cue h.prune(t) - } else { - t.interim[cue.ID] = cue } ev := Event{Streamer: streamer, Track: track, Cue: cue} for ch := range s.subs { @@ -106,6 +142,13 @@ func (h *Hub) prune(t *trackCaptions) { kept = append(kept, c) } t.final = kept + if over := len(t.final) - maxFinalCuesPerTrack; over > 0 { + for _, cue := range t.final[:over] { + delete(t.finalIDs, cue.ID) + } + copy(t.final, t.final[over:]) + t.final = t.final[:len(t.final)-over] + } } // Subscribe returns cue events for a streamer's tracks until ctx is done, when @@ -118,7 +161,12 @@ func (h *Hub) Subscribe(ctx context.Context, streamer string) <-chan Event { go func() { <-ctx.Done() h.mu.Lock() - delete(h.stream(streamer).subs, ch) + if s, ok := h.streams[streamer]; ok { + delete(s.subs, ch) + if len(s.subs) == 0 && len(s.tracks) == 0 { + delete(h.streams, streamer) + } + } close(ch) h.mu.Unlock() }() @@ -173,5 +221,24 @@ func (h *Hub) EndSession(streamer string) { defer h.mu.Unlock() if s, ok := h.streams[streamer]; ok { s.tracks = map[string]*trackCaptions{} + if len(s.subs) == 0 { + delete(h.streams, streamer) + } + } +} + +// Retention bounds the final history available for archival reconciliation. +func (h *Hub) Retention() time.Duration { return h.retention } + +// RemoveOrigin suppresses competing tracks when canonical captions appear. +func (h *Hub) RemoveOrigin(streamer string, origin Origin) { + h.mu.Lock() + defer h.mu.Unlock() + if s := h.streams[streamer]; s != nil { + for id, t := range s.tracks { + if t.track.Origin == origin { + delete(s.tracks, id) + } + } } } diff --git a/pkg/captions/hub_distribution_test.go b/pkg/captions/hub_distribution_test.go new file mode 100644 index 000000000..8d17894fd --- /dev/null +++ b/pkg/captions/hub_distribution_test.go @@ -0,0 +1,51 @@ +package captions + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestHubRejectsCrossOriginCollision(t *testing.T) { + h := NewHub(0) + now := time.Now() + canonical := Track{ID: "canonical-auto-en", Origin: OriginCanonical, Source: SourceAuto, Language: "en"} + h.Publish("alice", canonical, Cue{ID: "real", Text: "Real", Start: now, End: now.Add(time.Second), Final: true}) + forged := canonical + forged.Origin = OriginSidecar + h.Publish("alice", forged, Cue{ID: "forged", Text: "Forged", Start: now, End: now.Add(time.Second), Final: true}) + require.Equal(t, []Track{canonical}, h.Tracks("alice")) + require.Equal(t, "Real", h.Cues("alice", canonical.ID, now, now.Add(time.Second))[0].Text) + require.Len(t, h.Cues("alice", canonical.ID, now, now.Add(time.Second)), 1) +} + +func TestHubAnonymousCuesPreserveShortGap(t *testing.T) { + h := NewHub(0) + now := time.Now() + track := Track{ID: "canonical-ingest-en", Origin: OriginCanonical} + h.PublishCanonical("alice", track, Cue{ID: "muxl-1", Text: "Yes", Start: now, End: now.Add(time.Second), Final: true}) + h.PublishCanonical("alice", track, Cue{ID: "muxl-2", Text: "Yes", Start: now.Add(1050 * time.Millisecond), End: now.Add(2 * time.Second), Final: true}) + require.Empty(t, h.Cues("alice", track.ID, now.Add(time.Second), now.Add(1050*time.Millisecond)), "an intentional subtitle gap is not filled") +} + +func TestHubBoundsTrackAdmissionAndReleasesEndedStreams(t *testing.T) { + h := NewHub(0) + now := time.Now() + for i := range 1000 { + h.Publish("alice", Track{ID: fmt.Sprint(i), Origin: OriginSidecar}, Cue{ID: fmt.Sprint(i), Start: now, End: now.Add(time.Second)}) + } + require.LessOrEqual(t, len(h.Tracks("alice")), 64) + ctx, cancel := context.WithCancel(context.Background()) + events := h.Subscribe(ctx, "alice") + h.EndSession("alice") + cancel() + for range events { + } + h.mu.Lock() + _, exists := h.streams["alice"] + h.mu.Unlock() + require.False(t, exists, "ended streams with no remaining subscribers release their registry entry") +} diff --git a/pkg/captions/hub_test.go b/pkg/captions/hub_test.go index 14247944c..a6ff1b415 100644 --- a/pkg/captions/hub_test.go +++ b/pkg/captions/hub_test.go @@ -73,3 +73,17 @@ func TestHubSubscribeReceivesEventsAndCloses(t *testing.T) { for range ch { } } + +func TestHubCanonicalAnonymousContinuationAndReplay(t *testing.T) { + h := NewHub(time.Minute) + tr := Track{ID: "canonical-ingest-en", Origin: OriginCanonical} + h.PublishCanonical("s", tr, cue("muxl-9-100", 0.1, 1, "same line", true)) + h.PublishCanonical("s", tr, cue("muxl-9-1000", 1, 2, "same line", true)) + h.PublishCanonical("s", tr, cue("muxl-9-1000", 1, 2, "same line", true)) + h.PublishCanonical("s", tr, cue("muxl-9-3000", 3, 4, "same line", true)) + got := h.Cues("s", tr.ID, t0, t0.Add(time.Minute)) + require.Equal(t, []Cue{ + cue("muxl-9-100", 0.1, 2, "same line", true), + cue("muxl-9-3000", 3, 4, "same line", true), + }, got) +} diff --git a/pkg/captions/muxl.go b/pkg/captions/muxl.go new file mode 100644 index 000000000..f3423b9aa --- /dev/null +++ b/pkg/captions/muxl.go @@ -0,0 +1,125 @@ +package captions + +import ( + "bytes" + "context" + "fmt" + "io" + "sync" + "time" + + upstream "github.com/streamplace/muxl/go" + "stream.place/streamplace/pkg/captions/fmp4" +) + +var textEngineOnce sync.Once +var textEngine *upstream.WASMEngine +var textEngineErr error + +// TextEngine shares the timed-text wasm across live extraction and VOD reads. +func TextEngine() (*upstream.WASMEngine, error) { + textEngineOnce.Do(func() { textEngine, textEngineErr = upstream.NewWASM(context.Background()) }) + return textEngine, textEngineErr +} + +// CanonicalTrack decodes the fixed MUXL language/source convention. +func CanonicalTrack(t upstream.TextTrack, author string) Track { + lang := t.Language + if lang == "" { + lang = "und" + } + source := Source(t.Label) + switch source { + case SourceAuto, SourceIngest, SourceHuman: + default: + source = SourceIngest + } + return Track{ID: TrackID(OriginCanonical, source, lang), Language: lang, Kind: KindCaptions, Source: source, Origin: OriginCanonical, Author: author} +} + +// SegmentClock reads the reference AV decode time without a wasm invocation. +// header is the already synthesized presentation header; segment stays canonical. +func SegmentClock(header, segment []byte) (time.Duration, bool, error) { + tracks, err := fmp4.Tracks(header) + if err != nil { + return 0, false, err + } + hasText := false + var ref fmp4.TrackInfo + for _, t := range tracks { + switch t.Handler { + case "text", "sbtl", "subt": + hasText = true + case "vide": + ref = t + case "soun": + if ref.ID == 0 { + ref = t + } + } + } + if !hasText { + return 0, false, nil + } + frags, err := fmp4.Fragments(segment) + if err != nil { + return 0, true, err + } + for _, f := range frags { + if f.TrackID == ref.ID && ref.Timescale != 0 { + return time.Duration(f.BaseDecodeTime/uint64(ref.Timescale))*time.Second + time.Duration(f.BaseDecodeTime%uint64(ref.Timescale))*time.Second/time.Duration(ref.Timescale), true, nil + } + } + return 0, true, fmt.Errorf("caption reference track missing") +} + +// ReadCanonical reads one validated segment's text and places it on its signed +// wall clock. No text command is run for an AV-only segment. +func ReadCanonical(ctx context.Context, segment, header []byte, wall time.Time, author string) ([]Event, bool, error) { + media, has, err := SegmentClock(header, segment) + if err != nil || !has { + return nil, has, err + } + return ReadCanonicalWithClock(ctx, segment, media, wall, author) +} + +// ReadCanonicalWithClock reads cues after admission has classified the segment. +func ReadCanonicalWithClock(ctx context.Context, segment []byte, media time.Duration, wall time.Time, author string) ([]Event, bool, error) { + has := true + eng, err := TextEngine() + if err != nil { + return nil, has, err + } + tracks, err := eng.TextTracks(ctx, bytes.NewReader(segment)) + if err != nil { + return nil, has, err + } + var out []Event + for _, t := range tracks { + cues, err := eng.ReadTextCues(ctx, bytes.NewReader(segment), t.TrackID) + if err != nil { + return nil, has, err + } + track := CanonicalTrack(t, author) + for _, c := range cues { + if c.Text == "" || c.End <= c.Start { + continue + } + id := c.ID + if id == "" { + id = fmt.Sprintf("muxl-%d-%d", t.TrackID, c.Start) + } + out = append(out, Event{Track: track, Cue: Cue{ID: id, Text: c.Text, Start: wall.Add(time.Duration(c.Start)*time.Millisecond - media), End: wall.Add(time.Duration(c.End)*time.Millisecond - media), Final: true}}) + } + } + return out, has, nil +} + +// ReadTextCues is also used over a stored video's canonical fragment range. +func ReadTextCues(ctx context.Context, input io.Reader, track uint32) ([]upstream.TextCue, error) { + eng, err := TextEngine() + if err != nil { + return nil, err + } + return eng.ReadTextCues(ctx, input, track) +} diff --git a/pkg/captions/muxl_test.go b/pkg/captions/muxl_test.go new file mode 100644 index 000000000..56c81b37e --- /dev/null +++ b/pkg/captions/muxl_test.go @@ -0,0 +1,107 @@ +package captions + +import ( + "bytes" + "context" + "os" + "path/filepath" + "runtime" + "sort" + "testing" + "time" + + upstream "github.com/streamplace/muxl/go" + "github.com/stretchr/testify/require" +) + +func TestCanonicalMuxlClockAndContinuation(t *testing.T) { + ctx := context.Background() + eng, err := TextEngine() + require.NoError(t, err) + _, file, _, _ := runtime.Caller(0) + data, err := os.ReadFile(filepath.Join(filepath.Dir(file), "../../test/fixtures/h264-opus-frag.mp4")) + require.NoError(t, err) + ch := make(chan *upstream.Event, 32) + errCh := make(chan error, 1) + go func() { errCh <- eng.SegmentEvents(ctx, bytes.NewReader(data), ch); close(ch) }() + var segments []*upstream.Event + for ev := range ch { + if ev.Type == "segment" { + segments = append(segments, ev) + } + } + require.NoError(t, <-errCh) + require.GreaterOrEqual(t, len(segments), 2) + join := func(ev *upstream.Event) []byte { + keys := make([]string, 0, len(ev.Tracks)) + for id := range ev.Tracks { + keys = append(keys, id) + } + sort.Strings(keys) + var out []byte + for _, id := range keys { + out = append(out, ev.Tracks[id]...) + } + return out + } + var plainHeader bytes.Buffer + plain := join(segments[0]) + require.NoError(t, eng.Wrap(ctx, bytes.NewReader(plain), "flat", &plainHeader)) + _, has, err := ReadCanonical(ctx, plain, plainHeader.Bytes(), time.Unix(1000, 0), "did:plc:alice") + require.NoError(t, err) + require.False(t, has) + track := upstream.TextTrack{TrackID: 9, Language: "en", Label: "auto"} + // Discover actual media boundaries with a gap-covering track: the fixture + // deliberately starts away from zero, so receive-time/zero-based math fails. + withEmpty, err := eng.AddTextTrack(ctx, plain, track, nil) + require.NoError(t, err) + var header bytes.Buffer + require.NoError(t, eng.Wrap(ctx, bytes.NewReader(withEmpty), "flat", &header)) + media0, has, err := SegmentClock(header.Bytes(), plain) + require.NoError(t, err) + require.True(t, has) + plain1 := join(segments[1]) + withEmpty1, err := eng.AddTextTrack(ctx, plain1, track, nil) + require.NoError(t, err) + var header1 bytes.Buffer + require.NoError(t, eng.Wrap(ctx, bytes.NewReader(withEmpty1), "flat", &header1)) + media1, _, err := SegmentClock(header1.Bytes(), plain1) + require.NoError(t, err) + cue := upstream.TextCue{ID: "cross", Text: "Across the boundary", Start: uint64(media0.Milliseconds() + 250), End: uint64(media1.Milliseconds() + 500)} + wall := time.Unix(1700000000, 0).UTC() + hub := NewHub(0) + for i, seg := range [][]byte{plain, plain1} { + added, err := eng.AddTextTrack(ctx, seg, track, []upstream.TextCue{cue}) + require.NoError(t, err) + var hdr bytes.Buffer + require.NoError(t, eng.Wrap(ctx, bytes.NewReader(added), "flat", &hdr)) + start := wall + if i == 1 { + start = start.Add(media1 - media0) + } + events, has, err := ReadCanonical(ctx, added, hdr.Bytes(), start, "did:plc:alice") + require.NoError(t, err) + require.True(t, has) + for _, ev := range events { + hub.PublishCanonical("alice", ev.Track, ev.Cue) + hub.PublishCanonical("alice", ev.Track, ev.Cue) + } + } + got := hub.Cues("alice", "canonical-auto-en", wall, wall.Add(time.Minute)) + require.Len(t, got, 1) + require.Equal(t, "Across the boundary", got[0].Text) + require.Equal(t, wall.Add(time.Duration(cue.Start)*time.Millisecond-media0), got[0].Start) + require.Equal(t, wall.Add(time.Duration(cue.End)*time.Millisecond-media0), got[0].End) + // A lazy second language/source and an empty earlier track are independently + // discovered; no empty cue is published. + second, err := eng.AddTextTrack(ctx, withEmpty1, upstream.TextTrack{TrackID: 10, Language: "es", Label: "human"}, []upstream.TextCue{{ID: "es", Text: "Hola", Start: uint64(media1.Milliseconds() + 100), End: uint64(media1.Milliseconds() + 400)}}) + require.NoError(t, err) + var hdr bytes.Buffer + require.NoError(t, eng.Wrap(ctx, bytes.NewReader(second), "flat", &hdr)) + events, has, err := ReadCanonical(ctx, second, hdr.Bytes(), wall.Add(media1-media0), "did:plc:alice") + require.NoError(t, err) + require.True(t, has) + require.Len(t, events, 1) + require.Equal(t, "canonical-human-es", events[0].Track.ID) + require.Equal(t, "Hola", events[0].Cue.Text) +} diff --git a/pkg/captions/recognizer.go b/pkg/captions/recognizer.go index baea083a3..05d637a51 100644 --- a/pkg/captions/recognizer.go +++ b/pkg/captions/recognizer.go @@ -22,6 +22,9 @@ type RecognizerOptions struct { Hub *Hub Engine stt.Engine Layout CueLayout + // OnCoverage runs after finalized captions are published, with the absolute + // audio watermark before any still-provisional words. It is monotonic. + OnCoverage func(time.Time) // Step is how much new audio arrives between passes over the window. Step time.Duration @@ -103,6 +106,7 @@ type Recognizer struct { track Track grouper *Grouper passes int + coverage time.Time } // NewRecognizer leases recognition capacity and starts the worker. It @@ -189,6 +193,7 @@ func (r *Recognizer) run() { // finish everything with a forced final pass. r.drain() r.maybePass(true) + // A refused/failed final pass must still flush previously known words. r.finish() return case <-r.ctx.Done(): @@ -285,6 +290,7 @@ func (r *Recognizer) maybePass(force bool) { // window moves past the silence. r.commitAll() r.advance(len(r.buf)) + r.reportCoverage(r.bufStart) return } @@ -370,11 +376,27 @@ func (r *Recognizer) maybePass(force bool) { r.advance(len(pcm) - keep) } } - if !force { - // A forced pass is followed by a commit of everything, so an - // interim would only flicker. + if force { + r.finish() + } else { r.publishInterim() } + // Examining audio is not enough: the signer may consume only immutable + // finals, so never release it past a still-open cue or uncommitted word. + finalized := winEnd + if cue, ok := r.grouper.Current(r.prev); ok && cue.Start.Before(finalized) { + finalized = cue.Start + } + r.reportCoverage(finalized) +} + +func (r *Recognizer) reportCoverage(end time.Time) { + if end.After(r.coverage) { + r.coverage = end + if r.opts.OnCoverage != nil { + r.opts.OnCoverage(end) + } + } } func (r *Recognizer) trackLanguage() string { diff --git a/pkg/captions/recognizer_test.go b/pkg/captions/recognizer_test.go index 3276881a6..3163c81ce 100644 --- a/pkg/captions/recognizer_test.go +++ b/pkg/captions/recognizer_test.go @@ -423,3 +423,55 @@ func TestWrapLines(t *testing.T) { require.Equal(t, "aa bb\ncc dd\nee ff", WrapLines("aa bb cc dd ee ff", 5, 3)) require.Equal(t, "", WrapLines(" ", 37, 2)) } + +func TestRecognizerCoverageFollowsDecisionsAtMonotonicWindowEnd(t *testing.T) { + model := &fakeModel{} + var covered []time.Time + start := time.UnixMilli(1000) + r, err := NewRecognizer(context.Background(), RecognizerOptions{ + Streamer: "coverage", Origin: OriginCanonical, Hub: NewHub(time.Minute), + Engine: &fakeEngine{lease: &fakeLease{model: model}}, + Step: time.Second, MinWindow: 2 * time.Second, + OnCoverage: func(end time.Time) { covered = append(covered, end) }, + }) + require.NoError(t, err) + r.Push(start, speech(time.Second)) + r.settle() + require.Empty(t, covered, "PCM receipt alone must not release a held GoP") + r.Push(start.Add(time.Second), speech(time.Second)) + r.settle() + require.Equal(t, []time.Time{start.Add(2 * time.Second)}, covered) + // A pure-silence decision also covers its complete window. + r.Push(start.Add(3*time.Second), silence(2*time.Second)) + r.settle() + require.Equal(t, start.Add(5*time.Second), covered[len(covered)-1]) + // A backwards discontinuity cannot move the reported coverage backwards. + before := len(covered) + r.Push(start, silence(2*time.Second)) + r.settle() + require.Len(t, covered, before) + r.Close() +} + +func TestRecognizerEOFCoverageIncludesFinalVoicedWords(t *testing.T) { + hub := NewHub(time.Minute) + start := time.UnixMilli(1000) + var atCoverage []Cue + var covered time.Time + model := &fakeModel{script: []*stt.Result{result(w("last words", 0.1, 0.7))}} + r, err := NewRecognizer(context.Background(), RecognizerOptions{ + Streamer: "eof", Origin: OriginCanonical, Hub: hub, + Engine: &fakeEngine{lease: &fakeLease{model: model}}, MinWindow: 2 * time.Second, + OnCoverage: func(end time.Time) { + covered = end + atCoverage = hub.Cues("eof", TrackID(OriginCanonical, SourceAuto, "en"), start, end) + }, + }) + require.NoError(t, err) + r.Push(start, speech(time.Second)) + r.Close() + require.Equal(t, start.Add(time.Second), covered) + require.Len(t, atCoverage, 1, "the final voiced pass must publish before releasing the last GoP") + require.Equal(t, "last words", atCoverage[0].Text) + require.True(t, atCoverage[0].Final) +} diff --git a/pkg/captions/records/distribution_test.go b/pkg/captions/records/distribution_test.go new file mode 100644 index 000000000..35d2f453e --- /dev/null +++ b/pkg/captions/records/distribution_test.go @@ -0,0 +1,103 @@ +package records + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/captions/transcript" + "stream.place/streamplace/pkg/comatproto" + "stream.place/streamplace/pkg/placestream" +) + +type captionPDS struct { + calls []comatproto.RepoPutRecord_Input +} + +func (p *captionPDS) Do(_ context.Context, _, _, _ string, _ map[string]any, body, out any) error { + in := body.(comatproto.RepoPutRecord_Input) + p.calls = append(p.calls, in) + if len(p.calls) == 1 { + return errors.New("temporary PDS failure") + } + *out.(*comatproto.RepoPutRecord_Output) = comatproto.RepoPutRecord_Output{Uri: "at://" + in.Repo + "/place.stream.caption.transcript/" + in.Rkey, Cid: "bafy"} + return nil +} + +func TestDistributionWriterFinalFlushPDSRetry(t *testing.T) { + pds := &captionPDS{} + h := newHarness(t, func(cfg *Config) { + cfg.Publisher = &RepoPublisher{Clients: func(context.Context, string) (XRPCClient, error) { return pds, nil }} + }) + h.w.StartSessionWithOrigin(context.Background(), streamer, t0, true) + cue := captions.Cue{ID: "gop", Text: "A continuing cue", Start: t0.Add(time.Second), End: t0.Add(2 * time.Second), Final: true} + h.hub.PublishCanonical(streamer, autoTrack, cue) + cue.Start = t0.Add(2 * time.Second) + cue.End = t0.Add(3 * time.Second) + h.hub.PublishCanonical(streamer, autoTrack, cue) + h.w.StopSession(streamer) + require.Len(t, pds.calls, 2) + require.Equal(t, pds.calls[0].Rkey, pds.calls[1].Rkey, "a retry overwrites, rather than duplicating") + rec := pds.calls[1].Record.Val.(*placestream.CaptionTranscript) + require.Equal(t, streamer, pds.calls[1].Repo) + require.Equal(t, int64(1000), rec.StartMs) + words := transcript.Decode(transcript.Compact{StartMs: rec.StartMs, Text: rec.Text, Timings: rec.Timings}) + require.Equal(t, int64(3000), words[len(words)-1].EndMs) +} + +func TestDistributionRelayWritesOnlyOwnSidecar(t *testing.T) { + h := newHarness(t, nil) + h.w.StartSessionWithOrigin(context.Background(), streamer, t0, false) + say(h, autoTrack, "canonical", 0, "must not write streamer repo") + own := captions.Track{ID: "sidecar-auto-en", Language: "en", Kind: captions.KindCaptions, Source: captions.SourceAuto, Origin: captions.OriginSidecar, Author: nodeDID} + say(h, own, "own", 1000, "node speech") + upstream := own + upstream.ID = "upstream" + upstream.Author = "did:web:upstream.example" + say(h, upstream, "other", 2000, "must not copy upstream records") + h.w.StopSession(streamer) + calls := h.repos.snapshot() + require.Len(t, calls, 1) + require.Equal(t, Target{Repo: nodeDID, Node: true}, calls[0].target) + require.Equal(t, "node speech", calls[0].rec.Text) +} + +func TestDistributionWriterKeepsActiveCanonicalCueAcrossFlush(t *testing.T) { + h := newHarness(t, nil) + h.now = t0.Add(2 * time.Second) + h.w.StartSessionWithOrigin(context.Background(), streamer, t0, true) + cue := captions.Cue{ID: "continuing", Text: "Continuing speech", Start: t0.Add(time.Second), End: t0.Add(2 * time.Second), Final: true} + h.hub.PublishCanonical(streamer, autoTrack, cue) + h.buffered(streamer, 2) + h.flushed() + require.Empty(t, h.repos.snapshot(), "the newest canonical fragment can still extend") + cue.Start = t0.Add(2 * time.Second) + cue.End = t0.Add(3 * time.Second) + h.hub.PublishCanonical(streamer, autoTrack, cue) + h.w.StopSession(streamer) + calls := h.repos.snapshot() + require.Len(t, calls, 1) + words := decode(calls[0].rec) + require.Equal(t, int64(3000), words[len(words)-1].EndMs, "a flush boundary must not truncate a canonical cue") +} + +func TestDistributionWriterSettlesCanonicalCueAfterSilence(t *testing.T) { + h := newHarness(t, nil) + h.now = t0.Add(time.Second) + h.w.StartSessionWithOrigin(context.Background(), streamer, t0, true) + cue := captions.Cue{ID: "last", Text: "Last speech", Start: t0, End: t0.Add(time.Second), Final: true} + h.hub.PublishCanonical(streamer, autoTrack, cue) + h.buffered(streamer, 2) + h.flushed() + require.Empty(t, h.repos.snapshot()) + h.advance(DefaultFlushInterval) + h.flushed() + calls := h.repos.snapshot() + require.Len(t, calls, 1, "silence must not indefinitely defer the last canonical cue") + require.Equal(t, "Last speech", calls[0].rec.Text) + h.w.StopSession(streamer) + require.Len(t, h.repos.snapshot(), 1, "session flush does not duplicate the settled cue") +} diff --git a/pkg/captions/records/importer.go b/pkg/captions/records/importer.go index 0bcf721c8..3df5a9ca5 100644 --- a/pkg/captions/records/importer.go +++ b/pkg/captions/records/importer.go @@ -98,7 +98,7 @@ func (im *Importer) Import(ctx context.Context, client XRPCClient, caller string if err != nil { return nil, err } - chunks := transcript.Chunk(transcript.WordsFromCues(cues), im.Chunking) + chunks := transcript.ChunkAuthored(transcript.WordsFromCues(cues), im.Chunking) if len(chunks) == 0 { return nil, fmt.Errorf("%w: the file has no caption text", ErrInvalidCaptions) } diff --git a/pkg/captions/records/outbox.go b/pkg/captions/records/outbox.go new file mode 100644 index 000000000..028f348e3 --- /dev/null +++ b/pkg/captions/records/outbox.go @@ -0,0 +1,183 @@ +package records + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/placestream" +) + +// The outbox uses the existing idempotent PDS record key. An interrupted final +// flush therefore survives both the session and a node restart without a second +// transcript record. Files contain no OAuth credentials. +type outboxRecord struct { + Target Target `json:"target"` + Rkey string `json:"rkey"` + Record *placestream.CaptionTranscript `json:"record"` + RetryAt time.Time `json:"retryAt"` +} + +type outbox struct { + cfg Config + mu sync.Mutex + owners map[string]*session + cancel context.CancelFunc + done chan struct{} + wake chan struct{} +} + +func newOutbox(cfg Config) (*outbox, error) { + if err := os.MkdirAll(cfg.OutboxDir, 0700); err != nil { + return nil, fmt.Errorf("create caption outbox: %w", err) + } + ctx, cancel := context.WithCancel(context.Background()) + o := &outbox{cfg: cfg, owners: map[string]*session{}, cancel: cancel, done: make(chan struct{}), wake: make(chan struct{}, 1)} + go func() { defer close(o.done); o.run(ctx) }() + return o, nil +} + +func (o *outbox) save(owner *session, r outboxRecord) error { + o.mu.Lock() + o.owners[r.Rkey] = owner + o.mu.Unlock() + return o.write(r) +} + +func (o *outbox) write(r outboxRecord) error { + data, err := json.Marshal(r) + if err != nil { + return fmt.Errorf("encode caption outbox: %w", err) + } + f, err := os.CreateTemp(o.cfg.OutboxDir, ".pending-*") + if err != nil { + return err + } + name := f.Name() + defer os.Remove(name) + if _, err = f.Write(data); err != nil { + f.Close() + return err + } + if err = f.Sync(); err != nil { + f.Close() + return err + } + if err = f.Close(); err != nil { + return err + } + return os.Rename(name, filepath.Join(o.cfg.OutboxDir, r.Rkey+".json")) +} + +func (o *outbox) remove(key string) { + if err := os.Remove(filepath.Join(o.cfg.OutboxDir, key+".json")); err != nil && !os.IsNotExist(err) { + log.Warn(context.Background(), "remove delivered caption outbox record", "error", err) + } + o.mu.Lock() + delete(o.owners, key) + o.mu.Unlock() +} + +func (o *outbox) release(owner *session) { + o.mu.Lock() + for key, current := range o.owners { + if current == owner { + delete(o.owners, key) + } + } + o.mu.Unlock() + select { + case o.wake <- struct{}{}: + default: + } +} + +func (o *outbox) run(ctx context.Context) { + var limitedUntil time.Time + for ctx.Err() == nil { + delay := time.Minute + files, err := os.ReadDir(o.cfg.OutboxDir) + if err != nil { + log.Warn(ctx, "read caption outbox", "error", err) + } + for _, file := range files { + if ctx.Err() != nil { + return + } + if file.IsDir() || !strings.HasSuffix(file.Name(), ".json") { + continue + } + key := strings.TrimSuffix(file.Name(), ".json") + o.mu.Lock() + _, owned := o.owners[key] + if !owned { + o.owners[key] = nil + } + o.mu.Unlock() + if owned { + continue + } + data, err := os.ReadFile(filepath.Join(o.cfg.OutboxDir, file.Name())) + var r outboxRecord + if err == nil { + err = json.Unmarshal(data, &r) + } + if err != nil || r.Rkey != key || r.Record == nil || r.Target.Repo == "" { + log.Warn(ctx, "invalid caption outbox record", "file", file.Name(), "error", err) + o.mu.Lock() + delete(o.owners, key) + o.mu.Unlock() + continue + } + if limitedUntil.After(r.RetryAt) { + r.RetryAt = limitedUntil + if err := o.write(r); err != nil { + log.Warn(ctx, "retain shared caption rate limit", "error", err) + } + } + wait := r.RetryAt.Sub(o.cfg.Now()) + if wait <= 0 { + uri, err := o.cfg.Publisher.Publish(ctx, r.Target, r.Rkey, r.Record) + if err == nil { + if o.cfg.Index != nil { + if err := o.cfg.Index(ctx, r.Record, uri); err != nil { + log.Warn(ctx, "index recovered caption transcript", "error", err) + } + } + o.remove(key) + continue + } + wait, limited := retryAfter(err, o.cfg.Now()) + if !limited { + wait = backoffMin + } + r.RetryAt = o.cfg.Now().Add(wait) + if limited { + limitedUntil = r.RetryAt + } + if err := o.write(r); err != nil { + log.Warn(ctx, "retain caption outbox retry", "error", err) + } + } + delay = min(delay, max(time.Millisecond, r.RetryAt.Sub(o.cfg.Now()))) + o.mu.Lock() + delete(o.owners, key) + o.mu.Unlock() + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-o.wake: + timer.Stop() + case <-timer.C: + } + } +} diff --git a/pkg/captions/records/provider.go b/pkg/captions/records/provider.go index 1bd250cba..3c41f6a50 100644 --- a/pkg/captions/records/provider.go +++ b/pkg/captions/records/provider.go @@ -122,6 +122,9 @@ func (p *Provider) Cues(ctx context.Context, video, trackID string) ([]captions. if err != nil { return nil, err } + if authoredSource(g.track.Source) { + return transcript.AuthoredCues(words), nil + } return transcript.Cues(words, p.CueOptions), nil } return nil, ErrTrackNotFound @@ -294,7 +297,13 @@ func (p *Provider) words(ctx context.Context, v *videoView, g *group) ([]transcr log.Warn(ctx, "skipping unreadable caption transcript", "uri", row.URI, "error", err) continue } - ws := transcript.Decode(transcript.Compact{Text: rec.Text, StartMs: rec.StartMs, Timings: rec.Timings}) + compact := transcript.Compact{Text: rec.Text, StartMs: rec.StartMs, Timings: rec.Timings} + var ws []transcript.Word + if authoredSource(g.track.Source) { + ws = transcript.DecodeAuthored(compact) + } else { + ws = transcript.Decode(compact) + } switch g.class { case classVideo: out = append(out, ws...) @@ -314,6 +323,10 @@ func (p *Provider) words(ctx context.Context, v *videoView, g *group) ([]transcr return out, nil } +func authoredSource(source captions.Source) bool { + return source == captions.SourceImported || source == captions.SourceHuman +} + // shiftWords moves words by delta ms and cuts them to [0, limit). func shiftWords(ws []transcript.Word, delta, limit int64) []transcript.Word { var out []transcript.Word diff --git a/pkg/captions/records/provider_test.go b/pkg/captions/records/provider_test.go index f32e32995..7aaab47bf 100644 --- a/pkg/captions/records/provider_test.go +++ b/pkg/captions/records/provider_test.go @@ -212,9 +212,9 @@ func TestProviderCues(t *testing.T) { cues, err := p.Cues(ctx, videoURI, id) require.NoError(t, err) require.Equal(t, []captions.TimedCue{ - {ID: "1", Start: 1000 * time.Millisecond, End: 2000 * time.Millisecond, Text: "First bit."}, - {ID: "2", Start: 60_000 * time.Millisecond, End: 61_000 * time.Millisecond, Text: "Second bit."}, - }, cues, "chunks are merged in time order and grouped into cues of at least a second") + {ID: "1", Start: 1000 * time.Millisecond, End: 1800 * time.Millisecond, Text: "First bit."}, + {ID: "2", Start: 60_000 * time.Millisecond, End: 60_800 * time.Millisecond, Text: "Second bit."}, + }, cues, "authored chunks retain their exact spans in time order") _, err = p.Cues(ctx, videoURI, "record-human-en-00000000") require.ErrorIs(t, err, ErrTrackNotFound) @@ -239,8 +239,10 @@ func TestProviderToleratesBrokenRecords(t *testing.T) { require.NoError(t, f.m.UpsertCaptionTranscript(context.Background(), rec, syntax.ATURI("at://"+alice+"/place.stream.caption.transcript/odd"))) cues, err := f.provider(nil).Cues(context.Background(), videoURI, TrackID(alice, "en", "captions", "human")) require.NoError(t, err) - require.Len(t, cues, 1) - require.Equal(t, "one two three", cues[0].Text) + require.Equal(t, []captions.TimedCue{ + {ID: "1", Start: 0, End: time.Second, Text: "one two"}, + {ID: "2", Start: 1100 * time.Millisecond, End: 1600 * time.Millisecond, Text: "three"}, + }, cues, "extra durations cannot extend authored cues or swallow their silence") } // The recording of a livestream VOD: two objects with ten minutes between them @@ -372,7 +374,7 @@ func TestProviderClipsUseTheirSourceVideosCaptions(t *testing.T) { cues, err := f.provider(nil).Cues(ctx, clip, TrackID(alice, "en", "captions", "human")) require.NoError(t, err) require.Equal(t, map[string][2]int64{ - "straddles": {0, 1000}, + "straddles": {0, 500}, "inside.": {2000, 3000}, }, cueTimes(cues), "shifted by the clip's start and cut to its length") diff --git a/pkg/captions/records/publisher.go b/pkg/captions/records/publisher.go index cd550efd8..2055af813 100644 --- a/pkg/captions/records/publisher.go +++ b/pkg/captions/records/publisher.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "path/filepath" "regexp" "time" @@ -127,6 +128,7 @@ func NewNodeWriter(cli *config.CLI, hub *captions.Hub, m model.Model, clients Cl NodeDID: cli.ServerDID(), Subject: LatestLivestream(m), Publisher: &RepoPublisher{CLI: cli, Clients: clients}, + OutboxDir: filepath.Join(cli.DataDir, "captions", "transcripts"), Index: func(ctx context.Context, rec *placestream.CaptionTranscript, uri string) error { return m.UpsertCaptionTranscript(ctx, *rec, syntax.ATURI(uri)) }, @@ -144,7 +146,7 @@ func retryAfter(err error, now time.Time) (time.Duration, bool) { var xe *xrpc.Error if errors.As(err, &xe) && xe.StatusCode == http.StatusTooManyRequests { if xe.Ratelimit != nil && xe.Ratelimit.Reset.After(now) { - return clampWait(xe.Ratelimit.Reset.Sub(now)), true + return xe.Ratelimit.Reset.Sub(now), true } return backoffMin, true } @@ -152,17 +154,10 @@ func retryAfter(err error, now time.Time) (time.Duration, bool) { if errors.As(err, &he) && he.Code == http.StatusTooManyRequests { if m := resetRe.FindStringSubmatch(fmt.Sprint(he.Message)); m != nil { if reset, perr := time.Parse(time.RFC3339, m[1]); perr == nil && reset.After(now) { - return clampWait(reset.Sub(now)), true + return reset.Sub(now), true } } return backoffMin, true } return 0, false } - -// clampWait keeps a rate limit's wait to something a stream can ride out: a -// reset an hour away still makes us look again every backoffMax, in case the -// limit lifted early. -func clampWait(d time.Duration) time.Duration { - return min(max(d, time.Second), backoffMax) -} diff --git a/pkg/captions/records/writer.go b/pkg/captions/records/writer.go index 63640d5e3..71dc7c84e 100644 --- a/pkg/captions/records/writer.go +++ b/pkg/captions/records/writer.go @@ -87,6 +87,8 @@ type Config struct { NodeDID string Subject SubjectResolver Publisher Publisher + // OutboxDir persists encoded records until their PDS write succeeds. + OutboxDir string // Index, when set, is called with every record that was written so it is // searchable at once, without waiting for the firehose to bring it back. Index func(ctx context.Context, rec *placestream.CaptionTranscript, uri string) error @@ -116,6 +118,9 @@ type Writer struct { mu sync.Mutex sessions map[string]*session + active sync.WaitGroup + closed bool + outbox *outbox } // NewWriter returns a Writer, or an error when cfg lacks the hub, publisher, @@ -141,7 +146,15 @@ func NewWriter(cfg Config) (*Writer, error) { if cfg.RetryDelay <= 0 { cfg.RetryDelay = 2 * time.Second } - return &Writer{cfg: cfg, sessions: map[string]*session{}}, nil + w := &Writer{cfg: cfg, sessions: map[string]*session{}} + if cfg.OutboxDir != "" { + var err error + w.outbox, err = newOutbox(cfg) + if err != nil { + return nil, err + } + } + return w, nil } // TargetFor says where a live track of a streamer is written, and whether it is @@ -172,8 +185,17 @@ func targetFor(nodeDID, streamer string, track captions.Track) (Target, bool) { // startMs counts from. Starting a streamer that already has a session is a // no-op. The session ends with StopSession, Stop, or ctx. func (w *Writer) StartSession(ctx context.Context, streamer string, mediaStart time.Time) { + w.StartSessionWithOrigin(ctx, streamer, mediaStart, true) +} + +// StartSessionWithOrigin also records whether this node ingests the stream. +// A relay never writes another node's canonical captions to the streamer's repo. +func (w *Writer) StartSessionWithOrigin(ctx context.Context, streamer string, mediaStart time.Time, origin bool) { w.mu.Lock() defer w.mu.Unlock() + if w.closed { + return + } // A session that is already ending (its stream restarted) does not count: // the new one starts alongside it, and the old one finishes its flush. if old, ok := w.sessions[streamer]; ok && old.ctx.Err() == nil { @@ -185,6 +207,7 @@ func (w *Writer) StartSession(ctx context.Context, streamer string, mediaStart t mediaStart: mediaStart.UTC().Truncate(time.Millisecond), tracks: map[string]*trackBuf{}, done: make(chan struct{}), + origin: origin, } ctx = log.WithLogValues(ctx, "system", "caption-records", "streamer", streamer) ctx, s.cancel = context.WithCancel(ctx) @@ -192,8 +215,13 @@ func (w *Writer) StartSession(ctx context.Context, streamer string, mediaStart t w.sessions[streamer] = s events := w.cfg.Hub.Subscribe(ctx, streamer) s.backfill() + w.active.Add(1) go func() { + defer w.active.Done() defer close(s.done) + if w.outbox != nil { + defer w.outbox.release(s) + } defer func() { w.mu.Lock() if w.sessions[streamer] == s { @@ -209,29 +237,43 @@ func (w *Writer) StartSession(ctx context.Context, streamer string, mediaStart t // written (or for StopTimeout to give up). It is a no-op for a streamer with no // session. func (w *Writer) StopSession(streamer string) { + if done := w.FinishSession(streamer); done != nil { + <-done + } +} + +// FinishSession cancels precisely the current session and returns its flush +// completion signal, allowing media teardown to continue without PDS I/O. +func (w *Writer) FinishSession(streamer string) <-chan struct{} { w.mu.Lock() + defer w.mu.Unlock() s := w.sessions[streamer] - w.mu.Unlock() if s == nil { - return + return nil } + s.backfill() s.cancel() - <-s.done + return s.done } // Stop ends every session, waiting for each to flush. func (w *Writer) Stop() { w.mu.Lock() + w.closed = true all := make([]*session, 0, len(w.sessions)) for _, s := range w.sessions { all = append(all, s) } w.mu.Unlock() + if w.outbox != nil { + w.outbox.cancel() + } for _, s := range all { s.cancel() } - for _, s := range all { - <-s.done + w.active.Wait() + if w.outbox != nil { + <-w.outbox.done } } @@ -242,20 +284,24 @@ type session struct { ctx context.Context // ends the session cancel context.CancelFunc done chan struct{} + origin bool mu sync.Mutex // guards tracks, which the collector and the flusher share tracks map[string]*trackBuf - retryAt time.Time // flusher only - backoff time.Duration + retryAt time.Time // flusher only + backoff time.Duration + rateLimited bool } // trackBuf is what one live track has produced and not yet written. type trackBuf struct { - track captions.Track - seen map[string]struct{} // ids of final cues already taken in - words []transcript.Word // taken in, not yet part of a record - queue []*queued // encoded records, oldest first, waiting to be written + track captions.Track + seen map[string]time.Time // recent IDs; bounded to the reconciliation window + latestEnd, cutoff time.Time + pending map[string]captions.Cue + words []transcript.Word // taken in, not yet part of a record + queue []*queued // encoded records, oldest first, waiting to be written } type queued struct { @@ -282,6 +328,9 @@ func (s *session) take(ev captions.Event) { if !ev.Cue.Final || ev.Cue.End.Before(s.mediaStart) { return } + if ev.Track.Origin == captions.OriginCanonical && !s.origin { + return + } if _, ok := targetFor(s.w.cfg.NodeDID, s.streamer, ev.Track); !ok { return } @@ -289,15 +338,37 @@ func (s *session) take(ev captions.Event) { defer s.mu.Unlock() tb, ok := s.tracks[ev.Track.ID] if !ok { - tb = &trackBuf{seen: map[string]struct{}{}} + tb = &trackBuf{seen: map[string]time.Time{}, pending: map[string]captions.Cue{}} s.tracks[ev.Track.ID] = tb } tb.track = ev.Track + if ev.Cue.End.Before(tb.cutoff) { + return + } + if ev.Cue.End.After(tb.latestEnd) { + tb.latestEnd = ev.Cue.End + } + if len(tb.seen) >= maxPendingWords { + s.pruneSeen(tb) + } + if old, ok := tb.pending[ev.Cue.ID]; ok { + if ev.Track.Origin == captions.OriginCanonical && old.Text == ev.Cue.Text && ev.Cue.End.After(old.End) { + tb.pending[ev.Cue.ID] = ev.Cue + tb.seen[ev.Cue.ID] = ev.Cue.End + } + return + } if _, dup := tb.seen[ev.Cue.ID]; dup { return } - tb.seen[ev.Cue.ID] = struct{}{} - tb.words = append(tb.words, transcript.WordsFromCue(s.mediaStart, ev.Cue)...) + if len(tb.pending) >= maxPendingWords { + return + } + if len(tb.seen) >= maxPendingWords { + return + } + tb.seen[ev.Cue.ID] = ev.Cue.End + tb.pending[ev.Cue.ID] = ev.Cue if over := len(tb.words) - maxPendingWords; over > 0 { tb.words = tb.words[over:] } @@ -333,12 +404,15 @@ func (s *session) run(ctx context.Context, events <-chan captions.Event) { fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.w.cfg.StopTimeout) defer cancel() s.flush(fctx, true) + // Final reconciliation also covers viewer-subscription overflow. The media + // teardown takes a synchronous snapshot before clearing the live hub. } // flush encodes the buffered words of every track into records and writes the // queued records. Unless final, a backoff in progress skips the writes; a final // flush retries a few times before giving up. func (s *session) flush(ctx context.Context, final bool) { + s.backfill() s.mu.Lock() ids := make([]string, 0, len(s.tracks)) for id := range s.tracks { @@ -348,7 +422,11 @@ func (s *session) flush(ctx context.Context, final bool) { sort.Strings(ids) for _, id := range ids { - s.encode(ctx, id) + s.encode(ctx, id, final) + } + if err := s.persistOutbox(); err != nil { + log.Warn(ctx, "retain caption outbox", "error", err) + return } attempts := 1 if final { @@ -362,8 +440,17 @@ func (s *session) flush(ctx context.Context, final bool) { return } } - if !final && s.w.cfg.Now().Before(s.retryAt) { - return + if s.w.cfg.Now().Before(s.retryAt) { + if !final { + return + } + if s.rateLimited { + select { + case <-time.After(s.retryAt.Sub(s.w.cfg.Now())): + case <-ctx.Done(): + return + } + } } if s.publishQueued(ctx) { s.backoff, s.retryAt = 0, time.Time{} @@ -387,9 +474,29 @@ func (s *session) pending() int { // encode turns a track's buffered words into queued records. Without a subject // to attach them to, the words stay buffered. -func (s *session) encode(ctx context.Context, trackID string) { +func (s *session) encode(ctx context.Context, trackID string, final bool) { s.mu.Lock() tb := s.tracks[trackID] + s.pruneSeen(tb) + // MUXL finals are clipped to GoPs. Keep the newest canonical cue mutable + // until another cue follows, it settles for one flush interval, or the + // session ends; otherwise a flush can permanently truncate its next piece. + var latest time.Time + if !final && tb.track.Origin == captions.OriginCanonical { + for _, cue := range tb.pending { + if cue.End.After(latest) { + latest = cue.End + } + } + } + settled := s.w.cfg.Now().Add(-s.w.cfg.FlushInterval) + for id, cue := range tb.pending { + if cue.End.Equal(latest) && cue.End.After(settled) { + continue + } + tb.words = append(tb.words, transcript.WordsFromCue(s.mediaStart, cue)...) + delete(tb.pending, id) + } if len(tb.words) == 0 { s.mu.Unlock() return @@ -500,7 +607,13 @@ func (s *session) publishQueued(ctx context.Context) bool { uri, err := s.w.cfg.Publisher.Publish(ctx, target, head.rkey, head.rec) if err != nil { s.fail(ctx, id, err) + if err := s.persistOutbox(); err != nil { + log.Warn(ctx, "retain caption outbox retry", "error", err) + } ok = false + if s.rateLimited { + return false + } break } log.Log(ctx, "wrote caption transcript", "uri", uri, "track", id, "words", len(transcript.Tokens(head.rec.Text))) @@ -516,6 +629,9 @@ func (s *session) publishQueued(ctx context.Context) bool { } func (s *session) drop(trackID string, q *queued) { + if s.w.outbox != nil { + s.w.outbox.remove(q.rkey) + } s.mu.Lock() defer s.mu.Unlock() tb := s.tracks[trackID] @@ -534,5 +650,44 @@ func (s *session) fail(ctx context.Context, trackID string, err error) { wait = s.backoff } s.retryAt = now.Add(wait) + s.rateLimited = limited log.Warn(ctx, "failed to write caption transcript, will retry", "track", trackID, "retryIn", wait.String(), "rateLimited", limited, "error", err) } + +func (s *session) pruneSeen(tb *trackBuf) { + cutoff := tb.latestEnd.Add(-s.w.cfg.Hub.Retention()) + if cutoff.After(tb.cutoff) { + tb.cutoff = cutoff + } + for id, end := range tb.seen { + if end.Before(tb.cutoff) { + if _, mutable := tb.pending[id]; !mutable { + delete(tb.seen, id) + } + } + } +} + +func (s *session) persistOutbox() error { + if s.w.outbox == nil { + return nil + } + s.mu.Lock() + var records []outboxRecord + for _, tb := range s.tracks { + target, ok := targetFor(s.w.cfg.NodeDID, s.streamer, tb.track) + if !ok { + continue + } + for _, q := range tb.queue { + records = append(records, outboxRecord{Target: target, Rkey: q.rkey, Record: q.rec, RetryAt: s.retryAt}) + } + } + s.mu.Unlock() + for _, r := range records { + if err := s.w.outbox.save(s, r); err != nil { + return err + } + } + return nil +} diff --git a/pkg/captions/records/writer_review_test.go b/pkg/captions/records/writer_review_test.go new file mode 100644 index 000000000..5d7577292 --- /dev/null +++ b/pkg/captions/records/writer_review_test.go @@ -0,0 +1,121 @@ +package records + +import ( + "context" + "fmt" + "github.com/bluesky-social/indigo/xrpc" + "net/http" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/captions" +) + +func TestWriterReconcilesDroppedFinalEventsAtTeardown(t *testing.T) { + h := newHarness(t, nil) + h.w.StartSession(context.Background(), streamer, t0) + h.w.mu.Lock() + s := h.w.sessions[streamer] + h.w.mu.Unlock() + // Block the collector, not the nonblocking viewer path, to force overflow. + s.mu.Lock() + for i := range 1000 { + h.hub.Publish(streamer, autoTrack, captions.Cue{ID: fmt.Sprint(i), Text: fmt.Sprintf("word%d", i), Start: t0.Add(time.Duration(i) * time.Millisecond), End: t0.Add(time.Duration(i+1) * time.Millisecond), Final: true}) + } + s.mu.Unlock() + h.w.StopSession(streamer) + calls := h.repos.snapshot() + total := 0 + for _, call := range calls { + total += len(decode(call.rec)) + } + require.Equal(t, 1000, total, "archival delivery reconciles the hub after a lossy viewer subscription overflows") +} + +func TestWriterShutdownWaitsForReplacedFinishingSession(t *testing.T) { + h := newHarness(t, nil) + h.repos.block = make(chan struct{}) + defer close(h.repos.block) + ctx, cancel := context.WithCancel(context.Background()) + h.w.StartSession(ctx, streamer, t0) + say(h, autoTrack, "old", 0, "old", "speech") + h.buffered(streamer, 2) + h.w.mu.Lock() + old := h.w.sessions[streamer] + h.w.mu.Unlock() + cancel() + require.Eventually(t, func() bool { return old.pending() == 1 }, time.Second, time.Millisecond) + h.w.StartSession(context.Background(), streamer, t0.Add(time.Hour)) + done := make(chan struct{}) + go func() { h.w.Stop(); close(done) }() + select { + case <-done: + t.Fatal("shutdown returned while the replaced session's transcript was still blocked") + case <-time.After(30 * time.Millisecond): + } +} + +func TestWriterRateLimitedFinalBatchSurvivesRestart(t *testing.T) { + dir := t.TempDir() + reset := time.Now().Add(2 * time.Second) + h := newHarness(t, func(cfg *Config) { cfg.OutboxDir = dir; cfg.Now = time.Now; cfg.StopTimeout = 20 * time.Millisecond }) + h.repos.fail = func(int) error { + return &xrpc.Error{StatusCode: http.StatusTooManyRequests, Ratelimit: &xrpc.RatelimitInfo{Reset: reset}} + } + h.w.StartSession(context.Background(), streamer, t0) + say(h, autoTrack, "last", 0, "last", "words") + h.w.StopSession(streamer) + h.w.Stop() + calls := h.repos.snapshot() + require.Len(t, calls, 1, "a known reset must not consume final retry attempts early") + files, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, files, 1, "the deadline leaves a durable final record") + resumed := newHarness(t, func(cfg *Config) { cfg.OutboxDir = dir; cfg.Now = time.Now }) + require.Empty(t, resumed.repos.snapshot(), "recovery also honors the saved reset") + require.Eventually(t, func() bool { return len(resumed.repos.snapshot()) == 1 }, 4*time.Second, time.Millisecond) + require.Equal(t, calls[0].rkey, resumed.repos.snapshot()[0].rkey, "recovery retains the record key") + require.Equal(t, calls[0].rec, resumed.repos.snapshot()[0].rec) + require.Eventually(t, func() bool { files, err := os.ReadDir(dir); return err == nil && len(files) == 0 }, time.Second, time.Millisecond) +} + +func TestWriterBoundsDedupHistoryWithoutRearchivingExpiredReplay(t *testing.T) { + h := newHarness(t, nil) + s := &session{w: h.w, streamer: streamer, mediaStart: t0, origin: true, tracks: map[string]*trackBuf{}} + var first captions.Event + for i := range 100 { + ev := captions.Event{Streamer: streamer, Track: autoTrack, Cue: captions.Cue{ID: fmt.Sprint(i), Text: fmt.Sprintf("word%d", i), Start: t0.Add(time.Duration(i) * time.Minute), End: t0.Add(time.Duration(i)*time.Minute + time.Second), Final: true}} + if i == 0 { + first = ev + } + s.take(ev) + s.flush(context.Background(), false) + h.advance(time.Minute) + } + require.LessOrEqual(t, len(s.tracks[autoTrack.ID].seen), 16, "deduplication retains a bounded replay window, not a stream-lifetime ledger") + s.take(first) + s.flush(context.Background(), true) + calls := h.repos.snapshot() + require.Len(t, calls, 100, "an expired replay does not create another transcript") + require.Equal(t, "word99", calls[99].rec.Text) +} + +func TestWriterExtendedCanonicalIDStaysDeduplicated(t *testing.T) { + h := newHarness(t, func(cfg *Config) { cfg.Hub = captions.NewHub(time.Second) }) + h.hub = h.w.cfg.Hub + h.w.StartSession(context.Background(), streamer, t0) + cue := captions.Cue{ID: "long", Text: "A long line", Start: t0.Add(100 * time.Millisecond), End: t0.Add(300 * time.Millisecond), Final: true} + h.hub.PublishCanonical(streamer, autoTrack, cue) + h.buffered(streamer, 3) + cue.Start = cue.End + cue.End = t0.Add(2 * time.Second) + h.hub.PublishCanonical(streamer, autoTrack, cue) + h.flushed() + h.flushed() + h.w.StopSession(streamer) + calls := h.repos.snapshot() + require.Len(t, calls, 1, "extending a canonical cue updates its deduplication lifetime, not just its pending words") + require.Equal(t, int64(2000), decode(calls[0].rec)[2].EndMs) +} diff --git a/pkg/captions/records/writer_test.go b/pkg/captions/records/writer_test.go index 4ea3983d1..f3e8cec52 100644 --- a/pkg/captions/records/writer_test.go +++ b/pkg/captions/records/writer_test.go @@ -145,6 +145,9 @@ func (h *harness) buffered(streamer string, n int) { total := 0 for _, tb := range s.tracks { total += len(tb.words) + for _, cue := range tb.pending { + total += len(transcript.WordsFromCue(s.mediaStart, cue)) + } } return total >= n }, 5*time.Second, time.Millisecond) @@ -571,10 +574,6 @@ func TestRetryAfter(t *testing.T) { require.True(t, ok) require.Equal(t, 2*time.Minute, d) - d, ok = retryAfter(echo.NewHTTPError(http.StatusTooManyRequests, "http 429 from upstream (will reset at 2026-09-30T18:00:00Z)"), now) - require.True(t, ok) - require.Equal(t, backoffMax, d, "a distant reset is looked at again sooner") - d, ok = retryAfter(fmt.Errorf("wrapped: %w", echo.NewHTTPError(http.StatusTooManyRequests, "rate-limited by upstream, but ratelimit header not found")), now) require.True(t, ok) require.Equal(t, backoffMin, d) diff --git a/pkg/captions/syndication.go b/pkg/captions/syndication.go new file mode 100644 index 000000000..ff95bddd8 --- /dev/null +++ b/pkg/captions/syndication.go @@ -0,0 +1,25 @@ +package captions + +import "encoding/json" + +// SyndicationVersion is an opt-in query capability on subscribeSegments. Older +// peers continue receiving only binary media; capable peers also receive JSON +// text frames, on the same ordered socket as the stream they validated. +const SyndicationVersion = "1" + +type sidecarMessage struct { + Type string `json:"$type"` + Event Event `json:"event"` +} + +func EncodeSidecar(ev Event) ([]byte, error) { + return json.Marshal(sidecarMessage{Type: "place.stream.caption.sidecar#event", Event: ev}) +} + +func DecodeSidecar(data []byte) (Event, bool) { + var msg sidecarMessage + if json.Unmarshal(data, &msg) != nil || msg.Type != "place.stream.caption.sidecar#event" || msg.Event.Track.Origin != OriginSidecar { + return Event{}, false + } + return msg.Event, true +} diff --git a/pkg/captions/transcript/authored.go b/pkg/captions/transcript/authored.go new file mode 100644 index 000000000..274da53d8 --- /dev/null +++ b/pkg/captions/transcript/authored.go @@ -0,0 +1,104 @@ +package transcript + +import ( + "cmp" + "slices" + "strconv" + "strings" + + "stream.place/streamplace/pkg/captions" +) + +// authoredTokens keeps each word's preceding whitespace, including line breaks. +// The compact format still has exactly one duration per whitespace token. +func authoredTokens(text string) []string { + var out []string + start, inWord := 0, false + for i, r := range text { + if isSpace(r) { + if inWord { + out = append(out, text[start:i]) + start, inWord = i, false + } + } else { + inWord = true + } + } + if inWord { + out = append(out, text[start:]) + } + return out +} + +// WordsFromCues encodes authored cue boundaries as silence, not display-layout +// hints. Adjacent cues reserve 1 ms from the preceding cue's final word so the +// next cue retains its original start and the total timeline is unchanged. +func WordsFromCues(cues []captions.TimedCue) []Word { + sorted := slices.SortedStableFunc(slices.Values(cues), func(a, b captions.TimedCue) int { + return cmp.Compare(a.Start, b.Start) + }) + var out []Word + var cursor int64 + for i, cue := range sorted { + tokens := authoredTokens(strings.TrimSpace(cue.Text)) + if len(tokens) == 0 { + continue + } + start, end := max(durationMs(cue.Start), cursor), durationMs(cue.End) + if i+1 < len(sorted) { + if next := durationMs(sorted[i+1].Start); next > start { + end = min(end, next) + } + } + end = max(start, end) + if len(out) > 0 && start == out[len(out)-1].EndMs { + last := &out[len(out)-1] + if last.EndMs-last.StartMs > 1 { + last.EndMs-- + } else { + // A positive 1 ms word cannot be shortened further. + start++ + end = max(end, start+int64(len(tokens))) + } + } + out = append(out, spread(tokens, start, end)...) + cursor = end + } + return out +} + +// ChunkAuthored applies the same record limits as Chunk without normalizing +// authored whitespace. Chunk boundaries do not become cue boundaries. +func ChunkAuthored(words []Word, opts ChunkOptions) []Compact { + return chunkTokens(words, opts, func(tokens []Word) Compact { + return encodeTokens(tokens, true) + }) +} + +// DecodeAuthored retains whitespace while decoding the standard timings. +func DecodeAuthored(c Compact) []Word { + return decodeTokens(c, authoredTokens(c.Text)) +} + +// AuthoredCues reconstructs imported/human cue boundaries only at explicit +// silence gaps. Duration, punctuation and line-length heuristics must not +// rewrite an author's timing or text. +func AuthoredCues(words []Word) []captions.TimedCue { + var out []captions.TimedCue + for _, word := range words { + if len(out) == 0 || word.StartMs > out[len(out)-1].End.Milliseconds() { + out = append(out, captions.TimedCue{ + ID: strconv.Itoa(len(out) + 1), Start: msDuration(word.StartMs), + End: msDuration(word.EndMs), Text: strings.TrimLeftFunc(word.Text, isSpace), + }) + continue + } + cue := &out[len(out)-1] + if word.Text == strings.TrimLeftFunc(word.Text, isSpace) { + cue.Text += " " + } + cue.Text += word.Text + cue.End = max(cue.End, msDuration(word.EndMs)) + } + return out +} diff --git a/pkg/captions/transcript/authored_test.go b/pkg/captions/transcript/authored_test.go new file mode 100644 index 000000000..8c7eba056 --- /dev/null +++ b/pkg/captions/transcript/authored_test.go @@ -0,0 +1,31 @@ +package transcript + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/captions" +) + +func TestAuthoredAdjacentMinimumDuration(t *testing.T) { + input := []captions.TimedCue{ + {Start: 0, End: time.Millisecond, Text: "one"}, + {Start: time.Millisecond, End: 2 * time.Millisecond, Text: "two"}, + } + chunks := ChunkAuthored(WordsFromCues(input), ChunkOptions{}) + var words []Word + for _, chunk := range chunks { + for _, timing := range chunk.Timings { + require.NotZero(t, timing, "word durations must remain positive") + } + words = append(words, DecodeAuthored(chunk)...) + } + got := AuthoredCues(words) + require.Len(t, got, 2) + for i := range input { + require.Equal(t, input[i].Text, got[i].Text) + require.InDelta(t, input[i].Start.Milliseconds(), got[i].Start.Milliseconds(), 1) + require.InDelta(t, input[i].End.Milliseconds(), got[i].End.Milliseconds(), 1) + } +} diff --git a/pkg/captions/transcript/chunk.go b/pkg/captions/transcript/chunk.go index a6ac649d1..7322cecbb 100644 --- a/pkg/captions/transcript/chunk.go +++ b/pkg/captions/transcript/chunk.go @@ -63,8 +63,11 @@ func (o ChunkOptions) withDefaults() ChunkOptions { // a chunk past MaxSpan, or past the size limits. A chunk never splits a word, // and consecutive chunks together hold every word. func Chunk(words []Word, opts ChunkOptions) []Compact { + return chunkTokens(tokenize(words), opts, Encode) +} + +func chunkTokens(toks []Word, opts ChunkOptions, encode func([]Word) Compact) []Compact { opts = opts.withDefaults() - toks := tokenize(words) for i := range toks { toks[i] = clamp(toks[i], opts.MaxTextBytes) } @@ -83,7 +86,7 @@ func Chunk(words []Word, opts ChunkOptions) []Compact { add = 2 } if textBytes+1+len(w.Text) > opts.MaxTextBytes || entries+add > opts.MaxTimings || w.EndMs-toks[first].StartMs > maxSpan { - out = append(out, Encode(toks[first:i])) + out = append(out, encode(toks[first:i])) first = i textBytes, entries, cursor = len(w.Text), 1, w.EndMs continue @@ -98,12 +101,12 @@ func Chunk(words []Word, opts ChunkOptions) []Compact { span := w.EndMs - toks[first].StartMs next := toks[i+1] if span >= target && (endsSentence(w.Text) || next.StartMs-w.EndMs >= silence) { - out = append(out, Encode(toks[first:i+1])) + out = append(out, encode(toks[first:i+1])) first = i + 1 } } if first < len(toks) { - out = append(out, Encode(toks[first:])) + out = append(out, encode(toks[first:])) } return out } diff --git a/pkg/captions/transcript/cues.go b/pkg/captions/transcript/cues.go index bf11e1b0b..eb663cb8e 100644 --- a/pkg/captions/transcript/cues.go +++ b/pkg/captions/transcript/cues.go @@ -1,7 +1,6 @@ package transcript import ( - "cmp" "slices" "strconv" "strings" @@ -110,36 +109,6 @@ func wrap(lines []string, word string, maxChars, maxLines int) ([]string, bool) return append(slices.Clone(lines), word), true } -// WordsFromCues spreads the text of cues, which carry timing only for the cue -// as a whole (an imported VTT or SRT file), over words: each cue's words share -// its span evenly. A cue that begins before the previous one has finished -// pushes its words later, and a cue that is overlapped is cut short at the -// start of the next, because words in a transcript cannot overlap. -func WordsFromCues(cues []captions.TimedCue) []Word { - sorted := slices.SortedStableFunc(slices.Values(cues), func(a, b captions.TimedCue) int { - return cmp.Compare(a.Start, b.Start) - }) - var out []Word - var cursor int64 - for i, c := range sorted { - toks := Tokens(c.Text) - if len(toks) == 0 { - continue - } - start := max(durationMs(c.Start), cursor) - end := durationMs(c.End) - if i+1 < len(sorted) { - if next := durationMs(sorted[i+1].Start); next > start { - end = min(end, next) - } - } - end = max(end, start) - out = append(out, spread(toks, start, end)...) - cursor = end - } - return out -} - // WordsFromCaptions converts live words, which carry absolute times, into // words offset from base. Times round to the nearest millisecond. func WordsFromCaptions(base time.Time, words []captions.Word) []Word { diff --git a/pkg/captions/transcript/cues_test.go b/pkg/captions/transcript/cues_test.go index 9a96dffb0..14931872d 100644 --- a/pkg/captions/transcript/cues_test.go +++ b/pkg/captions/transcript/cues_test.go @@ -178,83 +178,6 @@ func TestCuesLimits(t *testing.T) { require.Equal(t, want, got, "every word appears exactly once, in order") } -func TestWordsFromCues(t *testing.T) { - tests := []struct { - name string - cues []captions.TimedCue - want []Word - }{ - { - name: "spreads the words evenly over the cue", - cues: []captions.TimedCue{{Start: ms(1000), End: ms(2000), Text: "one two three four"}}, - want: []Word{{"one", 1000, 1250}, {"two", 1250, 1500}, {"three", 1500, 1750}, {"four", 1750, 2000}}, - }, - { - name: "rounding never leaves a hole or runs past the cue", - cues: []captions.TimedCue{{Start: ms(0), End: ms(1000), Text: "a b c"}}, - want: []Word{{"a", 0, 333}, {"b", 333, 666}, {"c", 666, 1000}}, - }, - { - name: "multi-line cue text is split on any whitespace", - cues: []captions.TimedCue{{Start: ms(0), End: ms(400), Text: "first line\nsecond line"}}, - want: []Word{{"first", 0, 100}, {"line", 100, 200}, {"second", 200, 300}, {"line", 300, 400}}, - }, - { - name: "unicode and punctuation-only tokens", - cues: []captions.TimedCue{{Start: ms(0), End: ms(300), Text: "世界 — ♪"}}, - want: []Word{{"世界", 0, 100}, {"—", 100, 200}, {"♪", 200, 300}}, - }, - { - name: "empty cues are skipped and gaps are kept", - cues: []captions.TimedCue{ - {Start: ms(0), End: ms(500), Text: "a"}, - {Start: ms(600), End: ms(900), Text: " "}, - {Start: ms(5000), End: ms(5500), Text: "b"}, - }, - want: []Word{{"a", 0, 500}, {"b", 5000, 5500}}, - }, - { - name: "cues out of order are sorted", - cues: []captions.TimedCue{ - {Start: ms(2000), End: ms(2500), Text: "second"}, - {Start: ms(0), End: ms(500), Text: "first"}, - }, - want: []Word{{"first", 0, 500}, {"second", 2000, 2500}}, - }, - { - name: "an overlapped cue is cut short at the start of the next", - cues: []captions.TimedCue{ - {Start: ms(0), End: ms(2000), Text: "a b"}, - {Start: ms(1000), End: ms(2000), Text: "c d"}, - }, - want: []Word{{"a", 0, 500}, {"b", 500, 1000}, {"c", 1000, 1500}, {"d", 1500, 2000}}, - }, - { - name: "cues that start together are laid out one after another", - cues: []captions.TimedCue{ - {Start: ms(0), End: ms(1000), Text: "a"}, - {Start: ms(0), End: ms(1000), Text: "b"}, - }, - want: []Word{{"a", 0, 1000}, {"b", 1000, 1000}}, - }, - { - name: "a zero-length cue still yields its words", - cues: []captions.TimedCue{{Start: ms(700), End: ms(700), Text: "a b"}}, - want: []Word{{"a", 700, 700}, {"b", 700, 700}}, - }, - { - name: "none", - cues: nil, - want: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - require.Equal(t, tt.want, WordsFromCues(tt.cues)) - }) - } -} - // An imported file survives the trip into a record and back out as cues: // the cue-level timing comes back within the 1ms the encoding can lose. func TestImportedCuesRoundTrip(t *testing.T) { @@ -263,13 +186,15 @@ func TestImportedCuesRoundTrip(t *testing.T) { {Start: ms(4000), End: ms(6000), Text: "Today: ünïcode, — and ♪ music ♪"}, {Start: ms(70_000), End: ms(72_000), Text: "A cue after a long silence."}, } - chunks := Chunk(WordsFromCues(in), ChunkOptions{}) + chunks := ChunkAuthored(WordsFromCues(in), ChunkOptions{}) require.Len(t, chunks, 1) var words []Word for _, c := range chunks { - words = append(words, Decode(c)...) + words = append(words, DecodeAuthored(c)...) } - for i, c := range Cues(words, CueOptions{}) { + cues := AuthoredCues(words) + require.Len(t, cues, len(in)) + for i, c := range cues { require.Equal(t, in[i].Text, c.Text) require.InDelta(t, in[i].Start.Milliseconds(), c.Start.Milliseconds(), 1) require.InDelta(t, in[i].End.Milliseconds(), c.End.Milliseconds(), 1) diff --git a/pkg/captions/transcript/transcript.go b/pkg/captions/transcript/transcript.go index 1332ab1df..0f4d7efad 100644 --- a/pkg/captions/transcript/transcript.go +++ b/pkg/captions/transcript/transcript.go @@ -65,13 +65,23 @@ func Tokens(text string) []string { // form, so it is split and its span divided evenly between the pieces; a word // with no text is dropped. func Encode(words []Word) Compact { - tokens := tokenize(words) + return encodeTokens(tokenize(words), false) +} + +func encodeTokens(tokens []Word, authored bool) Compact { if len(tokens) == 0 { return Compact{Timings: []int64{}} } text := make([]string, len(tokens)) + separator := " " + if authored { + separator = "" + } for i, t := range tokens { text[i] = t.Text + if authored && i > 0 && t.Text == strings.TrimLeftFunc(t.Text, isSpace) { + text[i] = " " + t.Text + } } startMs := tokens[0].StartMs timings := make([]int64, 0, len(tokens)) @@ -83,7 +93,7 @@ func Encode(words []Word) Compact { timings = append(timings, max(t.EndMs-t.StartMs, 1)) cursor = t.EndMs } - return Compact{Text: strings.Join(text, " "), StartMs: startMs, Timings: timings} + return Compact{Text: strings.Join(text, separator), StartMs: startMs, Timings: timings} } // tokenize makes every Word hold exactly one whitespace-free token. @@ -121,7 +131,10 @@ func spread(tokens []string, startMs, endMs int64) []Word { // Decode unpacks a compact transcript into words with their spans. func Decode(c Compact) []Word { - tokens := Tokens(c.Text) + return decodeTokens(c, Tokens(c.Text)) +} + +func decodeTokens(c Compact, tokens []string) []Word { out := make([]Word, 0, len(tokens)) cursor := c.StartMs for _, v := range c.Timings { diff --git a/pkg/cmd/e2e.go b/pkg/cmd/e2e.go index 44f575720..a7130c127 100644 --- a/pkg/cmd/e2e.go +++ b/pkg/cmd/e2e.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net" "net/http" "net/url" @@ -23,13 +24,19 @@ import ( glex "github.com/streamplace/glex/runtime" urfavecli "github.com/urfave/cli/v3" "golang.org/x/sync/errgroup" + gormlogger "gorm.io/gorm/logger" "stream.place/streamplace/pkg/aqhttp" "stream.place/streamplace/pkg/atproto" + "stream.place/streamplace/pkg/blob" spcomatproto "stream.place/streamplace/pkg/comatproto" "stream.place/streamplace/pkg/config" "stream.place/streamplace/pkg/crypto/spkey" + "stream.place/streamplace/pkg/gstinit" "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/model" "stream.place/streamplace/pkg/placestream" + "stream.place/streamplace/pkg/statedb" + "stream.place/streamplace/pkg/vod" "stream.place/streamplace/test/remote" ) @@ -51,10 +58,93 @@ func createRecord(ctx context.Context, client *xrpc.Client, collection, repo str return out.Uri, nil } +// prepareE2EVideo uses the node's real upload processing and publication paths. +// Run before forking the node, so its server repo and databases are not being +// written by two processes while we seed the fixture. +func prepareE2EVideo(ctx context.Context, dataDir, broadcasterHost, fixture, did string, client *xrpc.Client) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + started := time.Now() + // stdout is the harness's shell-readable environment protocol. Reuse the + // node's stderr logger for dependencies that otherwise log to stdout. + defaultLogger := gormlogger.Default + gormlogger.Default = config.GormLogger + defer func() { gormlogger.Default = defaultLogger }() + cli := &config.CLI{ + DataDir: dataDir, DBURL: "sqlite://" + dataDir + "/state.sqlite", + BroadcasterHost: broadcasterHost, ServerHost: broadcasterHost, + } + mod, err := model.MakeDBConns(cli.DataFilePath([]string{"index"}), 1) + if err != nil { + return "", err + } + state, err := statedb.MakeDB(ctx, cli, nil, mod) + if err != nil { + return "", err + } + sqlDB, err := state.DB.DB() + if err != nil { + return "", err + } + defer sqlDB.Close() + serverRepo, err := atproto.MakeServerRepo(ctx, cli, state) + if err != nil { + return "", err + } + defer serverRepo.Close() + store, err := blob.NewFileStore(dataDir) + if err != nil { + return "", err + } + source, err := os.Open(fixture) + if err != nil { + return "", err + } + defer source.Close() + info, err := source.Stat() + if err != nil { + return "", err + } + uploadID := uuid.NewString() + key := "uploads/" + uploadID + writer, err := store.NewWriter(ctx, key, "video/mp4") + if err != nil { + return "", err + } + defer writer.Close() + if _, err := io.Copy(writer, source); err != nil { + return "", err + } + if err := writer.Complete(); err != nil { + return "", err + } + if err := state.CreateUpload(ctx, &statedb.Upload{ + ID: uploadID, RepoDID: did, MimeType: "video/mp4", + Filename: "e2e.mp4", Size: info.Size(), Backend: "file", + Location: store.URL(key), + }); err != nil { + return "", err + } + gstinit.InitGST() + if _, err := vod.ProcessVOD(ctx, cli, state, store, vod.Input{ + UploadID: uploadID, RepoDID: did, MimeType: "video/mp4", + Filename: "e2e.mp4", Size: info.Size(), Backend: "file", + Location: store.URL(key), + }); err != nil { + return "", fmt.Errorf("process fixture: %w", err) + } + uri, _, err := vod.PublishVideoWithClient(ctx, state, store, client, did, uploadID, &placestream.Video{Title: "e2e test video"}) + if err != nil { + return "", fmt.Errorf("publish fixture: %w", err) + } + log.Log(ctx, "prepared playable e2e VOD", "uri", uri, "elapsed", time.Since(started)) + return uri, nil +} + func makeE2eCommand(build *config.BuildFlags) *urfavecli.Command { return &urfavecli.Command{ Name: "e2e", - Usage: "start a self-contained e2e test environment with a test account and live stream", + Usage: "start a self-contained e2e test environment with a test account, live stream and playable VOD", Flags: []urfavecli.Flag{ &urfavecli.StringFlag{ Name: "dev-env", @@ -76,7 +166,7 @@ func makeE2eCommand(build *config.BuildFlags) *urfavecli.Command { }, &urfavecli.IntFlag{ Name: "https-port", - Usage: "port the HTTPS front end listens on; the URLs stay portless, so anything but 443 only works where loopback 443 is redirected to it", + Usage: "port the HTTPS front end listens on; server clients use the harness proxy, while direct device clients must redirect port 443 to this port", Value: 443, Sources: urfavecli.EnvVars("SP_E2E_HTTPS_PORT"), }, @@ -253,6 +343,11 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri return err } defer os.RemoveAll(dataDir) //nolint:errcheck + fixture := remote.RemoteFixture("3188c071b354f2e548d7f2d332699758e8e3ab1600280e5b07cb67eedc64f274/BigBuckBunny_1sGOP_240p30_NoBframes.mp4") + videoURI, err := prepareE2EVideo(ctx, dataDir, broadcasterHost, fixture, out.Did, xrpcc) + if err != nil { + return fmt.Errorf("prepare playable video: %w", err) + } nodeCmd := exec.CommandContext(ctx, self) // Inherit the parent environment (dev builds need LD_LIBRARY_PATH etc.) @@ -357,30 +452,11 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri if _, err := createRecord(ctx, xrpcc, "place.stream.livestream", out.Did, &livestream); err != nil { return fmt.Errorf("create livestream record: %w", err) } - // And a VOD, so flows have a video page to open. It has no source tracks: - // its metadata (title, author) loads, but there is nothing to play. - video := placestream.Video{ - LexiconTypeID: "place.stream.video", - CreatedAt: now, - Title: "e2e test video", - DurationMs: 10_000, - Source: placestream.Video_Source{ - MediaDefs_SourceTracks: &placestream.MediaDefs_SourceTracks{ - LexiconTypeID: "place.stream.media.defs#sourceTracks", - Tracks: []spcomatproto.RepoStrongRef{}, - }, - }, - } - videoURI, err := createRecord(ctx, xrpcc, "place.stream.video", out.Did, &video) - if err != nil { - return fmt.Errorf("create video record: %w", err) - } // Give the node a moment to index the key before we start streaming. time.Sleep(1 * time.Second) // Stream a test fixture in a loop so it outlasts any Maestro test run. - fixture := remote.RemoteFixture("3188c071b354f2e548d7f2d332699758e8e3ab1600280e5b07cb67eedc64f274/BigBuckBunny_1sGOP_240p30_NoBframes.mp4") g, streamCtx := errgroup.WithContext(ctx) g.Go(func() error { for { @@ -409,8 +485,8 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri // Print the env vars for the workflow to consume, in one write: callers // poll for SERVER_URL and then read the whole file. - vars := fmt.Sprintf("SERVER_URL=http://%s\nACCOUNT_HANDLE=%s\nACCOUNT_DID=%s\nACCOUNT_PASSWORD=%s\nVIDEO_URI=%s\n", - httpAddr, out.Handle, out.Did, password, videoURI) + vars := fmt.Sprintf("SERVER_URL=http://%s\nACCOUNT_HANDLE=%s\nACCOUNT_DID=%s\nACCOUNT_PASSWORD=%s\nVIDEO_URI=%s\nSTREAM_KEY=%s\n", + httpAddr, out.Handle, out.Did, password, videoURI, priv) if tlsEnv != nil { // The same node over HTTPS at its public name, plus what clients // need to reach and trust it (see e2e_https.go): a browser pins the diff --git a/pkg/cmd/e2e_https.go b/pkg/cmd/e2e_https.go index 800bfc199..e9590bd71 100644 --- a/pkg/cmd/e2e_https.go +++ b/pkg/cmd/e2e_https.go @@ -43,7 +43,7 @@ import ( // page to a same-site navigation (Sec-Fetch-Site: same-site). // // - mints a throwaway CA and one leaf for all of those and plc.directory; -// - terminates TLS on 127.0.0.1:443 and routes by SNI: the PDS hostname and +// - terminates TLS on 127.0.0.1:443 (or --https-port) and routes by SNI: the PDS hostname and // handles to the dev-env PDS, the station hostname to the node's // plain-HTTP listener (it runs with --behind-https-proxy), plc.directory // to the dev-env PLC; @@ -76,9 +76,9 @@ const e2ePLCHost = "plc.directory" // newE2EHTTPS mints the certificates and claims every listener up front, so a // harness that cannot bind its port fails before it has started anything else. // -// The front end listens on 127.0.0.1:port. The URLs it serves stay portless -// (https://), so any port other than 443 only works on a machine that -// redirects loopback 443 to it, e.g. with an iptables REDIRECT rule. +// The front end listens on 127.0.0.1:port while public URLs stay portless. +// Server-side Go/Node clients and proxied browsers reach it through CONNECT +// even on a high port. Direct devices must redirect their port 443 to it. func newE2EHTTPS(pdsHost, stationHost string, port int) (*e2eHTTPS, error) { h := &e2eHTTPS{pdsHost: pdsHost, stationHost: stationHost} var err error @@ -128,10 +128,17 @@ func (h *e2eHTTPS) ProxyURL() string { return "http://" + h.proxyLn.Addr().Strin // hostname, and trust in our CA for what it fetches over https (the node's // OAuth client metadata, lexicons from its own public URL). func (h *e2eHTTPS) DevEnvEnv() []string { - return []string{ + env := []string{ "DEV_ENV_PDS_HOSTNAME=" + h.pdsHost, "NODE_EXTRA_CA_CERTS=" + h.caPath, } + if h.frontLn.Addr().(*net.TCPAddr).Port != 443 { + // Native clients redirect only their own port 443, not the host's. + // Node fetch needs an explicit dispatcher on the pinned Node 22. + env = append(env, "DEV_ENV_HTTPS_PROXY=true", "HTTPS_PROXY="+h.ProxyURL(), + "NO_PROXY=localhost,127.0.0.1,::1") + } + return env } // NodeEnv is the extra environment for the forked node, whose broadcaster diff --git a/pkg/cmd/e2e_https_test.go b/pkg/cmd/e2e_https_test.go new file mode 100644 index 000000000..ccd18f888 --- /dev/null +++ b/pkg/cmd/e2e_https_test.go @@ -0,0 +1,69 @@ +package cmd + +import ( + "context" + "crypto/tls" + "crypto/x509" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/aqhttp" +) + +func TestE2EHighPortTrustedHTTPS(t *testing.T) { + // ProxyFromEnvironment caches its first environment snapshot process-wide. + // Isolate it so the test works after any other HTTP tests and cannot change + // their proxy settings. + const marker = "STREAMPLACE_E2E_PROXY_TEST_CHILD" + if os.Getenv(marker) != "1" { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestE2EHighPortTrustedHTTPS$") + cmd.Env = append(os.Environ(), marker+"=1") + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + return + } + backend := func(text string) *httptest.Server { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, text) + })) + t.Cleanup(s.Close) + return s + } + pds, plc, node := backend("pds"), backend("plc"), backend("node") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + h, err := newE2EHTTPS("pds.invalid", "station.invalid", 0) + require.NoError(t, err) + defer h.Close() + h.Serve(ctx, strings.TrimPrefix(pds.URL, "http://"), strings.TrimPrefix(plc.URL, "http://"), strings.TrimPrefix(node.URL, "http://")) + t.Setenv("HTTPS_PROXY", h.ProxyURL()) + t.Setenv("NO_PROXY", "") + t.Setenv("no_proxy", "") + ca, err := os.ReadFile(h.caPath) + require.NoError(t, err) + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(ca)) + transport := aqhttp.NewTrustedTransport() + base := transport.Base.(*http.Transport) + base.TLSClientConfig = &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12} + defer base.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 5 * time.Second} + for host, want := range map[string]string{"pds.invalid": "pds", "account.pds.invalid": "pds", "plc.directory": "plc", "station.invalid": "node"} { + resp, err := client.Get("https://" + host + "/xrpc/com.atproto.sync.getLatestCommit") + require.NoError(t, err, host) + body, err := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, want, string(body), host) + } +} diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index 93ffeaa8d..99d5cb83f 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -51,7 +51,7 @@ import ( "stream.place/streamplace/pkg/spmetrics" "stream.place/streamplace/pkg/statedb" "stream.place/streamplace/pkg/storage" - _ "stream.place/streamplace/pkg/stt" // Include bundled models even when automatic captions are disabled. + "stream.place/streamplace/pkg/stt" "stream.place/streamplace/pkg/upload" "stream.place/streamplace/pkg/viewlog" "stream.place/streamplace/pkg/vod" @@ -294,6 +294,22 @@ func runMain(ctx context.Context, build *config.BuildFlags, platformJobs []jobFu if err != nil { return err } + mm.STT, err = stt.NewEngine(ctx, cli) + if err != nil { + log.Warn(ctx, "speech engine unavailable; automatic captions disabled", "error", err) + } + if mm.STT != nil { + defer mm.STT.Close() + } + stopSpeechProxy, err := mm.StartCaptionEngineProxy(ctx) + if err != nil { + return err + } + defer stopSpeechProxy() + defer mm.ShutdownCaptions() + if err := mm.ConfigureCaptionRecords(ctx, state); err != nil { + return err + } // Every new playback session counts toward the streamer's running view // total, filed under their current livestream record. mm.SetViewRecorder(func(streamer string) { diff --git a/pkg/director/stream_session.go b/pkg/director/stream_session.go index e81f6927e..a6f84baf7 100644 --- a/pkg/director/stream_session.go +++ b/pkg/director/stream_session.go @@ -102,6 +102,7 @@ func exceedsMaxBitrate(dataLen int, durationNS int64, maxBitrate int) (int, bool func (ss *StreamSession) Start(ctx context.Context, notif *media.NewSegmentNotification) error { ctx, cancel := context.WithCancel(ctx) + defer ss.mm.EndCaptionSession(notif.Segment.RepoDID) spmetrics.StreamSessions.WithLabelValues(notif.Segment.RepoDID).Inc() ss.g, ctx = errgroup.WithContext(ctx) sid := livepeer.RandomTrailer(8) diff --git a/pkg/media/auth_stream_key.go b/pkg/media/auth_stream_key.go new file mode 100644 index 000000000..9a26066c1 --- /dev/null +++ b/pkg/media/auth_stream_key.go @@ -0,0 +1,76 @@ +package media + +import ( + "context" + "crypto" + "fmt" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/decred/dcrd/dcrec/secp256k1" + "github.com/mr-tron/base58" + "stream.place/streamplace/pkg/atproto" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/model" +) + +// AuthenticateStreamKey is shared by WHIP and pushCaptions. The latter requires +// a registered place.stream.key even for the legacy did:key-only encoding. +func AuthenticateStreamKey(ctx context.Context, cli *config.CLI, mod model.Model, syncer *atproto.ATProtoSynchronizer, keyStr string, requireRegistered bool) (string, crypto.Signer, error) { + if len(keyStr) < 2 || keyStr[0] != 'z' { + return "", nil, fmt.Errorf("invalid authorization key (not a multibase base58btc string)") + } + var addrBytes, didBytes []byte + priv, err := atcrypto.ParsePrivateMultibase(keyStr) + if err == nil { + addrBytes = priv.Bytes() + } else { + decoded, err := base58.Decode(keyStr[1:]) + if err != nil { + return "", nil, fmt.Errorf("invalid authorization key (not a base58btc string)") + } + if len(decoded) < 32 { + return "", nil, fmt.Errorf("invalid authorization key (not valid secp256k1)") + } + addrBytes, didBytes = decoded[:32], decoded[32:] + priv, err = atcrypto.ParsePrivateBytesK256(addrBytes) + if err != nil { + return "", nil, fmt.Errorf("invalid authorization key (not valid atproto): %w", err) + } + } + key, _ := secp256k1.PrivKeyFromBytes(addrBytes) + var signer crypto.Signer = key.ToECDSA() + pub, err := priv.PublicKey() + if err != nil { + return "", nil, fmt.Errorf("invalid authorization key (could not parse as atproto): %w", err) + } + did := string(didBytes) + if did != "" { + repo, err := syncer.SyncBlueskyRepo(ctx, did, mod) + if err != nil { + return "", nil, fmt.Errorf("could not resolve streamplace key: %w", err) + } + did = repo.DID + } else { + did = pub.DIDKey() + } + if err := cli.StreamIsAllowed(did); err != nil { + return "", nil, fmt.Errorf("user is not allowed to stream: %w", err) + } + if len(didBytes) > 0 || requireRegistered { + signingKey, err := mod.GetSigningKey(ctx, pub.DIDKey(), did) + if err != nil { + return "", nil, fmt.Errorf("signing key not found: %w", err) + } + if signingKey == nil { + return "", nil, fmt.Errorf("signing key not found") + } + } + labels, err := mod.GetActiveLabels(did) + if err != nil { + return "", nil, fmt.Errorf("failed to get active labels: %w", err) + } + if atproto.IsBanned(labels...) { + return "", nil, fmt.Errorf("user is banned") + } + return did, signer, nil +} diff --git a/pkg/media/auth_stream_key_test.go b/pkg/media/auth_stream_key_test.go new file mode 100644 index 000000000..163d79661 --- /dev/null +++ b/pkg/media/auth_stream_key_test.go @@ -0,0 +1,74 @@ +package media + +import ( + "context" + "fmt" + "testing" + + "github.com/mr-tron/base58" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/atproto" + "stream.place/streamplace/pkg/comatproto" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/crypto/spkey" + "stream.place/streamplace/pkg/model" +) + +type captionAuthModel struct { + model.Model + key *model.SigningKey + keyErr error + labels []*comatproto.LabelDefs_Label +} + +func (m *captionAuthModel) GetSigningKey(_ context.Context, did, repo string) (*model.SigningKey, error) { + if m.key != nil && (m.key.DID != did || m.key.RepoDID != repo) { + return nil, nil + } + return m.key, m.keyErr +} +func (m *captionAuthModel) GetActiveLabels(string) ([]*comatproto.LabelDefs_Label, error) { + return m.labels, nil +} + +func TestCaptionStreamKeyAuthorization(t *testing.T) { + priv, pub, err := spkey.GenerateStreamKey() + require.NoError(t, err) + key := "z" + base58.Encode(priv.Bytes()) + did := pub.DIDKey() + registered := &model.SigningKey{DID: did, RepoDID: did} + for _, tc := range []struct { + name string + cli config.CLI + mod captionAuthModel + key string + err string + }{ + {name: "registered", cli: config.CLI{WideOpen: true}, mod: captionAuthModel{key: registered}, key: key}, + {name: "unregistered", cli: config.CLI{WideOpen: true}, key: key, err: "signing key not found"}, + {name: "revoked", cli: config.CLI{WideOpen: true}, mod: captionAuthModel{keyErr: fmt.Errorf("signing key revoked")}, key: key, err: "signing key revoked"}, + {name: "banned", cli: config.CLI{WideOpen: true}, mod: captionAuthModel{key: registered, labels: []*comatproto.LabelDefs_Label{{Uri: did, Val: atproto.LabelDMCAViolation}}}, key: key, err: "user is banned"}, + {name: "disallowed", cli: config.CLI{AllowedStreams: []string{"did:plc:someoneelse"}}, mod: captionAuthModel{key: registered}, key: key, err: "user is not allowed to stream"}, + {name: "malformed short key", cli: config.CLI{WideOpen: true}, key: "z2", err: "invalid authorization key"}, + {name: "malformed base58", cli: config.CLI{WideOpen: true}, key: "z0", err: "invalid authorization key"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, signer, err := AuthenticateStreamKey(context.Background(), &tc.cli, &tc.mod, nil, tc.key, true) + if tc.err != "" { + require.ErrorContains(t, err, tc.err) + require.Empty(t, got) + require.Nil(t, signer) + return + } + require.NoError(t, err) + require.Equal(t, did, got) + parsed, err := spkey.KeyToSigner(priv) + require.NoError(t, err) + require.Equal(t, parsed.Public(), signer.Public()) + }) + } + // WHIP retains its legacy unregistered did:key mode; caption pushes do not. + got, _, err := AuthenticateStreamKey(context.Background(), &config.CLI{WideOpen: true}, &captionAuthModel{}, nil, key, false) + require.NoError(t, err) + require.Equal(t, did, got) +} diff --git a/pkg/media/captions_audio.go b/pkg/media/captions_audio.go index b675fcf9a..1059ade9d 100644 --- a/pkg/media/captions_audio.go +++ b/pkg/media/captions_audio.go @@ -93,7 +93,7 @@ func buildCaptionAudioPipeline(codec string) (*gst.Pipeline, error) { } pipeline, err := gst.NewPipelineFromString(strings.Join([]string{ "appsrc name=src ! qtdemux name=demux", - constants.Queue2Big + " name=vq ! fakesink sync=false", + constants.Queue2Big + " name=vq ! fakesink sync=false async=false", fmt.Sprintf("%s name=aq ! %s ! audioconvert ! audioresample ! audio/x-raw,format=F32LE,rate=%d,channels=1,layout=interleaved ! appsink name=sink sync=false", constants.Queue2Big, decode, stt.SampleRate), }, "\n")) if err != nil { diff --git a/pkg/media/captions_buffer.go b/pkg/media/captions_buffer.go new file mode 100644 index 000000000..f6a8f1a16 --- /dev/null +++ b/pkg/media/captions_buffer.go @@ -0,0 +1,129 @@ +package media + +import ( + "bytes" + "context" + "io" + "sync" + "time" + + "stream.place/streamplace/pkg/log" +) + +type arrivedBytes struct { + data []byte + at time.Time +} + +// ingestByteBuffer allows a normal caption hold without stalling appsink. +// Its bounded staging window backpressures a persistently slower consumer. +// Graceful Close drains queued media; CloseWithError discards it immediately. +type ingestByteBuffer struct { + mu sync.Mutex + ready *sync.Cond + chunks []arrivedBytes + offset int + closed bool + err error + bytes int + pressured bool + ctx context.Context + lastRead time.Time +} + +const ingestByteCapacity = 32 * 1024 * 1024 +const ingestByteChunk = 64 * 1024 + +func newIngestByteBuffer(contexts ...context.Context) *ingestByteBuffer { + ctx := context.Background() + if len(contexts) > 0 { + ctx = contexts[0] + } + b := &ingestByteBuffer{ctx: ctx} + b.ready = sync.NewCond(&b.mu) + return b +} +func (b *ingestByteBuffer) Write(p []byte) (int, error) { + return b.writeAt(p, time.Now()) +} +func (b *ingestByteBuffer) writeAt(p []byte, at time.Time) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + written := 0 + for len(p) > 0 { + for b.bytes == ingestByteCapacity && !b.closed { + if !b.pressured { + b.pressured = true + log.Warn(b.ctx, "caption media staging capacity reached; applying backpressure", "bytes", b.bytes) + } + b.ready.Wait() + } + if b.closed { + return written, io.ErrClosedPipe + } + n := min(len(p), ingestByteChunk, ingestByteCapacity-b.bytes) + b.chunks = append(b.chunks, arrivedBytes{data: bytes.Clone(p[:n]), at: at}) + b.bytes += n + written += n + p = p[n:] + b.ready.Broadcast() + } + if b.closed && written == 0 { + return 0, io.ErrClosedPipe + } + return written, nil +} +func (b *ingestByteBuffer) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + b.mu.Lock() + defer b.mu.Unlock() + for len(b.chunks) == 0 && !b.closed { + b.ready.Wait() + } + if len(b.chunks) == 0 { + if b.err != nil { + return 0, b.err + } + return 0, io.EOF + } + b.lastRead = b.chunks[0].at + n := copy(p, b.chunks[0].data[b.offset:]) + b.offset += n + if b.offset == len(b.chunks[0].data) { + b.bytes -= len(b.chunks[0].data) + b.chunks[0] = arrivedBytes{} + b.chunks = b.chunks[1:] + b.offset = 0 + b.ready.Broadcast() + } + return n, nil +} +func (b *ingestByteBuffer) Close() error { return b.CloseWithError(nil) } +func (b *ingestByteBuffer) CloseWithError(err error) error { + b.mu.Lock() + defer b.mu.Unlock() + b.closed = true + b.err = err + if err != nil { + b.chunks = nil + b.offset = 0 + b.bytes = 0 + } + b.ready.Broadcast() + return nil +} + +func (b *ingestByteBuffer) LastReadTime() time.Time { + b.mu.Lock() + defer b.mu.Unlock() + return b.lastRead +} + +func captionReadTime(input io.Reader) time.Time { + if timed, ok := input.(interface{ LastReadTime() time.Time }); ok { + return timed.LastReadTime() + } + return time.Now() +} diff --git a/pkg/media/captions_buffer_test.go b/pkg/media/captions_buffer_test.go new file mode 100644 index 000000000..a10d39935 --- /dev/null +++ b/pkg/media/captions_buffer_test.go @@ -0,0 +1,43 @@ +package media + +import ( + "bytes" + "context" + "io" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCaptionBufferBackpressureAndAbort(t *testing.T) { + b := newIngestByteBuffer() + prefix := bytes.Repeat([]byte{'a'}, 32*1024*1024) + _, err := b.Write(prefix) + require.NoError(t, err) + finished := make(chan error, 1) + started := make(chan struct{}) + go func() { close(started); _, err := b.Write([]byte("tail")); finished <- err }() + <-started + select { + case err := <-finished: + t.Fatalf("full queue must backpressure, write returned %v", err) + case <-time.After(20 * time.Millisecond): + } + got := make([]byte, 64*1024) + _, err = io.ReadFull(b, got) + require.NoError(t, err) + require.Equal(t, prefix[:len(got)], got) + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("reading must release a blocked producer") + } + require.NoError(t, b.CloseWithError(context.Canceled)) + got, err = io.ReadAll(b) + require.ErrorIs(t, err, context.Canceled) + require.Empty(t, got, "abort must release queued storage") + _, err = b.Write([]byte("after abort")) + require.ErrorIs(t, err, io.ErrClosedPipe) +} diff --git a/pkg/media/captions_clock_test.go b/pkg/media/captions_clock_test.go new file mode 100644 index 000000000..faaaf72c7 --- /dev/null +++ b/pkg/media/captions_clock_test.go @@ -0,0 +1,128 @@ +package media + +import ( + "bytes" + "context" + "encoding/json" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/captions/fmp4" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/muxl" +) + +func signedMediaDate(t *testing.T, ctx context.Context, data []byte) time.Time { + t.Helper() + output, err := muxl.RunMuxlVerify(ctx, bytes.NewReader(data)) + require.NoError(t, err) + var doc struct { + Segments []struct { + Manifest struct { + Assertions []struct { + Label string `json:"label"` + Data map[string]json.RawMessage `json:"data"` + } `json:"assertions"` + } `json:"manifest"` + } `json:"segments"` + } + require.NoError(t, json.Unmarshal([]byte(output), &doc)) + for _, segment := range doc.Segments { + for _, a := range segment.Manifest.Assertions { + if a.Label == "cawg.metadata" { + var date string + require.NoError(t, json.Unmarshal(a.Data["dc:date"], &date)) + parsed, err := time.Parse(time.RFC3339Nano, date) + require.NoError(t, err) + return parsed + } + } + } + t.Fatal("signed segment has no media date") + return time.Time{} +} +func TestCaptionMasterSignedDatesFollowMediaNotSigning(t *testing.T) { + ctx := context.Background() + fixture, err := os.ReadFile(getFixture("h264-opus-frag.mp4")) + require.NoError(t, err) + tracks, err := fmp4.Tracks(fixture) + require.NoError(t, err) + var timescale uint32 + for _, track := range tracks { + if track.ID == 1 { + timescale = track.Timescale + } + } + mm := NewOffline(&config.CLI{}) + ms := newBareSegmentSigner(t) + ms.PrebuiltManifest = captionManifest("off") + events := make(chan *muxl.MuxlEvent, 16) + done := make(chan error, 1) + input := newIngestByteBuffer() + wall := time.Now() + source := bytes.NewReader(fixture) + var moof []byte + for source.Len() > 0 { + box, kind, err := readCaptionBox(source) + require.NoError(t, err) + at := wall + if kind == "moof" { + moof = box + } + if kind == "mdat" && len(moof) > 0 { + fragments, err := fmp4.Fragments(append(moof, box...)) + require.NoError(t, err) + for _, fragment := range fragments { + for _, track := range tracks { + if fragment.TrackID == track.ID { + at = wall.Add(time.Duration(fragment.BaseDecodeTime) * time.Second / time.Duration(track.Timescale)) + break + } + } + break + } + } + _, err = input.writeAt(box, at) + require.NoError(t, err) + } + require.NoError(t, input.Close()) + var signed []*muxl.MuxlEvent + go func() { done <- mm.SignOriginStream(ctx, ms, input, events); close(events) }() + for event := range events { + if event.Type == "signed-segment" { + signed = append(signed, event) + } + } + require.NoError(t, <-done) + require.Len(t, signed, 2) + first := signedMediaDate(t, ctx, signed[0].Tracks["1"]) + second := signedMediaDate(t, ctx, signed[1].Tracks["1"]) + expected := time.Duration((signed[1].FirstDecodeTimes["1"]-signed[0].FirstDecodeTimes["1"])*1000/uint64(timescale)) * time.Millisecond + require.Equal(t, expected, second.Sub(first), "queued GoPs must retain their media-clock spacing") +} + +func TestCaptionMasterClockReanchorsPushAndSignedTimeTogether(t *testing.T) { + ctx := context.Background() + m := newCaptionMaster(ctx, "streamer", &config.CLI{}, nil) + m.setManifest(captionManifest("ingest")) + m.mediaFinished = true + wall := time.UnixMilli(1800000000000) + m.clockAt(time.UnixMilli(0), wall) + m.closeGopAt(1000, wall.Add(time.Second)) + require.Equal(t, wall.Add(time.Second), m.segmentTime(1000)) + m.closeGopAt(2000, wall.Add(5*time.Second)) + segmentWall := m.segmentTime(2000) + require.Equal(t, wall.Add(5*time.Second), segmentWall) + start, end := segmentWall.Add(200*time.Millisecond), segmentWall.Add(400*time.Millisecond) + require.NoError(t, m.push(masterTrack(captions.SourceHuman), []captions.Cue{{ID: "reanchored", Start: start, End: end, Text: "after stall", Final: true}})) + attached, err := m.text(ctx, muxl.TextRequest{StartMs: 2000, EndMs: 3000}) + require.NoError(t, err) + require.Len(t, attached.Tracks, 1) + require.Len(t, attached.Tracks[0].Cues, 1) + cue := attached.Tracks[0].Cues[0] + require.Equal(t, start, segmentWall.Add(time.Duration(cue.Start-2000)*time.Millisecond)) + require.Equal(t, end, segmentWall.Add(time.Duration(cue.End-2000)*time.Millisecond)) +} diff --git a/pkg/media/captions_distribution.go b/pkg/media/captions_distribution.go new file mode 100644 index 000000000..4a9b3fc54 --- /dev/null +++ b/pkg/media/captions_distribution.go @@ -0,0 +1,405 @@ +package media + +import ( + "bytes" + "context" + "fmt" + "strconv" + "sync" + "time" + + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/captions/fmp4" + "stream.place/streamplace/pkg/captions/records" + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/muxl" + "stream.place/streamplace/pkg/statedb" +) + +type captionDistribution struct { + mu sync.Mutex + ctx context.Context + writer *records.Writer + streams map[string]*captionStream + pending map[string][]captions.Event + closed bool + workers sync.WaitGroup + shutdown sync.Once +} + +type captionSegment struct { + vs *validatedSegment + segment []byte + media time.Duration + canonical bool +} + +type captionStream struct { + ctx context.Context + cancel context.CancelFunc + queue chan captionSegment + done chan struct{} + upstreamSignal chan struct{} + // Protected by distribution.mu. + upstream, canonical, published, recording bool + policy captions.Policy + // Owned by the stream worker. + decoder *captionAudioDecoder + recognizer *captions.Recognizer + recCancel context.CancelFunc + recDone chan struct{} + unavailable bool +} + +func (mm *MediaManager) captionState() *captionDistribution { + mm.captionDistributionOnce.Do(func() { + mm.captionDistribution = &captionDistribution{ + ctx: context.Background(), streams: map[string]*captionStream{}, + pending: map[string][]captions.Event{}, + } + }) + return mm.captionDistribution +} + +// ConfigureCaptionRecords binds transcript writes to the stored streamer OAuth +// sessions and the node server repo, before any media arrives. +func (mm *MediaManager) ConfigureCaptionRecords(ctx context.Context, state *statedb.StatefulDB) error { + d := mm.captionState() + w, err := records.NewNodeWriter(mm.cli, mm.bus.Captions, mm.model, records.SessionClients(state)) + if err != nil { + return err + } + d.ctx, d.writer = ctx, w + go func() { + <-ctx.Done() + mm.ShutdownCaptions() + }() + return nil +} + +func (mm *MediaManager) distributeCaptions(ctx context.Context, vs *validatedSegment, segment, header []byte) { + if mm.bus == nil || mm.bus.Captions == nil { + return + } + media, canonical, err := captions.SegmentClock(header, segment) + if err != nil { + log.Warn(ctx, "classify canonical captions", "error", err) + } + d := mm.captionState() + d.mu.Lock() + if d.closed || d.ctx.Err() != nil { + d.mu.Unlock() + return + } + s := d.streams[vs.repoDID] + // Stop/go-live can leave the encoder connected in preview. A new live + // session must not inherit the old cue IDs, clock, or transcript subject. + if s != nil && s.published && !vs.meta.Published { + mm.endCaptionSessionLocked(d, vs.repoDID) + s = nil + } + if s == nil { + sctx, cancel := context.WithCancel(context.WithoutCancel(d.ctx)) + s = &captionStream{ + ctx: sctx, cancel: cancel, queue: make(chan captionSegment, 32), + upstreamSignal: make(chan struct{}, 1), done: make(chan struct{}), + upstream: len(d.pending[vs.repoDID]) > 0, + } + d.streams[vs.repoDID] = s + d.workers.Add(1) + go func() { defer d.workers.Done(); mm.runCaptionStream(vs.repoDID, s) }() + } + if canonical { + s.canonical = true + mm.bus.Captions.RemoveOrigin(vs.repoDID, captions.OriginSidecar) + } + s.policy = captions.PolicyFromMetadata(vs.meta.MetadataConfiguration) + s.published = vs.meta.Published + if vs.meta.Published && !s.recording && d.writer != nil { + d.writer.StartSessionWithOrigin(context.WithoutCancel(d.ctx), vs.repoDID, vs.meta.StartTime.Time(), vs.local) + s.recording = true + } + select { + case s.queue <- captionSegment{vs: vs, segment: segment, media: media, canonical: canonical && err == nil}: + default: + log.Warn(ctx, "caption distribution behind, dropping segment", "streamer", vs.repoDID) + } + d.mu.Unlock() +} + +func (mm *MediaManager) runCaptionStream(streamer string, s *captionStream) { + defer close(s.done) + defer s.stopRecognition() + for { + var seg captionSegment + select { + case <-s.ctx.Done(): + d := mm.captionState() + d.mu.Lock() + drain := d.closed && d.streams[streamer] == s + d.mu.Unlock() + if !drain { + return + } + select { + case seg = <-s.queue: + default: + return + } + case <-s.upstreamSignal: + s.stopRecognition() + continue + case seg = <-s.queue: + } + var events []captions.Event + var err error + if seg.canonical { + events, _, err = captions.ReadCanonicalWithClock(context.WithoutCancel(s.ctx), seg.segment, seg.media, seg.vs.meta.StartTime.Time(), streamer) + } + if err != nil { + log.Warn(s.ctx, "extract canonical captions", "error", err, "streamer", streamer) + } + d := mm.captionState() + d.mu.Lock() + if d.streams[streamer] != s || (s.ctx.Err() != nil && !d.closed) { + d.mu.Unlock() + return + } + incoming := s.canonical || s.upstream + decision := captions.Decide(captions.Situation{Policy: s.policy, Origin: seg.vs.local, NodeCaptions: mm.cli.Captions, IncomingCaptions: incoming}) + if s.policy.AllowNodeCaptions && !s.canonical { + for _, ev := range d.pending[streamer] { + mm.bus.Captions.Publish(streamer, ev.Track, ev.Cue) + } + } + delete(d.pending, streamer) + for _, ev := range events { + mm.bus.Captions.PublishCanonical(streamer, ev.Track, ev.Cue) + } + d.mu.Unlock() + if s.ctx.Err() != nil || incoming || !decision.Recognize() || decision.Origin != captions.OriginSidecar { + s.stopRecognition() + continue + } + if s.unavailable || mm.STT == nil { + continue + } + if s.recognizer == nil { + if err := mm.startSidecar(streamer, s, decision); err != nil { + s.unavailable = true + log.Debug(s.ctx, "sidecar recognition unavailable", "error", err) + continue + } + } + if err := s.feedAudio(seg); err != nil { + log.Warn(s.ctx, "sidecar audio decode", "error", err) + s.stopRecognition() + s.unavailable = true + } + } +} + +func (mm *MediaManager) startSidecar(streamer string, s *captionStream, decision captions.Decision) error { + // Gate publication separately from the recognizer: a pass already running + // when an upstream track arrives must not publish a competing late final. + hub := captions.NewHub(0) + ctx, cancel := context.WithCancel(s.ctx) + events := hub.Subscribe(ctx, streamer) + r, err := captions.NewRecognizer(ctx, captions.RecognizerOptions{ + Streamer: streamer, Origin: captions.OriginSidecar, Author: mm.cli.ServerDID(), + Languages: decision.Languages, Hub: hub, Engine: mm.STT, + }) + if err != nil { + cancel() + return err + } + s.recognizer, s.recCancel, s.recDone = r, cancel, make(chan struct{}) + go func() { + defer close(s.recDone) + for ev := range events { + d := mm.captionState() + d.mu.Lock() + if d.streams[streamer] == s && !s.upstream && !s.canonical && s.policy.AllowNodeCaptions && ctx.Err() == nil { + mm.bus.Captions.Publish(streamer, ev.Track, ev.Cue) + } + d.mu.Unlock() + } + }() + return nil +} + +func (s *captionStream) stopRecognition() { + if s.recCancel != nil { + s.recCancel() + } + if s.decoder != nil { + s.decoder.close() + s.decoder = nil + } + if s.recognizer != nil { + s.recognizer.Close() + s.recognizer = nil + <-s.recDone + s.recCancel = nil + } +} + +func (s *captionStream) feedAudio(seg captionSegment) error { + events, err := unwrapMuxlEvents(s.ctx, seg.segment) + if err != nil { + return err + } + cat, tracks := catalogAndTracks(events) + if cat == nil || cat.Audio == nil { + return nil + } + var chosen, scale uint32 + var codec string + for _, a := range cat.Audio.Renditions { + if isAACCodec(a.Codec) { + chosen, scale, codec = a.TrackID(), a.Timescale(), "aac" + break + } + if isOpusCodec(a.Codec) { + chosen, scale, codec = a.TrackID(), a.Timescale(), "opus" + } + } + if chosen == 0 || scale == 0 { + return nil + } + audio := tracks[strconv.FormatUint(uint64(chosen), 10)] + frags, err := fmp4.Fragments(audio) + if err != nil { + return err + } + if len(frags) == 0 { + return fmt.Errorf("missing caption audio fragment") + } + tfdt := frags[0].BaseDecodeTime + selected := map[string][]byte{strconv.FormatUint(uint64(chosen), 10): audio} + if cat.Video != nil { + for _, v := range cat.Video.Renditions { + id := strconv.FormatUint(uint64(v.TrackID()), 10) + selected[id] = tracks[id] + } + } + decode := concatTracksByID(selected) + if s.decoder != nil && (s.decoder.codec != codec || s.decoder.backwards(tfdt) || s.decoder.failed()) { + s.decoder.close() + s.decoder = nil + } + var init bytes.Buffer + if s.decoder == nil { + if err := muxl.RunMuxlWrapInit(s.ctx, bytes.NewReader(decode), &init); err != nil { + return err + } + s.decoder, err = newCaptionAudioDecoder(s.ctx, codec, s.recognizer.Push) + if err != nil { + return err + } + } + media := time.Duration(tfdt/uint64(scale))*time.Second + time.Duration(tfdt%uint64(scale))*time.Second/time.Duration(scale) + return s.decoder.feed(init.Bytes(), decode, tfdt, media, seg.vs.meta.StartTime.Time()) +} + +// EndCaptionSession is the single teardown entry point, on timeout, shutdown, +// or the published livestream returning to preview. PDS I/O never blocks it. +func (mm *MediaManager) EndCaptionSession(streamer string) { + d := mm.captionState() + d.mu.Lock() + mm.endCaptionSessionLocked(d, streamer) + d.mu.Unlock() +} + +func (mm *MediaManager) endCaptionSessionLocked(d *captionDistribution, streamer string) { + s := d.streams[streamer] + delete(d.streams, streamer) + delete(d.pending, streamer) + if s != nil { + s.cancel() + } + if d.writer != nil { + d.writer.FinishSession(streamer) + } + if mm.bus != nil && mm.bus.Captions != nil { + mm.bus.Captions.EndSession(streamer) + } +} + +// CaptionSyndicationAllowed enforces the streamer's signed opt-out for both +// locally generated and passed-through sidecars. +func (mm *MediaManager) CaptionSyndicationAllowed(streamer string) bool { + d := mm.captionState() + d.mu.Lock() + defer d.mu.Unlock() + s := d.streams[streamer] + return s != nil && s.policy.AllowNodeCaptions +} + +// ReceiveSidecar accepts the connected upstream's stream and keeps its author. +// A short replay arriving before its first validated segment is held until the +// segment's signed policy has been checked; it is never exposed prematurely. +func (mm *MediaManager) ReceiveSidecar(streamer, upstream string, ev captions.Event) bool { + if upstream == "" || upstream == mm.cli.ServerDID() || ev.Streamer != streamer || ev.Track.Origin != captions.OriginSidecar || ev.Cue.ID == "" || !ev.Cue.End.After(ev.Cue.Start) { + return false + } + switch ev.Track.Source { + case captions.SourceAuto, captions.SourceIngest, captions.SourceHuman, captions.SourceImported: + default: + return false + } + if ev.Track.Language == "" { + ev.Track.Language = "und" + } + if len(ev.Track.Language) > 64 { + return false + } + ev.Track.ID = captions.TrackID(captions.OriginSidecar, ev.Track.Source, ev.Track.Language) + ev.Track.Author = upstream + d := mm.captionState() + d.mu.Lock() + defer d.mu.Unlock() + if d.closed || d.ctx.Err() != nil { + return false + } + s := d.streams[streamer] + if s == nil { + if len(d.pending[streamer]) < 256 { + d.pending[streamer] = append(d.pending[streamer], ev) + } + return true + } + if !s.policy.AllowNodeCaptions || s.canonical { + return false + } + s.upstream = true + select { + case s.upstreamSignal <- struct{}{}: + default: + } + mm.bus.Captions.Publish(streamer, ev.Track, ev.Cue) + return true +} + +// ShutdownCaptions stops admission and waits for workers, including already +// ending sessions, before waiting for all transcript final flushes. +func (mm *MediaManager) ShutdownCaptions() { + d := mm.captionState() + d.shutdown.Do(func() { + d.mu.Lock() + d.closed = true + for _, s := range d.streams { + s.cancel() + } + d.mu.Unlock() + d.workers.Wait() + d.mu.Lock() + for id := range d.streams { + mm.endCaptionSessionLocked(d, id) + } + d.mu.Unlock() + if d.writer != nil { + d.writer.Stop() + } + }) +} diff --git a/pkg/media/captions_distribution_review_test.go b/pkg/media/captions_distribution_review_test.go new file mode 100644 index 000000000..8675c28bb --- /dev/null +++ b/pkg/media/captions_distribution_review_test.go @@ -0,0 +1,105 @@ +package media + +import ( + "bytes" + "context" + "testing" + "time" + + upstream "github.com/streamplace/muxl/go" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/aqtime" + "stream.place/streamplace/pkg/bus" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/captions/records" + "stream.place/streamplace/pkg/comatproto" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/placestream" +) + +func TestDistributionSidecarCannotClaimCanonicalNamespace(t *testing.T) { + mm := &MediaManager{cli: &config.CLI{BroadcasterHost: "node.example"}, bus: bus.NewBus()} + d := mm.captionState() + d.streams["alice"] = &captionStream{policy: captions.Policy{AllowNodeCaptions: true}, upstreamSignal: make(chan struct{}, 1)} + now := time.Now() + ev := captions.Event{Streamer: "alice", Track: captions.Track{ID: "canonical-auto-en", Origin: captions.OriginSidecar, Source: captions.SourceAuto, Language: "en", Author: mm.cli.ServerDID()}, Cue: captions.Cue{ID: "forged", Text: "Peer speech", Start: now, End: now.Add(time.Second), Final: true}} + require.True(t, mm.ReceiveSidecar("alice", "did:web:upstream.example", ev)) + tracks := mm.bus.Captions.Tracks("alice") + require.Equal(t, "sidecar-auto-en", tracks[0].ID) + require.Equal(t, "did:web:upstream.example", tracks[0].Author, "the connected peer, not its claimed author, determines provenance") +} + +func TestDistributionCanonicalAdmissionPrecedesSidecarFrames(t *testing.T) { + seg, _ := distributionFixture(t) + eng, err := captions.TextEngine() + require.NoError(t, err) + canonical, err := eng.AddTextTrack(context.Background(), seg, upstream.TextTrack{TrackID: 9, Language: "en", Label: "ingest"}, nil) + require.NoError(t, err) + var hdr bytes.Buffer + require.NoError(t, eng.Wrap(context.Background(), bytes.NewReader(canonical), "flat", &hdr)) + mm := &MediaManager{cli: &config.CLI{Captions: true}, bus: bus.NewBus()} + allowed := true + now := time.Now() + vs := &validatedSegment{repoDID: "alice", meta: &SegmentMetadata{StartTime: aqtime.FromTime(now), Published: true, MetadataConfiguration: &placestream.MetadataConfiguration{CaptionPolicy: &placestream.MetadataCaptionPolicy{AllowNodeCaptions: &allowed}}}} + mm.distributeCaptions(context.Background(), vs, canonical, hdr.Bytes()) + d := mm.captionState() + d.mu.Lock() + s := d.streams["alice"] + d.mu.Unlock() + defer func() { mm.EndCaptionSession("alice"); <-s.done }() + ev := captions.Event{Streamer: "alice", Track: captions.Track{ID: "sidecar-auto-en", Origin: captions.OriginSidecar, Source: captions.SourceAuto, Language: "en", Author: "did:web:upstream.example"}, Cue: captions.Cue{ID: "peer", Text: "Competing", Start: now, End: now.Add(time.Second), Final: true}} + require.False(t, mm.ReceiveSidecar("alice", "did:web:upstream.example", ev), "the frame following canonical media must not enter the hub while extraction runs") + require.Empty(t, mm.bus.Captions.Tracks("alice")) +} + +func TestDistributionShutdownReleasesSpeechAndStopsAdmission(t *testing.T) { + seg, header := distributionFixture(t) + engine := &distributionEngine{leased: make(chan struct{}, 4), released: make(chan struct{}, 4)} + mm := &MediaManager{cli: &config.CLI{Captions: true, BroadcasterHost: "node.example"}, bus: bus.NewBus(), STT: engine} + vs := &validatedSegment{repoDID: "alice", meta: &SegmentMetadata{StartTime: aqtime.FromTime(time.Now()), Published: true}} + mm.distributeCaptions(context.Background(), vs, seg, header) + select { + case <-engine.leased: + case <-time.After(15 * time.Second): + t.Fatal("speech did not start") + } + mm.ShutdownCaptions() + select { + case <-engine.released: + default: + t.Fatal("caption shutdown returned before releasing speech") + } + require.NoError(t, engine.Close()) + mm.distributeCaptions(context.Background(), vs, seg, header) + ev := captions.Event{Streamer: "alice", Track: captions.Track{Origin: captions.OriginSidecar, Source: captions.SourceAuto, Language: "en"}, Cue: captions.Cue{ID: "late", Start: time.Now(), End: time.Now().Add(time.Second)}} + require.False(t, mm.ReceiveSidecar("alice", "did:web:upstream.example", ev), "shutdown cannot admit new caption work") + require.Empty(t, mm.bus.Captions.Tracks("alice")) +} + +type shutdownCaptionPDS struct{ text string } + +func (p *shutdownCaptionPDS) Publish(_ context.Context, _ records.Target, _ string, rec *placestream.CaptionTranscript) (string, error) { + p.text = rec.Text + return "at://alice/place.stream.caption.transcript/last", nil +} + +func TestDistributionShutdownArchivesAcceptedCanonicalQueue(t *testing.T) { + seg, _ := distributionFixture(t) + eng, err := captions.TextEngine() + require.NoError(t, err) + canonical, err := eng.AddTextTrack(context.Background(), seg, upstream.TextTrack{TrackID: 9, Language: "en", Label: "ingest"}, []upstream.TextCue{{ID: "last", Text: "Last accepted speech", Start: 100, End: 400}}) + require.NoError(t, err) + var hdr bytes.Buffer + require.NoError(t, eng.Wrap(context.Background(), bytes.NewReader(canonical), "flat", &hdr)) + mm := &MediaManager{cli: &config.CLI{}, bus: bus.NewBus()} + pds := &shutdownCaptionPDS{} + writer, err := records.NewWriter(records.Config{Hub: mm.bus.Captions, Publisher: pds, Subject: func(context.Context, string) (comatproto.RepoStrongRef, error) { + return comatproto.RepoStrongRef{Uri: "at://alice/place.stream.livestream/live", Cid: "bafy"}, nil + }}) + require.NoError(t, err) + mm.captionState().writer = writer + vs := &validatedSegment{repoDID: "alice", local: true, meta: &SegmentMetadata{StartTime: aqtime.FromTime(time.Now()), Published: true}} + mm.distributeCaptions(context.Background(), vs, canonical, hdr.Bytes()) + mm.ShutdownCaptions() + require.Equal(t, "Last accepted speech", pds.text, "shutdown drains accepted canonical segments before the final transcript snapshot") +} diff --git a/pkg/media/captions_distribution_test.go b/pkg/media/captions_distribution_test.go new file mode 100644 index 000000000..afd8f7142 --- /dev/null +++ b/pkg/media/captions_distribution_test.go @@ -0,0 +1,237 @@ +package media + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + upstream "github.com/streamplace/muxl/go" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/aqtime" + "stream.place/streamplace/pkg/bus" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/muxl" + "stream.place/streamplace/pkg/placestream" + "stream.place/streamplace/pkg/stt" +) + +type distributionEngine struct { + leased, released chan struct{} + err error +} + +func (e *distributionEngine) Lease(context.Context, stt.LeaseOptions) (stt.Lease, error) { + select { + case e.leased <- struct{}{}: + default: + } + if e.err != nil { + return nil, e.err + } + return &distributionLease{e: e}, nil +} +func (*distributionEngine) Models() []stt.ModelInfo { return nil } +func (*distributionEngine) Close() error { return nil } + +type distributionLease struct{ e *distributionEngine } + +func (l *distributionLease) Model() stt.Model { return distributionModel{} } +func (l *distributionLease) Release() { l.e.released <- struct{}{} } + +type distributionModel struct{} + +func (distributionModel) Info() stt.ModelInfo { return stt.ModelInfo{Name: "fake"} } +func (distributionModel) Transcribe(context.Context, []float32, stt.Options) (*stt.Result, error) { + return &stt.Result{Language: "en"}, nil +} + +func distributionFixture(t *testing.T) ([]byte, []byte) { + t.Helper() + data, err := os.ReadFile(getFixture("h264-opus-frag.mp4")) + require.NoError(t, err) + events, err := segmentMuxlEvents(context.Background(), data) + require.NoError(t, err) + for _, ev := range events { + if ev.Type == "segment" { + seg := concatTracksByID(ev.Tracks) + var header bytes.Buffer + require.NoError(t, muxl.RunMuxlWrap(context.Background(), bytes.NewReader(seg), "flat", &header)) + return seg, header.Bytes() + } + } + t.Fatal("fixture has no segments") + return nil, nil +} + +func TestDistributionSidecarPolicyAndUpstreamStop(t *testing.T) { + seg, header := distributionFixture(t) + eng, err := captions.TextEngine() + require.NoError(t, err) + // Add a cue as an observable barrier behind the AV-only policy decision. + canonical, err := eng.AddTextTrack(context.Background(), seg, upstream.TextTrack{TrackID: 9, Language: "en", Label: "ingest"}, []upstream.TextCue{{ID: "canonical", Text: "Signed captions", Start: 100, End: 400}}) + require.NoError(t, err) + var canonicalHeader bytes.Buffer + require.NoError(t, eng.Wrap(context.Background(), bytes.NewReader(canonical), "flat", &canonicalHeader)) + for _, tc := range []struct { + name, canonical string + local, allowed, want bool + }{ + {"origin off", "off", true, true, true}, + {"origin auto", "auto", true, true, false}, + {"origin ingest", "ingest", true, true, false}, + {"relay fallback", "auto", false, true, true}, + {"streamer opt out", "off", false, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + engine := &distributionEngine{leased: make(chan struct{}, 4), released: make(chan struct{}, 4)} + mm := &MediaManager{cli: &config.CLI{Captions: true, BroadcasterHost: "node.example"}, bus: bus.NewBus(), STT: engine} + vs := &validatedSegment{repoDID: "did:plc:alice", local: tc.local, meta: &SegmentMetadata{StartTime: aqtime.FromTime(time.Unix(1700000000, 0)), Published: true, MetadataConfiguration: &placestream.MetadataConfiguration{CaptionPolicy: &placestream.MetadataCaptionPolicy{Canonical: &tc.canonical, AllowNodeCaptions: &tc.allowed}}}} + mm.distributeCaptions(context.Background(), vs, seg, header) + d := mm.captionState() + d.mu.Lock() + s := d.streams[vs.repoDID] + d.mu.Unlock() + t.Cleanup(func() { mm.EndCaptionSession(vs.repoDID); <-s.done }) + if !tc.want { + mm.distributeCaptions(context.Background(), vs, canonical, canonicalHeader.Bytes()) + require.Eventually(t, func() bool { + return len(mm.bus.Captions.Cues(vs.repoDID, "canonical-ingest-en", vs.meta.StartTime.Time(), vs.meta.StartTime.Time().Add(time.Minute))) == 1 + }, 15*time.Second, time.Millisecond) + select { + case <-engine.leased: + t.Fatal("policy leased a speech model") + default: + } + return + } + select { + case <-engine.leased: + case <-time.After(15 * time.Second): + t.Fatal("allowed sidecar did not lease") + } + ev := captions.Event{Streamer: vs.repoDID, Track: captions.Track{ID: "sidecar-auto-en", Language: "en", Kind: captions.KindCaptions, Origin: captions.OriginSidecar, Source: captions.SourceAuto, Author: "did:web:upstream.example"}, Cue: captions.Cue{ID: "upstream", Text: "Upstream speech", Start: vs.meta.StartTime.Time(), End: vs.meta.StartTime.Time().Add(time.Second), Final: true}} + data, err := captions.EncodeSidecar(ev) + require.NoError(t, err) + decoded, ok := captions.DecodeSidecar(data) + require.True(t, ok) + require.True(t, mm.ReceiveSidecar(vs.repoDID, "did:web:upstream.example", decoded)) + select { + case <-engine.released: + case <-time.After(15 * time.Second): + t.Fatal("upstream sidecar did not stop recognition") + } + got := mm.bus.Captions.Cues(vs.repoDID, ev.Track.ID, ev.Cue.Start, ev.Cue.End) + require.Equal(t, []captions.Cue{ev.Cue}, got) + require.Equal(t, ev.Track.Author, mm.bus.Captions.Tracks(vs.repoDID)[0].Author) + mm.distributeCaptions(context.Background(), vs, seg, header) + mm.EndCaptionSession(vs.repoDID) + <-s.done + select { + case <-engine.leased: + t.Fatal("recognized after upstream appeared") + default: + } + require.Empty(t, mm.bus.Captions.Tracks(vs.repoDID)) + }) + } +} + +func TestDistributionCanonicalStopsSidecar(t *testing.T) { + seg, header := distributionFixture(t) + engine := &distributionEngine{leased: make(chan struct{}, 4), released: make(chan struct{}, 4)} + mm := &MediaManager{cli: &config.CLI{Captions: true, BroadcasterHost: "node.example"}, bus: bus.NewBus(), STT: engine} + vs := &validatedSegment{repoDID: "did:plc:alice", meta: &SegmentMetadata{StartTime: aqtime.FromTime(time.Unix(1700000000, 0)), Published: true}} + mm.distributeCaptions(context.Background(), vs, seg, header) + d := mm.captionState() + d.mu.Lock() + s := d.streams[vs.repoDID] + d.mu.Unlock() + t.Cleanup(func() { mm.EndCaptionSession(vs.repoDID); <-s.done }) + select { + case <-engine.leased: + case <-time.After(15 * time.Second): + t.Fatal("relay did not start recognition") + } + eng, err := captions.TextEngine() + require.NoError(t, err) + canonical, err := eng.AddTextTrack(context.Background(), seg, upstream.TextTrack{TrackID: 9, Language: "en", Label: "ingest"}, []upstream.TextCue{{ID: "canonical", Text: "Canonical wins", Start: 100, End: 400}}) + require.NoError(t, err) + var hdr bytes.Buffer + require.NoError(t, eng.Wrap(context.Background(), bytes.NewReader(canonical), "flat", &hdr)) + mm.distributeCaptions(context.Background(), vs, canonical, hdr.Bytes()) + select { + case <-engine.released: + case <-time.After(15 * time.Second): + t.Fatal("canonical track did not stop recognition") + } + got := mm.bus.Captions.Cues(vs.repoDID, "canonical-ingest-en", vs.meta.StartTime.Time(), vs.meta.StartTime.Time().Add(time.Minute)) + require.Len(t, got, 1) + require.Equal(t, "Canonical wins", got[0].Text) + mm.distributeCaptions(context.Background(), vs, seg, header) + mm.EndCaptionSession(vs.repoDID) + <-s.done + select { + case <-engine.leased: + t.Fatal("recognized after canonical track appeared") + default: + } +} + +func TestDistributionMissingEngineAndBudgetLeaveCaptionsPlayable(t *testing.T) { + seg, header := distributionFixture(t) + eng, err := captions.TextEngine() + require.NoError(t, err) + canonical, err := eng.AddTextTrack(context.Background(), seg, upstream.TextTrack{TrackID: 9, Language: "en", Label: "ingest"}, []upstream.TextCue{{ID: "canonical", Text: "Still available", Start: 100, End: 400}}) + require.NoError(t, err) + var hdr bytes.Buffer + require.NoError(t, eng.Wrap(context.Background(), bytes.NewReader(canonical), "flat", &hdr)) + for _, engine := range []stt.Engine{nil, &distributionEngine{leased: make(chan struct{}, 4), released: make(chan struct{}, 4), err: stt.ErrOverBudget}} { + mm := &MediaManager{cli: &config.CLI{Captions: true, BroadcasterHost: "node.example"}, bus: bus.NewBus(), STT: engine} + vs := &validatedSegment{repoDID: "did:plc:alice", meta: &SegmentMetadata{StartTime: aqtime.FromTime(time.Unix(1700000000, 0)), Published: true}} + mm.distributeCaptions(context.Background(), vs, seg, header) + mm.distributeCaptions(context.Background(), vs, canonical, hdr.Bytes()) + require.Eventually(t, func() bool { + cues := mm.bus.Captions.Cues(vs.repoDID, "canonical-ingest-en", vs.meta.StartTime.Time(), vs.meta.StartTime.Time().Add(time.Minute)) + return len(cues) == 1 && cues[0].Text == "Still available" + }, 15*time.Second, time.Millisecond) + for _, track := range mm.bus.Captions.Tracks(vs.repoDID) { + require.Equal(t, captions.OriginCanonical, track.Origin, "missing capacity must not create a sidecar") + } + d := mm.captionState() + d.mu.Lock() + s := d.streams[vs.repoDID] + d.mu.Unlock() + mm.EndCaptionSession(vs.repoDID) + <-s.done + } +} + +func TestDistributionCanonicalRejectsEarlySidecarReplay(t *testing.T) { + seg, _ := distributionFixture(t) + eng, err := captions.TextEngine() + require.NoError(t, err) + canonical, err := eng.AddTextTrack(context.Background(), seg, upstream.TextTrack{TrackID: 9, Language: "en", Label: "ingest"}, []upstream.TextCue{{ID: "canonical", Text: "Canonical replay precedence", Start: 100, End: 400}}) + require.NoError(t, err) + var hdr bytes.Buffer + require.NoError(t, eng.Wrap(context.Background(), bytes.NewReader(canonical), "flat", &hdr)) + mm := &MediaManager{cli: &config.CLI{Captions: true}, bus: bus.NewBus()} + allowed := true + vs := &validatedSegment{repoDID: "did:plc:alice", meta: &SegmentMetadata{StartTime: aqtime.FromTime(time.Unix(1700000000, 0)), Published: true, MetadataConfiguration: &placestream.MetadataConfiguration{CaptionPolicy: &placestream.MetadataCaptionPolicy{AllowNodeCaptions: &allowed}}}} + ev := captions.Event{Streamer: vs.repoDID, Track: captions.Track{ID: "sidecar-auto-en", Language: "en", Kind: captions.KindCaptions, Origin: captions.OriginSidecar, Source: captions.SourceAuto, Author: "did:web:upstream.example"}, Cue: captions.Cue{ID: "upstream", Text: "Competing replay", Start: vs.meta.StartTime.Time(), End: vs.meta.StartTime.Time().Add(time.Second), Final: true}} + require.True(t, mm.ReceiveSidecar(vs.repoDID, "did:web:upstream.example", ev)) + require.Empty(t, mm.bus.Captions.Tracks(vs.repoDID), "unvalidated replay remains private") + mm.distributeCaptions(context.Background(), vs, canonical, hdr.Bytes()) + d := mm.captionState() + d.mu.Lock() + s := d.streams[vs.repoDID] + d.mu.Unlock() + t.Cleanup(func() { mm.EndCaptionSession(vs.repoDID); <-s.done }) + require.Eventually(t, func() bool { + return len(mm.bus.Captions.Cues(vs.repoDID, "canonical-ingest-en", ev.Cue.Start, ev.Cue.Start.Add(time.Minute))) == 1 + }, 15*time.Second, time.Millisecond) + require.Empty(t, mm.bus.Captions.Cues(vs.repoDID, ev.Track.ID, ev.Cue.Start, ev.Cue.End), "canonical tracks suppress an earlier upstream sidecar replay") + require.False(t, mm.ReceiveSidecar(vs.repoDID, "did:web:upstream.example", ev), "later competing sidecars remain suppressed") +} diff --git a/pkg/media/captions_engine.go b/pkg/media/captions_engine.go new file mode 100644 index 000000000..adf0b4edc --- /dev/null +++ b/pkg/media/captions_engine.go @@ -0,0 +1,24 @@ +package media + +import ( + "context" + "fmt" + "path/filepath" + "stream.place/streamplace/pkg/stt" +) + +// StartCaptionEngineProxy shares the node scheduler with isolated media workers. +func (mm *MediaManager) StartCaptionEngineProxy(ctx context.Context) (func(), error) { + dir, err := mm.ingestWorkerSocketDir() + if err != nil { + return nil, err + } + // Detached-worker discovery reserves the .sock suffix for frame sockets. + path := filepath.Join(dir, "speech-engine.stt") + stop, err := stt.ServeEngine(ctx, path, mm.STT) + if err != nil { + return nil, fmt.Errorf("serve node speech engine: %w", err) + } + mm.CaptionEngineSocket = path + return stop, nil +} diff --git a/pkg/media/captions_engine_test.go b/pkg/media/captions_engine_test.go new file mode 100644 index 000000000..cd43b458e --- /dev/null +++ b/pkg/media/captions_engine_test.go @@ -0,0 +1,41 @@ +package media + +import ( + "context" + "net" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/config" +) + +func TestCaptionMasterEngineDiscovery(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + mm := NewOffline(&config.CLI{DataDir: t.TempDir()}) + stop, err := mm.StartCaptionEngineProxy(ctx) + require.NoError(t, err) + defer stop() + dir, err := mm.ingestWorkerSocketDir() + require.NoError(t, err) + framePath := filepath.Join(dir, "worker.sock") + frame, err := net.Listen("unix", framePath) + require.NoError(t, err) + defer frame.Close() + push, err := net.Listen("unix", framePath+".captions") + require.NoError(t, err) + defer push.Close() + require.NoError(t, writeWorkerMeta(framePath, workerMeta{})) + resumable, err := DiscoverWorkerSockets(dir) + require.NoError(t, err) + require.Equal(t, []string{framePath}, resumable, "engine and caption-control sockets are not frame transports") + removeWorkerFiles(framePath) + _, err = os.Stat(framePath + ".captions") + require.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(workerMetaPath(framePath)) + require.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(mm.CaptionEngineSocket) + require.NoError(t, err, "worker crash cleanup must not unlink the node scheduler") +} diff --git a/pkg/media/captions_language_test.go b/pkg/media/captions_language_test.go new file mode 100644 index 000000000..17f57b504 --- /dev/null +++ b/pkg/media/captions_language_test.go @@ -0,0 +1,31 @@ +package media + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/muxl" +) + +func TestCaptionMasterPushedLanguagesIgnoreRecognitionHints(t *testing.T) { + m := newCaptionMaster(context.Background(), "streamer", &config.CLI{}, nil) + m.setManifest(captionManifest("auto")) + m.clock(time.UnixMilli(0)) + m.mediaFinished = true + for _, language := range []string{"es", "fr"} { + track := captions.Track{Language: language, Source: captions.SourceHuman} + require.NoError(t, m.push(track, []captions.Cue{{ID: "line", Start: m.arrival.Add(100 * time.Millisecond), End: m.arrival.Add(500 * time.Millisecond), Text: language, Final: true}})) + } + got, err := m.text(context.Background(), muxl.TextRequest{EndMs: 1000}) + require.NoError(t, err) + var languages []string + for _, track := range got.Tracks { + languages = append(languages, track.Language) + require.Equal(t, "human", track.Label) + } + require.ElementsMatch(t, []string{"es", "fr"}, languages, "author languages must remain distinct despite en-US recognition hints") +} diff --git a/pkg/media/captions_master.go b/pkg/media/captions_master.go new file mode 100644 index 000000000..930b0f3a6 --- /dev/null +++ b/pkg/media/captions_master.go @@ -0,0 +1,365 @@ +package media + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + "sync" + "time" + + "github.com/google/uuid" + + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/muxl" + "stream.place/streamplace/pkg/placestream" + "stream.place/streamplace/pkg/stt" +) + +// CaptionTrackIDBase reserves canonical text IDs above node-added AV renditions. +const CaptionTrackIDBase uint32 = 100 + +type captionSession interface { + captionPolicy() (captions.Policy, error) + push(captions.Track, []captions.Cue) error +} + +type captionMaster struct { + ctx context.Context + streamer string + sessionID string + cli *config.CLI + engine stt.Engine + hub *captions.Hub + mu sync.Mutex + current captions.Policy + policyReady bool + stopped bool + ingestSeen bool + recognitionUnavailable bool + parsedUntil uint64 + mediaFinished bool + arrival time.Time + mediaOrigin time.Time + covered time.Time + changed chan struct{} + closes map[uint64]time.Time + gopTimes map[uint64]time.Time + signedUntil uint64 + consumed map[string]int64 + pending map[string]muxl.TextCue + tracks map[string]muxl.TextTrack + nextID uint32 +} + +func newCaptionMaster(ctx context.Context, streamer string, cli *config.CLI, engine stt.Engine) *captionMaster { + return &captionMaster{ctx: ctx, streamer: streamer, sessionID: uuid.NewString(), cli: cli, engine: engine, hub: captions.NewHub(0), current: captions.DefaultPolicy(), changed: make(chan struct{}), closes: make(map[uint64]time.Time), gopTimes: make(map[uint64]time.Time), consumed: make(map[string]int64), pending: make(map[string]muxl.TextCue), tracks: make(map[string]muxl.TextTrack)} +} + +func captionPolicyFromManifest(data []byte) captions.Policy { + var manifest struct { + Assertions []struct { + Label string `json:"label"` + Data json.RawMessage `json:"data"` + } `json:"assertions"` + } + if json.Unmarshal(data, &manifest) != nil { + return captions.DefaultPolicy() + } + for _, a := range manifest.Assertions { + if a.Label == "place.stream.metadata.configuration" { + var cfg placestream.MetadataConfiguration + if json.Unmarshal(a.Data, &cfg) == nil { + return captions.PolicyFromMetadata(&cfg) + } + } + } + return captions.DefaultPolicy() +} + +func (m *captionMaster) setManifest(data []byte) { + m.mu.Lock() + m.current = captionPolicyFromManifest(data) + m.policyReady = true + m.signal() + m.mu.Unlock() +} +func (m *captionMaster) policy() captions.Policy { m.mu.Lock(); defer m.mu.Unlock(); return m.current } +func (m *captionMaster) captionPolicy() (captions.Policy, error) { + m.mu.Lock() + defer m.mu.Unlock() + if !m.policyReady { + return captions.Policy{}, fmt.Errorf("caption manifest not ready") + } + return m.current, nil +} + +// waitPolicy prevents ingest from spending recognition budget on an unknown +// startup policy. The separate media queue keeps the signer free to fetch it. +func (m *captionMaster) waitPolicy() (captions.Policy, error) { + m.mu.Lock() + for !m.policyReady { + if m.stopped { + m.mu.Unlock() + return captions.Policy{}, context.Canceled + } + changed := m.changed + m.mu.Unlock() + select { + case <-m.ctx.Done(): + return captions.Policy{}, m.ctx.Err() + case <-changed: + } + m.mu.Lock() + } + policy := m.current + m.mu.Unlock() + return policy, nil +} +func (m *captionMaster) signal() { close(m.changed); m.changed = make(chan struct{}) } +func (m *captionMaster) coverage(end time.Time) { + m.mu.Lock() + if end.After(m.covered) { + m.covered = end + m.signal() + } + m.mu.Unlock() +} +func (m *captionMaster) clock(media time.Time) { m.clockAt(media, time.Now()) } +func (m *captionMaster) clockAt(media, at time.Time) { + m.mu.Lock() + if m.arrival.IsZero() { + m.arrival = at + m.mediaOrigin = time.UnixMilli(media.UnixMilli()) + m.signal() + } + m.mu.Unlock() +} + +func (m *captionMaster) segmentTime(start uint64) time.Time { + m.mu.Lock() + defer m.mu.Unlock() + prediction := m.arrival.Add(time.Duration(int64(start)-m.mediaOrigin.UnixMilli()) * time.Millisecond) + if at, ok := m.gopTimes[start]; ok { + prediction = at + } + for key := range m.gopTimes { + if key <= start { + delete(m.gopTimes, key) + } + } + return prediction +} +func (m *captionMaster) closeGop(end uint64) { + m.closeGopAt(end, time.Now()) +} +func (m *captionMaster) closeGopAt(end uint64, at time.Time) { + m.mu.Lock() + prediction := m.arrival.Add(time.Duration(int64(end)-m.mediaOrigin.UnixMilli()) * time.Millisecond) + if at.Sub(prediction).Abs() > time.Second { + m.arrival = at + m.mediaOrigin = time.UnixMilli(int64(end)) + prediction = at + } + m.gopTimes[end] = prediction + if end > m.parsedUntil { + m.parsedUntil = end + } + if end > m.signedUntil { + if _, ok := m.closes[end]; !ok { + m.closes[end] = at + } + } + m.signal() + m.mu.Unlock() +} +func (m *captionMaster) finishMedia() { m.mu.Lock(); m.mediaFinished = true; m.signal(); m.mu.Unlock() } +func (m *captionMaster) push(track captions.Track, cues []captions.Cue) error { + m.mu.Lock() + if m.arrival.IsZero() { + m.mu.Unlock() + return fmt.Errorf("caption media clock not ready") + } + arrival, origin := m.arrival, m.mediaOrigin + m.ingestSeen = true + m.signal() + m.mu.Unlock() + track.Origin = captions.OriginCanonical + track.ID = "push-" + captions.TrackID(track.Origin, track.Source, track.Language) + for _, cue := range cues { + cue.Start = origin.Add(cue.Start.Sub(arrival)) + cue.End = origin.Add(cue.End.Sub(arrival)) + m.hub.Publish(m.streamer, track, cue) + } + return nil +} + +// text holds only the signer, never appsink. It consumes immutable final cues +// from a private hub. A late final is carried into the next unsigned GoP. +func (m *captionMaster) text(ctx context.Context, req muxl.TextRequest) (*muxl.TextAttachment, error) { + m.mu.Lock() + closeAt, ok := m.closes[req.EndMs] + if !ok { + closeAt = time.Now() + } + for end := range m.closes { + if end <= req.EndMs { + delete(m.closes, end) + } + } + delay := m.cli.CaptionsMasterDelay + deadline := closeAt.Add(delay) + for { + decision := captions.Decide(captions.Situation{Policy: m.current, Origin: true, IngestCaptions: m.ingestSeen}) + tapReady := m.mediaFinished || m.parsedUntil >= req.EndMs + recognitionReady := !decision.Recognize() || m.engine == nil || m.recognitionUnavailable || m.covered.UnixMilli() >= int64(req.EndMs) + if m.current.Canonical == captions.CanonicalOff || (tapReady && recognitionReady) || !time.Now().Before(deadline) { + break + } + changed := m.changed + m.mu.Unlock() + timer := time.NewTimer(time.Until(deadline)) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-changed: + timer.Stop() + case <-timer.C: + } + m.mu.Lock() + } + p, ingest := m.current, m.ingestSeen + until := m.signedUntil + m.mu.Unlock() + attachment := &muxl.TextAttachment{} + if p.Canonical == captions.CanonicalOff { + m.mu.Lock() + m.signedUntil = req.EndMs + m.pending = make(map[string]muxl.TextCue) + m.mu.Unlock() + return attachment, nil + } + if m.nextID == 0 { + m.nextID = CaptionTrackIDBase + } + horizon := int64(req.StartMs) - captions.DefaultRetention.Milliseconds() + from, to := time.UnixMilli(horizon), time.UnixMilli(int64(req.EndMs)) + for key, end := range m.consumed { + if end <= horizon { + delete(m.consumed, key) + } + } + items := make(map[uint32]*muxl.TextTrackAttachment) + for _, track := range m.hub.Tracks(m.streamer) { + if track.Source == captions.SourceAuto && !strings.HasPrefix(track.ID, "push-") && (p.Canonical != captions.CanonicalAuto || ingest) { + for key := range m.pending { + if strings.HasPrefix(key, track.ID+"/") { + delete(m.pending, key) + } + } + continue + } + language := track.Language + if (language == "" || language == "und") && track.Source == captions.SourceIngest && len(p.Languages) > 0 { + language = p.Languages[0] + } + if language == "" { + language = "und" + } + trackKey := string(track.Source) + "/" + strings.ToLower(language) + for _, cue := range m.hub.Cues(m.streamer, track.ID, from, to) { + key := track.ID + "/" + cue.ID + if _, ok := m.consumed[key]; ok { + continue + } + m.consumed[key] = cue.End.UnixMilli() + start, end := cue.Start.UnixMilli(), cue.End.UnixMilli() + if end <= 0 { + continue + } + if start < 0 { + start = 0 + } + if start < int64(until) { + duration := end - start + start = int64(until) + if end <= start { + end = start + duration + } + } + m.pending[key] = muxl.TextCue{Start: uint64(start), End: uint64(end), Text: cue.Text, ID: m.sessionID + "/" + key} + if _, ok := m.tracks[trackKey]; !ok { + m.tracks[trackKey] = muxl.TextTrack{TrackID: m.nextID, Language: language, Label: string(track.Source)} + m.nextID++ + } + } + config, ok := m.tracks[trackKey] + if !ok { + continue + } + item := items[config.TrackID] + if item == nil { + item = &muxl.TextTrackAttachment{TextTrack: config} + items[config.TrackID] = item + } + for key, cue := range m.pending { + if !strings.HasPrefix(key, track.ID+"/") { + continue + } + if cue.Start < req.EndMs && cue.End > req.StartMs { + clipped := cue + clipped.Start = max(clipped.Start, req.StartMs) + clipped.End = min(clipped.End, req.EndMs) + item.Cues = append(item.Cues, clipped) + } + if cue.End <= req.EndMs { + delete(m.pending, key) + } + } + } + for _, item := range items { + attachment.Tracks = append(attachment.Tracks, *item) + } + m.mu.Lock() + m.signedUntil = req.EndMs + m.mu.Unlock() + return attachment, nil +} + +type captionManagerKey struct{} + +func withCaptionManager(ctx context.Context, mm *MediaManager) context.Context { + return context.WithValue(ctx, captionManagerKey{}, mm) +} +func (mm *MediaManager) registerCaptionMaster(streamer string, master captionSession) func() { + mm.captionMasters.Store(streamer, master) + return func() { mm.captionMasters.CompareAndDelete(streamer, master) } +} + +// OriginCaptionPolicy reports whether this node owns a live ingest session. +func (mm *MediaManager) OriginCaptionPolicy(streamer string) (captions.Policy, bool) { + v, ok := mm.captionMasters.Load(streamer) + if !ok { + return captions.Policy{}, false + } + p, err := v.(captionSession).captionPolicy() + return p, err == nil +} + +// PushCanonicalCaptions routes to that session's private master, never the node hub. +func (mm *MediaManager) PushCanonicalCaptions(streamer string, track captions.Track, cues []captions.Cue) error { + v, ok := mm.captionMasters.Load(streamer) + if !ok { + return fmt.Errorf("stream not live") + } + return v.(captionSession).push(track, cues) +} + +// SignOriginStream uses the node-wide engine and registers the session for +// pushCaptions while media is flowing through the streaming signer. +func (mm *MediaManager) SignOriginStream(ctx context.Context, ms MediaSigner, input io.Reader, events chan *muxl.MuxlEvent) error { + return ms.SignSegmentStream(withCaptionManager(ctx, mm), input, events) +} diff --git a/pkg/media/captions_master_control_test.go b/pkg/media/captions_master_control_test.go new file mode 100644 index 000000000..4cef8e97c --- /dev/null +++ b/pkg/media/captions_master_control_test.go @@ -0,0 +1,59 @@ +package media + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/muxl" +) + +func TestCaptionMasterWorkerControlAndReconnectIDs(t *testing.T) { + ctx := context.Background() + mm := NewOffline(&config.CLI{}) + var previousID string + for range 2 { + master := newCaptionMaster(ctx, "did:plc:streamer", &config.CLI{}, nil) + master.setManifest(captionManifest("ingest")) + master.clock(time.UnixMilli(200)) + master.mediaFinished = true + path := filepath.Join(t.TempDir(), "ingest.sock") + stop, err := master.servePush(path) + require.NoError(t, err) + unregister := mm.registerWorkerCaptionMaster(ctx, path, master.streamer) + policy, live := mm.OriginCaptionPolicy(master.streamer) + require.True(t, live) + require.Equal(t, captions.CanonicalIngest, policy.Canonical) + track := masterTrack(captions.SourceHuman) + cue := captions.Cue{ID: "a", Start: master.arrival.Add(100 * time.Millisecond), End: master.arrival.Add(300 * time.Millisecond), Text: "supplied words"} + require.NoError(t, mm.PushCanonicalCaptions(master.streamer, track, []captions.Cue{cue})) + interim, err := master.text(ctx, muxl.TextRequest{EndMs: 250}) + require.NoError(t, err) + require.Empty(t, interim.Tracks) + cue.Final = true + require.NoError(t, mm.PushCanonicalCaptions(master.streamer, track, []captions.Cue{cue})) + final, err := master.text(ctx, muxl.TextRequest{StartMs: 250, EndMs: 1000}) + require.NoError(t, err) + require.Len(t, final.Tracks, 1) + require.Equal(t, "human", final.Tracks[0].Label) + require.Len(t, final.Tracks[0].Cues, 1) + got := final.Tracks[0].Cues[0] + require.Equal(t, uint64(300), got.Start) + require.Equal(t, uint64(500), got.End) + require.Equal(t, "supplied words", got.Text) + require.NotEqual(t, previousID, got.ID, "rapid reconnect cannot reuse a public canonical cue ID") + previousID = got.ID + master.setManifest(captionManifest("off")) + policy, live = mm.OriginCaptionPolicy(master.streamer) + require.True(t, live) + require.Equal(t, captions.CanonicalOff, policy.Canonical) + stop() + unregister() + _, live = mm.OriginCaptionPolicy(master.streamer) + require.False(t, live) + } +} diff --git a/pkg/media/captions_master_feed.go b/pkg/media/captions_master_feed.go new file mode 100644 index 000000000..059c7bd26 --- /dev/null +++ b/pkg/media/captions_master_feed.go @@ -0,0 +1,232 @@ +package media + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "io" + "time" + + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/captions/fmp4" + "stream.place/streamplace/pkg/log" +) + +// tee stages a normal hold independently of the signer. Each bounded queue +// backpressures sustained overload rather than retaining unlimited media. +func (m *captionMaster) tee(input io.Reader) (io.Reader, func()) { + media, audio := newIngestByteBuffer(m.ctx), newIngestByteBuffer(m.ctx) + done := make(chan struct{}) + go func() { + select { + case <-m.ctx.Done(): + _ = media.CloseWithError(m.ctx.Err()) + _ = audio.CloseWithError(m.ctx.Err()) + if closer, ok := input.(io.ReadCloser); ok { + closer.Close() + } + case <-done: + } + }() + go func() { + defer close(done) + if err := m.readMedia(audio); err != nil { + log.Warn(m.ctx, "caption ingest tap stopped", "error", err) + } + // Continue draining if captions fail; media must still be signed. + _, _ = io.Copy(io.Discard, audio) + }() + go func() { + buf := make([]byte, 64*1024) + for { + n, err := input.Read(buf) + if n > 0 { + at := captionReadTime(input) + if _, writeErr := media.writeAt(buf[:n], at); writeErr != nil { + return + } + _, _ = audio.writeAt(buf[:n], at) + } + if err != nil { + if err == io.EOF { + err = nil + } + _ = media.CloseWithError(err) + _ = audio.CloseWithError(err) + return + } + } + }() + return media, func() { + m.mu.Lock() + m.stopped = true + m.signal() + m.mu.Unlock() + _ = media.CloseWithError(context.Canceled) + _ = audio.CloseWithError(context.Canceled) + if closer, ok := input.(io.ReadCloser); ok { + closer.Close() + } + <-done + } +} + +func readCaptionBox(r io.Reader) ([]byte, string, error) { + var h [8]byte + if _, err := io.ReadFull(r, h[:]); err != nil { + return nil, "", err + } + size := uint64(binary.BigEndian.Uint32(h[:4])) + header := 8 + var extra [8]byte + if size == 1 { + if _, err := io.ReadFull(r, extra[:]); err != nil { + return nil, "", err + } + size = binary.BigEndian.Uint64(extra[:]) + header = 16 + } + if size < uint64(header) || size > 256*1024*1024 { + return nil, "", fmt.Errorf("invalid caption MP4 box size %d", size) + } + data := make([]byte, int(size)) + copy(data, h[:]) + if header == 16 { + copy(data[8:], extra[:]) + } + if _, err := io.ReadFull(r, data[header:]); err != nil { + return nil, "", err + } + return data, string(h[4:]), nil +} + +func (m *captionMaster) readMedia(input io.Reader) error { + var init, moof []byte + var tracks []fmp4.TrackInfo + var referenceVideo uint32 + var decoder *captionAudioDecoder + var recognizer *captions.Recognizer + unavailable := false + var last time.Time + tap := captions.NewIngestTap(m.streamer, m.hub, captions.OriginCanonical, m.streamer, "und") + defer m.finishMedia() + defer func() { + if decoder != nil { + decoder.close() + } + if recognizer != nil { + recognizer.Close() + } + tap.Close(last) + }() + for { + data, kind, err := readCaptionBox(input) + if err == io.EOF { + return nil + } + if err != nil { + return err + } + switch kind { + case "ftyp": + init = data + case "moov": + init = append(init, data...) + tracks, err = fmp4.Tracks(init) + if err != nil { + return err + } + for _, track := range tracks { + if track.Handler == "vide" && (referenceVideo == 0 || track.ID < referenceVideo) { + referenceVideo = track.ID + } + } + case "moof": + moof = data + case "mdat": + if len(moof) == 0 { + continue + } + fragment := append(moof, data...) + moof = nil + frags, err := fmp4.Fragments(fragment) + if err != nil { + return err + } + for _, frag := range frags { + for _, track := range tracks { + if track.ID != frag.TrackID || track.Timescale == 0 { + continue + } + mediaTime := time.Duration(frag.BaseDecodeTime) * time.Second / time.Duration(track.Timescale) + m.clockAt(time.UnixMilli(0).Add(mediaTime), captionReadTime(input)) + p, err := m.waitPolicy() + if err != nil { + return err + } + if track.Handler == "vide" { + language := "und" + if len(p.Languages) > 0 { + language = p.Languages[0] + } + tap.SetTrack(captions.OriginCanonical, m.streamer, language) + tap.Publish(p.Canonical != captions.CanonicalOff) + for _, sample := range frag.Samples { + when := time.UnixMilli(0).Add(time.Duration(sample.PTS) * time.Second / time.Duration(track.Timescale)) + last = when + if sample.Sync && track.ID == referenceVideo { + m.closeGopAt(uint64(time.Duration(sample.DTS)*time.Second/time.Duration(track.Timescale)/time.Millisecond), captionReadTime(input)) + } + tap.Sample(sample.Data, when) + } + if tap.Seen() { + m.mu.Lock() + m.ingestSeen = true + m.signal() + m.mu.Unlock() + } + } + if track.Handler == "soun" && p.Canonical == captions.CanonicalAuto && m.engine != nil && !unavailable { + if decoder == nil { + codec := "aac" + if bytes.Contains(init, []byte("Opus")) { + codec = "opus" + } + recognizer, err = captions.NewRecognizer(context.WithoutCancel(m.ctx), captions.RecognizerOptions{Streamer: m.streamer, Origin: captions.OriginCanonical, Author: m.streamer, Languages: p.Languages, Hub: m.hub, Engine: m.engine, OnCoverage: m.coverage}) + if err != nil { + log.Warn(m.ctx, "canonical recognizer unavailable", "error", err) + unavailable = true + m.mu.Lock() + m.recognitionUnavailable = true + m.signal() + m.mu.Unlock() + continue + } + decoder, err = newCaptionAudioDecoder(context.WithoutCancel(m.ctx), codec, func(at time.Time, pcm []float32) { + m.mu.Lock() + decision := captions.Decide(captions.Situation{Policy: m.current, Origin: true, IngestCaptions: m.ingestSeen}) + m.mu.Unlock() + if decision.Recognize() && decision.Origin == captions.OriginCanonical { + recognizer.Push(at, pcm) + } + }) + if err != nil { + return err + } + } + if err := decoder.feedMedia(init, fragment, frag.BaseDecodeTime, mediaTime); err != nil { + return err + } + } + } + } + } + } +} + +// feedMedia is the origin-only synthetic clock mode. Distribution continues to +// call feed with each validated segment's wall-clock anchor unchanged. +func (d *captionAudioDecoder) feedMedia(init, seg []byte, tfdt uint64, media time.Duration) error { + return d.feed(init, seg, tfdt, media, time.UnixMilli(0).Add(media)) +} diff --git a/pkg/media/captions_master_socket.go b/pkg/media/captions_master_socket.go new file mode 100644 index 000000000..313fc1657 --- /dev/null +++ b/pkg/media/captions_master_socket.go @@ -0,0 +1,94 @@ +package media + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "time" + + "stream.place/streamplace/pkg/captions" +) + +// Detached ingest owns its media clock and master. A separate private control +// socket avoids replacing the main segment connection when CART tools push. +type captionControl struct { + Track captions.Track + Cues []captions.Cue +} +type captionControlReply struct { + Policy captions.Policy + Error string +} +type remoteCaptionMaster struct{ path string } + +func (m *captionMaster) servePush(path string) (func(), error) { + if path == "" { + return func() {}, nil + } + socket := path + ".captions" + ln, err := net.Listen("unix", socket) + if err != nil { + return nil, err + } + if err := os.Chmod(socket, 0600); err != nil { + ln.Close() + os.Remove(socket) + return nil, err + } + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + var req captionControl + if json.NewDecoder(conn).Decode(&req) != nil { + return + } + reply := captionControlReply{Policy: m.policy()} + if len(req.Cues) > 0 { + if err := m.push(req.Track, req.Cues); err != nil { + reply.Error = err.Error() + } + } + _ = json.NewEncoder(conn).Encode(reply) + }() + } + }() + return func() { ln.Close(); os.Remove(socket) }, nil +} +func (r *remoteCaptionMaster) call(req captionControl) (captionControlReply, error) { + conn, err := net.DialTimeout("unix", r.path+".captions", 2*time.Second) + if err != nil { + return captionControlReply{}, err + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + if err := json.NewEncoder(conn).Encode(req); err != nil { + return captionControlReply{}, err + } + var reply captionControlReply + if err := json.NewDecoder(conn).Decode(&reply); err != nil { + return reply, err + } + if reply.Error != "" { + return reply, fmt.Errorf("caption worker: %s", reply.Error) + } + return reply, nil +} +func (r *remoteCaptionMaster) captionPolicy() (captions.Policy, error) { + reply, err := r.call(captionControl{}) + return reply.Policy, err +} +func (r *remoteCaptionMaster) push(track captions.Track, cues []captions.Cue) error { + _, err := r.call(captionControl{Track: track, Cues: cues}) + return err +} +func (mm *MediaManager) registerWorkerCaptionMaster(ctx context.Context, path, streamer string) func() { + return mm.registerCaptionMaster(streamer, &remoteCaptionMaster{path: path}) +} diff --git a/pkg/media/captions_master_test.go b/pkg/media/captions_master_test.go new file mode 100644 index 000000000..c50ad6f40 --- /dev/null +++ b/pkg/media/captions_master_test.go @@ -0,0 +1,309 @@ +package media + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/muxl" + "stream.place/streamplace/pkg/stt" +) + +type captionTestEngine struct { + enter, release chan struct{} + once sync.Once + scripted *stt.Result +} + +func (e *captionTestEngine) Lease(context.Context, stt.LeaseOptions) (stt.Lease, error) { + return e, nil +} +func (e *captionTestEngine) Models() []stt.ModelInfo { return nil } +func (e *captionTestEngine) Close() error { return nil } +func (e *captionTestEngine) Model() stt.Model { return e } +func (e *captionTestEngine) Release() {} +func (e *captionTestEngine) Info() stt.ModelInfo { return stt.ModelInfo{Name: "fake"} } +func (e *captionTestEngine) Transcribe(ctx context.Context, _ []float32, _ stt.Options) (*stt.Result, error) { + if e.enter != nil { + e.once.Do(func() { close(e.enter) }) + select { + case <-e.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if e.scripted != nil { + return e.scripted, nil + } + return &stt.Result{Language: "en-US", Words: []stt.Word{{Text: "held words", Start: 250 * time.Millisecond, End: 750 * time.Millisecond, Prob: 0.99}}}, nil +} +func captionManifest(policy string) []byte { + return []byte(`{"title":"captions test","assertions":[{"label":"c2pa.actions","data":{"actions":[{"action":"c2pa.created"}]}},{"label":"place.stream.metadata.configuration","data":{"captionPolicy":{"canonical":"` + policy + `","languages":["en-US"]}}}]}`) +} +func masterTrack(source captions.Source) captions.Track { + return captions.Track{ID: captions.TrackID(captions.OriginCanonical, source, "en-US"), Language: "en-US", Source: source, Origin: captions.OriginCanonical, Kind: captions.KindCaptions} +} + +func TestCaptionMasterHoldsUntilRecognitionCoversSpan(t *testing.T) { + ctx := context.Background() + engine := &captionTestEngine{enter: make(chan struct{}), release: make(chan struct{})} + m := newCaptionMaster(ctx, "streamer", &config.CLI{CaptionsMasterDelay: time.Hour}, engine) + m.mediaFinished = true + r, err := captions.NewRecognizer(ctx, captions.RecognizerOptions{Streamer: m.streamer, Origin: captions.OriginCanonical, Hub: m.hub, Engine: engine, OnCoverage: m.coverage, Step: time.Millisecond, MinWindow: time.Millisecond, SilenceFlush: 100 * time.Millisecond}) + require.NoError(t, err) + defer r.Close() + pcm := make([]float32, stt.SampleRate*2) + for i := range stt.SampleRate { + pcm[i] = 0.2 + } + r.Push(time.UnixMilli(0), pcm) + <-engine.enter + result := make(chan *muxl.TextAttachment, 1) + started := make(chan struct{}) + go func() { + close(started) + attachment, err := m.text(ctx, muxl.TextRequest{StartMs: 0, EndMs: 1000}) + if err == nil { + result <- attachment + } + }() + <-started + select { + case <-result: + t.Fatal("signed before recognition finished") + default: + } + close(engine.release) + select { + case attachment := <-result: + require.Len(t, attachment.Tracks, 1) + require.Equal(t, "en-US", attachment.Tracks[0].Language) + require.Equal(t, "auto", attachment.Tracks[0].Label) + require.Len(t, attachment.Tracks[0].Cues, 1) + require.Equal(t, uint64(250), attachment.Tracks[0].Cues[0].Start) + require.GreaterOrEqual(t, attachment.Tracks[0].Cues[0].End, uint64(750)) + require.LessOrEqual(t, attachment.Tracks[0].Cues[0].End, uint64(1000)) + require.Equal(t, "held words", attachment.Tracks[0].Cues[0].Text) + case <-time.After(5 * time.Second): + t.Fatal("coverage did not release GoP") + } +} + +func TestCaptionMasterDeadlineClippingAndLateCarry(t *testing.T) { + m := newCaptionMaster(context.Background(), "streamer", &config.CLI{CaptionsMasterDelay: time.Second}, &captionTestEngine{}) + m.closes[1000] = time.Now().Add(-2 * time.Second) + first, err := m.text(context.Background(), muxl.TextRequest{StartMs: 0, EndMs: 1000}) + require.NoError(t, err) + require.Empty(t, first.Tracks) + track := masterTrack(captions.SourceAuto) + m.hub.Publish(m.streamer, track, captions.Cue{ID: "late", Start: time.UnixMilli(500), End: time.UnixMilli(800), Text: "late", Final: true}) + m.hub.Publish(m.streamer, track, captions.Cue{ID: "crossing", Start: time.UnixMilli(1400), End: time.UnixMilli(2300), Text: "crossing", Final: true}) + m.closes[2000] = time.Now().Add(-2 * time.Second) + second, err := m.text(context.Background(), muxl.TextRequest{StartMs: 1000, EndMs: 2000}) + require.NoError(t, err) + require.Len(t, second.Tracks, 1) + require.ElementsMatch(t, []muxl.TextCue{{Start: 1000, End: 1300, Text: "late", ID: m.sessionID + "/" + track.ID + "/late"}, {Start: 1400, End: 2000, Text: "crossing", ID: m.sessionID + "/" + track.ID + "/crossing"}}, second.Tracks[0].Cues) + m.coverage(time.UnixMilli(3000)) + m.mediaFinished = true + third, err := m.text(context.Background(), muxl.TextRequest{StartMs: 2000, EndMs: 3000}) + require.NoError(t, err) + require.Equal(t, []muxl.TextCue{{Start: 2000, End: 2300, Text: "crossing", ID: m.sessionID + "/" + track.ID + "/crossing"}}, third.Tracks[0].Cues) +} + +func TestCaptionMasterPolicySwitchPreservesTruthfulTracks(t *testing.T) { + m := newCaptionMaster(context.Background(), "streamer", &config.CLI{}, nil) + m.mediaFinished = true + m.clock(time.UnixMilli(0)) + auto := masterTrack(captions.SourceAuto) + m.hub.Publish(m.streamer, auto, captions.Cue{ID: "auto", Start: time.UnixMilli(100), End: time.UnixMilli(200), Text: "automatic", Final: true}) + first, err := m.text(context.Background(), muxl.TextRequest{EndMs: 1000}) + require.NoError(t, err) + require.Equal(t, CaptionTrackIDBase, first.Tracks[0].TrackID) + require.Equal(t, "auto", first.Tracks[0].Label) + m.setManifest(captionManifest("ingest")) + ingest := masterTrack(captions.SourceIngest) + require.NoError(t, m.push(ingest, []captions.Cue{{ID: "ingest", Start: m.arrival.Add(1200 * time.Millisecond), End: m.arrival.Add(1500 * time.Millisecond), Text: "supplied", Final: true}})) + second, err := m.text(context.Background(), muxl.TextRequest{StartMs: 1000, EndMs: 2000}) + require.NoError(t, err) + require.Len(t, second.Tracks, 1) + require.Equal(t, CaptionTrackIDBase+1, second.Tracks[0].TrackID) + require.Equal(t, "ingest", second.Tracks[0].Label) + m.setManifest(captionManifest("off")) + off, err := m.text(context.Background(), muxl.TextRequest{StartMs: 2000, EndMs: 3000}) + require.NoError(t, err) + require.Empty(t, off.Tracks) +} + +type pushedFixtureManifester struct { + mm *MediaManager + once sync.Once + err error +} + +func (p *pushedFixtureManifester) BuildManifest(ctx context.Context, streamer string, _ int64) ([]byte, error) { + p.once.Do(func() { + value, _ := p.mm.captionMasters.Load(streamer) + master := value.(*captionMaster) + for { + master.mu.Lock() + ready := !master.arrival.IsZero() + arrival := master.arrival + changed := master.changed + master.mu.Unlock() + if ready { + p.err = p.mm.PushCanonicalCaptions(streamer, masterTrack(captions.SourceIngest), []captions.Cue{{ID: "fixture", Start: arrival.Add(900 * time.Millisecond), End: arrival.Add(1100 * time.Millisecond), Text: "signed fixture", Final: true}}) + break + } + select { + case <-changed: + case <-ctx.Done(): + p.err = ctx.Err() + return + } + } + }) + return captionManifest("ingest"), p.err +} + +func TestCaptionMasterOriginStreamingSignedFixture(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + fixture, err := os.ReadFile(getFixture("h264-opus-frag.mp4")) + require.NoError(t, err) + mm := NewOffline(&config.CLI{CaptionsMasterDelay: time.Second}) + ms := newBareSegmentSigner(t) + ms.PrebuiltManifest = nil + ms.manifestBuilder = &pushedFixtureManifester{mm: mm} + events := make(chan *muxl.MuxlEvent, 16) + done := make(chan error, 1) + go func() { done <- mm.SignOriginStream(ctx, ms, bytes.NewReader(fixture), events); close(events) }() + var archived bytes.Buffer + count := 0 + for event := range events { + if event.Type != "signed-segment" { + continue + } + count++ + segment := concatTracksSorted(event.Tracks) + report, err := muxl.RunMuxlVerify(ctx, bytes.NewReader(segment)) + require.NoError(t, err) + var verified struct { + Segments []struct { + ValidationState string `json:"validation_state"` + } `json:"segments"` + } + require.NoError(t, json.Unmarshal([]byte(report), &verified)) + require.Len(t, verified.Segments, 3) + for _, v := range verified.Segments { + require.NotEqual(t, "Invalid", v.ValidationState) + } + archived.Write(segment) + } + require.NoError(t, <-done) + require.Equal(t, 2, count) + tracks, err := muxl.RunMuxlTextTracks(ctx, bytes.NewReader(archived.Bytes())) + require.NoError(t, err) + require.Equal(t, []muxl.TextTrack{{TrackID: CaptionTrackIDBase, Language: "en-US", Label: "ingest"}}, tracks) + cues, err := muxl.RunMuxlReadTextCues(ctx, bytes.NewReader(archived.Bytes()), CaptionTrackIDBase) + require.NoError(t, err) + require.Len(t, cues, 1) + require.Equal(t, uint64(900), cues[0].Start) + require.Equal(t, uint64(1100), cues[0].End) + require.Equal(t, "signed fixture", cues[0].Text) + require.Contains(t, cues[0].ID, "/push-canonical-ingest-en-us/fixture") +} + +func TestCaptionMasterBufferNeverDropsMediaOnHoldOrClose(t *testing.T) { + queue := newIngestByteBuffer() + for range 1000 { + _, err := queue.Write([]byte("media")) + require.NoError(t, err) + } + require.NoError(t, queue.Close()) + got, err := io.ReadAll(queue) + require.NoError(t, err) + require.Equal(t, bytes.Repeat([]byte("media"), 1000), got) +} + +func TestCaptionMasterOriginStreamingRecognizesDecodedAudio(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + fixture, err := os.ReadFile(getFixture("h264-opus-frag.mp4")) + require.NoError(t, err) + engine := &captionTestEngine{enter: make(chan struct{}), release: make(chan struct{})} + mm := NewOffline(&config.CLI{CaptionsMasterDelay: 500 * time.Millisecond}) + mm.STT = engine + ms := newBareSegmentSigner(t) + ms.PrebuiltManifest = captionManifest("auto") + events := make(chan *muxl.MuxlEvent, 16) + done := make(chan error, 1) + go func() { done <- mm.SignOriginStream(ctx, ms, bytes.NewReader(fixture), events); close(events) }() + var archived bytes.Buffer + first := true + for event := range events { + if event.Type != "signed-segment" { + continue + } + if first { + first = false + close(engine.release) + } + archived.Write(concatTracksSorted(event.Tracks)) + } + require.NoError(t, <-done) + tracks, err := muxl.RunMuxlTextTracks(ctx, bytes.NewReader(archived.Bytes())) + require.NoError(t, err) + require.Equal(t, []muxl.TextTrack{{TrackID: CaptionTrackIDBase, Language: "en-US", Label: "auto"}}, tracks) + cues, err := muxl.RunMuxlReadTextCues(ctx, bytes.NewReader(archived.Bytes()), CaptionTrackIDBase) + require.NoError(t, err) + require.Len(t, cues, 1) + require.Equal(t, "held words", cues[0].Text) + require.GreaterOrEqual(t, cues[0].Start, uint64(1000), "late words cannot be written into the already signed first GoP") + require.LessOrEqual(t, cues[0].End, uint64(2000)) + report, err := muxl.RunMuxlVerify(ctx, bytes.NewReader(archived.Bytes())) + require.NoError(t, err) + var verified struct { + Segments []struct { + ValidationState string `json:"validation_state"` + } `json:"segments"` + } + require.NoError(t, json.Unmarshal([]byte(report), &verified)) + for _, track := range verified.Segments { + require.NotEqual(t, "Invalid", track.ValidationState) + } +} + +func TestCaptionMasterVoicedEOFFinalsReachLastSignedGoP(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + fixture, err := os.ReadFile(getFixture("h264-opus-frag.mp4")) + require.NoError(t, err) + engine := &captionTestEngine{scripted: &stt.Result{Language: "en-US", Words: []stt.Word{{Text: "final voiced words", Start: 1500 * time.Millisecond, End: 1800 * time.Millisecond, Prob: 0.99}}}} + mm := NewOffline(&config.CLI{CaptionsMasterDelay: time.Second}) + mm.STT = engine + ms := newBareSegmentSigner(t) + ms.PrebuiltManifest = captionManifest("auto") + events := make(chan *muxl.MuxlEvent, 16) + done := make(chan error, 1) + go func() { done <- mm.SignOriginStream(ctx, ms, bytes.NewReader(fixture), events); close(events) }() + var last []byte + for event := range events { + if event.Type == "signed-segment" { + last = concatTracksSorted(event.Tracks) + } + } + require.NoError(t, <-done) + cues, err := muxl.RunMuxlReadTextCues(ctx, bytes.NewReader(last), CaptionTrackIDBase) + require.NoError(t, err) + require.Len(t, cues, 1) + require.Equal(t, "final voiced words", cues[0].Text) + require.Equal(t, uint64(1500), cues[0].Start) +} diff --git a/pkg/media/captions_order_test.go b/pkg/media/captions_order_test.go new file mode 100644 index 000000000..4a53796c7 --- /dev/null +++ b/pkg/media/captions_order_test.go @@ -0,0 +1,36 @@ +package media + +import ( + "bytes" + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/muxl" +) + +func TestCaptionMasterCanonicalNumericInterleave(t *testing.T) { + ctx := context.Background() + ms := newBareSegmentSigner(t) + fixture, err := os.ReadFile(getFixture("h264-opus-frag.mp4")) + require.NoError(t, err) + in := muxl.SignerInput{CertPEM: ms.Cert, Sign: muxl.SignerToCallback(ms.Signer, 32), TrackManifest: captionManifest("ingest"), TextFn: func(_ context.Context, req muxl.TextRequest) (*muxl.TextAttachment, error) { + return &muxl.TextAttachment{Tracks: []muxl.TextTrackAttachment{ + {TextTrack: muxl.TextTrack{TrackID: 3, Language: "en", Label: "human"}, Cues: []muxl.TextCue{{Start: req.StartMs, End: req.EndMs, Text: "third"}}}, + {TextTrack: muxl.TextTrack{TrackID: 10, Language: "es", Label: "human"}, Cues: []muxl.TextCue{{Start: req.StartMs, End: req.EndMs, Text: "tenth"}}}, + }}, nil + }} + events := make(chan *muxl.MuxlEvent, 16) + require.NoError(t, muxl.RunMuxlSignSegment(ctx, bytes.NewReader(fixture), in, nil, nil, events)) + close(events) + for event := range events { + if event.Type == "signed-segment" { + segment := concatTracksSorted(event.Tracks) + for _, id := range []string{"1", "2", "3", "10"} { + require.True(t, bytes.HasPrefix(segment, event.Tracks[id]), "canonical byte order at track %s", id) + segment = segment[len(event.Tracks[id]):] + } + } + } +} diff --git a/pkg/media/captions_transcode_test.go b/pkg/media/captions_transcode_test.go new file mode 100644 index 000000000..103730fb9 --- /dev/null +++ b/pkg/media/captions_transcode_test.go @@ -0,0 +1,94 @@ +package media + +import ( + "bytes" + "context" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/crypto/signers" + "stream.place/streamplace/pkg/muxl" + "sync" + "testing" + "time" +) + +func TestCaptionMasterCanonicalNamespaceSurvivesAudioCompletion(t *testing.T) { + for _, late := range []bool{false, true} { + name := "initial" + if late { + name = "late" + } + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + signer := newBareSegmentSigner(t) + key, err := signers.MarshalES256KPrivateKeyPEM(signer.Signer) + require.NoError(t, err) + // Three GoPs catch an unknown text track that qtdemux only rejects + // when the following AV fragment arrives; a two-GoP input misses it. + fixture := runSynthPipeline(t, ctx, + "videotestsrc num-buffers=90 pattern=ball ! video/x-raw,width=320,height=240,framerate=30/1 ! x264enc key-int-max=30 tune=zerolatency speed-preset=ultrafast ! h264parse ! mp4mux name=mux fragment-duration=500 ! appsink name=sink "+ + "audiotestsrc num-buffers=141 samplesperbuffer=1024 ! audio/x-raw,rate=48000,channels=2 ! audioconvert ! opusenc ! mux.") + events := make(chan *muxl.MuxlEvent, 16) + done := make(chan error, 1) + go func() { + done <- muxl.RunMuxlSignSegment(ctx, bytes.NewReader(fixture), muxl.SignerInput{ + CertPEM: signer.Cert, KeyPEM: key, TrackManifest: signer.PrebuiltManifest, + TextFn: func(_ context.Context, r muxl.TextRequest) (*muxl.TextAttachment, error) { + if late && r.StartMs == 0 { + return nil, nil + } + return &muxl.TextAttachment{Tracks: []muxl.TextTrackAttachment{{TextTrack: muxl.TextTrack{TrackID: CaptionTrackIDBase, Language: "en-US", Label: "human"}, Cues: []muxl.TextCue{{ID: "human", Start: r.StartMs + 10, End: r.StartMs + 50, Text: "words survive AAC"}}}}}, nil + }, + }, nil, nil, events) + close(events) + }() + mm := NewOffline(&config.CLI{BroadcasterHost: "test.example.com"}) + var mu sync.Mutex + var completed, sources [][]byte + tr := mm.newStreamTranscoder(ctx, "aac", signer.Cert, key, func(_ any, b []byte) { mu.Lock(); completed = append(completed, b); mu.Unlock() }) + defer tr.Close() + for event := range events { + if event.Type == "signed-segment" { + source := concatTracksSorted(event.Tracks) + sources = append(sources, source) + require.NoError(t, tr.Feed(source, nil)) + } + } + require.NoError(t, <-done) + require.NoError(t, tr.Close()) + mu.Lock() + defer mu.Unlock() + require.Len(t, completed, len(sources), "late track declarations must not terminate continuous audio completion") + if late { + require.GreaterOrEqual(t, len(sources), 3, "exercise another AV GoP after declaring text") + } + for index, segment := range completed { + require.True(t, bytes.HasPrefix(segment, sources[index]), "audio completion must retain every signed source byte") + events, err := unwrapMuxlEvents(ctx, segment) + require.NoError(t, err) + catalog, tracks := catalogAndTracks(events) + require.Contains(t, tracks, "1") + require.Contains(t, tracks, "2") + require.Contains(t, tracks, "3") + require.NotNil(t, catalog.Audio) + require.Len(t, catalog.Audio.Renditions, 2) + cues, err := muxl.RunMuxlReadTextCues(ctx, bytes.NewReader(segment), CaptionTrackIDBase) + if late && index == 0 { + require.NotContains(t, tracks, "100") + } else { + require.Contains(t, tracks, "100") + require.NoError(t, err) + require.Len(t, cues, 1) + require.Equal(t, "words survive AAC", cues[0].Text) + } + report, err := muxl.RunMuxlVerify(ctx, bytes.NewReader(segment)) + require.NoError(t, err) + require.NotContains(t, report, `"validation_state":"Invalid"`) + media, err := ValidateMP4Media(ctx, segment) + require.NoError(t, err) + require.Equal(t, 48000, media.MediaData.Audio[0].Rate, "the default audio rendition remains decodable") + } + }) + } +} diff --git a/pkg/media/ingest_daemon.go b/pkg/media/ingest_daemon.go index cab8825ba..b05e1a566 100644 --- a/pkg/media/ingest_daemon.go +++ b/pkg/media/ingest_daemon.go @@ -123,6 +123,7 @@ func readWorkerMeta(socketPath string) (workerMeta, error) { func removeWorkerFiles(socketPath string) { _ = os.Remove(socketPath) _ = os.Remove(workerMetaPath(socketPath)) + _ = os.Remove(socketPath + ".captions") } // ConsumeWorkerSocket connects to a worker's frame socket and feeds its segments @@ -135,6 +136,8 @@ func removeWorkerFiles(socketPath string) { // over the same socket (pushManifestUpdates) — so a pre-live → live transition // reaches a worker that has no model of its own. It's re-armed per connection. func (mm *MediaManager) ConsumeWorkerSocket(ctx context.Context, socketPath, streamer string, onSegment func([]byte) error, manifestSource func() ([]byte, error)) error { + unregister := mm.registerWorkerCaptionMaster(ctx, socketPath, streamer) + defer unregister() connectedOnce := false giveUp := time.Now().Add(workerConnectGrace) for { @@ -404,10 +407,12 @@ func (mm *MediaManager) WHIPIngestDetached(ctx context.Context, offerSDP string, return "", err } _ = conn.SetReadDeadline(time.Time{}) // clear; streaming has no deadline + unregister := mm.registerWorkerCaptionMaster(ctx, cfg.SocketPath, ms.Streamer()) // Consume the signed segments in the background; the HTTP handler returns the // answer now and the WebRTC media establishes directly to the worker. go func() { + defer unregister() // Ban / key revocation: watch on the detached worker's behalf and kill it. // Scoped to this consume's lifetime so it doesn't outlive the stream. wctx, wcancel := context.WithCancel(ctx) diff --git a/pkg/media/ingest_supervisor.go b/pkg/media/ingest_supervisor.go index 8fd05e10b..de0af372b 100644 --- a/pkg/media/ingest_supervisor.go +++ b/pkg/media/ingest_supervisor.go @@ -11,9 +11,11 @@ import ( "io" "os" "os/exec" + "path/filepath" "sync" "time" + "github.com/google/uuid" "stream.place/streamplace/pkg/crypto/signers" "stream.place/streamplace/pkg/ingestframe" "stream.place/streamplace/pkg/log" @@ -43,6 +45,13 @@ func (mm *MediaManager) MP4IngestIsolated(ctx context.Context, input io.Reader, if err != nil { return err } + dir, err := mm.ingestWorkerSocketDir() + if err != nil { + return err + } + cfg.CaptionSocketPath = filepath.Join(dir, uuid.NewString()+".sock") + defer mm.registerWorkerCaptionMaster(ctx, cfg.CaptionSocketPath, ms.Streamer())() + defer os.Remove(cfg.CaptionSocketPath + ".captions") cfgJSON, err := json.Marshal(cfg) if err != nil { return fmt.Errorf("marshal worker config: %w", err) @@ -264,11 +273,13 @@ func (mm *MediaManager) buildWorkerConfig(ctx context.Context, ms MediaSigner) ( return IngestWorkerConfig{}, fmt.Errorf("build manifest: %w", err) } cfg := IngestWorkerConfig{ - StreamerDID: ms.Streamer(), - KeyPEM: keyPEM, - CertPEM: local.Cert, - Manifest: manifest, - BroadcasterHost: mm.cli.BroadcasterHost, + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: local.Cert, + Manifest: manifest, + BroadcasterHost: mm.cli.BroadcasterHost, + CaptionEngineSocket: mm.CaptionEngineSocket, + CaptionsMasterDelay: mm.cli.CaptionsMasterDelay, } // Debug recording: main owns the per-stream setting (it needs the DB); the // worker carries out the recording (it owns the data path). A lookup failure diff --git a/pkg/media/ingest_worker.go b/pkg/media/ingest_worker.go index f2f54d4b5..b23cb9b91 100644 --- a/pkg/media/ingest_worker.go +++ b/pkg/media/ingest_worker.go @@ -7,6 +7,7 @@ import ( "io" "net/http/httputil" "sync" + "time" "github.com/go-gst/go-gst/gst" "stream.place/streamplace/pkg/config" @@ -14,6 +15,7 @@ import ( "stream.place/streamplace/pkg/log" "stream.place/streamplace/pkg/muxl" "stream.place/streamplace/pkg/s3" + "stream.place/streamplace/pkg/stt" ) // manifestHolder holds the worker's current C2PA manifest. It starts as the @@ -56,12 +58,10 @@ type IngestWorkerConfig struct { // forwarded verbatim, no reconstruction. KeyPEM []byte `json:"key_pem"` CertPEM []byte `json:"cert_pem"` - // Manifest is the C2PA manifest JSON, built ONCE by main at stream start. - // muxl-sign stamps each segment's signing time into it as it signs. NOTE: - // static for the worker's lifetime — mid-stream manifest changes (e.g. a - // pre-live → live transition) don't yet cross the boundary; that needs a - // control channel and is tracked as future work. - Manifest []byte `json:"manifest"` + // Manifest is refreshed through the worker's existing control channel. + Manifest []byte `json:"manifest"` + CaptionEngineSocket string `json:"caption_engine_socket"` + CaptionsMasterDelay time.Duration `json:"captions_master_delay"` // Node transcode signer + broadcaster identity. When set, the worker completes // each single-codec source segment to dual-codec (Opus+AAC) itself — the @@ -75,6 +75,9 @@ type IngestWorkerConfig struct { // it serves frames over this unix socket with buffered reconnect (survives a // main restart) instead of the fd-4 pipe. Empty → fd-4 pipe (Stage 1). SocketPath string `json:"socket_path,omitempty"` + // CaptionSocketPath supplies the private caption-control socket for the fd + // transport without switching its signed-segment transport to sockets. + CaptionSocketPath string `json:"caption_socket_path,omitempty"` // InputFD, when > 0, is the fd main passed the ingest CONNECTION on (fd-passing // the accepted, authed push). The worker reads media from it directly instead @@ -122,7 +125,7 @@ const IngestTransportWHIP = "whip" // handed its config over the handshake, else local disk under DataDir). Shared // by the MP4 and WHIP workers so both record to the same place main would. func (cfg IngestWorkerConfig) workerCLI() *config.CLI { - cli := &config.CLI{BroadcasterHost: cfg.BroadcasterHost, DataDir: cfg.DataDir} + cli := &config.CLI{BroadcasterHost: cfg.BroadcasterHost, DataDir: cfg.DataDir, CaptionsMasterDelay: cfg.CaptionsMasterDelay} if cfg.S3 != nil { cli.SetS3Config(*cfg.S3) } @@ -152,12 +155,30 @@ func WorkerInput(cfg IngestWorkerConfig, raw io.Reader) io.Reader { // workers. func workerSignStream(cfg IngestWorkerConfig, getManifest func() []byte) SignSegmentStreamFunc { return func(ctx context.Context, input io.Reader, eventCh chan *muxl.MuxlEvent) error { - fetchManifest := func() ([]byte, error) { return getManifest(), nil } + cli := cfg.workerCLI() + engine := stt.NewProxy(cfg.CaptionEngineSocket) + defer engine.Close() + master := newCaptionMaster(ctx, cfg.StreamerDID, cli, engine) + master.setManifest(getManifest()) + captionPath := cfg.CaptionSocketPath + if captionPath == "" { + captionPath = cfg.SocketPath + } + stop, err := master.servePush(captionPath) + if err != nil { + return fmt.Errorf("serve canonical caption pushes: %w", err) + } + defer stop() + input, finish := master.tee(input) + defer finish() + fetchManifest := func() ([]byte, error) { data := getManifest(); master.setManifest(data); return data, nil } return muxl.RunMuxlSignSegment(ctx, input, muxl.SignerInput{ CertPEM: cfg.CertPEM, KeyPEM: cfg.KeyPEM, TrackManifestFn: fetchManifest, WrapperManifestFn: fetchManifest, + TextFn: master.text, + SegmentTimeFn: master.segmentTime, }, nil, nil, eventCh) } } diff --git a/pkg/media/media.go b/pkg/media/media.go index 14f92680e..8b92082de 100644 --- a/pkg/media/media.go +++ b/pkg/media/media.go @@ -25,6 +25,7 @@ import ( "stream.place/streamplace/pkg/localdb" "stream.place/streamplace/pkg/model" "stream.place/streamplace/pkg/placestream" + "stream.place/streamplace/pkg/stt" "stream.place/streamplace/pkg/log" @@ -41,9 +42,13 @@ const SegmentsDir = "segments" const StreamplaceMetadata = "cawg.metadata" type MediaManager struct { - cli *config.CLI - liveWindows map[string]*livehls.Writer - liveWindowsMut sync.Mutex + cli *config.CLI + // STT is the node-wide speech engine; nil disables auto recognition safely. + STT stt.Engine + CaptionEngineSocket string + captionMasters sync.Map // streamer DID -> captionSession + liveWindows map[string]*livehls.Writer + liveWindowsMut sync.Mutex // liveWindowPublished is, per streamer, whether the latest segment fed // into the window was published; guarded by liveWindowsMut. liveWindowPublished map[string]bool @@ -103,7 +108,9 @@ type MediaManager struct { // per-DID transcoder rebuilds when a streamer reconnects rather than feeding // the restarted media timeline into the previous session's continuous encoder. // See withIngestSession / feedStreamTranscoder. - ingestSessionSeq atomic.Uint64 + ingestSessionSeq atomic.Uint64 + captionDistributionOnce sync.Once + captionDistribution *captionDistribution } // nextIngestSession claims a fresh monotonic ingest-session epoch for a new live diff --git a/pkg/media/media_signer.go b/pkg/media/media_signer.go index e94e5dad1..58d984345 100644 --- a/pkg/media/media_signer.go +++ b/pkg/media/media_signer.go @@ -22,6 +22,7 @@ import ( "stream.place/streamplace/pkg/log" "stream.place/streamplace/pkg/model" "stream.place/streamplace/pkg/muxl" + "stream.place/streamplace/pkg/stt" ) var signerTracer = otel.Tracer("signer") @@ -56,6 +57,7 @@ type MediaSignerLocal struct { manifestBuilder Manifester PrebuiltManifest []byte // Optional: use this manifest instead of building one sigs [][]byte + cli *config.CLI } func prepareCert(ctx context.Context, cli *config.CLI, signer crypto.Signer) ([]byte, error) { @@ -89,6 +91,7 @@ func MakeMediaSigner(ctx context.Context, cli *config.CLI, streamer string, sign AQPub: pub, did: did.DIDKey(), manifestBuilder: NewManifestBuilder(model, cli), + cli: cli, }, nil } @@ -156,17 +159,39 @@ func (ms *MediaSignerLocal) SignSegmentStream(ctx context.Context, input io.Read attribute.String("streamer", ms.StreamerName), )) defer span.End() + mm, _ := ctx.Value(captionManagerKey{}).(*MediaManager) + cli := ms.cli + if mm != nil { + cli = mm.cli + } + if cli == nil { + cli = &config.CLI{} + } + var engine stt.Engine + if mm != nil { + engine = mm.STT + } + master := newCaptionMaster(ctx, ms.StreamerName, cli, engine) + if mm != nil { + defer mm.registerCaptionMaster(ms.StreamerName, master)() + } // One callback shared by both kinds — track and wrapper manifests are the // same JSON in Streamplace today, so a single buildManifest call per GoP // covers both. If they ever diverge we split this in two. fetchManifest := func() ([]byte, error) { - return ms.buildManifest(ctx, time.Now().UnixMilli()) + data, err := ms.buildManifest(ctx, time.Now().UnixMilli()) + if err == nil { + master.setManifest(data) + } + return data, err } in := muxl.SignerInput{ CertPEM: ms.Cert, TrackManifestFn: fetchManifest, WrapperManifestFn: fetchManifest, + TextFn: master.text, + SegmentTimeFn: master.segmentTime, } if _, ok := ms.Signer.(*ecdsa.PrivateKey); ok { keyPEM, err := signers.MarshalES256KPrivateKeyPEM(ms.Signer) @@ -179,6 +204,8 @@ func (ms *MediaSignerLocal) SignSegmentStream(ctx context.Context, input io.Read in.Sign = muxl.SignerToCallback(ms.Signer, 32) span.SetAttributes(attribute.String("backend", "host-callback")) } + input, finish := master.tee(input) + defer finish() return muxl.RunMuxlSignSegment(ctx, input, in, nil, nil, eventCh) } diff --git a/pkg/media/muxl_segment.go b/pkg/media/muxl_segment.go index 112c2cbad..e1de19711 100644 --- a/pkg/media/muxl_segment.go +++ b/pkg/media/muxl_segment.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "sort" + "strconv" "github.com/go-gst/go-gst/gst" "github.com/go-gst/go-gst/gst/app" @@ -98,15 +99,15 @@ func muxlSignSegmentElem(ctx context.Context, cli *config.CLI, signStream SignSe return nil, nil, fmt.Errorf("failed to link mp4mux to appsink: %w", err) } - r, w := io.Pipe() + r := newIngestByteBuffer(ctx) go func() { <-ctx.Done() r.Close() }() // The signer and its event drain run on a non-cancellable ctx: cancelling - // ctx is the FLUSH signal, not an abort — it closes the input pipe above, - // the signer sees EOF, signs the final GoP, and exits cleanly. If the + // ctx is the FLUSH signal, not an abort — it closes the byte buffer above, + // the signer drains queued media, signs the final GoP, and exits cleanly. If the // cancelled ctx reached muxl's event parser instead, the parser would // abandon the stream mid-write and the signer wasm would deadlock against // the unread stdout pipe — done would never close and the caller's drain @@ -120,6 +121,9 @@ func muxlSignSegmentElem(ctx context.Context, cli *config.CLI, signStream SignSe eventCh := make(chan *muxl.MuxlEvent, 16) go func() { err := signStream(drainCtx, r, eventCh) + if err != nil { + _ = r.CloseWithError(err) + } close(eventCh) if err != nil && ctx.Err() == nil { log.Error(ctx, "error running muxl sign-segment", "error", err) @@ -142,7 +146,7 @@ func muxlSignSegmentElem(ctx context.Context, cli *config.CLI, signStream SignSe sink := app.SinkFromElement(appsink) sink.SetCallbacks(&app.SinkCallbacks{ - NewSampleFunc: WriterNewSample(ctx, w), + NewSampleFunc: WriterNewSample(ctx, r), }) return bin.Element, done, nil @@ -152,14 +156,21 @@ func muxlSignSegmentElem(ctx context.Context, cli *config.CLI, signStream SignSe // in ascending track-id order — the canonical interleave a multi-track .m4s // uses, which muxl's unwrap/verify/wrap all expect. func concatTracksSorted(tracks map[string][]byte) []byte { - keys := make([]string, 0, len(tracks)) - for k := range tracks { - keys = append(keys, k) + type numericTrack struct { + id uint32 + key string + } + keys := make([]numericTrack, 0, len(tracks)) + size := 0 + for k, data := range tracks { + id, _ := strconv.ParseUint(k, 10, 32) // signer event keys are uint32 track IDs + keys = append(keys, numericTrack{id: uint32(id), key: k}) + size += len(data) } - sort.Strings(keys) - var out []byte + sort.Slice(keys, func(i, j int) bool { return keys[i].id < keys[j].id }) + out := make([]byte, 0, size) for _, k := range keys { - out = append(out, tracks[k]...) + out = append(out, tracks[k.key]...) } return out } diff --git a/pkg/media/segmenter.go b/pkg/media/segmenter.go index 9269efb2b..3d8586d6b 100644 --- a/pkg/media/segmenter.go +++ b/pkg/media/segmenter.go @@ -186,6 +186,7 @@ func (mm *MediaManager) SegmentAndSignElem(ctx context.Context, ms MediaSigner) // per-DID continuous transcoder keys on this epoch and rebuilds rather than // feeding the restarted timeline into the previous session's encoder. ctx = withIngestSession(ctx, mm.nextIngestSession()) + ctx = withCaptionManager(ctx, mm) // muxl path: stream the fMP4 through the per-segment signer. Each GoP // arrives as a bare canonical .m4s, which ValidateMP4 verifies, archives diff --git a/pkg/media/transcode_stream.go b/pkg/media/transcode_stream.go index 79ce2d6b5..94f6891e5 100644 --- a/pkg/media/transcode_stream.go +++ b/pkg/media/transcode_stream.go @@ -249,11 +249,16 @@ func (t *streamTranscoder) Feed(src []byte, token any) error { t.started = true t.mu.Unlock() - // Synthesize and write the init (ftyp+moov) once, then the segment bytes — - // the same init-then-blind-concat the RTMP push feeder uses. + // Native qtdemux sees one immutable init, so a lazily declared text track + // cannot enter its input. Keep the original signed source in the completion + // job, but decode only video and the audio codec being transcoded. + decodeSrc, err := filterSegmentToCodec(t.ctx, src, t.target == "aac") + if err != nil { + return fmt.Errorf("filter transcoder input: %w", err) + } if first { var init bytes.Buffer - if err := muxl.RunMuxlWrapInit(t.ctx, bytes.NewReader(src), &init); err != nil { + if err := muxl.RunMuxlWrapInit(t.ctx, bytes.NewReader(decodeSrc), &init); err != nil { return fmt.Errorf("synthesize transcoder init: %w", err) } if _, err := t.feedW.Write(init.Bytes()); err != nil { @@ -267,7 +272,7 @@ func (t *streamTranscoder) Feed(src []byte, token any) error { case <-t.ctx.Done(): return t.ctx.Err() } - if _, err := t.feedW.Write(src); err != nil { + if _, err := t.feedW.Write(decodeSrc); err != nil { return err } return nil diff --git a/pkg/media/validate.go b/pkg/media/validate.go index 21f12a5e5..f51113a83 100644 --- a/pkg/media/validate.go +++ b/pkg/media/validate.go @@ -274,6 +274,7 @@ func (mm *MediaManager) distributeSegment(ctx context.Context, vs *validatedSegm if err := muxl.RunMuxlWrap(ctx, bytes.NewReader(seg), "flat", &playable); err != nil { return fmt.Errorf("wrap segment for distribution: %w", err) } + mm.distributeCaptions(ctx, vs, seg, playable.Bytes()) mm.notifySubscribers(ctx, &NewSegmentNotification{ Segment: dbSeg, diff --git a/pkg/multitest/captions_test.go b/pkg/multitest/captions_test.go new file mode 100644 index 000000000..2ac2edd5a --- /dev/null +++ b/pkg/multitest/captions_test.go @@ -0,0 +1,403 @@ +package multitest + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "github.com/gorilla/websocket" + "github.com/julienschmidt/httprouter" + "github.com/mr-tron/base58" + "github.com/slok/go-http-metrics/metrics" + "github.com/slok/go-http-metrics/middleware" + "github.com/streamplace/oatproxy/pkg/oatproxy" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/api" + "stream.place/streamplace/pkg/atproto" + "stream.place/streamplace/pkg/bus" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/crypto/spkey" + "stream.place/streamplace/pkg/gstinit" + "stream.place/streamplace/pkg/localdb" + "stream.place/streamplace/pkg/media" + "stream.place/streamplace/pkg/model" + "stream.place/streamplace/pkg/muxl" + "stream.place/streamplace/pkg/placestream" + "stream.place/streamplace/pkg/replication/websocketrep" + "stream.place/streamplace/pkg/spxrpc" + "stream.place/streamplace/pkg/stt" +) + +const captionTimeout = 30 * time.Second + +// These nodes run the real validator, signer, syndication client/server and +// public caption handlers in process. Unlike the binary-based legacy harness, +// this uses MediaManager.STT's existing injection point: no production seam or +// bundled model, CPU scheduling, remote fixture, PDS, or fixed port is needed. +type captionNode struct { + cli *config.CLI + mod model.Model + bus *bus.Bus + mm *media.MediaManager + server *httptest.Server + segments <-chan *media.NewSegmentNotification + engine *captionEngine +} + +type captionEngine struct { + leases atomic.Int32 + passes atomic.Int32 +} + +func (e *captionEngine) Lease(context.Context, stt.LeaseOptions) (stt.Lease, error) { + e.leases.Add(1) + return e, nil +} +func (e *captionEngine) Models() []stt.ModelInfo { return []stt.ModelInfo{e.Info()} } +func (*captionEngine) Close() error { return nil } +func (e *captionEngine) Model() stt.Model { return e } +func (*captionEngine) Release() {} +func (*captionEngine) Info() stt.ModelInfo { return stt.ModelInfo{Name: "multitest-deterministic"} } +func (e *captionEngine) Transcribe(ctx context.Context, _ []float32, _ stt.Options) (*stt.Result, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + e.passes.Add(1) + return &stt.Result{Language: "en", Words: []stt.Word{{Text: "Deterministic sidecar.", Start: 250 * time.Millisecond, End: 750 * time.Millisecond, Prob: 1}}}, nil +} + +func newCaptionNode(t *testing.T, ctx context.Context, name string) *captionNode { + t.Helper() + cli := &config.CLI{WideOpen: true, Captions: true, CaptionsMasterDelay: time.Second, DataDir: t.TempDir(), BroadcasterHost: name + ".example", ServerHost: name + ".example", Syndicate: []string{"*"}} + mod, err := model.MakeDB(":memory:") + require.NoError(t, err) + ldb, err := localdb.MakeDB(":memory:") + require.NoError(t, err) + b := bus.NewBus() + atsync := &atproto.ATProtoSynchronizer{CLI: cli, Model: mod, Bus: b} + mm, err := media.MakeMediaManager(ctx, cli, nil, mod, b, atsync, ldb) + require.NoError(t, err) + engine := &captionEngine{} + mm.STT = engine + op := oatproxy.New(&oatproxy.Config{Host: cli.BroadcasterHost, Scope: atproto.OAuthString, Public: true}) + xrpc, err := spxrpc.NewServer(ctx, cli, mod, nil, op, middleware.New(middleware.Config{Recorder: metrics.Dummy}), atsync, b, ldb, mm, nil, nil, nil, nil) + require.NoError(t, err) + a := &api.StreamplaceAPI{CLI: cli, Model: mod, LocalDB: ldb, MediaManager: mm, Bus: b, ATSync: atsync} + router := httprouter.New() + router.GET("/api/websocket/:repoDID", a.HandleWebsocket(ctx)) + router.Handler("GET", "/xrpc/*path", xrpc) + router.Handler("POST", "/xrpc/*path", xrpc) + server := httptest.NewServer(router) + t.Cleanup(server.Close) + cli.WebsocketURL = "ws" + strings.TrimPrefix(server.URL, "http") + return &captionNode{cli: cli, mod: mod, bus: b, mm: mm, server: server, segments: mm.NewSegment(), engine: engine} +} + +// Same bounded synthetic fMP4 pattern as media's ingest tests, with continuous +// non-silent Opus audio long enough for the real recognizer's two-pass agreement. +func captionMedia(t *testing.T, ctx context.Context) []byte { + t.Helper() + gstinit.InitGST() + pipeline, err := gst.NewPipelineFromString("videotestsrc num-buffers=180 ! video/x-raw,width=160,height=120,framerate=30/1 ! x264enc key-int-max=30 tune=zerolatency speed-preset=ultrafast ! h264parse ! mp4mux name=mux fragment-duration=100 ! appsink name=sink sync=false audiotestsrc num-buffers=300 samplesperbuffer=960 ! audio/x-raw,rate=48000,channels=1 ! audioconvert ! opusenc ! opusparse ! mux.") + require.NoError(t, err) + sink, err := pipeline.GetElementByName("sink") + require.NoError(t, err) + var data bytes.Buffer + app.SinkFromElement(sink).SetCallbacks(&app.SinkCallbacks{NewSampleFunc: media.WriterNewSample(ctx, &data)}) + done := make(chan error, 1) + go func() { done <- media.HandleBusMessages(ctx, pipeline) }() + require.NoError(t, pipeline.SetState(gst.StatePlaying)) + defer func() { _ = pipeline.SetState(gst.StateNull) }() + select { + case err := <-done: + require.NoError(t, err) + case <-ctx.Done(): + t.Fatal("caption media generator timed out") + } + return data.Bytes() +} + +func captionSigner(t *testing.T, ctx context.Context, origin *captionNode, canonical string, allowed bool) (media.MediaSigner, string, string) { + t.Helper() + priv, pub, err := spkey.GenerateStreamKey() + require.NoError(t, err) + did := pub.DIDKey() + require.NoError(t, origin.mod.UpdateSigningKey(&model.SigningKey{DID: did, RepoDID: did, RKey: "captions"})) + signer, err := spkey.KeyToSigner(priv) + require.NoError(t, err) + ms, err := media.MakeMediaSigner(ctx, origin.cli, did, signer, origin.mod) + require.NoError(t, err) + ms.(*media.MediaSignerLocal).PrebuiltManifest = []byte(fmt.Sprintf(`{"title":"caption multitest","assertions":[{"label":"c2pa.actions.v2","data":{"actions":[{"action":"c2pa.created"},{"action":"c2pa.published"}]}},{"label":"cawg.metadata","data":{"@context":{"dc":"http://purl.org/dc/elements/1.1/"},"dc:creator":%q,"dc:title":"captions","dc:date":"2026-10-01T00:00:00.000Z"}},{"label":"place.stream.metadata.configuration","data":{"captionPolicy":{"canonical":%q,"allowNodeCaptions":%t,"languages":["en"]}}}]}`, did, canonical, allowed)) + t.Cleanup(func() { origin.mm.EndCaptionSession(did) }) + return ms, did, "z" + base58.Encode(priv.Bytes()) +} + +func captionSegment(event *muxl.MuxlEvent) []byte { + // Keep the primary AV manifest first, including when IDs are not contiguous. + ids := make([]string, 0, len(event.Tracks)) + for id := range event.Tracks { + ids = append(ids, id) + } + sort.Strings(ids) + var data []byte + for _, id := range ids { + data = append(data, event.Tracks[id]...) + } + return data +} + +func ingestCaptionSegments(t *testing.T, ctx context.Context, origin *captionNode, segments [][]byte) { + t.Helper() + for _, segment := range segments { + require.NoError(t, origin.mm.ValidateMP4(ctx, bytes.NewReader(segment), true)) + select { + case not := <-origin.segments: + // The binary node's director publishes these validated notifications + // on the source bus. Keep only that boundary in this focused harness. + origin.bus.PublishSegment(ctx, not.Segment.RepoDID, "source", &bus.Seg{Muxl: not.Muxl, Published: not.Metadata.Published}) + case <-ctx.Done(): + t.Fatal("origin did not distribute its signed segment") + } + } +} + +func signCaptionSegments(t *testing.T, ctx context.Context, origin *captionNode, ms media.MediaSigner, input io.Reader) [][]byte { + t.Helper() + events := make(chan *muxl.MuxlEvent, 32) + done := make(chan error, 1) + go func() { done <- origin.mm.SignOriginStream(ctx, ms, input, events); close(events) }() + var segments [][]byte + for { + select { + case event, ok := <-events: + if !ok { + require.NoError(t, <-done) + return segments + } + if event.Type == "signed-segment" { + segments = append(segments, captionSegment(event)) + } + case <-ctx.Done(): + t.Fatal("origin signing timed out") + } + } +} + +func syndicateCaptionStream(t *testing.T, ctx context.Context, relay, origin *captionNode, did string, segments [][]byte) { + t.Helper() + wsURL := origin.cli.WebsocketURL + "/xrpc/place.stream.live.subscribeSegments?streamer=" + url.QueryEscape(did) + record := placestream.BroadcastOrigin{Streamer: did, Server: origin.cli.ServerDID(), WebsocketURL: &wsURL, UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano)} + uri := syntax.ATURI(fmt.Sprintf("at://%s/place.stream.broadcast.origin/%s::%s", did, did, record.Server)) + require.NoError(t, relay.mod.UpdateBroadcastOrigin(ctx, record, uri)) + r := websocketrep.NewWebsocketReplicator(relay.bus, relay.mod, relay.mm, nil) + done := make(chan error, 1) + go func() { done <- r.Start(ctx, relay.cli) }() + t.Cleanup(func() { relay.mm.EndCaptionSession(did) }) + // A published window alone only proves the first cached GoP arrived. + // Drain every replayed segment before asserting recognition stayed off. + replayed := segments[max(0, len(segments)-2):] + for _, expected := range replayed { + select { + case not := <-relay.segments: + require.Equal(t, did, not.Segment.RepoDID) + require.Equal(t, expected, not.Muxl, "syndication must preserve signed media bytes") + case <-ctx.Done(): + t.Fatal("relay never validated all syndicated public media") + } + } + require.True(t, relay.mm.LiveWindowPublished(did)) + select { + case err := <-done: + require.NoError(t, err) + default: + } +} + +func captionTracks(t *testing.T, node *captionNode, did string) []placestream.CaptionDefs_TrackView { + t.Helper() + client := &http.Client{Timeout: 2 * time.Second} + resp, err := client.Get(node.server.URL + "/xrpc/place.stream.caption.listTracks?streamer=" + url.QueryEscape(did)) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + var out placestream.CaptionListTracks_Output + require.NoError(t, json.NewDecoder(resp.Body).Decode(&out)) + return out.Tracks +} + +func publicCaption(t *testing.T, ctx context.Context, node *captionNode, did, text string) placestream.CaptionDefs_LiveCue { + t.Helper() + conn, _, err := websocket.DefaultDialer.DialContext(ctx, node.cli.WebsocketURL+"/api/websocket/"+did, nil) + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.SetReadDeadline(time.Now().Add(captionTimeout))) + for { + _, body, err := conn.ReadMessage() + require.NoError(t, err, "relay did not publish expected live cue %q", text) + var cue placestream.CaptionDefs_LiveCue + if json.Unmarshal(body, &cue) == nil && cue.LexiconTypeID == "place.stream.caption.defs#liveCue" && cue.Text == text { + return cue + } + } +} + +func TestCaptionCanonicalPassthrough(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + origin := newCaptionNode(t, ctx, "caption-origin") + relay := newCaptionNode(t, ctx, "caption-relay") + ms, did, key := captionSigner(t, ctx, origin, "ingest", true) + _, file, _, _ := runtime.Caller(0) + input, err := os.ReadFile(filepath.Join(filepath.Dir(file), "..", "..", "test", "fixtures", "h264-opus-frag.mp4")) + require.NoError(t, err) + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + events := make(chan *muxl.MuxlEvent, 32) + done := make(chan error, 1) + started := time.Now() + go func() { done <- origin.mm.SignOriginStream(ctx, ms, reader, events); close(events) }() + writeDone := make(chan error, 1) + go func() { _, err := writer.Write(input); writeDone <- err }() + // The first GoP establishes the signed ingest policy. Push into the next + // unsigned GoP, keeping input open so session ownership remains live. + for { + select { + case event := <-events: + require.NotNil(t, event) + if event.Type == "signed-segment" { + goto live + } + case <-ctx.Done(): + t.Fatal("first signed GoP did not arrive") + } + } +live: + id := "canonical-known" + body, err := json.Marshal(&placestream.CaptionPushCaptions_Input{Streamer: &did, Language: "en", Cues: []placestream.CaptionDefs_PushedCue{{Id: &id, StartTime: started.Add(250 * time.Millisecond).Format(time.RFC3339Nano), EndTime: started.Add(750 * time.Millisecond).Format(time.RFC3339Nano), Text: "Pushed words survive syndication."}}}) + require.NoError(t, err) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, origin.server.URL+"/xrpc/place.stream.caption.pushCaptions", bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+key) + resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + require.NoError(t, err) + responseBody, err := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode, "%s", responseBody) + require.Empty(t, origin.bus.Captions.Tracks(did), "pushed canonical cues must not bypass signed MUXL") + select { + case err := <-writeDone: + require.NoError(t, err) + case <-ctx.Done(): + t.Fatal("ingest input blocked") + } + require.NoError(t, writer.Close()) + var segments [][]byte + for { + select { + case event, ok := <-events: + if !ok { + goto mastered + } + if event.Type == "signed-segment" { + segments = append(segments, captionSegment(event)) + } + case <-ctx.Done(): + t.Fatal("canonical mastering timed out") + } + } +mastered: + require.NoError(t, <-done) + // Join once canonical text is introduced. Every segment this relay sees + // now contains the canonical track, so it must never start recognition. + require.NotEmpty(t, segments) + tracks, err := muxl.RunMuxlTextTracks(ctx, bytes.NewReader(segments[0])) + require.NoError(t, err) + require.Len(t, tracks, 1) + require.Equal(t, "human", tracks[0].Label) + archived := bytes.Join(segments, nil) + cues, err := muxl.RunMuxlReadTextCues(ctx, bytes.NewReader(archived), tracks[0].TrackID) + require.NoError(t, err) + require.Len(t, cues, 1) + require.Equal(t, "Pushed words survive syndication.", cues[0].Text) + ingestCaptionSegments(t, ctx, origin, segments) + syndicateCaptionStream(t, ctx, relay, origin, did, segments) + cue := publicCaption(t, ctx, relay, did, cues[0].Text) + require.Equal(t, "canonical", cue.Track.Origin) + require.True(t, cue.Final) + listed := captionTracks(t, relay, did) + require.Len(t, listed, 1) + require.Equal(t, "canonical", listed[0].Origin) + require.Zero(t, origin.engine.leases.Load(), "ingest-only origin must not recognize") + require.Zero(t, relay.engine.leases.Load(), "canonical relay must not recognize") + require.Zero(t, relay.engine.passes.Load()) +} + +func TestCaptionSidecarPassthrough(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + origin := newCaptionNode(t, ctx, "sidecar-origin") + relay := newCaptionNode(t, ctx, "sidecar-relay") + ms, did, _ := captionSigner(t, ctx, origin, "off", true) + segments := signCaptionSegments(t, ctx, origin, ms, bytes.NewReader(captionMedia(t, ctx))) + ingestCaptionSegments(t, ctx, origin, segments) + // Join after an upstream final exists, as a real late-joining relay does. + // The replay must prevent the relay from leasing its own recognizer even + // though the media contains no canonical text track. + require.Eventually(t, func() bool { + return len(origin.bus.Captions.Cues(did, "sidecar-auto-en", time.Time{}, time.Now().Add(time.Minute))) > 0 + }, captionTimeout, 10*time.Millisecond, "fake STT did not produce final sidecar captions from decoded audio") + require.Positive(t, origin.engine.passes.Load()) + syndicateCaptionStream(t, ctx, relay, origin, did, segments) + cue := publicCaption(t, ctx, relay, did, "Deterministic sidecar.") + require.Equal(t, "sidecar", cue.Track.Origin) + require.NotNil(t, cue.Track.Author) + require.Equal(t, origin.cli.ServerDID(), *cue.Track.Author) + listed := captionTracks(t, relay, did) + require.Len(t, listed, 1) + require.Equal(t, "sidecar", listed[0].Origin) + require.Equal(t, origin.cli.ServerDID(), *listed[0].Author) + require.Zero(t, relay.engine.leases.Load(), "upstream sidecar must prevent relay recognition") + require.Zero(t, relay.engine.passes.Load()) +} + +func TestCaptionNodeOptOut(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + origin := newCaptionNode(t, ctx, "optout-origin") + relay := newCaptionNode(t, ctx, "optout-relay") + ms, did, _ := captionSigner(t, ctx, origin, "off", false) + segments := signCaptionSegments(t, ctx, origin, ms, bytes.NewReader(captionMedia(t, ctx))) + ingestCaptionSegments(t, ctx, origin, segments) + syndicateCaptionStream(t, ctx, relay, origin, did, segments) + // A stale node sidecar on the origin's hub must not be syndicated after + // the streamer's signed opt-out. This also exercises the public output + // gate, rather than merely observing the absence of recognition. + now := time.Now() + track := captions.Track{ID: "sidecar-auto-en", Language: "en", Kind: captions.KindCaptions, Source: captions.SourceAuto, Origin: captions.OriginSidecar, Author: origin.cli.ServerDID()} + origin.bus.Captions.Publish(did, track, captions.Cue{ID: "forbidden", Start: now, End: now.Add(time.Second), Text: "Must not be distributed.", Final: true}) + require.Never(t, func() bool { + return len(captionTracks(t, relay, did)) > 0 || relay.engine.leases.Load() > 0 || origin.engine.leases.Load() > 0 + }, time.Second, 20*time.Millisecond, "opt-out must prevent recognition and public sidecars") + require.Empty(t, captionTracks(t, relay, did)) + require.Zero(t, relay.engine.passes.Load()) +} diff --git a/pkg/muxl/muxl.go b/pkg/muxl/muxl.go index 11b2166af..29c01836a 100644 --- a/pkg/muxl/muxl.go +++ b/pkg/muxl/muxl.go @@ -18,15 +18,20 @@ import ( // aliases, so values (events, catalogs, signer inputs) flow between this package // and upstream with no conversion and the existing call sites are unchanged. type ( - MuxlEvent = upstream.Event - MuxlCatalog = upstream.Catalog - MuxlCatalogVideo = upstream.CatalogVideo - MuxlCatalogAudio = upstream.CatalogAudio - MuxlVideoConfig = upstream.VideoConfig - MuxlAudioConfig = upstream.AudioConfig - MuxlContainer = upstream.Container - SignerInput = upstream.SignerInput - TranscodeInput = upstream.TranscodeInput + MuxlEvent = upstream.Event + MuxlCatalog = upstream.Catalog + MuxlCatalogVideo = upstream.CatalogVideo + MuxlCatalogAudio = upstream.CatalogAudio + MuxlVideoConfig = upstream.VideoConfig + MuxlAudioConfig = upstream.AudioConfig + MuxlContainer = upstream.Container + SignerInput = upstream.SignerInput + TextRequest = upstream.TextRequest + TextAttachment = upstream.TextAttachment + TextTrackAttachment = upstream.TextTrackAttachment + TextTrack = upstream.TextTrack + TextCue = upstream.TextCue + TranscodeInput = upstream.TranscodeInput ) // TranscodeIngredientLabel is the C2PA ingredient label SignTranscode assigns @@ -312,3 +317,21 @@ func failedConcatenator(err error) *Concatenator { closeFn: func() error { return err }, } } + +// RunMuxlTextTracks lists archival WebVTT tracks on the shared wasm engine. +func RunMuxlTextTracks(ctx context.Context, input io.Reader) ([]TextTrack, error) { + eng, err := getEngine() + if err != nil { + return nil, err + } + return eng.TextTracks(ctx, input) +} + +// RunMuxlReadTextCues reconstructs cues across canonical GoP boundaries. +func RunMuxlReadTextCues(ctx context.Context, input io.Reader, trackID uint32) ([]TextCue, error) { + eng, err := getEngine() + if err != nil { + return nil, err + } + return eng.ReadTextCues(ctx, input, trackID) +} diff --git a/pkg/placestream/captionpushcaptions.go b/pkg/placestream/captionpushcaptions.go index 82ea9cca4..73126f75f 100644 --- a/pkg/placestream/captionpushcaptions.go +++ b/pkg/placestream/captionpushcaptions.go @@ -66,7 +66,7 @@ func (t *CaptionPushCaptions_Output) UnmarshalCBOR(r io.Reader) error { // CaptionPushCaptions calls the XRPC method "place.stream.caption.pushCaptions". // -// Push live captions into a stream from a captioner: the Streamplace caption overlay, a CART stenographer, or an external tool. Requires the streamer's authorization. When the streamer's caption policy is `ingest`, pushed captions become the canonical caption track. +// Push live captions into a stream from a captioner, a CART stenographer, or an external tool. Requires the streamer's OAuth authorization or active registered stream key. Under canonical policy auto or ingest, pushed captions become streamer-signed MUXL text tracks. Under canonical policy off, pushed captions remain sidecar or local according to the streamer's node-caption policy. func CaptionPushCaptions(ctx context.Context, c glex.LexClient, input *CaptionPushCaptions_Input) (*CaptionPushCaptions_Output, error) { var out CaptionPushCaptions_Output diff --git a/pkg/replication/websocketrep/captions_test.go b/pkg/replication/websocketrep/captions_test.go new file mode 100644 index 000000000..415cf7eb8 --- /dev/null +++ b/pkg/replication/websocketrep/captions_test.go @@ -0,0 +1,163 @@ +package websocketrep + +import ( + "bytes" + "context" + "crypto/ecdsa" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/decred/dcrd/dcrec/secp256k1" + "github.com/gorilla/websocket" + glex "github.com/streamplace/glex/runtime" + upstream "github.com/streamplace/muxl/go" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/atproto" + "stream.place/streamplace/pkg/bus" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/crypto/signers" + "stream.place/streamplace/pkg/localdb" + "stream.place/streamplace/pkg/media" + "stream.place/streamplace/pkg/model" + "stream.place/streamplace/pkg/placestream" +) + +func replicationCaptionFixture(t *testing.T) (string, []byte) { + t.Helper() + ctx := context.Background() + priv, err := atcrypto.GeneratePrivateKeyK256() + require.NoError(t, err) + secp, _ := secp256k1.PrivKeyFromBytes(priv.Bytes()) + key := secp.ToECDSA() + pub, err := atproto.ParsePubKey(key.Public().(*ecdsa.PublicKey)) + require.NoError(t, err) + did := pub.DIDKey() + cert, err := signers.GenerateES256KCert(key) + require.NoError(t, err) + pem, err := signers.MarshalES256KPrivateKeyPEM(key) + require.NoError(t, err) + manifest := []byte(fmt.Sprintf(`{"title":"replication","assertions":[{"label":"c2pa.actions.v2","data":{"actions":[{"action":"c2pa.created"},{"action":"c2pa.published"}]}},{"label":"cawg.metadata","data":{"@context":{"dc":"http://purl.org/dc/elements/1.1/"},"dc:creator":%q,"dc:title":"replication","dc:date":"2026-09-30T00:00:00.000Z"}},{"label":"place.stream.metadata.configuration","data":{"captionPolicy":{"canonical":"off","allowNodeCaptions":true}}}]}`, did)) + _, file, _, _ := runtime.Caller(0) + data, err := os.ReadFile(filepath.Join(filepath.Dir(file), "../../../test/fixtures/h264-opus-frag.mp4")) + require.NoError(t, err) + eng, err := captions.TextEngine() + require.NoError(t, err) + ch := make(chan *upstream.Event, 32) + errs := make(chan error, 1) + go func() { + errs <- eng.SignSegment(ctx, bytes.NewReader(data), upstream.SignerInput{CertPEM: cert, KeyPEM: pem, TrackManifest: manifest, WrapperManifest: manifest}, nil, nil, ch) + close(ch) + }() + var segment []byte + for ev := range ch { + if ev.Type == "signed-segment" && segment == nil { + var ids []string + for id := range ev.Tracks { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + segment = append(segment, ev.Tracks[id]...) + } + } + } + require.NoError(t, <-errs) + require.NotEmpty(t, segment) + return did, segment +} + +func TestCaptionSyndicationWebsocketRoundTripAndLegacyOrigin(t *testing.T) { + did, segment := replicationCaptionFixture(t) + for _, supportsCaptions := range []bool{true, false} { + t.Run(fmt.Sprint(supportsCaptions), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cli := &config.CLI{WideOpen: true, Captions: true, BroadcasterHost: "relay.example", DataDir: t.TempDir()} + m, err := model.MakeDB(":memory:") + require.NoError(t, err) + ldb, err := localdb.MakeDB(":memory:") + require.NoError(t, err) + b := bus.NewBus() + mm, err := media.MakeMediaManager(ctx, cli, nil, m, b, nil, ldb) + require.NoError(t, err) + defer mm.EndCaptionSession(did) + segments := mm.NewSegment() + wall := time.Now().UTC() + ev := captions.Event{Streamer: did, Track: captions.Track{ID: "sidecar-auto-en", Language: "en", Kind: captions.KindCaptions, Source: captions.SourceAuto, Origin: captions.OriginSidecar, Author: "did:web:upstream.example"}, Cue: captions.Cue{ID: "line", Text: "Upstream captions", Start: wall, End: wall.Add(time.Second), Final: true}} + encoded, err := captions.EncodeSidecar(ev) + require.NoError(t, err) + capability := make(chan string, 1) + serverErr := make(chan error, 1) + closeServer := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capability <- r.URL.Query().Get("captions") + conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err != nil { + serverErr <- err + return + } + defer conn.Close() + // Replay can arrive before the first validated segment. It must be + // held until the segment's signed allowNodeCaptions policy is known. + if supportsCaptions { + if err = conn.WriteMessage(websocket.TextMessage, encoded); err != nil { + serverErr <- err + return + } + } + if err = conn.WriteMessage(websocket.BinaryMessage, segment); err != nil { + serverErr <- err + return + } + if supportsCaptions { + if err = conn.WriteMessage(websocket.TextMessage, encoded); err != nil { + serverErr <- err + return + } + } + serverErr <- nil + <-closeServer + })) + defer server.Close() + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(closeServer) }) } + defer release() + wsURL := strings.Replace(server.URL, "http://", "ws://", 1) + view := &placestream.BroadcastDefs_BroadcastOriginView{Record: &glex.LexiconTypeDecoder{Val: &placestream.BroadcastOrigin{Streamer: did, Server: "did:web:upstream.example", WebsocketURL: &wsURL}}} + r := NewWebsocketReplicator(b, m, mm, nil) + done := make(chan error, 1) + go func() { done <- r.openWebsocket(ctx, view) }() + require.Equal(t, captions.SyndicationVersion, <-capability) + require.NoError(t, <-serverErr) + if supportsCaptions { + require.Eventually(t, func() bool { return len(b.Captions.Cues(did, ev.Track.ID, wall, wall.Add(2*time.Second))) == 1 }, 20*time.Second, time.Millisecond) + require.Equal(t, []captions.Cue{ev.Cue}, b.Captions.Cues(did, ev.Track.ID, wall, wall.Add(2*time.Second))) + require.Equal(t, "did:web:upstream.example", b.Captions.Tracks(did)[0].Author) + } else { + select { + case not := <-segments: + require.Equal(t, did, not.Segment.RepoDID) + require.Equal(t, segment, not.Muxl, "legacy origin media remains byte-identical") + case <-time.After(20 * time.Second): + t.Fatal("legacy origin's binary media was not validated and distributed") + } + require.Empty(t, b.Captions.Tracks(did), "an older origin can send binary media only") + } + release() + err = <-done + require.Error(t, err, "socket close ends the pull") + require.NotContains(t, err.Error(), "expected binary message") + }) + } +} diff --git a/pkg/replication/websocketrep/websocket_replicator.go b/pkg/replication/websocketrep/websocket_replicator.go index a9f45a533..3abf50a08 100644 --- a/pkg/replication/websocketrep/websocket_replicator.go +++ b/pkg/replication/websocketrep/websocket_replicator.go @@ -14,6 +14,7 @@ import ( "golang.org/x/sync/errgroup" "stream.place/streamplace/pkg/appbsky" "stream.place/streamplace/pkg/bus" + "stream.place/streamplace/pkg/captions" "stream.place/streamplace/pkg/config" "stream.place/streamplace/pkg/log" "stream.place/streamplace/pkg/media" @@ -300,7 +301,15 @@ func (r *WebsocketReplicator) openWebsocket(ctx context.Context, view *placestre return fmt.Errorf("origin has no websocket URL") } dialCtx, cancel := context.WithTimeout(ctx, 15*time.Second) - conn, _, err := websocket.DefaultDialer.DialContext(dialCtx, *origin.WebsocketURL, nil) + u, err := url.Parse(*origin.WebsocketURL) + if err != nil { + cancel() + return err + } + q := u.Query() + q.Set("captions", captions.SyndicationVersion) + u.RawQuery = q.Encode() + conn, _, err := websocket.DefaultDialer.DialContext(dialCtx, u.String(), nil) cancel() if err != nil { spmetrics.ReplicationConnectErrorsTotal.Inc() @@ -326,6 +335,12 @@ func (r *WebsocketReplicator) openWebsocket(ctx context.Context, view *placestre spmetrics.ReplicationConnectErrorsTotal.Inc() return fmt.Errorf("could not read message: %w", err) } + if typ == websocket.TextMessage { + if ev, ok := captions.DecodeSidecar(msg); ok { + r.mm.ReceiveSidecar(origin.Streamer, origin.Server, ev) + } + continue + } if typ != websocket.BinaryMessage { log.Error(ctx, "expected binary message", "type", typ) return fmt.Errorf("expected binary message") diff --git a/pkg/spxrpc/caption_body_limit_test.go b/pkg/spxrpc/caption_body_limit_test.go new file mode 100644 index 000000000..e95719421 --- /dev/null +++ b/pkg/spxrpc/caption_body_limit_test.go @@ -0,0 +1,30 @@ +package spxrpc + +import ( + "github.com/labstack/echo/v4" + "github.com/stretchr/testify/require" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestPushCaptionsBoundsChunkedRequestBody(t *testing.T) { + e := echo.New() + e.Use(captionPushBodyLimitMiddleware()) + e.POST("/xrpc/place.stream.caption.pushCaptions", func(c echo.Context) error { + var value map[string]any + if err := c.Bind(&value); err != nil { + return err + } + return c.NoContent(http.StatusNoContent) + }) + body := `{"language":"en","padding":"` + strings.Repeat("x", 2<<20) + `"}` + request := httptest.NewRequest(http.MethodPost, "/xrpc/place.stream.caption.pushCaptions", io.NopCloser(strings.NewReader(body))) + request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + request.ContentLength = -1 + response := httptest.NewRecorder() + e.ServeHTTP(response, request) + require.Equal(t, http.StatusRequestEntityTooLarge, response.Code) +} diff --git a/pkg/spxrpc/place_stream_caption_authored_test.go b/pkg/spxrpc/place_stream_caption_authored_test.go new file mode 100644 index 000000000..8fb7032fa --- /dev/null +++ b/pkg/spxrpc/place_stream_caption_authored_test.go @@ -0,0 +1,116 @@ +package spxrpc + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/labstack/echo/v4" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/captions/records" + "stream.place/streamplace/pkg/captions/transcript" + "stream.place/streamplace/pkg/captions/webvtt" +) + +type authoredImportPDS struct{} + +func (authoredImportPDS) Do(context.Context, string, string, string, map[string]any, any, any) error { + return nil +} + +func TestAuthoredCaptionImportHTTPRoundTrip(t *testing.T) { + const srt = "1\n00:00:00,000 --> 00:01:00,000\nVOD caption proof timestamp\n\n2\n00:01:00,000 --> 00:01:30,000\nFirst line. Still here!\nSecond line continues.\n\n3\n00:01:30,010 --> 00:02:00,000\nAfter a ten millisecond gap\n" + want, err := webvtt.ParseSRT([]byte(srt)) + require.NoError(t, err) + for _, source := range []captions.Source{captions.SourceImported, captions.SourceHuman} { + t.Run(string(source), func(t *testing.T) { + s := capServer(t) + _, err := (&records.Importer{Store: s.model, Chunking: transcript.ChunkOptions{MaxTimings: 2}}).Import( + context.Background(), authoredImportPDS{}, "did:plc:owner", records.ImportInput{ + Video: capVideoURI, Language: "en", Format: "srt", Body: srt, + }) + require.NoError(t, err) + if source == captions.SourceHuman { + rows, err := s.model.GetCaptionTranscriptsBySubject(context.Background(), capVideoURI) + require.NoError(t, err) + for _, row := range rows { + rec, err := row.ToRecord() + require.NoError(t, err) + rec.Source = string(source) + require.NoError(t, s.model.UpsertCaptionTranscript(context.Background(), rec, mustATURI(t, row.URI))) + } + } + s.VideoCaptions = &records.Provider{Store: s.model} + e := echo.New() + e.GET("/xrpc/place.stream.caption.getCaptions", s.HandleGetCaptions) + httpServer := httptest.NewServer(e) + defer httpServer.Close() + trackID := records.TrackID("did:plc:owner", "en", "captions", string(source)) + // The same provider supplies the native/web VOD overlay. + overlay, err := s.VideoCaptions.Cues(context.Background(), capVideoURI, trackID) + require.NoError(t, err) + require.Len(t, overlay, len(want)) + for i := range want { + require.Equal(t, want[i].Text, overlay[i].Text) + require.InDelta(t, want[i].Start.Milliseconds(), overlay[i].Start.Milliseconds(), 1) + require.InDelta(t, want[i].End.Milliseconds(), overlay[i].End.Milliseconds(), 1) + } + for _, format := range []string{"vtt", "srt", "json"} { + q := url.Values{"video": {capVideoURI}, "track": {trackID}, "format": {format}} + resp, err := http.Get(httpServer.URL + "/xrpc/place.stream.caption.getCaptions?" + q.Encode()) + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode, string(body)) + var got []webvtt.Cue + switch format { + case "vtt": + got, err = webvtt.ParseVTT(body) + require.NoError(t, err) + case "srt": + got, err = webvtt.ParseSRT(body) + require.NoError(t, err) + case "json": + var doc struct { + Cues []struct { + Text string + StartMs, EndMs int64 + } + } + require.NoError(t, json.Unmarshal(body, &doc)) + for _, cue := range doc.Cues { + got = append(got, webvtt.Cue{Text: cue.Text, Start: time.Duration(cue.StartMs) * time.Millisecond, End: time.Duration(cue.EndMs) * time.Millisecond}) + } + } + require.Len(t, got, len(want), format) + for i := range want { + require.Equal(t, want[i].Text, got[i].Text, format) + require.InDelta(t, want[i].Start.Milliseconds(), got[i].Start.Milliseconds(), 1, format) + require.InDelta(t, want[i].End.Milliseconds(), got[i].End.Milliseconds(), 1, format) + } + } + // HLS selects overlapping authored cues without rewriting their text or span. + q := url.Values{"video": {capVideoURI}, "track": {trackID + ".vtt"}, "format": {"vtt"}, "start": {"0"}, "end": {"2000"}, "mpegts": {"0"}} + resp, err := http.Get(httpServer.URL + "/xrpc/place.stream.caption.getCaptions?" + q.Encode()) + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Contains(t, string(body), "X-TIMESTAMP-MAP") + hlsCues, err := webvtt.ParseVTT(body) + require.NoError(t, err) + require.Len(t, hlsCues, 1) + require.Equal(t, want[0].Text, hlsCues[0].Text) + require.Equal(t, time.Duration(0), hlsCues[0].Start) + require.InDelta(t, want[0].End.Milliseconds(), hlsCues[0].End.Milliseconds(), 1) + }) + } +} diff --git a/pkg/spxrpc/place_stream_caption_push.go b/pkg/spxrpc/place_stream_caption_push.go index 5aab04d87..ef8a88928 100644 --- a/pkg/spxrpc/place_stream_caption_push.go +++ b/pkg/spxrpc/place_stream_caption_push.go @@ -1,15 +1,111 @@ package spxrpc -// Integration placeholder owned by the CaptionSources slice; replaced before commit. - import ( "context" "net/http" + "strings" + "time" + "unicode/utf8" + "github.com/google/uuid" "github.com/labstack/echo/v4" + echomiddleware "github.com/labstack/echo/v4/middleware" + "github.com/streamplace/oatproxy/pkg/oatproxy" + "golang.org/x/text/language" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/media" "stream.place/streamplace/pkg/placestream" ) +// A maximum batch needs under 1 MiB even with four-byte Unicode and JSON escapes. +const captionPushBodyLimit = "2M" + +func captionPushBodyLimitMiddleware() echo.MiddlewareFunc { + return echomiddleware.BodyLimitWithConfig(echomiddleware.BodyLimitConfig{ + Limit: captionPushBodyLimit, + Skipper: func(c echo.Context) bool { + return c.Request().URL.Path != "/xrpc/place.stream.caption.pushCaptions" + }, + }) +} + +func (s *Server) captionPusher(ctx context.Context) (string, error) { + if session, _ := oatproxy.GetOAuthSession(ctx); session != nil { + return session.DID, nil + } + ec, _ := ctx.Value(echoContextKey).(echo.Context) + if ec == nil { + return "", echo.NewHTTPError(http.StatusUnauthorized, "streamer authorization required") + } + auth := strings.Fields(ec.Request().Header.Get("Authorization")) + if len(auth) != 2 || !strings.EqualFold(auth[0], "Bearer") { + return "", echo.NewHTTPError(http.StatusUnauthorized, "streamer authorization required") + } + did, _, err := media.AuthenticateStreamKey(ctx, s.cli, s.model, s.ATSync, auth[1], true) + if err != nil { + return "", echo.NewHTTPError(http.StatusUnauthorized, "invalid stream key", err) + } + return did, nil +} + func (s *Server) handlePlaceStreamCaptionPushCaptions(ctx context.Context, body *placestream.CaptionPushCaptions_Input) (*placestream.CaptionPushCaptions_Output, error) { - return nil, echo.NewHTTPError(http.StatusNotImplemented, "pushCaptions not implemented yet") + did, err := s.captionPusher(ctx) + if err != nil { + return nil, err + } + if body.Streamer != nil && *body.Streamer != "" && *body.Streamer != did { + return nil, echo.NewHTTPError(http.StatusForbidden, "Forbidden") + } + if len(body.Cues) > 100 { + return nil, echo.NewHTTPError(http.StatusBadRequest, "caption batch exceeds 100 cues") + } + if body.Language == "" { + return nil, echo.NewHTTPError(http.StatusBadRequest, "caption language required") + } + if _, err := language.Parse(body.Language); err != nil { + return nil, echo.NewHTTPError(http.StatusBadRequest, "invalid caption language", err) + } + policy, live := s.mm.OriginCaptionPolicy(did) + if !live { + return nil, echo.NewHTTPError(http.StatusBadRequest, "StreamNotLive") + } + source := captions.SourceHuman + if body.Source != nil { + source = captions.Source(*body.Source) + } + if source != captions.SourceHuman && source != captions.SourceAuto { + return nil, echo.NewHTTPError(http.StatusBadRequest, "invalid caption source") + } + origin := captions.PushedOrigin(policy) + track := captions.Track{ID: captions.TrackID(origin, source, body.Language), Language: body.Language, Kind: captions.KindCaptions, Source: source, Origin: origin, Author: did, Label: "Captions"} + cues := make([]captions.Cue, 0, len(body.Cues)) + for _, input := range body.Cues { + if utf8.RuneCountInString(input.Text) > 2000 || (input.Id != nil && utf8.RuneCountInString(*input.Id) > 64) { + return nil, echo.NewHTTPError(http.StatusBadRequest, "caption cue exceeds lexicon limits") + } + start, err := time.Parse(time.RFC3339Nano, input.StartTime) + if err != nil { + return nil, echo.NewHTTPError(http.StatusBadRequest, "invalid cue start", err) + } + end, err := time.Parse(time.RFC3339Nano, input.EndTime) + if err != nil || !end.After(start) { + return nil, echo.NewHTTPError(http.StatusBadRequest, "invalid cue end") + } + id := uuid.NewString() + if input.Id != nil && *input.Id != "" { + id = *input.Id + } + final := input.Final == nil || *input.Final + cues = append(cues, captions.Cue{ID: id, Start: start, End: end, Text: input.Text, Final: final}) + } + if origin == captions.OriginCanonical { + if err := s.mm.PushCanonicalCaptions(did, track, cues); err != nil { + return nil, echo.NewHTTPError(http.StatusServiceUnavailable, "caption master unavailable", err) + } + } else { + for _, cue := range cues { + s.bus.Captions.Publish(did, track, cue) + } + } + return &placestream.CaptionPushCaptions_Output{}, nil } diff --git a/pkg/spxrpc/place_stream_caption_push_test.go b/pkg/spxrpc/place_stream_caption_push_test.go new file mode 100644 index 000000000..2f2338c03 --- /dev/null +++ b/pkg/spxrpc/place_stream_caption_push_test.go @@ -0,0 +1,191 @@ +package spxrpc + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" + "time" + + "github.com/labstack/echo/v4" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/streamplace/oatproxy/pkg/oatproxy" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/bus" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/crypto/spkey" + "stream.place/streamplace/pkg/media" + "stream.place/streamplace/pkg/muxl" + "stream.place/streamplace/pkg/placestream" +) + +func captionOAuthContext(t *testing.T, ctx context.Context, did string) context.Context { + t.Helper() + raw, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + key, err := jwk.FromRaw(raw) + require.NoError(t, err) + data, err := json.Marshal(key) + require.NoError(t, err) + ctx = context.WithValue(ctx, oatproxy.OATProxyContextKey, &oatproxy.OATProxy{}) + return context.WithValue(ctx, oatproxy.OAuthSessionContextKey, &oatproxy.OAuthSession{DID: did, UpstreamDPoPPrivateJWK: string(data)}) +} +func requireCaptionHTTPError(t *testing.T, err error, code int, message string) { + t.Helper() + var httpErr *echo.HTTPError + require.ErrorAs(t, err, &httpErr) + require.Equal(t, code, httpErr.Code) + require.Contains(t, httpErr.Message, message) +} +func TestPushCaptionsAuthorizationAndLiveBoundary(t *testing.T) { + s := &Server{cli: &config.CLI{}, mm: media.NewOffline(&config.CLI{})} + input := &placestream.CaptionPushCaptions_Input{Language: "en-US"} + _, err := s.handlePlaceStreamCaptionPushCaptions(context.Background(), input) + requireCaptionHTTPError(t, err, http.StatusUnauthorized, "authorization") + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("Authorization", "Bearer z2") + ec := echo.New().NewContext(req, httptest.NewRecorder()) + _, err = s.handlePlaceStreamCaptionPushCaptions(context.WithValue(context.Background(), echoContextKey, ec), input) + requireCaptionHTTPError(t, err, http.StatusUnauthorized, "invalid stream key") + ctx := captionOAuthContext(t, context.Background(), "did:plc:owner") + other := "did:plc:other" + input.Streamer = &other + _, err = s.handlePlaceStreamCaptionPushCaptions(ctx, input) + requireCaptionHTTPError(t, err, http.StatusForbidden, "Forbidden") + input.Streamer = nil + _, err = s.handlePlaceStreamCaptionPushCaptions(ctx, input) + requireCaptionHTTPError(t, err, http.StatusBadRequest, "StreamNotLive") +} + +func TestPushCaptionsLiveCanonicalAndOffRoutes(t *testing.T) { + _, file, _, _ := runtime.Caller(0) + fixture, err := os.ReadFile(filepath.Join(filepath.Dir(file), "..", "..", "test", "fixtures", "h264-opus-frag.mp4")) + require.NoError(t, err) + for _, tc := range []struct { + name string + canonical string + nodes bool + origin captions.Origin + }{ + {"canonical", "ingest", true, captions.OriginCanonical}, + {"sidecar", "off", true, captions.OriginSidecar}, + {"local", "off", false, captions.OriginLocal}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + did := "did:plc:owner" + cli := &config.CLI{CaptionsMasterDelay: time.Second} + mm := media.NewOffline(cli) + priv, _, err := spkey.GenerateStreamKey() + require.NoError(t, err) + signer, err := spkey.KeyToSigner(priv) + require.NoError(t, err) + ms, err := media.MakeMediaSigner(ctx, cli, did, signer, nil) + require.NoError(t, err) + policy := map[string]any{"canonical": tc.canonical, "allowNodeCaptions": tc.nodes, "languages": []string{"en-US"}} + manifest, err := json.Marshal(map[string]any{"title": "push caption test", "assertions": []any{map[string]any{"label": "c2pa.actions", "data": map[string]any{"actions": []any{map[string]any{"action": "c2pa.created"}}}}, map[string]any{"label": "place.stream.metadata.configuration", "data": map[string]any{"captionPolicy": policy}}}}) + require.NoError(t, err) + ms.(*media.MediaSignerLocal).PrebuiltManifest = manifest + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + events := make(chan *muxl.MuxlEvent, 16) + done := make(chan error, 1) + streamStarted := time.Now() + go func() { done <- mm.SignOriginStream(ctx, ms, reader, events); close(events) }() + go func() { _, _ = writer.Write(fixture) }() + var archived bytes.Buffer + appendEvent := func(event *muxl.MuxlEvent) { + ids := make([]string, 0, len(event.Tracks)) + for id := range event.Tracks { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + archived.Write(event.Tracks[id]) + } + } + for { + select { + case event := <-events: + require.NotNil(t, event) + if event.Type == "signed-segment" { + appendEvent(event) + goto live + } + case <-ctx.Done(): + t.Fatal("first signed GoP did not arrive") + } + } + live: + hub := captions.NewHub(0) + s := &Server{cli: cli, mm: mm, bus: &bus.Bus{Captions: hub}} + auth := captionOAuthContext(t, ctx, did) + id := "cart-tools-cue" + final := false + start := streamStarted.Add(100 * time.Millisecond) + end := start.Add(300 * time.Millisecond) + input := &placestream.CaptionPushCaptions_Input{Language: "en-US", Cues: []placestream.CaptionDefs_PushedCue{{Id: &id, StartTime: start.Format(time.RFC3339Nano), EndTime: end.Format(time.RFC3339Nano), Text: "partial", Final: &final}}} + // A rejected batch must not publish its valid prefix to either route. + atomicID := "must-not-publish" + oversizedID := strings.Repeat("界", 65) + badBatch := &placestream.CaptionPushCaptions_Input{Language: "en-US", Cues: []placestream.CaptionDefs_PushedCue{ + {Id: &atomicID, StartTime: start.Format(time.RFC3339Nano), EndTime: end.Format(time.RFC3339Nano), Text: "rejected prefix"}, + {Id: &oversizedID, StartTime: start.Format(time.RFC3339Nano), EndTime: end.Format(time.RFC3339Nano), Text: "rejected suffix"}, + }} + _, err = s.handlePlaceStreamCaptionPushCaptions(auth, badBatch) + requireCaptionHTTPError(t, err, http.StatusBadRequest, "lexicon limits") + require.Empty(t, hub.Tracks(did)) + _, err = s.handlePlaceStreamCaptionPushCaptions(auth, input) + require.NoError(t, err) + final = true + input.Cues[0].Text = "CART words survive signing" + _, err = s.handlePlaceStreamCaptionPushCaptions(auth, input) + require.NoError(t, err) + if tc.origin == captions.OriginCanonical { + require.Empty(t, hub.Tracks(did), "canonical cues cannot bypass validated MUXL") + } else { + tracks := hub.Tracks(did) + require.Len(t, tracks, 1) + require.Equal(t, tc.origin, tracks[0].Origin) + cues := hub.Cues(did, tracks[0].ID, start.Add(-time.Second), end.Add(time.Second)) + require.Len(t, cues, 1) + require.Equal(t, "CART words survive signing", cues[0].Text) + require.True(t, cues[0].Final) + } + require.NoError(t, writer.Close()) + for event := range events { + if event.Type == "signed-segment" { + appendEvent(event) + } + } + require.NoError(t, <-done) + tracks, err := muxl.RunMuxlTextTracks(ctx, bytes.NewReader(archived.Bytes())) + require.NoError(t, err) + if tc.origin == captions.OriginCanonical { + require.Equal(t, []muxl.TextTrack{{TrackID: media.CaptionTrackIDBase, Language: "en-US", Label: "human"}}, tracks) + cues, err := muxl.RunMuxlReadTextCues(ctx, bytes.NewReader(archived.Bytes()), media.CaptionTrackIDBase) + require.NoError(t, err) + require.Len(t, cues, 1) + require.Equal(t, "CART words survive signing", cues[0].Text) + } else { + require.Empty(t, tracks, "canonical=off must not add a text track") + } + _, live := mm.OriginCaptionPolicy(did) + require.False(t, live, "session teardown must revoke push ownership") + }) + } +} diff --git a/pkg/spxrpc/place_stream_live.go b/pkg/spxrpc/place_stream_live.go index 0c12addc2..4c521c51d 100644 --- a/pkg/spxrpc/place_stream_live.go +++ b/pkg/spxrpc/place_stream_live.go @@ -21,6 +21,8 @@ import ( "github.com/streamplace/oatproxy/pkg/oatproxy" "stream.place/streamplace/pkg/appbsky" "stream.place/streamplace/pkg/atproto" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/captions/livecue" "stream.place/streamplace/pkg/comatproto" "stream.place/streamplace/pkg/log" "stream.place/streamplace/pkg/media" @@ -362,11 +364,36 @@ func (s *Server) handlePlaceStreamLiveSubscribeSegments(c echo.Context) error { // rest of the station gets the renditions. renChan := s.bus.SubscribeSegmentBuf(ctx, user, media.RenditionsChannel, 4) defer s.bus.UnsubscribeSegment(ctx, user, media.RenditionsChannel, renChan) + var captionEvents <-chan captions.Event + sendCaption := func(ev captions.Event) error { + if ev.Track.Origin != captions.OriginSidecar || !s.mm.LiveWindowPublished(user) || !s.mm.CaptionSyndicationAllowed(user) { + return nil + } + data, err := captions.EncodeSidecar(ev) + if err != nil { + return err + } + return ws.WriteMessage(websocket.TextMessage, data) + } + if c.QueryParam("captions") == captions.SyndicationVersion && s.bus.Captions != nil { + captionEvents = s.bus.Captions.Subscribe(ctx, user) + for _, ev := range livecue.Recent(s.bus.Captions, user, livecue.JoinWindow, time.Now()) { + if err := sendCaption(ev); err != nil { + cancel() + return + } + } + } for { select { case <-ctx.Done(): log.Debug(ctx, "exiting segment reader") return + case ev := <-captionEvents: + if err := sendCaption(ev); err != nil { + cancel() + return + } case file := <-segChan.C: if !file.Published { continue diff --git a/pkg/spxrpc/spxrpc.go b/pkg/spxrpc/spxrpc.go index 47ccf13bd..34fbbe61e 100644 --- a/pkg/spxrpc/spxrpc.go +++ b/pkg/spxrpc/spxrpc.go @@ -112,6 +112,7 @@ func NewServer(ctx context.Context, cli *config.CLI, model model.Model, stateful } e.Use(s.ErrorHandlingMiddleware()) e.Use(s.ContextPreservingMiddleware()) + e.Use(captionPushBodyLimitMiddleware()) e.Use(echomiddleware.Handler("", mdlw)) e.Use(s.ServiceAuthMiddleware()) e.Use(op.OAuthMiddleware) diff --git a/pkg/stt/proxy.go b/pkg/stt/proxy.go new file mode 100644 index 000000000..9e485c0c6 --- /dev/null +++ b/pkg/stt/proxy.go @@ -0,0 +1,287 @@ +package stt + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "net" + "os" + "sync" +) + +// One request carries at most a 30-second recognition window, with ample JSON +// overhead. Framing bounds allocation before decoding untrusted worker bytes. +const engineFrameLimit = 16 << 20 + +type engineRequest struct { + Op string + Lease LeaseOptions + PCM []float32 + Options Options +} +type engineResponse struct { + Error string + OverBudget bool + Models []ModelInfo + Info *ModelInfo + Result *Result +} + +func engineRead(c net.Conn, value any) error { + var header [4]byte + if _, err := io.ReadFull(c, header[:]); err != nil { + return err + } + n := binary.BigEndian.Uint32(header[:]) + if n > engineFrameLimit { + return fmt.Errorf("speech request exceeds frame limit") + } + data := make([]byte, n) + if _, err := io.ReadFull(c, data); err != nil { + return err + } + return json.Unmarshal(data, value) +} +func engineWrite(c net.Conn, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + if len(data) > engineFrameLimit { + return fmt.Errorf("speech response exceeds frame limit") + } + var header [4]byte + binary.BigEndian.PutUint32(header[:], uint32(len(data))) + if _, err = c.Write(header[:]); err != nil { + return err + } + _, err = c.Write(data) + return err +} + +// ServeEngine exposes the node's one scheduler to media-isolated workers. Each +// lease belongs to its connection; EOF cancels native inference and releases it. +func ServeEngine(ctx context.Context, path string, engine Engine) (func(), error) { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return nil, err + } + listener, err := net.Listen("unix", path) + if err != nil { + return nil, err + } + if err = os.Chmod(path, 0600); err != nil { + listener.Close() + return nil, err + } + ctx, cancel := context.WithCancel(ctx) + var once sync.Once + stop := func() { once.Do(func() { cancel(); listener.Close(); os.Remove(path) }) } + go func() { <-ctx.Done(); stop() }() + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go serveEngineConnection(ctx, conn, engine) + } + }() + return stop, nil +} +func serveEngineConnection(parent context.Context, c net.Conn, engine Engine) { + ctx, cancel := context.WithCancel(parent) + defer cancel() + defer c.Close() + requests := make(chan engineRequest) + go func() { + defer cancel() + defer close(requests) + for { + var r engineRequest + if engineRead(c, &r) != nil { + return + } + select { + case requests <- r: + case <-ctx.Done(): + return + } + } + }() + go func() { <-ctx.Done(); c.Close() }() + var lease Lease + defer func() { + if lease != nil { + lease.Release() + } + }() + for r := range requests { + var response engineResponse + var err error + switch r.Op { + case "models": + if engine != nil { + response.Models = engine.Models() + } + case "lease": + if lease != nil { + err = fmt.Errorf("connection already holds a lease") + } else if engine == nil { + err = ErrOverBudget + } else { + lease, err = engine.Lease(ctx, r.Lease) + } + case "model", "transcribe": + if lease == nil { + err = fmt.Errorf("speech lease required") + break + } + model := lease.Model() + if model == nil { + err = ErrOverBudget + break + } + info := model.Info() + response.Info = &info + if r.Op == "transcribe" { + if len(r.PCM) > 30*SampleRate { + err = fmt.Errorf("speech window exceeds 30 seconds") + } else { + response.Result, err = model.Transcribe(ctx, r.PCM, r.Options) + } + } + default: + err = fmt.Errorf("unknown speech operation") + } + if err != nil { + response.Error = err.Error() + response.OverBudget = err == ErrOverBudget + } + if engineWrite(c, response) != nil { + return + } + } +} + +type proxyEngine struct { + path string + mu sync.Mutex + closed bool + connections map[*proxyLease]struct{} +} +type proxyLease struct { + conn net.Conn + engine *proxyEngine + mu sync.Mutex + released bool +} +type proxyModel struct { + lease *proxyLease + info ModelInfo +} + +// NewProxy never loads a private model or substitutes an independent budget. +func NewProxy(path string) Engine { + return &proxyEngine{path: path, connections: make(map[*proxyLease]struct{})} +} +func (e *proxyEngine) Close() error { + e.mu.Lock() + e.closed = true + connections := e.connections + e.connections = nil + e.mu.Unlock() + for lease := range connections { + lease.Release() + } + return nil +} +func (e *proxyEngine) connect(ctx context.Context) (*proxyLease, error) { + c, err := (&net.Dialer{}).DialContext(ctx, "unix", e.path) + if err != nil { + return nil, err + } + e.mu.Lock() + defer e.mu.Unlock() + if e.closed { + c.Close() + return nil, fmt.Errorf("speech engine closed") + } + l := &proxyLease{conn: c, engine: e} + e.connections[l] = struct{}{} + return l, nil +} +func (e *proxyEngine) Models() []ModelInfo { + ctx := context.Background() + l, err := e.connect(ctx) + if err != nil { + return nil + } + defer l.Release() + r, err := l.call(ctx, engineRequest{Op: "models"}) + if err != nil { + return nil + } + return r.Models +} +func (e *proxyEngine) Lease(ctx context.Context, opts LeaseOptions) (Lease, error) { + l, err := e.connect(ctx) + if err != nil { + return nil, err + } + if _, err = l.call(ctx, engineRequest{Op: "lease", Lease: opts}); err != nil { + l.Release() + return nil, err + } + return l, nil +} +func (l *proxyLease) call(ctx context.Context, r engineRequest) (engineResponse, error) { + l.mu.Lock() + defer l.mu.Unlock() + var response engineResponse + if l.released { + return response, fmt.Errorf("speech lease released") + } + stop := context.AfterFunc(ctx, func() { l.conn.Close() }) + defer stop() + if err := engineWrite(l.conn, r); err != nil { + return response, err + } + if err := engineRead(l.conn, &response); err != nil { + if ctx.Err() != nil { + return response, ctx.Err() + } + return response, err + } + if response.OverBudget { + return response, ErrOverBudget + } + if response.Error != "" { + return response, fmt.Errorf("remote speech engine: %s", response.Error) + } + return response, nil +} +func (l *proxyLease) Release() { + l.conn.Close() + l.mu.Lock() + l.released = true + l.mu.Unlock() + if l.engine != nil { + l.engine.mu.Lock() + delete(l.engine.connections, l) + l.engine.mu.Unlock() + } +} +func (l *proxyLease) Model() Model { + r, err := l.call(context.Background(), engineRequest{Op: "model"}) + if err != nil || r.Info == nil { + return nil + } + return &proxyModel{lease: l, info: *r.Info} +} +func (m *proxyModel) Info() ModelInfo { return m.info } +func (m *proxyModel) Transcribe(ctx context.Context, pcm []float32, opts Options) (*Result, error) { + r, err := m.lease.call(ctx, engineRequest{Op: "transcribe", PCM: pcm, Options: opts}) + return r.Result, err +} diff --git a/pkg/stt/proxy_test.go b/pkg/stt/proxy_test.go new file mode 100644 index 000000000..d741d4e0e --- /dev/null +++ b/pkg/stt/proxy_test.go @@ -0,0 +1,119 @@ +package stt + +import ( + "context" + "encoding/binary" + "github.com/stretchr/testify/require" + "net" + "os" + "path/filepath" + "testing" + "time" +) + +func TestEngineProxySharesParentBudgetAndReleasesDisconnectedWorker(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + engine := fakeScheduler(1) + defer engine.Close() + path := filepath.Join(t.TempDir(), "engine.sock") + stop, err := ServeEngine(ctx, path, engine) + require.NoError(t, err) + defer stop() + stat, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), stat.Mode().Perm()) + parent, err := engine.Lease(ctx, LeaseOptions{Realtime: true, Model: "tiny"}) + require.NoError(t, err) + defer parent.Release() + worker, err := NewProxy(path).Lease(ctx, LeaseOptions{Realtime: true}) + require.NoError(t, err) + require.Equal(t, "tiny", worker.Model().Info().Name) + _, err = NewProxy(path).Lease(ctx, LeaseOptions{Realtime: true}) + require.ErrorIs(t, err, ErrOverBudget) + // Simulate SIGKILL: only the socket disappears; Release never runs in worker. + require.NoError(t, worker.(*proxyLease).conn.Close()) + require.Eventually(t, func() bool { engine.mu.Lock(); defer engine.mu.Unlock(); return len(engine.leases) == 1 }, time.Second, time.Millisecond) + proxy := NewProxy(path) + replacement, err := proxy.Lease(ctx, LeaseOptions{Realtime: true}) + require.NoError(t, err) + require.NoError(t, proxy.Close()) + require.Nil(t, replacement.Model()) + require.Eventually(t, func() bool { engine.mu.Lock(); defer engine.mu.Unlock(); return len(engine.leases) == 1 }, time.Second, time.Millisecond) +} +func TestEngineProxyCancelsWaitingLeaseAndBoundsFrames(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + engine := fakeScheduler(2) + defer engine.Close() + path := filepath.Join(t.TempDir(), "engine.sock") + stop, err := ServeEngine(ctx, path, engine) + require.NoError(t, err) + defer stop() + parent, err := engine.Lease(ctx, LeaseOptions{}) + require.NoError(t, err) + defer parent.Release() + short, stopShort := context.WithTimeout(ctx, 20*time.Millisecond) + defer stopShort() + _, err = NewProxy(path).Lease(short, LeaseOptions{}) + require.ErrorIs(t, err, context.DeadlineExceeded) + c, err := net.Dial("unix", path) + require.NoError(t, err) + defer c.Close() + var header [4]byte + binary.BigEndian.PutUint32(header[:], engineFrameLimit+1) + _, err = c.Write(header[:]) + require.NoError(t, err) + require.NoError(t, c.SetReadDeadline(time.Now().Add(time.Second))) + _, err = c.Read(header[:]) + require.Error(t, err) + require.NotErrorIs(t, err, os.ErrDeadlineExceeded) +} + +type cancelProxyModel struct { + entered chan struct{} + canceled chan struct{} +} + +func (m *cancelProxyModel) Info() ModelInfo { return ModelInfo{Name: "cancel-model"} } +func (m *cancelProxyModel) Transcribe(ctx context.Context, _ []float32, _ Options) (*Result, error) { + close(m.entered) + <-ctx.Done() + close(m.canceled) + return nil, ctx.Err() +} + +func TestEngineProxyWorkerCancellationStopsParentInference(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + engine := fakeScheduler(1) + defer engine.Close() + model := &cancelProxyModel{entered: make(chan struct{}), canceled: make(chan struct{})} + engine.models = []scheduledModel{{model: model, cost: .5}} + path := filepath.Join(t.TempDir(), "engine.sock") + stop, err := ServeEngine(ctx, path, engine) + require.NoError(t, err) + defer stop() + lease, err := NewProxy(path).Lease(ctx, LeaseOptions{Realtime: true}) + require.NoError(t, err) + defer lease.Release() + inference, stopInference := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { + _, err := lease.Model().Transcribe(inference, make([]float32, SampleRate), Options{}) + done <- err + }() + select { + case <-model.entered: + case <-ctx.Done(): + t.Fatal("parent inference never entered") + } + stopInference() + require.ErrorIs(t, <-done, context.Canceled) + select { + case <-model.canceled: + case <-ctx.Done(): + t.Fatal("parent inference did not observe worker cancellation") + } + require.Eventually(t, func() bool { engine.mu.Lock(); defer engine.mu.Unlock(); return len(engine.leases) == 0 }, time.Second, time.Millisecond) +} diff --git a/pkg/vod/captions.go b/pkg/vod/captions.go new file mode 100644 index 000000000..04232db9c --- /dev/null +++ b/pkg/vod/captions.go @@ -0,0 +1,283 @@ +package vod + +import ( + "bytes" + "context" + "fmt" + "io" + "sort" + "strconv" + "strings" + "time" + + upstream "github.com/streamplace/muxl/go" + "stream.place/streamplace/pkg/blob" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/captions/records" + "stream.place/streamplace/pkg/placestream" +) + +// VideoCaptions reads archival MUXL text first, supplementing it with imported +// and node transcript tracks. A streamer's record copy of a mastered track is +// omitted; sidecars and human corrections remain distinct choices. +type VideoCaptions struct { + Model interface { + GetVideoByURI(context.Context, string) (*placestream.Video, error) + GetMediaTrackByURI(context.Context, string) (*placestream.MediaTrack, error) + } + Store blob.Store + Records captions.VideoCaptions +} + +type videoCaptionView struct { + tracks []captions.Track + cues map[string][]captions.TimedCue +} + +func (p *VideoCaptions) Tracks(ctx context.Context, video string) ([]captions.Track, error) { + v, err := p.view(ctx, video, 0) + if err != nil { + return nil, err + } + return v.tracks, nil +} +func (p *VideoCaptions) Cues(ctx context.Context, video, track string) ([]captions.TimedCue, error) { + v, err := p.view(ctx, video, 0) + if err != nil { + return nil, err + } + if cues, ok := v.cues[track]; ok { + return cues, nil + } + return nil, records.ErrTrackNotFound +} + +func (p *VideoCaptions) view(ctx context.Context, uri string, depth int) (*videoCaptionView, error) { + if depth > 8 { + return nil, fmt.Errorf("caption video source cycle") + } + out := &videoCaptionView{cues: map[string][]captions.TimedCue{}} + rec, err := p.Model.GetVideoByURI(ctx, uri) + if err != nil { + return nil, err + } + if rec == nil { + return out, nil + } + if p.Store != nil { + if clip := rec.Source.MediaDefs_SourceClip; clip != nil { + parent, err := p.view(ctx, clip.Video, depth+1) + if err != nil { + return nil, err + } + for _, track := range parent.tracks { + if track.Origin != captions.OriginCanonical { + continue + } + out.tracks = append(out.tracks, track) + out.cues[track.ID] = clipCaptionCues(parent.cues[track.ID], time.Duration(clip.Start)*time.Millisecond, time.Duration(clip.End)*time.Millisecond) + } + } else if src := rec.Source.MediaDefs_SourceTracks; src != nil && len(src.Tracks) > 0 { + track, err := p.Model.GetMediaTrackByURI(ctx, src.Tracks[0].Uri) + if err != nil { + return nil, err + } + if track != nil && track.Track.MediaDefs_MuxlTrack != nil { + if err := p.readMuxl(ctx, track.Track.MediaDefs_MuxlTrack.Blob, uri, out); err != nil { + return nil, err + } + } + } + } + if p.Records != nil { + tracks, err := p.Records.Tracks(ctx, uri) + if err != nil { + return nil, err + } + for _, track := range tracks { + cues, err := p.Records.Cues(ctx, uri, track.ID) + if err != nil { + return nil, err + } + copy := false + for _, canonical := range out.tracks { + if canonical.Origin == captions.OriginCanonical && canonical.Language == track.Language && canonical.Kind == track.Kind && canonical.Source == track.Source && canonical.Author == track.Author && captionText(out.cues[canonical.ID]) == captionText(cues) { + copy = true + break + } + } + if copy { + continue + } + out.tracks = append(out.tracks, track) + out.cues[track.ID] = cues + } + } + sort.Slice(out.tracks, func(i, j int) bool { return out.tracks[i].ID < out.tracks[j].ID }) + return out, nil +} + +func (p *VideoCaptions) readMuxl(ctx context.Context, cid, uri string, out *videoCaptionView) error { + meta, err := readMetafile(ctx, p.Store, cid) + if err != nil { + return err + } + var reader blob.Reader + defer func() { + if reader != nil { + reader.Close() + } + }() + ref := captionReference(meta) + known := map[string]bool{} + for _, track := range out.tracks { + known[track.ID] = true + } + for tid, t := range meta.Tracks { + if t.Type != "text" { + continue + } + id, err := strconv.ParseUint(tid, 10, 32) + if err != nil { + return err + } + if reader == nil { + reader, err = p.Store.Open(ctx, BlobsPrefix+cid+".mp4") + if err != nil { + return err + } + } + refIndex := 0 + var elapsed uint64 + for _, seg := range t.Segments { + lang, label := t.Language, t.Label + if seg.CaptionConfigKnown { + lang, label = seg.CaptionLanguage, seg.CaptionLabel + } + track := captions.CanonicalTrack(upstream.TextTrack{TrackID: uint32(id), Language: lang, Label: label}, videoAuthor(uri)) + if !known[track.ID] { + out.tracks = append(out.tracks, track) + known[track.ID] = true + } + data, err := io.ReadAll(io.NewSectionReader(reader, meta.FlatHeaderSize+seg.Offset, seg.Size)) + if err != nil { + return err + } + offset := time.Duration(seg.CaptionOffsetNanos) + var baseOffset time.Duration + if seg.CaptionReferenceScale != 0 { + baseOffset = captionTicks(seg.CaptionReferenceTicks, seg.CaptionReferenceScale) + } else { + // Legacy indexes did not retain the containing GoP association. + for refIndex+1 < len(ref.Segments) && ref.Segments[refIndex+1].Offset <= seg.Offset { + elapsed += ref.Segments[refIndex].DurationTicks + refIndex++ + } + if len(ref.Segments) == 0 || ref.Timescale == 0 { + return fmt.Errorf("caption reference AV track missing") + } + reference := ref.Segments[refIndex] + base := reference.FirstDecodeTicks + if !reference.DecodeTimeKnown { + av, err := io.ReadAll(io.NewSectionReader(reader, meta.FlatHeaderSize+reference.Offset, reference.Size)) + if err != nil { + return err + } + var ok bool + base, ok = firstTFDT(av) + if !ok { + return fmt.Errorf("caption reference AV clock missing") + } + } + offset = captionTicks(elapsed, ref.Timescale) + baseOffset = captionTicks(base, ref.Timescale) + } + cues, err := captions.ReadTextCues(ctx, bytes.NewReader(data), uint32(id)) + if err != nil { + return err + } + for _, c := range cues { + out.cues[track.ID] = append(out.cues[track.ID], captions.TimedCue{ID: c.ID, Text: c.Text, Start: offset + time.Duration(c.Start)*time.Millisecond - baseOffset, End: offset + time.Duration(c.End)*time.Millisecond - baseOffset}) + } + } + } + for id, list := range out.cues { + sort.SliceStable(list, func(i, j int) bool { return list[i].Start < list[j].Start }) + joined := list[:0] + for _, cue := range list { + if len(joined) > 0 { + last := &joined[len(joined)-1] + if last.ID == cue.ID && last.Text == cue.Text && last.End == cue.Start { + last.End = cue.End + continue + } + } + joined = append(joined, cue) + } + out.cues[id] = joined + } + return nil +} + +func videoAuthor(uri string) string { + // All callers have an indexed AT URI; the authority is the signing owner. + for i := 5; i < len(uri); i++ { + if uri[i] == '/' { + return uri[5:i] + } + } + return "" +} + +func clipCaptionCues(cues []captions.TimedCue, start, end time.Duration) []captions.TimedCue { + var out []captions.TimedCue + for _, cue := range cues { + if cue.End <= start || cue.Start >= end { + continue + } + cue.Start = max(cue.Start, start) - start + cue.End = min(cue.End, end) - start + out = append(out, cue) + } + return out +} + +var _ captions.VideoCaptions = (*VideoCaptions)(nil) + +// captionReference is the same reference clock used by live extraction: +// the canonical video track, or audio when the video has none. +func captionReference(meta *Metafile) MetafileTrack { + keys := make([]string, 0, len(meta.Tracks)) + for id := range meta.Tracks { + keys = append(keys, id) + } + sort.Strings(keys) + var audio MetafileTrack + for _, id := range keys { + t := meta.Tracks[id] + if t.Type == "video" { + return t + } + if t.Type == "audio" && audio.Timescale == 0 { + audio = t + } + } + return audio +} + +func captionTicks(ticks uint64, scale uint32) time.Duration { + return time.Duration(ticks/uint64(scale))*time.Second + time.Duration(ticks%uint64(scale))*time.Second/time.Duration(scale) +} + +func captionText(cues []captions.TimedCue) string { + var text strings.Builder + for _, cue := range cues { + for _, word := range strings.Fields(cue.Text) { + if text.Len() > 0 { + text.WriteByte(' ') + } + text.WriteString(word) + } + } + return text.String() +} diff --git a/pkg/vod/captions_review_test.go b/pkg/vod/captions_review_test.go new file mode 100644 index 000000000..ff2396879 --- /dev/null +++ b/pkg/vod/captions_review_test.go @@ -0,0 +1,140 @@ +package vod + +import ( + "bytes" + "context" + "os" + "sort" + "strconv" + "testing" + "time" + + upstream "github.com/streamplace/muxl/go" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/blob" + "stream.place/streamplace/pkg/captions" +) + +func reconnectCaptionFixture(t *testing.T, changeLanguage, textFirst bool) (*videoCaptionView, time.Duration, error) { + t.Helper() + ctx := context.Background() + eng, err := captions.TextEngine() + require.NoError(t, err) + data, err := os.ReadFile(getFixture("h264-opus-frag.mp4")) + require.NoError(t, err) + ch := make(chan *upstream.Event, 32) + errs := make(chan error, 1) + go func() { errs <- eng.SegmentEvents(ctx, bytes.NewReader(data), ch); close(ch) }() + var source []*upstream.Event + var scale uint32 + for ev := range ch { + if ev.Catalog != nil && ev.Catalog.Video != nil { + for _, c := range ev.Catalog.Video.Renditions { + if c.TrackID() == 1 { + scale = c.Timescale() + } + } + } + if ev.Type == "segment" { + source = append(source, ev) + } + } + require.NoError(t, <-errs) + require.GreaterOrEqual(t, len(source), 2) + require.NotZero(t, scale) + var joined []byte + for i, index := range []int{0, 1, 0} { + ev := source[index] + var part []byte + for _, id := range []string{"1", "2"} { + part = append(part, ev.Tracks[id]...) + } + start := captionTicks(ev.FirstDecodeTimes["1"], scale).Milliseconds() + 100 + lang, label, text, id := "en", "auto", "English speech", "english" + if i == 1 { + id = "english-two" + } + if i == 2 { + id = "reconnected" + if changeLanguage { + lang, label, text = "es", "human", "Habla española" + } + } + part, err = eng.AddTextTrack(ctx, part, upstream.TextTrack{TrackID: 100, Language: lang, Label: label}, []upstream.TextCue{{ID: id, Text: text, Start: uint64(start), End: uint64(start + 300)}}) + require.NoError(t, err) + events := make(chan *upstream.Event, 8) + go func() { errs <- eng.UnwrapEvents(ctx, bytes.NewReader(part), events); close(events) }() + for event := range events { + if event.Type != "segment" { + continue + } + var keys []string + for key := range event.Tracks { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if textFirst && (keys[i] == "100" || keys[j] == "100") { + return keys[i] == "100" + } + a, _ := strconv.ParseUint(keys[i], 10, 32) + b, _ := strconv.ParseUint(keys[j], 10, 32) + return a < b + }) + for _, key := range keys { + joined = append(joined, event.Tracks[key]...) + } + } + require.NoError(t, <-errs) + } + store, err := blob.NewFileStore(t.TempDir()) + require.NoError(t, err) + const sourceKey = "reconnect-source.m4s" + w, err := store.NewWriter(ctx, sourceKey, "video/iso.segment") + require.NoError(t, err) + _, err = w.Write(joined) + require.NoError(t, err) + require.NoError(t, w.Complete()) + require.NoError(t, w.Close()) + cid, _, meta, err := hashAndBuildFragmentMetafile(ctx, store, []string{sourceKey}) + want := captionTicks(source[0].Durations["1"]+source[1].Durations["1"], scale) + 100*time.Millisecond + if textFirst { + require.Nil(t, meta, "non-canonical input must not emit mis-indexed byte ranges") + return nil, want, err + } + if err != nil { + return nil, want, err + } + w, err = store.NewWriter(ctx, BlobsPrefix+cid+".mp4", "video/mp4") + require.NoError(t, err) + _, err = w.Write(joined) + require.NoError(t, err) + require.NoError(t, w.Complete()) + require.NoError(t, w.Close()) + require.NoError(t, writeMetafile(ctx, store, cid, meta)) + out := &videoCaptionView{cues: map[string][]captions.TimedCue{}} + err = (&VideoCaptions{Store: store}).readMuxl(ctx, cid, "at://did:plc:alice/place.stream.video/vod", out) + return out, want, err +} + +func TestVideoCaptionsReconnectKeepsConfigurationEpochs(t *testing.T) { + out, want, err := reconnectCaptionFixture(t, true, false) + require.NoError(t, err) + english := out.cues["canonical-auto-en"] + spanish := out.cues["canonical-human-es"] + require.Len(t, english, 2, "a reused numeric ID must not relabel earlier English speech") + require.Equal(t, "English speech", english[0].Text) + require.Equal(t, []captions.TimedCue{{ID: "reconnected", Text: "Habla española", Start: want, End: want + 300*time.Millisecond}}, spanish) +} + +func TestVideoCaptionsReconnectUsesContainingGoPClock(t *testing.T) { + out, want, err := reconnectCaptionFixture(t, false, false) + require.NoError(t, err) + cues := out.cues["canonical-auto-en"] + require.Len(t, cues, 3) + require.Equal(t, want, cues[2].Start, "reset text uses its containing GoP's accumulated VOD clock") +} + +func TestVideoCaptionIndexerRejectsNoncanonicalTextFirstOrder(t *testing.T) { + _, _, err := reconnectCaptionFixture(t, false, true) + require.ErrorContains(t, err, "non-canonical MUXL track order") +} diff --git a/pkg/vod/captions_test.go b/pkg/vod/captions_test.go new file mode 100644 index 000000000..dc66df83a --- /dev/null +++ b/pkg/vod/captions_test.go @@ -0,0 +1,109 @@ +package vod + +import ( + "bytes" + "context" + "os" + "sort" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + upstream "github.com/streamplace/muxl/go" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/blob" + "stream.place/streamplace/pkg/captions" + "stream.place/streamplace/pkg/captions/records" + "stream.place/streamplace/pkg/comatproto" + "stream.place/streamplace/pkg/model" + "stream.place/streamplace/pkg/placestream" +) + +func TestVideoCaptionsMuxlPrecedenceAndLazyTrack(t *testing.T) { + ctx := context.Background() + eng, err := captions.TextEngine() + require.NoError(t, err) + data, err := os.ReadFile(getFixture("h264-opus-frag.mp4")) + require.NoError(t, err) + ch := make(chan *upstream.Event, 32) + errs := make(chan error, 1) + go func() { errs <- eng.SegmentEvents(ctx, bytes.NewReader(data), ch); close(ch) }() + var segments []*upstream.Event + for ev := range ch { + if ev.Type == "segment" { + segments = append(segments, ev) + } + } + require.NoError(t, <-errs) + require.GreaterOrEqual(t, len(segments), 2) + var joined []byte + var wantStart, wantEnd time.Duration + require.Zero(t, segments[0].FirstDecodeTimes["1"], "fixture video starts at zero") + for i, ev := range segments[:2] { + var seg []byte + var keys []string + for id := range ev.Tracks { + keys = append(keys, id) + } + sort.Strings(keys) + for _, id := range keys { + seg = append(seg, ev.Tracks[id]...) + } + if i == 1 { + // An empty text track gives SegmentClock the existing AV header. + empty, err := eng.AddTextTrack(ctx, seg, upstream.TextTrack{TrackID: 9, Language: "en", Label: "auto"}, nil) + require.NoError(t, err) + var hdr bytes.Buffer + require.NoError(t, eng.Wrap(ctx, bytes.NewReader(empty), "flat", &hdr)) + base, _, err := captions.SegmentClock(hdr.Bytes(), seg) + require.NoError(t, err) + wantStart = time.Duration(base.Milliseconds()+100) * time.Millisecond + wantEnd = time.Duration(base.Milliseconds()+500) * time.Millisecond + seg, err = eng.AddTextTrack(ctx, seg, upstream.TextTrack{TrackID: 9, Language: "en", Label: "auto"}, []upstream.TextCue{{ID: "archived", Text: "The mastered text", Start: uint64(base.Milliseconds() + 100), End: uint64(base.Milliseconds() + 500)}}) + require.NoError(t, err) + } + joined = append(joined, seg...) + } + store, err := blob.NewFileStore(t.TempDir()) + require.NoError(t, err) + const cid = "captions-fixture" + writer, err := store.NewWriter(ctx, BlobsPrefix+cid+".mp4", "video/mp4") + require.NoError(t, err) + _, err = writer.Write(joined) + require.NoError(t, err) + require.NoError(t, writer.Complete()) + require.NoError(t, writer.Close()) + mb := newFragmentMetafileBuilder(ctx, store) + ch = make(chan *upstream.Event, 32) + go func() { errs <- eng.UnwrapEvents(ctx, bytes.NewReader(joined), ch); close(ch) }() + for ev := range ch { + require.NoError(t, mb.Observe(ev)) + } + require.NoError(t, <-errs) + meta := mb.Finalize(cid, int64(len(joined))) + require.NoError(t, writeMetafile(ctx, store, cid, meta)) + require.Equal(t, "text", meta.Tracks["9"].Type) + require.Equal(t, []textProbeJSON{{TrackID: "9", Language: "en", Label: "auto"}}, metafileTextTracks(meta)) + m, err := model.MakeDB(":memory:") + require.NoError(t, err) + const video = "at://did:plc:alice/place.stream.video/vod" + const trackURI = "at://did:plc:alice/place.stream.media.track/track" + require.NoError(t, m.UpsertMediaTrack(ctx, placestream.MediaTrack{Track: placestream.MediaTrack_Track{MediaDefs_MuxlTrack: &placestream.MediaDefs_MuxlTrack{Blob: cid, TrackId: "1", MediaType: "video"}}}, syntax.ATURI(trackURI))) + require.NoError(t, m.UpsertVideo(ctx, placestream.Video{Source: placestream.Video_Source{MediaDefs_SourceTracks: &placestream.MediaDefs_SourceTracks{Tracks: []comatproto.RepoStrongRef{{Uri: trackURI, Cid: "bafy"}}}}}, syntax.ATURI(video))) + for _, source := range []string{"auto", "imported"} { + rec := placestream.CaptionTranscript{Subject: comatproto.RepoStrongRef{Uri: video, Cid: "bafy"}, Text: "The mastered text", StartMs: 0, Timings: []int64{100, 100, 200}, Language: "en", Source: source, CreatedAt: "2026-09-30T00:00:00Z"} + require.NoError(t, m.UpsertCaptionTranscript(ctx, rec, syntax.ATURI("at://did:plc:alice/place.stream.caption.transcript/"+source))) + } + p := &VideoCaptions{Model: m, Store: store, Records: &records.Provider{Store: m}} + tracks, err := p.Tracks(ctx, video) + require.NoError(t, err) + require.Len(t, tracks, 2) + got, err := p.Cues(ctx, video, "canonical-auto-en") + require.NoError(t, err) + require.Equal(t, []captions.TimedCue{{ID: "archived", Text: "The mastered text", Start: wantStart, End: wantEnd}}, got) + for _, track := range tracks { + if track.Origin == captions.OriginRecord { + require.Equal(t, captions.SourceImported, track.Source) + } + } +} diff --git a/pkg/vod/captions_tracks.go b/pkg/vod/captions_tracks.go new file mode 100644 index 000000000..394feaf16 --- /dev/null +++ b/pkg/vod/captions_tracks.go @@ -0,0 +1,87 @@ +package vod + +import ( + "sort" + "strconv" + "stream.place/streamplace/pkg/muxl" +) + +func metafileTextTracks(meta *Metafile) []textProbeJSON { + var tracks []textProbeJSON + for id, t := range meta.Tracks { + if t.Type == "text" { + tracks = append(tracks, textProbeJSON{TrackID: id, Language: t.Language, Label: t.Label}) + } + } + sort.Slice(tracks, func(i, j int) bool { return tracks[i].TrackID < tracks[j].TrackID }) + return tracks +} + +func mergeCatalog(dst, src *muxl.MuxlCatalog) *muxl.MuxlCatalog { + if dst == nil { + return src + } + if src == nil { + return dst + } + if src.Video != nil { + if dst.Video == nil { + dst.Video = src.Video + } else { + for k, v := range src.Video.Renditions { + dst.Video.Renditions[k] = v + } + } + } + if src.Audio != nil { + if dst.Audio == nil { + dst.Audio = src.Audio + } else { + for k, v := range src.Audio.Renditions { + dst.Audio.Renditions[k] = v + } + } + } + if src.Text != nil { + if dst.Text == nil { + dst.Text = src.Text + } else { + for k, v := range src.Text.Renditions { + dst.Text.Renditions[k] = v + } + } + } + return dst +} + +// catalogCaptionReference chooses an AV track actually present in this GoP. +// Keeping the association here avoids guessing it from byte ordering later. +func catalogCaptionReference(cat *muxl.MuxlCatalog, tracks map[string][]byte) (string, uint32) { + if cat == nil { + return "", 0 + } + var id uint32 + var scale uint32 + if cat.Video != nil { + for _, c := range cat.Video.Renditions { + key := strconv.FormatUint(uint64(c.TrackID()), 10) + if _, ok := tracks[key]; ok && (scale == 0 || c.TrackID() < id) { + id = c.TrackID() + scale = c.Timescale() + } + } + } + if scale == 0 && cat.Audio != nil { + for _, c := range cat.Audio.Renditions { + key := strconv.FormatUint(uint64(c.TrackID()), 10) + if _, ok := tracks[key]; ok && (scale == 0 || c.TrackID() < id) { + id = c.TrackID() + scale = c.Timescale() + } + } + } + if scale == 0 { + return "", 0 + } + return strconv.FormatUint(uint64(id), 10), scale +} diff --git a/pkg/vod/finalize_livestream.go b/pkg/vod/finalize_livestream.go index 026a7f5df..8ef047931 100644 --- a/pkg/vod/finalize_livestream.go +++ b/pkg/vod/finalize_livestream.go @@ -137,6 +137,7 @@ func FinalizeLivestreamVOD(ctx context.Context, cli *config.CLI, state *statedb. size: blobSize, mimeType: "video/mp4", probe: probe, + text: metafileTextTracks(metafile), signingKey: in.SigningKey, }) }); err != nil { diff --git a/pkg/vod/flat_vod.go b/pkg/vod/flat_vod.go index 73f3bc1ac..93c1b8b19 100644 --- a/pkg/vod/flat_vod.go +++ b/pkg/vod/flat_vod.go @@ -63,6 +63,7 @@ func hashAndBuildFragmentMetafile(ctx context.Context, store blob.Store, keys [] tee := io.TeeReader(io.MultiReader(readers...), io.MultiWriter(hasher, counter)) mb := newFragmentMetafileBuilder(ctx, store) + mb.canonicalHash = bdasl.NewWriter() eventCh := make(chan *muxl.MuxlEvent, 16) producerErr := make(chan error, 1) go func() { @@ -87,6 +88,9 @@ func hashAndBuildFragmentMetafile(ctx context.Context, store blob.Store, keys [] } muxlCID := hasher.CID() + if mb.canonicalHash.CID() != muxlCID { + return "", 0, nil, errors.New("non-canonical MUXL track order: fragments must use ascending numeric track_id within each GoP") + } size := counter.load() return muxlCID, size, mb.Finalize(muxlCID, size), nil } diff --git a/pkg/vod/metafile.go b/pkg/vod/metafile.go index 96bf46783..5d39db799 100644 --- a/pkg/vod/metafile.go +++ b/pkg/vod/metafile.go @@ -7,6 +7,7 @@ import ( "fmt" "sort" "strconv" + "time" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -93,6 +94,9 @@ type MetafileTrack struct { // Audio-only. Channels uint32 `json:"channels,omitempty"` SampleRate uint32 `json:"sampleRate,omitempty"` + // Text-track encoding: language is BCP 47, label is captions.Source. + Language string `json:"language,omitempty"` + Label string `json:"label,omitempty"` } // MetafileSegment is one GOP-sized byte range within the blob. @@ -101,6 +105,18 @@ type MetafileSegment struct { Size int64 `json:"size"` DurationTicks uint64 `json:"durationTicks"` SampleCount uint32 `json:"sampleCount"` + // DecodeTimeKnown distinguishes a genuine zero tfdt from legacy indexes + // that lack decode times. Caption placement uses the reference AV clock. + FirstDecodeTicks uint64 `json:"firstDecodeTicks,omitempty"` + DecodeTimeKnown bool `json:"decodeTimeKnown,omitempty"` + // Text identity and reference clock belong to this GoP, not to the final + // merged catalog or the physical ordering of neighboring byte ranges. + CaptionConfigKnown bool `json:"captionConfigKnown,omitempty"` + CaptionLanguage string `json:"captionLanguage,omitempty"` + CaptionLabel string `json:"captionLabel,omitempty"` + CaptionReferenceTicks uint64 `json:"captionReferenceTicks,omitempty"` + CaptionReferenceScale uint32 `json:"captionReferenceScale,omitempty"` + CaptionOffsetNanos int64 `json:"captionOffsetNanos,omitempty"` // Discontinuity marks a segment that begins a new continuous timeline — // its decode time jumped backward relative to the previous segment of the // same track. This happens when a recording concatenates multiple ingest @@ -141,8 +157,15 @@ type metafileBuilder struct { // lastTFDT / tfdtSeen track each track's previous baseMediaDecodeTime so a // backward jump (a concatenated reconnect/restart) can be flagged as a // discontinuity. See MetafileSegment.Discontinuity. - lastTFDT map[string]uint64 - tfdtSeen map[string]bool + lastTFDT map[string]uint64 + tfdtSeen map[string]bool + textConfigs map[string]textProbeJSON + referenceScale uint32 + referenceTicks uint64 + referenceOffset time.Duration + // Set only when indexing untouched archived fragments: compare their + // original content hash with the ascending-numeric event serialization. + canonicalHash *bdasl.Writer } func newMetafileBuilder(ctx context.Context, store blob.Store) *metafileBuilder { @@ -154,6 +177,7 @@ func newMetafileBuilder(ctx context.Context, store blob.Store) *metafileBuilder leadingInitInBlob: true, lastTFDT: map[string]uint64{}, tfdtSeen: map[string]bool{}, + textConfigs: map[string]textProbeJSON{}, } } @@ -172,15 +196,13 @@ func newFragmentMetafileBuilder(ctx context.Context, store blob.Store) *metafile func (b *metafileBuilder) Observe(ev *muxl.MuxlEvent) error { switch ev.Type { case "init": - if b.seenInit { - // Mid-stream init swap (catalog change). Doesn't happen in - // today's single-input VOD pipeline; if it ever does we'd - // need a richer schema (sub-archives per init). Warn loudly - // rather than silently produce a wrong metafile. - log.Warn(b.ctx, "metafile: mid-stream init swap; offsets after this point may be wrong") + if ev.Catalog != nil && ev.Catalog.Text != nil { + for _, c := range ev.Catalog.Text.Renditions { + id := strconv.FormatUint(uint64(c.TrackID()), 10) + b.textConfigs[id] = textProbeJSON{TrackID: id, Language: c.Language, Label: c.Label} + } } - b.seenInit = true - b.catalog = ev.Catalog + b.catalog = mergeCatalog(b.catalog, ev.Catalog) // Write per-track init bytes to the blob.Store keyed by their // own BDASL CID. The primary blob's init occupies bytes // [0, len(ev.Data)) in the output; advance runningOffset by @@ -195,12 +217,26 @@ func (b *metafileBuilder) Observe(ev *muxl.MuxlEvent) error { // Advance past the leading init only when it physically prefixes the // blob. For the flat-MP4 shape the fragments start at 0 (the flat-header // is added at serve/store time, not measured here). - if b.leadingInitInBlob { - b.runningOffset = int64(len(ev.Data)) - } else { - b.runningOffset = 0 + if !b.seenInit { + if b.leadingInitInBlob { + b.runningOffset = int64(len(ev.Data)) + } + b.seenInit = true } case "segment", "signed-segment": + refID, scale := catalogCaptionReference(b.catalog, ev.Tracks) + refTicks, referenceKnown := firstTFDT(ev.Tracks[refID]) + if b.referenceScale != scale { + if b.referenceScale != 0 { + b.referenceOffset += captionTicks(b.referenceTicks, b.referenceScale) + } + b.referenceScale = scale + b.referenceTicks = 0 + } + var captionOffset time.Duration + if scale != 0 { + captionOffset = b.referenceOffset + captionTicks(b.referenceTicks, scale) + } // Within a single segment event, per-track byte slices are // concatenated in sorted key order (matching ParseMuxlEvents' // byte-channel dispatch). Track that order here so offsets @@ -211,31 +247,55 @@ func (b *metafileBuilder) Observe(ev *muxl.MuxlEvent) error { for k := range ev.Tracks { keys = append(keys, k) } - sort.Strings(keys) + sort.Slice(keys, func(i, j int) bool { + a, _ := strconv.ParseUint(keys[i], 10, 32) + c, _ := strconv.ParseUint(keys[j], 10, 32) + return a < c + }) for _, tid := range keys { chunk := ev.Tracks[tid] + if b.canonicalHash != nil { + if _, err := b.canonicalHash.Write(chunk); err != nil { + return fmt.Errorf("hash canonical MUXL layout: %w", err) + } + } // Flag a discontinuity when this track's decode time jumps backward // vs its previous segment — the signature of a concatenated // reconnect/restart. A normal stream's tfdt is strictly increasing // (tfdt[n] = tfdt[n-1] + duration[n-1]), so this never fires for a // clean single-session recording. disc := false - if tfdt, ok := firstTFDT(chunk); ok { + tfdt, known := firstTFDT(chunk) + if known { if b.tfdtSeen[tid] && tfdt < b.lastTFDT[tid] { disc = true } b.lastTFDT[tid] = tfdt b.tfdtSeen[tid] = true } - b.trackSegments[tid] = append(b.trackSegments[tid], MetafileSegment{ - Offset: b.runningOffset, - Size: int64(len(chunk)), - DurationTicks: ev.Durations[tid], - SampleCount: ev.SampleCounts[tid], - Discontinuity: disc, - }) + entry := MetafileSegment{ + Offset: b.runningOffset, + Size: int64(len(chunk)), + DurationTicks: ev.Durations[tid], + SampleCount: ev.SampleCounts[tid], + Discontinuity: disc, + FirstDecodeTicks: tfdt, + DecodeTimeKnown: known, + } + if config, ok := b.textConfigs[tid]; ok { + entry.CaptionConfigKnown = true + entry.CaptionLanguage = config.Language + entry.CaptionLabel = config.Label + if referenceKnown { + entry.CaptionReferenceTicks = refTicks + entry.CaptionReferenceScale = scale + entry.CaptionOffsetNanos = int64(captionOffset) + } + } + b.trackSegments[tid] = append(b.trackSegments[tid], entry) b.runningOffset += int64(len(chunk)) } + b.referenceTicks += ev.Durations[refID] default: log.Warn(b.ctx, "metafile: unexpected event type; skipping", "type", ev.Type) } @@ -286,6 +346,38 @@ func (b *metafileBuilder) Finalize(cid string, size int64) *Metafile { } } } + if track.Type == "unknown" && b.catalog.Text != nil { + for _, c := range b.catalog.Text.Renditions { + if c.TrackID() == targetTID { + track.Type = "text" + track.Codec = c.Codec + track.Timescale = c.Timescale() + track.Language = c.Language + track.Label = c.Label + break + } + } + } + } + } + if track.Type == "text" { + first := true + for _, seg := range segments { + if !seg.CaptionConfigKnown { + continue + } + if first { + track.Language = seg.CaptionLanguage + track.Label = seg.CaptionLabel + first = false + continue + } + if track.Language != seg.CaptionLanguage { + track.Language = "und" + } + if track.Label != seg.CaptionLabel { + track.Label = "" + } } } tracks[tid] = track diff --git a/pkg/vod/publish.go b/pkg/vod/publish.go index 0e403e2a1..56be3686a 100644 --- a/pkg/vod/publish.go +++ b/pkg/vod/publish.go @@ -46,6 +46,7 @@ type publishParams struct { mimeType string probe media.VODResult signingKey string + text []textProbeJSON } // publishRecords does the post-processing record publish. With tracks @@ -95,6 +96,18 @@ func publishRecords(ctx context.Context, p publishParams) error { span.SetStatus(codes.Error, "marshal_probe") return fmt.Errorf("marshal probe: %w", err) } + if len(p.text) > 0 { + var shape probeJSONShape + if err := json.Unmarshal([]byte(probeJSON), &shape); err != nil { + return err + } + shape.Text = p.text + data, err := json.Marshal(shape) + if err != nil { + return err + } + probeJSON = string(data) + } if err := p.state.SetUploadProcessed(ctx, p.in.UploadID, p.probe.DurationMS, p.cid, p.signingKey, probeJSON, p.size); err != nil { span.RecordError(err) @@ -114,6 +127,7 @@ type probeJSONShape struct { DurationMS int64 `json:"durationMs"` Video *videoProbeJSON `json:"video,omitempty"` Audio *audioProbeJSON `json:"audio,omitempty"` + Text []textProbeJSON `json:"text,omitempty"` } type videoProbeJSON struct { Codec string `json:"codec"` @@ -129,6 +143,12 @@ type audioProbeJSON struct { MPEGVersion int `json:"mpegVersion"` } +type textProbeJSON struct { + TrackID string `json:"trackId"` + Language string `json:"language"` + Label string `json:"label"` +} + func marshalProbe(p media.VODResult) (string, error) { out := probeJSONShape{DurationMS: p.DurationMS} if p.Video != nil { @@ -220,7 +240,7 @@ func publishOrigin(ctx context.Context, cli *config.CLI, cid string, size int64, // this upload's segments — the same key signs every track of an upload. // Exactly one of videoMeta / audioMeta should be non-nil; the other // is ignored. -func publishTrack(ctx context.Context, client XRPCClient, did, cid string, blobSize, durationMS int64, trackID, mediaType, signingKey string, videoMeta *media.VODVideoTrack, audioMeta *media.VODAudioTrack) (*comatproto.RepoStrongRef, error) { +func publishTrack(ctx context.Context, client XRPCClient, did, cid string, blobSize, durationMS int64, trackID, mediaType, signingKey string, videoMeta *media.VODVideoTrack, audioMeta *media.VODAudioTrack, textMeta *textProbeJSON) (*comatproto.RepoStrongRef, error) { ctx, span := vodTracer.Start(ctx, "vod.publishTrack", trace.WithAttributes( attribute.String("cid", cid), attribute.String("track_id", trackID), @@ -252,6 +272,9 @@ func publishTrack(ctx context.Context, client XRPCClient, did, cid string, blobS Channels: int64(audioMeta.Channels), } } + if textMeta != nil { + meta.Language = &textMeta.Language + } rec := &placestream.MediaTrack{ LexiconTypeID: constants.PLACE_STREAM_MEDIA_TRACK, diff --git a/pkg/vod/publish_draft.go b/pkg/vod/publish_draft.go index 65342b816..dbb2092c7 100644 --- a/pkg/vod/publish_draft.go +++ b/pkg/vod/publish_draft.go @@ -236,19 +236,30 @@ func tracksForUpload(ctx context.Context, state *statedb.StatefulDB, client XRPC } var tracks []comatproto.RepoStrongRef if probe.Video != nil { - ref, err := publishTrack(ctx, client, did, upload.ContentCID, upload.BlobSize, probe.DurationMS, "1", "video", upload.SigningKey, probe.Video, nil) + ref, err := publishTrack(ctx, client, did, upload.ContentCID, upload.BlobSize, probe.DurationMS, "1", "video", upload.SigningKey, probe.Video, nil, nil) if err != nil { return nil, fmt.Errorf("publish video track: %w", err) } tracks = append(tracks, *ref) } if probe.Audio != nil { - ref, err := publishTrack(ctx, client, did, upload.ContentCID, upload.BlobSize, probe.DurationMS, "2", "audio", upload.SigningKey, nil, probe.Audio) + ref, err := publishTrack(ctx, client, did, upload.ContentCID, upload.BlobSize, probe.DurationMS, "2", "audio", upload.SigningKey, nil, probe.Audio, nil) if err != nil { return nil, fmt.Errorf("publish audio track: %w", err) } tracks = append(tracks, *ref) } + var shape probeJSONShape + if err := json.Unmarshal([]byte(upload.ProbeJSON), &shape); err != nil { + return nil, err + } + for _, text := range shape.Text { + ref, err := publishTrack(ctx, client, did, upload.ContentCID, upload.BlobSize, probe.DurationMS, text.TrackID, "text", upload.SigningKey, nil, nil, &text) + if err != nil { + return nil, fmt.Errorf("publish text track: %w", err) + } + tracks = append(tracks, *ref) + } if len(tracks) == 0 { return nil, nil } diff --git a/pkg/vod/publish_video.go b/pkg/vod/publish_video.go index 90e27a9c5..1b7c701c3 100644 --- a/pkg/vod/publish_video.go +++ b/pkg/vod/publish_video.go @@ -45,6 +45,20 @@ var ( // (the client may not supply one) using the same generateThumbnail path // vod-test exercises. func PublishVideo(ctx context.Context, state *statedb.StatefulDB, store blob.Store, did, uploadID string, video *placestream.Video) (string, string, error) { + return publishVideo(ctx, state, store, nil, did, uploadID, video) +} + +// PublishVideoWithClient publishes a processed upload with an already +// authenticated PDS client. It uses the same validation, track publication, +// thumbnail generation and video record creation as PublishVideo. +func PublishVideoWithClient(ctx context.Context, state *statedb.StatefulDB, store blob.Store, client XRPCClient, did, uploadID string, video *placestream.Video) (string, string, error) { + if client == nil { + return "", "", errors.New("authenticated PDS client required") + } + return publishVideo(ctx, state, store, client, did, uploadID, video) +} + +func publishVideo(ctx context.Context, state *statedb.StatefulDB, store blob.Store, client XRPCClient, did, uploadID string, video *placestream.Video) (string, string, error) { ctx = log.WithLogValues(ctx, "func", "PublishVideo", "did", did, "uploadId", uploadID) ctx, span := vodTracer.Start(ctx, "vod.PublishVideo", trace.WithAttributes( attribute.String("did", did), @@ -71,10 +85,12 @@ func PublishVideo(ctx context.Context, state *statedb.StatefulDB, store blob.Sto video.DurationMs = upload.DurationMS video.CreatedAt = time.Now().UTC().Format(time.RFC3339) - client, err := getUserXRPCClient(ctx, state, did) - if err != nil { - span.RecordError(err) - return "", "", fmt.Errorf("get user xrpc client: %w", err) + if client == nil { + client, err = getUserXRPCClient(ctx, state, did) + if err != nil { + span.RecordError(err) + return "", "", fmt.Errorf("get user xrpc client: %w", err) + } } // The track records are published here, at publish time, like the diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2e72e1ae..da18df7bd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -730,6 +730,9 @@ importers: uint8arrays: specifier: ^5.1.0 version: 5.1.0 + undici: + specifier: ^6.24.0 + version: 6.24.0 js/docs: dependencies: @@ -853,6 +856,9 @@ importers: '@atproto/oauth-client-browser': specifier: ^0.4.1 version: 0.4.1 + '@atproto/syntax': + specifier: ^0.6.1 + version: 0.6.1 '@base-ui/react': specifier: ^1.5.0 version: 1.5.0(@types/react@19.2.14)(react-dom@19.2.0(react@19.2.0))(react@19.2.0) -- 2.51.2