From ddef4c96b040e560ebf190ce3e2350d5ef47520c Mon Sep 17 00:00:00 2001 From: Eli Streams Date: Sun, 6 Apr 2025 02:55:19 +0000 Subject: [PATCH] add livepeer transcoding part 2! See merge request streamplace/streamplace!122 Changelog: feature --- Makefile | 19 +- js/app/components/livestream/livestream.tsx | 3 +- js/app/components/player/controls.tsx | 146 ++++- js/app/components/player/player.tsx | 28 +- js/app/components/player/props.tsx | 6 +- js/app/components/player/provider.tsx | 5 +- js/app/components/player/shared.tsx | 27 +- js/app/components/player/use-webrtc.tsx | 2 +- js/app/components/player/video-retry.tsx | 9 +- js/app/components/player/video.native.tsx | 9 +- js/app/components/player/video.tsx | 10 +- js/app/features/player/playerSlice.tsx | 130 +++- js/desktop/package.json | 2 +- lexicons/place/stream/defs.json | 17 + lexicons/place/stream/livestream.json | 2 + lexicons/place/stream/segment.json | 18 +- pkg/api/api.go | 40 +- pkg/api/api_internal.go | 96 ++- pkg/api/api_util.go | 12 + pkg/api/playback.go | 20 +- pkg/atproto/firehose.go | 30 + pkg/atproto/sync.go | 6 +- pkg/cmd/streamplace.go | 72 +-- pkg/config/config.go | 1 + pkg/director/director.go | 95 +++ pkg/director/stream_session.go | 251 ++++++++ pkg/gen/gen.go | 1 + pkg/livepeer/livepeer.go | 111 ++++ pkg/media/bus_handler.go | 2 +- pkg/media/concat.go | 44 +- pkg/media/ffmpeg_concat.go | 95 +++ pkg/media/gstreamer.go | 638 -------------------- pkg/media/m3u8.go | 239 ++++---- pkg/media/media.go | 250 +------- pkg/media/media_data_parser.go | 140 +++++ pkg/media/progressive.go | 198 ++++++ pkg/media/segchanman/segchanman.go | 76 +++ pkg/media/segment_conv.go | 131 ++++ pkg/media/segment_conv_test.go | 35 ++ pkg/media/segmenter.go | 81 +++ pkg/media/segmenter_hls.go | 296 +++++++++ pkg/media/thumbnail.go | 62 ++ pkg/media/validate.go | 104 ++++ pkg/media/webrtc.go | 12 +- pkg/model/segment.go | 18 +- pkg/renditions/renditions.go | 210 +++++++ pkg/renditions/renditions_test.go | 323 ++++++++++ pkg/spmetrics/spmetrics.go | 20 + pkg/streamplace/cbor_gen.go | 284 ++++++++- pkg/streamplace/streamdefs.go | 16 + pkg/streamplace/streamlivestream.go | 16 + pkg/streamplace/streamsegment.go | 15 +- 52 files changed, 3246 insertions(+), 1227 deletions(-) create mode 100644 pkg/api/api_util.go create mode 100644 pkg/director/director.go create mode 100644 pkg/director/stream_session.go create mode 100644 pkg/livepeer/livepeer.go create mode 100644 pkg/media/ffmpeg_concat.go create mode 100644 pkg/media/media_data_parser.go create mode 100644 pkg/media/progressive.go create mode 100644 pkg/media/segchanman/segchanman.go create mode 100644 pkg/media/segment_conv.go create mode 100644 pkg/media/segment_conv_test.go create mode 100644 pkg/media/segmenter.go create mode 100644 pkg/media/segmenter_hls.go create mode 100644 pkg/media/thumbnail.go create mode 100644 pkg/media/validate.go create mode 100644 pkg/renditions/renditions.go create mode 100644 pkg/renditions/renditions_test.go diff --git a/Makefile b/Makefile index e119561f..ff940fbb 100644 --- a/Makefile +++ b/Makefile @@ -187,7 +187,9 @@ ios: app mkdir -p .build \ && curl -L -o ./.build/bundletool.jar https://github.com/google/bundletool/releases/download/1.17.0/bundletool-all-1.17.0.jar -OPTS = -D "gst-plugins-base:audioresample=enabled" \ +OPTS = \ + --buildtype=debugoptimized \ + -D "gst-plugins-base:audioresample=enabled" \ -D "gst-plugins-base:playback=enabled" \ -D "gst-plugins-base:opus=enabled" \ -D "gst-plugins-base:gio-typefinder=enabled" \ @@ -197,6 +199,7 @@ OPTS = -D "gst-plugins-base:audioresample=enabled" \ -D "gst-plugins-base:compositor=enabled" \ -D "gst-plugins-base:videorate=enabled" \ -D "gst-plugins-base:app=enabled" \ + -D "gst-plugins-base:audiorate=enabled" \ -D "gst-plugins-base:audiotestsrc=enabled" \ -D "gst-plugins-base:audioconvert=enabled" \ -D "gst-plugins-good:matroska=enabled" \ @@ -211,22 +214,30 @@ OPTS = -D "gst-plugins-base:audioresample=enabled" \ -D "gst-plugins-good:audioparsers=enabled" \ -D "gst-plugins-bad:videoparsers=enabled" \ -D "gst-plugins-bad:mpegtsmux=enabled" \ + -D "gst-plugins-bad:mpegtsdemux=enabled" \ -D "gst-plugins-bad:codectimestamper=enabled" \ -D "gst-plugins-bad:opus=enabled" \ -D "gst-plugins-ugly:x264=enabled" \ -D "gst-plugins-ugly:gpl=enabled" \ -D "x264:asm=enabled" \ -D "gstreamer-full:gst-full=enabled" \ - -D "gstreamer-full:gst-full-plugins=libgstopusparse.a;libgstcodectimestamper.a;libgstrtp.a;libgstaudioresample.a;libgstlibav.a;libgstmatroska.a;libgstmultifile.a;libgstjpeg.a;libgstaudiotestsrc.a;libgstaudioconvert.a;libgstaudioparsers.a;libgstfdkaac.a;libgstisomp4.a;libgstapp.a;libgstvideoconvertscale.a;libgstvideobox.a;libgstvideorate.a;libgstpng.a;libgstcompositor.a;libgstx264.a;libgstopus.a;libgstvideotestsrc.a;libgstvideoparsersbad.a;libgstaudioparsers.a;libgstmpegtsmux.a;libgstplayback.a;libgsttypefindfunctions.a" \ + -D "gstreamer-full:gst-full-plugins=libgstopusparse.a;libgstcodectimestamper.a;libgstrtp.a;libgstaudioresample.a;libgstlibav.a;libgstmatroska.a;libgstmultifile.a;libgstjpeg.a;libgstaudiotestsrc.a;libgstaudioconvert.a;libgstaudioparsers.a;libgstfdkaac.a;libgstisomp4.a;libgstapp.a;libgstvideoconvertscale.a;libgstvideobox.a;libgstvideorate.a;libgstpng.a;libgstcompositor.a;libgstaudiorate.a;libgstx264.a;libgstopus.a;libgstvideotestsrc.a;libgstvideoparsersbad.a;libgstaudioparsers.a;libgstmpegtsmux.a;libgstmpegtsdemux.a;libgstplayback.a;libgsttypefindfunctions.a" \ -D "gstreamer-full:gst-full-libraries=gstreamer-controller-1.0,gstreamer-plugins-base-1.0,gstreamer-pbutils-1.0" \ -D "gstreamer-full:gst-full-target-type=static_library" \ - -D "gstreamer-full:gst-full-elements=coreelements:concat,filesrc,queue,queue2,multiqueue,typefind,tee,capsfilter,fakesink,identity" \ + -D "gstreamer-full:gst-full-elements=coreelements:concat,filesrc,filesink,queue,queue2,multiqueue,typefind,tee,capsfilter,fakesink,identity" \ -D "gstreamer-full:bad=enabled" \ -D "gstreamer-full:tls=disabled" \ -D "gstreamer-full:libav=enabled" \ -D "gstreamer-full:ugly=enabled" \ -D "gstreamer-full:gpl=enabled" \ - -D "gstreamer-full:gst-full-typefind-functions=" + -D "gstreamer-full:gst-full-typefind-functions=" \ + -D "gstreamer-full:glib_assert=false" \ + -D "gstreamer:glib_assert=false" \ + -D "gst-plugins-good:glib_assert=false" \ + -D "gst-plugins-bad:glib_assert=false" \ + -D "gst-plugins-base:glib_assert=false" \ + -D "gst-plugins-ugly:glib_assert=false" \ + -D "glib:glib_assert=false" .PHONY: meson-setup meson-setup: diff --git a/js/app/components/livestream/livestream.tsx b/js/app/components/livestream/livestream.tsx index 421c66fa..bf9201a9 100644 --- a/js/app/components/livestream/livestream.tsx +++ b/js/app/components/livestream/livestream.tsx @@ -30,7 +30,7 @@ export function LivestreamInner(props: Partial) { const telemetry = useAppSelector(selectTelemetry); const player = useAppSelector(usePlayer()); - const { src, protocol, ...extraProps } = props; + const { src, ...extraProps } = props; const dispatch = useAppDispatch(); const { width, height } = useWindowDimensions(); const video = player.segment?.video?.[0]; @@ -147,7 +147,6 @@ export function LivestreamInner(props: Partial) { ( + ); useEffect(() => { if (!media.sm && props.showControls === false) { setOpen(false); @@ -211,9 +225,7 @@ export function PopoverMenu(props: PlayerProps) { - - - + {gearMenu} - + {gearMenu} ); @@ -296,8 +308,26 @@ function LiveBubbleText() { return ; } -function GearMenu(props: PlayerProps) { +function GearMenu( + props: PlayerProps & { + renditions: Rendition[]; + selectedRendition: string; + protocol: string; + setSelectedRendition: (rendition: string) => void; + setProtocol: (protocol: string) => void; + dispatch: Dispatch; + }, +) { const [menu, setMenu] = useState("root"); + const { + renditions, + selectedRendition, + protocol, + setSelectedRendition, + setProtocol, + dispatch, + } = props; + return ( {menu == "root" && ( @@ -319,9 +349,10 @@ function GearMenu(props: PlayerProps) { hoverTheme pressTheme title="Quality" - subTitle="WIP" + subTitle="Adjust bandwidth usage" icon={Sparkle} iconAfter={ChevronRight} + onPress={() => setMenu("quality")} /> @@ -345,11 +376,11 @@ function GearMenu(props: PlayerProps) { title="HLS" subTitle="HTTP Live Streaming" icon={Star} - iconAfter={props.protocol === PROTOCOL_HLS ? CheckCircle : Circle} - onPress={() => props.setProtocol(PROTOCOL_HLS)} + iconAfter={protocol === PROTOCOL_HLS ? CheckCircle : Circle} + onPress={() => dispatch(setProtocol(PROTOCOL_HLS))} /> - + {/* props.setProtocol(PROTOCOL_PROGRESSIVE_MP4)} + onPress={() => dispatch(setProtocol(PROTOCOL_PROGRESSIVE_MP4))} /> @@ -374,13 +403,11 @@ function GearMenu(props: PlayerProps) { subTitle="WebM but loooong" icon={Squirrel} iconAfter={ - props.protocol === PROTOCOL_PROGRESSIVE_WEBM - ? CheckCircle - : Circle + protocol === PROTOCOL_PROGRESSIVE_WEBM ? CheckCircle : Circle } - onPress={() => props.setProtocol(PROTOCOL_PROGRESSIVE_WEBM)} + onPress={() => dispatch(setProtocol(PROTOCOL_PROGRESSIVE_WEBM))} /> - + */} dispatch(setProtocol(PROTOCOL_WEBRTC))} + /> + + + )} + {menu == "quality" && ( + <> + + setMenu("root")} + /> + + + {protocol === PROTOCOL_HLS && ( + <> + + dispatch(setSelectedRendition("auto"))} + /> + + + + )} + + props.setProtocol(PROTOCOL_WEBRTC)} + onPress={() => dispatch(setSelectedRendition("source"))} /> + {renditions.map((rendition) => ( + + + + dispatch(setSelectedRendition(rendition.name))} + /> + + + ))} )} diff --git a/js/app/components/player/player.tsx b/js/app/components/player/player.tsx index 43f232a2..9e1ff976 100644 --- a/js/app/components/player/player.tsx +++ b/js/app/components/player/player.tsx @@ -10,12 +10,15 @@ import { PlayerProps, PlayerStatus, PlayerStatusTracker, - PROTOCOL_WEBRTC, } from "./props"; import PlayerProvider from "./provider"; import { selectUserMuted } from "features/streamplace/streamplaceSlice"; import { useAppSelector } from "store/hooks"; -import { usePlayerSegment } from "features/player/playerSlice"; +import { + usePlayerRenditions, + usePlayerSegment, + usePlayerSelectedRendition, +} from "features/player/playerSlice"; const HIDE_CONTROLS_AFTER = 2000; const OFFLINE_THRESHOLD = 10000; @@ -55,19 +58,6 @@ export function PlayerInner(props: Partial) { setTouchTime(Date.now()); setShowControls(true); }; - // keeping this other logic for now in case we need a second-best choice - let defProto = PROTOCOL_WEBRTC; - // const plat = usePlatform(); - // if (plat.isIOS) { - // defProto = PROTOCOL_HLS; - // } else if (plat.isSafari) { - // defProto = PROTOCOL_HLS; - // } else if (plat.isFirefox) { - // defProto = PROTOCOL_HLS; - // } - if (props.forceProtocol) { - defProto = props.forceProtocol; - } const { url } = useStreamplaceNode(); const info = usePlatform(); const playerEvent = async ( @@ -98,7 +88,6 @@ export function PlayerInner(props: Partial) { }; const [status, setStatus] = usePlayerStatus(playerEvent); const [playTime, setPlayTime] = useState(0); - const [protocol, setProtocol] = useState(defProto); const [fullscreen, setFullscreen] = useState(false); const [offline, setOffline] = useState(true); @@ -107,6 +96,9 @@ export function PlayerInner(props: Partial) { const segment = useAppSelector(usePlayerSegment()); const [lastCheck, setLastCheck] = useState(0); + const renditions = useAppSelector(usePlayerRenditions()); + const selectedRendition = useAppSelector(usePlayerSelectedRendition()); + useEffect(() => { if (playing) { setOffline(false); @@ -143,8 +135,6 @@ export function PlayerInner(props: Partial) { setFullscreen: setFullscreen, fullscreen: fullscreen, offline: offline, - protocol: protocol, - setProtocol: setProtocol, showControls: props.showControls ?? showControls, userInteraction: userInteraction, playerEvent: playerEvent, @@ -154,6 +144,8 @@ export function PlayerInner(props: Partial) { setPlayTime: setPlayTime, ingestMediaSource: props.ingestMediaSource ?? IngestMediaSource.USER, ingestAutoStart: props.ingestAutoStart ?? false, + renditions: renditions ?? [], + selectedRendition: selectedRendition ?? "source", ...props, }; return ( diff --git a/js/app/components/player/props.tsx b/js/app/components/player/props.tsx index 4bc1b6b2..10af3809 100644 --- a/js/app/components/player/props.tsx +++ b/js/app/components/player/props.tsx @@ -1,3 +1,5 @@ +import { Rendition } from "lexicons/types/place/stream/defs"; + export enum IngestMediaSource { USER = "user", DISPLAY = "display", @@ -9,13 +11,11 @@ export type PlayerProps = { src: string; muted: boolean; fullscreen: boolean; - protocol: string; forceProtocol?: string; showControls: boolean; telemetry: boolean; setMuted: (isMuted: boolean) => void; setFullscreen: (isFullscreen: boolean) => void; - setProtocol: (protocol: string) => void; userInteraction: () => void; playerEvent: ( time: string, @@ -33,6 +33,8 @@ export type PlayerProps = { ingestAutoStart?: boolean; avSyncTest?: boolean; offline: boolean; + renditions: Rendition[]; + selectedRendition: string; }; export type PlayerEvent = { diff --git a/js/app/components/player/provider.tsx b/js/app/components/player/provider.tsx index 32b972ae..08390729 100644 --- a/js/app/components/player/provider.tsx +++ b/js/app/components/player/provider.tsx @@ -37,8 +37,11 @@ export function PlayerContextInitializer( if (props.playerId) { newPlayerAction.payload.playerId = props.playerId; } - setPlayerId(newPlayerAction.payload.playerId); + if (props.forceProtocol) { + newPlayerAction.payload.forceProtocol = props.forceProtocol; + } dispatch(newPlayerAction); + setPlayerId(newPlayerAction.payload.playerId); }, []); if (!playerId) { return <>; diff --git a/js/app/components/player/shared.tsx b/js/app/components/player/shared.tsx index e66c31a3..a607aeb5 100644 --- a/js/app/components/player/shared.tsx +++ b/js/app/components/player/shared.tsx @@ -15,7 +15,10 @@ const protocolSuffixes = { webrtc: PROTOCOL_WEBRTC, }; -export function srcToUrl(props: PlayerProps): { +export function srcToUrl( + props: PlayerProps, + protocol: string, +): { url: string; protocol: string; } { @@ -34,20 +37,24 @@ export function srcToUrl(props: PlayerProps): { } } let outUrl; - if (props.protocol === PROTOCOL_HLS) { - outUrl = `${url}/api/playback/${props.src}/hls/stream.m3u8`; - } else if (props.protocol === PROTOCOL_PROGRESSIVE_MP4) { + if (protocol === PROTOCOL_HLS) { + if (props.selectedRendition === "auto") { + outUrl = `${url}/api/playback/${props.src}/hls/index.m3u8`; + } else { + outUrl = `${url}/api/playback/${props.src}/hls/index.m3u8?rendition=${props.selectedRendition}`; + } + } else if (protocol === PROTOCOL_PROGRESSIVE_MP4) { outUrl = `${url}/api/playback/${props.src}/stream.mp4`; - } else if (props.protocol === PROTOCOL_PROGRESSIVE_WEBM) { + } else if (protocol === PROTOCOL_PROGRESSIVE_WEBM) { outUrl = `${url}/api/playback/${props.src}/stream.webm`; - } else if (props.protocol === PROTOCOL_WEBRTC) { - outUrl = `${url}/api/playback/${props.src}/webrtc`; + } else if (protocol === PROTOCOL_WEBRTC) { + outUrl = `${url}/api/playback/${props.src}/webrtc?rendition=${props.selectedRendition}`; } else { - throw new Error(`unknown playback protocol: ${props.protocol}`); + throw new Error(`unknown playback protocol: ${protocol}`); } return { - protocol: props.protocol, + protocol: protocol, url: outUrl, }; - }, [props.src, props.protocol, url]); + }, [props.src, protocol, url]); } diff --git a/js/app/components/player/use-webrtc.tsx b/js/app/components/player/use-webrtc.tsx index f238b47f..ef18aab0 100644 --- a/js/app/components/player/use-webrtc.tsx +++ b/js/app/components/player/use-webrtc.tsx @@ -126,7 +126,7 @@ export async function negotiateConnectionWithClientOffer( * This specifies how the client should communicate, * and what kind of media client and server have negotiated to exchange. */ - let response = await postSDPOffer(endpoint, ofr.sdp, bearerToken); + let response = await postSDPOffer(`${endpoint}`, ofr.sdp, bearerToken); if (response.status === 201) { let answerSDP = await response.text(); if ((peerConnection.connectionState as string) === "closed") { diff --git a/js/app/components/player/video-retry.tsx b/js/app/components/player/video-retry.tsx index a556af26..dc2fe8d3 100644 --- a/js/app/components/player/video-retry.tsx +++ b/js/app/components/player/video-retry.tsx @@ -1,5 +1,7 @@ import React, { useEffect, useState } from "react"; import { PlayerProps, PlayerStatus } from "./props"; +import { usePlayerSelectedRendition } from "features/player/playerSlice"; +import { useAppSelector } from "store/hooks"; export default function VideoRetry( props: PlayerProps & { children: React.ReactNode }, @@ -7,6 +9,7 @@ export default function VideoRetry( const [resetTime, setResetTime] = useState(Date.now()); const [retryCount, setRetryCount] = useState(0); const isPlaying = props.status === PlayerStatus.PLAYING; + const selectedRendition = useAppSelector(usePlayerSelectedRendition()); useEffect(() => { if (isPlaying) { @@ -30,5 +33,9 @@ export default function VideoRetry( return () => clearTimeout(handle); }, [isPlaying, resetTime, retryCount]); - return {props.children}; + return ( + + {props.children} + + ); } diff --git a/js/app/components/player/video.native.tsx b/js/app/components/player/video.native.tsx index a60984b7..3d67529e 100644 --- a/js/app/components/player/video.native.tsx +++ b/js/app/components/player/video.native.tsx @@ -6,6 +6,8 @@ import { PlayerProps, PlayerStatus, PROTOCOL_WEBRTC } from "./props"; import { srcToUrl } from "./shared"; import useWebRTC from "./use-webrtc"; import { MediaStream } from "react-native-webrtc"; +import { usePlayerProtocol } from "features/player/playerSlice"; +import { useAppSelector } from "store/hooks"; // export function Player() { // return ; @@ -14,10 +16,11 @@ import { MediaStream } from "react-native-webrtc"; export default function NativeVideo( props: PlayerProps & { videoRef: React.RefObject }, ) { - if (props.protocol === PROTOCOL_WEBRTC) { + const protocol = useAppSelector(usePlayerProtocol()); + if (protocol === PROTOCOL_WEBRTC) { return ; } - const { url } = srcToUrl(props); + const { url } = srcToUrl(props, protocol); useEffect(() => { return () => { props.setStatus(PlayerStatus.START); @@ -86,7 +89,7 @@ export default function NativeVideo( } export function NativeWHEP(props: PlayerProps) { - const { url } = srcToUrl(props); + const { url } = srcToUrl(props, PROTOCOL_WEBRTC); const [stream, stuck] = useWebRTC(url); useEffect(() => { if (stuck) { diff --git a/js/app/components/player/video.tsx b/js/app/components/player/video.tsx index 9db26042..1219113f 100644 --- a/js/app/components/player/video.tsx +++ b/js/app/components/player/video.tsx @@ -1,6 +1,6 @@ import streamKey from "components/live-dashboard/stream-key"; import { selectStoredKey } from "features/bluesky/blueskySlice"; -import { usePlayer } from "features/player/playerSlice"; +import { usePlayer, usePlayerProtocol } from "features/player/playerSlice"; import Hls from "hls.js"; import useStreamplaceNode from "hooks/useStreamplaceNode"; import { @@ -32,7 +32,8 @@ type VideoProps = PlayerProps & { url: string }; export default function WebVideo( props: PlayerProps & { videoRef: RefObject }, ) { - const { url, protocol } = srcToUrl(props); + const inProto = useAppSelector(usePlayerProtocol()); + const { url, protocol } = srcToUrl(props, inProto); useEffect(() => { if (props.playTime == 0) { return; @@ -53,7 +54,7 @@ export default function WebVideo( } else if (protocol === PROTOCOL_WEBRTC) { return ; } else { - throw new Error(`unknown playback protocol ${props.protocol}`); + throw new Error(`unknown playback protocol ${inProto}`); } } @@ -192,7 +193,8 @@ export function HLSPlayer( return; } if (Hls.isSupported()) { - var hls = new Hls(); + // workaround for not having quite the right number of audio frames :( + var hls = new Hls({ maxAudioFramesDrift: 20 }); hls.loadSource(props.url); try { hls.attachMedia(videoRef.current); diff --git a/js/app/features/player/playerSlice.tsx b/js/app/features/player/playerSlice.tsx index fefb0e34..75714ba9 100644 --- a/js/app/features/player/playerSlice.tsx +++ b/js/app/features/player/playerSlice.tsx @@ -1,5 +1,6 @@ import { AppBskyFeedDefs, AppBskyFeedPost } from "@atproto/api"; import { createAction } from "@reduxjs/toolkit"; +import { PROTOCOL_HLS, PROTOCOL_WEBRTC } from "components/player/props"; import { StreamplaceState } from "features/streamplace/streamplaceSlice"; import { uuidv7 } from "hooks/uuid"; import { @@ -9,8 +10,12 @@ import { import { createContext, useContext } from "react"; import { createAppSlice } from "../../hooks/createSlice"; import { Record as ChatMessageRecord } from "../../lexicons/types/place/stream/chat/message"; -import { BlockView, isBlockView } from "../../lexicons/types/place/stream/defs"; - +import { + BlockView, + isBlockView, + isRenditions, + Rendition, +} from "../../lexicons/types/place/stream/defs"; import { isLivestreamView, isViewerCount, @@ -62,6 +67,9 @@ export interface PlayerState { chatList: MessageViewHydrated[]; livestream: LivestreamViewHydrated | null; segment: Segment.Record | null; + renditions: Rendition[]; + selectedRendition: string | null; + protocol: string; } export interface PlayersState { @@ -72,7 +80,7 @@ const initialState: PlayersState = {}; export const newPlayer = createAction("player/newPlayer", function prepare() { return { - payload: { playerId: uuidv7() }, + payload: { playerId: uuidv7(), forceProtocol: PROTOCOL_WEBRTC }, }; }); @@ -134,18 +142,27 @@ export const playerSlice = createAppSlice({ initialState, extraReducers: (builder) => { - builder.addCase(newPlayer, (state, action) => { - state[action.payload.playerId] = { - ingestStarted: null, - ingestStarting: false, - ingestConnectionState: null, - viewers: null, - chat: {}, - chatList: [], - livestream: null, - segment: null, - }; - }); + builder.addCase( + newPlayer, + ( + state, + action: { payload: { playerId: string; forceProtocol: string } }, + ) => { + state[action.payload.playerId] = { + ingestStarted: null, + ingestStarting: false, + ingestConnectionState: null, + viewers: null, + protocol: action.payload.forceProtocol ?? PROTOCOL_WEBRTC, + chat: {}, + chatList: [], + livestream: null, + segment: null, + renditions: [], + selectedRendition: "source", + }; + }, + ); }, reducers: (create) => { @@ -241,6 +258,14 @@ export const playerSlice = createAppSlice({ [block], ), }; + } else if (isRenditions(message)) { + state = { + ...state, + [action.payload.playerId]: { + ...state[action.payload.playerId], + renditions: message.renditions, + }, + }; } } return state; @@ -377,6 +402,50 @@ export const playerSlice = createAppSlice({ }, }, ), + + setSelectedRendition: create.reducer( + ( + state, + action: { + payload: { playerId: string; rendition: string }; + type: string; + }, + ) => { + return { + ...state, + [action.payload.playerId]: { + ...state[action.payload.playerId], + selectedRendition: action.payload.rendition, + }, + }; + }, + ), + + setProtocol: create.reducer( + ( + state, + action: { + payload: { playerId: string; protocol: string }; + type: string; + }, + ) => { + const newPlayer = { + ...state[action.payload.playerId], + protocol: action.payload.protocol, + }; + if (action.payload.protocol === PROTOCOL_HLS) { + newPlayer.selectedRendition = "auto"; + } else { + if (newPlayer.selectedRendition === "auto") { + newPlayer.selectedRendition = "source"; + } + } + return { + ...state, + [action.payload.playerId]: newPlayer, + }; + }, + ), }; }, @@ -393,6 +462,15 @@ export const playerSlice = createAppSlice({ selectSegment: (state, playerId: string) => { return state[playerId].segment; }, + selectRenditions: (state, playerId: string) => { + return state[playerId].renditions; + }, + selectSelectedRendition: (state, playerId: string) => { + return state[playerId].selectedRendition; + }, + selectProtocol: (state, playerId: string) => { + return state[playerId].protocol; + }, }, }); @@ -418,6 +496,10 @@ export const usePlayerActions = () => { playerSlice.actions.pollSegment({ playerId, user }), handleWebSocketMessages: (messages: any[]) => playerSlice.actions.handleWebSocketMessages({ playerId, messages }), + setSelectedRendition: (rendition: string) => + playerSlice.actions.setSelectedRendition({ playerId, rendition }), + setProtocol: (protocol: string) => + playerSlice.actions.setProtocol({ playerId, protocol }), }; }; @@ -448,3 +530,21 @@ export const usePlayerSegment = (): ((state: { const playerId = usePlayerId(); return (state) => state.player[playerId].segment; }; +export const usePlayerRenditions = (): ((state: { + player: PlayersState; +}) => Rendition[]) => { + const playerId = usePlayerId(); + return (state) => state.player[playerId].renditions; +}; +export const usePlayerSelectedRendition = (): ((state: { + player: PlayersState; +}) => string | null) => { + const playerId = usePlayerId(); + return (state) => state.player[playerId].selectedRendition; +}; +export const usePlayerProtocol = (): ((state: { + player: PlayersState; +}) => string) => { + const playerId = usePlayerId(); + return (state) => state.player[playerId].protocol; +}; diff --git a/js/desktop/package.json b/js/desktop/package.json index aae38f3f..066add14 100644 --- a/js/desktop/package.json +++ b/js/desktop/package.json @@ -5,7 +5,7 @@ "description": "Streamplace Desktop Application", "main": ".webpack/main", "scripts": { - "start": "electron-forge start \"$@\" | cat", + "start": "PORT=38082 electron-forge start \"$@\" | cat", "start-with-node": "electron-forge start \"$@\" | cat", "package": "electron-forge package", "make": "electron-forge make", diff --git a/lexicons/place/stream/defs.json b/lexicons/place/stream/defs.json index 77cfe1a5..bd218aad 100644 --- a/lexicons/place/stream/defs.json +++ b/lexicons/place/stream/defs.json @@ -15,6 +15,23 @@ "record": { "type": "ref", "ref": "app.bsky.graph.block" }, "indexedAt": { "type": "string", "format": "datetime" } } + }, + "renditions": { + "type": "object", + "required": ["renditions"], + "properties": { + "renditions": { + "type": "array", + "items": { "type": "ref", "ref": "#rendition" } + } + } + }, + "rendition": { + "type": "object", + "required": ["name"], + "properties": { + "name": { "type": "string" } + } } } } diff --git a/lexicons/place/stream/livestream.json b/lexicons/place/stream/livestream.json index d0ec9902..6048495a 100644 --- a/lexicons/place/stream/livestream.json +++ b/lexicons/place/stream/livestream.json @@ -65,6 +65,8 @@ "#livestreamView", "#viewerCount", "place.stream.defs#blockView", + "place.stream.defs#renditions", + "place.stream.defs#rendition", "place.stream.chat.defs#messageView" ] } diff --git a/lexicons/place/stream/segment.json b/lexicons/place/stream/segment.json index 974d79bb..3e2f991c 100644 --- a/lexicons/place/stream/segment.json +++ b/lexicons/place/stream/segment.json @@ -23,6 +23,10 @@ "format": "datetime", "description": "When this segment started" }, + "duration": { + "type": "integer", + "description": "The duration of the segment in nanoseconds" + }, "creator": { "type": "string", "format": "did" @@ -59,7 +63,19 @@ "properties": { "codec": { "type": "string", "enum": ["h264"] }, "width": { "type": "integer" }, - "height": { "type": "integer" } + "height": { "type": "integer" }, + "framerate": { + "type": "ref", + "ref": "#framerate" + } + } + }, + "framerate": { + "type": "object", + "required": ["num", "den"], + "properties": { + "num": { "type": "integer" }, + "den": { "type": "integer" } } } } diff --git a/pkg/api/api.go b/pkg/api/api.go index 210cee45..d940244e 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -28,6 +28,7 @@ import ( "stream.place/streamplace/pkg/bus" "stream.place/streamplace/pkg/config" "stream.place/streamplace/pkg/crypto/signers/eip712" + "stream.place/streamplace/pkg/director" apierrors "stream.place/streamplace/pkg/errors" "stream.place/streamplace/pkg/linking" "stream.place/streamplace/pkg/log" @@ -35,6 +36,7 @@ import ( "stream.place/streamplace/pkg/mist/mistconfig" "stream.place/streamplace/pkg/model" "stream.place/streamplace/pkg/notifications" + "stream.place/streamplace/pkg/renditions" "stream.place/streamplace/pkg/spmetrics" "stream.place/streamplace/pkg/streamplace" ) @@ -49,12 +51,13 @@ type StreamplaceAPI struct { MediaManager *media.MediaManager MediaSigner media.MediaSigner // not thread-safe yet - Aliases map[string]string - Bus *bus.Bus - ATSync *atproto.ATProtoSynchronizer + Aliases map[string]string + Bus *bus.Bus + ATSync *atproto.ATProtoSynchronizer + Director *director.Director } -func MakeStreamplaceAPI(cli *config.CLI, mod model.Model, signer *eip712.EIP712Signer, noter notifications.FirebaseNotifier, mm *media.MediaManager, ms media.MediaSigner, bus *bus.Bus, atsync *atproto.ATProtoSynchronizer) (*StreamplaceAPI, error) { +func MakeStreamplaceAPI(cli *config.CLI, mod model.Model, signer *eip712.EIP712Signer, noter notifications.FirebaseNotifier, mm *media.MediaManager, ms media.MediaSigner, bus *bus.Bus, atsync *atproto.ATProtoSynchronizer, d *director.Director) (*StreamplaceAPI, error) { updater, err := PrepareUpdater(cli) if err != nil { return nil, err @@ -69,6 +72,7 @@ func MakeStreamplaceAPI(cli *config.CLI, mod model.Model, signer *eip712.EIP712S Aliases: map[string]string{}, Bus: bus, ATSync: atsync, + Director: d, } a.Mimes, err = updater.GetMimes() if err != nil { @@ -95,6 +99,13 @@ func (fs AppHostingFS) Open(name string) (http.File, error) { return nil, ErrorIndex } +// api/playback/iame.li/webrtc?rendition=source +// api/playback/iame.li/stream.mp4?rendition=source +// api/playback/iame.li/stream.webm?rendition=source +// api/playback/iame.li/hls/index.m3u8 +// api/playback/iame.li/hls/source/stream.m3u8 +// api/playback/iame.li/hls/source/000000000000.ts + func (a *StreamplaceAPI) Handler(ctx context.Context) (http.Handler, error) { router := httprouter.New() apiRouter := httprouter.New() @@ -107,12 +118,11 @@ func (a *StreamplaceAPI) Handler(ctx context.Context) (http.Handler, error) { apiRouter.POST("/api/webrtc/:stream", a.MistProxyHandler(ctx, "/webrtc/%s")) apiRouter.OPTIONS("/api/webrtc/:stream", a.MistProxyHandler(ctx, "/webrtc/%s")) apiRouter.DELETE("/api/webrtc/:stream", a.MistProxyHandler(ctx, "/webrtc/%s")) - apiRouter.GET("/api/hls/:stream/*resource", a.MistProxyHandler(ctx, "/hls/%s")) apiRouter.Handler("POST", "/api/segment", a.HandleSegment(ctx)) apiRouter.HandlerFunc("GET", "/api/healthz", a.HandleHealthz(ctx)) + apiRouter.GET("/api/playback/:user/hls/*file", a.HandleHLSPlayback(ctx)) apiRouter.GET("/api/playback/:user/stream.mp4", a.HandleMP4Playback(ctx)) apiRouter.GET("/api/playback/:user/stream.webm", a.HandleMKVPlayback(ctx)) - apiRouter.GET("/api/playback/:user/hls/:file", a.HandleHLSPlayback(ctx)) // they're, uh, not jpegs. but we used this once and i don't wanna break backwards compatibility apiRouter.GET("/api/playback/:user/stream.jpg", a.HandleThumbnailPlayback(ctx)) // this one is not a lie @@ -736,6 +746,24 @@ func (a *StreamplaceAPI) HandleWebsocket(ctx context.Context) httprouter.Handle return } initialBurst <- spSeg + if a.CLI.LivepeerGatewayURL != "" { + renditions, err := renditions.GenerateRenditions(spSeg) + if err != nil { + log.Error(ctx, "could not generate renditions", "error", err) + return + } + outRs := streamplace.Defs_Renditions{ + LexiconTypeID: "place.stream.defs#renditions", + } + outRs.Renditions = []*streamplace.Defs_Rendition{} + for _, r := range renditions { + outRs.Renditions = append(outRs.Renditions, &streamplace.Defs_Rendition{ + LexiconTypeID: "place.stream.defs#rendition", + Name: r.Name, + }) + } + initialBurst <- outRs + } }() go func() { diff --git a/pkg/api/api_internal.go b/pkg/api/api_internal.go index 7c242669..691d089e 100644 --- a/pkg/api/api_internal.go +++ b/pkg/api/api_internal.go @@ -29,6 +29,7 @@ import ( "stream.place/streamplace/pkg/mist/misttriggers" "stream.place/streamplace/pkg/model" notificationpkg "stream.place/streamplace/pkg/notifications" + "stream.place/streamplace/pkg/renditions" v0 "stream.place/streamplace/pkg/schema/v0" ) @@ -79,6 +80,8 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err router.Handler("GET", "/debug/pprof/heap", pprof.Handler("heap")) router.Handler("GET", "/debug/pprof/threadcreate", pprof.Handler("threadcreate")) router.Handler("GET", "/debug/pprof/block", pprof.Handler("block")) + router.Handler("GET", "/debug/pprof/allocs", pprof.Handler("allocs")) + router.Handler("GET", "/debug/pprof/mutex", pprof.Handler("mutex")) router.POST("/gc", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { runtime.GC() @@ -87,12 +90,17 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err router.Handler("GET", "/metrics", promhttp.Handler()) - router.GET("/playback/:user/concat", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + router.GET("/playback/:user/:rendition/concat", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { user := p.ByName("user") if user == "" { errors.WriteHTTPBadRequest(w, "user required", nil) return } + rendition := p.ByName("rendition") + if rendition == "" { + errors.WriteHTTPBadRequest(w, "rendition required", nil) + return + } user, err := a.NormalizeUser(ctx, user) if err != nil { errors.WriteHTTPBadRequest(w, "invalid user", err) @@ -102,11 +110,11 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err fmt.Fprintf(w, "ffconcat version 1.0\n") // intermittent reports that you need two here to make things work properly? shouldn't matter. for i := 0; i < 2; i += 1 { - fmt.Fprintf(w, "file '%s/playback/%s/latest.mp4'\n", a.CLI.OwnInternalURL(), user) + fmt.Fprintf(w, "file '%s/playback/%s/%s/latest.mp4'\n", a.CLI.OwnInternalURL(), user, rendition) } }) - router.GET("/playback/:user/latest.mp4", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + router.GET("/playback/:user/:rendition/latest.mp4", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { user := p.ByName("user") if user == "" { errors.WriteHTTPBadRequest(w, "user required", nil) @@ -117,13 +125,18 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err errors.WriteHTTPBadRequest(w, "invalid user", err) return } - file := <-a.MediaManager.SubscribeSegment(ctx, user) - base := filepath.Base(file) - w.Header().Set("Location", fmt.Sprintf("%s/playback/%s/segment/%s\n", a.CLI.OwnInternalURL(), user, base)) + rendition := p.ByName("rendition") + if rendition == "" { + errors.WriteHTTPBadRequest(w, "rendition required", nil) + return + } + seg := <-a.MediaManager.SubscribeSegment(ctx, user, rendition) + base := filepath.Base(seg.Filepath) + w.Header().Set("Location", fmt.Sprintf("%s/playback/%s/%s/segment/%s\n", a.CLI.OwnInternalURL(), user, rendition, base)) w.WriteHeader(301) }) - router.GET("/playback/:user/segment/:file", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + router.GET("/playback/:user/:rendition/segment/:file", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { user := p.ByName("user") if user == "" { errors.WriteHTTPBadRequest(w, "user required", nil) @@ -147,12 +160,17 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err http.ServeFile(w, r, fullpath) }) - router.GET("/playback/:user/stream.mkv", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + router.GET("/playback/:user/:rendition/stream.mkv", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { user := p.ByName("user") if user == "" { errors.WriteHTTPBadRequest(w, "user required", nil) return } + rendition := p.ByName("rendition") + if rendition == "" { + errors.WriteHTTPBadRequest(w, "rendition required", nil) + return + } user, err := a.NormalizeUser(ctx, user) if err != nil { errors.WriteHTTPBadRequest(w, "invalid user", err) @@ -160,18 +178,23 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err } w.Header().Set("Content-Type", "video/x-matroska") w.WriteHeader(200) - err = a.MediaManager.SegmentToMKVPlusOpus(ctx, user, w) + err = a.MediaManager.SegmentToMKVPlusOpus(ctx, user, rendition, w) if err != nil { log.Log(ctx, "stream.mkv error", "error", err) } }) - router.GET("/playback/:user/stream.mp4", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + router.GET("/playback/:user/:rendition/stream.mp4", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { user := p.ByName("user") if user == "" { errors.WriteHTTPBadRequest(w, "user required", nil) return } + rendition := p.ByName("rendition") + if rendition == "" { + errors.WriteHTTPBadRequest(w, "rendition required", nil) + return + } user, err := a.NormalizeUser(ctx, user) if err != nil { errors.WriteHTTPBadRequest(w, "invalid user", err) @@ -197,7 +220,7 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err pr, pw := io.Pipe() bufw := bufio.NewWriter(pw) g.Go(func() error { - return a.MediaManager.SegmentToMP4(ctx, user, bufw) + return a.MediaManager.SegmentToMP4(ctx, user, rendition, bufw) }) g.Go(func() error { time.Sleep(time.Duration(delayMS) * time.Millisecond) @@ -207,7 +230,7 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err g.Wait() }) - router.HEAD("/playback/:user/stream.mkv", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + router.HEAD("/playback/:user/:rendition/stream.mkv", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { user := p.ByName("user") if user == "" { errors.WriteHTTPBadRequest(w, "user required", nil) @@ -453,6 +476,55 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err w.WriteHeader(http.StatusNoContent) }) + router.POST("/livepeer-auth-webhook-url", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + var payload struct { + URL string `json:"url"` + } + // urls look like http://127.0.0.1:9999/live/did:plc:dkh4rwafdcda4ko7lewe43ml-uucbv40mdkcfat50/47.mp4 + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + errors.WriteHTTPBadRequest(w, "invalid request body (could not decode)", err) + return + } + parts := strings.Split(payload.URL, "/") + if len(parts) < 5 { + errors.WriteHTTPBadRequest(w, "invalid request body (too few parts)", nil) + return + } + didSession := parts[4] + idParts := strings.Split(didSession, "-") + if len(idParts) != 2 { + errors.WriteHTTPBadRequest(w, "invalid request body (invalid did session)", nil) + return + } + did := idParts[0] + // sessionID := idParts[1] + seg, err := a.Model.LatestSegmentForUser(did) + if err != nil { + errors.WriteHTTPInternalServerError(w, "unable to get latest segment", err) + return + } + spseg, err := seg.ToStreamplaceSegment() + if err != nil { + errors.WriteHTTPInternalServerError(w, "unable to convert segment to streamplace segment", err) + return + } + renditions, err := renditions.GenerateRenditions(spseg) + if err != nil { + errors.WriteHTTPInternalServerError(w, "unable to generate renditions", err) + return + } + out := map[string]any{ + "manifestID": didSession, + "profiles": renditions.ToLivepeerProfiles(), + } + bs, err := json.Marshal(out) + if err != nil { + errors.WriteHTTPInternalServerError(w, "unable to marshal json", err) + return + } + w.Write(bs) + }) + handler := sloghttp.Recovery(router) if log.Level(4) { handler = sloghttp.New(slog.Default())(handler) diff --git a/pkg/api/api_util.go b/pkg/api/api_util.go new file mode 100644 index 00000000..c53bc5b6 --- /dev/null +++ b/pkg/api/api_util.go @@ -0,0 +1,12 @@ +package api + +import "net/http" + +// get rendition from query params, defaulting to "source" +func getRendition(r *http.Request) string { + rendition := r.URL.Query().Get("rendition") + if rendition == "" { + rendition = "source" + } + return rendition +} diff --git a/pkg/api/playback.go b/pkg/api/playback.go index dba37fcf..46c071a3 100644 --- a/pkg/api/playback.go +++ b/pkg/api/playback.go @@ -52,6 +52,7 @@ func (a *StreamplaceAPI) HandleMP4Playback(ctx context.Context) httprouter.Handl errors.WriteHTTPBadRequest(w, "user required", nil) return } + rendition := getRendition(r) user, err := a.NormalizeUser(ctx, user) if err != nil { errors.WriteHTTPBadRequest(w, "invalid user", err) @@ -79,7 +80,7 @@ func (a *StreamplaceAPI) HandleMP4Playback(ctx context.Context) httprouter.Handl pr, pw := io.Pipe() bufw := bufio.NewWriter(pw) g.Go(func() error { - return a.MediaManager.SegmentToMP4(ctx, user, bufw) + return a.MediaManager.SegmentToMP4(ctx, user, rendition, bufw) }) g.Go(func() error { <-ctx.Done() @@ -103,6 +104,7 @@ func (a *StreamplaceAPI) HandleMKVPlayback(ctx context.Context) httprouter.Handl errors.WriteHTTPBadRequest(w, "user required", nil) return } + rendition := getRendition(r) user, err := a.NormalizeUser(ctx, user) if err != nil { errors.WriteHTTPBadRequest(w, "invalid user", err) @@ -130,7 +132,7 @@ func (a *StreamplaceAPI) HandleMKVPlayback(ctx context.Context) httprouter.Handl pr, pw := io.Pipe() bufw := bufio.NewWriter(pw) g.Go(func() error { - return a.MediaManager.SegmentToMKV(ctx, user, bufw) + return a.MediaManager.SegmentToMKV(ctx, user, rendition, bufw) }) g.Go(func() error { <-ctx.Done() @@ -154,6 +156,7 @@ func (a *StreamplaceAPI) HandleWebRTCPlayback(ctx context.Context) httprouter.Ha errors.WriteHTTPBadRequest(w, "user required", nil) return } + rendition := getRendition(r) user, err := a.NormalizeUser(ctx, user) if err != nil { errors.WriteHTTPBadRequest(w, "invalid user", err) @@ -165,7 +168,7 @@ func (a *StreamplaceAPI) HandleWebRTCPlayback(ctx context.Context) httprouter.Ha return } offer := webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: string(body)} - answer, err := a.MediaManager.WebRTCPlayback(ctx, user, &offer) + answer, err := a.MediaManager.WebRTCPlayback(ctx, user, rendition, &offer) if err != nil { errors.WriteHTTPInternalServerError(w, "error playing back", err) return @@ -346,25 +349,26 @@ func (a *StreamplaceAPI) HandleHLSPlayback(ctx context.Context) httprouter.Handl errors.WriteHTTPBadRequest(w, "file required", nil) return } - m3u8, err := a.MediaManager.SegmentToHLSOnce(ctx, user) + m3u8, err := a.Director.GetM3U8(ctx, user) if err != nil { - errors.WriteHTTPInternalServerError(w, "SegmentToHLSOnce failed", nil) + errors.WriteHTTPNotFound(w, "could not get m3u8", err) return } session := r.URL.Query().Get("session") - buf, err := m3u8.GetSegment(file, session) + rendition := r.URL.Query().Get("rendition") + buf, err := m3u8.GetFile(file, session, rendition) if err != nil { errors.WriteHTTPNotFound(w, "segment not found", err) return } if strings.HasSuffix(file, ".m3u8") { - w.Header().Set("Content-Type", "application/x-mpegURL") + w.Header().Set("Content-Type", "application/vnd.apple.mpegurl") } else { if session != "" { spmetrics.SessionSeen(user, session) } - w.Header().Set("Content-Type", "video/MP2T") + w.Header().Set("Content-Type", "video/mp2t") } http.ServeContent(w, r, file, time.Now(), bytes.NewReader(buf)) diff --git a/pkg/atproto/firehose.go b/pkg/atproto/firehose.go index aaeb6c95..24d603a6 100644 --- a/pkg/atproto/firehose.go +++ b/pkg/atproto/firehose.go @@ -39,6 +39,36 @@ type ATProtoSynchronizer struct { } func (atsync *ATProtoSynchronizer) StartFirehose(ctx context.Context) error { + retryCount := 0 + retryWindow := time.Now() + + for { + if ctx.Err() != nil { + return nil + } + err := atsync.StartFirehoseRetry(ctx) + if err != nil { + log.Error(ctx, "firehose error", "err", err) + + // Check if we're within the 1-minute window + now := time.Now() + if now.Sub(retryWindow) > time.Minute { + // Reset the counter if more than a minute has passed + retryCount = 1 + retryWindow = now + } else { + // Increment retry count if within the window + retryCount++ + if retryCount >= 3 { + log.Error(ctx, "firehose failed 3 times within a minute, crashing", "err", err) + return fmt.Errorf("firehose failed 3 times within a minute: %w", err) + } + } + } + } +} + +func (atsync *ATProtoSynchronizer) StartFirehoseRetry(ctx context.Context) error { ctx = log.WithLogValues(ctx, "func", "StartFirehose") ctx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/pkg/atproto/sync.go b/pkg/atproto/sync.go index 20de4159..40282bb9 100644 --- a/pkg/atproto/sync.go +++ b/pkg/atproto/sync.go @@ -55,7 +55,7 @@ func (atsync *ATProtoSynchronizer) handleCreateUpdate(ctx context.Context, userD // someone we don't know about return nil } - log.Warn(ctx, "creating block", "userDID", userDID, "subjectDID", rec.Subject) + log.Debug(ctx, "creating block", "userDID", userDID, "subjectDID", rec.Subject) block := &model.Block{ RKey: rkey.String(), RepoDID: userDID, @@ -86,13 +86,13 @@ func (atsync *ATProtoSynchronizer) handleCreateUpdate(ctx context.Context, userD if err != nil { return fmt.Errorf("failed to sync bluesky repo: %w", err) } - log.Warn(ctx, "streamplace.ChatMessage detected", "message", rec.Text, "repo", repo.Handle) + log.Debug(ctx, "streamplace.ChatMessage detected", "message", rec.Text, "repo", repo.Handle) block, err := atsync.Model.GetUserBlock(ctx, streamerRepo.DID, userDID) if err != nil { return fmt.Errorf("failed to get user block: %w", err) } if block != nil { - log.Warn(ctx, "excluding message from blocked user", "userDID", userDID, "subjectDID", streamerRepo.DID) + log.Debug(ctx, "excluding message from blocked user", "userDID", userDID, "subjectDID", streamerRepo.DID) return nil } mcm := &model.ChatMessage{ diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index 4065cfad..e2fb51e3 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -1,7 +1,6 @@ package cmd import ( - "bytes" "context" "crypto" "flag" @@ -13,15 +12,14 @@ import ( "runtime/pprof" "strconv" "syscall" - "time" "golang.org/x/term" "stream.place/streamplace/pkg/aqhttp" - "stream.place/streamplace/pkg/aqtime" "stream.place/streamplace/pkg/atproto" "stream.place/streamplace/pkg/bus" "stream.place/streamplace/pkg/crypto/signers" "stream.place/streamplace/pkg/crypto/signers/eip712" + "stream.place/streamplace/pkg/director" "stream.place/streamplace/pkg/log" "stream.place/streamplace/pkg/media" "stream.place/streamplace/pkg/notifications" @@ -29,7 +27,6 @@ import ( "stream.place/streamplace/pkg/replication/boring" v0 "stream.place/streamplace/pkg/schema/v0" "stream.place/streamplace/pkg/spmetrics" - "stream.place/streamplace/pkg/thumbnail" "github.com/ThalesGroup/crypto11" _ "github.com/go-gst/go-glib/glib" @@ -131,6 +128,7 @@ func start(build *config.BuildFlags, platformJobs []jobFunc) error { fs.StringVar(&cli.AppBundleID, "app-bundle-id", "", "bundle id of an app that we facilitate oauth login for") fs.StringVar(&cli.StreamerName, "streamer-name", "", "name of the person streaming from this streamplace node") fs.StringVar(&cli.FrontendProxy, "dev-frontend-proxy", "", "(FOR DEVELOPMENT ONLY) proxy frontend requests to this address instead of using the bundled frontend") + fs.StringVar(&cli.LivepeerGatewayURL, "livepeer-gateway-url", "", "URL of the Livepeer Gateway to use for transcoding") fs.BoolVar(&cli.WideOpen, "wide-open", false, "allow ALL streams to be uploaded to this node (not recommended for production)") cli.StringSliceFlag(fs, &cli.AllowedStreams, "allowed-streams", "", "if set, only allow these addresses or atproto DIDs to upload to this node") cli.StringSliceFlag(fs, &cli.Peers, "peers", "", "other streamplace nodes to replicate to") @@ -179,6 +177,7 @@ func start(build *config.BuildFlags, platformJobs []jobFunc) error { if *version { return nil } + spmetrics.Version.WithLabelValues(build.Version).Inc() aqhttp.UserAgent = fmt.Sprintf("streamplace/%s", build.Version) @@ -299,7 +298,9 @@ func start(build *config.BuildFlags, platformJobs []jobFunc) error { return err } - a, err := api.MakeStreamplaceAPI(&cli, mod, eip712signer, noter, mm, ms, b, atsync) + d := director.NewDirector(mm, mod, &cli, b) + + a, err := api.MakeStreamplaceAPI(&cli, mod, eip712signer, noter, mm, ms, b, atsync, d) if err != nil { return err } @@ -339,66 +340,7 @@ func start(build *config.BuildFlags, platformJobs []jobFunc) error { }) group.Go(func() error { - newSeg := mm.NewSegment() - for { - select { - case <-ctx.Done(): - return nil - case not := <-newSeg: - err := mod.CreateSegment(not.Segment) - if err != nil { - log.Error(ctx, "could not add segment to database", "error", err) - } - spseg, err := not.Segment.ToStreamplaceSegment() - if err != nil { - log.Error(ctx, "could not convert segment to streamplace segment", "error", err) - continue - } - b.Publish(spseg.Creator, spseg) - go func() { - err := func() error { - lock := thumbnail.GetThumbnailLock(not.Segment.RepoDID) - locked := lock.TryLock() - if !locked { - // we're already generating a thumbnail for this user, skip - return nil - } - defer lock.Unlock() - oldThumb, err := mod.LatestThumbnailForUser(not.Segment.RepoDID) - if err != nil { - return err - } - if oldThumb != nil && not.Segment.StartTime.Sub(oldThumb.Segment.StartTime) < time.Minute { - // we have a thumbnail <60sec old, skip generating a new one - return nil - } - r := bytes.NewReader(not.Data) - aqt := aqtime.FromTime(not.Segment.StartTime) - fd, err := cli.SegmentFileCreate(not.Segment.RepoDID, aqt, "jpg") - if err != nil { - return err - } - defer fd.Close() - err = mm.Thumbnail(ctx, r, fd) - if err != nil { - return err - } - thumb := &model.Thumbnail{ - Format: "jpg", - SegmentID: not.Segment.ID, - } - err = mod.CreateThumbnail(thumb) - if err != nil { - return err - } - return nil - }() - if err != nil { - log.Error(ctx, "could not create thumbnail", "error", err) - } - }() - } - } + return d.Start(ctx) }) if cli.TestStream { diff --git a/pkg/config/config.go b/pkg/config/config.go index 6d42b954..4a0fbe39 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -84,6 +84,7 @@ type CLI struct { NoFirehose bool PrintChat bool Color string + LivepeerGatewayURL string dataDirFlags []*string } diff --git a/pkg/director/director.go b/pkg/director/director.go new file mode 100644 index 00000000..8055b799 --- /dev/null +++ b/pkg/director/director.go @@ -0,0 +1,95 @@ +package director + +import ( + "context" + "fmt" + "sync" + + "golang.org/x/sync/errgroup" + "stream.place/streamplace/pkg/bus" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/media" + "stream.place/streamplace/pkg/model" +) + +// director is responsible for managing the lifecycle of a stream, making business +// logic decisions about when to do things like +// - size of the in-memory segment cache +// - transcoding +// - thumbnail generation + +type Director struct { + mm *media.MediaManager + mod model.Model + cli *config.CLI + bus *bus.Bus + streamSessions map[string]*StreamSession + streamSessionsMu sync.Mutex +} + +func NewDirector(mm *media.MediaManager, mod model.Model, cli *config.CLI, bus *bus.Bus) *Director { + return &Director{ + mm: mm, + mod: mod, + cli: cli, + bus: bus, + streamSessions: make(map[string]*StreamSession), + streamSessionsMu: sync.Mutex{}, + } +} + +func (d *Director) Start(ctx context.Context) error { + newSeg := d.mm.NewSegment() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + g, ctx := errgroup.WithContext(ctx) + for { + select { + case <-ctx.Done(): + cancel() + return g.Wait() + case not := <-newSeg: + d.streamSessionsMu.Lock() + ss, ok := d.streamSessions[not.Segment.RepoDID] + if !ok { + ss = &StreamSession{ + hls: nil, + lp: nil, + repoDID: not.Segment.RepoDID, + mm: d.mm, + mod: d.mod, + cli: d.cli, + bus: d.bus, + segmentChan: make(chan struct{}), + } + d.streamSessions[not.Segment.RepoDID] = ss + g.Go(func() error { + err := ss.Start(ctx, not) + if err != nil { + log.Error(ctx, "could not start stream session", "error", err) + } + d.streamSessionsMu.Lock() + delete(d.streamSessions, not.Segment.RepoDID) + d.streamSessionsMu.Unlock() + return nil + }) + } + d.streamSessionsMu.Unlock() + err := ss.NewSegment(ctx, not) + if err != nil { + log.Error(ctx, "could not add segment to stream session", "error", err) + } + } + } +} + +func (d *Director) GetM3U8(ctx context.Context, repoDID string) (*media.M3U8, error) { + d.streamSessionsMu.Lock() + defer d.streamSessionsMu.Unlock() + ss, ok := d.streamSessions[repoDID] + if !ok { + return nil, fmt.Errorf("stream session not found") + } + return ss.hls, nil +} diff --git a/pkg/director/stream_session.go b/pkg/director/stream_session.go new file mode 100644 index 00000000..46be869c --- /dev/null +++ b/pkg/director/stream_session.go @@ -0,0 +1,251 @@ +package director + +import ( + "bytes" + "context" + "fmt" + "time" + + "golang.org/x/sync/errgroup" + "stream.place/streamplace/pkg/aqtime" + "stream.place/streamplace/pkg/bus" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/livepeer" + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/media" + "stream.place/streamplace/pkg/media/segchanman" + "stream.place/streamplace/pkg/model" + "stream.place/streamplace/pkg/renditions" + "stream.place/streamplace/pkg/spmetrics" + "stream.place/streamplace/pkg/streamplace" + "stream.place/streamplace/pkg/thumbnail" +) + +type StreamSession struct { + mm *media.MediaManager + mod model.Model + cli *config.CLI + bus *bus.Bus + hls *media.M3U8 + lp *livepeer.LivepeerSession + repoDID string + segmentChan chan struct{} +} + +func (ss *StreamSession) Start(ctx context.Context, not *media.NewSegmentNotification) error { + + sid := livepeer.RandomTrailer(8) + ctx = log.WithLogValues(ctx, "sid", sid) + ctx, cancel := context.WithCancel(ctx) + log.Log(ctx, "starting stream session") + defer cancel() + spseg, err := not.Segment.ToStreamplaceSegment() + if err != nil { + return fmt.Errorf("could not convert segment to streamplace segment: %w", err) + } + var allRenditions renditions.Renditions + + if ss.cli.LivepeerGatewayURL != "" { + allRenditions, err = renditions.GenerateRenditions(spseg) + } else { + allRenditions = []renditions.Rendition{} + } + if err != nil { + return err + } + if spseg.Duration == nil { + return fmt.Errorf("segment duration is required to calculate bitrate") + } + dur := time.Duration(*spseg.Duration) + byteLen := len(not.Data) + bitrate := int(float64(byteLen) / dur.Seconds() * 8) + sourceRendition := renditions.Rendition{ + Name: "source", + Bitrate: bitrate, + Width: spseg.Video[0].Width, + Height: spseg.Video[0].Height, + } + allRenditions = append([]renditions.Rendition{sourceRendition}, allRenditions...) + ss.hls = media.NewM3U8(allRenditions) + + g, ctx := errgroup.WithContext(ctx) + + for _, r := range allRenditions { + g.Go(func() error { + for { + if ctx.Err() != nil { + return nil + } + err := ss.mm.ToHLS(ctx, spseg.Creator, r.Name, ss.hls) + if ctx.Err() != nil { + return nil + } + log.Warn(ctx, "hls failed, retrying in 5 seconds", "error", err) + time.Sleep(time.Second * 5) + } + }) + } + + for { + select { + case <-ss.segmentChan: + // reset timer + case <-ctx.Done(): + return g.Wait() + // case <-time.After(time.Minute * 1): + case <-time.After(time.Second * 60): + log.Log(ctx, "no new segments for 1 minute, shutting down") + cancel() + } + } +} + +func (ss *StreamSession) NewSegment(ctx context.Context, not *media.NewSegmentNotification) error { + if ctx.Err() != nil { + return nil + } + ss.segmentChan <- struct{}{} + aqt := aqtime.FromTime(not.Segment.StartTime) + ctx = log.WithLogValues(ctx, "segID", not.Segment.ID, "repoDID", not.Segment.RepoDID, "timestamp", aqt.FileSafeString()) + err := ss.mod.CreateSegment(not.Segment) + if err != nil { + return fmt.Errorf("could not add segment to database: %w", err) + } + spseg, err := not.Segment.ToStreamplaceSegment() + if err != nil { + return fmt.Errorf("could not convert segment to streamplace segment: %w", err) + } + + ss.bus.Publish(spseg.Creator, spseg) + + go func() { + err := ss.Thumbnail(ctx, spseg.Creator, not) + if err != nil { + log.Error(ctx, "could not create thumbnail", "error", err) + } + }() + + if ss.cli.LivepeerGatewayURL != "" { + go func() { + start := time.Now() + err := ss.Transcode(ctx, spseg, not.Data) + took := time.Since(start) + if err != nil { + log.Error(ctx, "could not transcode", "error", err, "took", took) + } else { + log.Log(ctx, "transcoded segment", "took", took) + } + }() + } + + return nil +} + +func (ss *StreamSession) Thumbnail(ctx context.Context, repoDID string, not *media.NewSegmentNotification) error { + lock := thumbnail.GetThumbnailLock(not.Segment.RepoDID) + locked := lock.TryLock() + if !locked { + // we're already generating a thumbnail for this user, skip + return nil + } + defer lock.Unlock() + oldThumb, err := ss.mod.LatestThumbnailForUser(not.Segment.RepoDID) + if err != nil { + return err + } + if oldThumb != nil && not.Segment.StartTime.Sub(oldThumb.Segment.StartTime) < time.Minute { + // we have a thumbnail <60sec old, skip generating a new one + return nil + } + r := bytes.NewReader(not.Data) + aqt := aqtime.FromTime(not.Segment.StartTime) + fd, err := ss.cli.SegmentFileCreate(not.Segment.RepoDID, aqt, "png") + if err != nil { + return err + } + defer fd.Close() + err = ss.mm.Thumbnail(ctx, r, fd) + if err != nil { + return err + } + thumb := &model.Thumbnail{ + Format: "png", + SegmentID: not.Segment.ID, + } + err = ss.mod.CreateThumbnail(thumb) + if err != nil { + return err + } + return nil +} + +func (ss *StreamSession) Transcode(ctx context.Context, spseg *streamplace.Segment, data []byte) error { + rs, err := renditions.GenerateRenditions(spseg) + if ss.lp == nil { + var err error + ss.lp, err = livepeer.NewLivepeerSession(ctx, spseg.Creator, ss.cli.LivepeerGatewayURL) + if err != nil { + return err + } + + } + spmetrics.TranscodeAttemptsTotal.Inc() + segs, err := ss.lp.PostSegmentToGateway(ctx, data, spseg) + if err != nil { + spmetrics.TranscodeErrorsTotal.Inc() + return err + } + if len(rs) != len(segs) { + spmetrics.TranscodeErrorsTotal.Inc() + return fmt.Errorf("expected %d renditions, got %d", len(rs), len(segs)) + } + spmetrics.TranscodeSuccessesTotal.Inc() + aqt, err := aqtime.FromString(spseg.StartTime) + if err != nil { + return err + } + for i, seg := range segs { + log.Debug(ctx, "publishing segment", "rendition", rs[i]) + fd, err := ss.cli.SegmentFileCreate(spseg.Creator, aqt, fmt.Sprintf("%s.mp4", rs[i].Name)) + if err != nil { + return err + } + defer fd.Close() + fd.Write(seg) + // go ss.TryAddToHLS(ctx, spseg, rs[i].Name, seg) + go ss.mm.PublishSegment(ctx, spseg.Creator, rs[i].Name, &segchanman.Seg{ + Filepath: fd.Name(), + Data: seg, + }) + } + return nil +} + +// func (ss *StreamSession) TryAddToHLS(ctx context.Context, spseg *streamplace.Segment, rendition string, data []byte) { +// ctx = log.WithLogValues(ctx, "rendition", rendition) +// err := ss.AddToHLS(ctx, spseg, rendition, data) +// if err != nil { +// log.Error(ctx, "could not add to hls", "error", err) +// } +// } + +// func (ss *StreamSession) AddToHLS(ctx context.Context, spseg *streamplace.Segment, rendition string, data []byte) error { +// buf := bytes.Buffer{} +// dur, err := media.MP4ToMPEGTS(ctx, bytes.NewReader(data), &buf) +// if err != nil { +// return err +// } +// newSeg := &streamplace.Segment{ +// LexiconTypeID: "place.stream.segment", +// Id: spseg.Id, +// Creator: spseg.Creator, +// StartTime: spseg.StartTime, +// Duration: &dur, +// Audio: spseg.Audio, +// Video: spseg.Video, +// SigningKey: spseg.SigningKey, +// } +// log.Debug(ctx, "transmuxed to mpegts, adding to hls", "rendition", rendition, "size", buf.Len()) +// ss.hls.NewSegment(newSeg, rendition, buf.Bytes()) +// return nil +// } diff --git a/pkg/gen/gen.go b/pkg/gen/gen.go index b00a571c..8aec6a2e 100644 --- a/pkg/gen/gen.go +++ b/pkg/gen/gen.go @@ -25,6 +25,7 @@ func main() { streamplace.Segment{}, streamplace.Segment_Audio{}, streamplace.Segment_Video{}, + streamplace.Segment_Framerate{}, streamplace.ChatMessage{}, streamplace.RichtextFacet{}, streamplace.ChatProfile{}, diff --git a/pkg/livepeer/livepeer.go b/pkg/livepeer/livepeer.go new file mode 100644 index 00000000..876334de --- /dev/null +++ b/pkg/livepeer/livepeer.go @@ -0,0 +1,111 @@ +package livepeer + +import ( + "bytes" + "context" + "fmt" + "io" + "math/rand" + "mime" + "mime/multipart" + "net/http" + "strings" + "sync" + "time" + + "golang.org/x/net/context/ctxhttp" + "stream.place/streamplace/pkg/aqhttp" + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/streamplace" +) + +type LivepeerSession struct { + SessionID string + Count int + GatewayURL string + SegLock sync.Mutex +} + +// borrowed from catalyst-api +func RandomTrailer(length int) string { + const charset = "abcdefghijklmnopqrstuvwxyz0123456789" + + res := make([]byte, length) + for i := 0; i < length; i++ { + res[i] = charset[rand.Intn(len(charset))] + } + return string(res) +} + +func NewLivepeerSession(ctx context.Context, did string, gatewayURL string) (*LivepeerSession, error) { + sessionID := RandomTrailer(8) + return &LivepeerSession{ + SessionID: fmt.Sprintf("%s-%s", did, sessionID), + Count: 0, + GatewayURL: gatewayURL, + }, nil +} + +func (ls *LivepeerSession) PostSegmentToGateway(ctx context.Context, buf []byte, seg *streamplace.Segment) ([][]byte, error) { + ctx = log.WithLogValues(ctx, "func", "PostSegmentToGateway") + ls.SegLock.Lock() + defer ls.SegLock.Unlock() + ctx, cancel := context.WithTimeout(ctx, time.Minute*5) + defer cancel() + url := fmt.Sprintf("%s/live/%s/%d.mp4", ls.GatewayURL, ls.SessionID, ls.Count) + ls.Count++ + + dur := time.Duration(*seg.Duration) + durationMs := int(dur.Milliseconds()) + log.Debug(ctx, "posting segment to livepeer gateway", "duration_ms", durationMs, "url", url) + + vid := seg.Video[0] + width := int(vid.Width) + height := int(vid.Height) + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(buf)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Accept", "multipart/mixed") + req.Header.Set("Content-Duration", fmt.Sprintf("%d", durationMs)) + req.Header.Set("Content-Resolution", fmt.Sprintf("%dx%d", width, height)) + + resp, err := ctxhttp.Do(ctx, &aqhttp.Client, req) + if err != nil { + return nil, fmt.Errorf("failed to send segment to gateway: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errOut, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("gateway returned non-OK status: %d, %s", resp.StatusCode, string(errOut)) + } + + var out [][]byte + + mediaType, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil { + return nil, fmt.Errorf("failed to parse media type: %w", err) + } + if strings.HasPrefix(mediaType, "multipart/") { + mr := multipart.NewReader(resp.Body, params["boundary"]) + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("failed to get next part: %w", err) + } + bs, err := io.ReadAll(p) + if err != nil { + return nil, fmt.Errorf("failed to read part: %w", err) + } + log.Debug(ctx, "got part back from livepeer gateway", "length", len(bs), "name", p.FileName()) + out = append(out, bs) + } + } + + return out, nil +} diff --git a/pkg/media/bus_handler.go b/pkg/media/bus_handler.go index 7e8ba03f..4b66d3f1 100644 --- a/pkg/media/bus_handler.go +++ b/pkg/media/bus_handler.go @@ -26,7 +26,7 @@ func HandleBusMessagesCustom(ctx context.Context, pipeline *gst.Pipeline, handle } switch msg.Type() { case gst.MessageEOS: // When end-of-stream is received flush the pipeline and stop the main loop - log.Log(ctx, "got gst.MessageEOS, exiting") + log.Debug(ctx, "got gst.MessageEOS, exiting") return case gst.MessageError: // Error messages are always fatal err := msg.ParseError() diff --git a/pkg/media/concat.go b/pkg/media/concat.go index f526fdf8..156479d2 100644 --- a/pkg/media/concat.go +++ b/pkg/media/concat.go @@ -1,26 +1,27 @@ package media import ( + "bytes" "context" "errors" "fmt" "io" - "os" "strings" "sync" "github.com/go-gst/go-gst/gst" "github.com/go-gst/go-gst/gst/app" "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/media/segchanman" ) type ConcatStreamer interface { - SubscribeSegment(ctx context.Context, user string) <-chan string - UnsubscribeSegment(ctx context.Context, user string, ch <-chan string) + SubscribeSegment(ctx context.Context, user string, rendition string) <-chan *segchanman.Seg + UnsubscribeSegment(ctx context.Context, user string, rendition string, ch <-chan *segchanman.Seg) } // This function remains in scope for the duration of a single users' playback -func ConcatStream(ctx context.Context, pipeline *gst.Pipeline, user string, streamer ConcatStreamer) (*gst.Element, <-chan struct{}, error) { +func ConcatStream(ctx context.Context, pipeline *gst.Pipeline, user string, rendition string, streamer ConcatStreamer) (*gst.Element, <-chan struct{}, error) { ctx = log.WithLogValues(ctx, "func", "ConcatStream") ctx, cancel := context.WithCancel(ctx) @@ -35,9 +36,6 @@ func ConcatStream(ctx context.Context, pipeline *gst.Pipeline, user string, stre if err != nil { return nil, nil, fmt.Errorf("failed to add input multiqueue to pipeline: %w", err) } - for _, tmpl := range inputQueue.GetPadTemplates() { - log.Warn(ctx, "pad template", "name", tmpl.GetName(), "direction", tmpl.Direction()) - } inputQueuePadVideoSink := inputQueue.GetRequestPad("sink_%u") if inputQueuePadVideoSink == nil { return nil, nil, fmt.Errorf("failed to get input queue video sink pad") @@ -125,19 +123,19 @@ func ConcatStream(ctx context.Context, pipeline *gst.Pipeline, user string, stre // this goroutine will read all the files from the segment queue and buffer // them in a pipe so that we don't miss any in between iterations of the output - allFiles := make(chan string, 1024) + allFiles := make(chan []byte, 1024) go func() { for { - ch := streamer.SubscribeSegment(ctx, user) + ch := streamer.SubscribeSegment(ctx, user, rendition) select { case <-ctx.Done(): - log.Warn(ctx, "exiting segment reader") - streamer.UnsubscribeSegment(ctx, user, ch) + log.Debug(ctx, "exiting segment reader") + streamer.UnsubscribeSegment(ctx, user, rendition, ch) return case file := <-ch: - log.Debug(ctx, "got segment", "file", file) - allFiles <- file - if file == "" { + log.Debug(ctx, "got segment", "file", file.Filepath) + allFiles <- file.Data + if len(file.Data) == 0 { log.Warn(ctx, "no more segments") return } @@ -156,23 +154,15 @@ func ConcatStream(ctx context.Context, pipeline *gst.Pipeline, user string, stre pr.Close() pw.Close() return - case fullpath := <-allFiles: - if fullpath == "" { + case bs := <-allFiles: + if len(bs) == 0 { log.Warn(ctx, "no more segments") cancel() return } - f, err := os.Open(fullpath) - log.Debug(ctx, "opening segment file", "file", fullpath) - if err != nil { - log.Debug(ctx, "failed to open segment file", "error", err, "file", fullpath) - cancel() - return - } - defer f.Close() - _, err = io.Copy(pw, f) + _, err = io.Copy(pw, bytes.NewReader(bs)) if err != nil { - log.Error(ctx, "failed to copy segment file", "error", err, "file", fullpath) + log.Error(ctx, "failed to copy segment file", "error", err) cancel() return } @@ -304,7 +294,7 @@ func ConcatStream(ctx context.Context, pipeline *gst.Pipeline, user string, stre done() return } else { - log.Error(ctx, "failed to read data", "error", err) + log.Debug(ctx, "failed to read data, ending stream", "error", err) cancel() return } diff --git a/pkg/media/ffmpeg_concat.go b/pkg/media/ffmpeg_concat.go new file mode 100644 index 00000000..0f95632e --- /dev/null +++ b/pkg/media/ffmpeg_concat.go @@ -0,0 +1,95 @@ +package media + +import ( + "context" + "fmt" + "io" + + "github.com/livepeer/lpms/ffmpeg" + "golang.org/x/sync/errgroup" +) + +func (mm *MediaManager) SegmentToMKV(ctx context.Context, user string, rendition string, w io.Writer) error { + muxer := ffmpeg.ComponentOptions{ + Name: "matroska", + } + return mm.SegmentToStream(ctx, user, rendition, muxer, w) +} + +func (mm *MediaManager) SegmentToMKVPlusOpus(ctx context.Context, user string, rendition string, w io.Writer) error { + muxer := ffmpeg.ComponentOptions{ + Name: "matroska", + } + pr, pw := io.Pipe() + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + return mm.SegmentToStream(ctx, user, rendition, muxer, pw) + }) + g.Go(func() error { + return AddOpusToMKV(ctx, pr, w) + }) + return g.Wait() +} + +func (mm *MediaManager) SegmentToMP4(ctx context.Context, user string, rendition string, w io.Writer) error { + muxer := ffmpeg.ComponentOptions{ + Name: "mp4", + Opts: map[string]string{ + "movflags": "frag_keyframe+empty_moov", + }, + } + return mm.SegmentToStream(ctx, user, rendition, muxer, w) +} + +func (mm *MediaManager) SegmentToStream(ctx context.Context, user string, rendition string, muxer ffmpeg.ComponentOptions, w io.Writer) error { + tc := ffmpeg.NewTranscoder() + defer tc.StopTranscoder() + ourl, or, odone, err := mm.HTTPPipe() + if err != nil { + return err + } + defer odone() + iname := fmt.Sprintf("%s/playback/%s/%s/concat", mm.cli.OwnInternalURL(), user, rendition) + in := &ffmpeg.TranscodeOptionsIn{ + Fname: iname, + Transmuxing: true, + Profile: ffmpeg.VideoProfile{}, + Loop: -1, + Demuxer: ffmpeg.ComponentOptions{ + Name: "concat", + Opts: map[string]string{ + "safe": "0", + "protocol_whitelist": "file,http,https,tcp,tls", + }, + }, + } + out := []ffmpeg.TranscodeOptions{ + { + Oname: ourl, + VideoEncoder: ffmpeg.ComponentOptions{ + Name: "copy", + }, + AudioEncoder: ffmpeg.ComponentOptions{ + Name: "copy", + }, + Profile: ffmpeg.VideoProfile{Format: ffmpeg.FormatNone}, + Muxer: muxer, + }, + } + g, _ := errgroup.WithContext(ctx) + g.Go(func() error { + <-ctx.Done() + or.Close() + return nil + }) + g.Go(func() error { + _, err := tc.Transcode(in, out) + tc.StopTranscoder() + return err + }) + g.Go(func() error { + _, err := io.Copy(w, or) + return err + }) + return g.Wait() +} diff --git a/pkg/media/gstreamer.go b/pkg/media/gstreamer.go index ee5624bb..954da5d4 100644 --- a/pkg/media/gstreamer.go +++ b/pkg/media/gstreamer.go @@ -15,12 +15,10 @@ import ( "github.com/go-gst/go-glib/glib" "github.com/go-gst/go-gst/gst" "github.com/go-gst/go-gst/gst/app" - "github.com/google/uuid" "github.com/skip2/go-qrcode" "golang.org/x/sync/errgroup" "stream.place/streamplace/pkg/aqtime" "stream.place/streamplace/pkg/log" - "stream.place/streamplace/pkg/model" "stream.place/streamplace/test" ) @@ -232,229 +230,6 @@ func SelfTest(ctx context.Context) error { return nil } -// #EXTM3U -// #EXT-X-VERSION:3 -// #EXT-X-MEDIA-SEQUENCE:281 -// #EXT-X-TARGETDURATION:1 - -// #EXTINF:1, -// segment00281.ts -// #EXTINF:1.0049999952316284, -// segment00282.ts -// #EXTINF:1, -// segment00283.ts -// #EXTINF:1.0010000467300415, -// segment00284.ts -// #EXTINF:1, -// segment00285.ts -// #EXT-X-ENDLIST - -func (mm *MediaManager) ToHLS(ctx context.Context, input io.Reader, m3u8 *M3U8) error { - ctx = log.WithLogValues(ctx, "GStreamerFunc", "ToHLS") - - splitmuxsink, err := gst.NewElementWithProperties("splitmuxsink", map[string]any{ - "name": "mux", - "async-finalize": true, - "sink-factory": "appsink", - "muxer-factory": "mpegtsmux", - "max-size-bytes": 1, - }) - if err != nil { - return err - } - - p := splitmuxsink.GetRequestPad("video") - if p == nil { - return fmt.Errorf("failed to get video pad") - } - p = splitmuxsink.GetRequestPad("audio_%u") - if p == nil { - return fmt.Errorf("failed to get audio pad") - } - - pipelineSlice := []string{ - "appsrc name=appsrc ! matroskademux name=demux", - "demux.video_0 ! queue ! h264parse name=videoparse", - "demux.audio_0 ! queue ! opusdec use-inband-fec=true ! audioresample ! fdkaacenc name=audioenc", - } - - pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) - if err != nil { - return fmt.Errorf("error creating ToHLS pipeline: %w", err) - } - - err = pipeline.Add(splitmuxsink) - if err != nil { - return fmt.Errorf("error adding splitmuxsink to ToHLS pipeline: %w", err) - } - - videoparse, err := pipeline.GetElementByName("videoparse") - if err != nil { - return fmt.Errorf("error getting videoparse from ToHLS pipeline: %w", err) - } - err = videoparse.Link(splitmuxsink) - if err != nil { - return fmt.Errorf("error linking videoparse to splitmuxsink: %w", err) - } - - audioenc, err := pipeline.GetElementByName("audioenc") - if err != nil { - return fmt.Errorf("error getting audioenc from ToHLS pipeline: %w", err) - } - err = audioenc.Link(splitmuxsink) - if err != nil { - return fmt.Errorf("error linking audioenc to splitmuxsink: %w", err) - } - - splitmuxsink.Connect("sink-added", func(split, sinkEle *gst.Element) { - vf, err := m3u8.GetNextSegment(ctx) - if err != nil { - panic(err) - } - appsink := app.SinkFromElement(sinkEle) - appsink.SetCallbacks(&app.SinkCallbacks{ - NewSampleFunc: WriterNewSample(ctx, vf.Buf), - EOSFunc: func(sink *app.Sink) { - m3u8.CloseSegment(ctx, vf) - }, - }) - }) - - appsrc, err := pipeline.GetElementByName("appsrc") - if err != nil { - return err - } - - src := app.SrcFromElement(appsrc) - src.SetCallbacks(&app.SourceCallbacks{ - NeedDataFunc: ReaderNeedData(ctx, input), - }) - - onPadAdded := func(element *gst.Element, pad *gst.Pad) { - caps := pad.GetCurrentCaps() - if caps == nil { - fmt.Println("Unable to get pad caps") - return - } - - fmt.Printf("New pad added: %s\n", pad.GetName()) - fmt.Printf("Caps: %s\n", caps.String()) - - structure := caps.GetStructureAt(0) - if structure == nil { - fmt.Println("Unable to get structure from caps") - return - } - - name := structure.Name() - fmt.Printf("Structure Name: %s\n", name) - - if name[:5] == "video" { - // Get some common video properties - widthVal, _ := structure.GetValue("width") - heightVal, _ := structure.GetValue("height") - - width, ok := widthVal.(int) - if ok { - m3u8.Width = uint64(width) - } - height, ok := heightVal.(int) - if ok { - m3u8.Height = uint64(height) - } - // framerate, ok := framerateVal.(string) - // if ok { - // fmt.Printf(" Framerate: %s\n", framerate) - // } - // pixelAspectRatio, ok := pixelAspectRatioVal.(string) - // if ok { - // fmt.Printf(" Pixel Aspect Ratio: %s\n", pixelAspectRatio) - // } - // if codecVal != nil { - // fmt.Printf(" Has codec data: true\n") - // } - } - - // if name[:5] == "audio" { - // // Get some common audio properties - // rateVal, _ := structure.GetValue("rate") - // channelsVal, _ := structure.GetValue("channels") - // formatVal, err := structure.GetValue("format") - // mpegversion, _ := structure.GetValue("mpegversion") - // log.Log(ctx, "format error", "error", err, "mpegversion", mpegversion) - - // fmt.Printf(" Structure: %s\n", structure.String()) - // rate, ok := rateVal.(int) - // if ok { - // fmt.Printf(" Rate: %d\n", rate) - // } - // channels, ok := channelsVal.(int) - // if ok { - // fmt.Printf(" Channels: %d\n", channels) - // } - // format, ok := formatVal.(int) - // if ok { - // fmt.Printf(" Format: %d\n", format) - // } - - // } - } - - demux, err := pipeline.GetElementByName("demux") - if err != nil { - return err - } - demux.Connect("pad-added", onPadAdded) - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - go func() { - HandleBusMessagesCustom(ctx, pipeline, func(msg *gst.Message) { - switch msg.Type() { - case gst.MessageElement: - structure := msg.GetStructure() - name := structure.Name() - if name == "splitmuxsink-fragment-opened" { - runningTime, err := structure.GetValue("running-time") - if err != nil { - log.Warn(ctx, "splitmuxsink-fragment-opened error", "error", err) - cancel() - } - runningTimeInt, ok := runningTime.(uint64) - if !ok { - log.Warn(ctx, "splitmuxsink-fragment-opened not a uint64") - cancel() - } - m3u8.FragmentOpened(ctx, runningTimeInt) - } - if name == "splitmuxsink-fragment-closed" { - runningTime, err := structure.GetValue("running-time") - if err != nil { - log.Warn(ctx, "splitmuxsink-fragment-closed error", "error", err) - cancel() - } - runningTimeInt, ok := runningTime.(uint64) - if !ok { - log.Warn(ctx, "splitmuxsink-fragment-closed not a uint64") - cancel() - } - m3u8.FragmentClosed(ctx, runningTimeInt) - } - } - }) - cancel() - }() - - // Start the pipeline - pipeline.SetState(gst.StatePlaying) - - <-ctx.Done() - - pipeline.BlockSetState(gst.StateNull) - - return nil -} - func (mm *MediaManager) IngestStream(ctx context.Context, input io.Reader, ms MediaSigner) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -643,416 +418,3 @@ func (mm *MediaManager) TestSource(ctx context.Context, ms MediaSigner) error { return g.Wait() } - -// element that takes the input stream, muxes to mp4, and signs the result -func (mm *MediaManager) SegmentAndSignElem(ctx context.Context, ms MediaSigner) (*gst.Element, error) { - // elem, err := gst.NewElement("splitmuxsink name=splitter async-finalize=true sink-factory=appsink muxer-factory=matroskamux max-size-bytes=1") - elem, err := gst.NewElementWithProperties("splitmuxsink", map[string]any{ - "name": "signer", - "async-finalize": true, - "sink-factory": "appsink", - "muxer-factory": "mp4mux", - "max-size-bytes": 1, - }) - if err != nil { - return nil, err - } - - p := elem.GetRequestPad("video") - if p == nil { - return nil, fmt.Errorf("failed to get video pad") - } - p = elem.GetRequestPad("audio_%u") - if p == nil { - return nil, fmt.Errorf("failed to get audio pad") - } - - resetTimer := make(chan struct{}) - - go func() { - for { - select { - case <-ctx.Done(): - return - case <-resetTimer: - continue - case <-time.After(time.Second * 10): - log.Warn(ctx, "no new segment for 10 seconds") - elem.ErrorMessage(gst.DomainCore, gst.CoreErrorFailed, "No new segment for 10 seconds", "No new segment for 10 seconds (debug)") - return - } - } - }() - - elem.Connect("sink-added", func(split, sinkEle *gst.Element) { - buf := &bytes.Buffer{} - appsink := app.SinkFromElement(sinkEle) - if appsink == nil { - panic("appsink should not be nil") - } - appsink.SetCallbacks(&app.SinkCallbacks{ - NewSampleFunc: WriterNewSample(ctx, buf), - EOSFunc: func(sink *app.Sink) { - resetTimer <- struct{}{} - bs, err := ms.SignMP4(ctx, bytes.NewReader(buf.Bytes()), time.Now().UnixMilli()) - if err != nil { - log.Error(ctx, "error signing segment", "error", err) - return - } - err = mm.ValidateMP4(ctx, bytes.NewReader(bs)) - if err != nil { - log.Error(ctx, "error validating segment", "error", err) - return - } - }, - }) - }) - - return elem, nil -} - -func (mm *MediaManager) Thumbnail(ctx context.Context, r io.Reader, w io.Writer) error { - ctx = log.WithLogValues(ctx, "function", "Thumbnail") - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - pipelineSlice := []string{ - "appsrc name=appsrc ! qtdemux ! decodebin ! videoconvert ! videoscale ! video/x-raw,width=[1,720],height=[1,720],pixel-aspect-ratio=1/1 ! pngenc snapshot=true ! appsink name=appsink", - } - - pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) - if err != nil { - return fmt.Errorf("error creating Thumbnail pipeline: %w", err) - } - appsrc, err := pipeline.GetElementByName("appsrc") - if err != nil { - return err - } - - src := app.SrcFromElement(appsrc) - src.SetCallbacks(&app.SourceCallbacks{ - NeedDataFunc: ReaderNeedData(ctx, r), - }) - - appsink, err := pipeline.GetElementByName("appsink") - if err != nil { - return err - } - - go func() { - HandleBusMessages(ctx, pipeline) - cancel() - }() - - sink := app.SinkFromElement(appsink) - sink.SetCallbacks(&app.SinkCallbacks{ - NewSampleFunc: WriterNewSample(ctx, w), - EOSFunc: func(sink *app.Sink) { - cancel() - }, - }) - - pipeline.SetState(gst.StatePlaying) - - <-ctx.Done() - - pipeline.BlockSetState(gst.StateNull) - - return nil -} - -func (mm *MediaManager) MP4Playback(ctx context.Context, user string, w io.Writer) error { - uu, err := uuid.NewV7() - if err != nil { - return err - } - ctx = log.WithLogValues(ctx, "playbackID", uu.String()) - ctx, cancel := context.WithCancel(ctx) - - ctx = log.WithLogValues(ctx, "mediafunc", "MP4Playback") - - pipelineSlice := []string{ - "mp4mux name=muxer fragment-mode=first-moov-then-finalise fragment-duration=1000 streamable=true ! appsink name=mp4sink", - "h264parse name=videoparse ! muxer.", - "opusparse name=audioparse ! muxer.", - } - - pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) - if err != nil { - return fmt.Errorf("failed to create GStreamer pipeline: %w", err) - } - - go func() { - HandleBusMessages(ctx, pipeline) - cancel() - }() - - outputQueue, done, err := ConcatStream(ctx, pipeline, user, mm) - if err != nil { - return fmt.Errorf("failed to get output queue: %w", err) - } - go func() { - select { - case <-ctx.Done(): - return - case <-done: - cancel() - } - }() - - videoParse, err := pipeline.GetElementByName("videoparse") - if err != nil { - return fmt.Errorf("failed to get video sink element from pipeline: %w", err) - } - err = outputQueue.Link(videoParse) - if err != nil { - return fmt.Errorf("failed to link output queue to video parse: %w", err) - } - - audioParse, err := pipeline.GetElementByName("audioparse") - if err != nil { - return fmt.Errorf("failed to get audio parse element from pipeline: %w", err) - } - err = outputQueue.Link(audioParse) - if err != nil { - return fmt.Errorf("failed to link output queue to audio parse: %w", err) - } - - go func() { - ticker := time.NewTicker(time.Second * 1) - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - state := pipeline.GetCurrentState() - log.Debug(ctx, "pipeline state", "state", state) - } - } - }() - - mp4sinkele, err := pipeline.GetElementByName("mp4sink") - if err != nil { - return fmt.Errorf("failed to get video sink element from pipeline: %w", err) - } - mp4sink := app.SinkFromElement(mp4sinkele) - mp4sink.SetCallbacks(&app.SinkCallbacks{ - NewSampleFunc: WriterNewSample(ctx, w), - EOSFunc: func(sink *app.Sink) { - log.Warn(ctx, "mp4sink EOSFunc") - cancel() - }, - }) - - pipeline.SetState(gst.StatePlaying) - - <-ctx.Done() - - pipeline.BlockSetState(gst.StateNull) - - return nil -} - -func (mm *MediaManager) MKVPlayback(ctx context.Context, user string, w io.Writer) error { - uu, err := uuid.NewV7() - if err != nil { - return err - } - ctx = log.WithLogValues(ctx, "playbackID", uu.String()) - ctx, cancel := context.WithCancel(ctx) - - ctx = log.WithLogValues(ctx, "mediafunc", "MKVPlayback") - - pipelineSlice := []string{ - "matroskamux name=muxer streamable=true ! appsink name=mkvsink", - "h264parse name=videoparse ! muxer.", - "opusparse name=audioparse ! muxer.", - } - - pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) - if err != nil { - return fmt.Errorf("failed to create GStreamer pipeline: %w", err) - } - - go func() { - HandleBusMessages(ctx, pipeline) - cancel() - }() - - outputQueue, done, err := ConcatStream(ctx, pipeline, user, mm) - if err != nil { - return fmt.Errorf("failed to get output queue: %w", err) - } - go func() { - select { - case <-ctx.Done(): - return - case <-done: - cancel() - } - }() - - videoParse, err := pipeline.GetElementByName("videoparse") - if err != nil { - return fmt.Errorf("failed to get video sink element from pipeline: %w", err) - } - err = outputQueue.Link(videoParse) - if err != nil { - return fmt.Errorf("failed to link output queue to video parse: %w", err) - } - - audioParse, err := pipeline.GetElementByName("audioparse") - if err != nil { - return fmt.Errorf("failed to get audio parse element from pipeline: %w", err) - } - err = outputQueue.Link(audioParse) - if err != nil { - return fmt.Errorf("failed to link output queue to audio parse: %w", err) - } - - go func() { - ticker := time.NewTicker(time.Second * 1) - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - state := pipeline.GetCurrentState() - log.Debug(ctx, "pipeline state", "state", state) - } - } - }() - - mkvsinkele, err := pipeline.GetElementByName("mkvsink") - if err != nil { - return fmt.Errorf("failed to get video sink element from pipeline: %w", err) - } - mkvsink := app.SinkFromElement(mkvsinkele) - mkvsink.SetCallbacks(&app.SinkCallbacks{ - NewSampleFunc: WriterNewSample(ctx, w), - EOSFunc: func(sink *app.Sink) { - log.Warn(ctx, "mp4sink EOSFunc") - cancel() - }, - }) - - pipeline.SetState(gst.StatePlaying) - - <-ctx.Done() - - pipeline.BlockSetState(gst.StateNull) - - return nil -} - -func (mm *MediaManager) ParseSegmentMediaData(ctx context.Context, mp4bs []byte) (*model.SegmentMediaData, error) { - ctx = log.WithLogValues(ctx, "GStreamerFunc", "ParseSegmentMediaData") - ctx, cancel := context.WithCancel(ctx) - defer cancel() - pipelineSlice := []string{ - "appsrc name=appsrc ! qtdemux name=demux ! fakesink", - } - - pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) - if err != nil { - return nil, fmt.Errorf("error creating SegmentMetadata pipeline: %w", err) - } - - var videoMetadata *model.SegmentMediadataVideo - var audioMetadata *model.SegmentMediadataAudio - - appsrc, err := pipeline.GetElementByName("appsrc") - if err != nil { - return nil, fmt.Errorf("error creating SegmentMetadata pipeline: %w", err) - } - - src := app.SrcFromElement(appsrc) - src.SetCallbacks(&app.SourceCallbacks{ - NeedDataFunc: ReaderNeedData(ctx, bytes.NewReader(mp4bs)), - }) - - onPadAdded := func(element *gst.Element, pad *gst.Pad) { - caps := pad.GetCurrentCaps() - if caps == nil { - log.Warn(ctx, "Unable to get pad caps") - cancel() - return - } - - structure := caps.GetStructureAt(0) - if structure == nil { - log.Warn(ctx, "Unable to get structure from caps") - cancel() - return - } - - name := structure.Name() - log.Debug(ctx, "Structure Name", "name", name) - - if name[:5] == "video" { - videoMetadata = &model.SegmentMediadataVideo{} - // Get some common video properties - widthVal, _ := structure.GetValue("width") - heightVal, _ := structure.GetValue("height") - - width, ok := widthVal.(int) - if ok { - videoMetadata.Width = width - } - height, ok := heightVal.(int) - if ok { - videoMetadata.Height = height - } - framerateVal, _ := structure.GetValue("framerate") - framerateStr := fmt.Sprintf("%v", framerateVal) - if framerateStr != "" { - videoMetadata.Framerate = framerateStr - } - } - - if name[:5] == "audio" { - audioMetadata = &model.SegmentMediadataAudio{} - // Get some common audio properties - rateVal, _ := structure.GetValue("rate") - channelsVal, _ := structure.GetValue("channels") - - rate, ok := rateVal.(int) - if ok { - audioMetadata.Rate = rate - } - channels, ok := channelsVal.(int) - if ok { - audioMetadata.Channels = channels - } - } - - if videoMetadata != nil && audioMetadata != nil { - cancel() - } - } - - demux, err := pipeline.GetElementByName("demux") - if err != nil { - return nil, fmt.Errorf("error creating SegmentMetadata pipeline: %w", err) - } - demux.Connect("pad-added", onPadAdded) - - go func() { - HandleBusMessages(ctx, pipeline) - cancel() - }() - - // Start the pipeline - pipeline.SetState(gst.StatePlaying) - - <-ctx.Done() - - meta := &model.SegmentMediaData{ - Video: []*model.SegmentMediadataVideo{videoMetadata}, - Audio: []*model.SegmentMediadataAudio{audioMetadata}, - } - - pipeline.BlockSetState(gst.StateNull) - - return meta, nil -} diff --git a/pkg/media/m3u8.go b/pkg/media/m3u8.go index 7ccc5228..9763704a 100644 --- a/pkg/media/m3u8.go +++ b/pkg/media/m3u8.go @@ -6,10 +6,12 @@ import ( "fmt" "math" "strings" + "sync" "time" "github.com/google/uuid" "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/renditions" ) // how many segments are served in the live playlist? @@ -18,136 +20,89 @@ const LIVE_PLAYLIST_SIZE = 8 // how long should we keep old segments around? const RETAIN_SEGMENT_SIZE = LIVE_PLAYLIST_SIZE * 3 -type Segment struct { - MSN uint64 // media sequence number - Buf *bytes.Buffer - StartTime *uint64 - EndTime *uint64 - Closed bool -} +const INDEX_M3U8 = "index.m3u8" -func (s *Segment) Duration() time.Duration { - return time.Duration(*s.EndTime - *s.StartTime) +type Segment struct { + MSN uint64 // media sequence number + Duration time.Duration + Buf *bytes.Buffer + Time time.Time + Closed bool + StartTS *uint64 + EndTS *uint64 } type M3U8 struct { curSeg uint64 - segments []*Segment pendingSegments []*Segment waits []chan struct{} - Bitrate uint64 - Width uint64 - Height uint64 -} - -func NewM3U8() *M3U8 { - return &M3U8{ - curSeg: 0, - } -} - -func (m *M3U8) GetNextSegment(ctx context.Context) (*Segment, error) { - log.Debug(ctx, "next segment") - msn := m.curSeg - m.curSeg += 1 - seg := &Segment{ - MSN: msn, - Buf: &bytes.Buffer{}, - } - m.pendingSegments = append(m.pendingSegments, seg) - return seg, nil + renditions []*M3U8Rendition } -func (m *M3U8) CloseSegment(ctx context.Context, seg *Segment) { - log.Debug(ctx, "close segment", "MSN", seg.MSN) - seg.Closed = true - m.checkSegments(ctx) +type M3U8Rendition struct { + Rendition renditions.Rendition + Segments []*Segment + SegmentLock sync.RWMutex + MSN uint64 } -func (m *M3U8) FragmentOpened(ctx context.Context, t uint64) error { - log.Debug(ctx, "fragment opened", "time", t) - if len(m.pendingSegments) == 0 { - return fmt.Errorf("no pending segments") - } - for _, seg := range m.pendingSegments { - if seg.StartTime == nil { - seg.StartTime = &t - break +func NewM3U8(renditions renditions.Renditions) *M3U8 { + rends := []*M3U8Rendition{} + for _, r := range renditions { + mr := &M3U8Rendition{ + Rendition: r, } + rends = append(rends, mr) } - m.checkSegments(ctx) - return nil -} - -func (m *M3U8) FragmentClosed(ctx context.Context, t uint64) error { - log.Debug(ctx, "fragment closed", "time", t) - if len(m.pendingSegments) == 0 { - return fmt.Errorf("no pending segments") - } - for _, seg := range m.pendingSegments { - if seg.EndTime == nil { - seg.EndTime = &t - if m.Bitrate == 0 { - dur := seg.Duration() - m.Bitrate = uint64(float64(seg.Buf.Len())/dur.Seconds()) * 8 - } - break - } + return &M3U8{ + curSeg: 0, + renditions: rends, } - m.checkSegments(ctx) - return nil } -// the tricky piece of the design here is that we need to expect GetNextSegment, -// CloseSegment, FragmentOpened, and FragmentClosed to be called in any order. So -// all of those functions call this one, and it checks if we have the necessary information -// to finalize a segment and add it to our playlist. -func (m *M3U8) checkSegments(ctx context.Context) { - pending := m.pendingSegments[0] - if pending.StartTime != nil && pending.EndTime != nil && pending.Closed { - m.segments = append(m.segments, pending) - m.pendingSegments = m.pendingSegments[1:] - log.Debug(ctx, "finalizing segment", "MSN", pending.MSN) - for _, wait := range m.waits { - go func(wait chan struct{}) { - wait <- struct{}{} - }(wait) - } - m.waits = []chan struct{}{} - } - if len(m.segments) > RETAIN_SEGMENT_SIZE { - startWith := len(m.segments) - RETAIN_SEGMENT_SIZE - m.segments = m.segments[startWith:] - } +func (r *M3U8Rendition) GetMediaLine(session string) string { + // m.waitForStart() + lines := []string{} + lines = append(lines, "#EXTM3U") + lines = append(lines, fmt.Sprintf("#EXT-X-STREAM-INF:BANDWIDTH=%d,RESOLUTION=%dx%d", r.Rendition.Bitrate, r.Rendition.Width, r.Rendition.Height)) + lines = append(lines, fmt.Sprintf("%s/%s?session=%s", r.Rendition.Name, INDEX_M3U8, session)) + return strings.Join(lines, "\n") } -func (m *M3U8) waitForStart() { - if len(m.segments) == 0 { - // todo: fix concurrent access here - wait := make(chan struct{}) - m.waits = append(m.waits, wait) - <-wait +func (r *M3U8Rendition) GetPlaylist(session string) []byte { + if session == "" { + uu, err := uuid.NewV7() + if err != nil { + panic(err) + } + session = uu.String() } -} - -func (m *M3U8) GetPlaylist(session string) []byte { - m.waitForStart() + r.SegmentLock.RLock() + defer r.SegmentLock.RUnlock() + // m.waitForStart() lines := []string{} lines = append(lines, "#EXTM3U") lines = append(lines, "#EXT-X-VERSION:3") - startWith := len(m.segments) - LIVE_PLAYLIST_SIZE + startWith := len(r.Segments) - LIVE_PLAYLIST_SIZE if startWith < 0 { startWith = 0 } - firstSeg := m.segments[startWith] - lastSeg := m.segments[len(m.segments)-1] - targetDuration := int64(math.Round(lastSeg.Duration().Seconds())) + if len(r.Segments) == 0 { + return []byte{} + } + firstSeg := r.Segments[startWith] + lastSeg := r.Segments[len(r.Segments)-1] + targetDuration := int64(math.Round(lastSeg.Duration.Seconds())) lines = append(lines, fmt.Sprintf("#EXT-X-MEDIA-SEQUENCE:%d", firstSeg.MSN)) - lines = append(lines, fmt.Sprintf("#EXT-X-TARGETDURATION:%d", targetDuration)) + lines = append(lines, fmt.Sprintf("#EXT-X-DISCONTINUITY-SEQUENCE:%d", firstSeg.MSN)) + lines = append(lines, fmt.Sprintf("#EXT-X-TARGETDURATION:%d", targetDuration+1)) + lines = append(lines, "#EXT-X-INDEPENDENT-SEGMENTS") lines = append(lines, "") - lastSegments := m.segments[startWith:] + lastSegments := r.Segments[startWith:] for _, seg := range lastSegments { - dur := seg.Duration() + dur := seg.Duration + lines = append(lines, "#EXT-X-DISCONTINUITY") + lines = append(lines, fmt.Sprintf("#EXT-X-PROGRAM-DATE-TIME:%s", seg.Time.Format(time.RFC3339Nano))) lines = append(lines, fmt.Sprintf("#EXTINF:%f,", dur.Seconds())) lines = append(lines, fmt.Sprintf("segment%05d.ts?session=%s", seg.MSN, session)) } @@ -155,31 +110,83 @@ func (m *M3U8) GetPlaylist(session string) []byte { return []byte(strings.Join(lines, "\n")) } -func (m *M3U8) GetMultivariantPlaylist() []byte { +func (r *M3U8Rendition) GetSegment(session string, filename string) []byte { + r.SegmentLock.RLock() + defer r.SegmentLock.RUnlock() + for _, seg := range r.Segments { + if fmt.Sprintf("segment%05d.ts", seg.MSN) == filename { + return seg.Buf.Bytes() + } + } + return nil +} + +func (m *M3U8) GetMultivariantPlaylist(rendition string) []byte { uu, err := uuid.NewV7() if err != nil { panic(err) } - m.waitForStart() + // m.waitForStart() lines := []string{} lines = append(lines, "#EXTM3U") - lines = append(lines, fmt.Sprintf("#EXT-X-STREAM-INF:BANDWIDTH=%d,RESOLUTION=%dx%d", m.Bitrate, m.Width, m.Height)) - lines = append(lines, fmt.Sprintf("media.m3u8?session=%s", uu.String())) + for _, r := range m.renditions { + if rendition == "" || r.Rendition.Name == rendition { + lines = append(lines, r.GetMediaLine(uu.String())) + } + } return []byte(strings.Join(lines, "\n")) } -// takes segment00015.ts and returns the corresponding segment -func (m *M3U8) GetSegment(str string, session string) ([]byte, error) { - if str == "stream.m3u8" { - return m.GetMultivariantPlaylist(), nil +// needs to handle: +// - index.m3u8 +// - 720p/stream.m3u8 +// - 720p/segment00015.ts +func (m *M3U8) GetFile(str string, session string, rendition string) ([]byte, error) { + str = strings.TrimPrefix(str, "/") + if str == INDEX_M3U8 { + return m.GetMultivariantPlaylist(rendition), nil } - if str == "media.m3u8" { - return m.GetPlaylist(session), nil + parts := strings.Split(str, "/") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid path") } - for _, seg := range m.segments { - if fmt.Sprintf("segment%05d.ts", seg.MSN) == str { - return seg.Buf.Bytes(), nil + rStr := parts[0] + fStr := parts[1] + rend := m.GetRendition(rStr) + log.Debug(context.Background(), "m3u8 get file", "str", str, "session", session, "rend", rStr, "file", fStr) + if rend == nil { + return nil, fmt.Errorf("rendition not found") + } + if fStr == INDEX_M3U8 { + return rend.GetPlaylist(session), nil + } + seg := rend.GetSegment(session, fStr) + if seg == nil { + return nil, fmt.Errorf("segment not found") + } + return seg, nil +} + +func (r *M3U8Rendition) NewSegment(seg *Segment) error { + r.SegmentLock.Lock() + defer r.SegmentLock.Unlock() + seg.MSN = r.MSN + r.MSN += 1 + r.Segments = append(r.Segments, seg) + if len(r.Segments) > RETAIN_SEGMENT_SIZE { + // Calculate how many segments to remove + removeCount := len(r.Segments) - RETAIN_SEGMENT_SIZE + // Remove the oldest segments (from the front of the slice) + r.Segments = r.Segments[removeCount:] + } + return nil +} + +func (m *M3U8) GetRendition(rendition string) *M3U8Rendition { + for _, r := range m.renditions { + if r.Rendition.Name == rendition { + return r } } - return nil, fmt.Errorf("segment not found") + return nil } diff --git a/pkg/media/media.go b/pkg/media/media.go index 121970c2..ff4345ea 100644 --- a/pkg/media/media.go +++ b/pkg/media/media.go @@ -1,32 +1,25 @@ package media import ( - "bytes" "context" "crypto" "encoding/json" "errors" "fmt" "io" - "strings" "sync" "github.com/go-gst/go-gst/gst" "github.com/google/uuid" - "github.com/livepeer/lpms/ffmpeg" - "golang.org/x/sync/errgroup" "stream.place/streamplace/pkg/aqtime" "stream.place/streamplace/pkg/atproto" "stream.place/streamplace/pkg/bus" "stream.place/streamplace/pkg/config" - "stream.place/streamplace/pkg/constants" - "stream.place/streamplace/pkg/crypto/signers" - "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/media/segchanman" "stream.place/streamplace/pkg/model" "stream.place/streamplace/pkg/replication" - "git.stream.place/streamplace/c2pa-go/pkg/c2pa" "git.stream.place/streamplace/c2pa-go/pkg/c2pa/generated/manifeststore" "github.com/piprate/json-gold/ld" ) @@ -38,8 +31,7 @@ var STREAMPLACE_METADATA = "place.stream.metadata" type MediaManager struct { cli *config.CLI - mp4subs map[string][]chan string - mp4subsmut sync.Mutex + segChanMan *segchanman.SegChanMan replicator replication.Replicator hlsRunning map[string]*M3U8 hlsRunningMut sync.Mutex @@ -71,7 +63,7 @@ func MakeMediaManager(ctx context.Context, cli *config.CLI, signer crypto.Signer } return &MediaManager{ cli: cli, - mp4subs: map[string][]chan string{}, + segChanMan: segchanman.MakeSegChanMan(), replicator: rep, hlsRunning: map[string]*M3U8{}, httpPipes: map[string]io.Writer{}, @@ -116,160 +108,17 @@ func (mm *MediaManager) NewSegment() <-chan *NewSegmentNotification { } // subscribe to the latest segments from a given user for livestreaming purposes -func (mm *MediaManager) SubscribeSegment(ctx context.Context, user string) <-chan string { - mm.mp4subsmut.Lock() - defer mm.mp4subsmut.Unlock() - _, ok := mm.mp4subs[user] - if !ok { - mm.mp4subs[user] = []chan string{} - } - c := make(chan string) - mm.mp4subs[user] = append(mm.mp4subs[user], c) - return c +func (mm *MediaManager) SubscribeSegment(ctx context.Context, user string, rendition string) <-chan *segchanman.Seg { + return mm.segChanMan.SubscribeSegment(ctx, user, rendition) } -func (mm *MediaManager) UnsubscribeSegment(ctx context.Context, user string, ch <-chan string) { - mm.mp4subsmut.Lock() - defer mm.mp4subsmut.Unlock() - for i, c := range mm.mp4subs[user] { - if c == ch { - mm.mp4subs[user] = append(mm.mp4subs[user][:i], mm.mp4subs[user][i+1:]...) - break - } - } +func (mm *MediaManager) UnsubscribeSegment(ctx context.Context, user string, rendition string, ch <-chan *segchanman.Seg) { + mm.segChanMan.UnsubscribeSegment(ctx, user, rendition, ch) } // subscribe to the latest segments from a given user for livestreaming purposes -func (mm *MediaManager) PublishSegment(ctx context.Context, user, file string) { - mm.mp4subsmut.Lock() - defer mm.mp4subsmut.Unlock() - for _, sub := range mm.mp4subs[user] { - go func() { - sub <- file - }() - } - mm.mp4subs[user] = []chan string{} -} - -func (mm *MediaManager) SegmentToMKV(ctx context.Context, user string, w io.Writer) error { - muxer := ffmpeg.ComponentOptions{ - Name: "matroska", - } - return mm.SegmentToStream(ctx, user, muxer, w) -} - -func (mm *MediaManager) SegmentToMKVPlusOpus(ctx context.Context, user string, w io.Writer) error { - muxer := ffmpeg.ComponentOptions{ - Name: "matroska", - } - pr, pw := io.Pipe() - g, ctx := errgroup.WithContext(ctx) - g.Go(func() error { - return mm.SegmentToStream(ctx, user, muxer, pw) - }) - g.Go(func() error { - return AddOpusToMKV(ctx, pr, w) - }) - return g.Wait() -} - -func (mm *MediaManager) SegmentToHLSOnce(ctx context.Context, user string) (*M3U8, error) { - mm.hlsRunningMut.Lock() - defer mm.hlsRunningMut.Unlock() - hls, ok := mm.hlsRunning[user] - if !ok { - hls = NewM3U8() - mm.hlsRunning[user] = hls - go func() { - err := mm.SegmentToHLS(ctx, user, hls) - if err != nil { - log.Log(ctx, "error in async segmentToHLS code", "error", err) - } - mm.hlsRunningMut.Lock() - defer mm.hlsRunningMut.Unlock() - delete(mm.hlsRunning, user) - }() - } - return hls, nil -} - -func (mm *MediaManager) SegmentToHLS(ctx context.Context, user string, m3u8 *M3U8) error { - muxer := ffmpeg.ComponentOptions{ - Name: "matroska", - } - - pr, pw := io.Pipe() - g, ctx := errgroup.WithContext(ctx) - g.Go(func() error { - return mm.SegmentToStream(ctx, user, muxer, pw) - }) - g.Go(func() error { - return mm.ToHLS(ctx, pr, m3u8) - }) - return g.Wait() -} - -func (mm *MediaManager) SegmentToMP4(ctx context.Context, user string, w io.Writer) error { - muxer := ffmpeg.ComponentOptions{ - Name: "mp4", - Opts: map[string]string{ - "movflags": "frag_keyframe+empty_moov", - }, - } - return mm.SegmentToStream(ctx, user, muxer, w) -} - -func (mm *MediaManager) SegmentToStream(ctx context.Context, user string, muxer ffmpeg.ComponentOptions, w io.Writer) error { - tc := ffmpeg.NewTranscoder() - defer tc.StopTranscoder() - ourl, or, odone, err := mm.HTTPPipe() - if err != nil { - return err - } - defer odone() - iname := fmt.Sprintf("%s/playback/%s/concat", mm.cli.OwnInternalURL(), user) - in := &ffmpeg.TranscodeOptionsIn{ - Fname: iname, - Transmuxing: true, - Profile: ffmpeg.VideoProfile{}, - Loop: -1, - Demuxer: ffmpeg.ComponentOptions{ - Name: "concat", - Opts: map[string]string{ - "safe": "0", - "protocol_whitelist": "file,http,https,tcp,tls", - }, - }, - } - out := []ffmpeg.TranscodeOptions{ - { - Oname: ourl, - VideoEncoder: ffmpeg.ComponentOptions{ - Name: "copy", - }, - AudioEncoder: ffmpeg.ComponentOptions{ - Name: "copy", - }, - Profile: ffmpeg.VideoProfile{Format: ffmpeg.FormatNone}, - Muxer: muxer, - }, - } - g, _ := errgroup.WithContext(ctx) - g.Go(func() error { - <-ctx.Done() - or.Close() - return nil - }) - g.Go(func() error { - _, err := tc.Transcode(in, out) - tc.StopTranscoder() - return err - }) - g.Go(func() error { - _, err := io.Copy(w, or) - return err - }) - return g.Wait() +func (mm *MediaManager) PublishSegment(ctx context.Context, user, rendition string, seg *segchanman.Seg) { + mm.segChanMan.PublishSegment(ctx, user, rendition, seg) } type obj map[string]any @@ -342,84 +191,3 @@ func ParseSegmentAssertions(mani *manifeststore.Manifest) (*SegmentMetadata, err } return &out, nil } - -func (mm *MediaManager) ValidateMP4(ctx context.Context, input io.Reader) error { - buf, err := io.ReadAll(input) - if err != nil { - return err - } - r := bytes.NewReader(buf) - reader, err := c2pa.FromStream(r, "video/mp4") - if err != nil { - return err - } - mani := reader.GetActiveManifest() - certs := reader.GetProvenanceCertChain() - pub, err := signers.ParseES256KCert([]byte(certs)) - if err != nil { - return err - } - meta, err := ParseSegmentAssertions(mani) - if err != nil { - return err - } - mediaData, err := mm.ParseSegmentMediaData(ctx, buf) - if err != nil { - return err - } - // special case for test signers that are only signed with a key - var repoDID string - var signingKeyDID string - if strings.HasPrefix(meta.Creator, constants.DID_KEY_PREFIX) { - signingKeyDID = meta.Creator - repoDID = meta.Creator - } else { - repo, err := mm.atsync.SyncBlueskyRepoCached(ctx, meta.Creator, mm.model) - if err != nil { - return err - } - signingKey, err := mm.model.GetSigningKey(pub.DIDKey(), repo.DID) - if err != nil { - return err - } - if signingKey == nil { - return fmt.Errorf("no signing key found for %s", pub.DIDKey()) - } - repoDID = repo.DID - signingKeyDID = signingKey.DID - } - - err = mm.cli.StreamIsAllowed(repoDID) - if err != nil { - return fmt.Errorf("got valid segment, but user %s is not allowed: %w", repoDID, err) - } - fd, err := mm.cli.SegmentFileCreate(repoDID, meta.StartTime, "mp4") - if err != nil { - return err - } - defer fd.Close() - go mm.replicator.NewSegment(ctx, buf) - r = bytes.NewReader(buf) - io.Copy(fd, r) - go mm.PublishSegment(ctx, repoDID, fd.Name()) - seg := &model.Segment{ - ID: *mani.Label, - SigningKeyDID: signingKeyDID, - RepoDID: repoDID, - StartTime: meta.StartTime.Time(), - Title: meta.Title, - MediaData: mediaData, - } - mm.newSegmentSubsMutex.RLock() - defer mm.newSegmentSubsMutex.RUnlock() - not := &NewSegmentNotification{ - Segment: seg, - Data: buf, - Metadata: meta, - } - for _, ch := range mm.newSegmentSubs { - go func() { ch <- not }() - } - log.Log(ctx, "successfully ingested segment", "user", repoDID, "signingKey", signingKeyDID, "timestamp", meta.StartTime, "segmentID", *mani.Label) - return nil -} diff --git a/pkg/media/media_data_parser.go b/pkg/media/media_data_parser.go new file mode 100644 index 00000000..107af538 --- /dev/null +++ b/pkg/media/media_data_parser.go @@ -0,0 +1,140 @@ +package media + +import ( + "bytes" + "context" + "fmt" + "strconv" + "strings" + + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/model" +) + +func (mm *MediaManager) ParseSegmentMediaData(ctx context.Context, mp4bs []byte) (*model.SegmentMediaData, error) { + ctx = log.WithLogValues(ctx, "GStreamerFunc", "ParseSegmentMediaData") + ctx, cancel := context.WithCancel(ctx) + defer cancel() + pipelineSlice := []string{ + "appsrc name=appsrc ! qtdemux name=demux ! fakesink", + } + + pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) + if err != nil { + return nil, fmt.Errorf("error creating SegmentMetadata pipeline: %w", err) + } + + var videoMetadata *model.SegmentMediadataVideo + var audioMetadata *model.SegmentMediadataAudio + + appsrc, err := pipeline.GetElementByName("appsrc") + if err != nil { + return nil, fmt.Errorf("error creating SegmentMetadata pipeline: %w", err) + } + + src := app.SrcFromElement(appsrc) + src.SetCallbacks(&app.SourceCallbacks{ + NeedDataFunc: ReaderNeedData(ctx, bytes.NewReader(mp4bs)), + }) + + onPadAdded := func(element *gst.Element, pad *gst.Pad) { + caps := pad.GetCurrentCaps() + if caps == nil { + log.Warn(ctx, "Unable to get pad caps") + cancel() + return + } + + structure := caps.GetStructureAt(0) + if structure == nil { + log.Warn(ctx, "Unable to get structure from caps") + cancel() + return + } + + name := structure.Name() + + if name[:5] == "video" { + videoMetadata = &model.SegmentMediadataVideo{} + // Get some common video properties + widthVal, _ := structure.GetValue("width") + heightVal, _ := structure.GetValue("height") + + width, ok := widthVal.(int) + if ok { + videoMetadata.Width = width + } + height, ok := heightVal.(int) + if ok { + videoMetadata.Height = height + } + framerateVal, _ := structure.GetValue("framerate") + framerateStr := fmt.Sprintf("%v", framerateVal) + parts := strings.Split(framerateStr, "/") + num := 0 + den := 0 + if len(parts) == 2 { + num, _ = strconv.Atoi(parts[0]) + den, _ = strconv.Atoi(parts[1]) + } + if num != 0 && den != 0 { + videoMetadata.FPSNum = num + videoMetadata.FPSDen = den + } + } + + if name[:5] == "audio" { + audioMetadata = &model.SegmentMediadataAudio{} + // Get some common audio properties + rateVal, _ := structure.GetValue("rate") + channelsVal, _ := structure.GetValue("channels") + + rate, ok := rateVal.(int) + if ok { + audioMetadata.Rate = rate + } + channels, ok := channelsVal.(int) + if ok { + audioMetadata.Channels = channels + } + } + + // if videoMetadata != nil && audioMetadata != nil { + // cancel() + // } + } + + demux, err := pipeline.GetElementByName("demux") + if err != nil { + return nil, fmt.Errorf("error creating SegmentMetadata pipeline: %w", err) + } + demux.Connect("pad-added", onPadAdded) + + go func() { + HandleBusMessages(ctx, pipeline) + cancel() + }() + + // Start the pipeline + pipeline.SetState(gst.StatePlaying) + + <-ctx.Done() + + meta := &model.SegmentMediaData{ + Video: []*model.SegmentMediadataVideo{videoMetadata}, + Audio: []*model.SegmentMediadataAudio{audioMetadata}, + } + + ok, dur := pipeline.QueryDuration(gst.FormatTime) + if !ok { + return nil, fmt.Errorf("error getting duration") + } else { + meta.Duration = dur + } + + pipeline.BlockSetState(gst.StateNull) + + return meta, nil +} diff --git a/pkg/media/progressive.go b/pkg/media/progressive.go new file mode 100644 index 00000000..8a08a018 --- /dev/null +++ b/pkg/media/progressive.go @@ -0,0 +1,198 @@ +package media + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "github.com/google/uuid" + "stream.place/streamplace/pkg/log" +) + +func (mm *MediaManager) MP4Playback(ctx context.Context, user string, rendition string, w io.Writer) error { + uu, err := uuid.NewV7() + if err != nil { + return err + } + ctx = log.WithLogValues(ctx, "playbackID", uu.String()) + ctx, cancel := context.WithCancel(ctx) + + ctx = log.WithLogValues(ctx, "mediafunc", "MP4Playback") + + pipelineSlice := []string{ + "mp4mux name=muxer fragment-mode=first-moov-then-finalise fragment-duration=1000 streamable=true ! appsink name=mp4sink", + "h264parse name=videoparse ! muxer.", + "opusparse name=audioparse ! muxer.", + } + + pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) + if err != nil { + return fmt.Errorf("failed to create GStreamer pipeline: %w", err) + } + + go func() { + HandleBusMessages(ctx, pipeline) + cancel() + }() + + outputQueue, done, err := ConcatStream(ctx, pipeline, user, rendition, mm) + if err != nil { + return fmt.Errorf("failed to get output queue: %w", err) + } + go func() { + select { + case <-ctx.Done(): + return + case <-done: + cancel() + } + }() + + videoParse, err := pipeline.GetElementByName("videoparse") + if err != nil { + return fmt.Errorf("failed to get video sink element from pipeline: %w", err) + } + err = outputQueue.Link(videoParse) + if err != nil { + return fmt.Errorf("failed to link output queue to video parse: %w", err) + } + + audioParse, err := pipeline.GetElementByName("audioparse") + if err != nil { + return fmt.Errorf("failed to get audio parse element from pipeline: %w", err) + } + err = outputQueue.Link(audioParse) + if err != nil { + return fmt.Errorf("failed to link output queue to audio parse: %w", err) + } + + go func() { + ticker := time.NewTicker(time.Second * 1) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + state := pipeline.GetCurrentState() + log.Debug(ctx, "pipeline state", "state", state) + } + } + }() + + mp4sinkele, err := pipeline.GetElementByName("mp4sink") + if err != nil { + return fmt.Errorf("failed to get video sink element from pipeline: %w", err) + } + mp4sink := app.SinkFromElement(mp4sinkele) + mp4sink.SetCallbacks(&app.SinkCallbacks{ + NewSampleFunc: WriterNewSample(ctx, w), + EOSFunc: func(sink *app.Sink) { + log.Warn(ctx, "mp4sink EOSFunc") + cancel() + }, + }) + + pipeline.SetState(gst.StatePlaying) + + <-ctx.Done() + + pipeline.BlockSetState(gst.StateNull) + + return nil +} + +func (mm *MediaManager) MKVPlayback(ctx context.Context, user string, rendition string, w io.Writer) error { + uu, err := uuid.NewV7() + if err != nil { + return err + } + ctx = log.WithLogValues(ctx, "playbackID", uu.String()) + ctx, cancel := context.WithCancel(ctx) + + ctx = log.WithLogValues(ctx, "mediafunc", "MKVPlayback") + + pipelineSlice := []string{ + "matroskamux name=muxer streamable=true ! appsink name=mkvsink", + "h264parse name=videoparse ! muxer.", + "opusparse name=audioparse ! muxer.", + } + + pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) + if err != nil { + return fmt.Errorf("failed to create GStreamer pipeline: %w", err) + } + + go func() { + HandleBusMessages(ctx, pipeline) + cancel() + }() + + outputQueue, done, err := ConcatStream(ctx, pipeline, user, rendition, mm) + if err != nil { + return fmt.Errorf("failed to get output queue: %w", err) + } + go func() { + select { + case <-ctx.Done(): + return + case <-done: + cancel() + } + }() + + videoParse, err := pipeline.GetElementByName("videoparse") + if err != nil { + return fmt.Errorf("failed to get video sink element from pipeline: %w", err) + } + err = outputQueue.Link(videoParse) + if err != nil { + return fmt.Errorf("failed to link output queue to video parse: %w", err) + } + + audioParse, err := pipeline.GetElementByName("audioparse") + if err != nil { + return fmt.Errorf("failed to get audio parse element from pipeline: %w", err) + } + err = outputQueue.Link(audioParse) + if err != nil { + return fmt.Errorf("failed to link output queue to audio parse: %w", err) + } + + go func() { + ticker := time.NewTicker(time.Second * 1) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + state := pipeline.GetCurrentState() + log.Debug(ctx, "pipeline state", "state", state) + } + } + }() + + mkvsinkele, err := pipeline.GetElementByName("mkvsink") + if err != nil { + return fmt.Errorf("failed to get video sink element from pipeline: %w", err) + } + mkvsink := app.SinkFromElement(mkvsinkele) + mkvsink.SetCallbacks(&app.SinkCallbacks{ + NewSampleFunc: WriterNewSample(ctx, w), + EOSFunc: func(sink *app.Sink) { + log.Warn(ctx, "mp4sink EOSFunc") + cancel() + }, + }) + + pipeline.SetState(gst.StatePlaying) + + <-ctx.Done() + + pipeline.BlockSetState(gst.StateNull) + + return nil +} diff --git a/pkg/media/segchanman/segchanman.go b/pkg/media/segchanman/segchanman.go new file mode 100644 index 00000000..346669cd --- /dev/null +++ b/pkg/media/segchanman/segchanman.go @@ -0,0 +1,76 @@ +package segchanman + +import ( + "context" + "fmt" + "sync" +) + +// it's a segment channel manager, you see + +type Seg struct { + Filepath string + Data []byte +} + +type SegChanMan struct { + segChans map[string][]chan *Seg + segChansMutex sync.Mutex +} + +func MakeSegChanMan() *SegChanMan { + return &SegChanMan{ + segChans: make(map[string][]chan *Seg), + } +} + +func segChanKey(user string, rendition string) string { + return fmt.Sprintf("%s::%s", user, rendition) +} + +func (s *SegChanMan) SubscribeSegment(ctx context.Context, user string, rendition string) <-chan *Seg { + key := segChanKey(user, rendition) + s.segChansMutex.Lock() + defer s.segChansMutex.Unlock() + chs, ok := s.segChans[key] + if !ok { + chs = []chan *Seg{} + s.segChans[key] = chs + } + ch := make(chan *Seg, 1024) + chs = append(chs, ch) + s.segChans[key] = chs + return ch +} + +func (s *SegChanMan) UnsubscribeSegment(ctx context.Context, user string, rendition string, ch <-chan *Seg) { + key := segChanKey(user, rendition) + s.segChansMutex.Lock() + defer s.segChansMutex.Unlock() + chs, ok := s.segChans[key] + if !ok { + return + } + for i, c := range chs { + if c == ch { + chs = append(chs[:i], chs[i+1:]...) + break + } + } + s.segChans[key] = chs +} + +func (s *SegChanMan) PublishSegment(ctx context.Context, user string, rendition string, seg *Seg) { + key := segChanKey(user, rendition) + s.segChansMutex.Lock() + defer s.segChansMutex.Unlock() + chs, ok := s.segChans[key] + if !ok { + return + } + for _, ch := range chs { + go func(ch chan *Seg) { + ch <- seg + }(ch) + } +} diff --git a/pkg/media/segment_conv.go b/pkg/media/segment_conv.go new file mode 100644 index 00000000..9bc547bd --- /dev/null +++ b/pkg/media/segment_conv.go @@ -0,0 +1,131 @@ +package media + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "golang.org/x/sync/errgroup" +) + +// MP4ToMPEGTS converts an MP4 file with H264 video and Opus audio to an MPEG-TS file with H264 video and AAC audio. +// It reads from the provided reader and writes the converted MPEG-TS to the writer. +func MP4ToMPEGTS(ctx context.Context, input io.Reader, output io.Writer) (int64, error) { + pipelineStr := strings.Join([]string{ + "appsrc name=appsrc ! qtdemux name=demux", + "mpegtsmux name=mux ! appsink name=appsink", + "demux.video_0 ! h264parse ! video/x-h264,stream-format=byte-stream ! queue name=videoqueue", + "demux.audio_0 ! opusdec use-inband-fec=true ! audioresample ! fdkaacenc ! aacparse ! queue name=audioqueue", + }, " ") + + pipeline, err := gst.NewPipelineFromString(pipelineStr) + if err != nil { + return 0, err + } + + mux, err := pipeline.GetElementByName("mux") + if err != nil { + return 0, err + } + muxVideoSinkPad := mux.GetRequestPad("sink_%d") + if muxVideoSinkPad == nil { + return 0, fmt.Errorf("failed to get video sink pad") + } + muxAudioSinkPad := mux.GetRequestPad("sink_%d") + if muxAudioSinkPad == nil { + return 0, fmt.Errorf("failed to get audio sink pad") + } + videoQueue, err := pipeline.GetElementByName("videoqueue") + if err != nil { + return 0, err + } + audioQueue, err := pipeline.GetElementByName("audioqueue") + if err != nil { + return 0, err + } + videoQueueSrcPad := videoQueue.GetStaticPad("src") + if videoQueueSrcPad == nil { + return 0, fmt.Errorf("failed to get video queue source pad") + } + audioQueueSrcPad := audioQueue.GetStaticPad("src") + if audioQueueSrcPad == nil { + return 0, fmt.Errorf("failed to get audio queue source pad") + } + + ok := videoQueueSrcPad.Link(muxVideoSinkPad) + if ok != gst.PadLinkOK { + return 0, fmt.Errorf("failed to link video queue source pad to mux video sink pad: %v", ok) + } + ok = audioQueueSrcPad.Link(muxAudioSinkPad) + if ok != gst.PadLinkOK { + return 0, fmt.Errorf("failed to link audio queue source pad to mux audio sink pad: %v", ok) + } + + // Get elements + appsrc, err := pipeline.GetElementByName("appsrc") + if err != nil { + return 0, err + } + appsink, err := pipeline.GetElementByName("appsink") + if err != nil { + return 0, err + } + + source := app.SrcFromElement(appsrc) + sink := app.SinkFromElement(appsink) + + // Set up source callbacks + source.SetCallbacks(&app.SourceCallbacks{ + NeedDataFunc: ReaderNeedData(ctx, input), + EnoughDataFunc: func(self *app.Source) { + // Nothing to do here + }, + SeekDataFunc: func(self *app.Source, offset uint64) bool { + return false // We don't support seeking + }, + }) + + // Set up sink callbacks + sink.SetCallbacks(&app.SinkCallbacks{ + NewSampleFunc: WriterNewSample(ctx, output), + NewPrerollFunc: func(self *app.Sink) gst.FlowReturn { + return gst.FlowOK + }, + }) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Handle bus messages in a separate goroutine + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + HandleBusMessages(ctx, pipeline) + cancel() + return nil + }) + + // Start the pipeline + err = pipeline.SetState(gst.StatePlaying) + if err != nil { + return 0, fmt.Errorf("failed to set pipeline state to playing: %w", err) + } + + // Wait for the pipeline to finish or context to be canceled + <-ctx.Done() + + durOk, dur := pipeline.QueryDuration(gst.FormatTime) + if !durOk { + return 0, fmt.Errorf("failed to query duration") + } + + // Clean up + err = pipeline.SetState(gst.StateNull) + if err != nil { + return 0, fmt.Errorf("failed to set pipeline state to null: %w", err) + } + + return dur, nil +} diff --git a/pkg/media/segment_conv_test.go b/pkg/media/segment_conv_test.go new file mode 100644 index 00000000..cdc572ad --- /dev/null +++ b/pkg/media/segment_conv_test.go @@ -0,0 +1,35 @@ +package media + +import ( + "context" + "os" + "testing" + + "github.com/go-gst/go-gst/gst" + "github.com/stretchr/testify/require" +) + +func TestMP4ToMPEGTS(t *testing.T) { + gst.Init(nil) + + // Open input file + inputFile, err := os.Open(getFixture("sample-segment.mp4")) + require.NoError(t, err) + defer inputFile.Close() + + // Create temporary output file + outputFile, err := os.CreateTemp("", "*.ts") + require.NoError(t, err) + defer os.Remove(outputFile.Name()) + defer outputFile.Close() + + // Convert MP4 to MPEG-TS + dur, err := MP4ToMPEGTS(context.Background(), inputFile, outputFile) + require.NoError(t, err) + require.Greater(t, dur, int64(0), "Duration should be greater than 0") + + // Verify output file has content + info, err := os.Stat(outputFile.Name()) + require.NoError(t, err) + require.Greater(t, info.Size(), int64(0), "Output file should not be empty") +} diff --git a/pkg/media/segmenter.go b/pkg/media/segmenter.go new file mode 100644 index 00000000..53c403c6 --- /dev/null +++ b/pkg/media/segmenter.go @@ -0,0 +1,81 @@ +package media + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "stream.place/streamplace/pkg/log" +) + +// element that takes the input stream, muxes to mp4, and signs the result +func (mm *MediaManager) SegmentAndSignElem(ctx context.Context, ms MediaSigner) (*gst.Element, error) { + // elem, err := gst.NewElement("splitmuxsink name=splitter async-finalize=true sink-factory=appsink muxer-factory=matroskamux max-size-bytes=1") + elem, err := gst.NewElementWithProperties("splitmuxsink", map[string]any{ + "name": "signer", + "async-finalize": true, + "sink-factory": "appsink", + "muxer-factory": "mp4mux", + "max-size-bytes": 1, + }) + if err != nil { + return nil, err + } + + p := elem.GetRequestPad("video") + if p == nil { + return nil, fmt.Errorf("failed to get video pad") + } + p = elem.GetRequestPad("audio_%u") + if p == nil { + return nil, fmt.Errorf("failed to get audio pad") + } + + resetTimer := make(chan struct{}) + + go func() { + for { + select { + case <-ctx.Done(): + return + case <-resetTimer: + continue + case <-time.After(time.Second * 10): + log.Warn(ctx, "no new segment for 10 seconds") + elem.ErrorMessage(gst.DomainCore, gst.CoreErrorFailed, "No new segment for 10 seconds", "No new segment for 10 seconds (debug)") + return + } + } + }() + + elem.Connect("sink-added", func(split, sinkEle *gst.Element) { + buf := &bytes.Buffer{} + appsink := app.SinkFromElement(sinkEle) + if appsink == nil { + panic("appsink should not be nil") + } + appsink.SetCallbacks(&app.SinkCallbacks{ + NewSampleFunc: WriterNewSample(ctx, buf), + EOSFunc: func(sink *app.Sink) { + resetTimer <- struct{}{} + now := time.Now().UnixMilli() + bs, err := ms.SignMP4(ctx, bytes.NewReader(buf.Bytes()), now) + if err != nil { + log.Error(ctx, "error signing segment", "error", err) + return + } + + err = mm.ValidateMP4(ctx, bytes.NewReader(bs)) + if err != nil { + log.Error(ctx, "error validating segment", "error", err) + return + } + }, + }) + }) + + return elem, nil +} diff --git a/pkg/media/segmenter_hls.go b/pkg/media/segmenter_hls.go new file mode 100644 index 00000000..5893fdb5 --- /dev/null +++ b/pkg/media/segmenter_hls.go @@ -0,0 +1,296 @@ +package media + +import ( + "bytes" + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "stream.place/streamplace/pkg/log" +) + +func (mm *MediaManager) ToHLS(ctx context.Context, user string, rendition string, m3u8 *M3U8) error { + ctx = log.WithLogValues(ctx, "GStreamerFunc", "ToHLS", "rendition", rendition) + + pipelineSlice := []string{ + "h264parse name=videoparse", + "opusdec use-inband-fec=true name=audioparse ! audioresample ! audiorate ! fdkaacenc name=audioenc", + } + + pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) + if err != nil { + return fmt.Errorf("error creating ToHLS pipeline: %w", err) + } + + outputQueue, done, err := ConcatStream(ctx, pipeline, user, rendition, mm) + if err != nil { + return fmt.Errorf("failed to get output queue: %w", err) + } + + videoParse, err := pipeline.GetElementByName("videoparse") + if err != nil { + return fmt.Errorf("failed to get video sink element from pipeline: %w", err) + } + err = outputQueue.Link(videoParse) + if err != nil { + return fmt.Errorf("failed to link output queue to video parse: %w", err) + } + + audioParse, err := pipeline.GetElementByName("audioparse") + if err != nil { + return fmt.Errorf("failed to get audio parse element from pipeline: %w", err) + } + err = outputQueue.Link(audioParse) + if err != nil { + return fmt.Errorf("failed to link output queue to audio parse: %w", err) + } + + splitmuxsink, err := gst.NewElementWithProperties("splitmuxsink", map[string]any{ + "name": "mux", + "async-finalize": true, + "sink-factory": "appsink", + "muxer-factory": "mpegtsmux", + "max-size-bytes": 1, + }) + if err != nil { + return err + } + + r := m3u8.GetRendition(rendition) + defer func() { r = nil }() + ps := NewPendingSegments(r) + defer func() { ps = nil }() + + p := splitmuxsink.GetRequestPad("video") + if p == nil { + return fmt.Errorf("failed to get video pad") + } + p = splitmuxsink.GetRequestPad("audio_%u") + if p == nil { + return fmt.Errorf("failed to get audio pad") + } + + err = pipeline.Add(splitmuxsink) + if err != nil { + return fmt.Errorf("error adding splitmuxsink to ToHLS pipeline: %w", err) + } + + videoparse, err := pipeline.GetElementByName("videoparse") + if err != nil { + return fmt.Errorf("error getting videoparse from ToHLS pipeline: %w", err) + } + err = videoparse.Link(splitmuxsink) + if err != nil { + return fmt.Errorf("error linking videoparse to splitmuxsink: %w", err) + } + + audioenc, err := pipeline.GetElementByName("audioenc") + if err != nil { + return fmt.Errorf("error getting audioenc from ToHLS pipeline: %w", err) + } + err = audioenc.Link(splitmuxsink) + if err != nil { + return fmt.Errorf("error linking audioenc to splitmuxsink: %w", err) + } + + ctx, cancel := context.WithCancel(ctx) + + go func() { + select { + case <-ctx.Done(): + return + case <-done: + cancel() + } + }() + + splitmuxsink.Connect("sink-added", func(split, sinkEle *gst.Element) { + log.Debug(ctx, "hls-check sink-added") + vf, err := ps.GetNextSegment(ctx) + if err != nil { + panic(err) + } + appsink := app.SinkFromElement(sinkEle) + appsink.SetCallbacks(&app.SinkCallbacks{ + NewSampleFunc: WriterNewSample(ctx, vf.Buf), + EOSFunc: func(sink *app.Sink) { + log.Debug(ctx, "hls-check Segment EOS", "buf", vf.Buf.Len()) + ps.CloseSegment(ctx, vf) + }, + }) + }) + + onPadAdded := func(element *gst.Element, pad *gst.Pad) { + caps := pad.GetCurrentCaps() + if caps == nil { + fmt.Println("Unable to get pad caps") + return + } + + log.Debug(ctx, "New pad added", "pad", pad.GetName(), "caps", caps.String()) + + structure := caps.GetStructureAt(0) + if structure == nil { + fmt.Println("Unable to get structure from caps") + return + } + + name := structure.Name() + fmt.Printf("Structure Name: %s\n", name) + + if name[:5] == "video" { + // Get some common video properties + // widthVal, _ := structure.GetValue("width") + // heightVal, _ := structure.GetValue("height") + + // width, ok := widthVal.(int) + // if ok { + // m3u8.Width = uint64(width) + // } + // height, ok := heightVal.(int) + // if ok { + // m3u8.Height = uint64(height) + // } + } + } + + splitmuxsink.Connect("pad-added", onPadAdded) + + defer cancel() + go func() { + HandleBusMessagesCustom(ctx, pipeline, func(msg *gst.Message) { + switch msg.Type() { + case gst.MessageElement: + structure := msg.GetStructure() + name := structure.Name() + if name == "splitmuxsink-fragment-opened" { + runningTime, err := structure.GetValue("running-time") + if err != nil { + log.Debug(ctx, "splitmuxsink-fragment-opened error", "error", err) + cancel() + } + runningTimeInt, ok := runningTime.(uint64) + if !ok { + log.Warn(ctx, "splitmuxsink-fragment-opened not a uint64") + cancel() + } + log.Debug(ctx, "hls-check splitmuxsink-fragment-opened", "runningTime", runningTimeInt) + ps.FragmentOpened(ctx, runningTimeInt) + } + if name == "splitmuxsink-fragment-closed" { + runningTime, err := structure.GetValue("running-time") + if err != nil { + log.Debug(ctx, "splitmuxsink-fragment-closed error", "error", err) + cancel() + } + runningTimeInt, ok := runningTime.(uint64) + if !ok { + log.Warn(ctx, "splitmuxsink-fragment-closed not a uint64") + cancel() + } + log.Debug(ctx, "hls-check splitmuxsink-fragment-closed", "runningTime", runningTimeInt) + ps.FragmentClosed(ctx, runningTimeInt) + } + } + }) + cancel() + }() + + // Start the pipeline + pipeline.SetState(gst.StatePlaying) + + <-ctx.Done() + + pipeline.BlockSetState(gst.StateNull) + + return nil +} + +type PendingSegments struct { + segments []*Segment + lock sync.Mutex + rendition *M3U8Rendition +} + +func NewPendingSegments(rendition *M3U8Rendition) *PendingSegments { + return &PendingSegments{ + segments: []*Segment{}, + lock: sync.Mutex{}, + rendition: rendition, + } +} + +func (ps *PendingSegments) GetNextSegment(ctx context.Context) (*Segment, error) { + ps.lock.Lock() + defer ps.lock.Unlock() + log.Debug(ctx, "next segment") + seg := &Segment{ + Buf: &bytes.Buffer{}, + Time: time.Now(), + Closed: false, + } + ps.segments = append(ps.segments, seg) + return seg, nil +} + +func (ps *PendingSegments) CloseSegment(ctx context.Context, seg *Segment) { + ps.lock.Lock() + defer ps.lock.Unlock() + log.Debug(ctx, "close segment", "MSN", seg.MSN) + seg.Closed = true + ps.checkSegments(ctx) +} + +func (ps *PendingSegments) FragmentOpened(ctx context.Context, t uint64) error { + ps.lock.Lock() + defer ps.lock.Unlock() + log.Debug(ctx, "fragment opened", "time", t) + if len(ps.segments) == 0 { + return fmt.Errorf("no pending segments") + } + for _, seg := range ps.segments { + if seg.StartTS == nil { + seg.StartTS = &t + break + } + } + ps.checkSegments(ctx) + return nil +} + +func (ps *PendingSegments) FragmentClosed(ctx context.Context, t uint64) error { + ps.lock.Lock() + defer ps.lock.Unlock() + log.Debug(ctx, "fragment closed", "time", t) + if len(ps.segments) == 0 { + return fmt.Errorf("no pending segments") + } + for _, seg := range ps.segments { + if seg.EndTS == nil { + seg.EndTS = &t + dur := *seg.EndTS - *seg.StartTS + seg.Duration = time.Duration(dur) + break + } + } + ps.checkSegments(ctx) + return nil +} + +// the tricky piece of the design here is that we need to expect GetNextSegment, +// CloseSegment, FragmentOpened, and FragmentClosed to be called in any order. So +// all of those functions call this one, and it checks if we have the necessary information +// to finalize a segment and add it to our playlist. +// only call if you're holding ps.lock! +func (ps *PendingSegments) checkSegments(ctx context.Context) { + pending := ps.segments[0] + if pending.StartTS != nil && pending.EndTS != nil && pending.Closed { + ps.rendition.NewSegment(pending) + log.Debug(ctx, "finalizing segment", "MSN", pending.MSN) + ps.segments = ps.segments[1:] + } +} diff --git a/pkg/media/thumbnail.go b/pkg/media/thumbnail.go new file mode 100644 index 00000000..16d63087 --- /dev/null +++ b/pkg/media/thumbnail.go @@ -0,0 +1,62 @@ +package media + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "stream.place/streamplace/pkg/log" +) + +func (mm *MediaManager) Thumbnail(ctx context.Context, r io.Reader, w io.Writer) error { + ctx = log.WithLogValues(ctx, "function", "Thumbnail") + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + pipelineSlice := []string{ + "appsrc name=appsrc ! qtdemux ! decodebin ! videoconvert ! videoscale ! video/x-raw,width=[1,720],height=[1,720],pixel-aspect-ratio=1/1 ! pngenc snapshot=true ! appsink name=appsink", + } + + pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) + if err != nil { + return fmt.Errorf("error creating Thumbnail pipeline: %w", err) + } + appsrc, err := pipeline.GetElementByName("appsrc") + if err != nil { + return err + } + + src := app.SrcFromElement(appsrc) + src.SetCallbacks(&app.SourceCallbacks{ + NeedDataFunc: ReaderNeedData(ctx, r), + }) + + appsink, err := pipeline.GetElementByName("appsink") + if err != nil { + return err + } + + go func() { + HandleBusMessages(ctx, pipeline) + cancel() + }() + + sink := app.SinkFromElement(appsink) + sink.SetCallbacks(&app.SinkCallbacks{ + NewSampleFunc: WriterNewSample(ctx, w), + EOSFunc: func(sink *app.Sink) { + cancel() + }, + }) + + pipeline.SetState(gst.StatePlaying) + + <-ctx.Done() + + pipeline.BlockSetState(gst.StateNull) + + return nil +} diff --git a/pkg/media/validate.go b/pkg/media/validate.go new file mode 100644 index 00000000..e7ed385b --- /dev/null +++ b/pkg/media/validate.go @@ -0,0 +1,104 @@ +package media + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + + "stream.place/streamplace/pkg/aqtime" + "stream.place/streamplace/pkg/constants" + "stream.place/streamplace/pkg/crypto/signers" + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/media/segchanman" + "stream.place/streamplace/pkg/model" + + "git.stream.place/streamplace/c2pa-go/pkg/c2pa" +) + +func (mm *MediaManager) ValidateMP4(ctx context.Context, input io.Reader) error { + buf, err := io.ReadAll(input) + if err != nil { + return err + } + r := bytes.NewReader(buf) + reader, err := c2pa.FromStream(r, "video/mp4") + if err != nil { + return err + } + mani := reader.GetActiveManifest() + certs := reader.GetProvenanceCertChain() + pub, err := signers.ParseES256KCert([]byte(certs)) + if err != nil { + return err + } + meta, err := ParseSegmentAssertions(mani) + if err != nil { + return err + } + mediaData, err := mm.ParseSegmentMediaData(ctx, buf) + if err != nil { + return err + } + // special case for test signers that are only signed with a key + var repoDID string + var signingKeyDID string + if strings.HasPrefix(meta.Creator, constants.DID_KEY_PREFIX) { + signingKeyDID = meta.Creator + repoDID = meta.Creator + } else { + repo, err := mm.atsync.SyncBlueskyRepoCached(ctx, meta.Creator, mm.model) + if err != nil { + return err + } + signingKey, err := mm.model.GetSigningKey(pub.DIDKey(), repo.DID) + if err != nil { + return err + } + if signingKey == nil { + return fmt.Errorf("no signing key found for %s", pub.DIDKey()) + } + repoDID = repo.DID + signingKeyDID = signingKey.DID + } + + err = mm.cli.StreamIsAllowed(repoDID) + if err != nil { + return fmt.Errorf("got valid segment, but user %s is not allowed: %w", repoDID, err) + } + fd, err := mm.cli.SegmentFileCreate(repoDID, meta.StartTime, "mp4") + if err != nil { + return err + } + defer fd.Close() + go mm.replicator.NewSegment(ctx, buf) + r = bytes.NewReader(buf) + io.Copy(fd, r) + scmSeg := &segchanman.Seg{ + Filepath: fd.Name(), + Data: buf, + } + go mm.PublishSegment(ctx, repoDID, "source", scmSeg) + seg := &model.Segment{ + ID: *mani.Label, + SigningKeyDID: signingKeyDID, + RepoDID: repoDID, + StartTime: meta.StartTime.Time(), + Title: meta.Title, + MediaData: mediaData, + } + mm.newSegmentSubsMutex.RLock() + defer mm.newSegmentSubsMutex.RUnlock() + not := &NewSegmentNotification{ + Segment: seg, + Data: buf, + Metadata: meta, + } + for _, ch := range mm.newSegmentSubs { + go func() { ch <- not }() + } + aqt := aqtime.FromTime(meta.StartTime.Time()) + log.Log(ctx, "successfully ingested segment", "user", repoDID, "signingKey", signingKeyDID, "timestamp", aqt.FileSafeString(), "segmentID", *mani.Label) + return nil +} diff --git a/pkg/media/webrtc.go b/pkg/media/webrtc.go index 97546afb..43c6165f 100644 --- a/pkg/media/webrtc.go +++ b/pkg/media/webrtc.go @@ -24,7 +24,7 @@ import ( var DEFAULT_DURATION = time.Duration(32 * time.Millisecond) // This function remains in scope for the duration of a single users' playback -func (mm *MediaManager) WebRTCPlayback(ctx context.Context, user string, offer *webrtc.SessionDescription) (*webrtc.SessionDescription, error) { +func (mm *MediaManager) WebRTCPlayback(ctx context.Context, user string, rendition string, offer *webrtc.SessionDescription) (*webrtc.SessionDescription, error) { uu, err := uuid.NewV7() if err != nil { return nil, err @@ -49,7 +49,7 @@ func (mm *MediaManager) WebRTCPlayback(ctx context.Context, user string, offer * cancel() }() - outputQueue, done, err := ConcatStream(ctx, pipeline, user, mm) + outputQueue, done, err := ConcatStream(ctx, pipeline, user, rendition, mm) if err != nil { return nil, fmt.Errorf("failed to get output queue: %w", err) } @@ -305,7 +305,6 @@ func (mm *MediaManager) WebRTCPlayback(ctx context.Context, user string, offer * }) <-ctx.Done() - log.Warn(ctx, "!!!!!!!!!!!!!!!!!!!!!!! ctx done") }() select { case <-gatherComplete: @@ -391,7 +390,7 @@ func (mm *MediaManager) WebRTCIngest(ctx context.Context, offer *webrtc.SessionD pipelineSlice := []string{ "multiqueue name=queue", "appsrc format=time is-live=true do-timestamp=true name=videosrc ! capsfilter caps=application/x-rtp ! rtph264depay ! capsfilter caps=video/x-h264,stream-format=byte-stream,alignment=nal ! h264parse ! h264timestamper ! identity ! queue.sink_0", - "appsrc format=time is-live=true do-timestamp=true name=audiosrc ! capsfilter caps=application/x-rtp,media=audio,encoding-name=OPUS,payload=111 ! rtpopusdepay ! queue.sink_1", + "appsrc format=time is-live=true do-timestamp=true name=audiosrc ! capsfilter caps=application/x-rtp,media=audio,encoding-name=OPUS,payload=111 ! rtpopusdepay ! opusdec use-inband-fec=true ! audiorate ! opusenc ! queue.sink_1", } pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) @@ -525,11 +524,11 @@ func (mm *MediaManager) WebRTCIngest(ctx context.Context, offer *webrtc.SessionD peerConnection.OnConnectionStateChange(func(s webrtc.PeerConnectionState) { log.Log(ctx, "Peer Connection State has changed", "state", s.String()) - if s == webrtc.PeerConnectionStateFailed { + if s == webrtc.PeerConnectionStateFailed || s == webrtc.PeerConnectionStateDisconnected { // Wait until PeerConnection has had no network activity for 30 seconds or another failure. It may be reconnected using an ICE Restart. // Use webrtc.PeerConnectionStateDisconnected if you are interested in detecting faster timeout. // Note that the PeerConnection may come back from PeerConnectionStateDisconnected. - log.Log(ctx, "Peer Connection has gone to failed exiting") + log.Log(ctx, "Peer Connection has ended, exiting", "state", s.String()) cancel() } }) @@ -614,7 +613,6 @@ func (mm *MediaManager) WebRTCIngest(ctx context.Context, offer *webrtc.SessionD }) <-ctx.Done() - log.Warn(ctx, "!!!!!!!!! context done, exiting") }() select { case <-gatherComplete: diff --git a/pkg/model/segment.go b/pkg/model/segment.go index 06d7f3ea..f6ac5171 100644 --- a/pkg/model/segment.go +++ b/pkg/model/segment.go @@ -13,9 +13,10 @@ import ( ) type SegmentMediadataVideo struct { - Width int `json:"width"` - Height int `json:"height"` - Framerate string `json:"framerate"` + Width int `json:"width"` + Height int `json:"height"` + FPSNum int `json:"fpsNum"` + FPSDen int `json:"fpsDen"` } type SegmentMediadataAudio struct { @@ -24,8 +25,9 @@ type SegmentMediadataAudio struct { } type SegmentMediaData struct { - Video []*SegmentMediadataVideo `json:"video"` - Audio []*SegmentMediadataAudio `json:"audio"` + Video []*SegmentMediadataVideo `json:"video"` + Audio []*SegmentMediadataAudio `json:"audio"` + Duration int64 `json:"duration"` } // Scan scan value into Jsonb, implements sql.Scanner interface @@ -68,17 +70,23 @@ func (s *Segment) ToStreamplaceSegment() (*streamplace.Segment, error) { if len(s.MediaData.Audio) == 0 || s.MediaData.Audio[0] == nil { return nil, fmt.Errorf("audio data is nil") } + duration := s.MediaData.Duration return &streamplace.Segment{ LexiconTypeID: "place.stream.segment", Creator: s.RepoDID, Id: s.ID, SigningKey: s.SigningKeyDID, StartTime: string(aqt), + Duration: &duration, Video: []*streamplace.Segment_Video{ { Codec: "h264", Width: int64(s.MediaData.Video[0].Width), Height: int64(s.MediaData.Video[0].Height), + Framerate: &streamplace.Segment_Framerate{ + Num: int64(s.MediaData.Video[0].FPSNum), + Den: int64(s.MediaData.Video[0].FPSDen), + }, }, }, Audio: []*streamplace.Segment_Audio{ diff --git a/pkg/renditions/renditions.go b/pkg/renditions/renditions.go new file mode 100644 index 00000000..f5b7cd12 --- /dev/null +++ b/pkg/renditions/renditions.go @@ -0,0 +1,210 @@ +package renditions + +import ( + "fmt" + "math" + + "stream.place/streamplace/pkg/streamplace" +) + +type FPS struct { + Passthrough bool + Num uint + Den uint +} + +type Rendition struct { + Width int64 + Height int64 + Bitrate int + Framerate FPS + Profile string + Name string + Parent *Rendition +} + +type JsonProfile struct { + Name string `json:"name,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + Bitrate int `json:"bitrate,omitempty"` + FPS uint `json:"fps,omitempty"` + FPSDen uint `json:"fpsDen,omitempty"` + Profile string `json:"profile,omitempty"` + GOP string `json:"gop,omitempty"` + Encoder string `json:"encoder,omitempty"` + Quality uint `json:"quality,omitempty"` +} + +func (r Rendition) ToLivepeerProfile() JsonProfile { + p := JsonProfile{ + Name: r.Name, + Bitrate: r.Bitrate, + FPS: r.Framerate.Num, + FPSDen: r.Framerate.Den, + Profile: r.Profile, + } + if r.Parent == nil { + p.Width = int(r.Width) + p.Height = int(r.Height) + } else { + // We want to set the dimension that is the same as the parent + if r.Width < r.Height { + if r.Parent.Width == r.Height { + p.Height = int(r.Parent.Width) + } else { + p.Width = int(r.Parent.Height) + } + } else { + if r.Parent.Height == r.Height { + p.Height = int(r.Parent.Height) + } else { + p.Width = int(r.Parent.Width) + } + } + } + return p +} + +type Renditions []Rendition + +func (rs Renditions) ToLivepeerProfiles() []JsonProfile { + profiles := make([]JsonProfile, len(rs)) + for i, r := range rs { + profiles[i] = r.ToLivepeerProfile() + } + return profiles +} + +var DesiredRenditions = []Rendition{ + { + Name: "1080p", + Width: 1920, + Height: 1080, + Bitrate: 6_000_000, + Framerate: FPS{ + Num: 60, + Den: 1, + }, + Profile: "h264constrainedhigh", + }, + { + Name: "720p", + Width: 1280, + Height: 720, + Bitrate: 3_000_000, + Framerate: FPS{ + Num: 60, + Den: 1, + }, + Profile: "h264constrainedhigh", + }, + { + Name: "360p", + Width: 640, + Height: 360, + Bitrate: 1_000_000, + Framerate: FPS{ + Num: 30, + Den: 1, + }, + Profile: "h264constrainedhigh", + }, + { + Name: "240p", + Width: 426, + Height: 240, + Bitrate: 500_000, + Framerate: FPS{ + Num: 30, + Den: 1, + }, + Profile: "h264constrainedhigh", + }, + { + Name: "160p", + Width: 284, + Height: 160, + Bitrate: 250_000, + Framerate: FPS{ + Num: 30, + Den: 1, + }, + Profile: "h264baseline", + }, +} + +// GenerateRenditions generates renditions for a given spseg +func GenerateRenditions(spseg *streamplace.Segment) (Renditions, error) { + vid := spseg.Video[0] + if vid == nil { + return nil, fmt.Errorf("no video stream found") + } + rs := []Rendition{} + for _, r := range DesiredRenditions { + vidWidth := int64(vid.Width) + vidHeight := int64(vid.Height) + vertical := vid.Height > vid.Width + // do all the math as if it's horizontal then flip at the end + if vertical { + vidWidth, vidHeight = vidHeight, vidWidth + } + if vidWidth <= r.Width && vidHeight <= r.Height { + continue + } + rAspectRatio := float64(r.Width) / float64(r.Height) + vidAspectRatio := float64(vidWidth) / float64(vidHeight) + if vidAspectRatio > rAspectRatio { + // vid is wider than r + // scale down to r.Width + scale := float64(r.Width) / float64(vidWidth) + vidWidth = r.Width + vidHeight = int64(math.Round(float64(vidHeight) * scale)) + } else { + // vid is taller than r + // scale down to r.Height + scale := float64(r.Height) / float64(vidHeight) + vidHeight = r.Height + vidWidth = int64(math.Round(float64(vidWidth) * scale)) + } + outR := Rendition{ + Name: r.Name, + Parent: &r, + Profile: r.Profile, + } + if vertical { + outR.Width = vidHeight + outR.Height = vidWidth + } else { + outR.Width = vidWidth + outR.Height = vidHeight + } + + // if vertical { + // ratio := float64(r.Height) / float64(vid.Height) + // outR.Height = int64(float64(vid.Width) * (16.0 / 9.0) * ratio) + // outR.Width = r.Height + // } else { + // ratio := float64(r.Width) / float64(vid.Width) + // outR.Width = r.Width + // outR.Height = int64(float64(vid.Width) * (9.0 / 16.0) * ratio) + // } + if vid.Framerate.Den > 0 { + vidFPS := float64(vid.Framerate.Num) / float64(vid.Framerate.Den) + rFPS := float64(r.Framerate.Num) / float64(r.Framerate.Den) + delta := rFPS / vidFPS + + if rFPS < vidFPS { + if delta < 0.75 { + outR.Framerate.Num = uint(vid.Framerate.Num) + outR.Framerate.Den = uint(vid.Framerate.Den * 2) + } + } + } + + outR.Bitrate = r.Bitrate + outR.Profile = r.Profile + rs = append(rs, outR) + } + return rs, nil +} diff --git a/pkg/renditions/renditions_test.go b/pkg/renditions/renditions_test.go new file mode 100644 index 00000000..480d1171 --- /dev/null +++ b/pkg/renditions/renditions_test.go @@ -0,0 +1,323 @@ +package renditions + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/streamplace" +) + +func seg(width int, height int, fpsNum int, fpsDen int) *streamplace.Segment { + return &streamplace.Segment{ + Video: []*streamplace.Segment_Video{ + { + Width: int64(width), + Height: int64(height), + Framerate: &streamplace.Segment_Framerate{ + Num: int64(fpsNum), + Den: int64(fpsDen), + }, + }, + }, + } +} + +var cases = []struct { + name string + spseg *streamplace.Segment + lp string +}{ + { + name: "4K 60fps", + spseg: seg(3840, 2160, 60, 1), + lp: ` + [ + { + "name": "1080p", + "height": 1080, + "bitrate": 6000000, + "profile": "h264constrainedhigh" + }, + { + "name": "720p", + "height": 720, + "bitrate": 3000000, + "profile": "h264constrainedhigh" + }, + { + "name": "360p", + "height": 360, + "bitrate": 1000000, + "profile": "h264constrainedhigh", + "fps": 60, + "fpsDen": 2 + }, + { + "name": "240p", + "height": 240, + "bitrate": 500000, + "profile": "h264constrainedhigh", + "fps": 60, + "fpsDen": 2 + }, + { + "name": "160p", + "height": 160, + "bitrate": 250000, + "profile": "h264baseline", + "fps": 60, + "fpsDen": 2 + } + ] + `, + }, + { + name: "2K with fractional framerate", + spseg: seg(2160, 1440, 60000, 1001), + lp: ` + [ + { + "name": "1080p", + "height": 1080, + "bitrate": 6000000, + "profile": "h264constrainedhigh" + }, + { + "name": "720p", + "height": 720, + "bitrate": 3000000, + "profile": "h264constrainedhigh" + }, + { + "name": "360p", + "height": 360, + "bitrate": 1000000, + "profile": "h264constrainedhigh", + "fps": 60000, + "fpsDen": 2002 + }, + { + "name": "240p", + "height": 240, + "bitrate": 500000, + "profile": "h264constrainedhigh", + "fps": 60000, + "fpsDen": 2002 + }, + { + "name": "160p", + "height": 160, + "bitrate": 250000, + "profile": "h264baseline", + "fps": 60000, + "fpsDen": 2002 + } + ] + `, + }, + { + name: "720p 50fps", + spseg: seg(1280, 720, 50, 1), + lp: ` + [ + { + "name": "360p", + "height": 360, + "bitrate": 1000000, + "profile": "h264constrainedhigh", + "fps": 50, + "fpsDen": 2 + }, + { + "name": "240p", + "height": 240, + "bitrate": 500000, + "profile": "h264constrainedhigh", + "fps": 50, + "fpsDen": 2 + }, + { + "name": "160p", + "height": 160, + "bitrate": 250000, + "profile": "h264baseline", + "fps": 50, + "fpsDen": 2 + } + ] + `, + }, + { + name: "720p 30fps", + spseg: seg(1280, 720, 30, 1), + lp: ` + [ + { + "name": "360p", + "height": 360, + "bitrate": 1000000, + "profile": "h264constrainedhigh" + }, + { + "name": "240p", + "height": 240, + "bitrate": 500000, + "profile": "h264constrainedhigh" + }, + { + "name": "160p", + "height": 160, + "bitrate": 250000, + "profile": "h264baseline" + } + ] + `, + }, + { + name: "720p 25fps", + spseg: seg(1280, 720, 25, 1), + lp: ` + [ + { + "name": "360p", + "height": 360, + "bitrate": 1000000, + "profile": "h264constrainedhigh" + }, + { + "name": "240p", + "height": 240, + "bitrate": 500000, + "profile": "h264constrainedhigh" + }, + { + "name": "160p", + "height": 160, + "bitrate": 250000, + "profile": "h264baseline" + } + ] + `, + }, + { + name: "Vertical video 60fps", + spseg: seg(480, 640, 60, 1), + lp: ` + [ + { + "name": "360p", + "width": 360, + "bitrate": 1000000, + "profile": "h264constrainedhigh", + "fps": 60, + "fpsDen": 2 + }, + { + "name": "240p", + "width": 240, + "bitrate": 500000, + "profile": "h264constrainedhigh", + "fps": 60, + "fpsDen": 2 + }, + { + "name": "160p", + "width": 160, + "bitrate": 250000, + "profile": "h264baseline", + "fps": 60, + "fpsDen": 2 + } + ] + `, + }, +} + +func TestRenditions(t *testing.T) { + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + rends, err := GenerateRenditions(c.spseg) + require.NoError(t, err) + lp := rends.ToLivepeerProfiles() + bs, err := json.Marshal(lp) + require.NoError(t, err) + require.JSONEq(t, c.lp, string(bs)) + }) + } +} + +var singleCases = []struct { + name string + spseg *streamplace.Segment + lp string + dimensions []int +}{ + { + name: "Nearly-Square Landscape 4K 60fps", + spseg: seg(3840, 3830, 60, 1), + dimensions: []int{1083, 1080}, + lp: ` + { + "name": "1080p", + "height": 1080, + "bitrate": 6000000, + "profile": "h264constrainedhigh" + } + `, + }, + { + name: "Nearly-Square Portrait 4K 60fps", + spseg: seg(3830, 3840, 60, 1), + dimensions: []int{1080, 1083}, + lp: ` + { + "name": "1080p", + "width": 1080, + "bitrate": 6000000, + "profile": "h264constrainedhigh" + } + `, + }, + { + name: "Stupidly-Wide Landscape 4K 60fps", + spseg: seg(5000, 2160, 60, 1), + dimensions: []int{1920, 829}, + lp: ` + { + "name": "1080p", + "width": 1920, + "bitrate": 6000000, + "profile": "h264constrainedhigh" + } + `, + }, + { + name: "Stupidly-Tall Portrait 4K 60fps", + spseg: seg(2160, 5000, 60, 1), + dimensions: []int{829, 1920}, + lp: ` + { + "name": "1080p", + "height": 1920, + "bitrate": 6000000, + "profile": "h264constrainedhigh" + } + `, + }, +} + +func TestSingleRendition(t *testing.T) { + for _, c := range singleCases { + t.Run(c.name, func(t *testing.T) { + rends, err := GenerateRenditions(c.spseg) + require.NoError(t, err) + first := rends[0] + require.Equal(t, c.dimensions, []int{int(first.Width), int(first.Height)}) + lp := rends.ToLivepeerProfiles() + bs, err := json.Marshal(lp[0]) + require.NoError(t, err) + require.JSONEq(t, c.lp, string(bs)) + }) + } +} diff --git a/pkg/spmetrics/spmetrics.go b/pkg/spmetrics/spmetrics.go index d3e7147b..e426a13a 100644 --- a/pkg/spmetrics/spmetrics.go +++ b/pkg/spmetrics/spmetrics.go @@ -27,6 +27,26 @@ var ViewersTotal = promauto.NewGauge(prometheus.GaugeOpts{ Help: "total number of viewers", }) +var TranscodeAttemptsTotal = promauto.NewCounter(prometheus.CounterOpts{ + Name: "streamplace_transcode_attempts_total", + Help: "total number of transcode attempts", +}) + +var TranscodeSuccessesTotal = promauto.NewCounter(prometheus.CounterOpts{ + Name: "streamplace_transcode_successes_total", + Help: "total number of transcode successes", +}) + +var TranscodeErrorsTotal = promauto.NewCounter(prometheus.CounterOpts{ + Name: "streamplace_transcode_errors_total", + Help: "total number of transcode errors", +}) + +var Version = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "streamplace_version", + Help: "version of streamplace", +}, []string{"version"}) + func ViewerInc(user string) { go func() { viewersLock.Lock() diff --git a/pkg/streamplace/cbor_gen.go b/pkg/streamplace/cbor_gen.go index e67df689..87b9e711 100644 --- a/pkg/streamplace/cbor_gen.go +++ b/pkg/streamplace/cbor_gen.go @@ -456,12 +456,16 @@ func (t *Segment) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) - fieldCount := 7 + fieldCount := 8 if t.Audio == nil { fieldCount-- } + if t.Duration == nil { + fieldCount-- + } + if t.Video == nil { fieldCount-- } @@ -593,6 +597,38 @@ func (t *Segment) MarshalCBOR(w io.Writer) error { return err } + // t.Duration (int64) (int64) + if t.Duration != nil { + + if len("duration") > 1000000 { + return xerrors.Errorf("Value in field \"duration\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("duration"))); err != nil { + return err + } + if _, err := cw.WriteString(string("duration")); err != nil { + return err + } + + if t.Duration == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if *t.Duration >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(*t.Duration)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-*t.Duration-1)); err != nil { + return err + } + } + } + + } + // t.StartTime (string) (string) if len("startTime") > 1000000 { return xerrors.Errorf("Value in field \"startTime\" was too long") @@ -813,6 +849,42 @@ func (t *Segment) UnmarshalCBOR(r io.Reader) (err error) { t.Creator = string(sval) } + // t.Duration (int64) (int64) + case "duration": + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Duration = (*int64)(&extraI) + } + } // t.StartTime (string) (string) case "startTime": @@ -1050,8 +1122,13 @@ func (t *Segment_Video) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) + fieldCount := 4 - if _, err := cw.Write([]byte{163}); err != nil { + if t.Framerate == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } @@ -1122,6 +1199,24 @@ func (t *Segment_Video) MarshalCBOR(w io.Writer) error { } } + // t.Framerate (streamplace.Segment_Framerate) (struct) + if t.Framerate != nil { + + if len("framerate") > 1000000 { + return xerrors.Errorf("Value in field \"framerate\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("framerate"))); err != nil { + return err + } + if _, err := cw.WriteString(string("framerate")); err != nil { + return err + } + + if err := t.Framerate.MarshalCBOR(cw); err != nil { + return err + } + } return nil } @@ -1150,7 +1245,7 @@ func (t *Segment_Video) UnmarshalCBOR(r io.Reader) (err error) { n := extra - nameBuf := make([]byte, 6) + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { @@ -1229,6 +1324,189 @@ func (t *Segment_Video) UnmarshalCBOR(r io.Reader) (err error) { t.Height = int64(extraI) } + // t.Framerate (streamplace.Segment_Framerate) (struct) + case "framerate": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Framerate = new(Segment_Framerate) + if err := t.Framerate.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Framerate pointer: %w", err) + } + } + + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *Segment_Framerate) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Den (int64) (int64) + if len("den") > 1000000 { + return xerrors.Errorf("Value in field \"den\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("den"))); err != nil { + return err + } + if _, err := cw.WriteString(string("den")); err != nil { + return err + } + + if t.Den >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Den)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Den-1)); err != nil { + return err + } + } + + // t.Num (int64) (int64) + if len("num") > 1000000 { + return xerrors.Errorf("Value in field \"num\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("num"))); err != nil { + return err + } + if _, err := cw.WriteString(string("num")); err != nil { + return err + } + + if t.Num >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Num)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Num-1)); err != nil { + return err + } + } + + return nil +} + +func (t *Segment_Framerate) UnmarshalCBOR(r io.Reader) (err error) { + *t = Segment_Framerate{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("Segment_Framerate: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 3) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Den (int64) (int64) + case "den": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Den = int64(extraI) + } + // t.Num (int64) (int64) + case "num": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Num = int64(extraI) + } default: // Field doesn't exist on this type, so ignore it diff --git a/pkg/streamplace/streamdefs.go b/pkg/streamplace/streamdefs.go index 977ff61a..954479a2 100644 --- a/pkg/streamplace/streamdefs.go +++ b/pkg/streamplace/streamdefs.go @@ -19,3 +19,19 @@ type Defs_BlockView struct { Record *appbskytypes.GraphBlock `json:"record" cborgen:"record"` Uri string `json:"uri" cborgen:"uri"` } + +// Defs_Rendition is a "rendition" in the place.stream.defs schema. +// +// RECORDTYPE: Defs_Rendition +type Defs_Rendition struct { + LexiconTypeID string `json:"$type,const=place.stream.defs#rendition" cborgen:"$type,const=place.stream.defs#rendition"` + Name string `json:"name" cborgen:"name"` +} + +// Defs_Renditions is a "renditions" in the place.stream.defs schema. +// +// RECORDTYPE: Defs_Renditions +type Defs_Renditions struct { + LexiconTypeID string `json:"$type,const=place.stream.defs#renditions" cborgen:"$type,const=place.stream.defs#renditions"` + Renditions []*Defs_Rendition `json:"renditions" cborgen:"renditions"` +} diff --git a/pkg/streamplace/streamlivestream.go b/pkg/streamplace/streamlivestream.go index ea9477bc..51a477a1 100644 --- a/pkg/streamplace/streamlivestream.go +++ b/pkg/streamplace/streamlivestream.go @@ -50,6 +50,8 @@ type Livestream_StreamplaceAnything_Livestream struct { Livestream_LivestreamView *Livestream_LivestreamView Livestream_ViewerCount *Livestream_ViewerCount Defs_BlockView *Defs_BlockView + Defs_Renditions *Defs_Renditions + Defs_Rendition *Defs_Rendition ChatDefs_MessageView *ChatDefs_MessageView } @@ -66,6 +68,14 @@ func (t *Livestream_StreamplaceAnything_Livestream) MarshalJSON() ([]byte, error t.Defs_BlockView.LexiconTypeID = "place.stream.defs#blockView" return json.Marshal(t.Defs_BlockView) } + if t.Defs_Renditions != nil { + t.Defs_Renditions.LexiconTypeID = "place.stream.defs#renditions" + return json.Marshal(t.Defs_Renditions) + } + if t.Defs_Rendition != nil { + t.Defs_Rendition.LexiconTypeID = "place.stream.defs#rendition" + return json.Marshal(t.Defs_Rendition) + } if t.ChatDefs_MessageView != nil { t.ChatDefs_MessageView.LexiconTypeID = "place.stream.chat.defs#messageView" return json.Marshal(t.ChatDefs_MessageView) @@ -88,6 +98,12 @@ func (t *Livestream_StreamplaceAnything_Livestream) UnmarshalJSON(b []byte) erro case "place.stream.defs#blockView": t.Defs_BlockView = new(Defs_BlockView) return json.Unmarshal(b, t.Defs_BlockView) + case "place.stream.defs#renditions": + t.Defs_Renditions = new(Defs_Renditions) + return json.Unmarshal(b, t.Defs_Renditions) + case "place.stream.defs#rendition": + t.Defs_Rendition = new(Defs_Rendition) + return json.Unmarshal(b, t.Defs_Rendition) case "place.stream.chat.defs#messageView": t.ChatDefs_MessageView = new(ChatDefs_MessageView) return json.Unmarshal(b, t.ChatDefs_MessageView) diff --git a/pkg/streamplace/streamsegment.go b/pkg/streamplace/streamsegment.go index 0a52c0a3..f956fab6 100644 --- a/pkg/streamplace/streamsegment.go +++ b/pkg/streamplace/streamsegment.go @@ -16,6 +16,8 @@ type Segment struct { LexiconTypeID string `json:"$type,const=place.stream.segment" cborgen:"$type,const=place.stream.segment"` Audio []*Segment_Audio `json:"audio,omitempty" cborgen:"audio,omitempty"` Creator string `json:"creator" cborgen:"creator"` + // duration: The duration of the segment in nanoseconds + Duration *int64 `json:"duration,omitempty" cborgen:"duration,omitempty"` // id: Unique identifier for the segment Id string `json:"id" cborgen:"id"` // signingKey: The DID of the signing key used for this segment @@ -32,9 +34,16 @@ type Segment_Audio struct { Rate int64 `json:"rate" cborgen:"rate"` } +// Segment_Framerate is a "framerate" in the place.stream.segment schema. +type Segment_Framerate struct { + Den int64 `json:"den" cborgen:"den"` + Num int64 `json:"num" cborgen:"num"` +} + // Segment_Video is a "video" in the place.stream.segment schema. type Segment_Video struct { - Codec string `json:"codec" cborgen:"codec"` - Height int64 `json:"height" cborgen:"height"` - Width int64 `json:"width" cborgen:"width"` + Codec string `json:"codec" cborgen:"codec"` + Framerate *Segment_Framerate `json:"framerate,omitempty" cborgen:"framerate,omitempty"` + Height int64 `json:"height" cborgen:"height"` + Width int64 `json:"width" cborgen:"width"` } -- 2.51.2