From a79ffc5bd0a67201a1b00df3a6061f1c4289bd5e Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 1 Jun 2026 18:16:17 -0700 Subject: [PATCH 01/17] media: isolate MKV/RTMP-push ingest in a worker subprocess (opt-in) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of per-stream process isolation: a single failing stream should never take down the node. Native faults in the ingest pipeline (a gst plugin segfault, an OOM, a deadlock) aren't recoverable in-process, so each MKV/RTMP push can run in its own `streamplace ingest-worker` subprocess that owns the pipeline and streams signed canonical .m4s segments back; the main process only reads frames and runs ValidateMP4 (memory-safe Go + wasm). Wire protocol (pkg/ingestframe): typed, length-prefixed, transport-agnostic frames (Segment/End/Error). A clean end is an End frame then EOF; a crash tears off mid-frame (io.ErrUnexpectedEOF) — that's how the supervisor tells "stream ended" from "worker died". Rides stdout today, a unix socket later (the zero-downtime detach/reattach path) with the same bytes. Worker (RunMKVIngestWorker): reads MKV from stdin, runs the shared buildMKVIngestPipeline (extracted from MKVIngest) into muxlSignSegmentElem — the new done-channel core of MuxlSignSegmentElem, parameterized by a SignSegmentStreamFunc so the worker forwards the streamer key PEM straight to muxl-sign without a MediaSigner/model. Per the locked design the worker signs everything; main hands it the key + cert + a once-built manifest over a dedicated config fd (off argv/env). Frames go out on a dedicated fd too, so stray stdout/stderr can't corrupt them. Supervisor (MKVIngestIsolated): re-execs the worker (the proc.RunMistServer pattern), pumps the body to its stdin, reads frames into the existing ValidateMP4 chokepoint, forwards stdout+stderr to the logger, and maps exit/EOF to clean-vs-crash. Opt-in via --isolated-ingest (SP_ISOLATED_INGEST); the in-process path stays the default. Tests: ingestframe round-trip / truncation / oversize / concurrency; the worker emits valid signed segments in-process; and a real-subprocess test (TestMain re-exec helper) proves fd passing, the framed wire protocol over a real pipe, a clean End frame, and a zero exit. The clean H264+AAC MKV the test needs is remuxed in-process from 5sec.mp4 (the only AAC fixture carries four audio tracks that wedge matroskademux — itself a node-killer this isolation contains). Co-Authored-By: Claude Opus 4.8 --- pkg/api/api_internal.go | 6 +- pkg/cmd/streamplace.go | 46 ++++++ pkg/config/config.go | 8 ++ pkg/ingestframe/frame.go | 147 +++++++++++++++++++ pkg/ingestframe/frame_test.go | 150 ++++++++++++++++++++ pkg/media/ingest_subprocess_test.go | 126 +++++++++++++++++ pkg/media/ingest_supervisor.go | 209 ++++++++++++++++++++++++++++ pkg/media/ingest_worker.go | 99 +++++++++++++ pkg/media/ingest_worker_test.go | 102 ++++++++++++++ pkg/media/leak_test.go | 6 + pkg/media/mkv_ingest.go | 93 +++++++------ pkg/media/muxl_segment.go | 46 ++++-- 12 files changed, 978 insertions(+), 60 deletions(-) create mode 100644 pkg/ingestframe/frame.go create mode 100644 pkg/ingestframe/frame_test.go create mode 100644 pkg/media/ingest_subprocess_test.go create mode 100644 pkg/media/ingest_supervisor.go create mode 100644 pkg/media/ingest_worker.go create mode 100644 pkg/media/ingest_worker_test.go diff --git a/pkg/api/api_internal.go b/pkg/api/api_internal.go index 84e203d1..f3f5a529 100644 --- a/pkg/api/api_internal.go +++ b/pkg/api/api_internal.go @@ -262,7 +262,11 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err return } - err = a.MediaManager.MKVIngest(reqCtx, r, mediaSigner) + if a.CLI.IsolatedIngest { + err = a.MediaManager.MKVIngestIsolated(reqCtx, r, mediaSigner) + } else { + err = a.MediaManager.MKVIngest(reqCtx, r, mediaSigner) + } if err != nil { log.Log(reqCtx, "stream error", "error", err) diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index e4393ee9..99dae045 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -4,9 +4,11 @@ import ( "bytes" "context" "crypto/rand" + "encoding/json" "errors" "flag" "fmt" + "io" "net/url" "os" "os/signal" @@ -30,6 +32,7 @@ import ( "stream.place/streamplace/pkg/bus" "stream.place/streamplace/pkg/director" "stream.place/streamplace/pkg/gstinit" + "stream.place/streamplace/pkg/ingestframe" "stream.place/streamplace/pkg/iroh/generated/iroh_streamplace" "stream.place/streamplace/pkg/localdb" "stream.place/streamplace/pkg/log" @@ -73,6 +76,7 @@ func start(build *config.BuildFlags, platformJobs []jobFunc) error { makeSelfTestCommand(build), makeVODTestCommand(build), makeStreamCommand(build), + makeIngestWorkerCommand(build), makeLiveCommand(build), makeWhepCommand(build), makeWhipCommand(build), @@ -834,6 +838,48 @@ func makeStreamCommand(build *config.BuildFlags) *urfavecli.Command { } } +// makeIngestWorkerCommand is the per-stream isolated ingest worker (Stage 1: +// MKV/RTMP push). The node spawns it; it is not meant for direct use. It reads +// the config handshake from fd 3, the MKV media from stdin, runs the mux + sign +// pipeline, and writes signed canonical .m4s frames to fd 4 — dedicated fds so +// stray stdout/stderr can't corrupt the frame stream. A clean run ends with an +// End frame; a fatal error emits an Error frame before exiting non-zero. +func makeIngestWorkerCommand(build *config.BuildFlags) *urfavecli.Command { + return &urfavecli.Command{ + Name: "ingest-worker", + Usage: "internal: per-stream isolated ingest worker (spawned by the node)", + Hidden: true, + Action: func(ctx context.Context, cmd *urfavecli.Command) error { + cfgFile := os.NewFile(3, "ingest-config") + if cfgFile == nil { + return fmt.Errorf("ingest-worker: missing config fd 3") + } + cfgBytes, err := io.ReadAll(cfgFile) + cfgFile.Close() + if err != nil { + return fmt.Errorf("ingest-worker: read config: %w", err) + } + var cfg media.IngestWorkerConfig + if err := json.Unmarshal(cfgBytes, &cfg); err != nil { + return fmt.Errorf("ingest-worker: parse config: %w", err) + } + + framesFile := os.NewFile(4, "ingest-frames") + if framesFile == nil { + return fmt.Errorf("ingest-worker: missing frames fd 4") + } + defer framesFile.Close() + frames := ingestframe.NewWriter(framesFile) + + if err := media.RunMKVIngestWorker(ctx, cfg, os.Stdin, frames); err != nil { + _ = frames.Error(err.Error()) + return err + } + return frames.End() + }, + } +} + func makeLiveCommand(build *config.BuildFlags) *urfavecli.Command { cli := config.CLI{Build: build} liveCmd := cli.NewCommand("live") diff --git a/pkg/config/config.go b/pkg/config/config.go index 6c50de9e..1a0ffecc 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -75,6 +75,7 @@ type CLI struct { RTMPSAddonAddr string Secure bool NoMist bool + IsolatedIngest bool MistAdminPort int MistHTTPPort int MistRTMPPort int @@ -228,6 +229,13 @@ func (cli *CLI) NewCommand(name string) *urfavecli.Command { Destination: &cli.Secure, Sources: urfavecli.EnvVars("SP_SECURE"), }, + &urfavecli.BoolFlag{ + Name: "isolated-ingest", + Usage: "Run each MKV/RTMP-push ingest in an isolated worker subprocess (fault isolation)", + Value: false, + Destination: &cli.IsolatedIngest, + Sources: urfavecli.EnvVars("SP_ISOLATED_INGEST"), + }, &urfavecli.StringFlag{ Name: "tls-cert", Usage: fmt.Sprintf(`Path to TLS certificate (default: "%s")`, filepath.Join(SPDataDir, "tls", "tls.crt")), diff --git a/pkg/ingestframe/frame.go b/pkg/ingestframe/frame.go new file mode 100644 index 00000000..31f16d39 --- /dev/null +++ b/pkg/ingestframe/frame.go @@ -0,0 +1,147 @@ +// Package ingestframe defines the wire protocol a per-stream ingest worker uses +// to stream canonical MUXL fragments back to the main streamplace process. +// +// Each incoming live stream is handled by an isolated worker subprocess that +// owns the socket, muxes + transcodes the media, and signs each GoP. It emits +// the resulting signed canonical .m4s segments to the main process as a sequence +// of typed, length-prefixed frames. +// +// The framing is deliberately transport-agnostic: today it rides the worker's +// stdout pipe, but a detached / reattachable worker (the zero-downtime-upgrade +// path, where workers keep buffering signed segments across a main restart) can +// carry the identical frames over a unix socket. Nothing above this package +// cares which. +package ingestframe + +import ( + "encoding/binary" + "fmt" + "io" + "sync" +) + +// Type identifies a frame's payload. +type Type uint8 + +const ( + // Segment carries one signed canonical .m4s segment — the unit main's + // ValidateMP4 ingests. Payload: the bare canonical segment bytes. + Segment Type = 1 + // End signals the worker finished the stream cleanly (graceful EOS). No + // payload. Its ABSENCE before EOF is how main tells a crash from a clean end. + End Type = 2 + // Error carries a worker-side fatal error message (UTF-8). The worker emits + // it just before exiting so main can log a cause, not a bare "worker exited". + Error Type = 3 +) + +func (t Type) String() string { + switch t { + case Segment: + return "segment" + case End: + return "end" + case Error: + return "error" + default: + return fmt.Sprintf("unknown(%d)", uint8(t)) + } +} + +// magic prefixes every frame so a desynced/corrupt stream is caught immediately +// rather than mis-parsed as a length. +var magic = [4]byte{'S', 'P', 'F', '1'} + +// MaxPayload bounds a single frame so a corrupt or hostile length can't make the +// reader allocate unboundedly. Canonical GoP segments are well under this. +const MaxPayload = 64 << 20 // 64 MiB + +const headerSize = 4 + 1 + 4 // magic + type + uint32 length + +// Writer serializes frames to an underlying stream. Safe for concurrent use: a +// worker emits segments from more than one goroutine (the source signer and the +// transcoder's completion callback), and frames must never interleave. +type Writer struct { + mu sync.Mutex + w io.Writer +} + +// NewWriter wraps w. w is typically the worker's os.Stdout. +func NewWriter(w io.Writer) *Writer { + return &Writer{w: w} +} + +// WriteFrame writes one whole frame atomically with respect to other WriteFrame +// calls on the same Writer. +func (fw *Writer) WriteFrame(t Type, payload []byte) error { + if len(payload) > MaxPayload { + return fmt.Errorf("ingestframe: payload %d exceeds max %d", len(payload), MaxPayload) + } + var hdr [headerSize]byte + copy(hdr[0:4], magic[:]) + hdr[4] = byte(t) + binary.BigEndian.PutUint32(hdr[5:9], uint32(len(payload))) + + fw.mu.Lock() + defer fw.mu.Unlock() + if _, err := fw.w.Write(hdr[:]); err != nil { + return err + } + if len(payload) > 0 { + if _, err := fw.w.Write(payload); err != nil { + return err + } + } + return nil +} + +// Segment frames a signed canonical .m4s segment. +func (fw *Writer) Segment(seg []byte) error { return fw.WriteFrame(Segment, seg) } + +// End frames a clean end-of-stream marker. +func (fw *Writer) End() error { return fw.WriteFrame(End, nil) } + +// Error frames a fatal worker-side error message. +func (fw *Writer) Error(msg string) error { return fw.WriteFrame(Error, []byte(msg)) } + +// Reader decodes frames from an underlying stream. +type Reader struct { + r io.Reader +} + +// NewReader wraps r, typically the worker's stdout pipe. +func NewReader(r io.Reader) *Reader { + return &Reader{r: r} +} + +// ReadFrame decodes the next frame. It returns io.EOF only at a clean frame +// boundary (the stream ended between frames); a stream that dies mid-frame +// surfaces as io.ErrUnexpectedEOF, so an abrupt worker death is distinguishable +// from a clean close. +func (fr *Reader) ReadFrame() (Type, []byte, error) { + var hdr [headerSize]byte + if _, err := io.ReadFull(fr.r, hdr[:]); err != nil { + // io.EOF here = clean boundary. io.ReadFull maps a partial read to + // ErrUnexpectedEOF, which we keep: a torn header is an abrupt death. + return 0, nil, err + } + if [4]byte(hdr[0:4]) != magic { + return 0, nil, fmt.Errorf("ingestframe: bad magic %q (stream desynced)", hdr[0:4]) + } + t := Type(hdr[4]) + n := binary.BigEndian.Uint32(hdr[5:9]) + if n > MaxPayload { + return 0, nil, fmt.Errorf("ingestframe: frame length %d exceeds max %d", n, MaxPayload) + } + if n == 0 { + return t, nil, nil + } + payload := make([]byte, n) + if _, err := io.ReadFull(fr.r, payload); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return 0, nil, err + } + return t, payload, nil +} diff --git a/pkg/ingestframe/frame_test.go b/pkg/ingestframe/frame_test.go new file mode 100644 index 00000000..70881fcf --- /dev/null +++ b/pkg/ingestframe/frame_test.go @@ -0,0 +1,150 @@ +package ingestframe + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "sort" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestRoundTrip writes a mix of frame types/sizes and reads them back verbatim, +// then confirms a clean EOF at the boundary after the last frame. +func TestRoundTrip(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(&buf) + + big := bytes.Repeat([]byte{0xAB}, 500_000) + require.NoError(t, w.Segment([]byte("seg-one"))) + require.NoError(t, w.Segment(nil)) // zero-length segment is legal + require.NoError(t, w.Segment(big)) + require.NoError(t, w.Error("something broke")) + require.NoError(t, w.End()) + + r := NewReader(&buf) + + assertFrame := func(wantT Type, wantPayload []byte) { + t.Helper() + gotT, got, err := r.ReadFrame() + require.NoError(t, err) + require.Equal(t, wantT, gotT) + require.Equal(t, wantPayload, got) + } + assertFrame(Segment, []byte("seg-one")) + assertFrame(Segment, nil) + assertFrame(Segment, big) + assertFrame(Error, []byte("something broke")) + assertFrame(End, nil) + + // Clean boundary after the last frame. + _, _, err := r.ReadFrame() + require.ErrorIs(t, err, io.EOF) +} + +// TestTruncatedFrameIsUnexpectedEOF is the crash-vs-clean-end distinction the +// supervisor relies on: a worker that dies mid-segment must NOT look like a +// graceful end. +func TestTruncatedFrameIsUnexpectedEOF(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, NewWriter(&buf).Segment(bytes.Repeat([]byte{1}, 1000))) + + // Lop off the back half of the payload — an abrupt death mid-frame. + full := buf.Bytes() + torn := full[:len(full)-400] + + _, _, err := NewReader(bytes.NewReader(torn)).ReadFrame() + require.ErrorIs(t, err, io.ErrUnexpectedEOF) +} + +// TestTornHeaderIsUnexpectedEOF: dying partway through the header is also an +// abrupt death, not a clean boundary. +func TestTornHeaderIsUnexpectedEOF(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, NewWriter(&buf).End()) + torn := buf.Bytes()[:headerSize-2] + + _, _, err := NewReader(bytes.NewReader(torn)).ReadFrame() + require.ErrorIs(t, err, io.ErrUnexpectedEOF) +} + +// TestBadMagicRejected: a desynced/corrupt stream is caught, not mis-parsed. +func TestBadMagicRejected(t *testing.T) { + junk := append([]byte("XXXX"), make([]byte, headerSize)...) + _, _, err := NewReader(bytes.NewReader(junk)).ReadFrame() + require.Error(t, err) + require.Contains(t, err.Error(), "bad magic") +} + +// TestOversizeLengthRejected: a hostile length can't trigger an unbounded alloc. +func TestOversizeLengthRejected(t *testing.T) { + var hdr [headerSize]byte + copy(hdr[0:4], magic[:]) + hdr[4] = byte(Segment) + binary.BigEndian.PutUint32(hdr[5:9], uint32(MaxPayload+1)) + + _, _, err := NewReader(bytes.NewReader(hdr[:])).ReadFrame() + require.Error(t, err) + require.Contains(t, err.Error(), "exceeds max") +} + +// TestWriteOversizeRejected: the writer refuses to emit an over-cap frame. +func TestWriteOversizeRejected(t *testing.T) { + err := NewWriter(io.Discard).Segment(make([]byte, MaxPayload+1)) + require.Error(t, err) + require.Contains(t, err.Error(), "exceeds max") +} + +// TestConcurrentWritesDoNotInterleave: the worker emits segments from multiple +// goroutines (source signer + transcoder completion). Frames must stay whole. +func TestConcurrentWritesDoNotInterleave(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(&buf) + + const writers = 8 + const each = 50 + var wg sync.WaitGroup + for g := 0; g < writers; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < each; i++ { + // Distinct, self-identifying payloads so interleaving is detectable. + payload := []byte(fmt.Sprintf("g%02d-i%02d-%s", g, i, bytes.Repeat([]byte("x"), i))) + require.NoError(t, w.Segment(payload)) + } + }(g) + } + wg.Wait() + + r := NewReader(&buf) + var got []string + for { + typ, payload, err := r.ReadFrame() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + require.Equal(t, Segment, typ) + // Every payload must be one of the well-formed strings — a torn/interleaved + // frame would fail this prefix shape or the count. + require.Regexp(t, `^g\d\d-i\d\d-x*$`, string(payload)) + got = append(got, string(payload)) + } + require.Len(t, got, writers*each, "every frame arrives exactly once, intact") + + // And every expected payload is present exactly once. + want := make([]string, 0, writers*each) + for g := 0; g < writers; g++ { + for i := 0; i < each; i++ { + want = append(want, fmt.Sprintf("g%02d-i%02d-%s", g, i, bytes.Repeat([]byte("x"), i))) + } + } + sort.Strings(got) + sort.Strings(want) + require.Equal(t, want, got) +} diff --git a/pkg/media/ingest_subprocess_test.go b/pkg/media/ingest_subprocess_test.go new file mode 100644 index 00000000..c852325c --- /dev/null +++ b/pkg/media/ingest_subprocess_test.go @@ -0,0 +1,126 @@ +package media + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/crypto/signers" + "stream.place/streamplace/pkg/ingestframe" + "stream.place/streamplace/pkg/muxl" +) + +// runIngestWorkerHelper is what the test binary becomes when re-exec'd with the +// `ingest-worker` arg (see TestMain). It mirrors makeIngestWorkerCommand exactly: +// config on fd 3, frames on fd 4, MKV on stdin; clean run ends with End, a fatal +// error with an Error frame and a non-zero exit. +func runIngestWorkerHelper() int { + cfgFile := os.NewFile(3, "ingest-config") + if cfgFile == nil { + return 2 + } + cfgBytes, err := io.ReadAll(cfgFile) + cfgFile.Close() + if err != nil { + return 2 + } + var cfg IngestWorkerConfig + if err := json.Unmarshal(cfgBytes, &cfg); err != nil { + return 2 + } + framesFile := os.NewFile(4, "ingest-frames") + if framesFile == nil { + return 2 + } + defer framesFile.Close() + frames := ingestframe.NewWriter(framesFile) + if err := RunMKVIngestWorker(context.Background(), cfg, os.Stdin, frames); err != nil { + _ = frames.Error(err.Error()) + return 1 + } + _ = frames.End() + return 0 +} + +// TestIngestWorkerSubprocess exercises the real process boundary: it spawns the +// worker as an actual subprocess (config over fd 3, MKV over stdin, frames over +// fd 4 — the exact wiring MKVIngestIsolated uses) and verifies the worker +// produces valid signed segments, a clean End frame, and a zero exit. This is +// the part the in-process worker test can't cover: fd passing, the framed wire +// protocol over a real pipe, and process lifecycle. +func TestIngestWorkerSubprocess(t *testing.T) { + ctx := context.Background() + ms := newBareSegmentSigner(t) + + keyPEM, err := signers.MarshalES256KPrivateKeyPEM(ms.Signer) + require.NoError(t, err) + manifest, err := ms.buildManifest(ctx, time.Now().UnixMilli()) + require.NoError(t, err) + cfgJSON, err := json.Marshal(IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: ms.Cert, + Manifest: manifest, + }) + require.NoError(t, err) + + mkv := makeH264AACMKV(t, ctx, getFixture("5sec.mp4")) + + exe, err := os.Executable() + require.NoError(t, err) + cmd := exec.CommandContext(ctx, exe, "ingest-worker") + // Quiet gst in the child (it inherits the parent test's verbose leak-tracer env). + cmd.Env = append(os.Environ(), "GST_DEBUG=0", "GST_TRACERS=") + cmd.Stdin = bytes.NewReader(mkv) + cmd.Stderr = os.Stderr + + cfgR, cfgW, err := os.Pipe() + require.NoError(t, err) + framesR, framesW, err := os.Pipe() + require.NoError(t, err) + cmd.ExtraFiles = []*os.File{cfgR, framesW} // → child fd 3, fd 4 + + require.NoError(t, cmd.Start()) + cfgR.Close() + framesW.Close() + go func() { + _, _ = cfgW.Write(cfgJSON) + cfgW.Close() + }() + + fr := ingestframe.NewReader(framesR) + var segs int + var sawEnd bool + for { + typ, payload, rerr := fr.ReadFrame() + if errors.Is(rerr, io.EOF) { + break + } + require.NoError(t, rerr) + switch typ { + case ingestframe.Segment: + require.False(t, sawEnd, "no segments after End") + out, verr := muxl.RunMuxlVerify(ctx, bytes.NewReader(payload)) + require.NoError(t, verr, "segment %d verify", segs) + require.NotContains(t, out, `"validation_state":"Invalid"`, "segment %d must validate", segs) + segs++ + case ingestframe.End: + sawEnd = true + case ingestframe.Error: + t.Fatalf("worker emitted error frame: %s", payload) + } + } + framesR.Close() + + require.NoError(t, cmd.Wait(), "worker subprocess exits cleanly") + require.GreaterOrEqual(t, segs, 1, "worker subprocess emitted at least one signed segment") + require.True(t, sawEnd, "worker subprocess emitted a clean End frame") + t.Logf("worker subprocess emitted %d valid signed segments + clean End", segs) +} diff --git a/pkg/media/ingest_supervisor.go b/pkg/media/ingest_supervisor.go new file mode 100644 index 00000000..ce558d7e --- /dev/null +++ b/pkg/media/ingest_supervisor.go @@ -0,0 +1,209 @@ +package media + +import ( + "bufio" + "bytes" + "context" + "crypto/ecdsa" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "sync" + "time" + + "stream.place/streamplace/pkg/crypto/signers" + "stream.place/streamplace/pkg/ingestframe" + "stream.place/streamplace/pkg/log" +) + +// MKVIngestIsolated is the process-isolated counterpart to MKVIngest. Instead of +// running the demux + sign pipeline in this process — where a native gst fault, +// OOM, or deadlock would take the whole node down — it spawns a dedicated +// `ingest-worker` subprocess that owns the pipeline and streams signed canonical +// .m4s segments back. This process only reads frames and runs ValidateMP4 +// (memory-safe Go + wasm), so a single failing stream can at worst kill its own +// worker; the node survives. +// +// Per the locked design the worker signs everything, so main hands it the +// streamer key + cert + a once-built manifest over a dedicated config fd (kept +// off argv/env). See buildWorkerConfig for the interim key-custody note. +func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, ms MediaSigner) error { + cfg, err := mm.buildWorkerConfig(ctx, ms) + if err != nil { + return err + } + cfgJSON, err := json.Marshal(cfg) + if err != nil { + return fmt.Errorf("marshal worker config: %w", err) + } + + // Optional recording stays in main: tee the raw input before it reaches the + // worker, so the worker needs no data-dir access. + if shouldRecord, rerr := mm.shouldRecord(ctx, ms.Streamer()); rerr == nil && shouldRecord { + log.Log(ctx, "recording RTMP stream to file", "streamer", ms.Streamer()) + pr, pw := io.Pipe() + input = io.TeeReader(input, pw) + go func() { + if derr := mm.dumpToFile(ctx, pr, ms.Streamer(), ".rtmp.mkv"); derr != nil { + log.Error(ctx, "error dumping to file", "error", derr) + } + }() + } + + exe, err := os.Executable() + if err != nil { + return fmt.Errorf("locate self: %w", err) + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + cmd := exec.CommandContext(ctx, exe, "ingest-worker") + + // Dedicated pipes: fd 3 carries the config in, fd 4 carries the frame stream + // out. Keeping frames off stdout means nothing the worker (or gst, or the + // self-test) writes to stdout/stderr can corrupt them; those stay plain logs. + cfgR, cfgW, err := os.Pipe() + if err != nil { + return fmt.Errorf("config pipe: %w", err) + } + defer cfgW.Close() + framesR, framesW, err := os.Pipe() + if err != nil { + cfgR.Close() + return fmt.Errorf("frames pipe: %w", err) + } + defer framesR.Close() + cmd.ExtraFiles = []*os.File{cfgR, framesW} // → child fd 3, fd 4 + + stdin, err := cmd.StdinPipe() + if err != nil { + cfgR.Close() + framesW.Close() + return err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + cfgR.Close() + framesW.Close() + return err + } + stderr, err := cmd.StderrPipe() + if err != nil { + cfgR.Close() + framesW.Close() + return err + } + + if err := cmd.Start(); err != nil { + cfgR.Close() + framesW.Close() + return fmt.Errorf("start ingest worker: %w", err) + } + cfgR.Close() // the child holds its own copy now + framesW.Close() // ditto; the parent only reads framesR + + go func() { + _, _ = cfgW.Write(cfgJSON) + cfgW.Close() // EOF so the worker's config read completes + }() + + // Pump the media to the worker; closing stdin on input EOF is the worker's + // end-of-stream. Managed here (not cmd.Stdin) so a worker exit can't wedge + // cmd.Wait on a still-blocked body read. + go func() { + defer stdin.Close() + _, _ = io.Copy(stdin, input) + }() + + // Forward stdout + stderr to the node logger. Drain both fully before Wait. + var logsWG sync.WaitGroup + logsWG.Add(2) + go func() { defer logsWG.Done(); streamWorkerLogs(ctx, stdout, ms.Streamer()) }() + go func() { defer logsWG.Done(); streamWorkerLogs(ctx, stderr, ms.Streamer()) }() + + // Read signed-segment frames and feed each into the normal chokepoint. + sawEnd, readErr := mm.consumeWorkerFrames(ctx, framesR, ms.Streamer()) + logsWG.Wait() + werr := cmd.Wait() + + switch { + case readErr != nil: + return fmt.Errorf("ingest worker stream: %w", readErr) + case werr != nil && !sawEnd: + // A non-zero exit without a clean End frame means the worker died — that + // failure is contained to the subprocess; the node is unaffected. + return fmt.Errorf("ingest worker exited: %w", werr) + case werr != nil: + log.Warn(ctx, "ingest worker signalled clean end but exited nonzero", "streamer", ms.Streamer(), "error", werr) + } + return nil +} + +// consumeWorkerFrames reads framed segments from the worker and runs ValidateMP4 +// over each. It returns whether a clean End frame was seen and the terminal read +// error: nil on a clean close (End then EOF), or io.ErrUnexpectedEOF / a desync +// error when the worker died mid-frame. +func (mm *MediaManager) consumeWorkerFrames(ctx context.Context, stdout io.Reader, streamer string) (sawEnd bool, _ error) { + fr := ingestframe.NewReader(stdout) + for { + typ, payload, err := fr.ReadFrame() + if err != nil { + if errors.Is(err, io.EOF) { + return sawEnd, nil + } + return sawEnd, err + } + switch typ { + case ingestframe.Segment: + if verr := mm.ValidateMP4(ctx, bytes.NewReader(payload), true); verr != nil { + log.Error(ctx, "ingest worker: validate segment failed", "streamer", streamer, "error", verr) + } + case ingestframe.End: + sawEnd = true + case ingestframe.Error: + log.Error(ctx, "ingest worker: reported error", "streamer", streamer, "error", string(payload)) + } + } +} + +// streamWorkerLogs forwards the worker's stderr lines into the node logger. +func streamWorkerLogs(ctx context.Context, stderr io.Reader, streamer string) { + scan := bufio.NewScanner(stderr) + scan.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scan.Scan() { + if line := scan.Text(); line != "" { + log.Log(ctx, "[ingest-worker] "+line, "streamer", streamer) + } + } +} + +// buildWorkerConfig extracts the handshake the worker needs to sign on main's +// behalf. INTERIM key custody: requires a software MediaSignerLocal — the +// MKV/RTMP push path always provides one; anything else errors so the caller can +// fall back to the in-process path. +func (mm *MediaManager) buildWorkerConfig(ctx context.Context, ms MediaSigner) (IngestWorkerConfig, error) { + local, ok := ms.(*MediaSignerLocal) + if !ok { + return IngestWorkerConfig{}, fmt.Errorf("isolated ingest requires a local signer, got %T", ms) + } + if _, ok := local.Signer.(*ecdsa.PrivateKey); !ok { + return IngestWorkerConfig{}, fmt.Errorf("isolated ingest requires a software signer, got %T", local.Signer) + } + keyPEM, err := signers.MarshalES256KPrivateKeyPEM(local.Signer) + if err != nil { + return IngestWorkerConfig{}, fmt.Errorf("marshal streamer key: %w", err) + } + manifest, err := local.buildManifest(ctx, time.Now().UnixMilli()) + if err != nil { + return IngestWorkerConfig{}, fmt.Errorf("build manifest: %w", err) + } + return IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: local.Cert, + Manifest: manifest, + }, nil +} diff --git a/pkg/media/ingest_worker.go b/pkg/media/ingest_worker.go new file mode 100644 index 00000000..22d64f5d --- /dev/null +++ b/pkg/media/ingest_worker.go @@ -0,0 +1,99 @@ +package media + +import ( + "context" + "fmt" + "io" + + "github.com/go-gst/go-gst/gst" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/gstinit" + "stream.place/streamplace/pkg/ingestframe" + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/muxl" +) + +// IngestWorkerConfig is the startup handshake the main process hands an ingest +// worker over a dedicated pipe fd — kept off argv/env so key material never +// lands in a process listing. +// +// INTERIM key custody: per the locked design the worker signs everything, so it +// receives the streamer key directly. This is the deliberately-temporary +// approach; the detach/reattach work will revisit how a worker holds keys. +type IngestWorkerConfig struct { + StreamerDID string `json:"streamer_did"` + // KeyPEM is the streamer's ES256K signing key in PEM. The MKV/RTMP push path + // always yields a software key, which is what muxl-sign wants here; it is + // forwarded verbatim, no reconstruction. + KeyPEM []byte `json:"key_pem"` + CertPEM []byte `json:"cert_pem"` + // Manifest is the C2PA manifest JSON, built ONCE by main at stream start. + // muxl-sign stamps each segment's signing time into it as it signs. NOTE: + // static for the worker's lifetime — mid-stream manifest changes (e.g. a + // pre-live → live transition) don't yet cross the boundary; that needs a + // control channel and is tracked as future work. + Manifest []byte `json:"manifest"` +} + +// RunMKVIngestWorker is the body of the `ingest-worker` subcommand. It reads an +// MKV stream from stdin, runs the same demux + Opus re-encode + muxl-sign +// pipeline as the in-process MKVIngest, and emits each signed canonical .m4s +// segment to frames; the main process reads those frames and runs ValidateMP4 +// over each, exactly as if onSegment had called it directly. +// +// It returns when the stream ends cleanly (EOS) or the pipeline errors. The +// caller frames End or Error accordingly. All segment frames are guaranteed +// flushed before it returns, so a trailing End can never race ahead of the last +// Segment. +func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Reader, frames *ingestframe.Writer) error { + gstinit.InitGST() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // The worker signs everything itself: forward the streamer key PEM + cert + + // prebuilt manifest straight to muxl-sign. No MediaSigner / model / DB needed. + signStream := func(ctx context.Context, input io.Reader, eventCh chan *muxl.MuxlEvent) error { + fetchManifest := func() ([]byte, error) { return cfg.Manifest, nil } + return muxl.RunMuxlSignSegment(ctx, input, muxl.SignerInput{ + CertPEM: cfg.CertPEM, + KeyPEM: cfg.KeyPEM, + TrackManifestFn: fetchManifest, + WrapperManifestFn: fetchManifest, + }, nil, nil, eventCh) + } + + onSegment := func(_ context.Context, segment []byte) error { + return frames.Segment(segment) + } + + signerElem, done, err := muxlSignSegmentElem(ctx, &config.CLI{}, signStream, onSegment) + if err != nil { + return fmt.Errorf("build signer element: %w", err) + } + pipeline, err := buildMKVIngestPipeline(ctx, stdin, signerElem) + if err != nil { + return fmt.Errorf("build pipeline: %w", err) + } + + busErr := make(chan error, 1) + go func() { + busErr <- HandleBusMessages(ctx, pipeline) + }() + + if err := pipeline.SetState(gst.StatePlaying); err != nil { + return fmt.Errorf("set playing: %w", err) + } + defer func() { + if err := pipeline.SetState(gst.StateNull); err != nil { + log.Error(ctx, "ingest worker: set null", "error", err) + } + }() + + // Wait for the pipeline to finish (EOS or error), then drain the signer: + // cancelling unblocks the signer's input pipe so it flushes the final GoP, + // and <-done guarantees every segment frame is written before we return. + pipeErr := <-busErr + cancel() + <-done + return pipeErr +} diff --git a/pkg/media/ingest_worker_test.go b/pkg/media/ingest_worker_test.go new file mode 100644 index 00000000..9eb8d6c1 --- /dev/null +++ b/pkg/media/ingest_worker_test.go @@ -0,0 +1,102 @@ +package media + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + "time" + + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/crypto/signers" + "stream.place/streamplace/pkg/gstinit" + "stream.place/streamplace/pkg/ingestframe" + "stream.place/streamplace/pkg/muxl" +) + +// makeH264AACMKV builds a clean, single-track, streamable H264+AAC MKV from an +// H264+Opus MP4 fixture (video passed through, audio transcoded Opus→AAC). The +// repo's only AAC fixture (sample-stream.mkv) carries four audio tracks, which +// the single-audio ingest pipeline leaves three of unlinked — wedging +// matroskademux with no EOS. This produces exactly the 1-video-1-audio AAC MKV +// the RTMP push path actually delivers. +func makeH264AACMKV(t *testing.T, ctx context.Context, srcMP4 string) []byte { + t.Helper() + gstinit.InitGST() + desc := strings.Join([]string{ + "filesrc location=" + srcMP4 + " ! qtdemux name=d", + "d. ! queue ! h264parse ! matroskamux name=mux streamable=true ! appsink name=sink", + "d. ! queue ! opusdec ! audioconvert ! audioresample ! fdkaacenc ! aacparse ! mux.", + }, "\n") + pipeline, err := gst.NewPipelineFromString(desc) + require.NoError(t, err) + + sinkEle, err := pipeline.GetElementByName("sink") + require.NoError(t, err) + var buf bytes.Buffer + app.SinkFromElement(sinkEle).SetCallbacks(&app.SinkCallbacks{ + NewSampleFunc: WriterNewSample(ctx, &buf), + }) + + busErr := make(chan error, 1) + go func() { busErr <- HandleBusMessages(ctx, pipeline) }() + require.NoError(t, pipeline.SetState(gst.StatePlaying)) + defer func() { _ = pipeline.SetState(gst.StateNull) }() + require.NoError(t, <-busErr, "remux to H264+AAC MKV") + require.NotEmpty(t, buf.Bytes(), "remux produced an MKV") + return buf.Bytes() +} + +// TestRunMKVIngestWorkerProducesValidSignedFrames drives the isolated ingest +// worker's core directly (no subprocess): feed it an H264+AAC MKV, collect the +// framed output, and verify every emitted segment is a valid signed canonical +// .m4s. This is the contract the supervisor relies on — frames it can hand +// straight to ValidateMP4. (The real subprocess spawn + fault injection is +// Stage 3.) +func TestRunMKVIngestWorkerProducesValidSignedFrames(t *testing.T) { + ctx := context.Background() + ms := newBareSegmentSigner(t) + + keyPEM, err := signers.MarshalES256KPrivateKeyPEM(ms.Signer) + require.NoError(t, err) + manifest, err := ms.buildManifest(ctx, time.Now().UnixMilli()) + require.NoError(t, err) + + cfg := IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: ms.Cert, + Manifest: manifest, + } + + mkv := makeH264AACMKV(t, ctx, getFixture("5sec.mp4")) + + // All frame writes complete before RunMKVIngestWorker returns (it waits on + // the signer drain), so reading the buffer single-threaded afterwards is safe. + var buf bytes.Buffer + frames := ingestframe.NewWriter(&buf) + require.NoError(t, RunMKVIngestWorker(ctx, cfg, bytes.NewReader(mkv), frames)) + + r := ingestframe.NewReader(&buf) + var segs int + for { + typ, payload, err := r.ReadFrame() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + require.Equal(t, ingestframe.Segment, typ, "worker emits only Segment frames; End is the subcommand's job") + require.NotEmpty(t, payload) + + out, err := muxl.RunMuxlVerify(ctx, bytes.NewReader(payload)) + require.NoError(t, err, "segment %d verify", segs) + require.NotContains(t, out, `"validation_state":"Invalid"`, "segment %d must validate", segs) + segs++ + } + require.GreaterOrEqual(t, segs, 1, "worker emitted at least one signed segment") + t.Logf("worker emitted %d valid signed segments", segs) +} diff --git a/pkg/media/leak_test.go b/pkg/media/leak_test.go index 1d129a6c..eb7d634d 100644 --- a/pkg/media/leak_test.go +++ b/pkg/media/leak_test.go @@ -48,6 +48,12 @@ var LeakReportMutex sync.Mutex var LeakDoneCh = make(chan struct{}) func TestMain(m *testing.M) { + // When the parent test re-execs us as an isolated ingest worker (mirroring the + // `streamplace ingest-worker` subcommand), act as that worker and exit — before + // any leak-tracer setup, so the child stays a clean media pipeline. + if len(os.Args) > 1 && os.Args[1] == "ingest-worker" { + os.Exit(runIngestWorkerHelper()) + } if os.Getenv(IgnoreLeaks) != "" { gstinit.InitGST() os.Exit(m.Run()) diff --git a/pkg/media/mkv_ingest.go b/pkg/media/mkv_ingest.go index f4f15bcc..ca8bf562 100644 --- a/pkg/media/mkv_ingest.go +++ b/pkg/media/mkv_ingest.go @@ -34,75 +34,76 @@ func (mm *MediaManager) MKVIngest(ctx context.Context, input io.Reader, ms Media } ctx, cancel := context.WithCancel(ctx) defer cancel() - pipelineSlice := []string{ - "appsrc name=streamsrc ! matroskademux name=demux", - "demux. ! queue ! h264parse name=parse", - "demux. ! queue ! fdkaacdec ! audioresample ! opusenc name=audioenc", - } - pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) - if err != nil { - return fmt.Errorf("error creating MKVIngest pipeline: %w", err) - } - - srcele, err := pipeline.GetElementByName("streamsrc") - if err != nil { - return err - } - // defer runtime.KeepAlive(srcele) - src := app.SrcFromElement(srcele) - src.SetCallbacks(&app.SourceCallbacks{ - NeedDataFunc: ReaderNeedDataIncremental(ctx, input), - }) - parseEle, err := pipeline.GetElementByName("parse") - if err != nil { - return err - } signer, err := mm.SegmentAndSignElem(ctx, ms) if err != nil { return err } - - err = pipeline.Add(signer) - if err != nil { - return err - } - err = parseEle.Link(signer) - if err != nil { - return err - } - audioenc, err := pipeline.GetElementByName("audioenc") - if err != nil { - return err - } - err = audioenc.Link(signer) + pipeline, err := buildMKVIngestPipeline(ctx, input, signer) if err != nil { return err } busErr := make(chan error) go func() { - err := HandleBusMessages(ctx, pipeline) - busErr <- err + busErr <- HandleBusMessages(ctx, pipeline) }() go mm.HandleKeyRevocation(ctx, ms, pipeline) - err = pipeline.SetState(gst.StatePlaying) - if err != nil { + if err := pipeline.SetState(gst.StatePlaying); err != nil { return err } - defer func() { - err := pipeline.SetState(gst.StateNull) - if err != nil { + if err := pipeline.SetState(gst.StateNull); err != nil { log.Error(ctx, "error setting pipeline to null state", "error", err) } }() - err = <-busErr + return <-busErr +} - return err +// buildMKVIngestPipeline builds the H264+AAC MKV demux graph (video → h264parse, +// audio → Opus re-encode) and links both branches into signerElem — the muxl +// signing bin that emits one bare canonical .m4s per GoP. Shared by the +// in-process MKVIngest and the isolated ingest worker, which differ only in +// where signerElem routes its segments (ValidateMP4 vs. a frame writer to the +// main process). +func buildMKVIngestPipeline(ctx context.Context, input io.Reader, signerElem *gst.Element) (*gst.Pipeline, error) { + pipelineSlice := []string{ + "appsrc name=streamsrc ! matroskademux name=demux", + "demux. ! queue ! h264parse name=parse", + "demux. ! queue ! fdkaacdec ! audioresample ! opusenc name=audioenc", + } + pipeline, err := gst.NewPipelineFromString(strings.Join(pipelineSlice, "\n")) + if err != nil { + return nil, fmt.Errorf("error creating MKVIngest pipeline: %w", err) + } + srcele, err := pipeline.GetElementByName("streamsrc") + if err != nil { + return nil, err + } + app.SrcFromElement(srcele).SetCallbacks(&app.SourceCallbacks{ + NeedDataFunc: ReaderNeedDataIncremental(ctx, input), + }) + parseEle, err := pipeline.GetElementByName("parse") + if err != nil { + return nil, err + } + if err := pipeline.Add(signerElem); err != nil { + return nil, err + } + if err := parseEle.Link(signerElem); err != nil { + return nil, err + } + audioenc, err := pipeline.GetElementByName("audioenc") + if err != nil { + return nil, err + } + if err := audioenc.Link(signerElem); err != nil { + return nil, err + } + return pipeline, nil } func (mm *MediaManager) dumpToFile(ctx context.Context, r io.Reader, user string, filesuffix string) error { diff --git a/pkg/media/muxl_segment.go b/pkg/media/muxl_segment.go index 4d6b94e2..1de14d82 100644 --- a/pkg/media/muxl_segment.go +++ b/pkg/media/muxl_segment.go @@ -14,6 +14,13 @@ import ( "stream.place/streamplace/pkg/muxl" ) +// SignSegmentStreamFunc drives muxl-sign's streaming per-segment signer over an +// fMP4 input, emitting one signed-segment event per GoP on eventCh. It is the +// only thing muxlSignSegmentElem needs from a signer, so the isolated ingest +// worker can supply a key-PEM-backed closure without a full MediaSigner (and +// without the model/DB a MediaSignerLocal carries). +type SignSegmentStreamFunc func(ctx context.Context, input io.Reader, eventCh chan *muxl.MuxlEvent) error + // MuxlSignSegmentElem builds the gstreamer bin that muxes the incoming // video+audio into a fragmented MP4 stream, then drives muxl-sign's streaming // per-segment signer over it. For each GoP it assembles the bare canonical @@ -23,6 +30,17 @@ import ( // produced here. Presentation headers are synthesized downstream (ValidateMP4 // / playback) only when needed. func MuxlSignSegmentElem(ctx context.Context, cli *config.CLI, ms MediaSigner, onSegment func(ctx context.Context, segment []byte) error) (*gst.Element, error) { + elem, _, err := muxlSignSegmentElem(ctx, cli, ms.SignSegmentStream, onSegment) + return elem, err +} + +// muxlSignSegmentElem is MuxlSignSegmentElem's core, parameterized by the raw +// sign-stream function and additionally returning a done channel that closes +// once every signed segment has been drained to onSegment (the signer goroutine +// has finished and the event loop has emptied). The isolated ingest worker waits +// on it to guarantee all segment frames are flushed before it signals a clean +// end-of-stream. +func muxlSignSegmentElem(ctx context.Context, cli *config.CLI, signStream SignSegmentStreamFunc, onSegment func(ctx context.Context, segment []byte) error) (*gst.Element, <-chan struct{}, error) { ctx = log.WithLogValues(ctx, "func", "MuxlSignSegmentElem") bin := gst.NewBin("muxl-segment-bin") elem, err := gst.NewElementWithProperties("mp4mux", map[string]any{ @@ -31,46 +49,46 @@ func MuxlSignSegmentElem(ctx context.Context, cli *config.CLI, ms MediaSigner, o "fragment-duration": 1, }) if err != nil { - return nil, err + return nil, nil, err } if err := bin.Add(elem); err != nil { - return nil, fmt.Errorf("failed to add mp4mux to bin: %w", err) + return nil, nil, fmt.Errorf("failed to add mp4mux to bin: %w", err) } videoPad := elem.GetRequestPad("video_%u") if videoPad == nil { - return nil, fmt.Errorf("failed to get video pad") + return nil, nil, fmt.Errorf("failed to get video pad") } videoGhost := gst.NewGhostPad("video_0", videoPad) if videoGhost == nil { - return nil, fmt.Errorf("failed to create video ghost pad") + return nil, nil, fmt.Errorf("failed to create video ghost pad") } audioPad := elem.GetRequestPad("audio_%u") if audioPad == nil { - return nil, fmt.Errorf("failed to get audio pad") + return nil, nil, fmt.Errorf("failed to get audio pad") } audioGhost := gst.NewGhostPad("audio_0", audioPad) if audioGhost == nil { - return nil, fmt.Errorf("failed to create audio ghost pad") + return nil, nil, fmt.Errorf("failed to create audio ghost pad") } if ok := bin.AddPad(videoGhost.Pad); !ok { - return nil, fmt.Errorf("failed to add video ghost pad to bin") + return nil, nil, fmt.Errorf("failed to add video ghost pad to bin") } if ok := bin.AddPad(audioGhost.Pad); !ok { - return nil, fmt.Errorf("failed to add audio ghost pad to bin") + return nil, nil, fmt.Errorf("failed to add audio ghost pad to bin") } appsink, err := gst.NewElementWithProperties("appsink", map[string]any{ "name": "muxl-appsink", }) if err != nil { - return nil, fmt.Errorf("failed to create appsink element: %w", err) + return nil, nil, fmt.Errorf("failed to create appsink element: %w", err) } if err := bin.Add(appsink); err != nil { - return nil, fmt.Errorf("failed to add appsink to bin: %w", err) + return nil, nil, fmt.Errorf("failed to add appsink to bin: %w", err) } if err := elem.Link(appsink); err != nil { - return nil, fmt.Errorf("failed to link mp4mux to appsink: %w", err) + return nil, nil, fmt.Errorf("failed to link mp4mux to appsink: %w", err) } r, w := io.Pipe() @@ -83,13 +101,15 @@ func MuxlSignSegmentElem(ctx context.Context, cli *config.CLI, ms MediaSigner, o // GoP's per-track signed canonical segments. eventCh := make(chan *muxl.MuxlEvent, 16) go func() { - err := ms.SignSegmentStream(ctx, r, eventCh) + err := signStream(ctx, r, eventCh) close(eventCh) if err != nil && ctx.Err() == nil { log.Error(ctx, "error running muxl sign-segment", "error", err) } }() + done := make(chan struct{}) go func() { + defer close(done) for ev := range eventCh { if ev.Type != "signed-segment" { continue @@ -107,7 +127,7 @@ func MuxlSignSegmentElem(ctx context.Context, cli *config.CLI, ms MediaSigner, o NewSampleFunc: WriterNewSample(ctx, w), }) - return bin.Element, nil + return bin.Element, done, nil } // concatTracksSorted joins the per-track canonical segment bytes for one GoP -- 2.51.2 From ec23bc0619e738c9ed30139a792d8c8c2bd129e5 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 1 Jun 2026 18:21:06 -0700 Subject: [PATCH 02/17] media: watchdog + fault-containment for isolated ingest workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The point of isolation is that one bad stream can't take the node down — this makes good on it. A worker that stops producing frames (a wedged native pipeline gst can't drain) is presumed dead and killed via the cancellable command context, bounded by ingestWorkerWatchdog (30s; reset on every frame). MKVIngestIsolated then returns an error instead of hanging the session. TestMKVIngestIsolatedWedgeContained proves it with a real node-killer: sample-stream.mkv carries four audio tracks, so the single-audio pipeline leaves three matroskademux pads unlinked and wedges with no EOS. In-process that hangs (or OOMs) the node; isolated, the watchdog kills the worker and the supervisor returns "ingest worker exited: signal: killed" in seconds — with the test process (the node) still running to assert it. Co-Authored-By: Claude Opus 4.8 --- pkg/media/ingest_subprocess_test.go | 27 +++++++++++++++++++++++++++ pkg/media/ingest_supervisor.go | 26 ++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/pkg/media/ingest_subprocess_test.go b/pkg/media/ingest_subprocess_test.go index c852325c..27b791f0 100644 --- a/pkg/media/ingest_subprocess_test.go +++ b/pkg/media/ingest_subprocess_test.go @@ -124,3 +124,30 @@ func TestIngestWorkerSubprocess(t *testing.T) { require.True(t, sawEnd, "worker subprocess emitted a clean End frame") t.Logf("worker subprocess emitted %d valid signed segments + clean End", segs) } + +// TestMKVIngestIsolatedWedgeContained is the isolation guarantee: sample-stream.mkv +// carries four audio tracks, so the single-audio ingest pipeline leaves three +// matroskademux pads unlinked and wedges with no EOS — exactly the kind of native +// wedge that would hang (or, with a runaway buffer, OOM-kill) an in-process +// ingest and take the node with it. Run in a worker, it must be contained: the +// watchdog kills the worker and MKVIngestIsolated returns an error, bounded in +// time, with THIS process — the node — still running to assert it. +func TestMKVIngestIsolatedWedgeContained(t *testing.T) { + old := ingestWorkerWatchdog + ingestWorkerWatchdog = 6 * time.Second + defer func() { ingestWorkerWatchdog = old }() + + mm, _ := getStaticTestMediaManager(t) + ms := newBareSegmentSigner(t) + + wedge, err := os.ReadFile(getFixture("sample-stream.mkv")) + require.NoError(t, err) + + start := time.Now() + err = mm.MKVIngestIsolated(context.Background(), bytes.NewReader(wedge), ms) + elapsed := time.Since(start) + + require.Error(t, err, "a wedged worker must surface as an error, not a hang") + require.Less(t, elapsed, 25*time.Second, "watchdog bounded the wedge") + t.Logf("wedged worker contained in %s: %v", elapsed.Round(time.Second), err) +} diff --git a/pkg/media/ingest_supervisor.go b/pkg/media/ingest_supervisor.go index ce558d7e..02e74711 100644 --- a/pkg/media/ingest_supervisor.go +++ b/pkg/media/ingest_supervisor.go @@ -19,6 +19,13 @@ import ( "stream.place/streamplace/pkg/log" ) +// ingestWorkerWatchdog bounds how long an isolated worker may go without +// producing a segment frame before it's presumed wedged (a native pipeline gst +// can't drain — e.g. a pathological stream) and killed. A healthy stream emits a +// segment every GoP (~1–2s), so this is generous. Var (not const) so tests can +// shorten it. +var ingestWorkerWatchdog = 30 * time.Second + // MKVIngestIsolated is the process-isolated counterpart to MKVIngest. Instead of // running the demux + sign pipeline in this process — where a native gst fault, // OOM, or deadlock would take the whole node down — it spawns a dedicated @@ -124,8 +131,20 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, go func() { defer logsWG.Done(); streamWorkerLogs(ctx, stdout, ms.Streamer()) }() go func() { defer logsWG.Done(); streamWorkerLogs(ctx, stderr, ms.Streamer()) }() + // Watchdog: a worker that stops producing frames is presumed wedged and + // killed (cancel → CommandContext SIGKILLs it), so a single bad stream can't + // hang its session forever. Reset on every frame. + watchdog := time.AfterFunc(ingestWorkerWatchdog, func() { + log.Warn(ctx, "ingest worker watchdog fired (no frames); killing worker", + "streamer", ms.Streamer(), "timeout", ingestWorkerWatchdog) + cancel() + }) + defer watchdog.Stop() + // Read signed-segment frames and feed each into the normal chokepoint. - sawEnd, readErr := mm.consumeWorkerFrames(ctx, framesR, ms.Streamer()) + sawEnd, readErr := mm.consumeWorkerFrames(ctx, framesR, ms.Streamer(), func() { + watchdog.Reset(ingestWorkerWatchdog) + }) logsWG.Wait() werr := cmd.Wait() @@ -146,7 +165,7 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, // over each. It returns whether a clean End frame was seen and the terminal read // error: nil on a clean close (End then EOF), or io.ErrUnexpectedEOF / a desync // error when the worker died mid-frame. -func (mm *MediaManager) consumeWorkerFrames(ctx context.Context, stdout io.Reader, streamer string) (sawEnd bool, _ error) { +func (mm *MediaManager) consumeWorkerFrames(ctx context.Context, stdout io.Reader, streamer string, onProgress func()) (sawEnd bool, _ error) { fr := ingestframe.NewReader(stdout) for { typ, payload, err := fr.ReadFrame() @@ -156,6 +175,9 @@ func (mm *MediaManager) consumeWorkerFrames(ctx context.Context, stdout io.Reade } return sawEnd, err } + if onProgress != nil { + onProgress() + } switch typ { case ingestframe.Segment: if verr := mm.ValidateMP4(ctx, bytes.NewReader(payload), true); verr != nil { -- 2.51.2 From 0595a86eca577b93b61633d857966976d0a10dc4 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 1 Jun 2026 18:27:16 -0700 Subject: [PATCH 03/17] media: complete dual-codec inside the ingest worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fuse the transcode into the worker so it emits finished dual-codec (Opus+AAC) signed segments, instead of single-codec source that main then has to complete. The worker feeds each signed source segment into a per-stream transcoder running in its OWN process; the completion callback frames the finished segment. main's ValidateMP4 then sees already-dual-codec input and distributes it directly — the whole mux + sign + transcode chain now lives in the isolated worker, and the node only does memory-safe validate + distribute. Mechanics: the worker gets the node transcode key + broadcaster host in the fd-3 handshake (main from mm.transcodeSigner(); absent → worker emits single-codec, logged). The transcoder runs on a context.WithoutCancel so draining the signer doesn't kill it; its ~1-GoP tail is flushed by Close before the worker signals End. Because one worker == one session, the per-DID transcoder-reuse hazard (the reconnect/timeline bug) structurally cannot occur here — a reconnect is a brand-new process. TestRunMKVIngestWorkerProducesValidSignedFrames now supplies a node key and asserts every emitted segment verifies and carries BOTH the source Opus and a worker-transcoded AAC track. Co-Authored-By: Claude Opus 4.8 --- pkg/media/ingest_supervisor.go | 23 +++++++++++---- pkg/media/ingest_worker.go | 50 ++++++++++++++++++++++++++++++--- pkg/media/ingest_worker_test.go | 32 +++++++++++++++++---- 3 files changed, 89 insertions(+), 16 deletions(-) diff --git a/pkg/media/ingest_supervisor.go b/pkg/media/ingest_supervisor.go index 02e74711..52d365eb 100644 --- a/pkg/media/ingest_supervisor.go +++ b/pkg/media/ingest_supervisor.go @@ -222,10 +222,21 @@ func (mm *MediaManager) buildWorkerConfig(ctx context.Context, ms MediaSigner) ( if err != nil { return IngestWorkerConfig{}, fmt.Errorf("build manifest: %w", err) } - return IngestWorkerConfig{ - StreamerDID: ms.Streamer(), - KeyPEM: keyPEM, - CertPEM: local.Cert, - Manifest: manifest, - }, nil + cfg := IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: local.Cert, + Manifest: manifest, + BroadcasterHost: mm.cli.BroadcasterHost, + } + // Node transcode signer lets the worker complete to dual-codec itself. If it's + // unavailable, the worker emits single-codec (the node doesn't re-transcode the + // worker's output) — an acceptable, logged fallback rather than a hard failure. + if nodeCert, nodeKeyPEM, serr := mm.transcodeSigner(); serr == nil { + cfg.NodeCertPEM = nodeCert + cfg.NodeKeyPEM = nodeKeyPEM + } else { + log.Warn(ctx, "node transcode signer unavailable; isolated worker will emit single-codec", "error", serr) + } + return cfg, nil } diff --git a/pkg/media/ingest_worker.go b/pkg/media/ingest_worker.go index 22d64f5d..84128fdc 100644 --- a/pkg/media/ingest_worker.go +++ b/pkg/media/ingest_worker.go @@ -33,6 +33,14 @@ type IngestWorkerConfig struct { // pre-live → live transition) don't yet cross the boundary; that needs a // control channel and is tracked as future work. Manifest []byte `json:"manifest"` + + // Node transcode signer + broadcaster identity. When set, the worker completes + // each single-codec source segment to dual-codec (Opus+AAC) itself — the + // transcode runs in this isolated process too — signing the added track under + // the node identity. Empty → the worker emits single-codec source segments. + NodeCertPEM []byte `json:"node_cert_pem,omitempty"` + NodeKeyPEM []byte `json:"node_key_pem,omitempty"` + BroadcasterHost string `json:"broadcaster_host,omitempty"` } // RunMKVIngestWorker is the body of the `ingest-worker` subcommand. It reads an @@ -50,6 +58,10 @@ func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Re ctx, cancel := context.WithCancel(ctx) defer cancel() + // Minimal manager: just the broadcaster identity the transcode completion + // (finishTranscodedSegment) stamps into the node-signed AAC track. + mm := &MediaManager{cli: &config.CLI{BroadcasterHost: cfg.BroadcasterHost}} + // The worker signs everything itself: forward the streamer key PEM + cert + // prebuilt manifest straight to muxl-sign. No MediaSigner / model / DB needed. signStream := func(ctx context.Context, input io.Reader, eventCh chan *muxl.MuxlEvent) error { @@ -62,11 +74,34 @@ func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Re }, nil, nil, eventCh) } + // With a node transcode key, the worker completes each single-codec source + // segment to dual-codec itself: feed the signed source segment into a + // per-stream transcoder running in THIS process; its completion callback + // frames the finished dual-codec segment. The transcoder runs on a + // non-cancellable context so draining the signer (cancel, below) can't kill it + // before its ~1-GoP tail is flushed by Close. One process == one session, so + // the per-DID transcoder-reuse hazard simply can't arise here. + var transcoder *streamTranscoder onSegment := func(_ context.Context, segment []byte) error { - return frames.Segment(segment) + if len(cfg.NodeKeyPEM) == 0 { + return frames.Segment(segment) // no node signer → single-codec + } + if transcoder == nil { + target, need := mm.audioCompletionTarget(ctx, segment) + if !need { + return frames.Segment(segment) // already dual-codec / no audio track + } + transcoder = mm.newStreamTranscoder(context.WithoutCancel(ctx), target, cfg.NodeCertPEM, cfg.NodeKeyPEM, + func(_ any, completed []byte) { + if ferr := frames.Segment(completed); ferr != nil { + log.Error(ctx, "ingest worker: frame completed segment", "error", ferr) + } + }) + } + return transcoder.Feed(segment, nil) } - signerElem, done, err := muxlSignSegmentElem(ctx, &config.CLI{}, signStream, onSegment) + signerElem, done, err := muxlSignSegmentElem(ctx, mm.cli, signStream, onSegment) if err != nil { return fmt.Errorf("build signer element: %w", err) } @@ -90,10 +125,17 @@ func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Re }() // Wait for the pipeline to finish (EOS or error), then drain the signer: - // cancelling unblocks the signer's input pipe so it flushes the final GoP, - // and <-done guarantees every segment frame is written before we return. + // cancelling unblocks the signer's input pipe so it flushes the final GoP, and + // <-done guarantees every source segment has been fed. Then flush the + // transcoder's tail so the last dual-codec completions are framed before we + // return (the caller's End can't race ahead of them). pipeErr := <-busErr cancel() <-done + if transcoder != nil { + if cerr := transcoder.Close(); cerr != nil { + log.Error(ctx, "ingest worker: transcoder close", "error", cerr) + } + } return pipeErr } diff --git a/pkg/media/ingest_worker_test.go b/pkg/media/ingest_worker_test.go index 9eb8d6c1..efb699c9 100644 --- a/pkg/media/ingest_worker_test.go +++ b/pkg/media/ingest_worker_test.go @@ -66,11 +66,16 @@ func TestRunMKVIngestWorkerProducesValidSignedFrames(t *testing.T) { manifest, err := ms.buildManifest(ctx, time.Now().UnixMilli()) require.NoError(t, err) + // Provide the node transcode signer (the same test key serves both roles + // here, as in the transcoder tests) so the worker completes to dual-codec. cfg := IngestWorkerConfig{ - StreamerDID: ms.Streamer(), - KeyPEM: keyPEM, - CertPEM: ms.Cert, - Manifest: manifest, + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: ms.Cert, + Manifest: manifest, + NodeCertPEM: ms.Cert, + NodeKeyPEM: keyPEM, + BroadcasterHost: "test.example.com", } mkv := makeH264AACMKV(t, ctx, getFixture("5sec.mp4")) @@ -95,8 +100,23 @@ func TestRunMKVIngestWorkerProducesValidSignedFrames(t *testing.T) { out, err := muxl.RunMuxlVerify(ctx, bytes.NewReader(payload)) require.NoError(t, err, "segment %d verify", segs) require.NotContains(t, out, `"validation_state":"Invalid"`, "segment %d must validate", segs) + + // With a node key the worker completes to dual-codec: every segment must + // carry both the source Opus and a worker-transcoded AAC track. + codecs := audioCodecsOf(t, ctx, payload) + hasOpus, hasAAC := false, false + for _, c := range codecs { + if isOpusCodec(c) { + hasOpus = true + } + if isAACCodec(c) { + hasAAC = true + } + } + require.True(t, hasOpus, "segment %d keeps source Opus (got %v)", segs, codecs) + require.True(t, hasAAC, "segment %d gains worker-transcoded AAC (got %v)", segs, codecs) segs++ } - require.GreaterOrEqual(t, segs, 1, "worker emitted at least one signed segment") - t.Logf("worker emitted %d valid signed segments", segs) + require.GreaterOrEqual(t, segs, 1, "worker emitted at least one signed dual-codec segment") + t.Logf("worker emitted %d valid dual-codec segments", segs) } -- 2.51.2 From c94b5ac1d29a2a1bf961c08eeb58d3076409e71d Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 1 Jun 2026 18:37:24 -0700 Subject: [PATCH 04/17] media: buffered frame server for zero-downtime worker reconnect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transport core of the detach/reattach path. frameServer delivers a worker's signed segments to main over a reconnectable socket, buffering — bounded, drop-oldest, counted — whenever no client is attached. That's what makes a main restart lossless: main disconnects to upgrade, the worker keeps signing segments into the buffer, and the reconnecting main replays the buffer in order before going live. An outage longer than the buffer window drops the oldest tail loudly (droppedCount) rather than growing without bound; a write to a dead client auto-detaches and re-buffers. RunMKVIngestWorker now takes a FrameWriter interface (satisfied by both *ingestframe.Writer for the Stage-1 pipe and *frameServer for the socket path), so the worker body is transport-agnostic. Tests (deterministic, no gst/subprocess): buffer-while-disconnected then replay-in-order across a simulated restart with zero loss; drop-oldest beyond the bound with correct accounting; and the real accept loop flushing buffered frames on connect then streaming live. Co-Authored-By: Claude Opus 4.8 --- pkg/media/frame_server.go | 136 ++++++++++++++++++++++++++++ pkg/media/frame_server_test.go | 158 +++++++++++++++++++++++++++++++++ pkg/media/ingest_worker.go | 3 +- 3 files changed, 295 insertions(+), 2 deletions(-) create mode 100644 pkg/media/frame_server.go create mode 100644 pkg/media/frame_server_test.go diff --git a/pkg/media/frame_server.go b/pkg/media/frame_server.go new file mode 100644 index 00000000..8c55e71c --- /dev/null +++ b/pkg/media/frame_server.go @@ -0,0 +1,136 @@ +package media + +import ( + "bytes" + "context" + "io" + "net" + "sync" + + "stream.place/streamplace/pkg/ingestframe" + "stream.place/streamplace/pkg/log" +) + +// FrameWriter is the worker's segment sink. Stage 1 uses a direct framed pipe +// (*ingestframe.Writer); the zero-downtime path uses *frameServer, which buffers +// across a disconnected main and replays on reconnect. Structurally satisfied by +// *ingestframe.Writer, so RunMKVIngestWorker is agnostic to which it gets. +type FrameWriter interface { + Segment(seg []byte) error + End() error + Error(msg string) error +} + +type bufferedFrame struct { + typ ingestframe.Type + payload []byte +} + +// frameServer delivers worker frames to the main process over a reconnectable +// transport (a per-session unix socket), buffering — bounded, drop-oldest — +// whenever no client is attached. This is the heart of the zero-downtime upgrade +// path: main disconnects for a restart, the worker keeps signing segments into +// the buffer, and the reconnecting main drains the buffer before going live, so +// segments produced during a brief restart are not lost. Drops are bounded and +// counted (a main outage longer than the buffer window loses the oldest tail, +// loudly, rather than growing without limit). +// +// Safe for concurrent push (the worker) vs attach/detach (the socket accept +// loop). A push to an attached-but-dead client fails the write, auto-detaches, +// and re-buffers that frame, so a hard main disconnect degrades to buffering. +type frameServer struct { + mu sync.Mutex + pending []bufferedFrame + conn net.Conn + w *ingestframe.Writer + maxBuf int + dropped int +} + +// newFrameServer creates a server that buffers up to maxBuf frames while no +// client is attached. +func newFrameServer(maxBuf int) *frameServer { + return &frameServer{maxBuf: maxBuf} +} + +func (s *frameServer) push(typ ingestframe.Type, payload []byte) { + s.mu.Lock() + defer s.mu.Unlock() + if s.w != nil { + if err := s.w.WriteFrame(typ, payload); err == nil { + return + } + // Client gone; drop it and buffer this frame instead. + s.conn, s.w = nil, nil + } + s.pending = append(s.pending, bufferedFrame{typ, bytes.Clone(payload)}) + for len(s.pending) > s.maxBuf { + s.pending = s.pending[1:] + s.dropped++ + } +} + +func (s *frameServer) Segment(seg []byte) error { s.push(ingestframe.Segment, seg); return nil } +func (s *frameServer) End() error { s.push(ingestframe.End, nil); return nil } +func (s *frameServer) Error(msg string) error { s.push(ingestframe.Error, []byte(msg)); return nil } + +// dropped reports how many buffered frames were discarded because the buffer +// overflowed (main was disconnected longer than the buffer window). +func (s *frameServer) droppedCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.dropped +} + +// attach binds a freshly-connected client, replaying buffered frames in order +// before going live. On a replay error the client is dropped and the buffer kept +// intact for the next reconnect. +func (s *frameServer) attach(conn net.Conn) { + s.mu.Lock() + defer s.mu.Unlock() + w := ingestframe.NewWriter(conn) + for _, f := range s.pending { + if err := w.WriteFrame(f.typ, f.payload); err != nil { + return + } + } + s.pending = nil + s.conn, s.w = conn, w +} + +// detachConn drops the named client if it's still the current one (a stale +// connection's teardown must not clobber a newer one that already reattached). +// The buffer and drop count are preserved. +func (s *frameServer) detachConn(conn net.Conn) { + s.mu.Lock() + defer s.mu.Unlock() + if s.conn == conn { + s.conn, s.w = nil, nil + } +} + +// serveFrameSocket accepts client connections on ln and attaches each to the +// server, replacing any prior client (main reconnecting after a restart). Each +// connection is watched for close so the server reverts to buffering. Returns +// when ctx is cancelled or the listener is closed. +func serveFrameSocket(ctx context.Context, ln net.Listener, s *frameServer) { + go func() { + <-ctx.Done() + ln.Close() + }() + for { + conn, err := ln.Accept() + if err != nil { + return // listener closed + } + log.Log(ctx, "ingest worker: main attached to frame socket") + s.attach(conn) + go func(c net.Conn) { + // Main only reads frames; this drains anything it sends (nothing today) + // and unblocks when it disconnects, at which point we revert to buffering. + _, _ = io.Copy(io.Discard, c) + s.detachConn(c) + log.Log(ctx, "ingest worker: main detached from frame socket") + }(conn) + } +} diff --git a/pkg/media/frame_server_test.go b/pkg/media/frame_server_test.go new file mode 100644 index 00000000..68b4007f --- /dev/null +++ b/pkg/media/frame_server_test.go @@ -0,0 +1,158 @@ +package media + +import ( + "context" + "fmt" + "net" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/ingestframe" +) + +// unixPair returns a connected (client, server) unix-socket pair. Unlike +// net.Pipe these are kernel-buffered, so small writes don't block on a reader — +// the frameServer can flush its buffer without a concurrent drainer. +func unixPair(t *testing.T) (client net.Conn, server net.Conn) { + t.Helper() + sock := filepath.Join(t.TempDir(), "p.sock") + ln, err := net.Listen("unix", sock) + require.NoError(t, err) + defer ln.Close() + type res struct { + c net.Conn + e error + } + ch := make(chan res, 1) + go func() { + c, e := ln.Accept() + ch <- res{c, e} + }() + client, err = net.Dial("unix", sock) + require.NoError(t, err) + r := <-ch + require.NoError(t, r.e) + t.Cleanup(func() { client.Close(); r.c.Close() }) + return client, r.c +} + +func seg(i int) []byte { return []byte(fmt.Sprintf("seg-%04d", i)) } + +func readSegs(t *testing.T, r *ingestframe.Reader, n int) [][]byte { + t.Helper() + out := make([][]byte, 0, n) + for i := 0; i < n; i++ { + typ, payload, err := r.ReadFrame() + require.NoError(t, err, "read frame %d", i) + require.Equal(t, ingestframe.Segment, typ) + out = append(out, payload) + } + return out +} + +// TestFrameServerBufferFlushReconnect is the zero-downtime guarantee at the +// transport level: segments produced while main is disconnected are buffered and +// replayed, in order, when main reconnects — nothing is lost across the gap. The +// attach/detach are driven explicitly so the assertion is deterministic. +func TestFrameServerBufferFlushReconnect(t *testing.T) { + srv := newFrameServer(1000) // ample buffer: no drops + + // Detached: the first 3 segments buffer. + for i := 0; i < 3; i++ { + require.NoError(t, srv.Segment(seg(i))) + } + + // Main connects: buffered 0,1,2 replayed, then 3,4 live. + clientA, serverA := unixPair(t) + srv.attach(serverA) + for i := 3; i < 5; i++ { + require.NoError(t, srv.Segment(seg(i))) + } + got := readSegs(t, ingestframe.NewReader(clientA), 5) + for i := 0; i < 5; i++ { + require.Equal(t, seg(i), got[i], "frame %d in order before the restart", i) + } + + // Main restarts: detach + drop the connection. Segments 5,6 produced while + // it's gone must buffer, not vanish. + srv.detachConn(serverA) + clientA.Close() + serverA.Close() + for i := 5; i < 7; i++ { + require.NoError(t, srv.Segment(seg(i))) + } + + // Main reconnects on a fresh connection: buffered 5,6 replayed, then 7 live. + clientB, serverB := unixPair(t) + srv.attach(serverB) + require.NoError(t, srv.Segment(seg(7))) + got = readSegs(t, ingestframe.NewReader(clientB), 3) + for i := 5; i < 8; i++ { + require.Equal(t, seg(i), got[i-5], "frame %d replayed/live after reconnect", i) + } + + require.Equal(t, 0, srv.droppedCount(), "ample buffer drops nothing across a brief restart") +} + +// TestFrameServerDropsOldestBeyondBound: a main outage longer than the buffer +// window drops the OLDEST frames (bounded memory), loudly via droppedCount — +// never grows without limit. +func TestFrameServerDropsOldestBeyondBound(t *testing.T) { + srv := newFrameServer(3) + for i := 0; i < 6; i++ { // 0,1,2 should be dropped; 3,4,5 retained + require.NoError(t, srv.Segment(seg(i))) + } + require.Equal(t, 3, srv.droppedCount(), "oldest 3 dropped") + + client, server := unixPair(t) + srv.attach(server) + got := readSegs(t, ingestframe.NewReader(client), 3) + for i := 3; i < 6; i++ { + require.Equal(t, seg(i), got[i-3], "newest 3 survive in order") + } +} + +// TestServeFrameSocketAttachAndFlush exercises the real accept loop: frames +// buffered before any client are flushed on connect, then live frames stream. +func TestServeFrameSocketAttachAndFlush(t *testing.T) { + sock := filepath.Join(t.TempDir(), "frames.sock") + ln, err := net.Listen("unix", sock) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + srv := newFrameServer(1000) + for i := 0; i < 3; i++ { // buffered before anyone connects + require.NoError(t, srv.Segment(seg(i))) + } + go serveFrameSocket(ctx, ln, srv) + + client, err := net.Dial("unix", sock) + require.NoError(t, err) + defer client.Close() + r := ingestframe.NewReader(client) + + // The 3 buffered frames are replayed once the accept loop attaches us. + got := readSegs(t, r, 3) + for i := 0; i < 3; i++ { + require.Equal(t, seg(i), got[i]) + } + // Reading them confirms we're attached; subsequent pushes stream live. + for i := 3; i < 6; i++ { + require.NoError(t, srv.Segment(seg(i))) + } + got = readSegs(t, r, 3) + for i := 3; i < 6; i++ { + require.Equal(t, seg(i), got[i-3]) + } + + // End frame then a clean EOF on cancel. + require.NoError(t, srv.End()) + typ, _, err := r.ReadFrame() + require.NoError(t, err) + require.Equal(t, ingestframe.End, typ) + + cancel() + ln.Close() +} diff --git a/pkg/media/ingest_worker.go b/pkg/media/ingest_worker.go index 84128fdc..a0f1c53f 100644 --- a/pkg/media/ingest_worker.go +++ b/pkg/media/ingest_worker.go @@ -8,7 +8,6 @@ import ( "github.com/go-gst/go-gst/gst" "stream.place/streamplace/pkg/config" "stream.place/streamplace/pkg/gstinit" - "stream.place/streamplace/pkg/ingestframe" "stream.place/streamplace/pkg/log" "stream.place/streamplace/pkg/muxl" ) @@ -53,7 +52,7 @@ type IngestWorkerConfig struct { // caller frames End or Error accordingly. All segment frames are guaranteed // flushed before it returns, so a trailing End can never race ahead of the last // Segment. -func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Reader, frames *ingestframe.Writer) error { +func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Reader, frames FrameWriter) error { gstinit.InitGST() ctx, cancel := context.WithCancel(ctx) defer cancel() -- 2.51.2 From f7e4abbef26acca6f8128b4a739a8ffbb85425cf Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 1 Jun 2026 18:43:49 -0700 Subject: [PATCH 05/17] media: worker serves frames over a unix socket (detach transport) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the buffered frameServer into the worker as an alternate transport. When the handshake carries a SocketPath, ServeMKVIngestWorkerSocket listens on a per-session unix socket, serves the signed-segment stream over it (buffering across any main disconnect via frameServer), runs the full ingest, frames a trailing End/Error, then lingers until main has drained the buffer — bounded by workerDrainGrace — before closing the connection (clean EOF) and removing the socket. The ingest-worker subcommand selects this path on SocketPath; otherwise it keeps the Stage-1 fd-4 pipe. This is the worker half of the zero-downtime story: the socket + buffer let main disconnect for a restart and reconnect without the worker losing the segments it signs in the meantime. TestWorkerServesFramesOverSocket runs the real mux+sign+transcode pipeline and asserts a client reads valid signed dual-codec segments through to a clean End over the socket. (Buffer-across-restart correctness is covered deterministically by the frameServer reconnect tests.) Co-Authored-By: Claude Opus 4.8 --- pkg/cmd/streamplace.go | 6 ++ pkg/media/frame_server.go | 108 +++++++++++++++++++++++++++-- pkg/media/frame_socket_e2e_test.go | 91 ++++++++++++++++++++++++ pkg/media/ingest_worker.go | 5 ++ 4 files changed, 204 insertions(+), 6 deletions(-) create mode 100644 pkg/media/frame_socket_e2e_test.go diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index 99dae045..715c1d79 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -864,6 +864,12 @@ func makeIngestWorkerCommand(build *config.BuildFlags) *urfavecli.Command { return fmt.Errorf("ingest-worker: parse config: %w", err) } + // Detach/reattach transport: serve frames over a unix socket with + // buffered reconnect (survives a main restart) instead of the fd-4 pipe. + if cfg.SocketPath != "" { + return media.ServeMKVIngestWorkerSocket(ctx, cfg, os.Stdin) + } + framesFile := os.NewFile(4, "ingest-frames") if framesFile == nil { return fmt.Errorf("ingest-worker: missing frames fd 4") diff --git a/pkg/media/frame_server.go b/pkg/media/frame_server.go index 8c55e71c..47585c77 100644 --- a/pkg/media/frame_server.go +++ b/pkg/media/frame_server.go @@ -3,14 +3,27 @@ package media import ( "bytes" "context" + "fmt" "io" "net" + "os" "sync" + "time" "stream.place/streamplace/pkg/ingestframe" "stream.place/streamplace/pkg/log" ) +// workerFrameBuffer bounds how many signed segments a worker holds while main is +// disconnected — ~10 min at one segment per ~1s GoP. Beyond this the oldest are +// dropped (a main outage longer than this loses the oldest tail, loudly). +const workerFrameBuffer = 600 + +// workerDrainGrace bounds how long a worker lingers after its stream ends waiting +// for main to drain the buffer. Generous enough for a main restart/upgrade; an +// orphaned worker (main never returns) exits after it. +const workerDrainGrace = 60 * time.Second + // FrameWriter is the worker's segment sink. Stage 1 uses a direct framed pipe // (*ingestframe.Writer); the zero-downtime path uses *frameServer, which buffers // across a disconnected main and replays on reconnect. Structurally satisfied by @@ -39,12 +52,13 @@ type bufferedFrame struct { // loop). A push to an attached-but-dead client fails the write, auto-detaches, // and re-buffers that frame, so a hard main disconnect degrades to buffering. type frameServer struct { - mu sync.Mutex - pending []bufferedFrame - conn net.Conn - w *ingestframe.Writer - maxBuf int - dropped int + mu sync.Mutex + pending []bufferedFrame + conn net.Conn + w *ingestframe.Writer + maxBuf int + dropped int + everAttached bool } // newFrameServer creates a server that buffers up to maxBuf frames while no @@ -96,6 +110,47 @@ func (s *frameServer) attach(conn net.Conn) { } s.pending = nil s.conn, s.w = conn, w + s.everAttached = true +} + +// waitDrained blocks until a client has attached and the buffer is fully written +// out to it (so main has the whole stream, including the trailing End), or until +// ctx is cancelled or grace elapses. The worker calls this after the stream ends +// so it doesn't exit — discarding the in-memory buffer — before a reconnecting +// main has drained it. grace bounds an orphaned worker whose main never returns. +func (s *frameServer) waitDrained(ctx context.Context, grace time.Duration) { + deadline := time.NewTimer(grace) + defer deadline.Stop() + tick := time.NewTicker(100 * time.Millisecond) + defer tick.Stop() + for { + s.mu.Lock() + drained := s.everAttached && len(s.pending) == 0 + s.mu.Unlock() + if drained { + return + } + select { + case <-ctx.Done(): + return + case <-deadline.C: + log.Warn(ctx, "ingest worker: main never drained the frame buffer; exiting", "pending", len(s.pending)) + return + case <-tick.C: + } + } +} + +// closeConn closes the current client connection, giving main a clean EOF after +// the trailing End — the signal that the worker is done and won't reconnect. +func (s *frameServer) closeConn() { + s.mu.Lock() + c := s.conn + s.conn, s.w = nil, nil + s.mu.Unlock() + if c != nil { + c.Close() + } } // detachConn drops the named client if it's still the current one (a stale @@ -109,6 +164,47 @@ func (s *frameServer) detachConn(conn net.Conn) { } } +// ServeMKVIngestWorkerSocket runs the ingest worker, delivering its signed +// segments to main over a per-session unix socket at cfg.SocketPath with +// buffered reconnect — the zero-downtime path. It listens, serves the frame +// stream (buffering across any main disconnect), runs the ingest, frames a +// trailing End/Error, then lingers until main has drained the buffer before +// removing the socket and returning. +func ServeMKVIngestWorkerSocket(ctx context.Context, cfg IngestWorkerConfig, stdin io.Reader) error { + if cfg.SocketPath == "" { + return fmt.Errorf("ServeMKVIngestWorkerSocket: empty socket path") + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + _ = os.Remove(cfg.SocketPath) // clear any stale socket from a prior worker + ln, err := net.Listen("unix", cfg.SocketPath) + if err != nil { + return fmt.Errorf("listen %s: %w", cfg.SocketPath, err) + } + defer func() { + ln.Close() + _ = os.Remove(cfg.SocketPath) + }() + + srv := newFrameServer(workerFrameBuffer) + go serveFrameSocket(ctx, ln, srv) + + runErr := RunMKVIngestWorker(ctx, cfg, stdin, srv) + if runErr != nil { + _ = srv.Error(runErr.Error()) + } else { + _ = srv.End() + } + + // Don't exit (and drop the in-memory buffer) until main has the whole stream, + // including the trailing End — so a brief main restart loses nothing. + srv.waitDrained(ctx, workerDrainGrace) + // Close the connection so main reads End then a clean EOF (worker is done). + srv.closeConn() + return runErr +} + // serveFrameSocket accepts client connections on ln and attaches each to the // server, replacing any prior client (main reconnecting after a restart). Each // connection is watched for close so the server reverts to buffering. Returns diff --git a/pkg/media/frame_socket_e2e_test.go b/pkg/media/frame_socket_e2e_test.go new file mode 100644 index 00000000..33734803 --- /dev/null +++ b/pkg/media/frame_socket_e2e_test.go @@ -0,0 +1,91 @@ +package media + +import ( + "bytes" + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/crypto/signers" + "stream.place/streamplace/pkg/ingestframe" + "stream.place/streamplace/pkg/muxl" +) + +// TestWorkerServesFramesOverSocket drives the zero-downtime transport end-to-end +// with a REAL ingest: ServeMKVIngestWorkerSocket runs the full mux+sign+transcode +// pipeline and serves the resulting signed dual-codec segments over a unix +// socket; a client connects and reads them through to a clean End. This proves +// the socket path carries real signed media (the frameServer reconnect tests +// cover the buffer-across-restart behavior deterministically). +func TestWorkerServesFramesOverSocket(t *testing.T) { + ctx := context.Background() + ms := newBareSegmentSigner(t) + keyPEM, err := signers.MarshalES256KPrivateKeyPEM(ms.Signer) + require.NoError(t, err) + manifest, err := ms.buildManifest(ctx, time.Now().UnixMilli()) + require.NoError(t, err) + + sock := filepath.Join(t.TempDir(), "ingest.sock") + cfg := IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: ms.Cert, + Manifest: manifest, + NodeCertPEM: ms.Cert, + NodeKeyPEM: keyPEM, + BroadcasterHost: "test.example.com", + SocketPath: sock, + } + + mkv := makeH264AACMKV(t, ctx, getFixture("5sec.mp4")) + + serveDone := make(chan error, 1) + go func() { serveDone <- ServeMKVIngestWorkerSocket(ctx, cfg, bytes.NewReader(mkv)) }() + + // Connect once the worker's listener is up (retry the dial briefly). + var conn net.Conn + for i := 0; i < 100; i++ { + if conn, err = net.Dial("unix", sock); err == nil { + break + } + time.Sleep(50 * time.Millisecond) + } + require.NoError(t, err, "connect to worker frame socket") + defer conn.Close() + + r := ingestframe.NewReader(conn) + var segs int + var sawEnd bool + for { + typ, payload, rerr := r.ReadFrame() + if rerr != nil { + break // EOF after the worker exits + } + switch typ { + case ingestframe.Segment: + require.False(t, sawEnd, "no segments after End") + out, verr := muxl.RunMuxlVerify(ctx, bytes.NewReader(payload)) + require.NoError(t, verr, "segment %d verify", segs) + require.NotContains(t, out, `"validation_state":"Invalid"`, "segment %d valid", segs) + segs++ + case ingestframe.End: + sawEnd = true + case ingestframe.Error: + t.Fatalf("worker error frame: %s", payload) + } + } + + require.GreaterOrEqual(t, segs, 1, "worker served at least one signed segment over the socket") + require.True(t, sawEnd, "worker served a clean End over the socket") + + select { + case serveErr := <-serveDone: + require.NoError(t, serveErr) + case <-time.After(30 * time.Second): + t.Fatal("ServeMKVIngestWorkerSocket did not return after the stream drained") + } + t.Logf("worker served %d signed segments + End over the socket", segs) +} diff --git a/pkg/media/ingest_worker.go b/pkg/media/ingest_worker.go index a0f1c53f..1b10a6f7 100644 --- a/pkg/media/ingest_worker.go +++ b/pkg/media/ingest_worker.go @@ -40,6 +40,11 @@ type IngestWorkerConfig struct { NodeCertPEM []byte `json:"node_cert_pem,omitempty"` NodeKeyPEM []byte `json:"node_key_pem,omitempty"` BroadcasterHost string `json:"broadcaster_host,omitempty"` + + // SocketPath, when set, switches the worker to the detach/reattach transport: + // it serves frames over this unix socket with buffered reconnect (survives a + // main restart) instead of the fd-4 pipe. Empty → fd-4 pipe (Stage 1). + SocketPath string `json:"socket_path,omitempty"` } // RunMKVIngestWorker is the body of the `ingest-worker` subcommand. It reads an -- 2.51.2 From 11a37ac74e49e127052a5e903d742a7a21407727 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 1 Jun 2026 19:01:20 -0700 Subject: [PATCH 06/17] media: worker ingests media from an fd-passed connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Input-ownership foundation for zero-downtime (the fd-passing direction for MKV/RTMP). When the handshake carries InputFD, the worker reads its media straight off that fd — the connection main fd-passed after authing the push — instead of stdin, so main is out of the gst pipeline's data path and the worker keeps ingesting across a main restart. The subcommand resolves the media reader from InputFD in socket mode, else stdin. TestWorkerIngestsFromPassedFD spawns a real worker, passes it the ingest connection on fd 4 (a pipe standing in for the hijacked socket; the test feeds its far end), and asserts the worker ingests from that fd and serves valid signed dual-codec segments through to End over its frame socket — main never touching the gst data path. Remaining for full zero-downtime (main side): hijack + fd-pass the real push connection (httputil.NewChunkedReader over conn+prebuf for the body framing, passed via the handshake), daemonize the worker (Setsid), main-side reconnecting socket consumer, and socket-dir discovery on restart. Co-Authored-By: Claude Opus 4.8 --- pkg/cmd/streamplace.go | 13 +++- pkg/media/ingest_subprocess_test.go | 116 ++++++++++++++++++++++++++++ pkg/media/ingest_worker.go | 6 ++ 3 files changed, 134 insertions(+), 1 deletion(-) diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index 715c1d79..b0faf281 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -866,8 +866,19 @@ func makeIngestWorkerCommand(build *config.BuildFlags) *urfavecli.Command { // Detach/reattach transport: serve frames over a unix socket with // buffered reconnect (survives a main restart) instead of the fd-4 pipe. + // Media comes from the fd-passed ingest connection (InputFD) when main + // handed one off, else stdin. if cfg.SocketPath != "" { - return media.ServeMKVIngestWorkerSocket(ctx, cfg, os.Stdin) + input := io.Reader(os.Stdin) + if cfg.InputFD > 0 { + f := os.NewFile(uintptr(cfg.InputFD), "ingest-input") + if f == nil { + return fmt.Errorf("ingest-worker: bad input fd %d", cfg.InputFD) + } + defer f.Close() + input = f + } + return media.ServeMKVIngestWorkerSocket(ctx, cfg, input) } framesFile := os.NewFile(4, "ingest-frames") diff --git a/pkg/media/ingest_subprocess_test.go b/pkg/media/ingest_subprocess_test.go index 27b791f0..ea9d48dd 100644 --- a/pkg/media/ingest_subprocess_test.go +++ b/pkg/media/ingest_subprocess_test.go @@ -6,8 +6,10 @@ import ( "encoding/json" "errors" "io" + "net" "os" "os/exec" + "path/filepath" "testing" "time" @@ -35,6 +37,25 @@ func runIngestWorkerHelper() int { if err := json.Unmarshal(cfgBytes, &cfg); err != nil { return 2 } + + // Socket mode (Stage 4): serve frames over the unix socket; media comes from + // the fd-passed ingest connection (InputFD) when present, else stdin. + if cfg.SocketPath != "" { + input := io.Reader(os.Stdin) + if cfg.InputFD > 0 { + f := os.NewFile(uintptr(cfg.InputFD), "ingest-input") + if f == nil { + return 2 + } + defer f.Close() + input = f + } + if err := ServeMKVIngestWorkerSocket(context.Background(), cfg, input); err != nil { + return 1 + } + return 0 + } + framesFile := os.NewFile(4, "ingest-frames") if framesFile == nil { return 2 @@ -151,3 +172,98 @@ func TestMKVIngestIsolatedWedgeContained(t *testing.T) { require.Less(t, elapsed, 25*time.Second, "watchdog bounded the wedge") t.Logf("wedged worker contained in %s: %v", elapsed.Round(time.Second), err) } + +// TestWorkerIngestsFromPassedFD proves the input-ownership mechanism for +// zero-downtime: main fd-passes the (authed) ingest connection to the worker, +// which reads media straight off that fd — main is NOT in the gst pipeline's +// data path — and serves signed segments over its frame socket. Here the passed +// fd is a pipe whose far end the test feeds; in production it's the hijacked push +// connection, so the worker keeps ingesting across a main restart. (HTTP body +// de-framing on a real connection is a separate layer over this mechanism.) +func TestWorkerIngestsFromPassedFD(t *testing.T) { + ctx := context.Background() + ms := newBareSegmentSigner(t) + keyPEM, err := signers.MarshalES256KPrivateKeyPEM(ms.Signer) + require.NoError(t, err) + manifest, err := ms.buildManifest(ctx, time.Now().UnixMilli()) + require.NoError(t, err) + + sock := filepath.Join(t.TempDir(), "ingest.sock") + cfgJSON, err := json.Marshal(IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: ms.Cert, + Manifest: manifest, + NodeCertPEM: ms.Cert, + NodeKeyPEM: keyPEM, + BroadcasterHost: "test.example.com", + SocketPath: sock, + InputFD: 4, // main fd-passes the ingest connection on fd 4 + }) + require.NoError(t, err) + + mkv := makeH264AACMKV(t, ctx, getFixture("5sec.mp4")) + + exe, err := os.Executable() + require.NoError(t, err) + cmd := exec.CommandContext(ctx, exe, "ingest-worker") + cmd.Env = append(os.Environ(), "GST_DEBUG=0", "GST_TRACERS=") + cmd.Stderr = os.Stderr + + cfgR, cfgW, err := os.Pipe() + require.NoError(t, err) + // The fd-passed "connection": the worker reads mediaR (its fd 4); the test + // feeds mediaW. The worker's gst pipeline reads the fd itself — main is out of + // the data path entirely. + mediaR, mediaW, err := os.Pipe() + require.NoError(t, err) + cmd.ExtraFiles = []*os.File{cfgR, mediaR} // → child fd 3, fd 4 + + require.NoError(t, cmd.Start()) + cfgR.Close() + mediaR.Close() + go func() { + _, _ = cfgW.Write(cfgJSON) + cfgW.Close() + }() + go func() { + _, _ = mediaW.Write(mkv) + mediaW.Close() + }() + + var conn net.Conn + for i := 0; i < 100; i++ { + if conn, err = net.Dial("unix", sock); err == nil { + break + } + time.Sleep(50 * time.Millisecond) + } + require.NoError(t, err, "connect to worker frame socket") + defer conn.Close() + + r := ingestframe.NewReader(conn) + var segs int + var sawEnd bool + for { + typ, payload, rerr := r.ReadFrame() + if rerr != nil { + break + } + switch typ { + case ingestframe.Segment: + out, verr := muxl.RunMuxlVerify(ctx, bytes.NewReader(payload)) + require.NoError(t, verr, "segment %d verify", segs) + require.NotContains(t, out, `"validation_state":"Invalid"`, "segment %d valid", segs) + segs++ + case ingestframe.End: + sawEnd = true + case ingestframe.Error: + t.Fatalf("worker error frame: %s", payload) + } + } + + require.NoError(t, cmd.Wait(), "worker subprocess exits cleanly") + require.GreaterOrEqual(t, segs, 1, "worker ingested from the passed fd and served segments") + require.True(t, sawEnd, "clean End over the socket") + t.Logf("worker ingested from passed fd, served %d signed segments + End", segs) +} diff --git a/pkg/media/ingest_worker.go b/pkg/media/ingest_worker.go index 1b10a6f7..3c7b59da 100644 --- a/pkg/media/ingest_worker.go +++ b/pkg/media/ingest_worker.go @@ -45,6 +45,12 @@ type IngestWorkerConfig struct { // it serves frames over this unix socket with buffered reconnect (survives a // main restart) instead of the fd-4 pipe. Empty → fd-4 pipe (Stage 1). SocketPath string `json:"socket_path,omitempty"` + + // InputFD, when > 0, is the fd main passed the ingest CONNECTION on (fd-passing + // the accepted, authed push). The worker reads media from it directly instead + // of stdin, so main is out of the media path and the worker keeps ingesting + // across a main restart. 0 → read media from stdin. + InputFD int `json:"input_fd,omitempty"` } // RunMKVIngestWorker is the body of the `ingest-worker` subcommand. It reads an -- 2.51.2 From b6848e6a4f5be37cb11dfed12721aa6227e88763 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 1 Jun 2026 19:10:29 -0700 Subject: [PATCH 07/17] =?UTF-8?q?media:=20detached=20worker=20lifecycle=20?= =?UTF-8?q?=E2=80=94=20spawn,=20discover,=20reconnecting=20consume?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The main-side half of zero-downtime. SpawnIngestWorkerDetached launches the worker in its own session (Setsid) so it outlives a main restart, fd-passing the authed ingest connection (fd 4) and config (fd 3, written synchronously so nothing must outlive a restarting main); it is not ctx-tied or waited on here. ConsumeWorkerSocket connects to the worker's frame socket and feeds segments to an injected handler (ValidateMP4 in prod), reconnecting across transient disconnects — the worker buffers + replays while disconnected, and ValidateMP4 dedup makes replayed overlap idempotent. DiscoverWorkerSockets lists running workers under the socket dir for a restarting main to reconnect to. consumeWorkerFrames now takes an injected onSegment so the same reader serves both the fd-4 pipe supervisor and the socket consumer. TestDetachedWorkerZeroDowntime runs it end to end: spawn a DETACHED worker (asserts it leads its own process group — the property that lets it survive a main restart) fd-passing real media, discover its socket the way a restarting main would, consume via the reconnecting consumer, and verify valid signed dual-codec segments through a clean End. Plus a DiscoverWorkerSockets unit test. Remaining: wire handleIncomingStream to hijack + fd-pass the real push connection into SpawnIngestWorkerDetached, and run discovery on main startup. Co-Authored-By: Claude Opus 4.8 --- pkg/media/ingest_daemon.go | 124 ++++++++++++++++++++++++++++++++ pkg/media/ingest_daemon_test.go | 100 ++++++++++++++++++++++++++ pkg/media/ingest_supervisor.go | 23 ++++-- 3 files changed, 242 insertions(+), 5 deletions(-) create mode 100644 pkg/media/ingest_daemon.go create mode 100644 pkg/media/ingest_daemon_test.go diff --git a/pkg/media/ingest_daemon.go b/pkg/media/ingest_daemon.go new file mode 100644 index 00000000..00ea09ed --- /dev/null +++ b/pkg/media/ingest_daemon.go @@ -0,0 +1,124 @@ +package media + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "stream.place/streamplace/pkg/log" +) + +// ingestReconnectBackoff paces redials when a worker is up but main's connection +// dropped (a restart in progress). +const ingestReconnectBackoff = 250 * time.Millisecond + +// SpawnIngestWorkerDetached launches an ingest worker in its OWN session +// (Setsid) so it outlives a main restart, fd-passing the (already authed) ingest +// connection as the worker's media input (fd 4) and having it serve signed +// segments over cfg.SocketPath with buffered reconnect. The config rides a pipe +// on fd 3, written synchronously so nothing has to outlive a restarting main. +// +// The worker is deliberately NOT tied to main's context and NOT waited on here: +// it self-terminates after its stream drains (or its watchdog/orphan-grace +// fires), and is reaped by init once main exits. The returned process lets a +// caller that stays alive reap it; a restarting main just lets it go. +func SpawnIngestWorkerDetached(cfg IngestWorkerConfig, media *os.File) (*os.Process, error) { + if cfg.SocketPath == "" { + return nil, fmt.Errorf("SpawnIngestWorkerDetached: empty socket path") + } + exe, err := os.Executable() + if err != nil { + return nil, err + } + cfgJSON, err := json.Marshal(cfg) + if err != nil { + return nil, err + } + cfgR, cfgW, err := os.Pipe() + if err != nil { + return nil, err + } + defer cfgR.Close() + defer cfgW.Close() + + cmd := exec.Command(exe, "ingest-worker") + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} // detach into its own session + cmd.ExtraFiles = []*os.File{cfgR, media} // → child fd 3 (config), fd 4 (media) + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("spawn ingest worker: %w", err) + } + // Small, synchronous write: the bytes sit in the pipe buffer for the worker to + // read even after we close + return (and even if main then exits/restarts). + if _, err := cfgW.Write(cfgJSON); err != nil { + _ = cmd.Process.Kill() + return nil, fmt.Errorf("send worker config: %w", err) + } + return cmd.Process, nil +} + +// ConsumeWorkerSocket connects to a worker's frame socket and feeds its segments +// to onSegment, reconnecting across transient disconnects. A worker that +// outlived a main restart keeps buffering and replays on reconnect; ValidateMP4's +// dedup makes any replayed overlap idempotent. Returns nil on a clean End, or an +// error if the socket vanishes without one (worker crashed — contained). +func (mm *MediaManager) ConsumeWorkerSocket(ctx context.Context, socketPath, streamer string, onSegment func([]byte) error) error { + connectedOnce := false + for { + conn, err := net.Dial("unix", socketPath) + if err != nil { + if !connectedOnce { + select { // worker may still be coming up + case <-ctx.Done(): + return ctx.Err() + case <-time.After(ingestReconnectBackoff): + continue + } + } + return fmt.Errorf("ingest worker socket gone before End: %w", err) + } + connectedOnce = true + sawEnd, _ := mm.consumeWorkerFrames(ctx, conn, streamer, onSegment, nil) + conn.Close() + if sawEnd { + return nil + } + if ctx.Err() != nil { + return ctx.Err() + } + // Transient disconnect: reconnect and let the worker replay its buffer. + log.Log(ctx, "ingest worker connection dropped; reconnecting", "streamer", streamer) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(ingestReconnectBackoff): + } + } +} + +// DiscoverWorkerSockets lists the worker frame sockets under dir — the running +// workers a restarting main should reconnect to and resume consuming. +func DiscoverWorkerSockets(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, err + } + var socks []string + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".sock") { + socks = append(socks, filepath.Join(dir, e.Name())) + } + } + return socks, nil +} diff --git a/pkg/media/ingest_daemon_test.go b/pkg/media/ingest_daemon_test.go new file mode 100644 index 00000000..7d2ab55c --- /dev/null +++ b/pkg/media/ingest_daemon_test.go @@ -0,0 +1,100 @@ +package media + +import ( + "bytes" + "context" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/crypto/signers" + "stream.place/streamplace/pkg/muxl" +) + +func TestDiscoverWorkerSockets(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "notes.txt"), []byte("x"), 0o600)) // ignored + for _, n := range []string{"a.sock", "b.sock"} { + require.NoError(t, os.WriteFile(filepath.Join(dir, n), nil, 0o600)) + } + socks, err := DiscoverWorkerSockets(dir) + require.NoError(t, err) + require.Len(t, socks, 2) + + socks, err = DiscoverWorkerSockets(filepath.Join(dir, "nope")) // missing dir → empty, no error + require.NoError(t, err) + require.Empty(t, socks) +} + +// TestDetachedWorkerZeroDowntime exercises the main-side lifecycle end to end: +// spawn a worker DETACHED (its own session, so it survives a main restart), +// fd-passing its media; discover its socket the way a restarting main would; +// consume via the reconnecting consumer; and verify it served valid signed +// dual-codec segments through a clean End. The session check confirms the +// detachment that lets the worker outlive main. +func TestDetachedWorkerZeroDowntime(t *testing.T) { + ctx := context.Background() + ms := newBareSegmentSigner(t) + keyPEM, err := signers.MarshalES256KPrivateKeyPEM(ms.Signer) + require.NoError(t, err) + manifest, err := ms.buildManifest(ctx, time.Now().UnixMilli()) + require.NoError(t, err) + + dir := t.TempDir() + sock := filepath.Join(dir, "stream.sock") + cfg := IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: ms.Cert, + Manifest: manifest, + NodeCertPEM: ms.Cert, + NodeKeyPEM: keyPEM, + BroadcasterHost: "test.example.com", + SocketPath: sock, + InputFD: 4, + } + + mkv := makeH264AACMKV(t, ctx, getFixture("5sec.mp4")) + mediaR, mediaW, err := os.Pipe() + require.NoError(t, err) + + proc, err := SpawnIngestWorkerDetached(cfg, mediaR) + require.NoError(t, err) + mediaR.Close() // the worker holds its own dup + go func() { + _, _ = mediaW.Write(mkv) + mediaW.Close() + }() + + // A (re)starting main discovers the running worker by scanning the socket dir. + var socks []string + require.Eventually(t, func() bool { + socks, _ = DiscoverWorkerSockets(dir) + return len(socks) == 1 + }, 15*time.Second, 100*time.Millisecond, "worker socket appears for discovery") + + // The worker runs in its own session/process group (Setsid) — the property + // that lets it outlive a main restart. + wpgid, err := syscall.Getpgid(proc.Pid) + require.NoError(t, err) + require.Equal(t, proc.Pid, wpgid, "worker leads its own process group (Setsid detached)") + require.NotEqual(t, syscall.Getpgrp(), wpgid, "worker group differs from the test's") + + // Consume through the reconnecting consumer; verify dual-codec signed output. + var segs int + onSegment := func(s []byte) error { + out, verr := muxl.RunMuxlVerify(ctx, bytes.NewReader(s)) + require.NoError(t, verr) + require.NotContains(t, out, `"validation_state":"Invalid"`) + segs++ + return nil + } + require.NoError(t, (&MediaManager{}).ConsumeWorkerSocket(ctx, socks[0], ms.Streamer(), onSegment)) + require.GreaterOrEqual(t, segs, 1, "detached worker served signed segments") + + _, _ = proc.Wait() // reap the detached worker + t.Logf("detached worker (pgid=%d) served %d dual-codec segments via discovery+reconnect", wpgid, segs) +} diff --git a/pkg/media/ingest_supervisor.go b/pkg/media/ingest_supervisor.go index 52d365eb..0dd018b1 100644 --- a/pkg/media/ingest_supervisor.go +++ b/pkg/media/ingest_supervisor.go @@ -142,7 +142,7 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, defer watchdog.Stop() // Read signed-segment frames and feed each into the normal chokepoint. - sawEnd, readErr := mm.consumeWorkerFrames(ctx, framesR, ms.Streamer(), func() { + sawEnd, readErr := mm.consumeWorkerFrames(ctx, framesR, ms.Streamer(), mm.validateSegment(ctx), func() { watchdog.Reset(ingestWorkerWatchdog) }) logsWG.Wait() @@ -165,8 +165,8 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, // over each. It returns whether a clean End frame was seen and the terminal read // error: nil on a clean close (End then EOF), or io.ErrUnexpectedEOF / a desync // error when the worker died mid-frame. -func (mm *MediaManager) consumeWorkerFrames(ctx context.Context, stdout io.Reader, streamer string, onProgress func()) (sawEnd bool, _ error) { - fr := ingestframe.NewReader(stdout) +func (mm *MediaManager) consumeWorkerFrames(ctx context.Context, r io.Reader, streamer string, onSegment func([]byte) error, onProgress func()) (sawEnd bool, _ error) { + fr := ingestframe.NewReader(r) for { typ, payload, err := fr.ReadFrame() if err != nil { @@ -180,8 +180,12 @@ func (mm *MediaManager) consumeWorkerFrames(ctx context.Context, stdout io.Reade } switch typ { case ingestframe.Segment: - if verr := mm.ValidateMP4(ctx, bytes.NewReader(payload), true); verr != nil { - log.Error(ctx, "ingest worker: validate segment failed", "streamer", streamer, "error", verr) + if onSegment != nil { + if serr := onSegment(payload); serr != nil { + // Per-segment failures are logged, not fatal to the stream — a + // bad GoP shouldn't tear down an otherwise-healthy ingest. + log.Error(ctx, "ingest worker: segment handler failed", "streamer", streamer, "error", serr) + } } case ingestframe.End: sawEnd = true @@ -191,6 +195,15 @@ func (mm *MediaManager) consumeWorkerFrames(ctx context.Context, stdout io.Reade } } +// validateSegment is the onSegment handler for ingested worker frames: it folds +// each signed segment into the normal ValidateMP4 chokepoint (verify → archive → +// live-HLS → notify). +func (mm *MediaManager) validateSegment(ctx context.Context) func([]byte) error { + return func(seg []byte) error { + return mm.ValidateMP4(ctx, bytes.NewReader(seg), true) + } +} + // streamWorkerLogs forwards the worker's stderr lines into the node logger. func streamWorkerLogs(ctx context.Context, stderr io.Reader, streamer string) { scan := bufio.NewScanner(stderr) -- 2.51.2 From d433cfb9255b2544ae70e0d46aed827f95b7fb94 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 1 Jun 2026 19:17:38 -0700 Subject: [PATCH 08/17] media: wire the detached zero-downtime ingest path end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connect the production entry points to the detached worker machinery. - Worker body de-framing: WorkerInput reconstructs the raw media off the fd-passed push connection — prepend the bytes main read past the headers (Prebuf) then de-chunk if the push used chunked transfer-encoding (httputil.NewChunkedReader). The socket-mode subcommand applies it. - MKVIngestDetached: main's hijacked push connection → fd-pass to a detached worker on a per-session unix socket → consume its frames into ValidateMP4 with reconnect; reap on clean end, leave running on a main-shutdown ctx cancel. - ResumeDetachedWorkers: at startup, discover sockets of workers that outlived a restart and resume draining them. Wired into runMain under --isolated-ingest. - handleIncomingStream: with --isolated-ingest, hijack the authed push and route to MKVIngestDetached (falling back to the fd-4-pipe isolation when the connection can't be hijacked). TestWorkerInputDeframes covers the prebuf + chunked de-framing deterministically. The detached lifecycle (spawn/discover/reconnect/serve) is proven by TestDetachedWorkerZeroDowntime; the buffer-across-restart by the frameServer tests. The api hijack glue is thin over those tested pieces and is the one bit that still needs a real-push integration check (no in-container harness for it). Co-Authored-By: Claude Opus 4.8 --- pkg/api/api_internal.go | 22 ++++++++ pkg/cmd/streamplace.go | 11 +++- pkg/media/ingest_daemon.go | 86 +++++++++++++++++++++++++++++ pkg/media/ingest_subprocess_test.go | 6 +- pkg/media/ingest_worker.go | 25 +++++++++ pkg/media/ingest_worker_test.go | 21 +++++++ 6 files changed, 165 insertions(+), 6 deletions(-) diff --git a/pkg/api/api_internal.go b/pkg/api/api_internal.go index f3f5a529..67b54ae3 100644 --- a/pkg/api/api_internal.go +++ b/pkg/api/api_internal.go @@ -263,6 +263,28 @@ func (a *StreamplaceAPI) InternalHandler(ctx context.Context) (http.Handler, err } if a.CLI.IsolatedIngest { + // Zero-downtime path: hijack the authed push connection and hand it to a + // DETACHED worker that owns the connection (so it survives a main + // restart) and serves signed segments back over its socket. + if hj, ok := w.(http.Hijacker); ok { + conn, bufrw, herr := hj.Hijack() + if herr != nil { + log.Error(reqCtx, "ingest hijack failed", "error", herr) + return + } + var prebuf []byte + if n := bufrw.Reader.Buffered(); n > 0 { + prebuf = make([]byte, n) + _, _ = io.ReadFull(bufrw.Reader, prebuf) + } + chunked := len(httpReq.TransferEncoding) > 0 && httpReq.TransferEncoding[0] == "chunked" + if derr := a.MediaManager.MKVIngestDetached(reqCtx, conn, prebuf, chunked, mediaSigner); derr != nil { + log.Log(reqCtx, "isolated stream ended", "error", derr) + } + return // connection hijacked; the HTTP response is ours now + } + // No hijack support (HTTP/2, some proxies) → fd-4-pipe isolation: still + // fault-isolated, just no restart-survival. err = a.MediaManager.MKVIngestIsolated(reqCtx, r, mediaSigner) } else { err = a.MediaManager.MKVIngest(reqCtx, r, mediaSigner) diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index b0faf281..8b3f1f29 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -263,6 +263,11 @@ func runMain(ctx context.Context, build *config.BuildFlags, platformJobs []jobFu if err != nil { return err } + if cli.IsolatedIngest { + // Reconnect to any ingest workers still running from before this restart + // and drain whatever they buffered while we were down (zero-downtime). + mm.ResumeDetachedWorkers(ctx) + } ms, err := media.MakeMediaSigner(ctx, cli, cli.StreamerName, signer, mod) if err != nil { @@ -869,16 +874,16 @@ func makeIngestWorkerCommand(build *config.BuildFlags) *urfavecli.Command { // Media comes from the fd-passed ingest connection (InputFD) when main // handed one off, else stdin. if cfg.SocketPath != "" { - input := io.Reader(os.Stdin) + raw := io.Reader(os.Stdin) if cfg.InputFD > 0 { f := os.NewFile(uintptr(cfg.InputFD), "ingest-input") if f == nil { return fmt.Errorf("ingest-worker: bad input fd %d", cfg.InputFD) } defer f.Close() - input = f + raw = f } - return media.ServeMKVIngestWorkerSocket(ctx, cfg, input) + return media.ServeMKVIngestWorkerSocket(ctx, cfg, media.WorkerInput(cfg, raw)) } framesFile := os.NewFile(4, "ingest-frames") diff --git a/pkg/media/ingest_daemon.go b/pkg/media/ingest_daemon.go index 00ea09ed..0af289fe 100644 --- a/pkg/media/ingest_daemon.go +++ b/pkg/media/ingest_daemon.go @@ -13,6 +13,7 @@ import ( "syscall" "time" + "github.com/google/uuid" "stream.place/streamplace/pkg/log" ) @@ -104,6 +105,91 @@ func (mm *MediaManager) ConsumeWorkerSocket(ctx context.Context, socketPath, str } } +// ingestWorkerSocketDir returns (creating it) the directory of per-session +// worker frame sockets — the set a restarting main scans to resume. +func (mm *MediaManager) ingestWorkerSocketDir() (string, error) { + dir := mm.cli.DataFilePath([]string{"ingest-workers"}) + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + return dir, nil +} + +// MKVIngestDetached is the production zero-downtime entry: main has authed the +// push and hijacked its connection; this fd-passes that connection to a DETACHED +// worker (own session, survives a main restart) which ingests the media directly +// and serves signed segments over a per-session unix socket, and then consumes +// those frames into ValidateMP4 with reconnect. prebuf is any body bytes main +// already read past the headers; chunked says the push body is chunked. +// +// Because the worker owns the connection and is detached, a main restart neither +// breaks the ingest nor loses output: the worker keeps signing into its buffer, +// and the restarted main rediscovers the socket (DiscoverWorkerSockets) and +// drains it. +func (mm *MediaManager) MKVIngestDetached(ctx context.Context, conn net.Conn, prebuf []byte, chunked bool, ms MediaSigner) error { + cfg, err := mm.buildWorkerConfig(ctx, ms) + if err != nil { + return err + } + dir, err := mm.ingestWorkerSocketDir() + if err != nil { + return err + } + cfg.SocketPath = filepath.Join(dir, uuid.NewString()+".sock") + cfg.InputFD = 4 + cfg.Prebuf = prebuf + cfg.Chunked = chunked + + tcp, ok := conn.(*net.TCPConn) + if !ok { + return fmt.Errorf("isolated ingest requires a TCP connection, got %T", conn) + } + connFile, err := tcp.File() // dup the fd to hand to the worker + if err != nil { + return fmt.Errorf("dup ingest connection: %w", err) + } + + proc, err := SpawnIngestWorkerDetached(cfg, connFile) + connFile.Close() // the worker holds its own dup + conn.Close() // main is out of the media path now + if err != nil { + return fmt.Errorf("spawn detached worker: %w", err) + } + + err = mm.ConsumeWorkerSocket(ctx, cfg.SocketPath, ms.Streamer(), mm.validateSegment(ctx)) + if err == nil { + go func() { _, _ = proc.Wait() }() // clean end: reap the exiting worker + } + // On ctx cancel (main shutting down) we deliberately leave the detached worker + // running; a restarting main reconnects via discovery. + return err +} + +// ResumeDetachedWorkers reconnects to any ingest workers still running from +// before a main restart and resumes consuming their frames (draining whatever +// they buffered while main was down). Intended to run once at main startup. +func (mm *MediaManager) ResumeDetachedWorkers(ctx context.Context) { + dir, err := mm.ingestWorkerSocketDir() + if err != nil { + log.Error(ctx, "resume ingest workers: socket dir", "error", err) + return + } + socks, err := DiscoverWorkerSockets(dir) + if err != nil { + log.Error(ctx, "resume ingest workers: discover", "error", err) + return + } + for _, sock := range socks { + sock := sock + log.Log(ctx, "resuming detached ingest worker", "socket", sock) + go func() { + if cerr := mm.ConsumeWorkerSocket(ctx, sock, "resumed", mm.validateSegment(ctx)); cerr != nil { + log.Error(ctx, "resumed ingest worker ended", "socket", sock, "error", cerr) + } + }() + } +} + // DiscoverWorkerSockets lists the worker frame sockets under dir — the running // workers a restarting main should reconnect to and resume consuming. func DiscoverWorkerSockets(dir string) ([]string, error) { diff --git a/pkg/media/ingest_subprocess_test.go b/pkg/media/ingest_subprocess_test.go index ea9d48dd..04d518d0 100644 --- a/pkg/media/ingest_subprocess_test.go +++ b/pkg/media/ingest_subprocess_test.go @@ -41,16 +41,16 @@ func runIngestWorkerHelper() int { // Socket mode (Stage 4): serve frames over the unix socket; media comes from // the fd-passed ingest connection (InputFD) when present, else stdin. if cfg.SocketPath != "" { - input := io.Reader(os.Stdin) + raw := io.Reader(os.Stdin) if cfg.InputFD > 0 { f := os.NewFile(uintptr(cfg.InputFD), "ingest-input") if f == nil { return 2 } defer f.Close() - input = f + raw = f } - if err := ServeMKVIngestWorkerSocket(context.Background(), cfg, input); err != nil { + if err := ServeMKVIngestWorkerSocket(context.Background(), cfg, WorkerInput(cfg, raw)); err != nil { return 1 } return 0 diff --git a/pkg/media/ingest_worker.go b/pkg/media/ingest_worker.go index 3c7b59da..9a9321bc 100644 --- a/pkg/media/ingest_worker.go +++ b/pkg/media/ingest_worker.go @@ -1,9 +1,11 @@ package media import ( + "bytes" "context" "fmt" "io" + "net/http/httputil" "github.com/go-gst/go-gst/gst" "stream.place/streamplace/pkg/config" @@ -51,6 +53,29 @@ type IngestWorkerConfig struct { // of stdin, so main is out of the media path and the worker keeps ingesting // across a main restart. 0 → read media from stdin. InputFD int `json:"input_fd,omitempty"` + + // Prebuf is the HTTP body bytes main had already read past the request headers + // when it hijacked the push connection — prepended to the fd stream so none are + // lost. Chunked says the body uses chunked transfer-encoding, so the worker + // de-chunks the (prebuf+fd) stream to recover the raw media. Both are unset for + // stdin / raw fd input. + Prebuf []byte `json:"prebuf,omitempty"` + Chunked bool `json:"chunked,omitempty"` +} + +// WorkerInput reconstructs the raw media stream the gst pipeline reads from the +// fd-passed push connection: prepend any bytes main already read past the headers +// (Prebuf), then de-chunk if the push used chunked transfer-encoding. For stdin +// or a raw fd (no prebuf, not chunked) it returns raw unchanged. +func WorkerInput(cfg IngestWorkerConfig, raw io.Reader) io.Reader { + r := raw + if len(cfg.Prebuf) > 0 { + r = io.MultiReader(bytes.NewReader(cfg.Prebuf), r) + } + if cfg.Chunked { + r = httputil.NewChunkedReader(r) + } + return r } // RunMKVIngestWorker is the body of the `ingest-worker` subcommand. It reads an diff --git a/pkg/media/ingest_worker_test.go b/pkg/media/ingest_worker_test.go index efb699c9..c1d73299 100644 --- a/pkg/media/ingest_worker_test.go +++ b/pkg/media/ingest_worker_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "strings" "testing" @@ -18,6 +19,26 @@ import ( "stream.place/streamplace/pkg/muxl" ) +// TestWorkerInputDeframes checks the body-deframing the worker applies to the +// fd-passed push connection: prebuf (bytes main read past the headers) is +// prepended, and a chunked transfer-encoding is decoded back to the raw media. +func TestWorkerInputDeframes(t *testing.T) { + payload := []byte("the-actual-media-bytes-pretend-this-is-mkv-data") + // A textbook chunked body: one chunk then the zero terminator. + body := []byte(fmt.Sprintf("%x\r\n%s\r\n0\r\n\r\n", len(payload), payload)) + + // prebuf = the slice main already read; the rest is still on the fd. + cfg := IngestWorkerConfig{Chunked: true, Prebuf: append([]byte(nil), body[:5]...)} + got, err := io.ReadAll(WorkerInput(cfg, bytes.NewReader(body[5:]))) + require.NoError(t, err) + require.Equal(t, payload, got, "prebuf + chunked fd de-frames to the original media") + + // Raw (no prebuf, not chunked) passes through unchanged. + got, err = io.ReadAll(WorkerInput(IngestWorkerConfig{}, bytes.NewReader(payload))) + require.NoError(t, err) + require.Equal(t, payload, got) +} + // makeH264AACMKV builds a clean, single-track, streamable H264+AAC MKV from an // H264+Opus MP4 fixture (video passed through, audio transcoded Opus→AAC). The // repo's only AAC fixture (sample-stream.mkv) carries four audio tracks, which -- 2.51.2 From d86d3de64f3366947a6b6072597a1b9d1b68145a Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 1 Jun 2026 19:51:41 -0700 Subject: [PATCH 09/17] media: gate ingest isolation to Linux; keep Windows/macOS building The worker-subprocess transport relies on Unix fd-passing (exec.Cmd.ExtraFiles, unsupported on Windows) and POSIX Setsid for detachment. syscall.SysProcAttr has no Setsid field on Windows, so ingest_daemon.go was a hard compile break there. Split the platform bit behind a build tag (mirroring the existing runMist/streamplace_{linux,notlinux}.go precedent): setDetached + IngestIsolationSupported live in ingest_isolation_linux.go (real) and ingest_isolation_notlinux.go (no-op / false). runMain forces --isolated-ingest off and logs when the platform doesn't support it, so non-Linux falls back cleanly to in-process ingest. The Setsid-using daemon test is now //go:build linux. go list confirms Windows now selects the stub and drops the linux file; macOS has the primitives and could be enabled later by widening the tag to `unix`. Co-Authored-By: Claude Opus 4.8 --- pkg/cmd/streamplace.go | 6 ++++++ pkg/media/ingest_daemon.go | 5 ++--- pkg/media/ingest_daemon_test.go | 2 ++ pkg/media/ingest_isolation_linux.go | 23 +++++++++++++++++++++++ pkg/media/ingest_isolation_notlinux.go | 17 +++++++++++++++++ 5 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 pkg/media/ingest_isolation_linux.go create mode 100644 pkg/media/ingest_isolation_notlinux.go diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index 8b3f1f29..36ce386f 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -263,6 +263,12 @@ func runMain(ctx context.Context, build *config.BuildFlags, platformJobs []jobFu if err != nil { return err } + if cli.IsolatedIngest && !media.IngestIsolationSupported() { + // The worker transport needs Unix fd-passing + Setsid (Linux today); fall + // back to in-process ingest elsewhere rather than break. + log.Log(ctx, "isolated ingest not supported on this platform; using in-process ingest", "goos", runtime.GOOS) + cli.IsolatedIngest = false + } if cli.IsolatedIngest { // Reconnect to any ingest workers still running from before this restart // and drain whatever they buffered while we were down (zero-downtime). diff --git a/pkg/media/ingest_daemon.go b/pkg/media/ingest_daemon.go index 0af289fe..c027f4bb 100644 --- a/pkg/media/ingest_daemon.go +++ b/pkg/media/ingest_daemon.go @@ -10,7 +10,6 @@ import ( "os/exec" "path/filepath" "strings" - "syscall" "time" "github.com/google/uuid" @@ -51,8 +50,8 @@ func SpawnIngestWorkerDetached(cfg IngestWorkerConfig, media *os.File) (*os.Proc defer cfgW.Close() cmd := exec.Command(exe, "ingest-worker") - cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} // detach into its own session - cmd.ExtraFiles = []*os.File{cfgR, media} // → child fd 3 (config), fd 4 (media) + setDetached(cmd) // own session, survives a main restart (Linux) + cmd.ExtraFiles = []*os.File{cfgR, media} // → child fd 3 (config), fd 4 (media) cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { return nil, fmt.Errorf("spawn ingest worker: %w", err) diff --git a/pkg/media/ingest_daemon_test.go b/pkg/media/ingest_daemon_test.go index 7d2ab55c..99502fd8 100644 --- a/pkg/media/ingest_daemon_test.go +++ b/pkg/media/ingest_daemon_test.go @@ -1,3 +1,5 @@ +//go:build linux + package media import ( diff --git a/pkg/media/ingest_isolation_linux.go b/pkg/media/ingest_isolation_linux.go new file mode 100644 index 00000000..f318c738 --- /dev/null +++ b/pkg/media/ingest_isolation_linux.go @@ -0,0 +1,23 @@ +//go:build linux + +package media + +import ( + "os/exec" + "syscall" +) + +// IngestIsolationSupported reports whether per-stream ingest isolation (worker +// subprocesses with fd-passing + Setsid detachment) is available on this +// platform. It relies on Unix fd inheritance (exec.Cmd.ExtraFiles — unsupported +// on Windows) and POSIX sessions (Setsid), so it is gated to Linux, the platform +// Streamplace targets in production. Elsewhere --isolated-ingest falls back to +// in-process ingest. macOS has these primitives too and could be enabled by +// widening the build tag to `unix` once verified there. +func IngestIsolationSupported() bool { return true } + +// setDetached puts a spawned worker in its own session so it survives a main +// restart (the zero-downtime path). +func setDetached(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} +} diff --git a/pkg/media/ingest_isolation_notlinux.go b/pkg/media/ingest_isolation_notlinux.go new file mode 100644 index 00000000..58fa155b --- /dev/null +++ b/pkg/media/ingest_isolation_notlinux.go @@ -0,0 +1,17 @@ +//go:build !linux + +package media + +import "os/exec" + +// IngestIsolationSupported is false off Linux: the worker-subprocess transport +// relies on Unix fd-passing (exec.Cmd.ExtraFiles, unsupported on Windows) and +// POSIX Setsid, so --isolated-ingest falls back to in-process ingest on these +// platforms (see the gate in runMain). This keeps the build green everywhere; +// Linux is where Streamplace runs in production. +func IngestIsolationSupported() bool { return false } + +// setDetached is a no-op off Linux — the detached path is gated off there, so +// this only needs to keep the build compiling (no Setsid field exists on +// Windows' syscall.SysProcAttr). +func setDetached(cmd *exec.Cmd) {} -- 2.51.2 From cb79f9c942c407a688307bebc04c7551863da74e Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Tue, 2 Jun 2026 13:19:07 -0700 Subject: [PATCH 10/17] media: isolate WHIP ingest in a detached worker (zero-downtime) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends per-stream isolation to WHIP/WebRTC. Unlike MKV there's no socket to fd-pass: the worker OWNS the PeerConnection (built from the offer, binding its own UDP sockets), so the media flows straight to it and the session survives a main restart. The new wrinkle is the SDP answer — the worker generates it and emits it as the FIRST frame on its socket (ingestframe.Answer); main reads that, returns it to the WHIP client, then consumes the signed dual-codec segments. - Shared the machinery: extracted newWebRTCAPI (pion API/config) so the worker builds its own; factored workerSignStream + workerSegmentSink (dual-codec transcode → frames) out of RunMKVIngestWorker for both workers to use. - Refactored WebRTCIngest into a pre-built-signer-element core (webRTCIngestPipeline): in-process passes SegmentAndSignElem + the streamer's signer (key revocation); the worker passes a muxlSignSegmentElem wired to its frame socket + a nil keyRevSigner. - ServeWHIPIngestWorkerSocket: build PC, run the pipeline, frame the answer first, serve segments over the socket with drain/close. cfg gains Transport("whip")+OfferSDP; the subcommand branches on it. - WHIPIngestDetached (main): spawn the detached worker (no media fd), read the Answer frame (bounded by whipAnswerTimeout), then consume segments with reconnect. HandleWebRTCIngest routes here under --isolated-ingest (forced off where unsupported, so the in-process path stays the default elsewhere). Tests: Answer frame round-trip; TestWHIPWorkerAnswersOffer (offer → valid SDP answer that applies as the client's remote description); and TestWHIPWorkerLoopback, the WHIP parity of the MKV worker e2e test — a real pion client connects over ICE/DTLS and streams H264+Opus RTP, and the worker mux+sign+transcodes it into a valid signed dual-codec segment served over its socket. Co-Authored-By: Claude Opus 4.8 --- pkg/api/playback.go | 35 +++-- pkg/cmd/streamplace.go | 6 + pkg/ingestframe/frame.go | 10 ++ pkg/ingestframe/frame_test.go | 2 + pkg/media/frame_server.go | 1 + pkg/media/ingest_daemon.go | 112 +++++++++++++++- pkg/media/ingest_worker.go | 103 +++++++++------ pkg/media/media.go | 46 +------ pkg/media/webrtc_api.go | 49 +++++++ pkg/media/webrtc_ingest.go | 39 ++++-- pkg/media/whip_worker.go | 105 +++++++++++++++ pkg/media/whip_worker_test.go | 237 ++++++++++++++++++++++++++++++++++ 12 files changed, 635 insertions(+), 110 deletions(-) create mode 100644 pkg/media/webrtc_api.go create mode 100644 pkg/media/whip_worker.go create mode 100644 pkg/media/whip_worker_test.go diff --git a/pkg/api/playback.go b/pkg/api/playback.go index 1445d37e..260060d5 100644 --- a/pkg/api/playback.go +++ b/pkg/api/playback.go @@ -105,15 +105,30 @@ func (a *StreamplaceAPI) HandleWebRTCIngest(ctx context.Context) httprouter.Hand return } offer := webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: string(body)} - pc, err := a.MediaManager.NewPeerConnection(ctx, mediaSigner.Streamer()) - if err != nil { - errors.WriteHTTPInternalServerError(w, "unable to create peer connection", err) - return - } - answer, err := a.MediaManager.WebRTCIngest(ctx, &offer, mediaSigner, pc, make(chan error, 1)) - if err != nil { - errors.WriteHTTPInternalServerError(w, fmt.Sprintf("error ingesting: %s", err.Error()), err) - return + + // Isolated WHIP: a detached worker owns the PeerConnection (and survives a + // main restart), returning the SDP answer over its frame socket. + // --isolated-ingest is forced off where unsupported (see runMain), so the + // flag alone gates this. + var answerSDP string + if a.CLI.IsolatedIngest { + answerSDP, err = a.MediaManager.WHIPIngestDetached(ctx, offer.SDP, mediaSigner) + if err != nil { + errors.WriteHTTPInternalServerError(w, fmt.Sprintf("error ingesting: %s", err.Error()), err) + return + } + } else { + pc, pcErr := a.MediaManager.NewPeerConnection(ctx, mediaSigner.Streamer()) + if pcErr != nil { + errors.WriteHTTPInternalServerError(w, "unable to create peer connection", pcErr) + return + } + answer, ingestErr := a.MediaManager.WebRTCIngest(ctx, &offer, mediaSigner, pc, make(chan error, 1)) + if ingestErr != nil { + errors.WriteHTTPInternalServerError(w, fmt.Sprintf("error ingesting: %s", ingestErr.Error()), ingestErr) + return + } + answerSDP = answer.SDP } host := r.Host if host == "" { @@ -127,7 +142,7 @@ func (a *StreamplaceAPI) HandleWebRTCIngest(ctx context.Context) httprouter.Hand log.Log(ctx, "location", "location", location) w.Header().Set("Location", location) w.WriteHeader(201) - if _, err := w.Write([]byte(answer.SDP)); err != nil { + if _, err := w.Write([]byte(answerSDP)); err != nil { log.Error(ctx, "error writing response", "error", err) } } diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index 36ce386f..7f166c24 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -875,6 +875,12 @@ func makeIngestWorkerCommand(build *config.BuildFlags) *urfavecli.Command { return fmt.Errorf("ingest-worker: parse config: %w", err) } + // WHIP transport: the worker owns the PeerConnection (built from the + // offer in the config) and serves frames over the socket — no media fd. + if cfg.Transport == media.IngestTransportWHIP { + return media.ServeWHIPIngestWorkerSocket(ctx, cfg) + } + // Detach/reattach transport: serve frames over a unix socket with // buffered reconnect (survives a main restart) instead of the fd-4 pipe. // Media comes from the fd-passed ingest connection (InputFD) when main diff --git a/pkg/ingestframe/frame.go b/pkg/ingestframe/frame.go index 31f16d39..c8a746a3 100644 --- a/pkg/ingestframe/frame.go +++ b/pkg/ingestframe/frame.go @@ -33,6 +33,11 @@ const ( // Error carries a worker-side fatal error message (UTF-8). The worker emits // it just before exiting so main can log a cause, not a bare "worker exited". Error Type = 3 + // Answer carries an SDP answer (UTF-8). The WHIP worker owns the + // PeerConnection, so it generates the answer and emits it as the FIRST frame + // on the socket; main reads it and returns it to the WHIP client before + // consuming segments. Payload: the answer SDP. + Answer Type = 4 ) func (t Type) String() string { @@ -43,6 +48,8 @@ func (t Type) String() string { return "end" case Error: return "error" + case Answer: + return "answer" default: return fmt.Sprintf("unknown(%d)", uint8(t)) } @@ -104,6 +111,9 @@ func (fw *Writer) End() error { return fw.WriteFrame(End, nil) } // Error frames a fatal worker-side error message. func (fw *Writer) Error(msg string) error { return fw.WriteFrame(Error, []byte(msg)) } +// Answer frames the WHIP SDP answer (emitted first, before any segments). +func (fw *Writer) Answer(sdp string) error { return fw.WriteFrame(Answer, []byte(sdp)) } + // Reader decodes frames from an underlying stream. type Reader struct { r io.Reader diff --git a/pkg/ingestframe/frame_test.go b/pkg/ingestframe/frame_test.go index 70881fcf..aa309f93 100644 --- a/pkg/ingestframe/frame_test.go +++ b/pkg/ingestframe/frame_test.go @@ -20,6 +20,7 @@ func TestRoundTrip(t *testing.T) { w := NewWriter(&buf) big := bytes.Repeat([]byte{0xAB}, 500_000) + require.NoError(t, w.Answer("v=0\r\no=- 1 1 IN IP4 0.0.0.0\r\n")) require.NoError(t, w.Segment([]byte("seg-one"))) require.NoError(t, w.Segment(nil)) // zero-length segment is legal require.NoError(t, w.Segment(big)) @@ -35,6 +36,7 @@ func TestRoundTrip(t *testing.T) { require.Equal(t, wantT, gotT) require.Equal(t, wantPayload, got) } + assertFrame(Answer, []byte("v=0\r\no=- 1 1 IN IP4 0.0.0.0\r\n")) assertFrame(Segment, []byte("seg-one")) assertFrame(Segment, nil) assertFrame(Segment, big) diff --git a/pkg/media/frame_server.go b/pkg/media/frame_server.go index 47585c77..20b4965b 100644 --- a/pkg/media/frame_server.go +++ b/pkg/media/frame_server.go @@ -87,6 +87,7 @@ func (s *frameServer) push(typ ingestframe.Type, payload []byte) { func (s *frameServer) Segment(seg []byte) error { s.push(ingestframe.Segment, seg); return nil } func (s *frameServer) End() error { s.push(ingestframe.End, nil); return nil } func (s *frameServer) Error(msg string) error { s.push(ingestframe.Error, []byte(msg)); return nil } +func (s *frameServer) Answer(sdp string) error { s.push(ingestframe.Answer, []byte(sdp)); return nil } // dropped reports how many buffered frames were discarded because the buffer // overflowed (main was disconnected longer than the buffer window). diff --git a/pkg/media/ingest_daemon.go b/pkg/media/ingest_daemon.go index c027f4bb..45525381 100644 --- a/pkg/media/ingest_daemon.go +++ b/pkg/media/ingest_daemon.go @@ -13,6 +13,7 @@ import ( "time" "github.com/google/uuid" + "stream.place/streamplace/pkg/ingestframe" "stream.place/streamplace/pkg/log" ) @@ -50,8 +51,13 @@ func SpawnIngestWorkerDetached(cfg IngestWorkerConfig, media *os.File) (*os.Proc defer cfgW.Close() cmd := exec.Command(exe, "ingest-worker") - setDetached(cmd) // own session, survives a main restart (Linux) - cmd.ExtraFiles = []*os.File{cfgR, media} // → child fd 3 (config), fd 4 (media) + setDetached(cmd) // own session, survives a main restart (Linux) + // fd 3 = config; fd 4 = the fd-passed media connection (MKV/RTMP). WHIP owns + // its own PeerConnection, so it passes no media fd. + cmd.ExtraFiles = []*os.File{cfgR} + if media != nil { + cmd.ExtraFiles = append(cmd.ExtraFiles, media) + } cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { return nil, fmt.Errorf("spawn ingest worker: %w", err) @@ -164,6 +170,108 @@ func (mm *MediaManager) MKVIngestDetached(ctx context.Context, conn net.Conn, pr return err } +// whipAnswerTimeout bounds how long main waits for the worker to produce the SDP +// answer (worker startup + ICE gathering) before giving up on the WHIP request. +const whipAnswerTimeout = 20 * time.Second + +// dialWorkerSocket connects to a worker's frame socket, retrying until it's up or +// ctx is done (a freshly-spawned worker takes a moment to start listening). +func dialWorkerSocket(ctx context.Context, socketPath string) (net.Conn, error) { + for { + conn, err := net.Dial("unix", socketPath) + if err == nil { + return conn, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(ingestReconnectBackoff): + } + } +} + +// readWHIPAnswer reads frames on conn until the worker's Answer frame and returns +// its SDP. An Error/End/EOF before the answer is a setup failure. +func readWHIPAnswer(conn net.Conn) (string, error) { + fr := ingestframe.NewReader(conn) + for { + typ, payload, err := fr.ReadFrame() + if err != nil { + return "", fmt.Errorf("read whip answer: %w", err) + } + switch typ { + case ingestframe.Answer: + return string(payload), nil + case ingestframe.Error: + return "", fmt.Errorf("whip worker error: %s", payload) + case ingestframe.End: + return "", fmt.Errorf("whip worker ended before sending an answer") + } + // A Segment before the Answer shouldn't happen; ignore it defensively. + } +} + +// WHIPIngestDetached is the WHIP zero-downtime entry. Main has authed the WHIP +// request; this spawns a DETACHED worker that owns the PeerConnection (built from +// offerSDP, binding its own UDP sockets) and serves signed segments over a +// per-session socket. It reads the worker's SDP answer (the first frame) to +// return to the client, then consumes segments into ValidateMP4 in the +// background with reconnect. Because the worker owns the WebRTC session and is +// detached, both the session and its buffered output survive a main restart (the +// restarted main reconnects via discovery). +func (mm *MediaManager) WHIPIngestDetached(ctx context.Context, offerSDP string, ms MediaSigner) (string, error) { + cfg, err := mm.buildWorkerConfig(ctx, ms) + if err != nil { + return "", err + } + dir, err := mm.ingestWorkerSocketDir() + if err != nil { + return "", err + } + cfg.SocketPath = filepath.Join(dir, uuid.NewString()+".sock") + cfg.Transport = IngestTransportWHIP + cfg.OfferSDP = offerSDP + + proc, err := SpawnIngestWorkerDetached(cfg, nil) // worker owns the PeerConnection + if err != nil { + return "", fmt.Errorf("spawn detached whip worker: %w", err) + } + + // Connect + read the SDP answer (the worker's first frame), bounded so a + // wedged setup can't hang the WHIP client. + answerCtx, answerCancel := context.WithTimeout(ctx, whipAnswerTimeout) + defer answerCancel() + conn, err := dialWorkerSocket(answerCtx, cfg.SocketPath) + if err != nil { + _ = proc.Kill() + return "", fmt.Errorf("connect to whip worker: %w", err) + } + if dl, ok := answerCtx.Deadline(); ok { + _ = conn.SetReadDeadline(dl) + } + answer, err := readWHIPAnswer(conn) + if err != nil { + conn.Close() + _ = proc.Kill() + return "", err + } + _ = conn.SetReadDeadline(time.Time{}) // clear; streaming has no deadline + + // Consume the signed segments in the background; the HTTP handler returns the + // answer now and the WebRTC media establishes directly to the worker. + go func() { + sawEnd, _ := mm.consumeWorkerFrames(ctx, conn, ms.Streamer(), mm.validateSegment(ctx), nil) + conn.Close() + if !sawEnd && ctx.Err() == nil { + // Connection dropped but the detached worker lives on — reconnect and + // drain its buffer. + _ = mm.ConsumeWorkerSocket(ctx, cfg.SocketPath, ms.Streamer(), mm.validateSegment(ctx)) + } + go func() { _, _ = proc.Wait() }() + }() + return answer, nil +} + // ResumeDetachedWorkers reconnects to any ingest workers still running from // before a main restart and resumes consuming their frames (draining whatever // they buffered while main was down). Intended to run once at main startup. diff --git a/pkg/media/ingest_worker.go b/pkg/media/ingest_worker.go index 9a9321bc..5941b691 100644 --- a/pkg/media/ingest_worker.go +++ b/pkg/media/ingest_worker.go @@ -61,8 +61,20 @@ type IngestWorkerConfig struct { // stdin / raw fd input. Prebuf []byte `json:"prebuf,omitempty"` Chunked bool `json:"chunked,omitempty"` + + // Transport selects the worker's ingest source: "" / "mkv" reads MKV media + // (stdin or InputFD); "whip" makes the worker own the WebRTC PeerConnection, + // built from OfferSDP — no media fd to pass. + Transport string `json:"transport,omitempty"` + // OfferSDP is the WHIP client's SDP offer (transport "whip"). The worker + // generates the answer and emits it as the first frame (ingestframe.Answer) + // so main can return it to the client before consuming segments. + OfferSDP string `json:"offer_sdp,omitempty"` } +// IngestTransportWHIP is the cfg.Transport value selecting the WHIP worker. +const IngestTransportWHIP = "whip" + // WorkerInput reconstructs the raw media stream the gst pipeline reads from the // fd-passed push connection: prepend any bytes main already read past the headers // (Prebuf), then de-chunk if the push used chunked transfer-encoding. For stdin @@ -78,28 +90,11 @@ func WorkerInput(cfg IngestWorkerConfig, raw io.Reader) io.Reader { return r } -// RunMKVIngestWorker is the body of the `ingest-worker` subcommand. It reads an -// MKV stream from stdin, runs the same demux + Opus re-encode + muxl-sign -// pipeline as the in-process MKVIngest, and emits each signed canonical .m4s -// segment to frames; the main process reads those frames and runs ValidateMP4 -// over each, exactly as if onSegment had called it directly. -// -// It returns when the stream ends cleanly (EOS) or the pipeline errors. The -// caller frames End or Error accordingly. All segment frames are guaranteed -// flushed before it returns, so a trailing End can never race ahead of the last -// Segment. -func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Reader, frames FrameWriter) error { - gstinit.InitGST() - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - // Minimal manager: just the broadcaster identity the transcode completion - // (finishTranscodedSegment) stamps into the node-signed AAC track. - mm := &MediaManager{cli: &config.CLI{BroadcasterHost: cfg.BroadcasterHost}} - - // The worker signs everything itself: forward the streamer key PEM + cert + - // prebuilt manifest straight to muxl-sign. No MediaSigner / model / DB needed. - signStream := func(ctx context.Context, input io.Reader, eventCh chan *muxl.MuxlEvent) error { +// workerSignStream returns the streaming muxl signer a worker uses: it forwards +// the streamer key PEM + cert + prebuilt manifest straight to muxl-sign, no +// MediaSigner / model / DB needed. Shared by the MKV and WHIP workers. +func workerSignStream(cfg IngestWorkerConfig) SignSegmentStreamFunc { + return func(ctx context.Context, input io.Reader, eventCh chan *muxl.MuxlEvent) error { fetchManifest := func() ([]byte, error) { return cfg.Manifest, nil } return muxl.RunMuxlSignSegment(ctx, input, muxl.SignerInput{ CertPEM: cfg.CertPEM, @@ -108,16 +103,19 @@ func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Re WrapperManifestFn: fetchManifest, }, nil, nil, eventCh) } +} - // With a node transcode key, the worker completes each single-codec source - // segment to dual-codec itself: feed the signed source segment into a - // per-stream transcoder running in THIS process; its completion callback - // frames the finished dual-codec segment. The transcoder runs on a - // non-cancellable context so draining the signer (cancel, below) can't kill it - // before its ~1-GoP tail is flushed by Close. One process == one session, so - // the per-DID transcoder-reuse hazard simply can't arise here. +// workerSegmentSink returns the onSegment handler a worker hands to +// muxlSignSegmentElem, plus a flush to call once the signer has drained. With a +// node transcode key it completes each single-codec source segment to dual-codec +// via an in-process transcoder (its completion callback frames the finished +// segment); flush Closes that transcoder so its ~1-GoP tail is framed before the +// worker exits. The transcoder runs on a non-cancellable context so draining the +// signer can't kill it early. One process == one session, so the per-DID +// transcoder-reuse hazard can't arise. Shared by the MKV and WHIP workers. +func (mm *MediaManager) workerSegmentSink(ctx context.Context, cfg IngestWorkerConfig, frames FrameWriter) (onSegment func(context.Context, []byte) error, flush func()) { var transcoder *streamTranscoder - onSegment := func(_ context.Context, segment []byte) error { + onSegment = func(_ context.Context, segment []byte) error { if len(cfg.NodeKeyPEM) == 0 { return frames.Segment(segment) // no node signer → single-codec } @@ -135,8 +133,37 @@ func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Re } return transcoder.Feed(segment, nil) } + flush = func() { + if transcoder != nil { + if cerr := transcoder.Close(); cerr != nil { + log.Error(ctx, "ingest worker: transcoder close", "error", cerr) + } + } + } + return onSegment, flush +} - signerElem, done, err := muxlSignSegmentElem(ctx, mm.cli, signStream, onSegment) +// RunMKVIngestWorker is the body of the `ingest-worker` subcommand. It reads an +// MKV stream from stdin, runs the same demux + Opus re-encode + muxl-sign +// pipeline as the in-process MKVIngest, and emits each signed canonical .m4s +// segment to frames; the main process reads those frames and runs ValidateMP4 +// over each, exactly as if onSegment had called it directly. +// +// It returns when the stream ends cleanly (EOS) or the pipeline errors. The +// caller frames End or Error accordingly. All segment frames are guaranteed +// flushed before it returns, so a trailing End can never race ahead of the last +// Segment. +func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Reader, frames FrameWriter) error { + gstinit.InitGST() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Minimal manager: just the broadcaster identity the transcode completion + // (finishTranscodedSegment) stamps into the node-signed AAC track. + mm := &MediaManager{cli: &config.CLI{BroadcasterHost: cfg.BroadcasterHost}} + onSegment, flush := mm.workerSegmentSink(ctx, cfg, frames) + + signerElem, done, err := muxlSignSegmentElem(ctx, mm.cli, workerSignStream(cfg), onSegment) if err != nil { return fmt.Errorf("build signer element: %w", err) } @@ -159,18 +186,12 @@ func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Re } }() - // Wait for the pipeline to finish (EOS or error), then drain the signer: - // cancelling unblocks the signer's input pipe so it flushes the final GoP, and - // <-done guarantees every source segment has been fed. Then flush the - // transcoder's tail so the last dual-codec completions are framed before we - // return (the caller's End can't race ahead of them). + // Pipeline done (EOS/error) → drain the signer (cancel flushes the final GoP; + // <-done means every source segment has been fed) → flush the transcoder tail + // so the last dual-codec completions are framed before we return. pipeErr := <-busErr cancel() <-done - if transcoder != nil { - if cerr := transcoder.Close(); cerr != nil { - log.Error(ctx, "ingest worker: transcoder close", "error", cerr) - } - } + flush() return pipeErr } diff --git a/pkg/media/media.go b/pkg/media/media.go index 258f4bae..45359817 100644 --- a/pkg/media/media.go +++ b/pkg/media/media.go @@ -12,8 +12,6 @@ import ( "sync/atomic" "github.com/google/uuid" - "github.com/pion/interceptor" - "github.com/pion/interceptor/pkg/intervalpli" "github.com/pion/webrtc/v4" "go.opentelemetry.io/otel" "stream.place/streamplace/pkg/aqtime" @@ -111,52 +109,10 @@ func MakeMediaManager(ctx context.Context, cli *config.CLI, signer crypto.Signer return nil, fmt.Errorf("error in gstreamer self-test: %w", err) } - m := &webrtc.MediaEngine{} - // Create a InterceptorRegistry. This is the user configurable RTP/RTCP Pipeline. - // This provides NACKs, RTCP Reports and other features. If you use `webrtc.NewPeerConnection` - // this is enabled by default. If you are manually managing You MUST create a InterceptorRegistry - // for each PeerConnection. - i := &interceptor.Registry{} - - // Register a intervalpli factory - // This interceptor sends a PLI every 3 seconds. A PLI causes a video keyframe to be generated by the sender. - // This makes our video seekable and more error resilent, but at a cost of lower picture quality and higher bitrates - // A real world application should process incoming RTCP packets from viewers and forward them to senders - intervalPliFactory, err := intervalpli.NewReceiverInterceptor() + api, config, err := newWebRTCAPI() if err != nil { - return nil, fmt.Errorf("failed to create intervalpli factory: %w", err) - } - i.Add(intervalPliFactory) - - if err := m.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, Channels: 0, SDPFmtpLine: "", RTCPFeedback: nil}, - PayloadType: 102, - }, webrtc.RTPCodecTypeVideo); err != nil { - return nil, err - } - if err := m.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 0, SDPFmtpLine: "", RTCPFeedback: nil}, - PayloadType: 111, - }, webrtc.RTPCodecTypeAudio); err != nil { return nil, err } - - // Use the default set of Interceptors - if err = webrtc.RegisterDefaultInterceptors(m, i); err != nil { - return nil, fmt.Errorf("failed to register default interceptors: %w", err) - } - - // Create the API object with the MediaEngine - api := webrtc.NewAPI(webrtc.WithMediaEngine(m), webrtc.WithInterceptorRegistry(i)) - - // Prepare the configuration - config := webrtc.Configuration{ - ICEServers: []webrtc.ICEServer{ - { - URLs: []string{"stun:stun.l.google.com:19302"}, - }, - }, - } return &MediaManager{ cli: cli, liveWindows: map[string]*livehls.Writer{}, diff --git a/pkg/media/webrtc_api.go b/pkg/media/webrtc_api.go new file mode 100644 index 00000000..e8bc019e --- /dev/null +++ b/pkg/media/webrtc_api.go @@ -0,0 +1,49 @@ +package media + +import ( + "fmt" + + "github.com/pion/interceptor" + "github.com/pion/interceptor/pkg/intervalpli" + "github.com/pion/webrtc/v4" +) + +// newWebRTCAPI builds the pion API + configuration Streamplace uses for WebRTC +// ingest (H264 video + Opus audio, default interceptors plus an interval PLI so +// the publisher keeps sending keyframes). Shared by MakeMediaManager and the +// isolated WHIP worker — the worker builds its own API since it owns the +// PeerConnection in its own process. +func newWebRTCAPI() (*webrtc.API, webrtc.Configuration, error) { + m := &webrtc.MediaEngine{} + i := &interceptor.Registry{} + + intervalPliFactory, err := intervalpli.NewReceiverInterceptor() + if err != nil { + return nil, webrtc.Configuration{}, fmt.Errorf("failed to create intervalpli factory: %w", err) + } + i.Add(intervalPliFactory) + + if err := m.RegisterCodec(webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000}, + PayloadType: 102, + }, webrtc.RTPCodecTypeVideo); err != nil { + return nil, webrtc.Configuration{}, err + } + if err := m.RegisterCodec(webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000}, + PayloadType: 111, + }, webrtc.RTPCodecTypeAudio); err != nil { + return nil, webrtc.Configuration{}, err + } + if err := webrtc.RegisterDefaultInterceptors(m, i); err != nil { + return nil, webrtc.Configuration{}, fmt.Errorf("failed to register default interceptors: %w", err) + } + + api := webrtc.NewAPI(webrtc.WithMediaEngine(m), webrtc.WithInterceptorRegistry(i)) + config := webrtc.Configuration{ + ICEServers: []webrtc.ICEServer{ + {URLs: []string{"stun:stun.l.google.com:19302"}}, + }, + } + return api, config, nil +} diff --git a/pkg/media/webrtc_ingest.go b/pkg/media/webrtc_ingest.go index e4fd50bc..70b6b2bf 100644 --- a/pkg/media/webrtc_ingest.go +++ b/pkg/media/webrtc_ingest.go @@ -15,19 +15,36 @@ import ( "stream.place/streamplace/pkg/rtcrec" ) -// This function remains in scope for the duration of a single users' playback +// WebRTCIngest is the in-process WHIP entry: it builds the signing element via +// SegmentAndSignElem (→ ValidateMP4) and runs the shared ingest pipeline. Stays +// in scope for the duration of a single stream. func (mm *MediaManager) WebRTCIngest(ctx context.Context, offer *webrtc.SessionDescription, signer MediaSigner, peerConnection rtcrec.PeerConnection, done chan error) (*webrtc.SessionDescription, error) { uu, err := uuid.NewV7() if err != nil { return nil, err } - ctx = log.WithLogValues(ctx, "webrtcID", uu.String(), "mediafunc", "WebRTCIngest", "streamer", signer.Streamer()) + ctx, cancel := context.WithCancel(ctx) + signerElem, err := mm.SegmentAndSignElem(ctx, signer) + if err != nil { + cancel() + return nil, fmt.Errorf("failed create signer element: %w", err) + } + return mm.webRTCIngestPipeline(ctx, cancel, offer, peerConnection, signerElem, signer, done) +} +// webRTCIngestPipeline runs WebRTC ingest over a pre-built signer element: +// depay/parse the incoming RTP into the muxl signing bin, answer the offer, and +// stream in the background. The in-process path passes a SegmentAndSignElem (→ +// ValidateMP4) and the streamer's signer (for key revocation); the isolated WHIP +// worker passes a muxlSignSegmentElem wired to its frame socket and a nil +// keyRevSigner. The cancellable ctx and signerElem are built by the caller (the +// signer element's goroutines are tied to ctx). +func (mm *MediaManager) webRTCIngestPipeline(ctx context.Context, cancel context.CancelFunc, offer *webrtc.SessionDescription, peerConnection rtcrec.PeerConnection, signerElem *gst.Element, keyRevSigner MediaSigner, done chan error) (*webrtc.SessionDescription, error) { // Allow us to receive 1 audio track, and 1 video track - if _, err = peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio); err != nil { + if _, err := peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio); err != nil { return nil, fmt.Errorf("failed to add audio transceiver: %w", err) - } else if _, err = peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo); err != nil { + } else if _, err := peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo); err != nil { return nil, fmt.Errorf("failed to add video transceiver: %w", err) } @@ -92,12 +109,7 @@ func (mm *MediaManager) WebRTCIngest(ctx context.Context, offer *webrtc.SessionD // Create channel that is blocked until ICE Gathering is complete gatherComplete := rtcrec.GatheringCompletePromise(peerConnection) - ctx, cancel := context.WithCancel(ctx) - signerElem, err := mm.SegmentAndSignElem(ctx, signer) - if err != nil { - cancel() - return nil, fmt.Errorf("failed create signer element: %w", err) - } + // cancel + signerElem are provided by the caller. err = pipeline.Add(signerElem) if err != nil { cancel() @@ -153,8 +165,11 @@ func (mm *MediaManager) WebRTCIngest(ctx context.Context, offer *webrtc.SessionD } }() - // subscription to bus messages for key revocation - go mm.HandleKeyRevocation(ctx, signer, pipeline) + // subscription to bus messages for key revocation (in-process only; the + // isolated worker has no model-backed signer to revoke against) + if keyRevSigner != nil { + go mm.HandleKeyRevocation(ctx, keyRevSigner, pipeline) + } go func() { <-ctx.Done() diff --git a/pkg/media/whip_worker.go b/pkg/media/whip_worker.go new file mode 100644 index 00000000..ffb88592 --- /dev/null +++ b/pkg/media/whip_worker.go @@ -0,0 +1,105 @@ +package media + +import ( + "context" + "fmt" + "net" + "os" + + "github.com/pion/webrtc/v4" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/gstinit" + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/rtcrec" +) + +// ServeWHIPIngestWorkerSocket is the WHIP counterpart of +// ServeMKVIngestWorkerSocket. Unlike MKV there's no socket/fd to pass in: the +// worker OWNS the PeerConnection, so it creates it from cfg.OfferSDP (binding its +// own UDP sockets), generates the SDP answer, and emits it as the FIRST frame on +// the unix socket — main reads that Answer frame and returns it to the WHIP +// client, then keeps reading the signed dual-codec segments. The worker is +// detached, so the WebRTC session (and the buffered segment stream) survive a +// main restart. +func ServeWHIPIngestWorkerSocket(ctx context.Context, cfg IngestWorkerConfig) error { + if cfg.SocketPath == "" { + return fmt.Errorf("ServeWHIPIngestWorkerSocket: empty socket path") + } + if cfg.OfferSDP == "" { + return fmt.Errorf("ServeWHIPIngestWorkerSocket: empty offer") + } + gstinit.InitGST() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + _ = os.Remove(cfg.SocketPath) // clear any stale socket from a prior worker + ln, err := net.Listen("unix", cfg.SocketPath) + if err != nil { + return fmt.Errorf("listen %s: %w", cfg.SocketPath, err) + } + defer func() { + ln.Close() + _ = os.Remove(cfg.SocketPath) + }() + + srv := newFrameServer(workerFrameBuffer) + go serveFrameSocket(ctx, ln, srv) + + // finish flushes the trailing End/Error, waits for main to drain the buffer + // (incl. the Answer), then closes the connection for a clean EOF. + finish := func(runErr error) error { + if runErr != nil { + _ = srv.Error(runErr.Error()) + } else { + _ = srv.End() + } + srv.waitDrained(ctx, workerDrainGrace) + srv.closeConn() + return runErr + } + + mm := &MediaManager{cli: &config.CLI{BroadcasterHost: cfg.BroadcasterHost}} + + // The worker owns the PeerConnection (its own UDP sockets), built with the + // same codec/interceptor setup as the in-process server. No recording here — + // the worker has no model-backed settings. + api, webrtcConfig, err := newWebRTCAPI() + if err != nil { + return finish(fmt.Errorf("webrtc api: %w", err)) + } + pionpc, err := api.NewPeerConnection(webrtcConfig) + if err != nil { + return finish(fmt.Errorf("peer connection: %w", err)) + } + pc, err := rtcrec.NewRecordingPeerConnection(ctx, *mm.cli, cfg.StreamerDID, pionpc, false) + if err != nil { + return finish(fmt.Errorf("peer connection wrapper: %w", err)) + } + + onSegment, flush := mm.workerSegmentSink(ctx, cfg, srv) + signerElem, signerDone, err := muxlSignSegmentElem(ctx, mm.cli, workerSignStream(cfg), onSegment) + if err != nil { + return finish(fmt.Errorf("build signer element: %w", err)) + } + + offer := &webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: cfg.OfferSDP} + streamDone := make(chan error, 1) + answer, err := mm.webRTCIngestPipeline(ctx, cancel, offer, pc, signerElem, nil, streamDone) + if err != nil { + return finish(fmt.Errorf("webrtc ingest: %w", err)) + } + + // Hand the answer back to main FIRST; the segment frames stream behind it. + if aerr := srv.Answer(answer.SDP); aerr != nil { + log.Error(ctx, "whip worker: frame answer", "error", aerr) + } + + // Streaming runs until the peer disconnects / errors; webRTCIngestPipeline + // cancels ctx then, which drains the signer. Wait for that, flush the + // transcoder tail, then finish. + streamErr := <-streamDone + cancel() + <-signerDone + flush() + return finish(streamErr) +} diff --git a/pkg/media/whip_worker_test.go b/pkg/media/whip_worker_test.go new file mode 100644 index 00000000..b0ac69fb --- /dev/null +++ b/pkg/media/whip_worker_test.go @@ -0,0 +1,237 @@ +package media + +import ( + "bytes" + "context" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + pionmedia "github.com/pion/webrtc/v4/pkg/media" + + "github.com/pion/webrtc/v4" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/crypto/signers" + "stream.place/streamplace/pkg/ingestframe" + "stream.place/streamplace/pkg/muxl" +) + +// whipClientOffer builds a WHIP-style SDP offer (H264 video + Opus audio tracks), +// the way a real WHIP client does, returning the client PC, its tracks, and the +// offer. +func whipClientOffer(t *testing.T) (*webrtc.PeerConnection, *webrtc.TrackLocalStaticSample, *webrtc.TrackLocalStaticSample, webrtc.SessionDescription) { + t.Helper() + pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + videoTrack, err := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264}, "video", "pion") + require.NoError(t, err) + if _, err = pc.AddTrack(videoTrack); err != nil { + t.Fatal(err) + } + audioTrack, err := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "pion") + require.NoError(t, err) + if _, err = pc.AddTrack(audioTrack); err != nil { + t.Fatal(err) + } + offer, err := pc.CreateOffer(nil) + require.NoError(t, err) + require.NoError(t, pc.SetLocalDescription(offer)) + return pc, videoTrack, audioTrack, offer +} + +// TestWHIPWorkerAnswersOffer verifies the WHIP worker's answer back-channel: from +// an offer it builds the PeerConnection, generates the SDP answer, and emits it +// as the FIRST frame on its socket — the synchronous reply main returns to the +// WHIP client. (Media flow → signed segments rides the same webRTCIngestPipeline +// the in-process path uses, plus the transcoder/frame machinery the MKV tests +// already cover.) +func TestWHIPWorkerAnswersOffer(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ms := newBareSegmentSigner(t) + keyPEM, err := signers.MarshalES256KPrivateKeyPEM(ms.Signer) + require.NoError(t, err) + manifest, err := ms.buildManifest(ctx, time.Now().UnixMilli()) + require.NoError(t, err) + + clientPC, _, _, offer := whipClientOffer(t) + defer clientPC.Close() + + sock := filepath.Join(t.TempDir(), "whip.sock") + cfg := IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: ms.Cert, + Manifest: manifest, + NodeCertPEM: ms.Cert, + NodeKeyPEM: keyPEM, + BroadcasterHost: "test.example.com", + SocketPath: sock, + Transport: IngestTransportWHIP, + OfferSDP: offer.SDP, + } + + serveDone := make(chan error, 1) + go func() { serveDone <- ServeWHIPIngestWorkerSocket(ctx, cfg) }() + + // Connect to the worker socket (retry until up) and read the Answer frame. + dctx, dcancel := context.WithTimeout(ctx, 20*time.Second) + defer dcancel() + conn, derr := dialWorkerSocket(dctx, sock) + require.NoError(t, derr) + defer conn.Close() + + answerSDP, rerr := readWHIPAnswer(conn) + require.NoError(t, rerr, "worker emits an SDP answer as its first frame") + require.Contains(t, answerSDP, "v=0", "valid SDP answer") + + // The answer must apply cleanly as the client's remote description — i.e. it's + // a real, negotiated answer to the offer. + require.NoError(t, clientPC.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeAnswer, SDP: answerSDP, + }), "answer applies as the client's remote description") + + t.Logf("whip worker produced a %d-byte SDP answer", len(answerSDP)) + + cancel() + select { + case <-serveDone: + case <-time.After(25 * time.Second): + t.Fatal("worker did not exit after cancel") + } +} + +// produceWHIPMedia streams synthetic H264 + Opus into the WHIP client's tracks +// via a gst encode pipeline until ctx is cancelled — i.e. a real WHIP publisher. +func produceWHIPMedia(t *testing.T, ctx context.Context, video, audio *webrtc.TrackLocalStaticSample) { + t.Helper() + desc := strings.Join([]string{ + "videotestsrc is-live=true ! video/x-raw,width=320,height=240,framerate=30/1 ! x264enc key-int-max=15 tune=zerolatency speed-preset=ultrafast ! h264parse ! video/x-h264,stream-format=byte-stream,alignment=au ! appsink name=vsink", + "audiotestsrc is-live=true ! audioconvert ! audioresample ! opusenc ! opusparse ! appsink name=asink", + }, "\n") + pipeline, err := gst.NewPipelineFromString(desc) + require.NoError(t, err) + + pump := func(name string, track *webrtc.TrackLocalStaticSample, dur time.Duration) { + ele, gerr := pipeline.GetElementByName(name) + require.NoError(t, gerr) + app.SinkFromElement(ele).SetCallbacks(&app.SinkCallbacks{ + NewSampleFunc: func(sink *app.Sink) gst.FlowReturn { + sample := sink.PullSample() + if sample == nil { + return gst.FlowEOS + } + buf := sample.GetBuffer() + data := buf.Map(gst.MapRead).Bytes() + buf.Unmap() + if werr := track.WriteSample(pionmedia.Sample{Data: data, Duration: dur}); werr != nil { + return gst.FlowError + } + return gst.FlowOK + }, + }) + } + pump("vsink", video, time.Second/30) + pump("asink", audio, 20*time.Millisecond) + + go func() { + <-ctx.Done() + _ = pipeline.SetState(gst.StateNull) + }() + require.NoError(t, pipeline.SetState(gst.StatePlaying)) +} + +// TestWHIPWorkerLoopback is the full WHIP media path: a pion client offers, +// connects to the worker (which owns the PeerConnection), and streams real +// H264+Opus RTP; the worker must mux+sign+transcode it and serve a valid signed +// dual-codec segment over its socket — the WHIP parity of the MKV worker e2e +// test. +func TestWHIPWorkerLoopback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ms := newBareSegmentSigner(t) + keyPEM, err := signers.MarshalES256KPrivateKeyPEM(ms.Signer) + require.NoError(t, err) + manifest, err := ms.buildManifest(ctx, time.Now().UnixMilli()) + require.NoError(t, err) + + clientPC, videoTrack, audioTrack, offer := whipClientOffer(t) + defer clientPC.Close() + + connected := make(chan struct{}) + var once sync.Once + clientPC.OnConnectionStateChange(func(s webrtc.PeerConnectionState) { + if s == webrtc.PeerConnectionStateConnected { + once.Do(func() { close(connected) }) + } + }) + + sock := filepath.Join(t.TempDir(), "whip.sock") + cfg := IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: ms.Cert, + Manifest: manifest, + NodeCertPEM: ms.Cert, + NodeKeyPEM: keyPEM, + BroadcasterHost: "test.example.com", + SocketPath: sock, + Transport: IngestTransportWHIP, + OfferSDP: offer.SDP, + } + serveDone := make(chan error, 1) + go func() { serveDone <- ServeWHIPIngestWorkerSocket(ctx, cfg) }() + + dctx, dcancel := context.WithTimeout(ctx, 20*time.Second) + defer dcancel() + conn, derr := dialWorkerSocket(dctx, sock) + require.NoError(t, derr) + defer conn.Close() + + answerSDP, rerr := readWHIPAnswer(conn) + require.NoError(t, rerr) + require.NoError(t, clientPC.SetRemoteDescription(webrtc.SessionDescription{Type: webrtc.SDPTypeAnswer, SDP: answerSDP})) + + select { + case <-connected: + case <-time.After(20 * time.Second): + t.Fatal("client PC did not connect to the worker") + } + + produceWHIPMedia(t, ctx, videoTrack, audioTrack) + + // Read signed segments; require at least one valid dual-codec one. + _ = conn.SetReadDeadline(time.Now().Add(45 * time.Second)) + r := ingestframe.NewReader(conn) + var segs int + for segs == 0 { + typ, payload, ferr := r.ReadFrame() + require.NoError(t, ferr, "reading worker frames") + if typ != ingestframe.Segment { + continue + } + out, verr := muxl.RunMuxlVerify(ctx, bytes.NewReader(payload)) + require.NoError(t, verr) + require.NotContains(t, out, `"validation_state":"Invalid"`, "segment must validate") + segs++ + } + _ = conn.SetReadDeadline(time.Time{}) + t.Logf("whip worker produced %d signed segment(s) from real RTP media", segs) + + // Close the connection before tearing down: in production main's connection + // breaks on shutdown, which detaches the worker's frame server so it drains + // into its buffer instead of blocking on an unread socket. + conn.Close() + cancel() + select { + case <-serveDone: + case <-time.After(25 * time.Second): + t.Fatal("worker did not exit after cancel") + } +} -- 2.51.2 From 2a6f309d0dcae0acf719a37ce93b1d316bb20cc1 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Sat, 6 Jun 2026 14:57:41 -0700 Subject: [PATCH 11/17] media: pass streamer DID on the ingest-worker argv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workers all showed up as a bare `libstreamplace ingest-worker` in a process listing, so an operator couldn't tell which user's session a given worker belonged to. Pass the streamer DID as a positional argv element at every spawn site (MKVIngestIsolated, SpawnIngestWorkerDetached) so `ps` reads `libstreamplace ingest-worker did:plc:...`. The DID is public, non-sensitive identity — unlike the signing key / manifest, which stay on the fd-3 config. The argv copy is purely for identification; the worker still reads the authoritative DID from fd 3. The subcommand logs it early ("ingest-worker starting") and threads it into the logger for log correlation too. Co-Authored-By: Claude Opus 4.8 --- pkg/cmd/streamplace.go | 15 ++++++++++++--- pkg/media/ingest_daemon.go | 4 +++- pkg/media/ingest_supervisor.go | 4 +++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index 7f166c24..aad9e0e3 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -857,10 +857,19 @@ func makeStreamCommand(build *config.BuildFlags) *urfavecli.Command { // End frame; a fatal error emits an Error frame before exiting non-zero. func makeIngestWorkerCommand(build *config.BuildFlags) *urfavecli.Command { return &urfavecli.Command{ - Name: "ingest-worker", - Usage: "internal: per-stream isolated ingest worker (spawned by the node)", - Hidden: true, + Name: "ingest-worker", + Usage: "internal: per-stream isolated ingest worker (spawned by the node)", + ArgsUsage: "[streamer-did]", + Hidden: true, Action: func(ctx context.Context, cmd *urfavecli.Command) error { + // The streamer DID is passed on argv purely so the worker is + // identifiable in a process listing (ps); the authoritative copy + // still arrives in the fd-3 config. Thread it into the logger for + // log correlation. + if did := cmd.Args().First(); did != "" { + ctx = log.WithLogValues(ctx, "streamer", did) + log.Log(ctx, "ingest-worker starting") + } cfgFile := os.NewFile(3, "ingest-config") if cfgFile == nil { return fmt.Errorf("ingest-worker: missing config fd 3") diff --git a/pkg/media/ingest_daemon.go b/pkg/media/ingest_daemon.go index 45525381..9d9c068b 100644 --- a/pkg/media/ingest_daemon.go +++ b/pkg/media/ingest_daemon.go @@ -50,7 +50,9 @@ func SpawnIngestWorkerDetached(cfg IngestWorkerConfig, media *os.File) (*os.Proc defer cfgR.Close() defer cfgW.Close() - cmd := exec.Command(exe, "ingest-worker") + // The streamer DID rides argv (public, non-sensitive) so a worker is + // identifiable in a process listing; key material stays on fd 3. + cmd := exec.Command(exe, "ingest-worker", cfg.StreamerDID) setDetached(cmd) // own session, survives a main restart (Linux) // fd 3 = config; fd 4 = the fd-passed media connection (MKV/RTMP). WHIP owns // its own PeerConnection, so it passes no media fd. diff --git a/pkg/media/ingest_supervisor.go b/pkg/media/ingest_supervisor.go index 0dd018b1..cdc7ae70 100644 --- a/pkg/media/ingest_supervisor.go +++ b/pkg/media/ingest_supervisor.go @@ -67,7 +67,9 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, ctx, cancel := context.WithCancel(ctx) defer cancel() - cmd := exec.CommandContext(ctx, exe, "ingest-worker") + // The streamer DID rides argv (public, non-sensitive) so a worker is + // identifiable in a process listing; key material stays on fd 3. + cmd := exec.CommandContext(ctx, exe, "ingest-worker", cfg.StreamerDID) // Dedicated pipes: fd 3 carries the config in, fd 4 carries the frame stream // out. Keeping frames off stdout means nothing the worker (or gst, or the -- 2.51.2 From 6afb43026b78d5dac48b38516886d471f6d79ef6 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Sat, 6 Jun 2026 15:12:00 -0700 Subject: [PATCH 12/17] media: isolate RTMP multistream push in a worker subprocess (opt-in) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A gst fault in the egress chain (qtdemux/flvmux/rtmp2sink) takes the whole node down today — RTMPPush runs that native pipeline in-process. This extends the per-stream isolation to the OUTBOUND direction: under --isolated-ingest each enabled multistream target runs its push pipeline in a dedicated `rtmp-push-worker` subprocess, so a crash there kills only that worker; the node survives. Boundary: the crash-prone native pipeline is the only thing that moves. Main keeps the bus subscription, segment assembly (AAC select → init synth → concat), and the DB status writes — all memory-safe Go/wasm. The split is at the io.Pipe seam that already existed in RTMPPush: - runRTMPPushPipeline: the shared native core (appsrc → qtdemux → flvmux → rtmp2sink → TCP/TLS forwarder → target), parameterized by an input source and a status `report` callback. The in-process RTMPPush and the worker run an IDENTICAL pipeline — only source (io.Pipe vs stdin) and report (DB write vs Event frame) differ, so the paths can't drift. - writeRTMPSource: the extracted muxl source loop, now writing to any io.Writer (the in-process pipe, or the worker's stdin). State across the boundary is one small reverse channel: the worker can't reach the DB, so its stats poll emits ingestframe.Event{status,message} frames on fd 4; the supervisor (RTMPPushIsolated) folds each into CreateMultistreamEvent. The target URL + key ride fd 3 (off argv); only the public streamer DID goes on argv, for ps identification. Lifecycle reuses the existing multistream control loop unchanged: the worker runs under exec.CommandContext(ctx), so the cancel HandleMultistreamTargets fires on target-disable tears it down, and a crash becomes a non-zero exit that StartMultistreamTarget turns into an "error" event + 5s retry — strictly better than today, where that crash was fatal to the node. Gated by --isolated-ingest (already forced off where fd-passing is unsupported); the in-process path stays the default. Tests: ingestframe Event round-trip; consumePushEvents translation + error/torn-stream mapping (deterministic, no gst); and TestRTMPPushWorkerContainsFailure — a real subprocess whose pipeline rejects a bad target dies in its own process while the test (the node) survives to assert it. The live-push happy path still needs a real RTMP target to exercise end to end. Co-Authored-By: Claude Opus 4.8 --- pkg/cmd/streamplace.go | 51 +++++++ pkg/director/stream_session.go | 10 +- pkg/ingestframe/frame.go | 10 ++ pkg/ingestframe/frame_test.go | 2 + pkg/media/leak_test.go | 3 + pkg/media/rtmp_push.go | 220 ++++++++++++++++------------- pkg/media/rtmp_push_worker.go | 211 +++++++++++++++++++++++++++ pkg/media/rtmp_push_worker_test.go | 128 +++++++++++++++++ 8 files changed, 533 insertions(+), 102 deletions(-) create mode 100644 pkg/media/rtmp_push_worker.go create mode 100644 pkg/media/rtmp_push_worker_test.go diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index aad9e0e3..2c6a9ca3 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -77,6 +77,7 @@ func start(build *config.BuildFlags, platformJobs []jobFunc) error { makeVODTestCommand(build), makeStreamCommand(build), makeIngestWorkerCommand(build), + makeRTMPPushWorkerCommand(build), makeLiveCommand(build), makeWhepCommand(build), makeWhipCommand(build), @@ -923,6 +924,56 @@ func makeIngestWorkerCommand(build *config.BuildFlags) *urfavecli.Command { } } +// makeRTMPPushWorkerCommand is the per-target isolated multistream egress +// worker. The node spawns it; it is not meant for direct use. It reads the +// config handshake (incl. the target URL + stream key) from fd 3, the assembled +// fMP4 source stream from stdin, runs the native RTMP push pipeline, and writes +// status Event frames to fd 4 — dedicated fds so stray stdout/stderr can't +// corrupt the frame stream. A clean run ends with an End frame; a fatal error +// emits an Error frame before exiting non-zero. +func makeRTMPPushWorkerCommand(build *config.BuildFlags) *urfavecli.Command { + return &urfavecli.Command{ + Name: "rtmp-push-worker", + Usage: "internal: per-target isolated RTMP push worker (spawned by the node)", + ArgsUsage: "[streamer-did]", + Hidden: true, + Action: func(ctx context.Context, cmd *urfavecli.Command) error { + // The streamer DID is passed on argv purely so the worker is + // identifiable in a process listing (ps); the target URL stays on fd 3. + if did := cmd.Args().First(); did != "" { + ctx = log.WithLogValues(ctx, "streamer", did) + log.Log(ctx, "rtmp-push-worker starting") + } + cfgFile := os.NewFile(3, "push-config") + if cfgFile == nil { + return fmt.Errorf("rtmp-push-worker: missing config fd 3") + } + cfgBytes, err := io.ReadAll(cfgFile) + cfgFile.Close() + if err != nil { + return fmt.Errorf("rtmp-push-worker: read config: %w", err) + } + var cfg media.RTMPPushWorkerConfig + if err := json.Unmarshal(cfgBytes, &cfg); err != nil { + return fmt.Errorf("rtmp-push-worker: parse config: %w", err) + } + + eventsFile := os.NewFile(4, "push-events") + if eventsFile == nil { + return fmt.Errorf("rtmp-push-worker: missing events fd 4") + } + defer eventsFile.Close() + events := ingestframe.NewWriter(eventsFile) + + if err := media.RunRTMPPushWorker(ctx, cfg, os.Stdin, events); err != nil { + _ = events.Error(err.Error()) + return err + } + return events.End() + }, + } +} + func makeLiveCommand(build *config.BuildFlags) *urfavecli.Command { cli := config.CLI{Build: build} liveCmd := cli.NewCommand("live") diff --git a/pkg/director/stream_session.go b/pkg/director/stream_session.go index 44caf1fb..96eff5bc 100644 --- a/pkg/director/stream_session.go +++ b/pkg/director/stream_session.go @@ -953,7 +953,15 @@ func (ss *StreamSession) HandleMultistreamTargets(ctx context.Context) error { func (ss *StreamSession) StartMultistreamTarget(ctx context.Context, targetView *streamplace.MultistreamDefs_TargetView) error { for { - err := ss.mm.RTMPPush(ctx, ss.repoDID, "source", targetView) + // Under --isolated-ingest the crash-prone native egress pipeline runs in a + // worker subprocess (a gst fault there can't take the node down); otherwise + // it runs in-process. The on/off + status flow is identical either way. + var err error + if ss.cli.IsolatedIngest { + err = ss.mm.RTMPPushIsolated(ctx, ss.repoDID, "source", targetView) + } else { + err = ss.mm.RTMPPush(ctx, ss.repoDID, "source", targetView) + } if err != nil { log.Error(ctx, "failed to push to RTMP server", "error", err) err := ss.statefulDB.CreateMultistreamEvent(targetView.Uri, err.Error(), "error") diff --git a/pkg/ingestframe/frame.go b/pkg/ingestframe/frame.go index c8a746a3..783293f6 100644 --- a/pkg/ingestframe/frame.go +++ b/pkg/ingestframe/frame.go @@ -38,6 +38,11 @@ const ( // on the socket; main reads it and returns it to the WHIP client before // consuming segments. Payload: the answer SDP. Answer Type = 4 + // Event carries a worker status update (UTF-8 JSON) on the reverse channel. + // The RTMP push worker uses it to report multistream status (e.g. "active" + // with bytes acked) back to main, which writes it to the DB — the worker has + // no DB access of its own. Payload: JSON {status, message}. + Event Type = 5 ) func (t Type) String() string { @@ -50,6 +55,8 @@ func (t Type) String() string { return "error" case Answer: return "answer" + case Event: + return "event" default: return fmt.Sprintf("unknown(%d)", uint8(t)) } @@ -114,6 +121,9 @@ func (fw *Writer) Error(msg string) error { return fw.WriteFrame(Error, []byte(m // Answer frames the WHIP SDP answer (emitted first, before any segments). func (fw *Writer) Answer(sdp string) error { return fw.WriteFrame(Answer, []byte(sdp)) } +// Event frames a worker status update (JSON payload). +func (fw *Writer) Event(payload []byte) error { return fw.WriteFrame(Event, payload) } + // Reader decodes frames from an underlying stream. type Reader struct { r io.Reader diff --git a/pkg/ingestframe/frame_test.go b/pkg/ingestframe/frame_test.go index aa309f93..a6ce8968 100644 --- a/pkg/ingestframe/frame_test.go +++ b/pkg/ingestframe/frame_test.go @@ -24,6 +24,7 @@ func TestRoundTrip(t *testing.T) { require.NoError(t, w.Segment([]byte("seg-one"))) require.NoError(t, w.Segment(nil)) // zero-length segment is legal require.NoError(t, w.Segment(big)) + require.NoError(t, w.Event([]byte(`{"status":"active","message":"wrote 1234 bytes"}`))) require.NoError(t, w.Error("something broke")) require.NoError(t, w.End()) @@ -40,6 +41,7 @@ func TestRoundTrip(t *testing.T) { assertFrame(Segment, []byte("seg-one")) assertFrame(Segment, nil) assertFrame(Segment, big) + assertFrame(Event, []byte(`{"status":"active","message":"wrote 1234 bytes"}`)) assertFrame(Error, []byte("something broke")) assertFrame(End, nil) diff --git a/pkg/media/leak_test.go b/pkg/media/leak_test.go index eb7d634d..2c76d5f3 100644 --- a/pkg/media/leak_test.go +++ b/pkg/media/leak_test.go @@ -54,6 +54,9 @@ func TestMain(m *testing.M) { if len(os.Args) > 1 && os.Args[1] == "ingest-worker" { os.Exit(runIngestWorkerHelper()) } + if len(os.Args) > 1 && os.Args[1] == "rtmp-push-worker" { + os.Exit(runRTMPPushWorkerHelper()) + } if os.Getenv(IgnoreLeaks) != "" { gstinit.InitGST() os.Exit(m.Run()) diff --git a/pkg/media/rtmp_push.go b/pkg/media/rtmp_push.go index cee48a80..d49f66bf 100644 --- a/pkg/media/rtmp_push.go +++ b/pkg/media/rtmp_push.go @@ -19,6 +19,11 @@ import ( "stream.place/streamplace/pkg/streamplace" ) +// RTMPPush is the in-process multistream egress: it assembles the streamer's +// source segments into a continuous fMP4 stream and runs the native RTMP push +// pipeline over it, reporting status straight to the DB. The isolated +// counterpart (RTMPPushIsolated) runs the same native pipeline in a worker +// subprocess so a gst fault in the egress chain can't take the node down. func (mm *MediaManager) RTMPPush(ctx context.Context, user string, rendition string, targetView *streamplace.MultistreamDefs_TargetView) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -27,14 +32,98 @@ func (mm *MediaManager) RTMPPush(ctx context.Context, user string, rendition str if !ok { return fmt.Errorf("failed to convert target view to multistream target") } - targetURL := rec.Url + + // Source: subscribe to the streamer's segments and assemble one continuous + // fMP4 stream for the push pipeline. Tied to ctx so it tears down when the + // pipeline returns. + pr, pw := io.Pipe() + go func() { + pw.CloseWithError(mm.writeRTMPSource(ctx, user, rendition, pw)) + }() + + // Status straight to the DB (in-process). The isolated worker reports the + // same events back over a frame channel instead, where the supervisor writes + // them — see runRTMPPushPipeline / RTMPPushIsolated. + report := func(status, message string) { + if err := mm.atsync.StatefulDB.CreateMultistreamEvent(targetView.Uri, message, status); err != nil { + log.Error(ctx, "failed to create multistream event", "error", err) + } + } + return mm.runRTMPPushPipeline(ctx, pr, rec.Url, report) +} + +// writeRTMPSource subscribes to the streamer's source segments, selects the AAC +// audio + video tracks from each dual-codec segment, synthesizes a single fMP4 +// init from the first segment, and writes one continuous fMP4 stream to w (init +// then every segment's canonical bytes concatenated). It returns when ctx is +// done or a select/encode/write fails; the caller owns closing w. +// +// MUXL segments carry per-track monotonic tfdt, so blind concatenation after a +// single synthesized init is a valid fMP4 timeline with no remux. The init +// reflects the first segment's catalog and is never re-emitted; muxl derives the +// catalog from the moov and does not parse the H.264 bitstream, so a mid-stream +// resolution/orientation change (carried in-band as new SPS/PPS at a keyframe) +// is invisible to it — the parameter sets pass through verbatim to +// h264parse/flvmux and the init's declared dimensions simply stay at the initial +// config. Reflecting such a change in container metadata would require parsing +// SPS/PPS. +func (mm *MediaManager) writeRTMPSource(ctx context.Context, user, rendition string, w io.Writer) error { + segChan := mm.bus.SubscribeSegment(ctx, user, rendition) + defer mm.bus.UnsubscribeSegment(ctx, user, rendition, segChan) + first := true + for { + select { + case <-ctx.Done(): + return ctx.Err() + case seg := <-segChan.C: + log.Debug(ctx, "segment received", "file", seg.Filepath) + if len(seg.Muxl) == 0 { + log.Warn(ctx, "source segment has no MUXL bytes, skipping", "file", seg.Filepath) + continue + } + // RTMP wants AAC: select video + the AAC audio track from the + // dual-codec segment and feed only those, so flvmux gets AAC with no + // transcode. + aacSeg, err := filterSegmentToCodec(ctx, seg.Muxl, false) + if err != nil { + return fmt.Errorf("select AAC audio: %w", err) + } + if first { + var init bytes.Buffer + if err := muxl.RunMuxlWrapInit(ctx, bytes.NewReader(aacSeg), &init); err != nil { + return fmt.Errorf("synthesize init segment: %w", err) + } + log.Debug(ctx, "init segment synthesized", "size", init.Len()) + if _, err := w.Write(init.Bytes()); err != nil { + return err + } + first = false + } + log.Debug(ctx, "writing segment", "size", len(aacSeg)) + if _, err := w.Write(aacSeg); err != nil { + return err + } + } + } +} + +// runRTMPPushPipeline builds and runs the native RTMP egress pipeline: +// appsrc(source) → qtdemux → {h264parse, aacparse} → flvmux → rtmp2sink → a +// local TCP/TLS forwarder → the target URL. Status updates (currently "active" +// once the server acks bytes) go through report. This is the crash-prone native +// core shared by the in-process RTMPPush and the isolated rtmp-push worker; only +// `source` and `report` differ between them, so the two paths run an identical +// gst pipeline and can't drift. +func (mm *MediaManager) runRTMPPushPipeline(ctx context.Context, source io.Reader, targetURL string, report func(status, message string)) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() pipelineSlice := []string{ "appsrc name=muxlsrc ! qtdemux name=demux", "flvmux name=muxer ! rtmp2sink name=rtmp2sink", fmt.Sprintf("%s name=videoqueue ! h264parse ! muxer.video", constants.Queue2Big), - // Segments carry AAC (we feed only the AAC track below), so pass it - // straight to flvmux — no Opus→AAC transcode. + // Segments carry AAC (we feed only the AAC track), so pass it straight to + // flvmux — no Opus→AAC transcode. fmt.Sprintf("%s name=audioqueue ! aacparse ! muxer.audio", constants.Queue2Big), } @@ -48,36 +137,32 @@ func (mm *MediaManager) RTMPPush(ctx context.Context, user string, rendition str return fmt.Errorf("failed to get rtmp2sink element from pipeline: %w", err) } + // rtmp2sink can't speak rtmps and doesn't do TLS, so we always point it at a + // localhost forwarder that relays (optionally over TLS) to the real target. u, err := url.Parse(targetURL) if err != nil { return fmt.Errorf("failed to parse target URL: %w", err) } - if u.Scheme == "rtmps" { - localAddr, err := mm.RunTLSFForwarder(ctx, targetURL) - if err != nil { - return fmt.Errorf("failed to run TLS forwarder: %w", err) - } - local := fmt.Sprintf("rtmp://%s%s", localAddr, u.Path) - log.Debug(ctx, "running TLS forwarder", "localAddr", local, "destination", targetURL) - err = rtmp2sink.SetProperty("location", local) - if err != nil { - return fmt.Errorf("failed to set rtmp2sink location: %w", err) - } - } else if u.Scheme == "rtmp" { - localAddr, err := mm.RunTCPForwarder(ctx, targetURL) - if err != nil { - return fmt.Errorf("failed to run TCP forwarder: %w", err) - } - local := fmt.Sprintf("rtmp://%s%s", localAddr, u.Path) - log.Debug(ctx, "running TCP forwarder", "localAddr", local, "destination", targetURL) - err = rtmp2sink.SetProperty("location", local) - if err != nil { - return fmt.Errorf("failed to set rtmp2sink location: %w", err) - } - } else { + var localAddr string + switch u.Scheme { + case "rtmps": + localAddr, err = mm.RunTLSFForwarder(ctx, targetURL) + case "rtmp": + localAddr, err = mm.RunTCPForwarder(ctx, targetURL) + default: return fmt.Errorf("invalid target URL scheme: %s", u.Scheme) } + if err != nil { + return fmt.Errorf("failed to run forwarder: %w", err) + } + local := fmt.Sprintf("rtmp://%s%s", localAddr, u.Path) + log.Debug(ctx, "running forwarder", "localAddr", local) + if err := rtmp2sink.SetProperty("location", local); err != nil { + return fmt.Errorf("failed to set rtmp2sink location: %w", err) + } + // Poll the sink and report "active" once the destination acks bytes; back off + // once it's flowing so we don't spam the status channel. go func() { pollFreq := time.Second * 1 for { @@ -87,16 +172,16 @@ func (mm *MediaManager) RTMPPush(ctx context.Context, user string, rendition str case <-time.After(pollFreq): prop, err := rtmp2sink.GetProperty("stats") if err != nil { - log.Error(ctx, "error getting rtmp2sink peak-kbps", "error", err) + log.Error(ctx, "error getting rtmp2sink stats", "error", err) continue } if prop == nil { - log.Error(ctx, "failed to get rtmp2sink peak-kbps", "prop", prop) + log.Error(ctx, "failed to get rtmp2sink stats", "prop", prop) continue } propVal, ok := prop.(*gst.Structure) if !ok { - log.Error(ctx, "failed to convert rtmp2sink peak-kbps", "prop", prop) + log.Error(ctx, "failed to convert rtmp2sink stats", "prop", prop) continue } outBytesAcked, err := propVal.GetValue("out-bytes-acked") @@ -110,88 +195,21 @@ func (mm *MediaManager) RTMPPush(ctx context.Context, user string, rendition str continue } if outBytesAckedVal > 0 { - err = mm.atsync.StatefulDB.CreateMultistreamEvent(targetView.Uri, fmt.Sprintf("wrote %d bytes", outBytesAckedVal), "active") - if err != nil { - log.Error(ctx, "failed to create multistream event", "error", err) - } - // increase pollFreq, once it's working we don't need to spam the database + report("active", fmt.Sprintf("wrote %d bytes", outBytesAckedVal)) + // once it's working we don't need to spam the status channel pollFreq = time.Second * 15 } log.Debug(ctx, "rtmp2sink out-bytes-acked", "outBytesAckedVal", outBytesAckedVal) } - } }() - // Reassemble the streamer's source segments into one continuous fMP4 - // stream for a single qtdemux: synthesize the init (ftyp+moov) from the - // first segment's embedded catalog, then blindly concatenate every - // segment's canonical bytes after it. MUXL segments carry per-track - // monotonic tfdt, so this is a valid fMP4 timeline with no remux. - // - // The init reflects the first segment's catalog and is never re-emitted. - // muxl derives the catalog from the moov and does not parse the H.264 - // bitstream, so a mid-stream resolution/orientation change (carried in-band - // as new SPS/PPS at a keyframe) is invisible to it: no second moov appears, - // the parameter sets pass through verbatim to h264parse/flvmux, and the - // init's declared dimensions simply stay at the initial config. Reflecting - // such a change in container metadata would require parsing SPS/PPS. - pr, pw := io.Pipe() - muxlLoop := func() { - segChan := mm.bus.SubscribeSegment(ctx, user, rendition) - defer mm.bus.UnsubscribeSegment(ctx, user, rendition, segChan) - first := true - for { - select { - case <-ctx.Done(): - pw.CloseWithError(ctx.Err()) - return - case seg := <-segChan.C: - log.Debug(ctx, "segment received", "file", seg.Filepath) - if len(seg.Muxl) == 0 { - log.Warn(ctx, "source segment has no MUXL bytes, skipping", "file", seg.Filepath) - continue - } - // RTMP wants AAC: select video + the AAC audio track from the - // dual-codec segment and feed only those, so flvmux gets AAC - // with no transcode. - aacSeg, err := filterSegmentToCodec(ctx, seg.Muxl, false) - if err != nil { - log.Error(ctx, "failed to select AAC audio", "error", err) - pw.CloseWithError(err) - return - } - if first { - var init bytes.Buffer - if err := muxl.RunMuxlWrapInit(ctx, bytes.NewReader(aacSeg), &init); err != nil { - pw.CloseWithError(fmt.Errorf("synthesize init segment: %w", err)) - return - } - log.Debug(ctx, "init segment synthesized", "size", init.Len()) - if _, err := pw.Write(init.Bytes()); err != nil { - log.Error(ctx, "failed to write init segment", "error", err) - pw.CloseWithError(err) - return - } - first = false - } - log.Debug(ctx, "writing segment", "size", len(aacSeg)) - if _, err := pw.Write(aacSeg); err != nil { - log.Error(ctx, "failed to write segment", "error", err) - pw.CloseWithError(err) - return - } - } - } - } - go muxlLoop() - muxlSrc, err := pipeline.GetElementByName("muxlsrc") if err != nil { return fmt.Errorf("failed to get appsrc element from pipeline: %w", err) } app.SrcFromElement(muxlSrc).SetCallbacks(&app.SourceCallbacks{ - NeedDataFunc: ReaderNeedDataIncremental(ctx, pr), + NeedDataFunc: ReaderNeedDataIncremental(ctx, source), }) videoQueue, err := pipeline.GetElementByName("videoqueue") @@ -242,14 +260,14 @@ func (mm *MediaManager) RTMPPush(ctx context.Context, user string, rendition str defer func() { log.Log(ctx, "shutting down RTMP push pipeline") - err = pipeline.SetState(gst.StateNull) - if err != nil { + if err := pipeline.SetState(gst.StateNull); err != nil { log.Error(ctx, "failed to set pipeline state to null", "error", err) } }() return <-errCh } + func (mm *MediaManager) RunTLSFForwarder(ctx context.Context, dest string) (string, error) { destURL, err := url.Parse(dest) if err != nil { diff --git a/pkg/media/rtmp_push_worker.go b/pkg/media/rtmp_push_worker.go new file mode 100644 index 00000000..89e33ecc --- /dev/null +++ b/pkg/media/rtmp_push_worker.go @@ -0,0 +1,211 @@ +package media + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "sync" + + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/gstinit" + "stream.place/streamplace/pkg/ingestframe" + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/streamplace" +) + +// RTMPPushWorkerConfig is the startup handshake main hands an rtmp-push worker +// over a dedicated pipe fd. TargetURL embeds the destination stream key, so it +// rides fd 3 and is kept off argv/env — only the (public) streamer DID goes on +// the command line, for process-listing identification. +type RTMPPushWorkerConfig struct { + StreamerDID string `json:"streamer_did"` + // TargetURL is the rtmp(s):// destination including its stream key. Sensitive + // — fd-3 only. + TargetURL string `json:"target_url"` +} + +// pushEvent is the worker→main status payload carried in an ingestframe.Event. +// Main writes it verbatim as a multistream event against the target's AT-URI +// (the URI lives main-side; the worker only knows status semantics). +type pushEvent struct { + Status string `json:"status"` + Message string `json:"message"` +} + +// RunRTMPPushWorker is the body of the `rtmp-push-worker` subcommand. It reads +// the assembled fMP4 source stream from `source` (the worker's stdin, fed by +// main) and runs the native RTMP egress pipeline, reporting status back to main +// as Event frames instead of writing the DB directly (the worker has no DB). +// Returns when the pipeline ends (source EOS) or errors; the caller frames +// End/Error accordingly. +func RunRTMPPushWorker(ctx context.Context, cfg RTMPPushWorkerConfig, source io.Reader, events *ingestframe.Writer) error { + gstinit.InitGST() + // Minimal manager: the push pipeline + forwarder need no model/DB/bus, only a + // CLI shell. + mm := &MediaManager{cli: &config.CLI{}} + report := func(status, message string) { + payload, err := json.Marshal(pushEvent{Status: status, Message: message}) + if err != nil { + log.Error(ctx, "rtmp push worker: marshal event", "error", err) + return + } + if err := events.Event(payload); err != nil { + log.Error(ctx, "rtmp push worker: emit event", "error", err) + } + } + return mm.runRTMPPushPipeline(ctx, source, cfg.TargetURL, report) +} + +// RTMPPushIsolated is the process-isolated counterpart to RTMPPush: the +// crash-prone native egress pipeline (qtdemux/flvmux/rtmp2sink) runs in a +// dedicated `rtmp-push-worker` subprocess, so a gst fault there kills only the +// worker — the node survives. The bus subscription, segment assembly, and DB +// status writes stay here in main; only the native pipeline moves. +// +// Lifecycle maps onto the existing multistream control loop: the worker runs +// under exec.CommandContext(ctx), so the cancel HandleMultistreamTargets fires +// when a target is disabled tears the worker down, and a crash surfaces as a +// non-zero exit that StartMultistreamTarget turns into an "error" event + retry. +func (mm *MediaManager) RTMPPushIsolated(ctx context.Context, user string, rendition string, targetView *streamplace.MultistreamDefs_TargetView) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + ctx = log.WithLogValues(ctx, "mediafunc", "RTMPPushIsolated") + rec, ok := targetView.Record.Val.(*streamplace.MultistreamTarget) + if !ok { + return fmt.Errorf("failed to convert target view to multistream target") + } + + cfgJSON, err := json.Marshal(RTMPPushWorkerConfig{StreamerDID: user, TargetURL: rec.Url}) + if err != nil { + return fmt.Errorf("marshal push worker config: %w", err) + } + + exe, err := os.Executable() + if err != nil { + return fmt.Errorf("locate self: %w", err) + } + // The streamer DID rides argv (public) so a worker is identifiable in a + // process listing; the target URL (with its key) stays on fd 3. + cmd := exec.CommandContext(ctx, exe, "rtmp-push-worker", user) + + // fd 3 carries the config in; fd 4 carries the status-event stream out. Off + // stdout so stray gst/log noise can't corrupt the frames. + cfgR, cfgW, err := os.Pipe() + if err != nil { + return fmt.Errorf("config pipe: %w", err) + } + defer cfgW.Close() + eventsR, eventsW, err := os.Pipe() + if err != nil { + cfgR.Close() + return fmt.Errorf("events pipe: %w", err) + } + defer eventsR.Close() + cmd.ExtraFiles = []*os.File{cfgR, eventsW} // → child fd 3, fd 4 + + stdin, err := cmd.StdinPipe() + if err != nil { + cfgR.Close() + eventsW.Close() + return err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + cfgR.Close() + eventsW.Close() + return err + } + stderr, err := cmd.StderrPipe() + if err != nil { + cfgR.Close() + eventsW.Close() + return err + } + + if err := cmd.Start(); err != nil { + cfgR.Close() + eventsW.Close() + return fmt.Errorf("start rtmp push worker: %w", err) + } + cfgR.Close() // the child holds its own copy now + eventsW.Close() // ditto; the parent only reads eventsR + + go func() { + _, _ = cfgW.Write(cfgJSON) + cfgW.Close() // EOF so the worker's config read completes + }() + + // Feed the worker the continuous fMP4 source stream (bus → AAC select → + // init+concat). Closing stdin on source EOF/cancel is the worker's EOS. + go func() { + defer stdin.Close() + if serr := mm.writeRTMPSource(ctx, user, rendition, stdin); serr != nil && ctx.Err() == nil { + log.Error(ctx, "rtmp push source ended", "error", serr) + } + }() + + // Forward stdout + stderr to the node logger. Drain both fully before Wait. + var logsWG sync.WaitGroup + logsWG.Add(2) + go func() { defer logsWG.Done(); streamWorkerLogs(ctx, stdout, user) }() + go func() { defer logsWG.Done(); streamWorkerLogs(ctx, stderr, user) }() + + // Read status events from the worker and write each as a multistream event + // against this target — the DB side the worker can't reach itself. + report := func(status, message string) { + if cerr := mm.atsync.StatefulDB.CreateMultistreamEvent(targetView.Uri, message, status); cerr != nil { + log.Error(ctx, "failed to create multistream event", "error", cerr) + } + } + workerErr := consumePushEvents(ctx, eventsR, report) + logsWG.Wait() + werr := cmd.Wait() + + switch { + case ctx.Err() != nil: + // Target disabled (cancel) — a clean stop, mirroring in-process RTMPPush + // returning ctx.Err() from the bus handler. + return ctx.Err() + case workerErr != nil: + return fmt.Errorf("rtmp push worker stream: %w", workerErr) + case werr != nil: + // A non-zero exit without a clean end means the worker died — contained to + // the subprocess; StartMultistreamTarget records it and retries. + return fmt.Errorf("rtmp push worker exited: %w", werr) + } + return nil +} + +// consumePushEvents reads status frames from the worker and hands each to +// report (which main wires to the multistream-event DB write). Returns nil on a +// clean End/EOF, the worker's reported message on an Error frame, or the +// terminal read error if the frame stream tore mid-frame (worker died). +func consumePushEvents(ctx context.Context, r io.Reader, report func(status, message string)) error { + fr := ingestframe.NewReader(r) + for { + typ, payload, err := fr.ReadFrame() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + switch typ { + case ingestframe.Event: + var ev pushEvent + if uerr := json.Unmarshal(payload, &ev); uerr != nil { + log.Error(ctx, "rtmp push: bad event frame", "error", uerr) + continue + } + report(ev.Status, ev.Message) + case ingestframe.Error: + return fmt.Errorf("%s", string(payload)) + case ingestframe.End: + return nil + } + } +} diff --git a/pkg/media/rtmp_push_worker_test.go b/pkg/media/rtmp_push_worker_test.go new file mode 100644 index 00000000..58144af6 --- /dev/null +++ b/pkg/media/rtmp_push_worker_test.go @@ -0,0 +1,128 @@ +package media + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + "testing" + "time" + + lexutil "github.com/bluesky-social/indigo/lex/util" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/bus" + "stream.place/streamplace/pkg/ingestframe" + "stream.place/streamplace/pkg/streamplace" +) + +// runRTMPPushWorkerHelper is what the test binary becomes when re-exec'd with +// the `rtmp-push-worker` arg (see TestMain). It mirrors makeRTMPPushWorkerCommand: +// config on fd 3, the fMP4 source on stdin, status Event frames on fd 4; a clean +// run ends with End, a fatal error with an Error frame and a non-zero exit. +func runRTMPPushWorkerHelper() int { + cfgFile := os.NewFile(3, "push-config") + if cfgFile == nil { + return 2 + } + cfgBytes, err := io.ReadAll(cfgFile) + cfgFile.Close() + if err != nil { + return 2 + } + var cfg RTMPPushWorkerConfig + if err := json.Unmarshal(cfgBytes, &cfg); err != nil { + return 2 + } + eventsFile := os.NewFile(4, "push-events") + if eventsFile == nil { + return 2 + } + defer eventsFile.Close() + events := ingestframe.NewWriter(eventsFile) + if err := RunRTMPPushWorker(context.Background(), cfg, os.Stdin, events); err != nil { + _ = events.Error(err.Error()) + return 1 + } + if err := events.End(); err != nil { + return 1 + } + return 0 +} + +// TestConsumePushEvents covers the main-side status translation deterministically +// (no gst, no subprocess): Event frames are reported in order, an Error frame is +// surfaced as the returned error, and a clean End yields nil. +func TestConsumePushEvents(t *testing.T) { + t.Run("events then clean end", func(t *testing.T) { + pr, pw := io.Pipe() + w := ingestframe.NewWriter(pw) + go func() { + _ = w.Event([]byte(`{"status":"active","message":"wrote 5 bytes"}`)) + _ = w.Event([]byte(`{"status":"active","message":"wrote 99 bytes"}`)) + _ = w.End() + pw.Close() + }() + + var got []pushEvent + err := consumePushEvents(context.Background(), pr, func(status, message string) { + got = append(got, pushEvent{Status: status, Message: message}) + }) + require.NoError(t, err) + require.Equal(t, []pushEvent{ + {Status: "active", Message: "wrote 5 bytes"}, + {Status: "active", Message: "wrote 99 bytes"}, + }, got) + }) + + t.Run("error frame becomes the returned error", func(t *testing.T) { + pr, pw := io.Pipe() + w := ingestframe.NewWriter(pw) + go func() { + _ = w.Error("invalid target URL scheme: http") + pw.Close() + }() + + called := false + err := consumePushEvents(context.Background(), pr, func(string, string) { called = true }) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid target URL scheme") + require.False(t, called, "report must not fire on an error frame") + }) + + t.Run("torn stream is a non-nil error (worker died)", func(t *testing.T) { + // A frame header with a payload length that never arrives = an abrupt death + // mid-frame, which must NOT look like a clean end. + var buf bytes.Buffer + require.NoError(t, ingestframe.NewWriter(&buf).Event([]byte("0123456789"))) + torn := buf.Bytes()[:buf.Len()-5] // lop off the tail of the payload + err := consumePushEvents(context.Background(), bytes.NewReader(torn), func(string, string) {}) + require.Error(t, err) + }) +} + +// TestRTMPPushWorkerContainsFailure proves the containment property end to end: +// a worker whose push pipeline fails (here an invalid target scheme, which the +// shared pipeline core rejects before streaming) dies in its own subprocess and +// the supervisor returns an error — with the test process (the node) still +// running to make the assertion. In-process this same failure path stays inside +// the node; isolated, it can't take the node down. +func TestRTMPPushWorkerContainsFailure(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // A bus is needed so the source goroutine (writeRTMPSource) doesn't nil-panic; + // it just blocks on the empty bus until the worker fails and we cancel. + mm := &MediaManager{bus: bus.NewBus()} + targetView := &streamplace.MultistreamDefs_TargetView{ + Uri: "at://did:plc:test/place.stream.multistream.target/abc", + Record: &lexutil.LexiconTypeDecoder{ + Val: &streamplace.MultistreamTarget{Url: "http://127.0.0.1:1/nope"}, + }, + } + + err := mm.RTMPPushIsolated(ctx, "did:plc:test", "source", targetView) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid target URL scheme", + "the worker should reach the pipeline core and reject the bad scheme") +} -- 2.51.2 From 6644061c15d5a8160e6ce241c534301c4eb166e8 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Sat, 6 Jun 2026 15:38:39 -0700 Subject: [PATCH 13/17] config: default isolated-ingest to true --- pkg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 1a0ffecc..26cfc0f8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -232,7 +232,7 @@ func (cli *CLI) NewCommand(name string) *urfavecli.Command { &urfavecli.BoolFlag{ Name: "isolated-ingest", Usage: "Run each MKV/RTMP-push ingest in an isolated worker subprocess (fault isolation)", - Value: false, + Value: true, Destination: &cli.IsolatedIngest, Sources: urfavecli.EnvVars("SP_ISOLATED_INGEST"), }, -- 2.51.2 From 51824d5440dca94fdbd36c77c0bfcbc9b5c04d1d Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Sat, 6 Jun 2026 15:41:03 -0700 Subject: [PATCH 14/17] ingestframe: switch the worker wire format to DRISL CBOR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker↔main protocol was a hand-rolled magic+type+length framing. Swap it for a stream of concatenated DRISL CBOR items — the same codec muxl uses for its own stdio protocol — so the whole stack speaks one self-describing, debuggable format and we stop maintaining a bespoke parser. CBOR data items are self-delimiting (the length/count is in each item's head), so the separate length prefix is redundant and goes away. Crucially this PRESERVES the crash-vs-clean-end signal the supervisor depends on: the decoder returns io.EOF at an item boundary (clean end) and io.ErrUnexpectedEOF mid-item (a worker that died) — verified against the real hyphacoop/cbor decoder, not assumed. A garbage/desynced stream now fails to decode rather than being mis-parsed (replaces the magic-tag guard); the explicit MaxPayload cap is dropped (the decoder grows incrementally rather than pre-allocating a hostile declared length, and the peer is a trusted local subprocess). - frame.go: each frame is one DRISL CBOR map {type, payload} (payload omitted when empty, e.g. End). Writer marshals via drisl.Marshal then writes under the lock (frames still never interleave); Reader wraps drisl.NewDecoder. The public API (Type constants, Writer/Reader, WriteFrame, the per-type helpers) is unchanged, so consumers don't move — except: - A streaming CBOR decoder READS AHEAD, so a Reader now owns its stream for the stream's lifetime. consumeWorkerFrames takes a *Reader instead of an io.Reader, and the WHIP path reuses ONE Reader across readWHIPAnswer → the segment consume (a second Reader would lose buffered read-ahead). The reconnecting socket consumer still makes a fresh Reader per connection, which is correct — a reconnect replays the worker's buffer from the start. go-dasl moves from indirect to a direct dependency (used via its drisl subpackage); hyphacoop/cbor stays indirect (a small interface avoids importing it here). Tests: round-trip across all frame types; truncation = ErrUnexpectedEOF, torn-head = ErrUnexpectedEOF, clean-boundary = EOF, garbage = decode error, all re-proven against the real codec; concurrent-writer integrity. The whole worker stack still passes over real pipes/sockets/ICE — subprocess framing, buffered socket reconnect, and the WHIP Answer-then-segments single-decoder path (TestWHIPWorkerLoopback). Co-Authored-By: Claude Opus 4.8 --- go.mod | 2 +- pkg/ingestframe/frame.go | 129 +++++++++++++++------------------ pkg/ingestframe/frame_test.go | 42 ++++------- pkg/media/ingest_daemon.go | 21 ++++-- pkg/media/ingest_supervisor.go | 5 +- pkg/media/whip_worker_test.go | 11 ++- 6 files changed, 97 insertions(+), 113 deletions(-) diff --git a/go.mod b/go.mod index ad90ee77..be1d8449 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/golangci/golangci-lint/v2 v2.1.6 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/hyphacoop/go-dasl v0.8.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-ipld-cbor v0.2.0 github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 @@ -312,7 +313,6 @@ require ( github.com/hexops/gotextdiff v1.0.3 // indirect github.com/holiman/uint256 v1.3.2 // indirect github.com/hyphacoop/cbor/v2 v2.0.0-20251007204234-2a4fa83e606e // indirect - github.com/hyphacoop/go-dasl v0.8.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/yaml v0.3.1 // indirect github.com/ipfs/bbloom v0.0.4 // indirect diff --git a/pkg/ingestframe/frame.go b/pkg/ingestframe/frame.go index 783293f6..22e52a42 100644 --- a/pkg/ingestframe/frame.go +++ b/pkg/ingestframe/frame.go @@ -1,26 +1,37 @@ // Package ingestframe defines the wire protocol a per-stream ingest worker uses -// to stream canonical MUXL fragments back to the main streamplace process. +// to stream canonical MUXL fragments (and status) back to the main streamplace +// process. // // Each incoming live stream is handled by an isolated worker subprocess that -// owns the socket, muxes + transcodes the media, and signs each GoP. It emits -// the resulting signed canonical .m4s segments to the main process as a sequence -// of typed, length-prefixed frames. +// owns the socket, muxes + transcodes + signs the media, and emits the resulting +// signed canonical .m4s segments to the main process as a sequence of typed +// messages. The same channel carries control messages (a clean end, a fatal +// error, a WHIP SDP answer, a status event). // -// The framing is deliberately transport-agnostic: today it rides the worker's -// stdout pipe, but a detached / reattachable worker (the zero-downtime-upgrade -// path, where workers keep buffering signed segments across a main restart) can -// carry the identical frames over a unix socket. Nothing above this package -// cares which. +// The wire format is a stream of concatenated DRISL CBOR items — the same codec +// muxl uses for its own stdio protocol. CBOR data items are self-delimiting (the +// length/count lives in each item's head), so no separate length prefix is +// needed, and a decoder reads exactly one item per call. Crucially this PRESERVES +// the crash-vs-clean-end signal the supervisor relies on: the decoder returns +// io.EOF at an item boundary (the stream ended cleanly between messages) and +// io.ErrUnexpectedEOF mid-item (a worker that died). A garbage/desynced stream +// fails to decode rather than being mis-parsed. +// +// The format is transport-agnostic: today it rides the worker's stdout pipe or a +// per-session unix socket (the zero-downtime detach/reattach path, where workers +// keep buffering signed segments across a main restart). Nothing above this +// package cares which. package ingestframe import ( - "encoding/binary" "fmt" "io" "sync" + + "github.com/hyphacoop/go-dasl/drisl" ) -// Type identifies a frame's payload. +// Type identifies a message's payload. type Type uint8 const ( @@ -28,13 +39,14 @@ const ( // ValidateMP4 ingests. Payload: the bare canonical segment bytes. Segment Type = 1 // End signals the worker finished the stream cleanly (graceful EOS). No - // payload. Its ABSENCE before EOF is how main tells a crash from a clean end. + // payload. It's the in-band "done" marker; its absence before EOF (together + // with the worker's exit code) is how main tells a crash from a clean end. End Type = 2 // Error carries a worker-side fatal error message (UTF-8). The worker emits // it just before exiting so main can log a cause, not a bare "worker exited". Error Type = 3 // Answer carries an SDP answer (UTF-8). The WHIP worker owns the - // PeerConnection, so it generates the answer and emits it as the FIRST frame + // PeerConnection, so it generates the answer and emits it as the FIRST message // on the socket; main reads it and returns it to the WHIP client before // consuming segments. Payload: the answer SDP. Answer Type = 4 @@ -62,15 +74,21 @@ func (t Type) String() string { } } -// magic prefixes every frame so a desynced/corrupt stream is caught immediately -// rather than mis-parsed as a length. -var magic = [4]byte{'S', 'P', 'F', '1'} - -// MaxPayload bounds a single frame so a corrupt or hostile length can't make the -// reader allocate unboundedly. Canonical GoP segments are well under this. -const MaxPayload = 64 << 20 // 64 MiB +// message is the on-wire DRISL CBOR item: one self-delimiting map per frame. +// Payload is a CBOR byte string (raw for Segment, UTF-8/JSON for the rest) and is +// omitted entirely for an empty body (e.g. End), so a bodyless frame is just +// {"type": N}. +type message struct { + Type Type `cbor:"type"` + Payload []byte `cbor:"payload,omitempty"` +} -const headerSize = 4 + 1 + 4 // magic + type + uint32 length +// frameDecoder is the streaming-decode surface we need (satisfied by drisl's +// *cbor.Decoder). Kept as an interface so this package needn't import the cbor +// module directly. +type frameDecoder interface { + Decode(v any) error +} // Writer serializes frames to an underlying stream. Safe for concurrent use: a // worker emits segments from more than one goroutine (the source signer and the @@ -80,33 +98,23 @@ type Writer struct { w io.Writer } -// NewWriter wraps w. w is typically the worker's os.Stdout. +// NewWriter wraps w. w is typically the worker's frame fd or a socket conn. func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } // WriteFrame writes one whole frame atomically with respect to other WriteFrame -// calls on the same Writer. +// calls on the same Writer. The CBOR item is encoded up front, then written under +// the lock, so concurrent writers never interleave a frame's bytes. func (fw *Writer) WriteFrame(t Type, payload []byte) error { - if len(payload) > MaxPayload { - return fmt.Errorf("ingestframe: payload %d exceeds max %d", len(payload), MaxPayload) + b, err := drisl.Marshal(message{Type: t, Payload: payload}) + if err != nil { + return fmt.Errorf("ingestframe: encode %s: %w", t, err) } - var hdr [headerSize]byte - copy(hdr[0:4], magic[:]) - hdr[4] = byte(t) - binary.BigEndian.PutUint32(hdr[5:9], uint32(len(payload))) - fw.mu.Lock() defer fw.mu.Unlock() - if _, err := fw.w.Write(hdr[:]); err != nil { - return err - } - if len(payload) > 0 { - if _, err := fw.w.Write(payload); err != nil { - return err - } - } - return nil + _, err = fw.w.Write(b) + return err } // Segment frames a signed canonical .m4s segment. @@ -124,44 +132,27 @@ func (fw *Writer) Answer(sdp string) error { return fw.WriteFrame(Answer, []byte // Event frames a worker status update (JSON payload). func (fw *Writer) Event(payload []byte) error { return fw.WriteFrame(Event, payload) } -// Reader decodes frames from an underlying stream. +// Reader decodes frames from an underlying stream. The decoder buffers/reads +// ahead, so a Reader OWNS its stream for the stream's lifetime — don't create a +// second Reader on the same connection (it would lose the first's buffered +// read-ahead). type Reader struct { - r io.Reader + dec frameDecoder } -// NewReader wraps r, typically the worker's stdout pipe. +// NewReader wraps r, typically the worker's frame fd or a socket conn. func NewReader(r io.Reader) *Reader { - return &Reader{r: r} + return &Reader{dec: drisl.NewDecoder(r)} } -// ReadFrame decodes the next frame. It returns io.EOF only at a clean frame -// boundary (the stream ended between frames); a stream that dies mid-frame +// ReadFrame decodes the next frame. It returns io.EOF only at a clean item +// boundary (the stream ended between frames); a stream that dies mid-item // surfaces as io.ErrUnexpectedEOF, so an abrupt worker death is distinguishable -// from a clean close. +// from a clean close. A malformed/desynced item surfaces as a decode error. func (fr *Reader) ReadFrame() (Type, []byte, error) { - var hdr [headerSize]byte - if _, err := io.ReadFull(fr.r, hdr[:]); err != nil { - // io.EOF here = clean boundary. io.ReadFull maps a partial read to - // ErrUnexpectedEOF, which we keep: a torn header is an abrupt death. - return 0, nil, err - } - if [4]byte(hdr[0:4]) != magic { - return 0, nil, fmt.Errorf("ingestframe: bad magic %q (stream desynced)", hdr[0:4]) - } - t := Type(hdr[4]) - n := binary.BigEndian.Uint32(hdr[5:9]) - if n > MaxPayload { - return 0, nil, fmt.Errorf("ingestframe: frame length %d exceeds max %d", n, MaxPayload) - } - if n == 0 { - return t, nil, nil - } - payload := make([]byte, n) - if _, err := io.ReadFull(fr.r, payload); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } + var m message + if err := fr.dec.Decode(&m); err != nil { return 0, nil, err } - return t, payload, nil + return m.Type, m.Payload, nil } diff --git a/pkg/ingestframe/frame_test.go b/pkg/ingestframe/frame_test.go index a6ce8968..015028ed 100644 --- a/pkg/ingestframe/frame_test.go +++ b/pkg/ingestframe/frame_test.go @@ -2,7 +2,6 @@ package ingestframe import ( "bytes" - "encoding/binary" "errors" "fmt" "io" @@ -52,7 +51,8 @@ func TestRoundTrip(t *testing.T) { // TestTruncatedFrameIsUnexpectedEOF is the crash-vs-clean-end distinction the // supervisor relies on: a worker that dies mid-segment must NOT look like a -// graceful end. +// graceful end. CBOR's self-delimiting framing gives this for free — a byte +// string that declares more bytes than arrive surfaces as ErrUnexpectedEOF. func TestTruncatedFrameIsUnexpectedEOF(t *testing.T) { var buf bytes.Buffer require.NoError(t, NewWriter(&buf).Segment(bytes.Repeat([]byte{1}, 1000))) @@ -65,42 +65,26 @@ func TestTruncatedFrameIsUnexpectedEOF(t *testing.T) { require.ErrorIs(t, err, io.ErrUnexpectedEOF) } -// TestTornHeaderIsUnexpectedEOF: dying partway through the header is also an -// abrupt death, not a clean boundary. +// TestTornHeaderIsUnexpectedEOF: dying partway through the CBOR item head (here +// after the map header byte, before the first key) is also an abrupt death, not +// a clean boundary. func TestTornHeaderIsUnexpectedEOF(t *testing.T) { var buf bytes.Buffer require.NoError(t, NewWriter(&buf).End()) - torn := buf.Bytes()[:headerSize-2] + torn := buf.Bytes()[:1] // just the map header; the rest never arrives _, _, err := NewReader(bytes.NewReader(torn)).ReadFrame() require.ErrorIs(t, err, io.ErrUnexpectedEOF) } -// TestBadMagicRejected: a desynced/corrupt stream is caught, not mis-parsed. -func TestBadMagicRejected(t *testing.T) { - junk := append([]byte("XXXX"), make([]byte, headerSize)...) - _, _, err := NewReader(bytes.NewReader(junk)).ReadFrame() +// TestGarbageRejected: a desynced/corrupt stream is caught, not mis-parsed. A +// complete-but-wrong-shaped CBOR item (a bare integer, not a frame map) must +// surface as a decode error distinct from EOF / a torn frame. +func TestGarbageRejected(t *testing.T) { + _, _, err := NewReader(bytes.NewReader([]byte{0x01})).ReadFrame() require.Error(t, err) - require.Contains(t, err.Error(), "bad magic") -} - -// TestOversizeLengthRejected: a hostile length can't trigger an unbounded alloc. -func TestOversizeLengthRejected(t *testing.T) { - var hdr [headerSize]byte - copy(hdr[0:4], magic[:]) - hdr[4] = byte(Segment) - binary.BigEndian.PutUint32(hdr[5:9], uint32(MaxPayload+1)) - - _, _, err := NewReader(bytes.NewReader(hdr[:])).ReadFrame() - require.Error(t, err) - require.Contains(t, err.Error(), "exceeds max") -} - -// TestWriteOversizeRejected: the writer refuses to emit an over-cap frame. -func TestWriteOversizeRejected(t *testing.T) { - err := NewWriter(io.Discard).Segment(make([]byte, MaxPayload+1)) - require.Error(t, err) - require.Contains(t, err.Error(), "exceeds max") + require.NotErrorIs(t, err, io.EOF) + require.NotErrorIs(t, err, io.ErrUnexpectedEOF) } // TestConcurrentWritesDoNotInterleave: the worker emits segments from multiple diff --git a/pkg/media/ingest_daemon.go b/pkg/media/ingest_daemon.go index 9d9c068b..bf5a0edf 100644 --- a/pkg/media/ingest_daemon.go +++ b/pkg/media/ingest_daemon.go @@ -94,7 +94,9 @@ func (mm *MediaManager) ConsumeWorkerSocket(ctx context.Context, socketPath, str return fmt.Errorf("ingest worker socket gone before End: %w", err) } connectedOnce = true - sawEnd, _ := mm.consumeWorkerFrames(ctx, conn, streamer, onSegment, nil) + // Fresh Reader per connection: a reconnect is a new stream where the worker + // replays its buffer from the start. + sawEnd, _ := mm.consumeWorkerFrames(ctx, ingestframe.NewReader(conn), streamer, onSegment, nil) conn.Close() if sawEnd { return nil @@ -192,10 +194,11 @@ func dialWorkerSocket(ctx context.Context, socketPath string) (net.Conn, error) } } -// readWHIPAnswer reads frames on conn until the worker's Answer frame and returns -// its SDP. An Error/End/EOF before the answer is a setup failure. -func readWHIPAnswer(conn net.Conn) (string, error) { - fr := ingestframe.NewReader(conn) +// readWHIPAnswer reads frames until the worker's Answer frame and returns its +// SDP. An Error/End/EOF before the answer is a setup failure. It reads through +// the caller's Reader so the same decoder (and its buffered read-ahead) carries +// on to the segment stream. +func readWHIPAnswer(fr *ingestframe.Reader) (string, error) { for { typ, payload, err := fr.ReadFrame() if err != nil { @@ -248,10 +251,14 @@ func (mm *MediaManager) WHIPIngestDetached(ctx context.Context, offerSDP string, _ = proc.Kill() return "", fmt.Errorf("connect to whip worker: %w", err) } + // One Reader owns this connection for its whole lifetime: the streaming CBOR + // decoder reads ahead, so the Answer and the segments that follow must come + // through the SAME Reader (a second one would lose buffered read-ahead). + fr := ingestframe.NewReader(conn) if dl, ok := answerCtx.Deadline(); ok { _ = conn.SetReadDeadline(dl) } - answer, err := readWHIPAnswer(conn) + answer, err := readWHIPAnswer(fr) if err != nil { conn.Close() _ = proc.Kill() @@ -262,7 +269,7 @@ func (mm *MediaManager) WHIPIngestDetached(ctx context.Context, offerSDP string, // Consume the signed segments in the background; the HTTP handler returns the // answer now and the WebRTC media establishes directly to the worker. go func() { - sawEnd, _ := mm.consumeWorkerFrames(ctx, conn, ms.Streamer(), mm.validateSegment(ctx), nil) + sawEnd, _ := mm.consumeWorkerFrames(ctx, fr, ms.Streamer(), mm.validateSegment(ctx), nil) conn.Close() if !sawEnd && ctx.Err() == nil { // Connection dropped but the detached worker lives on — reconnect and diff --git a/pkg/media/ingest_supervisor.go b/pkg/media/ingest_supervisor.go index cdc7ae70..8ba69db1 100644 --- a/pkg/media/ingest_supervisor.go +++ b/pkg/media/ingest_supervisor.go @@ -144,7 +144,7 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, defer watchdog.Stop() // Read signed-segment frames and feed each into the normal chokepoint. - sawEnd, readErr := mm.consumeWorkerFrames(ctx, framesR, ms.Streamer(), mm.validateSegment(ctx), func() { + sawEnd, readErr := mm.consumeWorkerFrames(ctx, ingestframe.NewReader(framesR), ms.Streamer(), mm.validateSegment(ctx), func() { watchdog.Reset(ingestWorkerWatchdog) }) logsWG.Wait() @@ -167,8 +167,7 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, // over each. It returns whether a clean End frame was seen and the terminal read // error: nil on a clean close (End then EOF), or io.ErrUnexpectedEOF / a desync // error when the worker died mid-frame. -func (mm *MediaManager) consumeWorkerFrames(ctx context.Context, r io.Reader, streamer string, onSegment func([]byte) error, onProgress func()) (sawEnd bool, _ error) { - fr := ingestframe.NewReader(r) +func (mm *MediaManager) consumeWorkerFrames(ctx context.Context, fr *ingestframe.Reader, streamer string, onSegment func([]byte) error, onProgress func()) (sawEnd bool, _ error) { for { typ, payload, err := fr.ReadFrame() if err != nil { diff --git a/pkg/media/whip_worker_test.go b/pkg/media/whip_worker_test.go index b0ac69fb..717ebe96 100644 --- a/pkg/media/whip_worker_test.go +++ b/pkg/media/whip_worker_test.go @@ -86,7 +86,7 @@ func TestWHIPWorkerAnswersOffer(t *testing.T) { require.NoError(t, derr) defer conn.Close() - answerSDP, rerr := readWHIPAnswer(conn) + answerSDP, rerr := readWHIPAnswer(ingestframe.NewReader(conn)) require.NoError(t, rerr, "worker emits an SDP answer as its first frame") require.Contains(t, answerSDP, "v=0", "valid SDP answer") @@ -194,7 +194,11 @@ func TestWHIPWorkerLoopback(t *testing.T) { require.NoError(t, derr) defer conn.Close() - answerSDP, rerr := readWHIPAnswer(conn) + // One Reader for the whole connection: the streaming decoder reads ahead, so + // the Answer and the segments after it must come through the same Reader (the + // production WHIP path does the same). + fr := ingestframe.NewReader(conn) + answerSDP, rerr := readWHIPAnswer(fr) require.NoError(t, rerr) require.NoError(t, clientPC.SetRemoteDescription(webrtc.SessionDescription{Type: webrtc.SDPTypeAnswer, SDP: answerSDP})) @@ -208,10 +212,9 @@ func TestWHIPWorkerLoopback(t *testing.T) { // Read signed segments; require at least one valid dual-codec one. _ = conn.SetReadDeadline(time.Now().Add(45 * time.Second)) - r := ingestframe.NewReader(conn) var segs int for segs == 0 { - typ, payload, ferr := r.ReadFrame() + typ, payload, ferr := fr.ReadFrame() require.NoError(t, ferr, "reading worker frames") if typ != ingestframe.Segment { continue -- 2.51.2 From f2c72925145d98aa5c2af9453c7bea36aac60145 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Sat, 6 Jun 2026 16:15:10 -0700 Subject: [PATCH 15/17] media: keep debug recording on the isolated ingest paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Debug recording (the DebugRecording per-stream setting → a debug-recordings// dump) only worked on the in-process paths and the fd-4-pipe fallback, where main is in the data path and can tee. The zero-downtime detached MKV path and the WHIP worker put main OUT of the data path, so recording silently stopped there. Move the recording into the worker, uniformly across the isolated paths. main still owns the DECISION (shouldRecord needs the DB), evaluated once in buildWorkerConfig and passed as cfg.Record + cfg.DataDir in the handshake; the worker, which owns the data path, carries it out: - RunMKVIngestWorker tees its ingest media to dumpToFile when cfg.Record. - ServeWHIPIngestWorkerSocket passes cfg.Record to NewRecordingPeerConnection. - The now-redundant main-side tee is removed from MKVIngestIsolated, so there's one recording path (no double-record) and main stays out of the data path on the fd-4 path too. The in-process MKVIngest / NewPeerConnection recordings are unchanged. Bonus: because the worker owns the recording, a recording now survives a main restart along with the rest of the detached session. cfg.DataDir is only set when recording (rtcrec/dumpToFile panic on an empty DataDir, but they're only reached when enabled). Test: TestRunMKVIngestWorkerRecords runs the worker with Record + a temp DataDir and asserts debug-recordings//*.rtmp.mkv is written verbatim from the ingested media. Co-Authored-By: Claude Opus 4.8 --- pkg/media/ingest_supervisor.go | 23 ++++++++--------- pkg/media/ingest_worker.go | 34 +++++++++++++++++++++--- pkg/media/ingest_worker_test.go | 46 +++++++++++++++++++++++++++++++++ pkg/media/whip_worker.go | 8 +++--- 4 files changed, 91 insertions(+), 20 deletions(-) diff --git a/pkg/media/ingest_supervisor.go b/pkg/media/ingest_supervisor.go index 8ba69db1..47fa4bc4 100644 --- a/pkg/media/ingest_supervisor.go +++ b/pkg/media/ingest_supervisor.go @@ -47,18 +47,8 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, return fmt.Errorf("marshal worker config: %w", err) } - // Optional recording stays in main: tee the raw input before it reaches the - // worker, so the worker needs no data-dir access. - if shouldRecord, rerr := mm.shouldRecord(ctx, ms.Streamer()); rerr == nil && shouldRecord { - log.Log(ctx, "recording RTMP stream to file", "streamer", ms.Streamer()) - pr, pw := io.Pipe() - input = io.TeeReader(input, pw) - go func() { - if derr := mm.dumpToFile(ctx, pr, ms.Streamer(), ".rtmp.mkv"); derr != nil { - log.Error(ctx, "error dumping to file", "error", derr) - } - }() - } + // Debug recording now happens inside the worker (cfg.Record), uniformly across + // the isolated paths — main stays out of the data path here too. exe, err := os.Executable() if err != nil { @@ -243,6 +233,15 @@ func (mm *MediaManager) buildWorkerConfig(ctx context.Context, ms MediaSigner) ( Manifest: manifest, BroadcasterHost: mm.cli.BroadcasterHost, } + // Debug recording: main owns the per-stream setting (it needs the DB); the + // worker carries out the recording (it owns the data path). A lookup failure + // is non-fatal — just don't record. + if rec, rerr := mm.shouldRecord(ctx, ms.Streamer()); rerr != nil { + log.Warn(ctx, "could not read debug-recording setting; not recording", "streamer", ms.Streamer(), "error", rerr) + } else if rec { + cfg.Record = true + cfg.DataDir = mm.cli.DataDir + } // Node transcode signer lets the worker complete to dual-codec itself. If it's // unavailable, the worker emits single-codec (the node doesn't re-transcode the // worker's output) — an acceptable, logged fallback rather than a hard failure. diff --git a/pkg/media/ingest_worker.go b/pkg/media/ingest_worker.go index 5941b691..cc95cf73 100644 --- a/pkg/media/ingest_worker.go +++ b/pkg/media/ingest_worker.go @@ -62,6 +62,16 @@ type IngestWorkerConfig struct { Prebuf []byte `json:"prebuf,omitempty"` Chunked bool `json:"chunked,omitempty"` + // Record, when true, makes the worker write a debug recording of this session + // (the MKV/RTMP push body, or the WHIP session) under + // DataDir/debug-recordings//. main evaluates the per-stream DebugRecording + // setting (which needs the DB) and the worker carries it out — so debug + // recording keeps working on the isolated paths without main being in the data + // path, and a recording even survives a main restart. DataDir is set (only when + // Record) to the node data dir the worker writes recordings under. + Record bool `json:"record,omitempty"` + DataDir string `json:"data_dir,omitempty"` + // Transport selects the worker's ingest source: "" / "mkv" reads MKV media // (stdin or InputFD); "whip" makes the worker own the WebRTC PeerConnection, // built from OfferSDP — no media fd to pass. @@ -158,16 +168,32 @@ func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Re ctx, cancel := context.WithCancel(ctx) defer cancel() - // Minimal manager: just the broadcaster identity the transcode completion - // (finishTranscodedSegment) stamps into the node-signed AAC track. - mm := &MediaManager{cli: &config.CLI{BroadcasterHost: cfg.BroadcasterHost}} + // Minimal manager: the broadcaster identity the transcode completion + // (finishTranscodedSegment) stamps into the node-signed AAC track, plus the + // data dir for an optional debug recording. + mm := &MediaManager{cli: &config.CLI{BroadcasterHost: cfg.BroadcasterHost, DataDir: cfg.DataDir}} onSegment, flush := mm.workerSegmentSink(ctx, cfg, frames) + // Debug recording: tee the ingest media to a file before it reaches gst. main + // decided this (cfg.Record) and handed us DataDir; recording here keeps main + // out of the data path and lets the recording survive a main restart. + media := stdin + if cfg.Record { + log.Log(ctx, "recording ingest media to file", "streamer", cfg.StreamerDID) + pr, pw := io.Pipe() + media = io.TeeReader(stdin, pw) + go func() { + if derr := mm.dumpToFile(ctx, pr, cfg.StreamerDID, ".rtmp.mkv"); derr != nil { + log.Error(ctx, "ingest worker: dump recording to file", "error", derr) + } + }() + } + signerElem, done, err := muxlSignSegmentElem(ctx, mm.cli, workerSignStream(cfg), onSegment) if err != nil { return fmt.Errorf("build signer element: %w", err) } - pipeline, err := buildMKVIngestPipeline(ctx, stdin, signerElem) + pipeline, err := buildMKVIngestPipeline(ctx, media, signerElem) if err != nil { return fmt.Errorf("build pipeline: %w", err) } diff --git a/pkg/media/ingest_worker_test.go b/pkg/media/ingest_worker_test.go index c1d73299..ee8ce64f 100644 --- a/pkg/media/ingest_worker_test.go +++ b/pkg/media/ingest_worker_test.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "io" + "os" + "path/filepath" "strings" "testing" "time" @@ -141,3 +143,47 @@ func TestRunMKVIngestWorkerProducesValidSignedFrames(t *testing.T) { require.GreaterOrEqual(t, segs, 1, "worker emitted at least one signed dual-codec segment") t.Logf("worker emitted %d valid dual-codec segments", segs) } + +// TestRunMKVIngestWorkerRecords proves debug recording works INSIDE the worker: +// with cfg.Record set and a DataDir handed over, the worker tees its ingest +// media to debug-recordings//.rtmp.mkv. This is what keeps debug +// recording working on the isolated paths where main is out of the data path +// (it can't tee the bytes itself), so main decides and the worker records. +func TestRunMKVIngestWorkerRecords(t *testing.T) { + ctx := context.Background() + ms := newBareSegmentSigner(t) + + keyPEM, err := signers.MarshalES256KPrivateKeyPEM(ms.Signer) + require.NoError(t, err) + manifest, err := ms.buildManifest(ctx, time.Now().UnixMilli()) + require.NoError(t, err) + + dataDir := t.TempDir() + cfg := IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: ms.Cert, + Manifest: manifest, + BroadcasterHost: "test.example.com", + Record: true, + DataDir: dataDir, + } + + mkv := makeH264AACMKV(t, ctx, getFixture("5sec.mp4")) + + // Frames are irrelevant here (recording is on the input side); discard them. + require.NoError(t, RunMKVIngestWorker(ctx, cfg, bytes.NewReader(mkv), ingestframe.NewWriter(io.Discard))) + + // The recording lands at debug-recordings//.rtmp.mkv and + // must contain exactly the media the worker ingested. The dump goroutine + // flushes asynchronously, so allow it a moment to finish the last write. + glob := filepath.Join(dataDir, "debug-recordings", "*", "*.rtmp.mkv") + require.Eventually(t, func() bool { + matches, _ := filepath.Glob(glob) + if len(matches) != 1 { + return false + } + got, rerr := os.ReadFile(matches[0]) + return rerr == nil && bytes.Equal(got, mkv) + }, 10*time.Second, 25*time.Millisecond, "worker records the ingest media verbatim") +} diff --git a/pkg/media/whip_worker.go b/pkg/media/whip_worker.go index ffb88592..ff53f0af 100644 --- a/pkg/media/whip_worker.go +++ b/pkg/media/whip_worker.go @@ -58,11 +58,11 @@ func ServeWHIPIngestWorkerSocket(ctx context.Context, cfg IngestWorkerConfig) er return runErr } - mm := &MediaManager{cli: &config.CLI{BroadcasterHost: cfg.BroadcasterHost}} + mm := &MediaManager{cli: &config.CLI{BroadcasterHost: cfg.BroadcasterHost, DataDir: cfg.DataDir}} // The worker owns the PeerConnection (its own UDP sockets), built with the - // same codec/interceptor setup as the in-process server. No recording here — - // the worker has no model-backed settings. + // same codec/interceptor setup as the in-process server. Debug recording is + // decided by main (cfg.Record) and written by the worker under cfg.DataDir. api, webrtcConfig, err := newWebRTCAPI() if err != nil { return finish(fmt.Errorf("webrtc api: %w", err)) @@ -71,7 +71,7 @@ func ServeWHIPIngestWorkerSocket(ctx context.Context, cfg IngestWorkerConfig) er if err != nil { return finish(fmt.Errorf("peer connection: %w", err)) } - pc, err := rtcrec.NewRecordingPeerConnection(ctx, *mm.cli, cfg.StreamerDID, pionpc, false) + pc, err := rtcrec.NewRecordingPeerConnection(ctx, *mm.cli, cfg.StreamerDID, pionpc, cfg.Record) if err != nil { return finish(fmt.Errorf("peer connection wrapper: %w", err)) } -- 2.51.2 From 159d5ea3cab3a02c6a0ec0f5bfffcd474f33935c Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Sat, 6 Jun 2026 16:38:16 -0700 Subject: [PATCH 16/17] media: worker-side self-watchdog for the detached/WHIP ingest paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wedge containment had a hole: the supervisor watchdog (MKVIngestIsolated) only covers the fd-4 fallback. On the default detached path and WHIP, the worker is detached (not tied to main's context), so main can't kill a stuck one — and a wedged native pipeline (e.g. the 4-audio MKV that leaves matroskademux pads unlinked, never emits EOS, produces no frames) would run forever as an orphan. That's the exact failure isolation is supposed to contain. Add a worker-side watchdog: a FrameWriter wrapper kicks a timer on every frame the worker emits; going ingestWorkerWatchdog with no output means the pipeline is wedged, so it cancels the worker's context. muxlSignSegmentElem's ctx-done hook closes the signer pipe, so the worker actually returns and the process exits — the fault stays contained to the subprocess. Wired into both RunMKVIngestWorker and ServeWHIPIngestWorkerSocket (which kicks once after the SDP answer, so the first-segment clock starts post-negotiation; a pre-answer hang is already bounded by main's whipAnswerTimeout). The kick is mutex-guarded since a worker emits frames from more than one goroutine. This is additive on the fd-4 path (the supervisor watchdog still fires first there); it's the ONLY wedge containment on the detached/WHIP paths. Test: TestRunMKVIngestWorkerSelfWatchdog feeds the wedging MKV straight to RunMKVIngestWorker (no supervisor) and asserts it self-terminates rather than hanging. Co-Authored-By: Claude Opus 4.8 --- pkg/media/ingest_worker.go | 7 ++++ pkg/media/ingest_worker_test.go | 45 +++++++++++++++++++++ pkg/media/whip_worker.go | 10 ++++- pkg/media/worker_watchdog.go | 69 +++++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 pkg/media/worker_watchdog.go diff --git a/pkg/media/ingest_worker.go b/pkg/media/ingest_worker.go index cc95cf73..09d47263 100644 --- a/pkg/media/ingest_worker.go +++ b/pkg/media/ingest_worker.go @@ -168,6 +168,13 @@ func RunMKVIngestWorker(ctx context.Context, cfg IngestWorkerConfig, stdin io.Re ctx, cancel := context.WithCancel(ctx) defer cancel() + // Self-watchdog: if the pipeline wedges and stops emitting frames, tear it + // down so the worker exits and the fault stays contained (the only wedge + // containment on the detached path — main can't kill a detached worker). + wd := newWorkerWatchdog(ctx, ingestWorkerWatchdog, cancel, cfg.StreamerDID) + defer wd.stop() + frames = wd.wrap(frames) + // Minimal manager: the broadcaster identity the transcode completion // (finishTranscodedSegment) stamps into the node-signed AAC track, plus the // data dir for an optional debug recording. diff --git a/pkg/media/ingest_worker_test.go b/pkg/media/ingest_worker_test.go index ee8ce64f..618d170a 100644 --- a/pkg/media/ingest_worker_test.go +++ b/pkg/media/ingest_worker_test.go @@ -187,3 +187,48 @@ func TestRunMKVIngestWorkerRecords(t *testing.T) { return rerr == nil && bytes.Equal(got, mkv) }, 10*time.Second, 25*time.Millisecond, "worker records the ingest media verbatim") } + +// TestRunMKVIngestWorkerSelfWatchdog proves the worker's OWN watchdog contains a +// wedge. This is the only wedge containment on the detached/WHIP paths, where +// main can't kill a detached worker — so the worker has to notice it's stuck and +// exit itself. The 4-audio sample-stream.mkv leaves matroskademux pads unlinked, +// so it wedges with no EOS and emits no frames; the watchdog must tear the +// pipeline down and return rather than hang forever. (The fd-4 path's +// supervisor-side watchdog is covered separately by +// TestMKVIngestIsolatedWedgeContained.) +func TestRunMKVIngestWorkerSelfWatchdog(t *testing.T) { + old := ingestWorkerWatchdog + ingestWorkerWatchdog = 3 * time.Second + defer func() { ingestWorkerWatchdog = old }() + + ctx := context.Background() + ms := newBareSegmentSigner(t) + keyPEM, err := signers.MarshalES256KPrivateKeyPEM(ms.Signer) + require.NoError(t, err) + manifest, err := ms.buildManifest(ctx, time.Now().UnixMilli()) + require.NoError(t, err) + cfg := IngestWorkerConfig{ + StreamerDID: ms.Streamer(), + KeyPEM: keyPEM, + CertPEM: ms.Cert, + Manifest: manifest, + BroadcasterHost: "test.example.com", + } + + wedge, err := os.ReadFile(getFixture("sample-stream.mkv")) + require.NoError(t, err) + + start := time.Now() + done := make(chan error, 1) + go func() { + done <- RunMKVIngestWorker(ctx, cfg, bytes.NewReader(wedge), ingestframe.NewWriter(io.Discard)) + }() + select { + case <-done: + elapsed := time.Since(start) + require.Less(t, elapsed, 25*time.Second, "watchdog bounded the wedge") + t.Logf("worker self-terminated on wedge in %s", elapsed.Round(time.Second)) + case <-time.After(30 * time.Second): + t.Fatal("worker-side watchdog did not contain the wedge (RunMKVIngestWorker hung)") + } +} diff --git a/pkg/media/whip_worker.go b/pkg/media/whip_worker.go index ff53f0af..09bb221e 100644 --- a/pkg/media/whip_worker.go +++ b/pkg/media/whip_worker.go @@ -76,7 +76,14 @@ func ServeWHIPIngestWorkerSocket(ctx context.Context, cfg IngestWorkerConfig) er return finish(fmt.Errorf("peer connection wrapper: %w", err)) } - onSegment, flush := mm.workerSegmentSink(ctx, cfg, srv) + // Self-watchdog: a connected-but-silent publisher (or a wedged pipeline) that + // stops producing frames tears the worker down so the fault stays contained. + // The clock to the first segment starts at the answer (kick below), after ICE + // negotiation; a pre-answer hang is bounded by main's whipAnswerTimeout. + wd := newWorkerWatchdog(ctx, ingestWorkerWatchdog, cancel, cfg.StreamerDID) + defer wd.stop() + + onSegment, flush := mm.workerSegmentSink(ctx, cfg, wd.wrap(srv)) signerElem, signerDone, err := muxlSignSegmentElem(ctx, mm.cli, workerSignStream(cfg), onSegment) if err != nil { return finish(fmt.Errorf("build signer element: %w", err)) @@ -93,6 +100,7 @@ func ServeWHIPIngestWorkerSocket(ctx context.Context, cfg IngestWorkerConfig) er if aerr := srv.Answer(answer.SDP); aerr != nil { log.Error(ctx, "whip worker: frame answer", "error", aerr) } + wd.kick() // negotiation done + answer sent; start the first-segment clock here // Streaming runs until the peer disconnects / errors; webRTCIngestPipeline // cancels ctx then, which drains the signer. Wait for that, flush the diff --git a/pkg/media/worker_watchdog.go b/pkg/media/worker_watchdog.go new file mode 100644 index 00000000..1abf748e --- /dev/null +++ b/pkg/media/worker_watchdog.go @@ -0,0 +1,69 @@ +package media + +import ( + "context" + "sync" + "time" + + "stream.place/streamplace/pkg/log" +) + +// workerWatchdog self-terminates a wedged ingest worker. A healthy stream emits +// an output frame every GoP (~1–2s); going ingestWorkerWatchdog with no frame +// means the native pipeline is wedged (gst can't drain — e.g. a pathological +// stream that leaves demux pads unlinked) and won't recover. The watchdog then +// fires onWedge — the worker's context cancel — tearing the pipeline down so the +// process exits and the fault stays contained to this subprocess. +// +// This is the worker-side counterpart to MKVIngestIsolated's main-side watchdog, +// and the ONLY wedge containment on the detached and WHIP paths: those workers +// are detached (not tied to main's context), so main can't kill a stuck one — +// the worker has to notice and exit itself. +type workerWatchdog struct { + mu sync.Mutex + timer *time.Timer + to time.Duration +} + +func newWorkerWatchdog(ctx context.Context, timeout time.Duration, onWedge func(), streamer string) *workerWatchdog { + return &workerWatchdog{ + to: timeout, + timer: time.AfterFunc(timeout, func() { + log.Warn(ctx, "ingest worker watchdog fired (no output frames); self-terminating", + "streamer", streamer, "timeout", timeout) + onWedge() + }), + } +} + +// kick resets the no-output timer. Safe for concurrent use — a worker emits +// frames from more than one goroutine (the signer event loop and the +// transcoder's completion callback). +func (w *workerWatchdog) kick() { + w.mu.Lock() + defer w.mu.Unlock() + w.timer.Reset(w.to) +} + +// stop halts the watchdog; call it once the pipeline has ended so it can't fire +// during the post-stream drain. +func (w *workerWatchdog) stop() { + w.mu.Lock() + defer w.mu.Unlock() + w.timer.Stop() +} + +// wrap returns a FrameWriter that kicks the watchdog on every frame it writes, +// so any output the worker produces counts as liveness. +func (w *workerWatchdog) wrap(inner FrameWriter) FrameWriter { + return watchdogFrameWriter{FrameWriter: inner, wd: w} +} + +type watchdogFrameWriter struct { + FrameWriter + wd *workerWatchdog +} + +func (w watchdogFrameWriter) Segment(seg []byte) error { w.wd.kick(); return w.FrameWriter.Segment(seg) } +func (w watchdogFrameWriter) End() error { w.wd.kick(); return w.FrameWriter.End() } +func (w watchdogFrameWriter) Error(msg string) error { w.wd.kick(); return w.FrameWriter.Error(msg) } -- 2.51.2 From 90b0e5848867d1f8ddbc64f348e9e36855031ff3 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Sat, 6 Jun 2026 16:38:50 -0700 Subject: [PATCH 17/17] media: reap crashed detached workers, reap stale sockets, add crash metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three loose ends on the detached path, all in the direction of "a contained fault should leave nothing behind and not go unnoticed": - Zombie on crash: MKVIngestDetached only reaped (proc.Wait) on a clean End, so a worker that crashed left a zombie until main exited. Reap whenever ctx.Err()==nil (clean end OR crash); still skip on main shutdown, where the worker is deliberately left detached for a restarting main to rediscover. - Stale-socket spin + accumulation: ConsumeWorkerSocket retried a dead socket forever while it had never connected — so a SIGKILLed worker's leftover .sock would spin a reconnect goroutine at 250ms indefinitely, and accumulate one per restart via ResumeDetachedWorkers. Bound the initial connect with workerConnectGrace (15s); on give-up, and on a socket that vanishes after we'd connected, os.Remove the leftover (no-op on a clean exit, which the worker already unlinked). - Observability: the whole point of isolation is surviving faults, but a crash-looping stream was invisible (logs only). Add main-side metrics — streamplace_ingest_worker_starts_total{transport} and _exits_total{transport,outcome=clean|crash} — instrumented across the fd-4, detached, WHIP, and resumed paths via a shared recordWorkerExit (a ctx-cancel main shutdown isn't counted as a fault). Crash rate is now alertable. Metrics are main-side only; the short-lived worker has no scrape endpoint. Co-Authored-By: Claude Opus 4.8 --- pkg/media/ingest_daemon.go | 42 ++++++++++++++++++++++++++++------ pkg/media/ingest_supervisor.go | 28 +++++++++++++++++++++++ pkg/spmetrics/spmetrics.go | 19 +++++++++++++++ 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/pkg/media/ingest_daemon.go b/pkg/media/ingest_daemon.go index bf5a0edf..804400b4 100644 --- a/pkg/media/ingest_daemon.go +++ b/pkg/media/ingest_daemon.go @@ -15,12 +15,19 @@ import ( "github.com/google/uuid" "stream.place/streamplace/pkg/ingestframe" "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/spmetrics" ) // ingestReconnectBackoff paces redials when a worker is up but main's connection // dropped (a restart in progress). const ingestReconnectBackoff = 250 * time.Millisecond +// workerConnectGrace bounds the INITIAL connect to a worker's socket. A freshly +// spawned worker takes a moment to listen; a socket that never accepts within +// this window is a dead/never-started worker — or a stale socket a SIGKILLed +// worker left behind — so we give up and unlink it rather than spin forever. +const workerConnectGrace = 15 * time.Second + // SpawnIngestWorkerDetached launches an ingest worker in its OWN session // (Setsid) so it outlives a main restart, fd-passing the (already authed) ingest // connection as the worker's media input (fd 4) and having it serve signed @@ -80,10 +87,18 @@ func SpawnIngestWorkerDetached(cfg IngestWorkerConfig, media *os.File) (*os.Proc // error if the socket vanishes without one (worker crashed — contained). func (mm *MediaManager) ConsumeWorkerSocket(ctx context.Context, socketPath, streamer string, onSegment func([]byte) error) error { connectedOnce := false + giveUp := time.Now().Add(workerConnectGrace) for { conn, err := net.Dial("unix", socketPath) if err != nil { if !connectedOnce { + if time.Now().After(giveUp) { + // Never came up within the grace: a dead/never-started worker, or a + // stale socket a SIGKILLed worker left behind. Unlink it so it can't + // linger or re-spin a consumer on the next restart. + _ = os.Remove(socketPath) + return fmt.Errorf("ingest worker never came up at %s: %w", socketPath, err) + } select { // worker may still be coming up case <-ctx.Done(): return ctx.Err() @@ -91,6 +106,10 @@ func (mm *MediaManager) ConsumeWorkerSocket(ctx context.Context, socketPath, str continue } } + // Connected before, now the socket's gone: the worker exited. A clean + // exit already unlinked its socket; a SIGKILL leaves it behind, so remove + // the leftover (no-op if already gone). + _ = os.Remove(socketPath) return fmt.Errorf("ingest worker socket gone before End: %w", err) } connectedOnce = true @@ -164,13 +183,17 @@ func (mm *MediaManager) MKVIngestDetached(ctx context.Context, conn net.Conn, pr if err != nil { return fmt.Errorf("spawn detached worker: %w", err) } + spmetrics.IngestWorkerStarts.WithLabelValues("mkv").Inc() err = mm.ConsumeWorkerSocket(ctx, cfg.SocketPath, ms.Streamer(), mm.validateSegment(ctx)) - if err == nil { - go func() { _, _ = proc.Wait() }() // clean end: reap the exiting worker + recordWorkerExit("mkv", err, ctx.Err()) + // Reap the worker unless we're deliberately leaving it running across a main + // restart (ctx cancel). On a clean end OR a crash the worker has exited, so + // Wait() clears the zombie; only on main shutdown do we let it stay detached + // for a restarting main to rediscover via DiscoverWorkerSockets. + if ctx.Err() == nil { + go func() { _, _ = proc.Wait() }() } - // On ctx cancel (main shutting down) we deliberately leave the detached worker - // running; a restarting main reconnects via discovery. return err } @@ -241,6 +264,7 @@ func (mm *MediaManager) WHIPIngestDetached(ctx context.Context, offerSDP string, if err != nil { return "", fmt.Errorf("spawn detached whip worker: %w", err) } + spmetrics.IngestWorkerStarts.WithLabelValues("whip").Inc() // Connect + read the SDP answer (the worker's first frame), bounded so a // wedged setup can't hang the WHIP client. @@ -271,11 +295,13 @@ func (mm *MediaManager) WHIPIngestDetached(ctx context.Context, offerSDP string, go func() { sawEnd, _ := mm.consumeWorkerFrames(ctx, fr, ms.Streamer(), mm.validateSegment(ctx), nil) conn.Close() + var exitErr error if !sawEnd && ctx.Err() == nil { // Connection dropped but the detached worker lives on — reconnect and - // drain its buffer. - _ = mm.ConsumeWorkerSocket(ctx, cfg.SocketPath, ms.Streamer(), mm.validateSegment(ctx)) + // drain its buffer. Its terminal result is the worker's true outcome. + exitErr = mm.ConsumeWorkerSocket(ctx, cfg.SocketPath, ms.Streamer(), mm.validateSegment(ctx)) } + recordWorkerExit("whip", exitErr, ctx.Err()) go func() { _, _ = proc.Wait() }() }() return answer, nil @@ -299,9 +325,11 @@ func (mm *MediaManager) ResumeDetachedWorkers(ctx context.Context) { sock := sock log.Log(ctx, "resuming detached ingest worker", "socket", sock) go func() { - if cerr := mm.ConsumeWorkerSocket(ctx, sock, "resumed", mm.validateSegment(ctx)); cerr != nil { + cerr := mm.ConsumeWorkerSocket(ctx, sock, "resumed", mm.validateSegment(ctx)) + if cerr != nil { log.Error(ctx, "resumed ingest worker ended", "socket", sock, "error", cerr) } + recordWorkerExit("resumed", cerr, ctx.Err()) }() } } diff --git a/pkg/media/ingest_supervisor.go b/pkg/media/ingest_supervisor.go index 47fa4bc4..42454874 100644 --- a/pkg/media/ingest_supervisor.go +++ b/pkg/media/ingest_supervisor.go @@ -17,6 +17,7 @@ import ( "stream.place/streamplace/pkg/crypto/signers" "stream.place/streamplace/pkg/ingestframe" "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/spmetrics" ) // ingestWorkerWatchdog bounds how long an isolated worker may go without @@ -103,6 +104,7 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, } cfgR.Close() // the child holds its own copy now framesW.Close() // ditto; the parent only reads framesR + spmetrics.IngestWorkerStarts.WithLabelValues("mkv-fd").Inc() go func() { _, _ = cfgW.Write(cfgJSON) @@ -140,6 +142,17 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, logsWG.Wait() werr := cmd.Wait() + // Classify the exit for the lifecycle metrics: a crash is a read error or a + // nonzero exit without a clean End; a clean End (even with a nonzero exit) is + // clean. ctx cancel (main shutdown SIGKILLing the worker) isn't counted. + var exitErr error + if readErr != nil { + exitErr = readErr + } else if werr != nil && !sawEnd { + exitErr = werr + } + recordWorkerExit("mkv-fd", exitErr, ctx.Err()) + switch { case readErr != nil: return fmt.Errorf("ingest worker stream: %w", readErr) @@ -153,6 +166,21 @@ func (mm *MediaManager) MKVIngestIsolated(ctx context.Context, input io.Reader, return nil } +// recordWorkerExit folds a worker's terminal state into the lifecycle metrics: +// a clean end vs a crash (any non-nil error). A ctx cancel (main shutting down, +// the worker forcibly stopped or deliberately left detached) is not a fault we +// count. +func recordWorkerExit(transport string, exitErr, ctxErr error) { + switch { + case ctxErr != nil: + // main shutdown — not a stream-level fault + case exitErr == nil: + spmetrics.IngestWorkerExits.WithLabelValues(transport, "clean").Inc() + default: + spmetrics.IngestWorkerExits.WithLabelValues(transport, "crash").Inc() + } +} + // consumeWorkerFrames reads framed segments from the worker and runs ValidateMP4 // over each. It returns whether a clean End frame was seen and the terminal read // error: nil on a clean close (End then EOF), or io.ErrUnexpectedEOF / a desync diff --git a/pkg/spmetrics/spmetrics.go b/pkg/spmetrics/spmetrics.go index 1113bc27..9c25f351 100644 --- a/pkg/spmetrics/spmetrics.go +++ b/pkg/spmetrics/spmetrics.go @@ -121,6 +121,25 @@ var FirehoseEventsDedupedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Help: "firehose events dropped as cross-relay duplicates, by kind", }, []string{"kind"}) +// --- isolated ingest workers ------------------------------------------------ + +// IngestWorkerStarts counts isolated ingest worker subprocesses spawned, by +// transport ("mkv-fd" = fd-4 fallback, "mkv" = detached, "whip" = detached WHIP). +var IngestWorkerStarts = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "streamplace_ingest_worker_starts_total", + Help: "isolated ingest worker subprocesses spawned, by transport", +}, []string{"transport"}) + +// IngestWorkerExits counts isolated ingest worker exits by transport and outcome +// ("clean" | "crash"). A rising crash rate is the signal that a stream is +// repeatedly faulting — the contained fault the node now survives but which would +// otherwise be invisible. A worker left running across a main shutdown is not an +// exit and is not counted; "resumed" is a worker reattached after a restart. +var IngestWorkerExits = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "streamplace_ingest_worker_exits_total", + Help: "isolated ingest worker exits, by transport and outcome (clean|crash)", +}, []string{"transport", "outcome"}) + // --- VOD processing --------------------------------------------------------- // VODProcessAttemptsTotal increments once per task dequeued for VOD -- 2.51.2