From 4efa14d54fa8ecea475f2c8829fecdf4b59ce6c4 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Fri, 9 Oct 2026 17:20:04 -0700 Subject: [PATCH] obs-plugin: harden connect wait, framerate math and the send-queue limit Wait for a connect with poll on POSIX, since OBS often has descriptors past FD_SETSIZE. Ignore server framerates outside 32 bits, which could overflow the divisor to zero. Count samples held in the muxer toward the 64 MiB limit, so a track that never starts can't grow memory unbounded. --- obs-plugin/src/fmp4.c | 3 +++ obs-plugin/src/fmp4.h | 2 ++ obs-plugin/src/http.c | 17 +++++++++++------ obs-plugin/src/streamplace-output.c | 13 +++++++------ 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/obs-plugin/src/fmp4.c b/obs-plugin/src/fmp4.c index c7cd20763..aa4b91fc3 100644 --- a/obs-plugin/src/fmp4.c +++ b/obs-plugin/src/fmp4.c @@ -72,6 +72,7 @@ void fmp4_free(struct fmp4_mux *mux) da_free(mux->tracks); mux->sequence = 0; mux->init_written = false; + mux->held_bytes = 0; } size_t fmp4_add_track(struct fmp4_mux *mux, const struct fmp4_track *info) @@ -390,6 +391,7 @@ void fmp4_push(struct fmp4_mux *mux, size_t track_idx, struct encoder_packet *pa complete_pending(track, delta > 0 ? (uint32_t)(delta * track->timebase_num) : 0); } track->pending = *packet; + mux->held_bytes += packet->size; track->has_pending = true; } @@ -470,6 +472,7 @@ static void write_fragment(struct fmp4_mux *mux, struct serializer *s, bool forc struct encoder_packet *packet = &track->samples.array[j].packet; s_write(s, packet->data, packet->size); data_offset += (uint32_t)packet->size; + mux->held_bytes -= packet->size; obs_encoder_packet_release(packet); } da_resize(track->samples, 0); diff --git a/obs-plugin/src/fmp4.h b/obs-plugin/src/fmp4.h index 3d08973bf..5d0445571 100644 --- a/obs-plugin/src/fmp4.h +++ b/obs-plugin/src/fmp4.h @@ -50,6 +50,8 @@ struct fmp4_mux { DARRAY(struct fmp4_track) tracks; uint32_t sequence; bool init_written; + /* Sample bytes held in the muxer, not yet written to a fragment. */ + size_t held_bytes; }; void fmp4_free(struct fmp4_mux *mux); diff --git a/obs-plugin/src/http.c b/obs-plugin/src/http.c index 63cafd94e..3eac102ba 100644 --- a/obs-plugin/src/http.c +++ b/obs-plugin/src/http.c @@ -17,7 +17,7 @@ typedef SOCKET socket_t; #include #include #include -#include +#include #include #include typedef int socket_t; @@ -118,18 +118,23 @@ static bool connect_with_timeout(socket_t s, const struct sockaddr *addr, sockle #ifdef _WIN32 if (WSAGetLastError() != WSAEWOULDBLOCK) return false; -#else - if (errno != EINPROGRESS) - return false; -#endif + /* Winsock's fd_set is a list of sockets, so any socket value fits. */ fd_set wfds, efds; FD_ZERO(&wfds); FD_ZERO(&efds); FD_SET(s, &wfds); FD_SET(s, &efds); struct timeval tv = {CONNECT_TIMEOUT_SEC, 0}; - if (select((int)s + 1, NULL, &wfds, &efds, &tv) <= 0) + if (select(0, NULL, &wfds, &efds, &tv) <= 0) return false; +#else + if (errno != EINPROGRESS) + return false; + /* poll, not select: descriptors past FD_SETSIZE are common in OBS. */ + struct pollfd pfd = {.fd = s, .events = POLLOUT}; + if (poll(&pfd, 1, CONNECT_TIMEOUT_SEC * 1000) <= 0) + return false; +#endif int so_error = 0; socklen_t so_len = sizeof(so_error); diff --git a/obs-plugin/src/streamplace-output.c b/obs-plugin/src/streamplace-output.c index 8f14d008c..c1c242370 100644 --- a/obs-plugin/src/streamplace-output.c +++ b/obs-plugin/src/streamplace-output.c @@ -263,11 +263,12 @@ static obs_encoder_t *create_video_encoder(obs_data_t *config, size_t idx, const obs_encoder_set_preferred_video_format( encoder, lookup_enum(video_formats, obs_data_get_string(config, "format"), VIDEO_FORMAT_NV12)); + /* Bounded to 32 bits so the products below cannot overflow to zero. */ obs_data_t *framerate = obs_data_get_obj(config, "framerate"); - uint64_t num = (uint64_t)obs_data_get_int(framerate, "numerator"); - uint64_t den = (uint64_t)obs_data_get_int(framerate, "denominator"); - if (num && den) { - uint64_t divisor = ((uint64_t)ovi->fps_num * den) / (num * ovi->fps_den); + long long num = obs_data_get_int(framerate, "numerator"); + long long den = obs_data_get_int(framerate, "denominator"); + if (num > 0 && den > 0 && num <= UINT32_MAX && den <= UINT32_MAX) { + uint64_t divisor = ((uint64_t)ovi->fps_num * (uint64_t)den) / ((uint64_t)num * ovi->fps_den); if (divisor > 1) obs_encoder_set_frame_rate_divisor(encoder, (uint32_t)divisor); } @@ -509,8 +510,8 @@ static void sp_encoded_packet(void *data, struct encoder_packet *packet) if (track == 0) fmp4_write_fragment(&sp->mux, &sp->queue_s); - if (sp->queue.bytes.num > MAX_QUEUED_BYTES) { - do_log(LOG_ERROR, "The server is not keeping up; over %d MiB are queued", + if (sp->queue.bytes.num + sp->mux.held_bytes > MAX_QUEUED_BYTES) { + do_log(LOG_ERROR, "Over %d MiB of stream data are waiting to be sent", MAX_QUEUED_BYTES / (1024 * 1024)); finish_with_error(sp, OBS_OUTPUT_DISCONNECTED); } -- 2.51.2