From fe8a08cd5a86614389f1abe4ce3757a4764f51ae Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sat, 4 Apr 2026 13:50:41 -0500 Subject: [PATCH] add broadcast queue push-lock spin counter separates push-lock contention from queue-full contention so operator can distinguish: producers fighting for the CAS lock vs producers blocked on a full ring buffer. new metric: relay_broadcast_queue_push_lock_spins_total clarified: relay_broadcast_queue_full_total HELP text (ring capacity) Co-Authored-By: Claude Opus 4.6 --- src/broadcaster.zig | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/broadcaster.zig b/src/broadcaster.zig index c294cf9..2d8898e 100644 --- a/src/broadcaster.zig +++ b/src/broadcaster.zig @@ -59,6 +59,7 @@ pub const Stats = struct { pool_queued_bytes: std.atomic.Value(u64) = .{ .raw = 0 }, // persist/broadcast pipeline contention persist_order_spins: std.atomic.Value(u64) = .{ .raw = 0 }, + broadcast_queue_push_lock_spins: std.atomic.Value(u64) = .{ .raw = 0 }, broadcast_queue_full: std.atomic.Value(u64) = .{ .raw = 0 }, broadcast_queue_depth_hwm: std.atomic.Value(u32) = .{ .raw = 0 }, broadcast_no_consumers: std.atomic.Value(u64) = .{ .raw = 0 }, @@ -275,9 +276,14 @@ pub const BroadcastQueue = struct { var full_spins: u32 = 0; while (true) { // acquire spinlock + var lock_spins: u32 = 0; while (self.push_lock.cmpxchgWeak(0, 1, .acquire, .monotonic) != null) { + lock_spins += 1; std.atomic.spinLoopHint(); } + if (lock_spins > 0) { + _ = stats.broadcast_queue_push_lock_spins.fetchAdd(lock_spins, .monotonic); + } const tail = self.tail.load(.monotonic); const next_tail = (tail + 1) % CAPACITY; @@ -970,8 +976,12 @@ pub fn formatPrometheusMetrics(stats: *const Stats, cache_entries: usize, attrib \\# HELP relay_persist_order_spins_total spin iterations waiting for persist ordering lock \\relay_persist_order_spins_total {d} \\ + \\# TYPE relay_broadcast_queue_push_lock_spins_total counter + \\# HELP relay_broadcast_queue_push_lock_spins_total spin iterations waiting for broadcast queue push lock + \\relay_broadcast_queue_push_lock_spins_total {d} + \\ \\# TYPE relay_broadcast_queue_full_total counter - \\# HELP relay_broadcast_queue_full_total spin iterations on full broadcast queue + \\# HELP relay_broadcast_queue_full_total spin iterations on full broadcast queue (ring capacity) \\relay_broadcast_queue_full_total {d} \\ \\# TYPE relay_broadcast_queue_depth_hwm gauge @@ -984,6 +994,7 @@ pub fn formatPrometheusMetrics(stats: *const Stats, cache_entries: usize, attrib \\ , .{ stats.persist_order_spins.load(.acquire), + stats.broadcast_queue_push_lock_spins.load(.acquire), stats.broadcast_queue_full.load(.acquire), stats.broadcast_queue_depth_hwm.load(.acquire), stats.broadcast_no_consumers.load(.acquire), -- 2.51.2