From 66187087634727e2cfb24f33ed270cb949985eed Mon Sep 17 00:00:00 2001
From: zzstoatzz <thrast36@gmail.com>
Date: Mon, 02 Mar 2026 18:55:31 +0000
Subject: [PATCH] feat: add disk usage prometheus metrics (relay_disk_total_bytes, relay_disk_available_bytes)

disk pressure killed the pod with no warning in metrics. uses statvfs on
the data directory to export filesystem total and available bytes.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 src/broadcaster.zig | 31 +++++++++++++++++++++++++++++--
 src/main.zig | 8 +++++---
 2 file(s) changed, 34 insertion(s)(+), 5 deletion(s)(-)

diff --git a/src/broadcaster.zig b/src/broadcaster.zig
--- a/src/broadcaster.zig
+++ b/src/broadcaster.zig
@@ -543,7 +543,7 @@
     }
 };
 
-pub fn formatPrometheusMetrics(stats: *const Stats, cache_entries: usize, migration_queue_len: usize, buf: []u8) []const u8 {
+pub fn formatPrometheusMetrics(stats: *const Stats, cache_entries: usize, migration_queue_len: usize, data_dir: []const u8, buf: []u8) []const u8 {
     const uptime: i64 = std.time.timestamp() - stats.start_time;
     var fbs = std.io.fixedBufferStream(buf);
     const w = fbs.writer();
@@ -621,6 +621,7 @@
     // linux-only process metrics from /proc
     if (comptime builtin.os.tag == .linux) {
         appendProcMetrics(w);
+        appendDiskMetrics(w, data_dir);
     }
 
     return fbs.getWritten();
@@ -669,6 +670,32 @@
             }
         }
     } else |_| {}
+}
+
+const posix_vfs = @cImport(@cInclude("sys/statvfs.h"));
+
+fn appendDiskMetrics(w: anytype, data_dir: []const u8) void {
+    // statvfs needs a null-terminated path
+    var path_buf: [4096]u8 = undefined;
+    if (data_dir.len >= path_buf.len) return;
+    @memcpy(path_buf[0..data_dir.len], data_dir);
+    path_buf[data_dir.len] = 0;
+
+    var stat: posix_vfs.struct_statvfs = undefined;
+    if (posix_vfs.statvfs(@ptrCast(&path_buf), &stat) != 0) return;
+
+    const block_size: u64 = stat.f_frsize;
+    const total = stat.f_blocks * block_size;
+    const available = stat.f_bavail * block_size;
+
+    std.fmt.format(w,
+        \\# TYPE relay_disk_total_bytes gauge
+        \\relay_disk_total_bytes {d}
+        \\
+        \\# TYPE relay_disk_available_bytes gauge
+        \\relay_disk_available_bytes {d}
+        \\
+    , .{ total, available }) catch {};
 }
 
 pub fn formatStatsResponse(stats: *const Stats, buf: []u8) []const u8 {
@@ -781,7 +808,7 @@
     stats.cache_misses.store(100, .release);
 
     var buf: [8192]u8 = undefined;
-    const output = formatPrometheusMetrics(&stats, 42, 3, &buf);
+    const output = formatPrometheusMetrics(&stats, 42, 3, "/tmp", &buf);
 
     try std.testing.expect(std.mem.indexOf(u8, output, "relay_frames_received_total 10000") != null);
     try std.testing.expect(std.mem.indexOf(u8, output, "relay_frames_broadcast_total 9000") != null);
diff --git a/src/main.zig b/src/main.zig
--- a/src/main.zig
+++ b/src/main.zig
@@ -47,6 +47,7 @@
     server: std.net.Server,
     stats: *broadcaster.Stats,
     validator: *validator_mod.Validator,
+    data_dir: []const u8,
 
     fn run(self: *MetricsServer) void {
         while (!shutdown_flag.load(.acquire)) {
@@ -55,12 +56,12 @@
                 log.debug("metrics accept error: {s}", .{@errorName(err)});
                 continue;
             };
-            handleMetricsConn(conn.stream, self.stats, self.validator);
+            handleMetricsConn(conn.stream, self.stats, self.validator, self.data_dir);
         }
     }
 };
 
-fn handleMetricsConn(stream: std.net.Stream, stats: *broadcaster.Stats, validator: *validator_mod.Validator) void {
+fn handleMetricsConn(stream: std.net.Stream, stats: *broadcaster.Stats, validator: *validator_mod.Validator, data_dir: []const u8) void {
     defer stream.close();
 
     var recv_buf: [4096]u8 = undefined;
@@ -75,7 +76,7 @@
     const migration_queue_len = validator.migrationQueueLen();
 
     var metrics_buf: [8192]u8 = undefined;
-    const body = broadcaster.formatPrometheusMetrics(stats, cache_entries, migration_queue_len, &metrics_buf);
+    const body = broadcaster.formatPrometheusMetrics(stats, cache_entries, migration_queue_len, data_dir, &metrics_buf);
     request.respond(body, .{ .status = .ok, .keep_alive = false, .extra_headers = &.{
         .{ .name = "content-type", .value = "text/plain; version=0.0.4; charset=utf-8" },
         .{ .name = "server", .value = "zlay (atproto-relay)" },
@@ -180,6 +181,7 @@
         },
         .stats = &bc.stats,
         .validator = &val,
+        .data_dir = data_dir,
     };
     const metrics_thread = try std.Thread.spawn(.{ .stack_size = default_stack_size }, MetricsServer.run, .{&metrics_srv});
 

--
tangled.sh