diff --git a/build.zig.zon b/build.zig.zon index 4541c50..2aa8967 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -13,8 +13,8 @@ .hash = "websocket-0.1.10-ZPISdUowBQAXIl_X9VpZozIxBLVf_o5wYjOMMdobrXF5", }, .zio = .{ - .url = "https://tangled.org/zzstoatzz.io/zio/archive/6790455.tar.gz", - .hash = "zio-0.0.0-psl_OZh_AgBCz_4i-lIoPKs_wybKDL-tereRD7bh6hjo", + .url = "https://tangled.org/zzstoatzz.io/zio/archive/1321be0.tar.gz", + .hash = "zio-0.0.0-psl_ORGCAgCYCLRsoLjOIdaYZQI0FSlB81xG20_tPWGn", }, .pg = .{ .url = "https://tangled.org/zzstoatzz.io/pg.zig/archive/c5a5607.tar.gz", diff --git a/src/main.zig b/src/main.zig index e41d25f..32d9f83 100644 --- a/src/main.zig +++ b/src/main.zig @@ -146,7 +146,61 @@ const MetricsServer = struct { }; var metrics_buf: [65536]u8 = undefined; - const body = broadcaster.formatPrometheusMetrics(self.stats, cache_entries, attribution, self.data_dir, &metrics_buf, self.io); + const base_body = broadcaster.formatPrometheusMetrics(self.stats, cache_entries, attribution, self.data_dir, &metrics_buf, self.io); + + // zio runtime internals. a wedged loop, fibers piling up on a + // lost wake, or an eroded poller table are otherwise invisible + // from outside the process — the 2026-08-04 canary's failure had + // to be inferred from what stopped answering. the scrape is the + // one forensic channel that survives a pod dying, so put them + // there. this handler runs as a fiber on the loop thread, which + // is where debugStats() must be called from. + var zio_buf: [2048]u8 = undefined; + const body = if (comptime backend_config.use_zio) blk: { + const d = backend.debugStats(); + const extra = std.fmt.bufPrint(&zio_buf, + \\# HELP zio_sched_switches_total fiber context switches (flat = loop wedged) + \\# TYPE zio_sched_switches_total counter + \\zio_sched_switches_total {d} + \\# HELP zio_sched_fibers live fibers by state + \\# TYPE zio_sched_fibers gauge + \\zio_sched_fibers{{state="live"}} {d} + \\zio_sched_fibers{{state="ready"}} {d} + \\zio_sched_fibers{{state="parked"}} {d} + \\# HELP zio_sched_futex_waiters fibers parked on futex words (climbing = lost wakes) + \\# TYPE zio_sched_futex_waiters gauge + \\zio_sched_futex_waiters {d} + \\# HELP zio_sched_timers timer heap depth + \\# TYPE zio_sched_timers gauge + \\zio_sched_timers {d} + \\# HELP zio_sched_inject_pending cross-thread messages awaiting the loop + \\# TYPE zio_sched_inject_pending gauge + \\zio_sched_inject_pending {d} + \\# HELP zio_sched_free_fibers recycled fiber structs + \\# TYPE zio_sched_free_fibers gauge + \\zio_sched_free_fibers {d} + \\# HELP zio_poller_registrations fds with an armed direction (collapse = erosion) + \\# TYPE zio_poller_registrations gauge + \\zio_poller_registrations {d} + \\# HELP zio_dns_spill DNS spill pool depth and workers + \\# TYPE zio_dns_spill gauge + \\zio_dns_spill{{kind="queued"}} {d} + \\zio_dns_spill{{kind="workers"}} {d} + \\zio_dns_spill{{kind="idle"}} {d} + \\ + , .{ + d.sched.switches, d.sched.live, d.sched.ready, + d.sched.parked, d.sched.futex_waiters, d.sched.timers, + d.sched.inject_pending, d.sched.free_list, d.poller_registrations, + d.dns_queued, d.dns_workers, d.dns_idle, + }) catch ""; + if (base_body.len + extra.len <= metrics_buf.len) { + @memcpy(metrics_buf[base_body.len..][0..extra.len], extra); + break :blk metrics_buf[0 .. base_body.len + extra.len]; + } + break :blk base_body; + } else base_body; + request.respond(body, .{ .status = .ok, .keep_alive = false, .extra_headers = &.{ .{ .name = "content-type", .value = "text/plain; version=0.0.4; charset=utf-8" }, .{ .name = "server", .value = "zlay (atproto-relay)" },