Something went wrong. Try again.
semantic bufo search find-bufo.com
bufo
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973const std = @import("std");const zat = @import("zat");const mem = std.mem;const json = std.json;const Allocator = mem.Allocator;const Thread = std.Thread;const Io = std.Io;const net = Io.net;const http = std.http;const template = @import("stats_template.zig");const posts_index = @import("posts_index.zig");/// the traffic charts live in a real js file (served at /traffic.js) rather/// than inside the format-string template, where every brace must be doubledconst traffic_js = @embedFile("traffic.js");const posts_js = @embedFile("posts.js");
const STATS_PATH = "/data/stats.json";const STATS_TMP_PATH = "/data/stats.json.tmp";const MONTH_KEY_LEN = 7;
// module state — initialized via init(), not from a globalvar io: Io = undefined;
pub fn init(app_io: Io) void { io = app_io;}
fn timestamp() i64 { return @intCast(@divFloor(Io.Timestamp.now(io, .real).nanoseconds, std.time.ns_per_s));}
fn civilFromDays(days_since_epoch: i64) struct { year: i32, month: u32 } { const z = days_since_epoch + 719468; const era = @divFloor(if (z >= 0) z else z - 146096, 146097); const doe = z - era * 146097; const yoe = @divFloor(doe - @divFloor(doe, 1460) + @divFloor(doe, 36524) - @divFloor(doe, 146096), 365); const y = yoe + era * 400; const doy = doe - (365 * yoe + @divFloor(yoe, 4) - @divFloor(yoe, 100)); const mp = @divFloor(5 * doy + 2, 153); const month: i64 = mp + if (mp < 10) @as(i64, 3) else @as(i64, -9); const year: i64 = y + if (month <= 2) @as(i64, 1) else @as(i64, 0); return .{ .year = @intCast(year), .month = @intCast(month) };}
// year 9999 upper bound keeps civilFromDays' @intCast and the 4-digit// format below safe even if the clock source returns garbageconst MAX_SANE_TS: i64 = 253402300799;
fn monthKey(ts: i64, buf: *[MONTH_KEY_LEN]u8) []const u8 { const clamped = std.math.clamp(ts, 0, MAX_SANE_TS); const days = @divFloor(clamped, 86400); const date = civilFromDays(days); // year must be unsigned here: zig 0.16 zero-pads signed ints with a // leading '+', which overflows the 7-byte buffer const year: u32 = @intCast(date.year); return std.fmt.bufPrint(buf, "{d:0>4}-{d:0>2}", .{ year, date.month }) catch "unknown";}
test "monthKey handles sane and insane timestamps without panicking" { var buf: [MONTH_KEY_LEN]u8 = undefined; try std.testing.expectEqualStrings("2026-07", monthKey(1783468800, &buf)); try std.testing.expectEqualStrings("1970-01", monthKey(0, &buf)); // negative (pre-epoch) clamps to epoch try std.testing.expectEqualStrings("1970-01", monthKey(-123456789, &buf)); // nanosecond-scale value (the bug that crashed the bot) clamps to year 9999 try std.testing.expectEqualStrings("9999-12", monthKey(1783468800 * std.time.ns_per_s, &buf)); try std.testing.expectEqualStrings("9999-12", monthKey(std.math.maxInt(i64), &buf));}
pub const TrackedPost = struct { our_rkey: []const u8, original_uri: []const u8, original_did: []const u8, timestamp: i64,};
pub const Stats = struct { allocator: Allocator, start_time: i64, posts_checked: std.atomic.Value(u64) = .init(0), matches_found: std.atomic.Value(u64) = .init(0), posts_created: std.atomic.Value(u64) = .init(0), // opt-in gate counters: a match either belongs to a follower (eligible to // post) or a non-follower (skipped). only follower matches reach the // posting path. followers_matched: std.atomic.Value(u64) = .init(0), non_followers_skipped: std.atomic.Value(u64) = .init(0), errors: std.atomic.Value(u64) = .init(0), bufos_loaded: u64 = 0, jetstream_host_buf: [256]u8 = undefined, jetstream_host_len: std.atomic.Value(usize) = .init(0),
last_snapshot_hour: i64 = -1, // hour-of-day of last snapshot (-1 = none yet)
// track per-bufo match counts: name -> {count, url} bufo_matches: std.StringHashMap(BufoMatchData), monthly_bufo_matches: std.StringHashMap(BufoMatchData), bufo_mutex: Io.Mutex = Io.Mutex.init, // track our quote-posts for cleanup on delete/block tracked_posts: std.ArrayList(TrackedPost),
const BufoMatchData = struct { count: u64, url: []const u8, };
pub fn initStats(allocator: Allocator) Stats { var self = Stats{ .allocator = allocator, .start_time = timestamp(), .bufo_matches = std.StringHashMap(BufoMatchData).init(allocator), .monthly_bufo_matches = std.StringHashMap(BufoMatchData).init(allocator), .tracked_posts = .empty, }; self.load(); return self; }
pub fn deinit(self: *Stats) void { self.save(); var iter = self.bufo_matches.iterator(); while (iter.next()) |entry| { self.allocator.free(entry.key_ptr.*); self.allocator.free(entry.value_ptr.url); } self.bufo_matches.deinit(); var monthly_iter = self.monthly_bufo_matches.iterator(); while (monthly_iter.next()) |entry| { self.allocator.free(entry.key_ptr.*); self.allocator.free(entry.value_ptr.url); } self.monthly_bufo_matches.deinit(); for (self.tracked_posts.items) |tp| { self.allocator.free(tp.our_rkey); self.allocator.free(tp.original_uri); self.allocator.free(tp.original_did); } self.tracked_posts.deinit(self.allocator); }
fn load(self: *Stats) void { const file = Io.Dir.openFileAbsolute(io, STATS_PATH, .{}) catch return; defer file.close(io);
var buf: [256 * 1024]u8 = undefined; const len = file.readPositionalAll(io, &buf, 0) catch return; if (len == 0) return;
const parsed = json.parseFromSlice(json.Value, self.allocator, buf[0..len], .{}) catch return; defer parsed.deinit();
const root = parsed.value;
if (zat.json.getInt(root, "posts_checked")) |v| { self.posts_checked.store(@intCast(@max(0, v)), .monotonic); } if (zat.json.getInt(root, "matches_found")) |v| { self.matches_found.store(@intCast(@max(0, v)), .monotonic); } if (zat.json.getInt(root, "posts_created")) |v| { self.posts_created.store(@intCast(@max(0, v)), .monotonic); } if (zat.json.getInt(root, "followers_matched")) |v| { self.followers_matched.store(@intCast(@max(0, v)), .monotonic); } if (zat.json.getInt(root, "non_followers_skipped")) |v| { self.non_followers_skipped.store(@intCast(@max(0, v)), .monotonic); } if (zat.json.getInt(root, "errors")) |v| { self.errors.store(@intCast(@max(0, v)), .monotonic); } // load bufo_matches (or legacy bufo_posts) const matches_key = if (zat.json.getPath(root, "bufo_matches") != null) "bufo_matches" else "bufo_posts"; if (zat.json.getPath(root, matches_key)) |bp| { if (bp == .object) { var iter = bp.object.iterator(); while (iter.next()) |entry| { if (entry.value_ptr.* == .object) { // format: {"count": N, "url": "..."} const count_val = zat.json.getInt(entry.value_ptr.*, "count") orelse continue; const url_val = zat.json.getString(entry.value_ptr.*, "url") orelse continue;
const key = self.allocator.dupe(u8, entry.key_ptr.*) catch continue; const url = self.allocator.dupe(u8, url_val) catch { self.allocator.free(key); continue; }; self.bufo_matches.put(key, .{ .count = @intCast(@max(0, count_val)), .url = url, }) catch { self.allocator.free(key); self.allocator.free(url); }; } else if (entry.value_ptr.* == .integer) { // legacy format: just integer count - construct URL from name const key = self.allocator.dupe(u8, entry.key_ptr.*) catch continue; var url_buf: [256]u8 = undefined; const constructed_url = std.fmt.bufPrint(&url_buf, "https://all-the.bufo.zone/{s}", .{entry.key_ptr.*}) catch continue; const url = self.allocator.dupe(u8, constructed_url) catch { self.allocator.free(key); continue; }; self.bufo_matches.put(key, .{ .count = @intCast(@max(0, entry.value_ptr.integer)), .url = url, }) catch { self.allocator.free(key); self.allocator.free(url); }; } } } }
if (zat.json.getPath(root, "monthly_bufo_matches")) |mbp| { if (mbp == .object) { var iter = mbp.object.iterator(); while (iter.next()) |entry| { if (entry.value_ptr.* != .object) continue; const count_val = zat.json.getInt(entry.value_ptr.*, "count") orelse continue; const url_val = zat.json.getString(entry.value_ptr.*, "url") orelse continue;
const key = self.allocator.dupe(u8, entry.key_ptr.*) catch continue; const url = self.allocator.dupe(u8, url_val) catch { self.allocator.free(key); continue; }; self.monthly_bufo_matches.put(key, .{ .count = @intCast(@max(0, count_val)), .url = url, }) catch { self.allocator.free(key); self.allocator.free(url); }; } } }
// load tracked posts if (zat.json.getArray(root, "tracked_posts")) |tp| { for (tp) |item| { const rkey_val = zat.json.getString(item, "our_rkey") orelse continue; const uri_val = zat.json.getString(item, "original_uri") orelse continue; const did_val = zat.json.getString(item, "original_did") orelse continue; const ts_val = zat.json.getInt(item, "timestamp") orelse continue;
const rkey = self.allocator.dupe(u8, rkey_val) catch continue; const uri = self.allocator.dupe(u8, uri_val) catch { self.allocator.free(rkey); continue; }; const did = self.allocator.dupe(u8, did_val) catch { self.allocator.free(rkey); self.allocator.free(uri); continue; }; self.tracked_posts.append(self.allocator, .{ .our_rkey = rkey, .original_uri = uri, .original_did = did, .timestamp = ts_val, }) catch { self.allocator.free(rkey); self.allocator.free(uri); self.allocator.free(did); }; } }
std.debug.print("loaded stats from {s} ({} tracked posts)\n", .{ STATS_PATH, self.tracked_posts.items.len }); }
/// json string escaping for hand-rolled writes: quotes, backslashes, and /// control chars (monthly keys contain a literal tab — writing it raw /// produced invalid json that load() silently rejected, resetting stats /// on every restart). fn writeJsonString(w: *Io.Writer, s: []const u8) !void { try w.print("\"", .{}); for (s) |c| switch (c) { '"' => try w.print("\\\"", .{}), '\\' => try w.print("\\\\", .{}), 0x00...0x1f => try w.print("\\u{x:0>4}", .{c}), else => try w.print("{c}", .{c}), }; try w.print("\"", .{}); }
pub fn save(self: *Stats) void { self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io); self.saveUnlocked(); }
/// this process's uptime. every other counter survives a deploy via /// /data/stats.json; uptime is the one number that should not pub fn totalUptime(self: *Stats) i64 { return timestamp() - self.start_time; }
pub fn incPostsChecked(self: *Stats) void { _ = self.posts_checked.fetchAdd(1, .monotonic); }
pub fn incMatchesFound(self: *Stats) void { _ = self.matches_found.fetchAdd(1, .monotonic); }
pub fn incBufoMatch(self: *Stats, bufo_name: []const u8, bufo_url: []const u8) void { self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io);
if (self.bufo_matches.getPtr(bufo_name)) |data| { data.count += 1; } else { const key = self.allocator.dupe(u8, bufo_name) catch return; const url = self.allocator.dupe(u8, bufo_url) catch { self.allocator.free(key); return; }; self.bufo_matches.put(key, .{ .count = 1, .url = url }) catch { self.allocator.free(key); self.allocator.free(url); }; }
var month_buf: [MONTH_KEY_LEN]u8 = undefined; const month = monthKey(timestamp(), &month_buf); const monthly_key = std.fmt.allocPrint(self.allocator, "{s}\t{s}", .{ month, bufo_name }) catch return; if (self.monthly_bufo_matches.getPtr(monthly_key)) |data| { data.count += 1; self.allocator.free(monthly_key); } else { const url = self.allocator.dupe(u8, bufo_url) catch { self.allocator.free(monthly_key); return; }; self.monthly_bufo_matches.put(monthly_key, .{ .count = 1, .url = url }) catch { self.allocator.free(monthly_key); self.allocator.free(url); }; } self.saveUnlocked(); }
pub fn incPostsCreated(self: *Stats) void { _ = self.posts_created.fetchAdd(1, .monotonic); }
fn saveUnlocked(self: *Stats) void { // called when mutex is already held const file = Io.Dir.createFileAbsolute(io, STATS_TMP_PATH, .{}) catch return;
const now = timestamp(); var buf: [1024 * 1024]u8 = undefined; var w: Io.Writer = .fixed(&buf);
w.print("{{", .{}) catch return; w.print("\"posts_checked\":{},", .{self.posts_checked.load(.monotonic)}) catch return; w.print("\"matches_found\":{},", .{self.matches_found.load(.monotonic)}) catch return; w.print("\"posts_created\":{},", .{self.posts_created.load(.monotonic)}) catch return; w.print("\"followers_matched\":{},", .{self.followers_matched.load(.monotonic)}) catch return; w.print("\"non_followers_skipped\":{},", .{self.non_followers_skipped.load(.monotonic)}) catch return; w.print("\"errors\":{},", .{self.errors.load(.monotonic)}) catch return; w.print("\"bufo_matches\":{{", .{}) catch return;
var first = true; var iter = self.bufo_matches.iterator(); while (iter.next()) |entry| { if (!first) w.print(",", .{}) catch return; first = false; writeJsonString(&w, entry.key_ptr.*) catch return; w.print(":{{\"count\":{},\"url\":", .{entry.value_ptr.count}) catch return; writeJsonString(&w, entry.value_ptr.url) catch return; w.print("}}", .{}) catch return; }
w.print("}},", .{}) catch return; w.print("\"monthly_bufo_matches\":{{", .{}) catch return;
first = true; var monthly_iter = self.monthly_bufo_matches.iterator(); while (monthly_iter.next()) |entry| { if (!first) w.print(",", .{}) catch return; first = false; writeJsonString(&w, entry.key_ptr.*) catch return; w.print(":{{\"count\":{},\"url\":", .{entry.value_ptr.count}) catch return; writeJsonString(&w, entry.value_ptr.url) catch return; w.print("}}", .{}) catch return; }
w.print("}},", .{}) catch return;
// write tracked posts w.print("\"tracked_posts\":[", .{}) catch return; for (self.tracked_posts.items, 0..) |tp, i| { if (i > 0) w.print(",", .{}) catch return; w.print("{{\"our_rkey\":", .{}) catch return; writeJsonString(&w, tp.our_rkey) catch return; w.print(",\"original_uri\":", .{}) catch return; writeJsonString(&w, tp.original_uri) catch return; w.print(",\"original_did\":", .{}) catch return; writeJsonString(&w, tp.original_did) catch return; w.print(",\"timestamp\":{}}}", .{tp.timestamp}) catch return; } w.print("]}}", .{}) catch return;
const written = buf[0..w.end]; file.writeStreamingAll(io, written) catch return; file.close(io); Io.Dir.renameAbsolute(STATS_TMP_PATH, STATS_PATH, io) catch return;
// hourly snapshot: copy to /data/stats.snapshot.HH (24 rolling files) const hour = @mod(@divTrunc(now, 3600), 24); if (hour != self.last_snapshot_hour) { self.last_snapshot_hour = hour; var snap_path: [48]u8 = undefined; const snap = std.fmt.bufPrint(&snap_path, "/data/stats.snapshot.{d:0>2}", .{@as(u64, @intCast(hour))}) catch return; const snap_file = Io.Dir.createFileAbsolute(io, snap, .{}) catch return; snap_file.writeStreamingAll(io, written) catch {}; snap_file.close(io); } }
pub fn addTrackedPost(self: *Stats, our_rkey: []const u8, original_uri: []const u8, original_did: []const u8, ts: i64) void { self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io);
const rkey = self.allocator.dupe(u8, our_rkey) catch return; const uri = self.allocator.dupe(u8, original_uri) catch { self.allocator.free(rkey); return; }; const did = self.allocator.dupe(u8, original_did) catch { self.allocator.free(rkey); self.allocator.free(uri); return; }; self.tracked_posts.append(self.allocator, .{ .our_rkey = rkey, .original_uri = uri, .original_did = did, .timestamp = ts, }) catch { self.allocator.free(rkey); self.allocator.free(uri); self.allocator.free(did); return; }; self.saveUnlocked(); }
/// atomically find+remove a tracked post by original URI, returning the caller-owned rkey pub fn removeByOriginalUri(self: *Stats, uri: []const u8) ?[]const u8 { self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io);
for (self.tracked_posts.items, 0..) |tp, i| { if (mem.eql(u8, tp.original_uri, uri)) { const removed = self.tracked_posts.orderedRemove(i); self.allocator.free(removed.original_uri); self.allocator.free(removed.original_did); self.saveUnlocked(); return removed.our_rkey; // caller owns this } } return null; }
/// find+remove a tracked post by our rkey, returning true if found pub fn removeByOurRkey(self: *Stats, rkey: []const u8) bool { self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io);
for (self.tracked_posts.items, 0..) |tp, i| { if (mem.eql(u8, tp.our_rkey, rkey)) { const removed = self.tracked_posts.orderedRemove(i); self.allocator.free(removed.our_rkey); self.allocator.free(removed.original_uri); self.allocator.free(removed.original_did); self.saveUnlocked(); return true; } } return false; }
/// check if a given rkey is already tracked pub fn isTracked(self: *Stats, rkey: []const u8) bool { self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io);
for (self.tracked_posts.items) |tp| { if (mem.eql(u8, tp.our_rkey, rkey)) return true; } return false; }
pub fn hasOriginalUri(self: *Stats, uri: []const u8) bool { self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io);
for (self.tracked_posts.items) |tp| { if (mem.eql(u8, tp.original_uri, uri)) return true; } return false; }
/// collect rkeys of tracked posts matching a given original DID, then remove them pub fn removeByOriginalDid(self: *Stats, did: []const u8, buf: [][]const u8) [][]const u8 { self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io);
var count: usize = 0; var i: usize = 0; while (i < self.tracked_posts.items.len and count < buf.len) { if (mem.eql(u8, self.tracked_posts.items[i].original_did, did)) { const tp = self.tracked_posts.orderedRemove(i); buf[count] = tp.our_rkey; self.allocator.free(tp.original_uri); self.allocator.free(tp.original_did); count += 1; } else { i += 1; } } if (count > 0) self.saveUnlocked(); return buf[0..count]; }
pub fn pruneOldPosts(self: *Stats, max_age_secs: i64) void { self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io);
const now = timestamp(); var i: usize = 0; var pruned: usize = 0; while (i < self.tracked_posts.items.len) { if (now - self.tracked_posts.items[i].timestamp > max_age_secs) { const tp = self.tracked_posts.orderedRemove(i); self.allocator.free(tp.our_rkey); self.allocator.free(tp.original_uri); self.allocator.free(tp.original_did); pruned += 1; } else { i += 1; } } if (pruned > 0) { std.debug.print("pruned {} old tracked posts\n", .{pruned}); self.saveUnlocked(); } }
pub fn incFollowerMatched(self: *Stats) void { _ = self.followers_matched.fetchAdd(1, .monotonic); }
pub fn incNonFollowerSkipped(self: *Stats) void { _ = self.non_followers_skipped.fetchAdd(1, .monotonic); }
pub fn incErrors(self: *Stats) void { _ = self.errors.fetchAdd(1, .monotonic); }
pub fn setBufosLoaded(self: *Stats, count: u64) void { self.bufos_loaded = count; }
pub fn setJetstreamHost(self: *Stats, host: []const u8) void { const len = @min(host.len, self.jetstream_host_buf.len); @memcpy(self.jetstream_host_buf[0..len], host[0..len]); self.jetstream_host_len.store(len, .release); }
pub fn getJetstreamHost(self: *Stats) []const u8 { const len = self.jetstream_host_len.load(.acquire); if (len == 0) return "(connecting...)"; return self.jetstream_host_buf[0..len]; }
fn formatUptime(seconds: i64, buf: []u8) []const u8 { const s: u64 = @intCast(@max(0, seconds)); const days = s / 86400; const hours = (s % 86400) / 3600; const mins = (s % 3600) / 60; const secs = s % 60;
if (days > 0) { return std.fmt.bufPrint(buf, "{}d {}h {}m", .{ days, hours, mins }) catch "?"; } else if (hours > 0) { return std.fmt.bufPrint(buf, "{}h {}m {}s", .{ hours, mins, secs }) catch "?"; } else if (mins > 0) { return std.fmt.bufPrint(buf, "{}m {}s", .{ mins, secs }) catch "?"; } else { return std.fmt.bufPrint(buf, "{}s", .{secs}) catch "?"; } }
pub fn renderHtml(self: *Stats, allocator: Allocator) ![]const u8 { const uptime = self.totalUptime();
var uptime_buf: [64]u8 = undefined; const uptime_str = formatUptime(uptime, &uptime_buf);
const BufoEntry = struct { name: []const u8, count: u64, url: []const u8,
fn compare(_: void, a: @This(), b: @This()) bool { return a.count > b.count; } };
// collect top bufos var top_bufos: std.ArrayList(BufoEntry) = .empty; defer top_bufos.deinit(allocator);
{ self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io);
var iter = self.bufo_matches.iterator(); while (iter.next()) |entry| { try top_bufos.append(allocator, .{ .name = entry.key_ptr.*, .count = entry.value_ptr.count, .url = entry.value_ptr.url }); } }
// sort by count descending mem.sort(BufoEntry, top_bufos.items, {}, BufoEntry.compare);
// build top bufos grid html var top_html: std.ArrayList(u8) = .empty; defer top_html.deinit(allocator);
// find max count for scaling var max_count: u64 = 1; for (top_bufos.items) |entry| { if (entry.count > max_count) max_count = entry.count; }
for (top_bufos.items) |entry| { // scale size: min 60px, max 160px based on count ratio const ratio = @as(f64, @floatFromInt(entry.count)) / @as(f64, @floatFromInt(max_count)); const size: u32 = @intFromFloat(60.0 + ratio * 100.0);
// strip extension for display name var display_name = entry.name; if (mem.endsWith(u8, entry.name, ".gif")) { display_name = entry.name[0 .. entry.name.len - 4]; } else if (mem.endsWith(u8, entry.name, ".png")) { display_name = entry.name[0 .. entry.name.len - 4]; } else if (mem.endsWith(u8, entry.name, ".jpg")) { display_name = entry.name[0 .. entry.name.len - 4]; }
try top_html.print(allocator, \\<div class="bufo-card" style="width:{}px;height:{}px;" title="{s} ({} matches)" data-name="{s}" role="button" tabindex="0"> \\<img src="{s}" alt="{s}" loading="lazy"> \\<span class="bufo-count">{}</span> \\</div> , .{ size, size, display_name, entry.count, display_name, entry.url, display_name, entry.count }); }
const top_section = if (top_bufos.items.len > 0) top_html.items else "<p class=\"no-bufos\">no posts yet</p>";
const html = try std.fmt.allocPrint(allocator, template.html, .{ self.posts_checked.load(.monotonic), self.posts_checked.load(.monotonic), self.matches_found.load(.monotonic), self.matches_found.load(.monotonic), self.posts_created.load(.monotonic), self.posts_created.load(.monotonic), self.errors.load(.monotonic), self.errors.load(.monotonic), uptime, uptime_str, self.getJetstreamHost(), self.bufos_loaded, self.bufos_loaded, self.followers_matched.load(.monotonic), self.followers_matched.load(.monotonic), self.non_followers_skipped.load(.monotonic), self.non_followers_skipped.load(.monotonic), top_section, });
return html; }
pub fn renderPublicJson(self: *Stats, allocator: Allocator) ![]const u8 { const Entry = struct { name: []const u8, url: []const u8, count: u64,
fn lessThan(_: void, a: @This(), b: @This()) bool { if (a.count != b.count) return a.count > b.count; return mem.lessThan(u8, a.name, b.name); } }; self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io); var bufos: std.ArrayList(Entry) = .empty; defer bufos.deinit(allocator); var iter = self.bufo_matches.iterator(); while (iter.next()) |entry| { try bufos.append(allocator, .{ .name = entry.key_ptr.*, .url = entry.value_ptr.url, .count = entry.value_ptr.count }); } mem.sort(Entry, bufos.items, {}, Entry.lessThan); return json.Stringify.valueAlloc(allocator, .{ .posts_checked = self.posts_checked.load(.monotonic), .matches_found = self.matches_found.load(.monotonic), .posts_created = self.posts_created.load(.monotonic), .followers_matched = self.followers_matched.load(.monotonic), .non_followers_skipped = self.non_followers_skipped.load(.monotonic), .errors = self.errors.load(.monotonic), .bufos_loaded = self.bufos_loaded, .uptime_seconds = @max(0, self.totalUptime()), .bufos = bufos.items, }, .{}); }
pub fn renderJson(self: *Stats, allocator: Allocator) ![]const u8 { var output: Io.Writer.Allocating = .init(allocator); defer output.deinit(); var stream: json.Stringify = .{ .writer = &output.writer }; try stream.beginObject(); try stream.objectField("monthly_bufo_matches"); try stream.beginObject(); self.bufo_mutex.lockUncancelable(io); defer self.bufo_mutex.unlock(io); var iter = self.monthly_bufo_matches.iterator(); while (iter.next()) |entry| { try stream.objectField(entry.key_ptr.*); try stream.write(entry.value_ptr.*); } try stream.endObject(); try stream.endObject(); return output.toOwnedSlice(); }};
pub const StatsServer = struct { allocator: Allocator, health: *@import("health.zig").Health, stats: *Stats, index: *posts_index.PostsIndex, port: u16,
pub fn initServer(allocator: Allocator, s: *Stats, index: *posts_index.PostsIndex, health: *@import("health.zig").Health, port: u16) StatsServer { return .{ .allocator = allocator, .stats = s, .health = health, .index = index, .port = port, }; }
pub fn run(self: *StatsServer) void { // spawn periodic save ticker (every 60s) const ticker = Thread.spawn(.{}, saveTicker, .{self.stats}) catch |err| { std.debug.print("failed to start save ticker: {}\n", .{err}); return; }; ticker.detach();
self.serve() catch |err| { std.debug.print("stats server error: {}\n", .{err}); }; }
fn saveTicker(s: *Stats) void { while (true) { io.sleep(.{ .nanoseconds = 60 * std.time.ns_per_s }, .awake) catch {}; s.save(); } }
fn serve(self: *StatsServer) !void { var address = try net.IpAddress.parse("::", self.port); var server = try net.IpAddress.listen(&address, io, .{ .reuse_address = true }); defer server.deinit(io);
std.debug.print("stats server listening on http://[::]:{} \n", .{self.port});
while (true) { const stream = server.accept(io) catch |err| { std.debug.print("accept error: {}\n", .{err}); continue; };
const t = Thread.spawn(.{}, handleConnection, .{ self, stream }) catch |err| { std.debug.print("spawn error: {}\n", .{err}); stream.close(io); continue; }; t.detach(); } }
fn handleConnection(self: *StatsServer, stream: net.Stream) void { defer stream.close(io);
var read_buffer: [4096]u8 = undefined; var write_buffer: [8192]u8 = undefined;
var reader = net.Stream.Reader.init(stream, io, &read_buffer); var writer = net.Stream.Writer.init(stream, io, &write_buffer);
var server = http.Server.init(&reader.interface, &writer.interface);
var request = server.receiveHead() catch return;
const target = request.head.target; if (mem.eql(u8, target, "/health/freshness")) { const now: i64 = @intCast(@divFloor(Io.Timestamp.now(io, .awake).nanoseconds, std.time.ns_per_s)); const report = self.health.report(now); const body = json.Stringify.valueAlloc(self.allocator, report, .{}) catch return; defer self.allocator.free(body); request.respond(body, .{ .status = if (report.ok) .ok else .service_unavailable, .extra_headers = &.{ .{ .name = "content-type", .value = "application/json" }, .{ .name = "cache-control", .value = "no-store" }, }, }) catch return; return; } if (mem.eql(u8, target, "/traffic.js")) { request.respond(traffic_js, .{ .extra_headers = &.{ .{ .name = "content-type", .value = "application/javascript; charset=utf-8" }, .{ .name = "cache-control", .value = "public, max-age=300" }, }, }) catch return; return; } if (mem.eql(u8, target, "/posts.js")) { request.respond(posts_js, .{ .extra_headers = &.{ .{ .name = "content-type", .value = "application/javascript; charset=utf-8" }, .{ .name = "cache-control", .value = "public, max-age=300" }, }, }) catch return; return; } if (mem.startsWith(u8, target, "/posts/")) { self.servePosts(&request, target["/posts/".len..]); return; } if (mem.eql(u8, target, "/monthly.json") or mem.eql(u8, target, "/stats.json")) { const json_body = (if (mem.eql(u8, target, "/stats.json")) self.stats.renderPublicJson(self.allocator) else self.stats.renderJson(self.allocator)) catch |err| { std.debug.print("render json error: {}\n", .{err}); return; }; defer self.allocator.free(json_body);
request.respond(json_body, .{ .extra_headers = &.{ .{ .name = "content-type", .value = "application/json" }, .{ .name = "cache-control", .value = "public, max-age=300" }, }, }) catch return; return; }
const html = self.stats.renderHtml(self.allocator) catch |err| { std.debug.print("render error: {}\n", .{err}); return; }; defer self.allocator.free(html);
request.respond(html, .{ .extra_headers = &.{ .{ .name = "content-type", .value = "text/html; charset=utf-8" }, .{ .name = "cache-control", .value = "public, max-age=60" }, }, }) catch return; }
/// /posts/<bufo>.json and /posts/by-did/<did>.json — the post viewer's /// data, straight from the bot's own index fn servePosts(self: *StatsServer, request: *http.Server.Request, rest: []const u8) void { const by_did = mem.startsWith(u8, rest, "by-did/"); const raw = if (by_did) rest["by-did/".len..] else rest; if (!mem.endsWith(u8, raw, ".json") or raw.len > 512) { request.respond("not found", .{ .status = .not_found }) catch return; return; } var key_buf: [512]u8 = undefined; const encoded = raw[0 .. raw.len - ".json".len]; @memcpy(key_buf[0..encoded.len], encoded); const key = std.Uri.percentDecodeInPlace(key_buf[0..encoded.len]);
const body = (if (by_did) self.index.renderByDid(self.allocator, key) else self.index.renderBufo(self.allocator, key)) catch |err| { std.debug.print("render posts error: {}\n", .{err}); return; }; defer self.allocator.free(body); request.respond(body, .{ .extra_headers = &.{ .{ .name = "content-type", .value = "application/json" }, .{ .name = "cache-control", .value = "public, max-age=60" }, }, }) catch return; }};
test "public stats serialize all sorted bufos and omit internal records" { init(std.testing.io); const allocator = std.testing.allocator; var stats = Stats{ .allocator = allocator, .start_time = timestamp() - 60, .bufo_matches = std.StringHashMap(Stats.BufoMatchData).init(allocator), .monthly_bufo_matches = std.StringHashMap(Stats.BufoMatchData).init(allocator), .tracked_posts = .empty, }; defer stats.bufo_matches.deinit(); defer stats.monthly_bufo_matches.deinit(); defer stats.tracked_posts.deinit(allocator); stats.posts_checked.store(123, .monotonic); stats.matches_found.store(7, .monotonic); stats.posts_created.store(4, .monotonic); stats.followers_matched.store(5, .monotonic); stats.non_followers_skipped.store(2, .monotonic); stats.errors.store(1, .monotonic); stats.bufos_loaded = 100; const special = "bufo \"quoted\"\n🐸.gif"; const url = "https://example.com/\"bufo\".gif"; try stats.bufo_matches.put("z.gif", .{ .count = 3, .url = "https://example.com/z.gif" }); try stats.bufo_matches.put(special, .{ .count = 4, .url = url }); try stats.bufo_matches.put("a.gif", .{ .count = 3, .url = "https://example.com/a.gif" }); try stats.tracked_posts.append(allocator, .{ .our_rkey = "private-record", .original_uri = "private-uri", .original_did = "private-did", .timestamp = 1 }); const body = try stats.renderPublicJson(allocator); defer allocator.free(body); const parsed = try json.parseFromSlice(json.Value, allocator, body, .{}); defer parsed.deinit(); const root = parsed.value.object; try std.testing.expectEqual(@as(usize, 9), root.count()); inline for (.{ .{ "posts_checked", 123 }, .{ "matches_found", 7 }, .{ "posts_created", 4 }, .{ "followers_matched", 5 }, .{ "non_followers_skipped", 2 }, .{ "errors", 1 }, .{ "bufos_loaded", 100 } }) |pair| { try std.testing.expectEqual(@as(i64, pair[1]), root.get(pair[0]).?.integer); } try std.testing.expect(root.get("uptime_seconds").?.integer >= 60); try std.testing.expect(mem.indexOf(u8, body, "private-") == null); const bufos = root.get("bufos").?.array.items; try std.testing.expectEqual(@as(usize, 3), bufos.len); try std.testing.expectEqualStrings(special, bufos[0].object.get("name").?.string); try std.testing.expectEqualStrings(url, bufos[0].object.get("url").?.string); try std.testing.expectEqualStrings("a.gif", bufos[1].object.get("name").?.string); try std.testing.expectEqualStrings("z.gif", bufos[2].object.get("name").?.string);
try stats.monthly_bufo_matches.put(special, .{ .count = 4, .url = url }); const monthly = try stats.renderJson(allocator); defer allocator.free(monthly); const parsed_monthly = try json.parseFromSlice(json.Value, allocator, monthly, .{}); defer parsed_monthly.deinit(); const entry = parsed_monthly.value.object.get("monthly_bufo_matches").?.object.get(special).?.object; try std.testing.expectEqualStrings(url, entry.get("url").?.string); try std.testing.expectEqual(@as(i64, 4), entry.get("count").?.integer);
stats.bufo_matches.clearRetainingCapacity(); const empty = try stats.renderPublicJson(allocator); defer allocator.free(empty); const parsed_empty = try json.parseFromSlice(json.Value, allocator, empty, .{}); defer parsed_empty.deinit(); try std.testing.expectEqual(@as(usize, 0), parsed_empty.value.object.get("bufos").?.array.items.len);}