Something went wrong. Try again.
semantic bufo search find-bufo.com
bufo
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444//! every quote-post the bot has made, by bufo — the source behind the stats//! page's post viewer. the bot is the only writer of its own repo, so it can//! keep this index itself: entries are added as posts go out, removed as they//! are cleaned up, and reconciled against the bot's own PDS at startup//! (`com.atproto.repo.listRecords`, no appview). persisted at//! /data/posts_index.json next to stats.json.
const std = @import("std");const zat = @import("zat");const mem = std.mem;const json = std.json;const Allocator = mem.Allocator;const Io = std.Io;const http = std.http;
const INDEX_PATH = "/data/posts_index.json";const INDEX_TMP_PATH = "/data/posts_index.json.tmp";const PAGE_LIMIT = 100;const MAX_PAGES = 500;
var io: Io = undefined;
pub fn init(app_io: Io) void { io = app_io;}
pub const Entry = struct { rkey: []const u8, ts: i64, bufo: []const u8, quoted: []const u8,};
const Row = struct { []const u8, i64, []const u8, []const u8 };
const Document = struct { posts: []Entry };
pub const PostsIndex = struct { allocator: Allocator, entries: std.ArrayList(Entry) = .empty, known: std.StringHashMap(void), mutex: Io.Mutex = Io.Mutex.init,
pub fn initIndex(allocator: Allocator) PostsIndex { var self = PostsIndex{ .allocator = allocator, .known = std.StringHashMap(void).init(allocator), }; self.load(); return self; }
pub fn deinit(self: *PostsIndex) void { for (self.entries.items) |e| self.freeEntry(e); self.entries.deinit(self.allocator); self.known.deinit(); }
fn freeEntry(self: *PostsIndex, e: Entry) void { self.allocator.free(e.rkey); self.allocator.free(e.bufo); self.allocator.free(e.quoted); }
fn load(self: *PostsIndex) void { const file = Io.Dir.openFileAbsolute(io, INDEX_PATH, .{}) catch return; defer file.close(io); const buf = self.allocator.alloc(u8, 16 * 1024 * 1024) catch return; defer self.allocator.free(buf); const len = file.readPositionalAll(io, buf, 0) catch return; if (len == 0) return;
const parsed = json.parseFromSlice(Document, self.allocator, buf[0..len], .{ .ignore_unknown_fields = true }) catch |err| { std.debug.print("posts index: {s} unreadable ({s}), starting empty\n", .{ INDEX_PATH, @errorName(err) }); return; }; defer parsed.deinit(); for (parsed.value.posts) |e| self.appendUnlocked(e) catch continue; std.debug.print("posts index: loaded {} posts\n", .{self.entries.items.len}); }
/// dupes every string; a duplicate rkey is ignored fn appendUnlocked(self: *PostsIndex, e: Entry) !void { if (self.known.contains(e.rkey)) return; const rkey = try self.allocator.dupe(u8, e.rkey); errdefer self.allocator.free(rkey); const bufo = try self.allocator.dupe(u8, e.bufo); errdefer self.allocator.free(bufo); const quoted = try self.allocator.dupe(u8, e.quoted); errdefer self.allocator.free(quoted); try self.entries.append(self.allocator, .{ .rkey = rkey, .ts = e.ts, .bufo = bufo, .quoted = quoted }); errdefer _ = self.entries.pop(); try self.known.put(rkey, {}); }
fn saveUnlocked(self: *PostsIndex) void { const body = json.Stringify.valueAlloc(self.allocator, Document{ .posts = self.entries.items }, .{}) catch return; defer self.allocator.free(body); const file = Io.Dir.createFileAbsolute(io, INDEX_TMP_PATH, .{}) catch return; file.writeStreamingAll(io, body) catch { file.close(io); return; }; file.close(io); Io.Dir.renameAbsolute(INDEX_TMP_PATH, INDEX_PATH, io) catch return; }
pub fn add(self: *PostsIndex, rkey: []const u8, ts: i64, bufo_name: []const u8, quoted_uri: []const u8) void { self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); self.appendUnlocked(.{ .rkey = rkey, .ts = ts, .bufo = bufoName(bufo_name), .quoted = quoted_uri }) catch return; self.saveUnlocked(); }
pub fn remove(self: *PostsIndex, rkey: []const u8) void { self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); for (self.entries.items, 0..) |e, i| { if (!mem.eql(u8, e.rkey, rkey)) continue; _ = self.known.remove(e.rkey); self.freeEntry(self.entries.orderedRemove(i)); self.saveUnlocked(); return; } }
/// owned copies of the newest `n` entries, newest first pub fn recent(self: *PostsIndex, allocator: Allocator, n: usize) ![]Entry { self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); const take = @min(n, self.entries.items.len); var out = try allocator.alloc(Entry, take); errdefer allocator.free(out); var filled: usize = 0; errdefer for (out[0..filled]) |e| freeEntryWith(allocator, e); while (filled < take) : (filled += 1) { const e = self.entries.items[self.entries.items.len - 1 - filled]; out[filled] = .{ .rkey = try allocator.dupe(u8, e.rkey), .ts = e.ts, .bufo = try allocator.dupe(u8, e.bufo), .quoted = try allocator.dupe(u8, e.quoted), }; } return out; }
pub fn count(self: *PostsIndex) usize { self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); return self.entries.items.len; }
/// reconcile against the bot's own repo. walks listRecords newest-first /// and stops at the first page that adds nothing (a full walk when the /// index starts empty). runs on its own thread at startup. pub fn backfill(self: *PostsIndex, pds_endpoint: []const u8, did: []const u8) void { const started_empty = self.count() == 0; var cursor_buf: [128]u8 = undefined; var cursor: ?[]const u8 = null; var added: usize = 0; var pages: usize = 0;
while (pages < MAX_PAGES) : (pages += 1) { var url_buf: [1024]u8 = undefined; const url = if (cursor) |c| std.fmt.bufPrint(&url_buf, "{s}/xrpc/com.atproto.repo.listRecords?repo={s}&collection=app.bsky.feed.post&limit={d}&cursor={s}", .{ pds_endpoint, did, PAGE_LIMIT, c }) catch return else std.fmt.bufPrint(&url_buf, "{s}/xrpc/com.atproto.repo.listRecords?repo={s}&collection=app.bsky.feed.post&limit={d}", .{ pds_endpoint, did, PAGE_LIMIT }) catch return;
const body = self.fetch(url) catch |err| { std.debug.print("posts index: backfill page {} failed: {s}\n", .{ pages, @errorName(err) }); break; }; defer self.allocator.free(body);
const page = self.ingestPage(body, &cursor_buf) catch |err| { std.debug.print("posts index: backfill page {} unreadable: {s}\n", .{ pages, @errorName(err) }); break; }; added += page.added; if (page.added == 0 and !started_empty) break; cursor = page.cursor orelse break; }
self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); mem.sort(Entry, self.entries.items, {}, entryBefore); if (added > 0) self.saveUnlocked(); std.debug.print("posts index: backfill added {} ({} pages), {} posts total\n", .{ added, pages, self.entries.items.len }); }
fn entryBefore(_: void, a: Entry, b: Entry) bool { return a.ts < b.ts; }
fn fetch(self: *PostsIndex, url: []const u8) ![]u8 { var client: http.Client = .{ .allocator = self.allocator, .io = io }; defer client.deinit(); var aw: Io.Writer.Allocating = .init(self.allocator); defer aw.deinit(); const result = try client.fetch(.{ .location = .{ .url = url }, .method = .GET, .headers = .{ .accept_encoding = .{ .override = "identity" } }, .response_writer = &aw.writer, }); if (result.status != .ok) return error.ListRecordsFailed; return try self.allocator.dupe(u8, aw.written()); }
const Page = struct { added: usize, cursor: ?[]const u8 };
fn ingestPage(self: *PostsIndex, body: []const u8, cursor_buf: *[128]u8) !Page { const parsed = try json.parseFromSlice(json.Value, self.allocator, body, .{}); defer parsed.deinit(); if (parsed.value != .object) return error.BadPage;
var added: usize = 0; if (zat.json.getArray(parsed.value, "records")) |records| { self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); for (records) |rec| { const entry = parseRecord(rec) orelse continue; if (self.known.contains(entry.rkey)) continue; const name = dashed(self.allocator, entry.bufo) catch continue; defer self.allocator.free(name); self.appendUnlocked(.{ .rkey = entry.rkey, .ts = entry.ts, .bufo = name, .quoted = entry.quoted }) catch continue; added += 1; } }
var cursor: ?[]const u8 = null; if (zat.json.getString(parsed.value, "cursor")) |c| if (c.len > 0 and c.len <= cursor_buf.len) { @memcpy(cursor_buf[0..c.len], c); cursor = cursor_buf[0..c.len]; }; return .{ .added = added, .cursor = cursor }; }
pub fn renderBufo(self: *PostsIndex, allocator: Allocator, name: []const u8) ![]u8 { return self.render(allocator, name, .{ .bufo = name }); }
pub fn renderByDid(self: *PostsIndex, allocator: Allocator, did: []const u8) ![]u8 { return self.render(allocator, did, .{ .quoted_did = did }); }
const Filter = union(enum) { bufo: []const u8, quoted_did: []const u8 };
/// newest first: [rkey, unix seconds, quoted at-uri, bufo] fn render(self: *PostsIndex, allocator: Allocator, name: []const u8, filter: Filter) ![]u8 { var rows: std.ArrayList(Row) = .empty; defer rows.deinit(allocator);
self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); var i = self.entries.items.len; while (i > 0) { i -= 1; const e = self.entries.items[i]; const keep = switch (filter) { .bufo => |b| mem.eql(u8, e.bufo, b), .quoted_did => |d| quotedDid(e.quoted) != null and mem.eql(u8, quotedDid(e.quoted).?, d), }; if (keep) try rows.append(allocator, .{ e.rkey, e.ts, e.quoted, e.bufo }); } return json.Stringify.valueAlloc(allocator, .{ .name = name, .count = rows.items.len, .posts = rows.items }, .{}); }};
fn freeEntryWith(allocator: Allocator, e: Entry) void { allocator.free(e.rkey); allocator.free(e.bufo); allocator.free(e.quoted);}
pub fn freeEntries(allocator: Allocator, entries: []Entry) void { for (entries) |e| freeEntryWith(allocator, e); allocator.free(entries);}
fn quotedDid(uri: []const u8) ?[]const u8 { if (!mem.startsWith(u8, uri, "at://")) return null; var parts = mem.splitScalar(u8, uri[5..], '/'); return parts.next();}
/// the bot names a bufo in the media alt with dashes turned to spaces and the/// file extension dropped; the index keys on the dashed, extension-free name/// (what the stats grid calls it), so live adds and backfilled alts agreefn bufoName(name: []const u8) []const u8 { return name[0 .. mem.indexOfScalar(u8, name, '.') orelse name.len];}
/// one listRecords entry → Entry (strings borrowed from the parsed value),/// or null when it is not a bufo quote-postfn parseRecord(rec: json.Value) ?Entry { const uri = zat.json.getString(rec, "uri") orelse return null; const rkey = uri[(mem.lastIndexOfScalar(u8, uri, '/') orelse return null) + 1 ..]; const media = zat.json.getPath(rec, "value.embed.media") orelse return null;
// An images field takes precedence even when malformed or empty. const alt = if (zat.json.getPath(media, "images")) |images| blk: { if (images != .array or images.array.items.len == 0) return null; break :blk zat.json.getString(images.array.items[0], "alt"); } else zat.json.getString(media, "alt"); const alt_text = alt orelse return null; if (alt_text.len == 0) return null;
const created = zat.json.getString(rec, "value.createdAt") orelse return null; const ts = parseIsoTimestamp(created) orelse return null; const quoted = zat.json.getString(rec, "value.embed.record.record.uri") orelse "";
return .{ .rkey = rkey, .ts = ts, .bufo = alt_text, .quoted = quoted };}
/// the alt text is the bufo name with spaces for dashes; invert it in place/// at add time — parseRecord hands back the raw alt, so the index does the/// swap when dupingfn dashed(allocator: Allocator, alt: []const u8) ![]u8 { const out = try allocator.dupe(u8, alt); for (out) |*c| if (c.* == ' ') { c.* = '-'; }; return out;}
fn daysFromCivil(y_in: i64, m: i64, d: i64) i64 { const y = if (m <= 2) y_in - 1 else y_in; const era = @divFloor(y, 400); const yoe = y - era * 400; const mp = @mod(m + 9, 12); const doy = @divFloor(153 * mp + 2, 5) + d - 1; const doe = yoe * 365 + @divFloor(yoe, 4) - @divFloor(yoe, 100) + doy; return era * 146097 + doe - 719468;}
/// "YYYY-MM-DDTHH:MM:SS[.fff]Z" → unix seconds; anything else → nullfn parseIsoTimestamp(s: []const u8) ?i64 { if (s.len < 19 or s[4] != '-' or s[7] != '-' or s[10] != 'T' or s[13] != ':' or s[16] != ':') return null; const y = std.fmt.parseInt(i64, s[0..4], 10) catch return null; const mo = std.fmt.parseInt(i64, s[5..7], 10) catch return null; const d = std.fmt.parseInt(i64, s[8..10], 10) catch return null; const h = std.fmt.parseInt(i64, s[11..13], 10) catch return null; const mi = std.fmt.parseInt(i64, s[14..16], 10) catch return null; const sec = std.fmt.parseInt(i64, s[17..19], 10) catch return null; if (mo < 1 or mo > 12 or d < 1 or d > 31 or h > 23 or mi > 59 or sec > 60) return null; return daysFromCivil(y, mo, d) * 86400 + h * 3600 + mi * 60 + sec;}
test "iso timestamps" { try std.testing.expectEqual(@as(?i64, 0), parseIsoTimestamp("1970-01-01T00:00:00Z")); try std.testing.expectEqual(@as(?i64, 951868800), parseIsoTimestamp("2000-03-01T00:00:00.000Z")); try std.testing.expectEqual(@as(?i64, 1704067200), parseIsoTimestamp("2024-01-01T00:00:00Z")); try std.testing.expectEqual(@as(?i64, 1704067200 + 3661), parseIsoTimestamp("2024-01-01T01:01:01.5Z")); try std.testing.expectEqual(@as(?i64, null), parseIsoTimestamp("not a date"));}
test "bufo names" { try std.testing.expectEqualStrings("bufo-lgtm", bufoName("bufo-lgtm.png")); try std.testing.expectEqualStrings("bufo-lgtm", bufoName("bufo-lgtm")); const a = try dashed(std.testing.allocator, "bufo on the ceiling"); defer std.testing.allocator.free(a); try std.testing.expectEqualStrings("bufo-on-the-ceiling", a);}
test "parseRecord reads images and video embeds and skips the rest" { const allocator = std.testing.allocator; const images = \\{"uri":"at://did:plc:bot/app.bsky.feed.post/3aaa","value":{"createdAt":"2024-01-01T00:00:00Z","embed":{"$type":"app.bsky.embed.recordWithMedia","media":{"$type":"app.bsky.embed.images","images":[{"alt":"bufo on the ceiling","image":{}}]},"record":{"record":{"uri":"at://did:plc:user/app.bsky.feed.post/q1","cid":"x"}}}}} ; var p = try json.parseFromSlice(json.Value, allocator, images, .{}); defer p.deinit(); const e = parseRecord(p.value).?; try std.testing.expectEqualStrings("3aaa", e.rkey); try std.testing.expectEqual(@as(i64, 1704067200), e.ts); try std.testing.expectEqualStrings("bufo on the ceiling", e.bufo); try std.testing.expectEqualStrings("at://did:plc:user/app.bsky.feed.post/q1", e.quoted); try std.testing.expectEqualStrings("did:plc:user", quotedDid(e.quoted).?);
const video = \\{"uri":"at://did:plc:bot/app.bsky.feed.post/3bbb","value":{"createdAt":"2024-01-01T00:00:00Z","embed":{"media":{"$type":"app.bsky.embed.video","alt":"bufo lgtm","video":{}},"record":{"record":{"uri":"at://did:plc:user/app.bsky.feed.post/q2"}}}}} ; var v = try json.parseFromSlice(json.Value, allocator, video, .{}); defer v.deinit(); try std.testing.expectEqualStrings("bufo lgtm", parseRecord(v.value).?.bufo);
const plain = \\{"uri":"at://did:plc:bot/app.bsky.feed.post/3ccc","value":{"createdAt":"2024-01-01T00:00:00Z","text":"hi"}} ; var t = try json.parseFromSlice(json.Value, allocator, plain, .{}); defer t.deinit(); try std.testing.expect(parseRecord(t.value) == null);}
test "render filters by bufo and by quoted did, newest first" { const allocator = std.testing.allocator; var idx = PostsIndex{ .allocator = allocator, .known = std.StringHashMap(void).init(allocator) }; defer idx.deinit(); try idx.appendUnlocked(.{ .rkey = "3a", .ts = 10, .bufo = "bufo-a", .quoted = "at://did:plc:u1/app.bsky.feed.post/x" }); try idx.appendUnlocked(.{ .rkey = "3b", .ts = 20, .bufo = "bufo-a", .quoted = "at://did:plc:u2/app.bsky.feed.post/y" }); try idx.appendUnlocked(.{ .rkey = "3c", .ts = 30, .bufo = "bufo-b", .quoted = "at://did:plc:u1/app.bsky.feed.post/z" }); try idx.appendUnlocked(.{ .rkey = "3c", .ts = 30, .bufo = "bufo-b", .quoted = "" }); try std.testing.expectEqual(@as(usize, 3), idx.entries.items.len);
const by_bufo = try idx.renderBufo(allocator, "bufo-a"); defer allocator.free(by_bufo); try std.testing.expectEqualStrings( \\{"name":"bufo-a","count":2,"posts":[["3b",20,"at://did:plc:u2/app.bsky.feed.post/y","bufo-a"],["3a",10,"at://did:plc:u1/app.bsky.feed.post/x","bufo-a"]]} , by_bufo);
const by_did = try idx.renderByDid(allocator, "did:plc:u1"); defer allocator.free(by_did); try std.testing.expectEqualStrings( \\{"name":"did:plc:u1","count":2,"posts":[["3c",30,"at://did:plc:u1/app.bsky.feed.post/z","bufo-b"],["3a",10,"at://did:plc:u1/app.bsky.feed.post/x","bufo-a"]]} , by_did);}
test "parseRecord preserves images precedence over video alt" { const allocator = std.testing.allocator; const prefix = "{\"uri\":\"at://did:plc:bot/app.bsky.feed.post/rkey\",\"value\":{\"createdAt\":\"2024-01-01T00:00:00Z\",\"embed\":{\"media\":"; const suffix = "}}}"; for ([_][]const u8{ "{\"images\":null,\"alt\":\"video fallback\"}", "{\"images\":{},\"alt\":\"video fallback\"}", "{\"images\":[],\"alt\":\"video fallback\"}", "{\"images\":[null],\"alt\":\"video fallback\"}", "{\"images\":[{\"alt\":7}],\"alt\":\"video fallback\"}", "{\"images\":[{\"alt\":\"\"}],\"alt\":\"video fallback\"}", }) |media| { const body = try std.mem.concat(allocator, u8, &.{ prefix, media, suffix }); defer allocator.free(body); var parsed = try json.parseFromSlice(json.Value, allocator, body, .{}); defer parsed.deinit(); try std.testing.expect(parseRecord(parsed.value) == null); } var parsed = try json.parseFromSlice(json.Value, allocator, prefix ++ "{\"alt\":\"video fallback\"}" ++ suffix, .{}); defer parsed.deinit(); const entry = parseRecord(parsed.value).?; try std.testing.expectEqualStrings("video fallback", entry.bufo); try std.testing.expectEqualStrings("", entry.quoted);}