From 92c0d155f7ba0b5c4f808bb75d8d499a172b951c Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Thu, 11 Jun 2026 00:45:13 -0500 Subject: [PATCH] perf(pg): cache host_id, index account(host_id), short-circuit count (v0.0.3) tier 1 of the postgres workload handoff: cut the per-event query train and the new-account-admission seq scan. - host_id read-through/write-through cache (uid-keyed LRU). statement #2 (SELECT host_id ... every event, no cache) now hits the DB only on cache miss. setAccountHostId is the sole writer, so the cache stays coherent. - CREATE INDEX CONCURRENTLY on account(host_id): the per-host COUNT(*) on new-account admission was an 8.7M-row seq scan (no index existed). - getEffectiveAccountCount: CASE instead of COALESCE so the COUNT subquery only runs when no account_limit is set (COALESCE doesn't short-circuit the aggregate). regression test covers host_id cache coherence across set/update/clear/miss. Co-Authored-By: Claude Opus 4.8 --- build.zig.zon | 2 +- .../HANDOFF-2026-06-11-postgres-workload.md | 145 ++++++++++++++++++ src/internal/event_log.zig | 81 +++++++++- 3 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 docs/handoffs/HANDOFF-2026-06-11-postgres-workload.md diff --git a/build.zig.zon b/build.zig.zon index 49036ad..7c62b95 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = .zlay, - .version = "0.0.2", + .version = "0.0.3", .fingerprint = 0x31343ede133f3e58, .minimum_zig_version = "0.16.0", .dependencies = .{ diff --git a/docs/handoffs/HANDOFF-2026-06-11-postgres-workload.md b/docs/handoffs/HANDOFF-2026-06-11-postgres-workload.md new file mode 100644 index 0000000..3c1f21f --- /dev/null +++ b/docs/handoffs/HANDOFF-2026-06-11-postgres-workload.md @@ -0,0 +1,145 @@ +# operator handoff 2026-06-11 — postgres workload profile + +operator report, measured against the production instance (`zlay-db-postgresql-0`, +postgres 18.3, 85 days of accumulated stats + live sampling). nothing here is +broken — this is a cost profile. the motivation: stack-for-stack against the +indigo relay, the zlay **binary** wins big (1.06 vs 2.98 avg cores, 2.4 vs 8.9 +GiB), but zlay's **postgres** runs 2.16 avg cores (p95 4.4) vs indigo's 0.39, +which eats most of the binary's CPU advantage at the whole-stack level. this +doc lays out where those cycles go, with the data mapped to source lines. +what (if anything) to change, and how, is your call — there are operator-side +config levers too, listed at the end, which I'll coordinate separately. + +## headline numbers + +| measure | value | how measured | +|---|---|---| +| transactions | **2,656 tx/s** sustained | pg_stat_database diff over 129s | +| lifetime tx | 24.4B over 85d (~3,325/s avg) | pg_stat_database | +| event rate during measurement | ~350 frames/s in | relay_frames_received rate | +| → **queries per event** | **~7–8** | ratio | +| tuple updates | 342/s | snapshot diff | +| WAL | ~640 KB/s (~55 GB/day) | pg_current_wal_lsn diff | +| foreground WAL fsyncs | **2.52B lifetime (~343/s)** | pg_stat_io, client backend | +| account point lookups (now) | 634 idx_scan/s | pg_stat_user_tables diff over 90s | + +## the per-commit query train (mapped to source) + +every #commit event runs, in sequence, each as its own autocommit transaction +over the wire: + +1. `SELECT uid FROM account WHERE did=$1` — only on did_cache miss + (event_log.zig:505). the did→uid cache works: only 231M lifetime scans on + `account_did_key` vs 6.03B on `account_pkey`. +2. `SELECT host_id FROM account WHERE uid=$1` — **every event, no cache** + (uidForDidFromHost → getAccountHostId, event_log.zig:445/559) +3. `SELECT status, upstream_status FROM account WHERE uid=$1` — **every + commit, no cache** (isAccountActive, event_log.zig:640; called from + frame_worker.zig:171 and subscriber.zig:627) +4. `SELECT rev, commit_data_cid FROM account_repo WHERE uid=$1` — every + commit (getAccountState, event_log.zig:529; chain-break detection) +5. `INSERT INTO account_repo ... ON CONFLICT (uid) DO UPDATE ... WHERE + account_repo.rev < EXCLUDED.rev` — every validated commit + (updateAccountState, event_log.zig:548). this is the single hottest + statement: 16 of 29 active-query samples, 2.11B lifetime executions. + +steps 2–4 are uid-keyed point reads of data that either changes rarely +(host_id, status — mutations are ~0.8 rows/s on account) or that zlay itself +wrote on the previous commit for that uid (rev/data_cid — zlay is the only +writer of account_repo). lifetime: 6.03B account_pkey + 4.98B +account_repo_pkey index scans. + +observed live (30s of pg_stat_activity sampling, 29 active-query hits): +16× the upsert, 4× status, 4× host_id, 3× rev read, 2× did→uid. + +## where the postgres CPU actually goes + +- **per-statement transaction overhead.** 2,656 tx/s of single-statement + autocommit round trips: parse/plan/execute/commit each time, and — because + `synchronous_commit=on` — every *write* transaction does its own foreground + WAL flush. pg_stat_io: client backends performed 2.52B WAL fsyncs lifetime + (~343/s, ≈ the write-tx rate); the walwriter did only 12M. backends spend + their lives committing tiny transactions one at a time. process-level `ps` + agrees: CPU is smeared evenly across the 20 pool connections (~3% each, + mostly idle-between-statements), no single heavy query. +- **buffer churn on `account`.** lifetime heap hit ratio on `account` is 24% + (53.7B heap blocks read vs 17.3B hit — ~400 TB re-read over 85d). the heap + is 877 MB; `shared_buffers` is 128 MB (stock bitnami). `account_repo` stays + ~89% hot because the same pages are hammered continuously. current steady + state is gentler (95% overall hit, ~2.5 MB/s reads) — the horror numbers + are dominated by the collection-backfill era plus the seq-scan pattern + below, but the per-event uid lookups are still ~634 scans/s through a + 128 MB sieve. + +## episodic, not steady-state (but worth knowing) + +- **`SELECT COUNT(*) FROM account WHERE host_id=$1`** (event_log.zig:574, + also xrpc.zig:592, and the COALESCE/account_limit variant at + event_log.zig:590). `account` has **no index on host_id**, so each + execution is a (parallel) seq scan over 8.7M rows. lifetime: 3.32M seq + scans ≈ one per new account ingested; pg_stat_io shows ~430 TB of bulkread + traffic attributable to this class. currently **0 per 90s** — it fires on + new-account admission, so it's invisible in steady state and shows up + exactly when a new PDS ramps or a backfill runs (i.e. when you least want + it). note the COALESCE form computes the COUNT even when account_limit is + set — SQL doesn't short-circuit the aggregate. +- **collection-index backfill** drove most of the lifetime tup_returned + (5 trillion) and bulkread volume. done/idle now, but any future re-import + repeats the pattern. + +## what is already good + +- `host.last_seq` cursor flushes are debounced through CursorMap (host_ops.zig + flush sweep) — 549M updates over 85d ≈ 75/s across ~2,400 active hosts, all + HOT updates. the table autovacuums every ~84s, which is noisy but cheap + (1.1 MB table). +- HOT-update ratios are excellent everywhere (account_repo 99.95%) — no + fillfactor/bloat problem. dead tuple counts are healthy. zero rollbacks, + zero deadlocks, zero temp spills. +- the did→uid cache demonstrably works (26× fewer did-key scans than + uid-key scans). +- DbRequestQueue (2 workers, pool size 20) keeps the pipeline decoupled — + this profile is about cost, not stalls. + +## observations to consider (not prescriptions) + +1. three of the five per-commit statements read state that zlay either wrote + itself (rev/data_cid) or that changes ~5 orders of magnitude slower than + it's read (host_id, status). whether that belongs in an app-side cache, + a combined statement, or somewhere else entirely is a design call — but + each one eliminated removes ~350 tx/s. +2. the read-then-upsert pair (steps 4+5) and the guarded upsert overlap in + purpose: the upsert already enforces `rev < EXCLUDED.rev`. the separate + read exists for chain-break observability. there may be a shape where one + round trip serves both. +3. ~343 foreground WAL fsyncs/s for bookkeeping data raises the question of + what durability this data actually needs — all of it is reconstructible + from the network (re-crawl) or self-healing (rev guard rejects stale + writes). semantics worth a deliberate decision rather than the default. +4. the host_id COUNT pattern: an index would make it cheap; a counter would + make it free; computing it on the admission path at all is the deeper + question. it's also the only query class that punishes new-host ramps. +5. if per-statement round trips stay, batching adjacent events' statements + (the queue already serializes through 2 workers) changes the tx:event + ratio without changing any semantics. + +## operator-side levers (mine, listed for completeness) + +- `shared_buffers` 128 MB inside a 1Gi-limit pod, `work_mem` 4 MB, + `max_wal_size` 400 MB — all stock bitnami defaults, never tuned. the node + has headroom; I can resize the pod + buffers independently of any code + change. +- `pg_stat_statements` is not loaded (`shared_preload_libraries=pgaudit` + only), which is why this profile leans on pg_stat_io/activity sampling. + I'll add it at the next planned db restart so future profiles have + per-query timings. +- `synchronous_commit` is a config-side knob too (per-database or per-role), + but it changes the same durability semantics as observation 3, so it + should be one decision, not two. + +context: longer term we may consider running lightrail next to zlay and +un-inlining the collection index, but the postgres levers above come first — +they're cheaper and the measurements say most of the cost is in the per-event +query train, not the index. + +— operator diff --git a/src/internal/event_log.zig b/src/internal/event_log.zig index 8649996..7a01e99 100644 --- a/src/internal/event_log.zig +++ b/src/internal/event_log.zig @@ -231,6 +231,13 @@ pub const DiskPersist = struct { // DID → UID cache (matches indigo's bidirectional ARC cache) did_cache: lru.LruCache(u64), + // UID → host_id cache. host_id mutates ~5 orders of magnitude slower than + // it's read (every event hits getAccountHostId) and setAccountHostId is its + // sole writer, so a read-through cache eliminates a per-event point read. + // keyed by the uid's raw bytes; only nonzero host_ids are cached (0 means + // "not yet assigned", a transient state that resolves on the next commit). + host_id_cache: lru.LruCache(u64), + // write buffer (flushed periodically or when threshold hit) outbuf: std.ArrayListUnmanaged(u8) = .empty, evtbuf: std.ArrayListUnmanaged(PersistJob) = .empty, @@ -325,6 +332,12 @@ pub const DiskPersist = struct { _ = pool.exec("ALTER TABLE account ADD COLUMN IF NOT EXISTS host_id BIGINT NOT NULL DEFAULT 0", .{}) catch {}; _ = pool.exec("ALTER TABLE account ADD COLUMN IF NOT EXISTS upstream_status TEXT NOT NULL DEFAULT 'active'", .{}) catch {}; + // index host_id so the per-host account COUNT(*) (new-account admission) + // is an index scan instead of an 8.7M-row seq scan. CONCURRENTLY avoids + // locking the live account table during the build; IF NOT EXISTS makes + // re-runs (and resuming a failed build) safe. + _ = pool.exec("CREATE INDEX CONCURRENTLY IF NOT EXISTS account_host_id_idx ON account (host_id)", .{}) catch {}; + _ = try pool.exec( \\CREATE TABLE IF NOT EXISTS account_repo ( \\ uid BIGINT PRIMARY KEY REFERENCES account(uid), @@ -390,6 +403,7 @@ pub const DiskPersist = struct { .dir = dir, .db = pool, .did_cache = lru.LruCache(u64).init(allocator, 500_000, io), + .host_id_cache = lru.LruCache(u64).init(allocator, 500_000, io), .io = io, }; @@ -418,6 +432,7 @@ pub const DiskPersist = struct { self.outbuf.deinit(self.allocator); self.did_cache.deinit(); + self.host_id_cache.deinit(); if (self.current_file) |f| f.close(self.io); if (self.current_file_path) |p| self.allocator.free(p); @@ -553,15 +568,30 @@ pub const DiskPersist = struct { // --- account status --- + /// raw little-endian bytes of a uid, for use as an LRU key. + fn uidKey(uid: u64) [8]u8 { + return @bitCast(std.mem.nativeToLittle(u64, uid)); + } + /// get the host_id for an account. returns 0 if not set. + /// read-through cache: only nonzero host_ids are cached (0 is the transient + /// pre-assignment state, which resolves via setAccountHostId on next commit). pub fn getAccountHostId(self: *DiskPersist, uid: u64) !u64 { + const key = uidKey(uid); + if (self.host_id_cache.get(&key)) |hid| return hid; + var row = (try self.db.rowUnsafe( "SELECT host_id FROM account WHERE uid = $1", .{@as(i64, @intCast(uid))}, )) orelse return 0; defer row.deinit() catch {}; const hid = row.get(i64, 0); - return if (hid > 0) @intCast(hid) else 0; + if (hid > 0) { + const host_id: u64 = @intCast(hid); + self.host_id_cache.put(&key, host_id) catch {}; + return host_id; + } + return 0; } /// count accounts on a host (Threaded pool) @@ -585,9 +615,11 @@ pub const DiskPersist = struct { } /// uses admin-configured limit if set, otherwise actual COUNT(*). + /// CASE (not COALESCE) so the COUNT subquery is only evaluated when no + /// account_limit is set — SQL doesn't short-circuit a COALESCE's aggregate. fn getEffectiveAccountCountImpl(host_id: u64, db: *pg.Pool) u64 { var row = (db.rowUnsafe( - "SELECT COALESCE(h.account_limit, COUNT(a.uid)) FROM host h LEFT JOIN account a ON a.host_id = h.id WHERE h.id = $1 GROUP BY h.id", + "SELECT CASE WHEN h.account_limit IS NOT NULL THEN h.account_limit ELSE (SELECT COUNT(*) FROM account a WHERE a.host_id = h.id) END FROM host h WHERE h.id = $1", .{@as(i64, @intCast(host_id))}, ) catch return 0) orelse return 0; defer row.deinit() catch {}; @@ -622,6 +654,13 @@ pub const DiskPersist = struct { "UPDATE account SET host_id = $2 WHERE uid = $1", .{ @as(i64, @intCast(uid)), @as(i64, @intCast(host_id)) }, ); + // write-through: sole writer, so the cache stays coherent with the DB. + const key = uidKey(uid); + if (host_id > 0) { + self.host_id_cache.put(&key, host_id) catch {}; + } else { + _ = self.host_id_cache.remove(&key); + } } /// update the upstream status for an account (from #account events). @@ -1783,6 +1822,44 @@ test "uidForDid assigns and caches UIDs" { try std.testing.expect(uid1 != uid2); } +test "host_id cache stays coherent with setAccountHostId" { + const base_url = try requireDatabaseUrl(); + var tdb = try TestDb.create(base_url); + defer tdb.destroy(); + const database_url = tdb.url; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const dir_path = try tmpDirRealPath(std.testing.allocator, tmp); + defer std.testing.allocator.free(dir_path); + + var dp = try DiskPersist.init(std.testing.allocator, dir_path, database_url, 5, std.testing.io); + defer dp.deinit(); + + const uid = try dp.uidForDid("did:plc:hostcache"); + + // unset host_id reads as 0 and is not cached (transient pre-assignment state) + try std.testing.expectEqual(@as(u64, 0), try dp.getAccountHostId(uid)); + + // write-through: set populates the cache, read returns it + try dp.setAccountHostId(uid, 42); + try std.testing.expectEqual(@as(u64, 42), try dp.getAccountHostId(uid)); + + // a host change updates the cache in place — no stale read + try dp.setAccountHostId(uid, 99); + try std.testing.expectEqual(@as(u64, 99), try dp.getAccountHostId(uid)); + + // clearing to 0 evicts the entry; read falls through to the DB (0) + try dp.setAccountHostId(uid, 0); + try std.testing.expectEqual(@as(u64, 0), try dp.getAccountHostId(uid)); + + // value survives a cache miss: clear the cache, read repopulates from DB + try dp.setAccountHostId(uid, 7); + _ = dp.host_id_cache.remove(&DiskPersist.uidKey(uid)); + try std.testing.expectEqual(@as(u64, 7), try dp.getAccountHostId(uid)); +} + test "uidForDid survives reinit" { const base_url = try requireDatabaseUrl(); var tdb = try TestDb.create(base_url); -- 2.51.2