diff --git a/docs/design.md b/docs/design.md index 612db8f..e9335cd 100644 --- a/docs/design.md +++ b/docs/design.md @@ -54,6 +54,7 @@ additionally: | flush thread | 1 | 8 MB | batched fsync of event log (100ms or 400 events) | | GC thread | 1 | 8 MB | event log file cleanup every 10 minutes | | crawl queue thread | 1 | 8 MB | process `requestCrawl` — validate hostname, describeServer, spawn worker | +| reconcile thread | 1 | 8 MB | every 5 min, query active hosts from DB, respawn missing workers | | metrics server | 1 | 8 MB | HTTP on internal port, prometheus scrape | | main thread | 1 | default | signal handling, shutdown coordination | @@ -127,6 +128,9 @@ tables: - `account` — uid, did, status, upstream_status, host_id - `account_repo` — uid, rev, commit_data_cid (latest repo state) - `host` — id, hostname, status, last_seq, failed_attempts + - status lifecycle: `active` → `dormant` (after max failures, thread stops) + → `active` (on successful reconnect via reconciliation or crawl request). + `blocked` is permanent (admin action). `idle` is set by FutureCursor. - `log_file_refs` — seq→file mapping for cursor binary search - `domain_ban` — banned domain suffixes - `backfill_progress` — collection backfill cursor tracking @@ -228,3 +232,21 @@ zlay uses an 8K-entry per-consumer ring buffer (vs indigo's 16K-entry channel). can be tuned independently based on observed `ConsumerTooSlow` disconnect rate. the ring buffer is lock-free (atomic read/write indices), so the bottleneck is consumer write throughput, not buffer contention. + +### host retention model + +indigo marks hosts as "exhausted" after max failures and never retries. zlay +uses a dormant/reconciliation model instead: + +- after max consecutive failures (15), the host is marked `dormant` and its + thread stops. backoff grows to 30 min max (vs indigo's permanent death). +- every 5 minutes, a reconciliation thread queries active hosts from the DB + and respawns workers for any missing from the workers map. +- on successful reconnect, `reset_failures` also sets the host back to + `active`, so dormant hosts that come back online are automatically recovered. +- a connect gate (`MAX_CONCURRENT_CONNECTS`, default 50) bounds the DNS/TLS + storm during startup and reconnect waves, preventing health probe starvation. + +the net effect: hosts that go down temporarily are retried with increasing +backoff, and hosts that come back are picked up within 5 minutes. no manual +intervention needed unless the host is permanently dead (admin can block it). diff --git a/src/api/xrpc.zig b/src/api/xrpc.zig index c6db9ce..78aad66 100644 --- a/src/api/xrpc.zig +++ b/src/api/xrpc.zig @@ -584,6 +584,8 @@ const GetHostStatusReq = struct { "banned" else if (std.mem.eql(u8, raw_status, "exhausted")) "offline" + else if (std.mem.eql(u8, raw_status, "dormant")) + "offline" else raw_status; diff --git a/src/host_ops.zig b/src/host_ops.zig index 0dd1448..86ef7e5 100644 --- a/src/host_ops.zig +++ b/src/host_ops.zig @@ -281,8 +281,8 @@ pub const HostOpsQueue = struct { .increment_failures => { const failures = self.persist.incrementHostFailures(op.host_id) catch 0; if (failures >= self.max_consecutive_failures) { - log.warn("host_ops: host_id={d} exhausted after {d} failures", .{ op.host_id, failures }); - self.persist.updateHostStatus(op.host_id, "exhausted") catch {}; + log.warn("host_ops: host_id={d} dormant after {d} failures", .{ op.host_id, failures }); + self.persist.updateHostStatus(op.host_id, "dormant") catch {}; op.payload.host_shutdown.store(true, .release); } }, @@ -290,6 +290,10 @@ pub const HostOpsQueue = struct { self.persist.resetHostFailures(op.host_id) catch |err| { log.debug("host_ops: reset failures failed for host_id={d}: {s}", .{ op.host_id, @errorName(err) }); }; + // flip dormant hosts back to active on successful reconnect + self.persist.updateHostStatus(op.host_id, "active") catch |err| { + log.debug("host_ops: reset status failed for host_id={d}: {s}", .{ op.host_id, @errorName(err) }); + }; }, .update_status => { self.persist.updateHostStatus(op.host_id, op.payload.status.slice()) catch |err| { diff --git a/src/slurper.zig b/src/slurper.zig index 4a036fb..0f71307 100644 --- a/src/slurper.zig +++ b/src/slurper.zig @@ -5,7 +5,8 @@ //! - spawning/stopping subscriber workers //! - processing crawl requests (adding new hosts) //! - host validation (format, domain ban, describeServer, relay loop detection) -//! - tracking host lifecycle (active → exhausted → blocked) +//! - host lifecycle: active → dormant (after max failures) → active (on reconnect) +//! - reconciliation: every 5 min, respawns workers for active hosts missing from the map //! //! all downstream components (Broadcaster, DiskPersist, Validator) are //! thread-safe for N concurrent producers, so this just orchestrates. @@ -259,6 +260,7 @@ pub const Slurper = struct { // background tasks startup_future: ?Io.Future(void) = null, crawl_future: ?Io.Future(void) = null, + reconcile_future: ?Io.Future(void) = null, io: Io, /// dedicated Threaded io for the frame worker pool — safe from plain OS threads @@ -309,6 +311,7 @@ pub const Slurper = struct { // pullHosts + listActiveHosts + spawnWorker all happen in the background thread. self.startup_future = try self.io.concurrent(spawnWorkers, .{self}); self.crawl_future = try self.io.concurrent(processCrawlQueue, .{self}); + self.reconcile_future = try self.io.concurrent(reconcileHosts, .{self}); } /// pull PDS host list from the seed relay's com.atproto.sync.listHosts endpoint. @@ -693,6 +696,80 @@ pub const Slurper = struct { log.info("startup complete: {d} host(s) spawned", .{hosts.len}); } + /// background fiber: every 5 minutes, query active hosts from DB and respawn + /// any that are missing from the workers map. handles hosts that exited due to + /// transient errors or were marked dormant and later flipped back to active. + fn reconcileHosts(self: *Slurper) void { + const reconcile_interval: u64 = 5 * 60; // 5 minutes + + while (!self.shutdown.load(.acquire)) { + // sleep in 1-second increments so we can check shutdown + var remaining: u64 = reconcile_interval; + while (remaining > 0 and !self.shutdown.load(.acquire)) { + self.io.sleep(Io.Duration.fromSeconds(1), .awake) catch return; + remaining -= 1; + } + if (self.shutdown.load(.acquire)) return; + + const db_queue = self.db_queue orelse continue; + + // load active hosts via DbRequestQueue + const ListActiveHostsReq = struct { + base: event_log_mod.DbRequest = .{ .callback = &execute }, + alloc: Allocator, + result: ?[]event_log_mod.DiskPersist.Host = null, + + fn execute(b: *event_log_mod.DbRequest, dp: *event_log_mod.DiskPersist) void { + const s: *@This() = @fieldParentPtr("base", b); + s.result = dp.listActiveHosts(s.alloc) catch |e| { + b.err = e; + return; + }; + } + }; + var list_req: ListActiveHostsReq = .{ .alloc = self.allocator }; + db_queue.push(&list_req.base); + list_req.base.wait(self.io, self.shutdown); + + if (list_req.base.err != null or list_req.result == null) { + log.warn("reconcile: failed to load active hosts", .{}); + continue; + } + + const hosts = list_req.result.?; + defer { + for (hosts) |h| { + self.allocator.free(h.hostname); + self.allocator.free(h.status); + } + self.allocator.free(hosts); + } + + var respawned: usize = 0; + for (hosts) |host| { + if (self.shutdown.load(.acquire)) break; + + // check if already running + const has_worker = blk: { + self.workers_mutex.lockUncancelable(self.io); + defer self.workers_mutex.unlock(self.io); + break :blk self.workers.contains(host.id); + }; + if (has_worker) continue; + + self.spawnWorker(host.id, host.hostname, host.last_seq) catch |err| { + log.warn("reconcile: failed to respawn {s}: {s}", .{ host.hostname, @errorName(err) }); + continue; + }; + respawned += 1; + } + + if (respawned > 0) { + log.info("reconcile: respawned {d} host(s)", .{respawned}); + } + } + } + /// background thread: process crawl requests fn processCrawlQueue(self: *Slurper) void { while (!self.shutdown.load(.acquire)) { @@ -745,6 +822,7 @@ pub const Slurper = struct { if (self.startup_future) |*f| f.cancel(self.io); self.crawl_cond.signal(self.io); if (self.crawl_future) |*f| f.cancel(self.io); + if (self.reconcile_future) |*f| f.cancel(self.io); // collect futures to cancel (can't cancel while holding workers_mutex) var futures_to_cancel: std.ArrayListUnmanaged(Io.Future(void)) = .empty; diff --git a/src/subscriber.zig b/src/subscriber.zig index d78b35e..38eae7a 100644 --- a/src/subscriber.zig +++ b/src/subscriber.zig @@ -229,6 +229,9 @@ pub const Subscriber = struct { last_upstream_seq: ?u64 = null, last_cursor_flush: i64 = 0, rate_limiter: RateLimiter = .{}, + /// exponential backoff (seconds) between reconnect attempts — persists across retries, + /// reset to 1 on successful connect. max 1800s (30 min) for dormant-eligible hosts. + backoff: u64 = 1, // per-host shutdown (e.g. FutureCursor — stops only this subscriber) host_shutdown: std.atomic.Value(bool) = .{ .raw = false }, @@ -266,10 +269,9 @@ pub const Subscriber = struct { } /// run the subscriber loop. reconnects with exponential backoff. - /// blocks until shutdown flag is set or host is exhausted. + /// blocks until shutdown or host marked dormant. pub fn run(self: *Subscriber) void { - var backoff: u64 = 1; - const max_backoff: u64 = 60; + const max_backoff: u64 = 1800; // cursor is set at spawn time by slurper if (self.last_upstream_seq) |seq| { @@ -293,7 +295,7 @@ pub const Subscriber = struct { self.connectAndRead() catch |err| { if (self.shouldStop()) return; - log.err("host {s}: error: {s}, reconnecting in {d}s...", .{ self.options.hostname, @errorName(err), backoff }); + log.err("host {s}: error: {s}, reconnecting in {d}s...", .{ self.options.hostname, @errorName(err), self.backoff }); }; if (self.shouldStop()) return; @@ -310,13 +312,13 @@ pub const Subscriber = struct { } // backoff sleep in small increments so we can check shutdown - var remaining: u64 = backoff; + var remaining: u64 = self.backoff; while (remaining > 0 and !self.shouldStop()) { const chunk = @min(remaining, 1); self.io.sleep(Io.Duration.fromSeconds(@intCast(chunk)), .awake) catch {}; remaining -= chunk; } - backoff = @min(backoff * 2, max_backoff); + self.backoff = @min(self.backoff * 2, max_backoff); } } @@ -366,7 +368,8 @@ pub const Subscriber = struct { try client.handshake(path, .{ .headers = host_header }); log.info("host {s}: connected", .{self.options.hostname}); - // reset failures on successful connect (via host_ops queue) + // reset failures + backoff on successful connect (via host_ops queue) + self.backoff = 1; if (self.options.host_id > 0) { if (self.host_ops) |hq| { hq.push(.{