diff --git a/src/internal/api/admin.zig b/src/internal/api/admin.zig index 5918222..94bbc39 100644 --- a/src/internal/api/admin.zig +++ b/src/internal/api/admin.zig @@ -297,6 +297,67 @@ pub fn handleAdminUnblockHost(conn: *h.Conn, body: []const u8, headers: *const w h.respondJson(conn, .ok, "{\"success\":true}"); } +/// force a host's worker to drop and re-establish its connection. +/// +/// In-band recovery for a worker that is alive but deaf. Unlike +/// block+unblock this never changes the host's status, so a host cannot be +/// left stranded as blocked if the operator's second call does not land. +pub fn handleAdminReconnectHost(conn: *h.Conn, body: []const u8, headers: *const websocket.Handshake.KeyValue, ctx: *HttpContext) void { + if (!checkAdmin(conn, headers)) return; + + const parsed = std.json.parseFromSlice(struct { hostname: []const u8 }, ctx.persist.allocator, body, .{ .ignore_unknown_fields = true }) catch { + h.respondJson(conn, .bad_request, "{\"error\":\"BadRequest\",\"message\":\"invalid JSON\"}"); + return; + }; + defer parsed.deinit(); + + // resolve the host WITHOUT creating it: reconnecting a host we have never + // seen is a caller error, not a reason to add one. + const LookupReq = struct { + base: DbRequest = .{ .callback = &execute }, + hostname_buf: [256]u8 = undefined, + hostname_len: usize = 0, + host_id: u64 = 0, + found: bool = false, + + fn execute(b: *DbRequest, dp: *DiskPersist) void { + const self: *@This() = @fieldParentPtr("base", b); + const hn = self.hostname_buf[0..self.hostname_len]; + const maybe_id = dp.getHostIdForHostname(hn) catch |e| { + b.err = e; + return; + }; + if (maybe_id) |id| { + self.host_id = id; + self.found = true; + } + } + }; + var req: LookupReq = .{}; + const copy_len = @min(parsed.value.hostname.len, req.hostname_buf.len); + @memcpy(req.hostname_buf[0..copy_len], parsed.value.hostname[0..copy_len]); + req.hostname_len = copy_len; + ctx.db_queue.push(&req.base); + req.base.wait(ctx.io, ctx.shutdown); + + if (req.base.err != null) { + h.respondJson(conn, .internal_server_error, "{\"error\":\"DatabaseError\",\"message\":\"operation failed\"}"); + return; + } + if (!req.found) { + h.respondJson(conn, .not_found, "{\"error\":\"NotFound\",\"message\":\"unknown host\"}"); + return; + } + + if (!ctx.slurper.forceReconnect(req.host_id)) { + h.respondJson(conn, .not_found, "{\"error\":\"NoWorker\",\"message\":\"host has no active worker\"}"); + return; + } + + log.info("admin: forced reconnect for host {s} (id={d})", .{ parsed.value.hostname, req.host_id }); + h.respondJson(conn, .ok, "{\"success\":true}"); +} + /// set or clear the account_limit override for a host. pub fn handleAdminChangeLimits(conn: *h.Conn, body: []const u8, headers: *const websocket.Handshake.KeyValue, ctx: *HttpContext) void { if (!checkAdmin(conn, headers)) return; diff --git a/src/internal/api/router.zig b/src/internal/api/router.zig index 2a00aba..7a6c79d 100644 --- a/src/internal/api/router.zig +++ b/src/internal/api/router.zig @@ -134,6 +134,8 @@ fn handlePost(conn: *websocket.Conn, path: []const u8, query: []const u8, body: admin.handleAdminBlockHost(conn, body, headers, ctx); } else if (std.mem.eql(u8, path, "/admin/hosts/unblock")) { admin.handleAdminUnblockHost(conn, body, headers, ctx); + } else if (std.mem.eql(u8, path, "/admin/hosts/reconnect")) { + admin.handleAdminReconnectHost(conn, body, headers, ctx); } else if (std.mem.eql(u8, path, "/admin/hosts/changeLimits")) { admin.handleAdminChangeLimits(conn, body, headers, ctx); } else if (std.mem.eql(u8, path, "/admin/backfill-collections")) { diff --git a/src/internal/slurper.zig b/src/internal/slurper.zig index 0642b46..454a546 100644 --- a/src/internal/slurper.zig +++ b/src/internal/slurper.zig @@ -564,6 +564,53 @@ pub const Slurper = struct { } } + /// force a host's worker to drop and re-establish its connection. + /// + /// This is the in-band recovery for a worker that is alive but deaf -- + /// parked forever on a read that will never complete (see the 2026-08-18 + /// incident). Neither existing path could do it: `addHost` dedupes on + /// `workers.contains`, so requestCrawl is a no-op while the dead worker + /// still exists, and admin block/unblock only writes DB status, leaving + /// the host stranded as blocked without ever tearing the worker down. + /// + /// Cancelling is what actually reaches a wedged fiber: a flag cannot wake + /// a fiber parked in netRead, but cancellation readies it and makes the + /// read return `error.Canceled`. `host_shutdown` is set first so a + /// *healthy* worker exits its reconnect loop cleanly instead of racing. + /// `future.cancel` awaits completion, so by the time it returns runWorker + /// has already freed the cursor slot, dropped the map entry and + /// decremented `connected_inbound` -- which is why the respawn below no + /// longer trips the dedup. + /// + /// Returns false if the host has no worker. + pub fn forceReconnect(self: *Slurper, host_id: u64) bool { + var hostname_buf: [256]u8 = undefined; + var hostname_len: usize = 0; + var future: Io.Future(void) = undefined; + { + self.workers_mutex.lockUncancelable(self.io); + defer self.workers_mutex.unlock(self.io); + const entry = self.workers.get(host_id) orelse return false; + // copy the hostname now: runWorker frees it once the future ends. + const hn = entry.subscriber.options.hostname; + hostname_len = @min(hn.len, hostname_buf.len); + @memcpy(hostname_buf[0..hostname_len], hn[0..hostname_len]); + entry.subscriber.host_shutdown.store(true, .release); + future = entry.future; + } + + // must not hold workers_mutex here: runWorker takes it on the way out. + future.cancel(self.io); + + const hostname = hostname_buf[0..hostname_len]; + self.addCrawlRequest(hostname) catch |e| { + log.err("forceReconnect: host_id={d} ({s}) torn down but respawn enqueue failed: {s}", .{ host_id, hostname, @errorName(e) }); + return false; + }; + log.info("forceReconnect: host_id={d} ({s}) worker torn down, respawn queued", .{ host_id, hostname }); + return true; + } + /// shutdown all workers and clean up pub fn deinit(self: *Slurper) void { // cancel background tasks