From 584571aaba376bf7b18babc13637c96056d11f7f Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Thu, 09 Apr 2026 04:32:10 +0000 Subject: [PATCH] disable keep_alive on host authority resolver pool + log resolve errors 100% of host_authority rejects on 2026-04-08 were in the resolve branch (39,621 / 40,072 over 48min). plc.directory is reachable from the pod, cold resolvers in resolveLoop work fine, and websockets to 2785 PDSes are healthy — isolates the failure to the pooled + long-lived keep_alive HTTP path. pool was added on 0.15 (1639565) and never re-validated after the 0.16 migration (9cc1ba3). workaround: disable keep_alive on the pool. cost is one TLS handshake per is_new / host_changed DID, which is low-rate enough to absorb. keep the pool itself for socket churn savings across fiber callers. also wire sampleLogReject into the resolve and parse_did branches with @errorName of the resolver error — previous commit incremented counters for those branches but never logged, so we had no diagnostic data when the reject rate spiked. if the workaround doesn't fully fix it we now see the actual error kind without a second redeploy cycle. --- src/validator.zig | 31 +++++++++++++++++++++++++++---- 1 file(s) changed, 27 insertion(s)(+), 4 deletion(s)(-) diff --git a/src/validator.zig b/src/validator.zig --- a/src/validator.zig +++ b/src/validator.zig @@ -122,9 +122,25 @@ slot.* = try self.io.concurrent(resolveLoop, .{self}); } - // init host authority resolver pool (reused across calls) + // init host authority resolver pool (reused across calls). + // + // keep_alive = false: workaround for 100% rejection rate observed + // 2026-04-08. hypothesis is that zig 0.16 std.http.Client doesn't + // recover stale keep-alive connections on the pooled resolvers — + // pool was added 2026-03-18 on zig 0.15, never re-validated after + // the 0.16 migration on 2026-04-05. plc.directory is reachable + // from the pod and cold resolvers (resolveLoop) work fine, so it's + // specifically the pooled + long-lived keep_alive path. + // + // cost: one TLS handshake per host authority check (~tens of ms). + // host authority checks only fire on is_new or host_changed, so the + // steady-state rate is low. keep the pool for the socket churn + // savings across multiple fiber callers even without keep_alive. + // + // TODO: remove once upstream zig fix lands. file issue when we + // have the actual error kind from the sampled warn logs below. for (&self.host_resolvers) |*r| { - r.* = zat.DidResolver.initWithOptions(self.io, self.allocator, .{}); + r.* = zat.DidResolver.initWithOptions(self.io, self.allocator, .{ .keep_alive = false }); } for (&self.host_resolver_available) |*a| { a.store(true, .release); @@ -553,6 +569,7 @@ const persist = self.persist orelse return .migrate; // no DB — can't check const parsed = zat.Did.parse(did) orelse { _ = self.stats.host_authority_reject_parse_did.fetchAdd(1, .monotonic); + self.sampleLogReject("parse_did", did, "", incoming_host_id, 0); return .reject; }; @@ -562,10 +579,16 @@ var resolver = &self.host_resolvers[idx]; // first resolve attempt - var doc = resolver.resolve(parsed) catch { + var doc = resolver.resolve(parsed) catch |err1| { // retry once on network failure - var doc2 = resolver.resolve(parsed) catch { + var doc2 = resolver.resolve(parsed) catch |err2| { _ = self.stats.host_authority_reject_resolve.fetchAdd(1, .monotonic); + // log the second-attempt error kind — first-attempt kind is + // dropped because resolver.resolve already swallows it into + // DidResolutionFailed upstream, so both errors look the same + // here. detail field captures @errorName for upstream triage. + self.sampleLogReject("resolve", did, @errorName(err2), incoming_host_id, 0); + _ = err1; return .reject; }; defer doc2.deinit(); -- tangled.sh