From bf242a6033bd461c2771590ae43a363154ad7dca Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sun, 3 May 2026 11:28:45 -0500 Subject: [PATCH] host_authority: per-branch reject breakdown + sampled warn log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit restores the observability piece of ee4e368 (collaterally reverted with host retention on 39a0886). pure counter plumbing — no behavior change. splits failed_host_authority into 6 branches: parse_did, resolve, no_endpoint, bad_url, unknown_host, host_mismatch emitted as relay_host_authority_reject{branch=...}. invariant: sum(branches) == failed_host_authority. needed before acting on the post-attack handoff's ask #2 — the proposed "gate on cache miss" assumes a cache-miss-then-broadcast model that doesn't exist for host_authority (only for signing keys). once this is in prod, host_mismatch rate vs others tells us whether the 740-DID lead is stale-cached forgeries (no_check path) or genuine resolver failures. Co-Authored-By: Claude Opus 4 (1M context) --- src/broadcaster.zig | 25 ++++++++++++++++++++ src/validator.zig | 57 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/src/broadcaster.zig b/src/broadcaster.zig index 7701d73..a14ed46 100644 --- a/src/broadcaster.zig +++ b/src/broadcaster.zig @@ -55,6 +55,16 @@ pub const Stats = struct { host_authority_is_new: std.atomic.Value(u64) = .{ .raw = 0 }, host_authority_host_changed: std.atomic.Value(u64) = .{ .raw = 0 }, host_authority_time_us: std.atomic.Value(u64) = .{ .raw = 0 }, + // per-branch reject breakdown (subsets of failed_host_authority). + // lets us tell whether rejects come from stale cached forgeries + // (host_mismatch), genuinely forged DIDs (resolve/no_endpoint), + // or a mismatch between DID doc endpoint and our host table. + host_authority_reject_parse_did: std.atomic.Value(u64) = .{ .raw = 0 }, + host_authority_reject_resolve: std.atomic.Value(u64) = .{ .raw = 0 }, + host_authority_reject_no_endpoint: std.atomic.Value(u64) = .{ .raw = 0 }, + host_authority_reject_bad_url: std.atomic.Value(u64) = .{ .raw = 0 }, + host_authority_reject_unknown_host: std.atomic.Value(u64) = .{ .raw = 0 }, + host_authority_reject_host_mismatch: std.atomic.Value(u64) = .{ .raw = 0 }, // frame pool memory pressure pool_queued_bytes: std.atomic.Value(u64) = .{ .raw = 0 }, // persist/broadcast pipeline contention @@ -1075,6 +1085,15 @@ pub fn formatPrometheusMetrics(stats: *const Stats, cache_entries: usize, attrib \\relay_validation_failed{{reason="host_authority"}} {d} \\relay_validation_failed{{reason="future_rev"}} {d} \\ + \\# TYPE relay_host_authority_reject counter + \\# HELP relay_host_authority_reject host authority reject breakdown by branch + \\relay_host_authority_reject{{branch="parse_did"}} {d} + \\relay_host_authority_reject{{branch="resolve"}} {d} + \\relay_host_authority_reject{{branch="no_endpoint"}} {d} + \\relay_host_authority_reject{{branch="bad_url"}} {d} + \\relay_host_authority_reject{{branch="unknown_host"}} {d} + \\relay_host_authority_reject{{branch="host_mismatch"}} {d} + \\ , .{ stats.failed_bad_did.load(.acquire), stats.failed_bad_rev.load(.acquire), @@ -1083,6 +1102,12 @@ pub fn formatPrometheusMetrics(stats: *const Stats, cache_entries: usize, attrib stats.failed_bad_structure.load(.acquire), stats.failed_host_authority.load(.acquire), stats.failed_future_rev.load(.acquire), + stats.host_authority_reject_parse_did.load(.acquire), + stats.host_authority_reject_resolve.load(.acquire), + stats.host_authority_reject_no_endpoint.load(.acquire), + stats.host_authority_reject_bad_url.load(.acquire), + stats.host_authority_reject_unknown_host.load(.acquire), + stats.host_authority_reject_host_mismatch.load(.acquire), }) catch return w.buffered(); // memory attribution — internal capacities help identify what's consuming RSS diff --git a/src/validator.zig b/src/validator.zig index c1d453c..49118e9 100644 --- a/src/validator.zig +++ b/src/validator.zig @@ -551,7 +551,10 @@ pub const Validator = struct { /// .reject — DID doc does not confirm, caller should drop the event pub fn resolveHostAuthority(self: *Validator, did: []const u8, incoming_host_id: u64) HostAuthority { const persist = self.persist orelse return .migrate; // no DB — can't check - const parsed = zat.Did.parse(did) orelse return .reject; + const parsed = zat.Did.parse(did) orelse { + _ = self.stats.host_authority_reject_parse_did.fetchAdd(1, .monotonic); + return .reject; + }; const idx = self.acquireHostResolver(); defer self.releaseHostResolver(idx); @@ -561,12 +564,15 @@ pub const Validator = struct { // first resolve attempt var doc = resolver.resolve(parsed) catch { // retry once on network failure - var doc2 = resolver.resolve(parsed) catch return .reject; + var doc2 = resolver.resolve(parsed) catch { + _ = self.stats.host_authority_reject_resolve.fetchAdd(1, .monotonic); + return .reject; + }; defer doc2.deinit(); - return self.checkPdsHost(&doc2, persist, incoming_host_id); + return self.checkPdsHost(&doc2, persist, did, incoming_host_id); }; defer doc.deinit(); - return self.checkPdsHost(&doc, persist, incoming_host_id); + return self.checkPdsHost(&doc, persist, did, incoming_host_id); } /// acquire a resolver from the pool. spins until one is available. @@ -586,14 +592,47 @@ pub const Validator = struct { self.host_resolver_available[idx].store(true, .release); } - fn checkPdsHost(self: *Validator, doc: *zat.DidDocument, persist: *event_log_mod.DiskPersist, incoming_host_id: u64) HostAuthority { - _ = self; - const pds_endpoint = doc.pdsEndpoint() orelse return .reject; - const pds_host = extractHostFromUrl(pds_endpoint) orelse return .reject; - const pds_host_id = (persist.getHostIdForHostname(pds_host) catch null) orelse return .reject; + fn checkPdsHost(self: *Validator, doc: *zat.DidDocument, persist: *event_log_mod.DiskPersist, did: []const u8, incoming_host_id: u64) HostAuthority { + const pds_endpoint = doc.pdsEndpoint() orelse { + _ = self.stats.host_authority_reject_no_endpoint.fetchAdd(1, .monotonic); + self.sampleLogReject("no_endpoint", did, "", incoming_host_id, 0); + return .reject; + }; + const pds_host = extractHostFromUrl(pds_endpoint) orelse { + _ = self.stats.host_authority_reject_bad_url.fetchAdd(1, .monotonic); + self.sampleLogReject("bad_url", did, pds_endpoint, incoming_host_id, 0); + return .reject; + }; + const pds_host_id = (persist.getHostIdForHostname(pds_host) catch null) orelse { + _ = self.stats.host_authority_reject_unknown_host.fetchAdd(1, .monotonic); + self.sampleLogReject("unknown_host", did, pds_host, incoming_host_id, 0); + return .reject; + }; if (pds_host_id == incoming_host_id) return .migrate; + _ = self.stats.host_authority_reject_host_mismatch.fetchAdd(1, .monotonic); + self.sampleLogReject("host_mismatch", did, pds_host, incoming_host_id, pds_host_id); return .reject; } + + /// log a rejection sample at 1-in-2048 rate. at ~10 rejections/sec that's + /// one log line every ~3.5min. total rejections per branch are available + /// via relay_host_authority_reject{branch=...} in prometheus. + fn sampleLogReject( + self: *Validator, + branch: []const u8, + did: []const u8, + detail: []const u8, + incoming_host_id: u64, + resolved_host_id: u64, + ) void { + const count = self.stats.failed_host_authority.load(.monotonic); + // parens mandatory: `&` and `!=` precedence differs from C in zig. + if ((count & 0x7ff) != 0) return; + log.warn( + "host_authority reject branch={s} did={s} detail={s} incoming_host_id={d} resolved_host_id={d}", + .{ branch, did, detail, incoming_host_id, resolved_host_id }, + ); + } }; /// extract hostname from a URL like "https://pds.example.com" or "https://pds.example.com:443/path" -- 2.51.2