From d41f71fc714697d64605afe53a97274ecd2afa60 Mon Sep 17 00:00:00 2001 From: Guido X Jansen Date: Wed, 2 Sep 2026 02:21:58 +0300 Subject: [PATCH] fix(stats): send a User-Agent so the extrapolation anchor stops returning null bsky-search.jazco.io/stats rejects urllib's default "Python-urllib/3.x" with a 403, so bluesky_total_users has been null in every daily run since at least 2026-08-30 -- the anchor the whole estimate extrapolates from, silently absent. curl from the same host returns 200, which is what made it look like a network problem rather than a header one. Adds a shared fetch_json() with a real User-Agent and routes the AppView handle resolution through it too. Verified on the NAS: anchor now returns 46.293.001. --- scripts/estimate-nl-users.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/estimate-nl-users.py b/scripts/estimate-nl-users.py index 4ae5e11..1a6ae8f 100644 --- a/scripts/estimate-nl-users.py +++ b/scripts/estimate-nl-users.py @@ -116,11 +116,22 @@ DURATION, OUT, LANGS, AUTHOR_LANGS = parse_args(sys.argv[1:]) FOREVER = DURATION <= 0 # 0 (or negative) = run indefinitely +# urllib announces itself as "Python-urllib/3.x", which bsky-search.jazco.io rejects with +# a 403. That silently nulled bluesky_total_users -- the extrapolation anchor -- from at +# least 2026-08-30 until it was caught on 2026-09-02. Always send a real User-Agent. +USER_AGENT = "atproto.nl-stats/1.0 (+https://atproto.nl)" + + +def fetch_json(url, timeout=20): + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r) + + def bluesky_total(): """Current Bluesky account total (extrapolation anchor).""" try: - with urllib.request.urlopen("https://bsky-search.jazco.io/stats", timeout=20) as r: - return json.load(r).get("total_users") + return fetch_json("https://bsky-search.jazco.io/stats").get("total_users") except Exception as e: print(f" bluesky total fetch failed: {e}", file=sys.stderr) return None @@ -135,8 +146,7 @@ def resolve_handles(dids): batch = dids[i:i + 25] qs = "&".join("actors=" + urllib.parse.quote(d) for d in batch) try: - with urllib.request.urlopen(f"{base}?{qs}", timeout=20) as r: - data = json.load(r) + data = fetch_json(f"{base}?{qs}") for p in data.get("profiles", []): handles[p["did"]] = p.get("handle", "") except Exception as e: -- 2.51.2