diff --git a/scripts/estimate-nl-users.py b/scripts/estimate-nl-users.py index 4ae5e11..1a6ae8f 100644 --- a/scripts/estimate-nl-users.py +++ b/scripts/estimate-nl-users.py @@ -116,11 +116,22 @@ DURATION, OUT, LANGS, AUTHOR_LANGS = parse_args(sys.argv[1:]) FOREVER = DURATION <= 0 # 0 (or negative) = run indefinitely +# urllib announces itself as "Python-urllib/3.x", which bsky-search.jazco.io rejects with +# a 403. That silently nulled bluesky_total_users -- the extrapolation anchor -- from at +# least 2026-08-30 until it was caught on 2026-09-02. Always send a real User-Agent. +USER_AGENT = "atproto.nl-stats/1.0 (+https://atproto.nl)" + + +def fetch_json(url, timeout=20): + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r) + + def bluesky_total(): """Current Bluesky account total (extrapolation anchor).""" try: - with urllib.request.urlopen("https://bsky-search.jazco.io/stats", timeout=20) as r: - return json.load(r).get("total_users") + return fetch_json("https://bsky-search.jazco.io/stats").get("total_users") except Exception as e: print(f" bluesky total fetch failed: {e}", file=sys.stderr) return None @@ -135,8 +146,7 @@ def resolve_handles(dids): batch = dids[i:i + 25] qs = "&".join("actors=" + urllib.parse.quote(d) for d in batch) try: - with urllib.request.urlopen(f"{base}?{qs}", timeout=20) as r: - data = json.load(r) + data = fetch_json(f"{base}?{qs}") for p in data.get("profiles", []): handles[p["did"]] = p.get("handle", "") except Exception as e: