diff --git a/scripts/README.md b/scripts/README.md index 57146cb..65cedb6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -341,3 +341,41 @@ another detector, so the union drops from 9.441 to 9.112 (−3,5%). Non-destructive: it writes a cleaned file, never the live feeder, unless `--out` points there. + +## nl-did-facts.py + nl-lexicons.py + nl-facts-aggregate.py + +History we can read today instead of accumulating forward. Every `did:plc` identity has +a public audit log, and every repo lists the collections it holds. + +```sh +uv run scripts/nl-did-facts.py --dids nl-first-seen.jsonl --out nl-did-facts.jsonl +uv run scripts/nl-lexicons.py --facts nl-did-facts.jsonl --out nl-lexicons.jsonl --stale-days 30 +uv run scripts/nl-facts-aggregate.py --dir . --out nl-facts.json +``` + +- **`nl-did-facts.py`** — one audit-log request per DID yields three things: `createdAt` + (the first operation), every handle the DID has ever had with the date it took it, and + the PDS it lives on now. Supersedes the creation-date-only version. +- **`nl-lexicons.py`** — `com.atproto.repo.describeRepo` against each account's own PDS + lists its collections, which is a direct answer to "which apps does this account use". + It counts **accounts, not records**: a repo listing says a collection exists, not how + much is in it. `--stale-days 30` re-reads a repo monthly, which is how new adoption + surfaces. +- **`nl-facts-aggregate.py`** — folds both into `nl-facts.json` (counts only, no DIDs, + no handles) for `src/data/nl-facts.json`. + +Both collectors are resumable and append-only, so an interrupted run costs nothing and a +weekly re-run only fetches what is new. `nl-collector-day.sh` runs them on Sundays; +creation dates and PDS moves are rare and app adoption is slow, so daily would be wasted +requests against other people's infrastructure. + +Two things the aggregate deliberately does: + +- Bluesky's hosted fleet (`*.host.bsky.network`) collapses to one operator. Counting each + shard separately would say "spread across 40 hosts", which is false. +- A lexicon used by fewer than three accounts is dropped. Naming a one-account lexicon + invites over-reading a coincidence. + +**What this population is:** every account OUR detectors identified as Dutch. A creation +curve from it shows when the Dutch accounts we can see joined, not when Dutch people +joined atproto, and detection favours the visible. The page says so where it renders it. diff --git a/scripts/nl-collector-day.sh b/scripts/nl-collector-day.sh index 95f3862..607e27d 100755 --- a/scripts/nl-collector-day.sh +++ b/scripts/nl-collector-day.sh @@ -54,6 +54,27 @@ fi cp "$DIR/nl-live.json" "$DIR/days/nl-$(date -u +%F).json" echo "[$DAY] done; archived to days/nl-$(date -u +%F).json" >&2 +# Once a week, re-read what the directory and the repos say about every identified +# account: creation dates and PDS move rarely, and app adoption changes slowly, so +# daily would be wasted requests against other people's infrastructure. Sunday. +# +# Both scripts are resumable and append-only, so a run interrupted by a reboot simply +# continues the next week rather than starting over. +if [ "$(date -u +%u)" = "7" ] && [ -f "$SCRIPTS_DIR/nl-did-facts.py" ]; then + echo "[$DAY] weekly: refreshing DID facts + lexicon usage" >&2 + python3 "$SCRIPTS_DIR/nl-did-facts.py" --dids "$DIR/nl-first-seen.jsonl" \ + --out "$DIR/nl-did-facts.jsonl" >&2 || echo "[$DAY] did-facts failed; continuing" >&2 + # --stale-days 30: re-check a repo monthly, which is how new app adoption surfaces. + python3 "$SCRIPTS_DIR/nl-lexicons.py" --facts "$DIR/nl-did-facts.jsonl" \ + --out "$DIR/nl-lexicons.jsonl" --stale-days 30 >&2 || echo "[$DAY] lexicons failed; continuing" >&2 +fi + +# Cheap, and keeps nl-facts.json in step with whatever the two files above hold. +if [ -f "$SCRIPTS_DIR/nl-facts-aggregate.py" ]; then + python3 "$SCRIPTS_DIR/nl-facts-aggregate.py" --dir "$DIR" --out "$DIR/nl-facts.json" >&2 \ + || echo "[$DAY] facts-aggregate failed; continuing" >&2 +fi + # Prune the nlHandle feeder. It is a current-state snapshot by design -- a lapsed .nl # handle should drop out -- but the scanner only ever appends, so without this the floor # accumulates accounts that no longer meet the criterion that admitted them. The first diff --git a/scripts/nl-did-facts.py b/scripts/nl-did-facts.py new file mode 100644 index 0000000..21da467 --- /dev/null +++ b/scripts/nl-did-facts.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [] +# /// +"""Read what the PLC directory already knows about every identified Dutch account. + +One audit-log request per DID yields three things that would otherwise take months to +accumulate forward: + + createdAt the first operation in the log: when the account was created + handles every handle the DID has ever had, with the date it took it + pds the PDS it lives on today + +None of this needs the firehose and none of it needs waiting: the history is already +recorded in the directory. Supersedes nl-did-created.py, which read the same log for the +creation date alone. + + uv run scripts/nl-did-facts.py --dids nl-first-seen.jsonl --out nl-did-facts.jsonl + uv run scripts/nl-did-facts.py --out nl-did-facts.jsonl --summary + uv run scripts/nl-did-facts.py --self-test + +Resumable and append-only: DIDs already in the output are skipped, so an interrupted run +costs nothing and a weekly re-run only fetches what is new. Pass --refresh to re-read +DIDs already present (handles and PDS change; creation dates do not). + +What these numbers are, and are not: + + The population is every account OUR detectors identified as Dutch. A creation curve + built from it shows when the Dutch accounts we can see joined, NOT when Dutch people + joined atproto: every account we never detect is missing, and detection favours the + visible. Treat it as a shape, not a census. + + did:web identities have no PLC log and are recorded with a null createdAt. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from collections import Counter + +PLC = "https://plc.directory" +UA = "atproto.nl-did-facts/1.0" + + +def read_dids(path: str) -> list[str]: + """Accepts a plain .dids file or JSONL rows carrying a `did`.""" + out: list[str] = [] + if not path or not os.path.exists(path): + return out + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("{"): + try: + did = json.loads(line).get("did") + except ValueError: + continue + if did: + out.append(did) + elif line.startswith("did:"): + out.append(line) + return out + + +def read_facts(path: str) -> dict[str, dict]: + facts: dict[str, dict] = {} + if not path or not os.path.exists(path): + return facts + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except ValueError: + continue # a partial line from an interrupted run + if row.get("did"): + facts[row["did"]] = row # later lines win, so --refresh overwrites + return facts + + +def parse_log(did: str, log: list) -> dict: + """Pull creation, handle history and current PDS out of one audit log.""" + created = None + handles: list[dict] = [] + pds = None + last_handle = None + for op in log: + if op.get("nullified"): + continue # a rotated-away branch of history: never happened, effectively + stamp = op.get("createdAt") + if stamp and (created is None or stamp < created): + created = stamp + operation = op.get("operation") or {} + + aka = operation.get("alsoKnownAs") or [] + handle = None + for entry in aka: + if isinstance(entry, str) and entry.startswith("at://"): + handle = entry[len("at://") :] + break + # Legacy create ops carry the handle directly instead of in alsoKnownAs. + handle = handle or operation.get("handle") + if handle and handle != last_handle: + handles.append({"handle": handle, "at": stamp}) + last_handle = handle + + services = operation.get("services") or {} + atproto = services.get("atproto_pds") or {} + endpoint = atproto.get("endpoint") or operation.get("service") + if endpoint: + pds = endpoint + return {"did": did, "createdAt": created, "handles": handles, "pds": pds} + + +def fetch_facts(did: str, retries: int = 4) -> dict: + url = f"{PLC}/{urllib.parse.quote(did, safe=':')}/log/audit" + for attempt in range(retries): + try: + req = urllib.request.Request(url, headers={"accept": "application/json", "user-agent": UA}) + with urllib.request.urlopen(req, timeout=30) as resp: + log = json.load(resp) + if isinstance(log, list) and log: + return parse_log(did, log) + return {"did": did, "createdAt": None, "handles": [], "pds": None} + except urllib.error.HTTPError as exc: + if exc.code in (400, 404, 410): + return {"did": did, "createdAt": None, "handles": [], "pds": None} + if exc.code not in (429, 500, 502, 503, 504): + raise + time.sleep(2**attempt) + except Exception: + time.sleep(2**attempt) + return {"did": did, "createdAt": None, "handles": [], "pds": None} + + +def host_of(endpoint: str | None) -> str | None: + if not endpoint: + return None + try: + return urllib.parse.urlparse(endpoint).hostname + except ValueError: + return None + + +def summarise(path: str) -> int: + facts = read_facts(path) + dated = [f for f in facts.values() if f.get("createdAt")] + by_month = Counter(f["createdAt"][:7] for f in dated) + hosts = Counter(host_of(f.get("pds")) or "(unknown)" for f in facts.values()) + print(f"{len(facts)} DIDs, {len(dated)} with a creation date") + for month in sorted(by_month): + print(f" {month} {by_month[month]:>5} {'#' * min(60, by_month[month] // 20 + 1)}") + print("\ntop PDS hosts:") + for host, n in hosts.most_common(12): + print(f" {n:>5} {host}") + return 0 + + +def self_test() -> int: + log = [ + { + "createdAt": "2023-05-01T00:00:00.000Z", + "operation": {"handle": "old.bsky.social", "service": "https://bsky.social"}, + }, + { + "createdAt": "2024-11-02T00:00:00.000Z", + "operation": { + "alsoKnownAs": ["at://new.nl"], + "services": {"atproto_pds": {"endpoint": "https://pds.example.nl"}}, + }, + }, + { # a nullified branch must not contribute anything + "nullified": True, + "createdAt": "2020-01-01T00:00:00.000Z", + "operation": {"alsoKnownAs": ["at://wrong.example"]}, + }, + ] + facts = parse_log("did:plc:x", log) + assert facts["createdAt"] == "2023-05-01T00:00:00.000Z", facts + assert [h["handle"] for h in facts["handles"]] == ["old.bsky.social", "new.nl"], facts + assert facts["pds"] == "https://pds.example.nl", facts + assert host_of("https://pds.example.nl") == "pds.example.nl" + assert host_of(None) is None + print("self-test ok") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--dids", help="input .dids or .jsonl file") + ap.add_argument("--out", default="nl-did-facts.jsonl") + ap.add_argument("--sleep", type=float, default=0.12) + ap.add_argument("--limit", type=int, default=None) + ap.add_argument("--refresh", action="store_true", help="re-read DIDs already recorded") + ap.add_argument("--summary", action="store_true") + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args() + + if args.self_test: + return self_test() + if args.summary: + return summarise(args.out) + if not args.dids: + ap.error("--dids is required") + + dids = read_dids(args.dids) + known = read_facts(args.out) + todo = dids if args.refresh else [d for d in dids if d not in known] + print(f"{len(dids)} DIDs, {len(known)} recorded, {len(todo)} to fetch", file=sys.stderr) + + done = 0 + with open(args.out, "a", encoding="utf-8") as fh: + for did in todo: + if args.limit and done >= args.limit: + break + fh.write(json.dumps(fetch_facts(did), sort_keys=True) + "\n") + fh.flush() + done += 1 + if done % 250 == 0: + print(f" {done}/{len(todo)}", file=sys.stderr) + time.sleep(args.sleep) + + print(f"[facts] {done} fetched -> {args.out}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/nl-facts-aggregate.py b/scripts/nl-facts-aggregate.py new file mode 100644 index 0000000..7e2ae31 --- /dev/null +++ b/scripts/nl-facts-aggregate.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [] +# /// +"""Fold the per-DID facts into the counts-only JSON the site publishes. + +Reads nl-did-facts.jsonl (creation dates, handle history, PDS) and nl-lexicons.jsonl +(collections per repo), writes nl-facts.json: aggregates only, no DIDs, no handles. Same +rule as nl-stats.json -- the per-account files stay on the collector host. + + uv run scripts/nl-facts-aggregate.py --dir . --out nl-facts.json + uv run scripts/nl-facts-aggregate.py --self-test +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +import urllib.parse +from collections import Counter + +# Bluesky's own collections answer nothing: nearly every account has them. +BSKY_PREFIXES = ("app.bsky.", "chat.bsky.") +# Bluesky's hosted PDS fleet is ..host.bsky.network. Counting each shard +# separately would say "the community is spread over 40 hosts", which is false: they are +# one operator. What matters is hosted-by-Bluesky versus hosted elsewhere. +BSKY_PDS_SUFFIX = ".host.bsky.network" +# A lexicon used by one or two accounts is noise, and naming it invites over-reading. +MIN_LEXICON_ACCOUNTS = 3 + + +def read_jsonl(path: str) -> list[dict]: + rows = [] + if not path or not os.path.exists(path): + return rows + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except ValueError: + continue # a partial line from a run still in flight + return rows + + +def latest_by_did(rows: list[dict]) -> dict[str, dict]: + out: dict[str, dict] = {} + for row in rows: + if row.get("did"): + out[row["did"]] = row # append-only files: the last line is the current one + return out + + +def host_of(endpoint: str | None) -> str | None: + if not endpoint: + return None + try: + return urllib.parse.urlparse(endpoint).hostname + except ValueError: + return None + + +def is_custom_domain(handle: str | None) -> bool: + """A handle the account holder controls, rather than one a PDS handed out.""" + if not handle: + return False + h = handle.lower() + return not (h.endswith(".bsky.social") or h == "handle.invalid") + + +def build(facts: dict[str, dict], lexicons: dict[str, dict], now: str) -> dict: + dated = [f for f in facts.values() if f.get("createdAt")] + creation_by_month = Counter(f["createdAt"][:7] for f in dated) + + hosts = Counter() + bsky_hosted = 0 + for fact in facts.values(): + host = host_of(fact.get("pds")) + if not host: + continue + if host.endswith(BSKY_PDS_SUFFIX) or host == "bsky.social": + bsky_hosted += 1 + else: + hosts[host] += 1 + + # When each account FIRST took a handle it controls. Domain adoption is a maturity + # signal for this community specifically: it is the step from "I have an account" to + # "this is my identity", and it is the same step that makes an account detectable + # as Dutch by handle. + custom_by_month = Counter() + custom_now = 0 + for fact in facts.values(): + handles = fact.get("handles") or [] + for entry in handles: + if is_custom_domain(entry.get("handle")) and entry.get("at"): + custom_by_month[entry["at"][:7]] += 1 + break + if handles and is_custom_domain(handles[-1].get("handle")): + custom_now += 1 + + checked = [row for row in lexicons.values() if row.get("collections") is not None] + lex_counter: Counter[str] = Counter() + beyond_users = 0 + for row in checked: + beyond = [c for c in row["collections"] if not c.startswith(BSKY_PREFIXES)] + if beyond: + beyond_users += 1 + for nsid in beyond: + lex_counter[nsid] += 1 + + return { + "generatedAt": now, + "accounts": { + "known": len(facts), + "dated": len(dated), + "reposChecked": len(checked), + }, + # Every series below covers only the accounts our detectors identified, so it is + # a shape, not a census. The site says so where it renders them. + "creationByMonth": dict(sorted(creation_by_month.items())), + "pds": { + "blueskyHosted": bsky_hosted, + "elsewhere": sum(hosts.values()), + "topOtherHosts": [{"host": h, "accounts": n} for h, n in hosts.most_common(10)], + }, + "handles": { + "customDomainNow": custom_now, + "firstCustomDomainByMonth": dict(sorted(custom_by_month.items())), + }, + "lexicons": { + "accountsUsingSomething": beyond_users, + "byNsid": [ + {"nsid": nsid, "accounts": n} + for nsid, n in lex_counter.most_common() + if n >= MIN_LEXICON_ACCOUNTS + ], + }, + } + + +def self_test() -> int: + facts = { + "did:plc:a": { + "did": "did:plc:a", + "createdAt": "2024-11-05T00:00:00.000Z", + "handles": [ + {"handle": "a.bsky.social", "at": "2024-11-05T00:00:00.000Z"}, + {"handle": "a.nl", "at": "2024-12-01T00:00:00.000Z"}, + ], + "pds": "https://morel.us-east.host.bsky.network", + }, + "did:plc:b": { + "did": "did:plc:b", + "createdAt": "2023-05-01T00:00:00.000Z", + "handles": [{"handle": "b.bsky.social", "at": "2023-05-01T00:00:00.000Z"}], + "pds": "https://pds.example.nl", + }, + "did:plc:c": {"did": "did:plc:c", "createdAt": None, "handles": [], "pds": None}, + } + lexicons = { + "did:plc:a": {"collections": ["app.bsky.feed.post", "id.sifa.profile", "sh.tangled.repo"]}, + "did:plc:b": {"collections": ["app.bsky.feed.post"]}, + "did:plc:c": {"collections": None}, # unreadable repo + } + out = build(facts, lexicons, "2026-08-30T00:00:00Z") + assert out["accounts"] == {"known": 3, "dated": 2, "reposChecked": 2}, out["accounts"] + assert out["creationByMonth"] == {"2023-05": 1, "2024-11": 1}, out["creationByMonth"] + # Bluesky's shards collapse to one operator; a self-hosted PDS is counted separately. + assert out["pds"]["blueskyHosted"] == 1 and out["pds"]["elsewhere"] == 1, out["pds"] + assert out["pds"]["topOtherHosts"] == [{"host": "pds.example.nl", "accounts": 1}], out["pds"] + # 'a' moved to a custom domain in 2024-12; 'b' never did. + assert out["handles"] == { + "customDomainNow": 1, + "firstCustomDomainByMonth": {"2024-12": 1}, + }, out["handles"] + # One account uses something beyond Bluesky, but each lexicon is below the floor. + assert out["lexicons"]["accountsUsingSomething"] == 1, out["lexicons"] + assert out["lexicons"]["byNsid"] == [], out["lexicons"] + print("self-test OK") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--dir", default=".") + ap.add_argument("--facts", default=None) + ap.add_argument("--lexicons", default=None) + ap.add_argument("--out", default="nl-facts.json") + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args() + + if args.self_test: + return self_test() + + facts = latest_by_did(read_jsonl(args.facts or os.path.join(args.dir, "nl-did-facts.jsonl"))) + lexicons = latest_by_did(read_jsonl(args.lexicons or os.path.join(args.dir, "nl-lexicons.jsonl"))) + out = build(facts, lexicons, time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())) + with open(args.out, "w", encoding="utf-8") as fh: + fh.write(json.dumps(out, indent=2, ensure_ascii=False) + "\n") + print( + f"[facts-aggregate] {out['accounts']['known']} accounts, " + f"{out['accounts']['dated']} dated, {out['accounts']['reposChecked']} repos, " + f"{len(out['lexicons']['byNsid'])} lexicons -> {args.out}", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/nl-lexicons.py b/scripts/nl-lexicons.py new file mode 100644 index 0000000..85aaf61 --- /dev/null +++ b/scripts/nl-lexicons.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [] +# /// +"""Which lexicons does the Dutch community actually use, beyond Bluesky? + +`com.atproto.repo.describeRepo` lists the collections present in a repo, which is a +direct answer to "what apps is this account using": Sifa profiles, Smoke Signal events, +WhiteWind blogs, Frontpage posts, Tangled repos, Leaflet documents, Skylights reviews. +One request per account, no firehose, no waiting. + + uv run scripts/nl-lexicons.py --facts nl-did-facts.jsonl --out nl-lexicons.jsonl + uv run scripts/nl-lexicons.py --out nl-lexicons.jsonl --summary + uv run scripts/nl-lexicons.py --self-test + +It reads the PDS endpoint from nl-did-facts.jsonl, because describeRepo is served by the +account's own PDS rather than a central index. Accounts whose PDS is unknown are skipped. + +Resumable and append-only, so the weekly re-run only fetches accounts it has not seen. +`--stale-days N` also re-reads accounts last checked more than N days ago, which is how +new app adoption shows up: the collection list changes when someone starts using an app. + +What it counts: ACCOUNTS, not records. "142 accounts have a Sifa profile" is a claim this +data supports; "142 Sifa profiles were created" is not, because a repo listing says a +collection exists, not how much is in it. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from collections import Counter +from datetime import datetime, timedelta, timezone + +UA = "atproto.nl-lexicons/1.0" + +# Bluesky's own collections are the uninteresting answer: nearly every account has them. +# Kept in the raw data, excluded from the "beyond Bluesky" view. +BSKY_PREFIXES = ("app.bsky.", "chat.bsky.") + + +def read_facts(path: str) -> dict[str, dict]: + out: dict[str, dict] = {} + if not path or not os.path.exists(path): + return out + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except ValueError: + continue + if row.get("did"): + out[row["did"]] = row + return out + + +def read_seen(path: str) -> dict[str, dict]: + seen: dict[str, dict] = {} + if not path or not os.path.exists(path): + return seen + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except ValueError: + continue + if row.get("did"): + seen[row["did"]] = row + return seen + + +def describe_repo(pds: str, did: str, retries: int = 3) -> list[str] | None: + """Collections in this repo, or None if the PDS would not say.""" + url = f"{pds.rstrip('/')}/xrpc/com.atproto.repo.describeRepo?{urllib.parse.urlencode({'repo': did})}" + for attempt in range(retries): + try: + req = urllib.request.Request(url, headers={"accept": "application/json", "user-agent": UA}) + with urllib.request.urlopen(req, timeout=20) as resp: + data = json.load(resp) + cols = data.get("collections") + return sorted(cols) if isinstance(cols, list) else [] + except urllib.error.HTTPError as exc: + # 400 covers deactivated and taken-down repos; not worth retrying. + if exc.code in (400, 404, 410): + return None + if exc.code not in (429, 500, 502, 503, 504): + return None + time.sleep(2**attempt) + except Exception: + time.sleep(2**attempt) + return None + + +def beyond_bluesky(collections: list[str]) -> list[str]: + return [c for c in collections if not c.startswith(BSKY_PREFIXES)] + + +def summarise(path: str) -> int: + rows = read_seen(path) + with_repo = [r for r in rows.values() if r.get("collections")] + counter: Counter[str] = Counter() + for row in with_repo: + for nsid in beyond_bluesky(row["collections"]): + counter[nsid] += 1 + users = sum(1 for r in with_repo if beyond_bluesky(r["collections"])) + print(f"{len(rows)} accounts checked, {len(with_repo)} readable, {users} use something beyond Bluesky") + for nsid, n in counter.most_common(40): + print(f" {n:>5} {nsid}") + return 0 + + +def self_test() -> int: + cols = ["app.bsky.feed.post", "app.bsky.actor.profile", "id.sifa.profile", "sh.tangled.repo"] + assert beyond_bluesky(cols) == ["id.sifa.profile", "sh.tangled.repo"], beyond_bluesky(cols) + assert beyond_bluesky(["chat.bsky.convo"]) == [] + import tempfile + + d = tempfile.mkdtemp() + p = os.path.join(d, "seen.jsonl") + with open(p, "w") as fh: + fh.write(json.dumps({"did": "did:plc:a", "collections": cols, "checkedAt": "2026-08-30"}) + "\n") + fh.write("{broken\n") + seen = read_seen(p) + assert list(seen) == ["did:plc:a"], seen + print("self-test ok") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--facts", default="nl-did-facts.jsonl", help="source of DIDs + their PDS") + ap.add_argument("--out", default="nl-lexicons.jsonl") + ap.add_argument("--sleep", type=float, default=0.1) + ap.add_argument("--limit", type=int, default=None) + ap.add_argument( + "--stale-days", + type=int, + default=None, + help="also re-check accounts last read more than N days ago (weekly refresh)", + ) + ap.add_argument("--summary", action="store_true") + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args() + + if args.self_test: + return self_test() + if args.summary: + return summarise(args.out) + + facts = read_facts(args.facts) + seen = read_seen(args.out) + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + cutoff = None + if args.stale_days: + cutoff = (datetime.now(timezone.utc) - timedelta(days=args.stale_days)).strftime("%Y-%m-%d") + + todo = [] + for did, fact in facts.items(): + if not fact.get("pds"): + continue # describeRepo lives on the PDS; without one there is nothing to ask + prev = seen.get(did) + if prev is None or (cutoff and (prev.get("checkedAt") or "") < cutoff): + todo.append((did, fact["pds"])) + + print(f"{len(facts)} accounts, {len(seen)} already read, {len(todo)} to check", file=sys.stderr) + + done = unreadable = 0 + with open(args.out, "a", encoding="utf-8") as fh: + for did, pds in todo: + if args.limit and done >= args.limit: + break + cols = describe_repo(pds, did) + if cols is None: + unreadable += 1 + fh.write( + json.dumps({"did": did, "collections": cols, "checkedAt": today}, sort_keys=True) + "\n" + ) + fh.flush() + done += 1 + if done % 250 == 0: + print(f" {done}/{len(todo)} ({unreadable} unreadable)", file=sys.stderr) + time.sleep(args.sleep) + + print(f"[lexicons] {done} checked, {unreadable} unreadable -> {args.out}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/publish-nl-stats.sh b/scripts/publish-nl-stats.sh index 184281b..ce1ba38 100755 --- a/scripts/publish-nl-stats.sh +++ b/scripts/publish-nl-stats.sh @@ -69,10 +69,26 @@ else rm -f "$hist_tmp" fi +# The facts file (creation cohorts, PDS split, lexicon usage) rides along the same way: +# counts only, no DIDs, so it is safe to publish. +FACTS_DEST="$REPO/src/data/nl-facts.json" +facts_tmp="$(mktemp)" +if ssh "$NL_STATS_SSH" "cat '$NL_STATS_DIR/nl-facts.json' 2>/dev/null" > "$facts_tmp" && [ -s "$facts_tmp" ]; then + if python3 -c "import json,sys; json.load(open(sys.argv[1]))" "$facts_tmp"; then + mv "$facts_tmp" "$FACTS_DEST" + else + echo "facts pull was not valid JSON; keeping the committed copy" >&2 + rm -f "$facts_tmp" + fi +else + rm -f "$facts_tmp" +fi + cd "$REPO" # The history file only exists once the aggregator on the host has written one. TRACKED=("$DEST") [ -f "$HIST_DEST" ] && TRACKED+=("$HIST_DEST") +[ -f "$FACTS_DEST" ] && TRACKED+=("$FACTS_DEST") if git diff --quiet -- "${TRACKED[@]}"; then echo "nl-stats.json unchanged; nothing to commit." exit 0 diff --git a/src/components/pages/CommunityFacts.astro b/src/components/pages/CommunityFacts.astro new file mode 100644 index 0000000..3b906c2 --- /dev/null +++ b/src/components/pages/CommunityFacts.astro @@ -0,0 +1,277 @@ +--- +// What the accounts themselves say: when they were created, what they run on, and which +// lexicons they use beyond Bluesky. All of it read from the PLC directory and the repos, +// so it is history we could read today rather than history we have to accumulate. +// +// Every block is conditional. Before the first collection run these render nothing at +// all rather than empty axes, and they populate on their own once the data lands. +import { + nlFacts, + hasCreation, + hasLexicons, + hasPds, + hasAnyFacts, + creationSeries, + appLabel, +} from '../../data/nl-facts'; + +interface Props { + lang: 'en' | 'nl'; +} +const { lang } = Astro.props; +const isNl = lang === 'nl'; +const nf = (n: number) => n.toLocaleString('nl-NL'); + +const series = creationSeries(); +const peak = Math.max(1, ...series.map((s) => s.count)); +const lex = nlFacts.lexicons.byNsid.slice(0, 14); +const lexPeak = Math.max(1, ...lex.map((l) => l.accounts)); +const pdsTotal = nlFacts.pds.blueskyHosted + nlFacts.pds.elsewhere; +const elsewherePct = pdsTotal ? Math.round((nlFacts.pds.elsewhere / pdsTotal) * 1000) / 10 : 0; + +// Label roughly every sixth month, so the axis stays readable at any range length. +const tickEvery = Math.max(1, Math.ceil(series.length / 8)); + +const t = { + h2: isNl ? 'Wat deze accounts laten zien' : 'What these accounts show', + intro: isNl + ? `Elke did:plc-identiteit heeft een openbaar logboek, en elke repo vertelt welke collecties erin zitten. Daaruit valt af te lezen wanneer deze accounts zijn aangemaakt, waar ze draaien en welke apps ze gebruiken. Dit gaat over de ${nf(nlFacts.accounts.known)} accounts die wij herkennen, niet over alle Nederlandse accounts.` + : `Every did:plc identity has a public log, and every repo lists the collections it holds. That tells us when these accounts were created, where they run, and which apps they use. This covers the ${nf(nlFacts.accounts.known)} accounts we identify, not every Dutch account.`, + h3created: isNl ? 'Wanneer deze accounts zijn aangemaakt' : 'When these accounts were created', + createdCap: isNl + ? `Aanmaakdatum uit de PLC-directory, per maand. Dit is wanneer het account is aangemaakt, niet wanneer wij het herkenden.` + : `Creation date from the PLC directory, by month. This is when the account was made, not when we identified it.`, + h3lex: isNl ? 'Welke apps ze gebruiken, naast Bluesky' : 'Which apps they use, beyond Bluesky', + lexCap: isNl + ? `Aantal accounts met records van die lexicon in hun repo. Bluesky's eigen collecties zijn weggelaten: bijna iedereen heeft die. Dit telt accounts, geen records.` + : `Accounts holding records of that lexicon in their repo. Bluesky's own collections are left out, since nearly everyone has them. This counts accounts, not records.`, + lexNone: isNl + ? 'Nog niet genoeg repos gelezen om hier iets zinnigs te tonen.' + : 'Not enough repos read yet to show anything meaningful here.', + h3pds: isNl ? 'Waar ze draaien' : 'Where they run', + pdsCap: isNl + ? `${nf(nlFacts.pds.elsewhere)} van de ${nf(pdsTotal)} accounts (${elsewherePct}%) staan op een PDS buiten Bluesky's eigen hosting.` + : `${nf(nlFacts.pds.elsewhere)} of ${nf(pdsTotal)} accounts (${elsewherePct}%) live on a PDS outside Bluesky's own hosting.`, + bskyHosted: isNl ? 'Bluesky-hosting' : 'Bluesky hosted', + elsewhere: isNl ? 'elders' : 'elsewhere', + accounts: isNl ? 'accounts' : 'accounts', +}; +--- + +{ + hasAnyFacts && ( +
+

{t.h2}

+

{t.intro}

+ + {hasCreation && ( + <> +

{t.h3created}

+
+
+ + )} + + {hasLexicons && ( + <> +

{t.h3lex}

+
+ {lex.map((row) => ( +
+ + {appLabel(row.nsid)} {row.nsid} + + + + + {nf(row.accounts)} +
+ ))} +
+

{t.lexCap}

+ + )} + + {hasPds && ( + <> +

{t.h3pds}

+ +
+ {t.bskyHosted} {nf(nlFacts.pds.blueskyHosted)} + {t.elsewhere} {nf(nlFacts.pds.elsewhere)} +
+

{t.pdsCap}

+ {nlFacts.pds.topOtherHosts.length > 0 && ( +
    + {nlFacts.pds.topOtherHosts.map((h) => ( +
  • + {h.host} {nf(h.accounts)} {t.accounts} +
    • + ))} +
+ )} + + )} +
+ ) +} + + diff --git a/src/components/pages/CommunitySizePage.astro b/src/components/pages/CommunitySizePage.astro index 74d851e..1799f5d 100644 --- a/src/components/pages/CommunitySizePage.astro +++ b/src/components/pages/CommunitySizePage.astro @@ -6,6 +6,7 @@ // -> then you check it yourself. Both numbers are always adjacent, because the failure // mode of this page is a reader conflating a count with a model. import { nlStats, activeSources, overlapCount, summedSources, hasRange, type NlStatsSource } from '../../data/nl-stats'; +import CommunityFacts from './CommunityFacts.astro'; interface Props { lang: 'en' | 'nl'; @@ -308,6 +309,8 @@ const caveatConclusion = isNl + +

{t.h2model}

{t.modelIntro}

diff --git a/src/data/nl-facts.json b/src/data/nl-facts.json new file mode 100644 index 0000000..5afc95c --- /dev/null +++ b/src/data/nl-facts.json @@ -0,0 +1,8 @@ +{ + "generatedAt": "1970-01-01T00:00:00Z", + "accounts": { "known": 0, "dated": 0, "reposChecked": 0 }, + "creationByMonth": {}, + "pds": { "blueskyHosted": 0, "elsewhere": 0, "topOtherHosts": [] }, + "handles": { "customDomainNow": 0, "firstCustomDomainByMonth": {} }, + "lexicons": { "accountsUsingSomething": 0, "byNsid": [] } +} diff --git a/src/data/nl-facts.ts b/src/data/nl-facts.ts new file mode 100644 index 0000000..29bd65a --- /dev/null +++ b/src/data/nl-facts.ts @@ -0,0 +1,85 @@ +// What the PLC directory and the repos themselves say about the accounts we identified. +// +// Produced off-site by scripts/nl-facts-aggregate.py and committed here as counts only: +// no DIDs, no handles, same rule as nl-stats.json. The placeholder file ships with +// zeroes so the page builds before the first collection run lands; every section that +// reads it checks `has*` first and renders nothing until there is something to show. + +import raw from './nl-facts.json'; + +export interface NlFacts { + generatedAt: string; + accounts: { known: number; dated: number; reposChecked: number }; + /** "YYYY-MM" -> accounts created that month. */ + creationByMonth: Record; + pds: { + blueskyHosted: number; + elsewhere: number; + topOtherHosts: { host: string; accounts: number }[]; + }; + handles: { + customDomainNow: number; + /** "YYYY-MM" -> accounts that took a handle they control that month. */ + firstCustomDomainByMonth: Record; + }; + lexicons: { + accountsUsingSomething: number; + byNsid: { nsid: string; accounts: number }[]; + }; +} + +export const nlFacts: NlFacts = raw as NlFacts; + +export const hasCreation = Object.keys(nlFacts.creationByMonth).length > 0; +export const hasLexicons = nlFacts.lexicons.byNsid.length > 0; +export const hasPds = nlFacts.pds.blueskyHosted + nlFacts.pds.elsewhere > 0; +export const hasAnyFacts = hasCreation || hasLexicons || hasPds; + +/** Months from the first recorded to the last, with gaps filled as zero, so a quiet + * month renders as a gap in the chart instead of being silently skipped. */ +export function creationSeries(): { month: string; count: number }[] { + const months = Object.keys(nlFacts.creationByMonth).sort(); + if (!months.length) return []; + const out: { month: string; count: number }[] = []; + const [startY, startM] = months[0].split('-').map(Number); + const [endY, endM] = months[months.length - 1].split('-').map(Number); + for (let y = startY, m = startM; y < endY || (y === endY && m <= endM); ) { + const key = `${y}-${String(m).padStart(2, '0')}`; + out.push({ month: key, count: nlFacts.creationByMonth[key] ?? 0 }); + m += 1; + if (m > 12) { + m = 1; + y += 1; + } + } + return out; +} + +/** The app an NSID belongs to, for labelling. Falls back to the reversed authority, + * so an unknown lexicon still renders as something a reader can place. */ +export function appLabel(nsid: string): string { + const known: Record = { + 'id.sifa': 'Sifa', + 'sh.tangled': 'Tangled', + 'com.whtwnd': 'WhiteWind', + 'fyi.unravel.frontpage': 'Frontpage', + 'events.smokesignal': 'Smoke Signal', + 'blue.flashes': 'Flashes', + 'my.skylights': 'Skylights', + 'pub.leaflet': 'Leaflet', + 'app.popsky': 'Popsky', + 'blue.linkat': 'Linkat', + 'blue.badge': 'Badge', + 'social.grain': 'Grain', + 'fm.teal': 'teal.fm', + 'app.graze': 'Graze', + 'chat.roomy': 'Roomy', + 'space.aoi.shiharai': 'Shiharai', + 'com.atproto': 'AT Protocol core', + }; + for (const [prefix, label] of Object.entries(known)) { + if (nsid === prefix || nsid.startsWith(prefix + '.')) return label; + } + const parts = nsid.split('.'); + return parts.length >= 2 ? parts.slice(0, 2).reverse().join('.') : nsid; +}