jetstream v2 in zig stream.waow.tech
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142#!/usr/bin/env python3"""Manual, bounded refresh of shared profile metadata; never runs per visitor."""import jsonimport reimport timeimport subprocessimport urllib.parseimport urllib.errorfrom pathlib import Path
path = Path(__file__).with_name('identity-catalog.json')cache = json.loads(path.read_text()) if path.exists() else {}now = int(time.time())def get(url): return json.loads(subprocess.check_output(['curl', '--fail', '--silent', '--show-error', '--max-time', '20', '--max-filesize', '4194304', url]))rows = get('https://strata.zat.dev/api/collections')['collections']namespaces = sorted(ns for ns in set(row['ns'] for row in rows) if all(re.fullmatch(r'[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?', part) for part in ns.split('.')) and len(ns.split('.'))>=2 and re.fullmatch(r'[a-zA-Z][a-zA-Z0-9-]*', ns.split('.')[0]))pending = [ns for ns in namespaces if now-cache.get(ns, {}).get('checkedAt', 0) > (604800 if cache.get(ns, {}).get('profile') else 86400)]requests = 0for offset in range(0, len(pending), 25): if requests >= 100: break batch = pending[offset:offset+25] domains = {'.'.join(reversed(ns.split('.'))): ns for ns in batch} query = urllib.parse.urlencode([('actors', domain) for domain in domains]) requests += 1 try: profiles = get('https://public.api.bsky.app/xrpc/app.bsky.actor.getProfiles?' + query)['profiles'] except (subprocess.CalledProcessError, ValueError) as error: print(f'Stopping refresh after request {requests}: {type(error).__name__}', flush=True) break for ns in batch: cache[ns] = {'checkedAt': now, 'profile': None} for profile in profiles: handle = profile.get('handle', '').lower() if handle not in domains: continue cache[domains[handle]] = {'checkedAt': now, 'profile': {key: profile[key] for key in ('handle', 'displayName', 'avatar') if key in profile}} path.write_text(json.dumps(cache, separators=(',', ':'))) time.sleep(.3)print(f'{requests} profile batch requests; {sum(bool(v.get("profile")) for v in cache.values())} matching profiles cached; {len(cache)} namespaces checked')