Something went wrong. Try again.
A lightweight proxy service, allowing you to use old Twitter clients with Bluesky
Something went wrong. Try again.
4.4 kB · 94 lines
Python
at main
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495#!/usr/bin/env python3"""Fetch archived 2009-2010 Twitter API docs from the Wayback Machine.
For each apiwiki method page, pick the latest 200 snapshot, download the rawpage (id_ flag), and extract <pre> example blocks into testdata/wayback/.Usage: python3 scripts/fetch_wayback.py"""import html, json, os, re, sys, time, urllib.parse, urllib.request
OUT = os.path.join(os.path.dirname(__file__), "..", "testdata", "wayback")RAW = os.path.join(OUT, "raw")# (pattern, latest timestamp to consider). apiwiki pages became "this page# has moved" stubs in late 2010, so they are capped; dev.twitter.com/doc# replaced them and carries the late-2010 response shapes (with id_str).PATTERNS = [ ("apiwiki.twitter.com/Twitter-REST-API-Method*", "20100731"), ("apiwiki.twitter.com/Twitter-Search-API-Method*", "20100731"), ("apiwiki.twitter.com/Return-Values*", "20100731"), ("apiwiki.twitter.com/HTTP-Response-Codes-and-Errors*", "20100731"), ("apiwiki.twitter.com/Rate-limiting*", "20100731"), ("apiwiki.twitter.com/Authentication*", "20100731"), ("apiwiki.twitter.com/OAuth-FAQ*", "20100731"), ("apiwiki.twitter.com/Things-Every-Developer-Should-Know*", "20100731"), ("dev.twitter.com/doc/get/*", "20110331"), ("dev.twitter.com/doc/post/*", "20110331"),]
def get(url, tries=6): for i in range(tries): try: req = urllib.request.Request(url, headers={"User-Agent": "mockingbird-research/1.0"}) with urllib.request.urlopen(req, timeout=60) as r: b = r.read() if b"Temporarily Offline" in b[:2000]: raise IOError("archive offline") return b except Exception as e: print(" retry", i, url[:100], e, file=sys.stderr) time.sleep(3 + i * 4) return None
def slug(u): p = urllib.parse.unquote(urllib.parse.urlsplit(u).path).strip("/") p = p.replace(" ", "-").replace(":", "").replace(" ", "-") return re.sub(r"[^A-Za-z0-9._-]+", "-", p) or "index"
def main(): os.makedirs(RAW, exist_ok=True) latest = {} for pat, to in PATTERNS: cdx = "https://web.archive.org/cdx/search/cdx?" + urllib.parse.urlencode( {"url": pat, "output": "json", "filter": "statuscode:200", "fl": "original,timestamp", "to": to}) b = get(cdx) if not b: print("cdx failed", pat, file=sys.stderr); continue rows = json.loads(b or b"[]")[1:] for orig, ts in rows: if "SearchFor=" in orig or "%E2%97%8F" in orig: continue if "dev.twitter.com" in orig: # The dev.twitter.com index is polluted with URLs scraped # from JavaScript; keep only real method pages. path = urllib.parse.unquote(urllib.parse.urlsplit(orig).path) if not re.match(r"^/doc/(get|post|delete)[/ ][a-z_]+(/[a-z_:]+)*/?$", path, re.I): continue key = slug(orig.split("?")[0].replace(":80/", "/")) if "mode=print" in orig: # prefer print views, they have less chrome ts = ts + "p" if key not in latest or ts > latest[key][1]: latest[key] = (orig, ts) index = {} for key, (orig, ts) in sorted(latest.items()): dest = os.path.join(RAW, key + ".html") if not os.path.exists(dest): b = get("https://web.archive.org/web/%sid_/%s" % (ts.rstrip("p"), orig)) if not b: continue open(dest, "wb").write(b) s = open(dest, encoding="utf-8", errors="replace").read() if "This page has moved" in s: os.remove(dest) continue pres = [html.unescape(re.sub(r"<[^>]+>", "", p)) for p in re.findall(r"<pre[^>]*>(.*?)</pre>", s, re.S)] text = html.unescape(re.sub(r"<[^>]+>", "", re.sub(r"<script.*?</script>|<style.*?</style>", "", s, flags=re.S))) text = re.sub(r"\n\s*\n+", "\n", text) open(os.path.join(OUT, key + ".txt"), "w").write(text) if pres: open(os.path.join(OUT, key + ".pre.txt"), "w").write("\n\n=====\n\n".join(pres)) index[key] = {"url": orig, "timestamp": ts.rstrip("p"), "examples": len(pres)} print(key, ts, len(pres)) json.dump(index, open(os.path.join(OUT, "index.json"), "w"), indent=1, sort_keys=True)
if __name__ == "__main__": main()