jetstream v2 in zig stream.waow.tech
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101#!/usr/bin/env python3"""Build and publish the public, static Strata playground independently of Stream."""from pathlib import Pathimport datetimeimport gzipimport jsonimport subprocessimport tempfileimport hashlibimport argparseimport shutilROOT = Path(__file__).resolve().parents[1]SOURCE = ROOT / 'strata'HOST = 'root@89.167.122.160'SSH = ['-i', str(Path.home() / '.ssh/waow_ed25519'), '-o', 'IdentitiesOnly=yes']def fetch(url, raw=False): body = subprocess.check_output(['curl', '--fail', '--silent', '--show-error', '--max-time', '45', url]) return body.decode() if raw else json.loads(body)parser = argparse.ArgumentParser()parser.add_argument('--output-dir', type=Path, help='Build locally without publishing')parser.add_argument('--overview', type=Path, help='Reuse a previously fetched overview for a local build')args = parser.parse_args()with tempfile.TemporaryDirectory(prefix='strata-preview-') as tmp: out = Path(tmp) / datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ') out.mkdir() for name in ['index.html', 'style.css', 'app.js', 'sphere-map.js', 'storage-view.js', 'storage-parser.js', 'profile-parser.js', 'map-parser.js', 'd3.min.js', 'd3.LICENSE']: (out / name).write_bytes((SOURCE / name).read_bytes()) index = (out / 'index.html').read_text().replace('strata · local preview', 'strata') (out / 'index.html').write_text(index) app = (out / 'app.js').read_text().replace("json('/api/overview')", "json('overview.json')").replace('Static preview snapshot.', 'Static summary.').replace('This preview opens', 'This map opens') (out / 'app.js').write_text(app) overview = json.loads(args.overview.read_text()) if args.overview else { 'mode': 'static-preview', 'progress': fetch('https://strata.zat.dev/api/progress'), 'collections': fetch('https://strata.zat.dev/api/collections')['collections'], 'buckets': fetch('https://strata.zat.dev/api/heat?step=100&top=1')['buckets'], 'status': fetch('https://stream.waow.tech/status', raw=True), 'fetchedAt': int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000), 'source': 'https://stream.waow.tech', 'summarySource': 'https://strata.zat.dev', 'identityProfiles': [entry['profile'] for entry in json.loads((SOURCE / 'identity-catalog.json').read_text()).values() if entry.get('profile')], 'storage': json.loads((SOURCE / 'storage-report.json').read_text()), } assert overview['collections'] and overview['buckets'] and overview['progress']['maxSeq'] > 0 # The opening map needs source totals, not every individual record type. groups = {} totals = {} for row in overview['collections']: ns = row['ns'] assert row['nsid'] == ns or row['nsid'].startswith(ns + '.') groups.setdefault(ns, []).append([row['nsid'][len(ns) + 1:], row['count']]) totals[ns] = totals.get(ns, 0) + row['count'] assets = out / 'assets' assets.mkdir() def asset(name, body): path = assets / (hashlib.sha256(body).hexdigest()[:16] + '.' + name) path.write_bytes(body) return 'assets/' + path.name details_url = asset('json', json.dumps([{'ns': ns, 'records': [{'suffix': suffix, 'count': count} for suffix, count in rows]} for ns, rows in groups.items()], separators=(',', ':')).encode()) core = {k: v for k, v in overview.items() if k not in ('collections', 'buckets')} core.update(collections=[], collectionsUrl=details_url, recordTypeCount=len(overview['collections']), namespaces=[{'key': ns, 'count': count} for ns, count in sorted(totals.items(), key=lambda pair: -pair[1])]) # One script download instead of a serial module graph. `bun install` in strata/ first. (out / 'entry.js').write_text("import d3 from './d3.min.js'; globalThis.d3=d3; import('./app.js');") subprocess.run([str(SOURCE / 'node_modules/.bin/esbuild'), str(out / 'entry.js'), '--bundle', '--minify', '--format=iife', '--target=es2022', '--outfile=' + str(out / 'bundle.js')], check=True) script_url = asset('js', (out / 'bundle.js').read_bytes()) style_url = asset('css', (out / 'style.css').read_bytes()) index = index.replace('href="style.css"', 'href="' + style_url + '"') # Escape HTML-significant characters in externally sourced profile text. embedded = json.dumps(core, separators=(',', ':')).replace('<', '\\u003c') index = index.replace('<script src="d3.min.js"></script><script type="module" src="app.js"></script>', '<script id="map-data" type="application/json">' + embedded + '</script><script defer src="' + script_url + '"></script>') (out / 'index.html').write_text(index) for name in ['entry.js', 'bundle.js', 'bundle.js.LEGAL.txt']: (out / name).unlink(missing_ok=True) # Keep old filenames for pages already open during the rollout. (out / 'overview.json').write_text(json.dumps(overview, separators=(',', ':'))) for asset in list(out.rglob("*")): if asset.suffix in ('.html', '.js', '.css', '.json'): asset.with_name(asset.name + '.gz').write_bytes(gzip.compress(asset.read_bytes(), mtime=0)) print(f'Opening page: {len((out / "index.html.gz").read_bytes()):,} gzip bytes; record types load on demand.') if args.output_dir: shutil.copytree(out, args.output_dir, dirs_exist_ok=True) raise SystemExit(0) remote = '/opt/stream-experiment/site/.strata-preview-releases' subprocess.run(['ssh', *SSH, HOST, f'mkdir -p {remote}'], check=True) subprocess.run(['scp', '-q', *SSH, '-r', str(out), f'{HOST}:{remote}/'], check=True) # Retain immutable assets for cached pages from previous releases. subprocess.run(['ssh', *SSH, HOST, f'if [ -d /opt/stream-experiment/site/strata/assets ]; then cp -an /opt/stream-experiment/site/strata/assets/. {remote}/{out.name}/assets/; fi'], check=True) # Atomic pointer switch; retain earlier releases for rollback. subprocess.run(['ssh', *SSH, HOST, f'cd /opt/stream-experiment/site && ln -s .strata-preview-releases/{out.name} .strata-next && mv -Tf .strata-next strata'], check=True) print(f'Published {out.name}: https://stream.waow.tech/strata/') print(f'Snapshot: {len(overview["collections"])} record types; no runtime archive queries.')