jetstream v2 in zig stream.waow.tech
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428// strata's page: the layers picture (one DOM lane per app), the reader line, and "look inside an app"// (duckdb-wasm over the app's public parquet files). served by the worker at ui.js; talks to api/* relatively.
const fmt = new Intl.NumberFormat("en-US");const compact = new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 });const dayFmt = new Intl.DateTimeFormat("en-CA", { timeZone: "UTC", year: "numeric", month: "2-digit", day: "2-digit" });const timeFmt = new Intl.DateTimeFormat("en-GB", { timeZone: "UTC", hour: "2-digit", minute: "2-digit" });const STEPS = [1, 2, 5, 10, 25, 50, 100];const FLOOR = 0.14;const OTHER = "other";const OTHER_LABEL = "everything else";// the whole-network bootstrap ran 2026-07-28 → 2026-08-03 ~07:00 utc (stream /status: "backfill took 6d 2h")const BOOTSTRAP_END_US = 1785740400000000;const ERA_LABEL = { bootstrap: "history, replayed", live: "as it happened" };
function byId(id, ctor) { const node = document.getElementById(id); if (!(node instanceof ctor)) throw new Error(`missing #${id} (${ctor.name})`); return node;}const el = { layers: byId("layers", HTMLDivElement), cursor: document.createElement("div"), reader: byId("reader", HTMLDivElement), lede: byId("lede", HTMLParagraphElement), find: byId("find", HTMLInputElement), nsList: byId("ns-list", HTMLDataListElement), more: byId("more", HTMLButtonElement), tableToggle: byId("table-toggle", HTMLButtonElement), table: byId("table", HTMLDivElement), live: byId("live", HTMLSpanElement), updated: byId("updated", HTMLSpanElement), legendBar: byId("legend-bar", HTMLSpanElement), askNs: byId("ask-ns", HTMLSelectElement), askSize: byId("ask-size", HTMLSpanElement), askRun: byId("ask-run", HTMLButtonElement), askEdit: byId("ask-edit", HTMLButtonElement), askSql: byId("ask-sql", HTMLTextAreaElement), askFine: byId("ask-fine", HTMLParagraphElement), askOut: byId("ask-out", HTMLDivElement), askTable: byId("ask-table", HTMLDivElement), prompts: byId("prompts", HTMLDivElement),};
const esc = (s) => String(s).replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ }[c]));const displayName = (key) => (key === OTHER ? OTHER_LABEL : key);const shortName = (nsid) => nsid.slice(nsid.indexOf(".", nsid.indexOf(".") + 1) + 1);// one hue, dark → light, with a floor so a cell with anything in it never fades into the empty tonefunction shade(t) { const u = FLOOR + (1 - FLOOR) * Math.max(0, Math.min(1, t)); return `oklch(${(0.27 + 0.66 * u).toFixed(3)} ${(0.045 + 0.08 * u * (1 - u)).toFixed(3)} 222)`;}const rowHeight = (count, maxCount) => Math.round(12 + 34 * (Math.log10(Math.max(1, count)) / Math.log10(Math.max(10, maxCount))));
async function api(path) { const r = await fetch(path); return r.json();}
const view = { top: 10, expanded: new Map(), pinned: [], namespaces: [], step: 10 };let picture = null; // { rows, buckets, step }let hover = null; // { ri, bi }
async function loadProgress() { const p = await api("api/progress"); el.lede.innerHTML = `<em>strata</em> — what's in the archive. every app on the network writes its own kinds of records; this is how much of each there is, and when it arrived. so far: <em>${compact.format(p.events)}</em> records from <em>${fmt.format(p.namespaces)}</em> apps, in <em>${fmt.format(p.collections)}</em> record types.`; if (p.updatedAt) { const d = new Date(p.updatedAt); const partial = p.archiveSegments && p.archiveSegments > p.segments; el.live.innerHTML = partial ? `· showing the first <b>${fmt.format(p.segments)}</b> of <b>${fmt.format(p.archiveSegments)}</b> chunks — still reading` : `· <b>${fmt.format(p.segments)}</b> chunks read`; el.updated.textContent = `updated ${dayFmt.format(d)} ${timeFmt.format(d)} utc`; } view.step = STEPS.find((s) => Math.ceil(p.segments / s) <= 120) ?? STEPS[STEPS.length - 1]; if (!view.namespaces.length) { const { namespaces } = await api("api/namespaces"); view.namespaces = namespaces; el.nsList.innerHTML = namespaces.map((n) => `<option value="${esc(n.ns)}">${fmt.format(n.count)} records · ${n.members} record type${n.members === 1 ? "" : "s"}</option>`).join(""); el.askNs.innerHTML = `<option value="">—</option>` + namespaces.filter((n) => n.ns !== "app.bsky").map((n) => `<option value="${esc(n.ns)}">${esc(n.ns)} (${compact.format(n.count)})</option>`).join(""); }}
function seriesFor(cells, key, bucketIndex, n) { const a = new Float64Array(n); for (const c of cells) { if (c.key !== key) continue; const i = bucketIndex.get(c.bucket); if (i !== undefined) a[i] += c.count; } return a;}
async function load() { await loadProgress(); const step = view.step; const heat = await api(`api/heat?step=${step}&top=${view.top}`); if (heat.error) { el.reader.innerHTML = `<span class="status">${esc(heat.error)}</span>`; return; } const buckets = heat.buckets; const bucketIndex = new Map(buckets.map((b, i) => [b.bucket, i])); const keys = new Map(heat.keys.map((k) => [k.key, k])); const order = heat.keys.map((k) => k.key); const extra = view.pinned.filter((p) => !keys.has(p.ns)); const extraHeat = await Promise.all(extra.map((p) => api(`api/heat?step=${step}&ns=${p.ns}`))); extra.forEach((p, i) => { keys.set(p.ns, { key: p.ns, count: p.count, members: p.members, cells: extraHeat[i].cells }); order.push(p.ns); }); if (heat.cells.some((c) => c.key === OTHER)) { order.push(OTHER); keys.set(OTHER, { key: OTHER, count: 0, members: 0 }); } const rows = []; for (const key of order) { const k = keys.get(key); const values = k.cells ? seriesFor(k.cells, key, bucketIndex, buckets.length) : seriesFor(heat.cells, key, bucketIndex, buckets.length); const total = key === OTHER ? values.reduce((a, b) => a + b, 0) : k.count; rows.push({ key, depth: 0, total, members: k.members, values, expandable: key !== OTHER && k.members > 1 }); const sub = view.expanded.get(key); if (sub && sub.step === step) rows.push(...sub.rows); } picture = { rows, buckets, step }; for (const [key, sub] of view.expanded) if (sub.step !== step) expand(key); render();}
async function expand(key) { const step = view.step; const sub = await api(`api/heat?step=${step}&ns=${key}`); const bucketIndex = new Map(sub.buckets.map((b, i) => [b.bucket, i])); const rows = sub.keys.map((k) => ({ key: k.key, depth: 1, total: k.count, members: 1, values: seriesFor(sub.cells, k.key, bucketIndex, sub.buckets.length), expandable: false })); view.expanded.set(key, { step, rows }); if (picture && picture.step === step) { dropChildren(key); const at = picture.rows.findIndex((r) => r.key === key && r.depth === 0); if (at >= 0) picture.rows.splice(at + 1, 0, ...rows); } render();}function dropChildren(key) { const at = picture.rows.findIndex((r) => r.key === key && r.depth === 0); if (at < 0) return; let end = at + 1; while (end < picture.rows.length && picture.rows[end].depth === 1) end++; picture.rows.splice(at + 1, end - at - 1);}function collapse(key) { view.expanded.delete(key); if (picture) dropChildren(key); render();}
const eraOf = (b) => (b.min_witnessed_at < BOOTSTRAP_END_US ? "bootstrap" : "live");
function render() { if (!picture) return; const { rows, buckets } = picture; const n = buckets.length; const maxTotal = Math.max(...rows.filter((r) => r.depth === 0).map((r) => r.total)); const maxPerRow = rows.map((r) => Math.max(0, ...r.values));
const lanes = rows.map((r, ri) => { const h = Math.max(r.depth ? 11 : 12, rowHeight(r.total, maxTotal) - (r.depth ? 8 : 0)); const rects = []; for (let bi = 0; bi < n; bi++) { const v = r.values[bi]; if (v <= 0) continue; rects.push(`<rect x="${bi}" width="1" height="1" fill="${shade(v / maxPerRow[ri])}"/>`); } const glyph = r.expandable ? (view.expanded.has(r.key) ? "▾" : "▸") : r.depth ? "·" : ""; const label = r.depth ? shortName(r.key) : displayName(r.key); const expandAttr = r.expandable ? ` aria-expanded="${view.expanded.has(r.key)}"` : ""; return `<div class="lane${r.depth ? " child" : ""}" data-ri="${ri}">` + `<button class="name" type="button" data-key="${esc(r.key)}"${expandAttr} title="${esc(displayName(r.key))}"><span class="glyph">${glyph}</span><span class="label"><bdi>${esc(label)}</bdi></span></button>` + `<div class="strip" style="--h:${h}px"><svg viewBox="0 0 ${n} 1" preserveAspectRatio="none" shape-rendering="crispEdges" aria-hidden="true">${rects.join("")}</svg></div>` + `<span class="count">${compact.format(r.total)}</span></div>`; });
const eras = []; let start = 0; for (let bi = 1; bi <= n; bi++) { if (bi === n || eraOf(buckets[bi]) !== eraOf(buckets[start])) { const era = eraOf(buckets[start]); eras.push(`<div class="era ${era}" style="left:${(100 * start) / n}%;width:${(100 * (bi - start)) / n}%"></div>`); start = bi; } } // ticks: the first day of each era, then days spaced at least a tenth of the strip apart; the // boundary between eras is always labelled even when the later era is a single column const ticks = []; let lastAt = -Infinity; const eraAts = []; for (let bi = 0; bi < n; bi++) if (bi === 0 || eraOf(buckets[bi]) !== eraOf(buckets[bi - 1])) eraAts.push(bi / n); const place = (bi, text, cls) => { const at = bi / n; // an era label near the right edge is anchored to its right and reaches ~a quarter of the strip leftward const shadowed = eraAts.some((e) => (e > 0.8 ? at > e - 0.27 && at < e + 0.05 : at > e - 0.05 && at < e + 0.22)); if (!cls && (at - lastAt < 0.1 || shadowed)) return; ticks.push(`<span class="tick${at > 0.8 ? " end" : ""}${cls ? ` ${cls}` : ""}" style="left:${100 * at}%">${text}</span>`); lastAt = at; }; let lastDay = ""; for (let bi = 0; bi < n; bi++) { const day = dayFmt.format(new Date(buckets[bi].min_witnessed_at / 1000)); const eraStart = bi === 0 || eraOf(buckets[bi]) !== eraOf(buckets[bi - 1]); if (eraStart) place(bi, `${ERA_LABEL[eraOf(buckets[bi])]} · ${day}`, "era-tick"); else if (day !== lastDay) place(bi, day); lastDay = day; } const axis = `<div class="lane axis"><span></span><div class="strip">${eras.join("")}${ticks.join("")}</div><span></span></div>`;
el.layers.innerHTML = `${lanes.join("")}${axis}<div class="cursor" id="cursor"></div>`; el.cursor = byId("cursor", HTMLDivElement); el.legendBar.style.background = `linear-gradient(90deg, ${[0, 0.25, 0.5, 0.75, 1].map(shade).join(",")})`; el.more.textContent = view.top >= 60 ? "all shown" : `show more (${fmt.format(view.namespaces.length - rows.filter((r) => r.depth === 0 && r.key !== OTHER).length)} more apps)`; renderTable();}
function cellFromPointer(ev) { if (!picture) return null; const lane = ev.target instanceof Element ? ev.target.closest(".lane[data-ri]") : null; if (!lane) return null; const strip = lane.querySelector(".strip"); if (!(strip instanceof HTMLElement)) return null; const rect = strip.getBoundingClientRect(); const ri = Number(lane.dataset.ri); const x = (ev.clientX - rect.left) / rect.width; const bi = x < 0 || x >= 1 ? -1 : Math.floor(x * picture.buckets.length); return { ri, bi, rect };}
function readCell(c) { const r = picture.rows[c.ri]; const what = r.depth ? "record type" : r.key === OTHER ? "the apps not shown" : "app"; if (c.bi < 0) { el.reader.innerHTML = `<span><b>${esc(displayName(r.key))}</b> · ${what} · <b>${fmt.format(r.total)}</b> records in the archive${r.expandable ? ` across <b>${r.members}</b> record types · click the name to open` : ""}</span>`; return; } const b = picture.buckets[c.bi]; const v = r.values[c.bi]; const from0 = dayFmt.format(new Date(b.min_witnessed_at / 1000)); if (v <= 0) { el.reader.innerHTML = `<span>nothing from <b>${esc(displayName(r.key))}</b> in this slice (received ${from0} utc)</span>`; return; } const share = b.event_count ? (100 * v) / b.event_count : 0; const from = dayFmt.format(new Date(b.min_witnessed_at / 1000)); const to = dayFmt.format(new Date(b.max_witnessed_at / 1000)); const chunk = b.last_idx !== b.first_idx ? `chunks #${b.first_idx}–${b.last_idx}` : `chunk #${b.first_idx}`; el.reader.innerHTML = `<span><b>${esc(displayName(r.key))}</b> wrote <b>${fmt.format(v)}</b> records in this slice — <i>${share < 0.01 && v ? "under 0.01" : share.toFixed(2)}%</i> of the ${compact.format(b.event_count)} in it</span><br>` + `<span>archive ${chunk} · received ${from}${to !== from ? ` → ${to}` : ""} utc · record numbers ${fmt.format(b.min_seq)} – ${fmt.format(b.max_seq)}</span>`;}
function showCursor(c) { if (c.bi < 0) { el.cursor.style.display = "none"; return; } const host = el.layers.getBoundingClientRect(); const w = c.rect.width / picture.buckets.length; el.cursor.style.display = "block"; el.cursor.style.left = `${c.rect.left - host.left + c.bi * w}px`; el.cursor.style.width = `${Math.max(w, 1)}px`;}
function clearHover() { hover = null; el.cursor.style.display = "none"; el.reader.innerHTML = '<span class="status">move over the picture to read a cell · open an app to see its record types</span>';}
el.layers.addEventListener("pointermove", (ev) => { const c = cellFromPointer(ev); if (!c) { clearHover(); return; } hover = { ri: c.ri, bi: c.bi }; showCursor(c); readCell(c);});el.layers.addEventListener("pointerleave", clearHover);el.layers.addEventListener("click", (ev) => { const button = ev.target instanceof Element ? ev.target.closest("button.name[aria-expanded]") : null; if (!(button instanceof HTMLButtonElement)) return; const key = button.dataset.key ?? ""; if (view.expanded.has(key)) collapse(key); else expand(key);});
function renderTable() { if (!picture || el.table.hidden) return; const { rows, buckets } = picture; el.table.innerHTML = `<table><thead><tr><th>app / record type</th><th>records</th>${buckets.map((b) => `<th>#${b.first_idx}</th>`).join("")}</tr></thead><tbody>` + rows.map((r) => `<tr><td>${r.depth ? " · " : ""}${esc(displayName(r.key))}</td><td>${fmt.format(r.total)}</td>${Array.from(r.values, (v) => `<td>${v ? fmt.format(v) : ""}</td>`).join("")}</tr>`).join("") + `</tbody></table>`;}el.tableToggle.addEventListener("click", () => { el.table.hidden = !el.table.hidden; el.tableToggle.setAttribute("aria-pressed", String(!el.table.hidden)); el.tableToggle.textContent = el.table.hidden ? "show the numbers as a table" : "hide the table"; renderTable();});el.more.addEventListener("click", () => { view.top = Math.min(60, view.top + 12); el.more.disabled = view.top >= 60; load();});el.find.addEventListener("change", () => { const ns = el.find.value.trim().toLowerCase(); const hit = view.namespaces.find((n) => n.ns === ns); if (!hit) { el.reader.innerHTML = `<span class="status">no app called <b>${esc(ns)}</b> has written to the archive yet</span>`; return; } if (!view.pinned.some((p) => p.ns === ns)) view.pinned.push(hit); el.find.value = ""; load();});window.addEventListener("resize", () => { if (hover) el.cursor.style.display = "none";});
// ---- look inside an app ----const PROMPTS = [ ["what it writes", "select collection as record_type, op as action, count(*) as records, count(distinct did) as accounts\nfrom rows group by all order by records desc"], ["people per day", "select strftime(make_timestamp(time_us), '%Y-%m-%d') as day, count(distinct did) as accounts, count(*) as records\nfrom rows where kind = 'commit' group by 1 order by 1"], ["when each type first appeared", "select collection as record_type, strftime(make_timestamp(min(time_us)), '%Y-%m-%d %H:%M') as first_seen, count(*) as records\nfrom rows where kind = 'commit' group by 1 order by min(time_us)"], ["most active accounts", "select did as account, count(*) as records, count(distinct collection) as record_types\nfrom rows where kind = 'commit' group by 1 order by 2 desc limit 20"], ["creates vs deletes", "select op as action, count(*) as records from rows where kind = 'commit' group by 1 order by 2 desc"],];PROMPTS.forEach(([label, sql], i) => { if (i > 0) el.prompts.append(" · "); const b = document.createElement("button"); b.type = "button"; b.className = "linkish"; b.textContent = label; b.addEventListener("click", () => { el.askSql.value = sql; if (el.askNs.value) runAsk(); }); el.prompts.append(b);});{ el.prompts.append(" · "); const edit = el.askEdit; edit.className = "linkish"; el.prompts.append(edit);}el.askSql.value = PROMPTS[0][1];el.askEdit.addEventListener("click", () => { el.askSql.hidden = !el.askSql.hidden; el.askFine.hidden = el.askSql.hidden; el.askEdit.setAttribute("aria-pressed", String(!el.askSql.hidden)); if (!el.askSql.hidden) el.askSql.focus();});
let duck = null;let partition = null;async function duckdb() { if (duck) return duck; el.askOut.innerHTML = "loading the query engine…"; const mod = await import("https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.29.0/+esm"); const bundle = await mod.selectBundle(mod.getJsDelivrBundles()); const worker = new Worker(URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], { type: "text/javascript" }))); const db = new mod.AsyncDuckDB(new mod.ConsoleLogger(mod.LogLevel.WARNING), worker); await db.instantiate(bundle.mainModule, bundle.pthreadWorker); duck = await db.connect(); return duck;}el.askNs.addEventListener("change", async () => { partition = null; el.askRun.disabled = true; el.askTable.innerHTML = ""; el.askOut.innerHTML = ""; if (!el.askNs.value) { el.askSize.textContent = ""; return; } el.askSize.textContent = "checking…"; const p = await api(`api/partitions?ns=${el.askNs.value}`); if (p.error) { el.askSize.textContent = p.error; return; } partition = p; const size = p.bytes < 1 << 20 ? `${Math.round(p.bytes / 1024)} KB` : `${(p.bytes / (1 << 20)).toFixed(1)} MB`; el.askSize.textContent = p.files.length ? `${size} to read from ${p.files.length} chunk${p.files.length === 1 ? "" : "s"}` : "not copied out of the archive yet"; el.askRun.disabled = !p.files.length;});function formatCell(value, numeric) { if (value === null || value === undefined) return ""; return numeric ? fmt.format(value) : esc(value);}async function runAsk() { if (!partition || !partition.files.length) return; el.askRun.disabled = true; try { const conn = await duckdb(); await conn.query(`create or replace view rows as select * from read_parquet([${partition.files.map((f) => `'${f}'`).join(", ")}])`); const t = performance.now(); const res = await conn.query(el.askSql.value); const fields = res.schema.fields.map((f) => ({ name: f.name, numeric: /Int|Float|Decimal/.test(String(f.type)) })); const out = res.toArray().map((r) => r.toJSON()); el.askOut.innerHTML = `<b>${fmt.format(out.length)}</b> row${out.length === 1 ? "" : "s"} in <b>${(performance.now() - t).toFixed(0)} ms</b>`; el.askTable.innerHTML = `<table><thead><tr>${fields.map((f) => `<th>${esc(f.name.replaceAll("_", " "))}</th>`).join("")}</tr></thead><tbody>` + out.slice(0, 500).map((r) => `<tr>${fields.map((f) => `<td>${formatCell(r[f.name], f.numeric)}</td>`).join("")}</tr>`).join("") + `</tbody></table>` + (out.length > 500 ? `<p class="status">showing 500 of ${fmt.format(out.length)}</p>` : ""); } catch (err) { el.askOut.innerHTML = `<span class="err">${esc(err instanceof Error ? err.message : err)}</span>`; } finally { el.askRun.disabled = false; }}el.askRun.addEventListener("click", runAsk);el.askSql.addEventListener("keydown", (ev) => { if ((ev.metaKey || ev.ctrlKey) && ev.key === "Enter") runAsk();});
load();