jetstream v2 in zig stream.waow.tech
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover"><title>stream · stats</title><link rel="icon" type="image/svg+xml" href="/favicon.svg"><meta name="theme-color" content="#040708"><meta name="description" content="archive size, throughput, and delivery stats for stream.waow.tech"><meta property="og:title" content="stream · stats"><meta property="og:description" content="archive size, throughput, and delivery stats for stream.waow.tech"><meta property="og:type" content="website"><meta property="og:url" content="https://stream.waow.tech/stats"><meta name="twitter:card" content="summary"><style>
.grid{display:flex;flex-wrap:wrap;justify-content:center;gap:2.2rem 3ch;width:100%;max-width:66ch}.grid .stat{flex:0 1 18ch}.stat{display:flex;flex-direction:column;align-items:center;gap:.2rem;text-align:center}.stat .v{font-size:1.5rem;font-weight:300;color:var(--hot);letter-spacing:.04em;font-variant-numeric:tabular-nums}.stat .v small{font-size:.9rem;color:var(--dim)}.stat .l{font-size:.7rem;color:var(--dim);letter-spacing:.22em;text-transform:uppercase}.stat .s{font-size:.74rem;color:var(--dim)}.wrap{position:relative}@media (max-width:640px){.grid{gap:1.6rem 2ch}.grid .stat{flex:0 1 14ch}.wrap{padding:0 1.5ch}}</style><link rel="stylesheet" href="/brand.css?v=2"></head><body data-page="stats"><div id="atmosphere" aria-hidden="true"><canvas id="depth" data-current></canvas></div><div class="wrap"><header> <a class="brand" href="/">~ s t r e a m ~</a> <span class="livedot" id="live">· connecting</span></header><main> <div class="sect">the archive</div> <div class="grid"> <div class="stat"><span class="v" id="disk">—</span><span class="l">on disk</span><span class="s" id="disk-sub"></span></div> <div class="stat"><span class="v" id="events">—</span><span class="l">events archived</span><span class="s" id="events-sub"></span></div> <div class="stat"><span class="v" id="repos">—</span><span class="l">repositories</span><span class="s" id="repos-sub"></span></div> <div class="stat"><span class="v" id="format">jss v1</span><span class="l">format</span><span class="s">zstd blocks · byte-compatible with jetstream</span></div> <div class="stat"><span class="v" id="segs">—</span><span class="l">sealed segments</span><span class="s" id="segs-sub"></span></div> <div class="stat"><span class="v" id="lookback">—</span><span class="l">subscribe lookback</span><span class="s">deeper history via the archive api</span></div> </div> <div class="sect">delivery</div> <div class="grid"> <div class="stat"><span class="v" id="sent">—</span><span class="l">events sent</span><span class="s">to subscribers since last restart</span></div> <div class="stat"><span class="v" id="subs">—</span><span class="l">subscribers now</span><span class="s" id="subs-sub"></span></div> <div class="stat"><span class="v" id="served">—</span><span class="l">blocks served</span><span class="s" id="served-sub"></span></div> <div class="stat"><span class="v" id="ingest">—</span><span class="l">ingest rate</span><span class="s" id="ingest-sub">upstream seq advance</span></div> </div> <div class="sect" id="expected">event rate by hour (utc)</div> <canvas id="profile" width="1400" height="520" style="width:100%;height:auto;touch-action:pan-y;cursor:crosshair"></canvas> <p id="profile-reading" aria-live="polite"></p> <p class="s" style="color:var(--dim);font-size:.74rem;text-align:center">median · p25–75 · p10–90 · trailing 90 days via <a href="https://relay-eval.waow.tech" style="color:var(--water)">relay-eval</a> · touch the chart to read any hour</p> <div class="sect">the machine</div> <div class="grid"> <div class="stat"><span class="v" id="rss">—</span><span class="l">memory</span><span class="s">resident set</span></div> <div class="stat"><span class="v" id="free">—</span><span class="l">disk free</span><span class="s">on the archive volume</span></div> <div class="stat"><span class="v" id="uptime">—</span><span class="l">uptime</span><span class="s" id="uptime-sub"></span></div> </div></main><footer> <a href="/">home</a> <a href="/stats" aria-current="page">stats</a> <a href="/status">status</a> <a href="/strata/">strata</a> <a id="commit" href="https://tangled.org/zat.dev/stream">build</a> <a href="https://tangled.org/zat.dev/stream">source</a></footer></div><script src="/ambient.js?v=crystal-1"></script><script>const $ = (id) => document.getElementById(id);let _prevSeq = null;let _tailFlowing = null;let _prevSeqAt = 0, _rateEma = null;// cross-page continuity: polls persist {seq, t, ema}; the next page load// compares its first read against a fresh stored point instead of waiting// for a second poll. sessionStorage is per-tab, same-origin.try { const c = JSON.parse(sessionStorage.getItem("seqCont") || "null"); if (c && Date.now() - c.t < 60_000 && BigInt(c.seq) > 0n) { _prevSeq = BigInt(c.seq); _prevSeqAt = performance.now() - (Date.now() - c.t); _rateEma = c.ema; }} catch {}function saveCont(upN){ try { sessionStorage.setItem("seqCont", JSON.stringify({ seq: upN.toString(), t: Date.now(), ema: _rateEma })); } catch {}}// median firehose rate by utc hour: 7,014 five-minute relay-eval windows,// 2026-03-13..08-08 (monthly medians stable 343-371/s; hour is the signal)const EXPECTED = [360,351,335,314,278,250,240,228,220,224,249,296,346,384,413,424,428,422,419,418,423,418,400,378];// live baseline: relay-eval recomputes hour-of-day quantiles from its own// accumulating history; the embedded table is only the pre-fetch fallback.let PROFILE = null;fetch("https://relay-eval.waow.tech/api/rate-profile").then(r => r.json()) .then(d => { PROFILE = d.hours; for (let h = 0; h < 24; h++) EXPECTED[h] = d.hours[h].med; }) .catch(() => {});const expectedNow = () => EXPECTED[new Date().getUTCHours()];function noteRate(upN){ const now = performance.now(); if (_prevSeq !== null && upN > _prevSeq && _prevSeqAt > 0) { const inst = Number(upN - _prevSeq) * 1000 / (now - _prevSeqAt); _rateEma = _rateEma === null ? inst : 0.6 * _rateEma + 0.4 * inst; } else if (upN === _prevSeq) { _rateEma = _rateEma === null ? 0 : 0.6 * _rateEma; } _prevSeqAt = now; if ($("ingest")) { $("ingest").textContent = _rateEma === null || upN === 0n ? "—" : Math.round(_rateEma).toLocaleString() + "/s"; if (_rateEma !== null && upN !== 0n) { const x = _rateEma / expectedNow(); $("ingest-sub").textContent = x > 1.5 ? x.toFixed(1) + "× the hour's typical rate" : "upstream seq advance"; } } // The chart caption prints this same EMA. Its own timer fires while tick() // is still awaiting its fetch, so on a moving rate it painted the previous // poll's value and the page showed two different "live" numbers at once // (377 vs 330, 2026-08-09). Repaint here so both come from one value. drawProfile();}async function fetchStatus(){ const t = await (await fetch("/status",{headers:{Accept:"text/plain"},cache:"no-store"})).text(); const g = (re)=>{const m=t.match(re);return m?m[1]:null}; const build = g(/build ([0-9a-f]{7,})/); if(build && $("commit")){ $("commit").textContent = build; $("commit").href = "https://tangled.org/zat.dev/stream/commits/" + build; } const up = g(/upstream seq\s+([\d,]+)/); const upN = up ? BigInt(up.replace(/,/g,"")) : 0n; if (upN > 0n) dispatchEvent(new CustomEvent("stream-sequence", {detail:upN.toString()})); const flowing = upN > 0n && _prevSeq !== null && upN > _prevSeq; const stalled = upN > 0n && _prevSeq !== null && upN === _prevSeq; _tailFlowing = flowing ? true : stalled || upN === 0n ? false : null; // a booting instance reports upstream seq 0: feeding that into the rate // math (or persisting it) makes the next real read look like a delta of // the entire relay sequence space — the '503M/s' artifact if (upN > 0n) { noteRate(upN); saveCont(upN); _prevSeq = upN; } if($("live")) $("live").innerHTML = flowing ? "<b>●</b> live tail flowing" : stalled ? "· live tail stalled · archive serving" : upN > 0n ? "· checking tail…" : "· live tail paused · archive serving"; return { t, g };}
const fmtBytes = (b) => { if(b >= 1e12) return [(b/1e12).toFixed(2), "TB"]; if(b >= 1e9) return [(b/1e9).toFixed(1), "GB"]; if(b >= 1e6) return [(b/1e6).toFixed(1), "MB"]; return [String(b), "B"];};const fmtCount = (n) => { if(n >= 1e9) return (n/1e9).toFixed(2) + "B"; if(n >= 1e6) return (n/1e6).toFixed(1) + "M"; if(n >= 1e3) return (n/1e3).toFixed(1) + "k"; return String(n);};const setBytes = (id, b) => { const [v,u] = fmtBytes(b); $(id).innerHTML = v + " <small>" + u + "</small>";};
async function fetchMetrics(){ const t = await (await fetch("/metrics",{cache:"no-store"})).text(); const m = (name)=>{ const re = new RegExp("^" + name + "(?:\\{[^}]*\\})? ([0-9.e+-]+)$","gm"); let sum = null; for(const x of t.matchAll(re)) sum = (sum ?? 0) + Number(x[1]); return sum; }; const sent = m("jetstream_subscribe_events_sent_total"); if(sent !== null) $("sent").textContent = fmtCount(sent); const subs = m("stream_subscribers_active") ?? m("jetstream_subscribe_subscribers"); if(subs !== null) $("subs").textContent = String(subs); const blocks = m("jetstream_getblock_requests_total"); const blockBytes = m("jetstream_getblock_served_bytes_total"); if(blocks !== null) $("served").textContent = fmtCount(blocks); if(blockBytes !== null) $("served-sub").textContent = fmtBytes(blockBytes).join(" ") + " of compressed history since last restart"; const rss = m("process_resident_memory_bytes"); if(rss !== null) setBytes("rss", rss); const free = m("jetstream_data_dir_free_bytes"); if(free !== null) setBytes("free", free);}
// true archive size: sum sizeBytes over every sealed segment (paged metadata)async function sumArchive(){ let total = 0, count = 0, cursor = null; for(let i = 0; i < 40; i++){ const u = "/xrpc/network.bsky.jetstream.listSegments?limit=1000" + (cursor ? "&cursor=" + cursor : ""); const r = await (await fetch(u)).json(); if(!r.segments || !r.segments.length) break; for(const s of r.segments){ total += s.sizeBytes; count++; } if(!r.cursor || r.segments.length < 1000) break; cursor = r.cursor; } if(total > 0){ setBytes("disk", total); $("disk-sub").textContent = "sealed archive · " + fmtCount(count) + " segments summed"; } else { // key-gated listSegments: fall back to the public metrics gauge try{ const m = await (await fetch("/metrics",{cache:"no-store"})).text(); const mb = m.match(/^stream_archive_sealed_bytes (\d+)$/m); const ms = m.match(/^stream_archive_sealed_segments (\d+)$/m); if(mb){ setBytes("disk", Number(mb[1])); $("disk-sub").textContent = "sealed archive · " + (ms ? fmtCount(Number(ms[1])) : "?") + " segments"; return; } }catch(e){} $("disk-sub").textContent = "archive metadata is key-gated; size not shown"; }}
async function tick(){ try{ const { g } = await fetchStatus(); const num = (s)=> s ? Number(s.replace(/,/g,"")) : null; const next = num(g(/next seq\s+([\d,]+)/)); if(next) { $("events").textContent = fmtCount(next); $("events-sub").textContent = "seq 1 → today"; } const total = num(g(/total\s+([\d,]+)/)), complete = num(g(/complete\s+([\d,]+)/)); if(total){ $("repos").textContent = fmtCount(total); if(complete) $("repos-sub").textContent = Math.round(complete/total*1000)/10 + "% complete"; } const segs = g(/sealed segs\s+([\d,]+)/); if(segs) $("segs").textContent = segs; const oldest = g(/oldest retained\s+([\d,]+)/); if(oldest) $("segs-sub").textContent = "oldest live-replayable seq " + oldest; const lb = g(/cursor lookback\s+(\S+ ?\S*)/); if(lb) $("lookback").textContent = lb.trim(); const up = g(/uptime\s+(\S+ ?\S*)/); if(up) $("uptime").textContent = up.trim(); const phase = g(/phase\s+(\S+ for \S+ ?\S*)/); if(phase) $("uptime-sub").textContent = phase.trim(); const ls = g(/listeners\s+([\d,]+)/); if(ls) $("subs-sub").textContent = "on /subscribe + subscribeEvents"; await fetchMetrics(); }catch(e){ $("live").textContent = "· unreachable"; }}tick(); setInterval(tick, 5000);let _scrub = null; // fractional hour under the pointer, null = follow nowfunction drawProfile(){ const cv = $("profile"); if (!cv || !PROFILE) return; const ctx = cv.getContext("2d"), W = cv.clientWidth, H = 240, L=42, R=12, T=15, B=30; const dpr = Math.min(devicePixelRatio || 1, 2); cv.width = Math.round(W * dpr); cv.height = Math.round(H * dpr); ctx.setTransform(dpr,0,0,dpr,0,0); const css = n => getComputedStyle(document.documentElement).getPropertyValue(n).trim(); ctx.clearRect(0,0,W,H); const ymax = 500, X = h => L+(W-L-R)*h/24, Y = v => T+(H-T-B)*(1-v/ymax); const P24 = PROFILE.concat([{...PROFILE[0], h: 24}]); // hour 0 again at the right edge ctx.strokeStyle = css("--faint"); ctx.fillStyle = css("--dim"); ctx.font = "11px ui-monospace,Menlo,monospace"; for (let v=0; v<=ymax; v+=100){ ctx.beginPath(); ctx.moveTo(L,Y(v)); ctx.lineTo(W-R,Y(v)); ctx.stroke(); ctx.fillText(v,10,Y(v)+4); } for (let h=0; h<=23; h+=3) ctx.fillText(String(h).padStart(2,"0"), X(h)-6, H-8); function band(lo,hi,alpha){ ctx.beginPath(); P24.forEach((q,i)=> i?ctx.lineTo(X(q.h),Y(q[hi])):ctx.moveTo(X(q.h),Y(q[hi]))); [...P24].reverse().forEach(q=> ctx.lineTo(X(q.h),Y(q[lo]))); ctx.closePath(); ctx.fillStyle = "rgba(111,156,176," + alpha + ")"; ctx.fill(); } band("p10","p90",.12); band("p25","p75",.18); ctx.beginPath(); ctx.strokeStyle = css("--water"); ctx.lineWidth = 1.5; P24.forEach((q,i)=> i?ctx.lineTo(X(q.h),Y(q.med)):ctx.moveTo(X(q.h),Y(q.med))); ctx.stroke();
const now = new Date(), nowH = now.getUTCHours() + now.getUTCMinutes()/60; // now marker always present ctx.setLineDash([5,5]); ctx.strokeStyle = css("--dim"); ctx.lineWidth = 1; ctx.beginPath(); ctx.moveTo(X(nowH),Y(0)); ctx.lineTo(X(nowH),Y(ymax)); ctx.stroke(); ctx.setLineDash([]); if (_rateEma !== null){ ctx.beginPath(); ctx.arc(X(nowH), Y(Math.min(_rateEma,ymax)), 6, 0, 7); ctx.fillStyle = css("--hot"); ctx.fill(); } // readout: the scrubbed hour, or now const sh = _scrub === null ? nowH : _scrub; const hi = Math.round(sh) % 24, q = PROFILE[hi]; if (_scrub !== null){ ctx.strokeStyle = css("--hot"); ctx.lineWidth = 1.4; ctx.beginPath(); ctx.moveTo(X(sh),Y(0)); ctx.lineTo(X(sh),Y(ymax)); ctx.stroke(); ctx.beginPath(); ctx.arc(X(hi), Y(q.med), 5, 0, 7); ctx.fillStyle = css("--water"); ctx.fill(); } const isNow = _scrub === null || Math.round(sh) % 24 === now.getUTCHours(); let line = String(hi).padStart(2,"0") + ":00z · typically " + q.med + "/s (" + q.p25 + "–" + q.p75 + ")"; // during a pause the EMA decays toward zero — that is not a live rate, // and the header already knows the tail state; say the same thing here if (isNow) line += _tailFlowing && _rateEma !== null ? " · live " + Math.round(_rateEma) + "/s" : _tailFlowing === null ? " · checking tail…" : " · tail paused · archive serving"; $("profile-reading").textContent = line; ctx.textAlign = "start";}(() => { const cv = $("profile"); if (!cv) return; const frac = e => { const r = cv.getBoundingClientRect(); const x = e.clientX - r.left; return Math.max(0, Math.min(23.99, (x - 42) / (cv.clientWidth - 42 - 12) * 24)); }; cv.addEventListener("pointermove", e => { _scrub = frac(e); drawProfile(); }, {passive:true}); cv.addEventListener("pointerdown", e => { _scrub = frac(e); drawProfile(); }, {passive:true}); cv.addEventListener("pointerleave", () => { _scrub = null; drawProfile(); });})();new ResizeObserver(drawProfile).observe($("profile"));setInterval(drawProfile, 5000); setTimeout(drawProfile, 1200);sumArchive();</script></body></html>