jetstream v2 in zig stream.waow.tech
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>stream · experiment 6</title><style>:root{ --bg:#0d1013; --panel:#141a1f; --line:#232c34; --ink:#dfe7ec; --dim:#8296a4; --accent:#5fd3a6; --warn:#e8b465; --bad:#e8756a; --done:#4a6b7c; --mono:ui-monospace,"SF Mono",Menlo,Consolas,monospace;}@media (prefers-color-scheme:light){:root{ --bg:#f6f7f5; --panel:#fff; --line:#e0e3df; --ink:#1c2429; --dim:#63757f; --accent:#1a8a63; --warn:#a9700f; --bad:#b4402f; --done:#9aacb5;}}:root[data-theme=dark]{--bg:#0d1013;--panel:#141a1f;--line:#232c34;--ink:#dfe7ec;--dim:#8296a4;--accent:#5fd3a6;--warn:#e8b465;--bad:#e8756a;--done:#4a6b7c}:root[data-theme=light]{--bg:#f6f7f5;--panel:#fff;--line:#e0e3df;--ink:#1c2429;--dim:#63757f;--accent:#1a8a63;--warn:#a9700f;--bad:#b4402f;--done:#9aacb5}*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.55 var(--mono); padding:2rem 1.25rem 4rem;-webkit-font-smoothing:antialiased}.wrap{max-width:920px;margin:0 auto;display:flex;flex-direction:column;gap:1.5rem}header{display:flex;flex-wrap:wrap;align-items:baseline;gap:.6rem 1rem;border-bottom:1px solid var(--line);padding-bottom:1rem}h1{font-size:1.1rem;margin:0;letter-spacing:.02em;font-weight:600}.sub{color:var(--dim);font-size:.82rem}.live{margin-left:auto;font-size:.75rem;color:var(--accent);display:flex;align-items:center;gap:.4rem}.dot{width:7px;height:7px;border-radius:50%;background:var(--accent);animation:p 2s infinite}@keyframes p{0%,100%{opacity:1}50%{opacity:.25}}.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(170px,1fr));gap:.75rem}.card{background:var(--panel);border:1px solid var(--line);border-radius:7px;padding:.8rem .9rem}.k{color:var(--dim);font-size:.7rem;text-transform:uppercase;letter-spacing:.09em}.v{font-size:1.5rem;margin-top:.3rem;font-variant-numeric:tabular-nums;font-weight:600}.v small{font-size:.72rem;color:var(--dim);font-weight:400;letter-spacing:0}.bar{height:7px;background:var(--line);border-radius:4px;overflow:hidden;margin-top:.55rem}.bar>i{display:block;height:100%;background:var(--accent);transition:width .6s}section h2{font-size:.72rem;text-transform:uppercase;letter-spacing:.09em;color:var(--dim);margin:0 0 .7rem}table{width:100%;border-collapse:collapse;font-size:.84rem}td{padding:.34rem 0;border-bottom:1px solid var(--line);vertical-align:top}td:last-child{text-align:right;font-variant-numeric:tabular-nums}.tl{list-style:none;margin:0;padding:0}.tl li{position:relative;padding:0 0 1rem 1.5rem;border-left:2px solid var(--line)}.tl li:last-child{border-left-color:transparent;padding-bottom:0}.tl li::before{content:"";position:absolute;left:-6px;top:.42rem;width:10px;height:10px; border-radius:50%;background:var(--done);border:2px solid var(--bg)}.tl li.now::before{background:var(--accent);box-shadow:0 0 0 4px color-mix(in srgb,var(--accent) 22%,transparent)}.tl li.future::before{background:var(--line)}.tl li.bad::before{background:var(--bad)}.tl .when{font-size:.7rem;color:var(--dim);letter-spacing:.03em}.tl .what{font-size:.88rem;margin-top:.1rem}.tl .note{font-size:.76rem;color:var(--dim);margin-top:.15rem}.tl li.future .what{color:var(--dim)}.scroll{overflow-x:auto}footer{color:var(--dim);font-size:.72rem;border-top:1px solid var(--line);padding-top:.9rem}.err{color:var(--bad)}</style></head><body><div class="wrap"><header> <h1>stream · experiment 6</h1> <span class="sub">whole-network atproto backfill → jetstream v2</span> <span class="live"><span class="dot"></span><span id="age">connecting</span></span></header>
<div class="grid" id="tiles"></div>
<section><h2>lifecycle</h2><ul class="tl" id="timeline"></ul></section>
<section><h2>detail</h2><div class="scroll"><table id="detail"></table></div></section>
<footer id="foot"></footer></div><script>const $=s=>document.querySelector(s);const nf=n=>n==null?"—":n.toLocaleString("en-US");const gb=b=>b==null?"—":(b/1e9).toFixed(1)+" GB";const tb=b=>b==null?"—":(b/1e12).toFixed(2)+" TB";const ts=s=>new Date(s*1000).toISOString().replace("T"," ").slice(0,16)+"Z";function dur(sec){const s=Math.abs(sec);const d=Math.floor(s/86400),h=Math.floor(s%86400/3600),m=Math.floor(s%3600/60); return (d?d+"d ":"")+(d||h?h+"h ":"")+m+"m";}
function tiles(d){ const done=d.buckets.complete||0, pend=d.buckets.pending||0; const total=Object.values(d.buckets).reduce((a,b)=>a+b,0); const left=d.deadline-d.generated; return [ ["phase", d.phase||"—", d.phase_n?`stage ${d.phase_n} of 3`:""], ["repos complete", nf(done), total?`of ${nf(total)} enumerated`:""], ["elapsed", dur(d.generated-d.start), `since ${ts(d.start)}`], ["replay window", (()=>{const r=d.cursor_expiry-d.generated; return r>0?dur(r):"<span class='err'>EXPIRED</span>";})(), `firehose gap since ${ts(d.capture_stopped)}`], ["watchdog", (left>0?dur(left):"EXPIRED"), left>0?`destroys at ${ts(d.deadline)}`:"resources deleted"], ["spent", "€"+d.cost_eur.toFixed(0), `€${d.rate_eur_h.toFixed(2)}/h`], ["serving", d.serving_http==200?"live":"gated", d.serving_http?`listSegments → ${d.serving_http}`:""], ].map(([k,v,s])=>`<div class="card"><div class="k">${k}</div><div class="v">${v}${s?`<br><small>${s}</small>`:""}</div></div>`).join("");}
function timeline(d){ const ev=[ [1785214907,"provisioned + bootstrap begins","ccx43, 2.5 TB volume, 100 backfill workers","past"], [1785305160,"single oom-kill","only one all run; RSS is resident bloom filters","bad"], [1785438000,"deadline 120h → 180h, volume 2.5 → 3.0 TB","grown online with xfs_growfs","past"], [1785704698,"backfill cut over to merge","tail was unfetchable — brid.gy repos return RepoNotFound, so the crawl stalled at ~7.5 repos/s","past"], [1785708354,"merge drain complete","255,799,323 kept / 126,488,978 dropped across 137 sources","past"], [1785708400,"pending-repo pass","resyncing repos whose bootstrap download was interrupted","now"], [1785769956,"deadline 180h → 228h","extended to prove or disprove serving","past"], [null,"discovery · rotate · compaction","compaction over a 1.6 TB archive — never exercised at this size","future"], [null,"steady_state → serving ungates","listSegments returns 200 instead of 503","future"], [1786000698,"firehose replay window closes","72h of bsky.network retention from the capture cutover — miss this and the archive has a permanent hole, which binds 10h before the watchdog","future"], [1786035707,"watchdog destroys the project","deletes servers and the volume — archive and snapshot both","future"], ]; return ev.map(([t,what,note,cls])=>{ let c=cls; if(t&&cls!=="bad"&&cls!=="future"&&t<=d.generated)c=(cls==="now")?"now":""; return `<li class="${c}"><div class="when">${t?ts(t):"pending"}</div> <div class="what">${what}</div><div class="note">${note}</div></li>`; }).join("");}
function detail(d){ const b=d.buckets, m=d.merge; const rows=[ ["repos complete",nf(b.complete)],["repos pending",nf(b.pending)], ["repos failed",nf(b.failed)],["repos unavailable",nf(b.unavailable)], ["merge events kept",nf(m.kept)],["merge events dropped",nf(m.dropped)], ["merge segments consumed",nf(m.segments)],["repo revs updated",nf(m.revs_updated)], ["sealed segments",nf(d.segments)],["stale .tmp files",nf(d.stale_tmp)], ["backfill tree",gb(d.backfill_tree)], ["archive on disk",d.disk?tb(d.disk.used):"—"], ["disk free",d.disk?tb(d.disk.avail):"—"], ["process RSS",gb(d.rss)],["container restarts",nf(d.restarts)], ]; return rows.map(([k,v])=>`<tr><td>${k}</td><td>${v}</td></tr>`).join("");}
async function tick(){ try{ const d=await (await fetch("data.json?t="+Date.now(),{cache:"no-store"})).json(); $("#tiles").innerHTML=tiles(d); $("#timeline").innerHTML=timeline(d); $("#detail").innerHTML=detail(d); const age=Math.round(Date.now()/1000-d.generated); $("#age").textContent=age<90?"live":`stale ${dur(age)}`; $("#foot").innerHTML=`snapshot ${ts(d.generated)} · refreshes every 60s · `+ `served from the experiment host, which the watchdog deletes at ${ts(d.deadline)}`; }catch(e){ $("#age").innerHTML='<span class="err">unreachable</span>'; }}tick(); setInterval(tick,30000);</script></body></html>