Something went wrong. Try again.
a tiny pet with an even tinier brain, living in your browser
Something went wrong. Try again.
38 kB · 575 lines
HTML
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576<!doctype html><!-- Brain experiment benchmark, rendered by site/build.sh with its existing runtime/model placeholders. One fresh page (and one worker) per setting. Query: model=ID, dtype=q4|q8|fp32, device=wasm|webgpu, threads=0..256 qdq=0|1, prepack=0|1, repeats=1..100, warmup=0..20, timeout=1000..3600000 (whole-page deadline in milliseconds), prompts=short|long|both, mode=greedy|sampled|both, workload=pet|fixed|both. Fixed workload: diagnostic compute only, 32 greedy tokens ignoring EOS/wire stops, on the original fitted pet prompts; not useful output or behavior evidence. Defaults: release model, q4, wasm, ORT threads, fp32 folding/no prepacking, three repeats, one warmup, 120000 ms, both prompts and both modes. window.res.status is running/success/error; window.mem() measures page MB. Throughput counts actual post-first-token decoding, not a fixed output budget divided by prefill+decode time. Pet turns stop normally, with real fixture lookup results and the app's retry budgets; no canned model output is used.--><meta charset="utf-8"><title>subtokenling bench</title><meta name="viewport" content="width=device-width, initial-scale=1"><link rel="icon" href="data:,"><style> body { margin: 1rem; font: 14px/1.5 system-ui, sans-serif; } pre { max-width: 100%; white-space: pre-wrap; overflow-wrap: anywhere; word-break: break-word; } #generated { min-height: 1.5em; }</style><pre id="generated" aria-live="polite"></pre><pre id="out">loading…</pre><script type="module">const output = document.getElementById("out"), shown = document.getElementById("generated");const res = window.res = { status: "running", config: {}, hardware: { user_agent: navigator.userAgent, platform: navigator.platform, cores: navigator.hardwareConcurrency, device_memory_gib: navigator.deviceMemory ?? null, isolated: crossOriginIsolated, shared_array_buffer: typeof SharedArrayBuffer !== "undefined", language: navigator.language, screen: { width: screen.width, height: screen.height, dpr: devicePixelRatio },}, results: [], fixed_results: [], errors: [] };const render = () => { output.textContent = JSON.stringify(res, null, 2); };let worker, timer, workerUrl;const visible = new Map();let activeRun = null;function fail(error) { if (res.status !== "running") return; res.status = "error"; res.errors.push({ message: error?.message || String(error), stack: error?.stack || null }); clearTimeout(timer); worker?.terminate(); if (workerUrl) URL.revokeObjectURL(workerUrl); render();}window.addEventListener("error", (event) => fail(event.error || event.message));window.addEventListener("unhandledrejection", (event) => fail(event.reason));window.mem = async () => { if (!performance.measureUserAgentSpecificMemory) throw new Error("page memory measurement is unavailable in this browser/context"); const bytes = (await performance.measureUserAgentSpecificMemory()).bytes; res.memory = { bytes, mb: bytes / 1e6, source: "measureUserAgentSpecificMemory (page and resident model worker)" }; render(); return Math.round(bytes / 1e6);};
// The worker isolates inference from the page's watchdog and presentation.// A blocked or failing runtime cannot leave window.res loading indefinitely.async function benchmarkWorker() { const absoluteNow = () => performance.timeOrigin + performance.now(); const send = (type, value) => self.postMessage({ type, ...value }); let releaseProfile; const resources = () => performance.getEntriesByType("resource").map((entry) => ({ name: entry.name, duration_ms: entry.duration, transfer_bytes: entry.transferSize, encoded_bytes: entry.encodedBodySize, decoded_bytes: entry.decodedBodySize })); const checkpoint = (phase, detail = {}) => new Promise((resolve) => { releaseProfile = resolve; send("private-memory", { phase, ...detail, resource_timing: resources() }); }); function tensorShapes(value, depth = 0) { if (!value || typeof value !== "object" || depth > 3) return {}; return Object.fromEntries(Object.entries(value).flatMap(([name, tensor]) => { if (!tensor || typeof tensor !== "object") return []; if (Array.isArray(tensor.dims)) return [[name, { dims: [...tensor.dims], type: tensor.type ?? null }]]; const nested = tensorShapes(tensor, depth + 1); return Object.keys(nested).length ? [[name, nested]] : []; })); } self.onmessage = async ({ data: cfg }) => { if (cfg.type === "private-memory-continue") { releaseProfile?.(); releaseProfile = null; return; } try { if (new URL(cfg.models).origin !== self.location.origin || new URL(cfg.runtime).origin !== self.location.origin) { throw new Error("experiment model and runtime assets must stay on this origin"); } const T = await import(cfg.runtime + "transformers.min.js"); if (cfg.profile_memory) await checkpoint("runtime_imported_no_session"); // Transformers.js 4.3.1's metadata registry skips HTTP localModelPath // entries. Use its HTTP-host contract, still confined to our static origin. Object.assign(T.env, { allowRemoteModels: true, allowLocalModels: false, remoteHost: cfg.models, remotePathTemplate: "{model}/", useBrowserCache: false }); T.env.backends.onnx.wasm.wasmPaths = cfg.runtime; T.env.backends.onnx.wasm.numThreads = cfg.threads; if (cfg.threads > 1 && (!self.crossOriginIsolated || typeof SharedArrayBuffer === "undefined")) { throw new Error("multiple WASM threads requested without cross-origin isolation/SharedArrayBuffer; no single-thread fallback"); } let gpu = null; if (cfg.device === "webgpu") { if (!navigator.gpu) throw new Error("WebGPU requested but navigator.gpu is unavailable; no WASM fallback"); const adapter = await navigator.gpu.requestAdapter({ forceFallbackAdapter: false }); if (!adapter) throw new Error("WebGPU requested but no adapter is available; no WASM fallback"); const info = adapter.info || (adapter.requestAdapterInfo ? await adapter.requestAdapterInfo() : {}); const description = [info.vendor, info.architecture, info.device, info.description].filter(Boolean).join(" "); const software = adapter.isFallbackAdapter === true || /swiftshader|llvmpipe|software/i.test(description); gpu = { vendor: info.vendor || null, architecture: info.architecture || null, device: info.device || null, description: info.description || null, is_fallback_adapter: adapter.isFallbackAdapter ?? null, software_adapter_detected: software, adapter_info_available: Boolean(description), source: "worker adapter probe; runtime selects its own adapter" }; send("hardware", { gpu }); if (software) throw new Error("software/fallback WebGPU adapter detected; refusing a hardware-GPU timing claim"); } send("phase", { phase: "loading" }); const startLoad = performance.now(); const tok = await T.AutoTokenizer.from_pretrained(cfg.model); const model = await T.AutoModelForCausalLM.from_pretrained(cfg.model, { device: cfg.device, dtype: cfg.dtype, session_options: { extra: { session: { disable_quant_qdq: cfg.qdq ? "0" : "1", disable_prepacking: cfg.prepack ? "0" : "1" } } }, progress_callback: (progress) => send("progress", { progress }), }); let lastForward = null; if (cfg.profile_memory) { if (typeof model.forward !== "function") throw new Error("private memory profile cannot observe model.forward cache tensors"); const forward = model.forward; model.forward = async (...args) => { const result = await forward.apply(model, args); lastForward = { inputs: tensorShapes(args[0]), outputs: tensorShapes(result) }; return result; }; } const load_ms = performance.now() - startLoad; // Keep the loaded session resident after success so window.mem() includes // the actual model worker, not the small page left after terminating it. self.benchmarkModel = model; self.benchmarkTokenizer = tok; const modelConfig = model.config; const eos = model.generation_config?.eos_token_id ?? modelConfig.eos_token_id ?? tok.eos_token_id; const eosIds = new Set((Array.isArray(eos) ? eos : eos == null ? [] : [eos]).map(Number)); const maxContext = modelConfig.max_position_embeddings || modelConfig.n_positions; if (!Number.isInteger(maxContext) || maxContext < 128) throw new Error("model has no supported explicit context budget"); const modelBase = new URL(cfg.model.split("/").map(encodeURIComponent).join("/") + "/", cfg.models); async function receipt(name) { const response = await fetch(new URL(name, modelBase)); if (response.status === 404) return null; // Older published brains have no experiment receipt. if (!response.ok) throw new Error(`${name}: HTTP ${response.status}`); return response.json(); } const provenance = {}; for (const name of ["export.json", "quantization.json", "preparation.json", "training.json"]) { provenance[name] = await receipt(name); } send("loaded", { load_ms, effective_threads: T.env.backends.onnx.wasm.numThreads || null, model_config: modelConfig, provenance, gpu });
// Frozen wire markers and generic names. Header/lookup fixtures correspond // to world.header/toolResult; only the model writes the pet continuation. const memories = ["Sam's favorite color is green", "Sam likes frogs", "Sam has a cat named Bo", "Sam likes drawing"]; const header = "## tokenling chat\npet: Mochi · kid · day 3 · curious, foodie\nowner: Sam\n" + "now: 21:40 night, rainy\nneeds: tummy hungry, fun happy, energy sleepy, clean\n" + "remembers: Sam likes frogs; Sam has a cat named Bo; Sam likes drawing\n" + "today: ate a cookie; played catch (4 stars); had a bath\n---\n"; const history = [ ["hi Mochi!", "hi Sam! [emote:wave]"], ["I saw a frog outside.", "a tiny green visitor! [emote:wow]"], ["do you like rainy days?", "the window goes pitter patter. [emote:happy]"], ["what is on your shelf?", "[look]{shelf: pebble, flower · toy: ball · window: rainy} my little treasures!"], ["the flower is pretty.", "I picked it for you. [emote:blush]"], ["Bo is sleeping on the sofa.", "Bo found a comfy spot!"], ["I am making tea.", "can I sniff the warm cup?"], ["we can draw together later.", "frogs and cookies on the same page! [emote:giggle]"], ["what time is it?", "[clock]{21:40 night} it is getting late!"], ["I like the sound of rain.", "me too, it makes the room cozy."], ["remember that my favorite color is green.", "[remember:Sam's favorite color is green] green like the frog!"], ["do you want to dance?", "a little wiggle before bed. [emote:dance]"], ["I brought a drawing of a moon.", "a moon for our rainy window! [emote:love]"], ["the clouds will clear tomorrow.", "maybe I will see a star."], ["what should we draw next?", "a frog wearing a tiny hat!"], ["I will put the crayons away.", "keep the green one close."], ["Bo just woke up.", "hello Bo! a soft paw wave. [emote:wave]"], ["I think it is almost bedtime.", "my eyes are a little heavy. [emote:sleepy]"], ].flatMap(([owner, pet]) => [`Sam: ${owner}`, `Mochi: ${pet}`]); const scenarios = [ { name: "short_idle", kind: "idle", ctx: 512, max_new: 40, lines: ["Sam: hi!", "Mochi: hello Sam! [emote:wave]"] }, { name: "long_react", kind: "react", ctx: 1024, max_new: 64, lines: [...history, "Sam: what time is it, and what is my cat's name?"] }, ].filter((scenario) => cfg.prompts === "both" || scenario.name.startsWith(cfg.prompts)); const modes = cfg.mode === "both" ? ["greedy", "sampled"] : [cfg.mode]; const lookupEnd = /\[(clock|look|yesterday|recall:[^\]\n]{1,80})\]\[?$/; const lookupTail = /\[(clock|look|yesterday|recall)(?::([^\]\n]{1,80}))?\]$/; const tag = /\[(\w+)(?::([^\]\n]{1,80}))?\](\{[^}\n]*\})?/g; const notCute = /\b(sex\w*|porn\w*|nude\w*|naked|panties|penis|vagina|cock|dick|pussy|boob\w*|tits?|fuck\w*|shit\w*|bitch\w*|cunt|whore|slut|rape\w*|nigg\w*|fag\w*|retard\w*|kill (you|yourself)|suicide)\b/i; const clean = (text) => text.replace(tag, " ").replace(/[\[\]{}]/g, "").replace(/\s+/g, " ").trim(); const stopWords = new Set(["the", "and", "you", "your", "are", "was", "has", "have", "for", "with", "what", "who", "does", "did", "about", "that", "this", "like", "likes", "she", "her", "his", "him", "they", "them", "named", "name", "is"]); function words(text) { return new Set((text.toLowerCase().match(/[a-z0-9']+/g) || []).map((word) => word.replace(/^'+|'+$/g, "")) .filter((word) => word.length >= 3 && !stopWords.has(word)).map((word) => word.length > 3 && word.endsWith("s") ? word.slice(0, -1) : word)); } function lookup(name, argument) { if (name === "clock") return "21:40 night"; if (name === "look") return "shelf: pebble, flower · toy: ball · window: rainy"; if (name === "yesterday") return "dear diary, Sam and I watched the clouds together."; if (name === "recall") { const query = words(argument || ""); const hits = [...memories].reverse().filter((memory) => [...words(memory)].some((word) => query.has(word))); return hits.slice(0, 2).join("; ") || "nothing"; } throw new Error(`unsupported lookup ${name}`); } function fit(scenario, prefix) { const context = Math.min(scenario.ctx, maxContext), budget = context - scenario.max_new - 4; let used = tok.encode(header).length + tok.encode(prefix).length; if (used > budget) throw new Error("pet header/prefix exceeds the context budget"); const kept = []; for (let i = scenario.lines.length - 1; i >= 0; i--) { const count = tok.encode(scenario.lines[i] + "\n").length; if (used + count > budget) break; kept.unshift(scenario.lines[i]); used += count; } return { prompt: header + kept.map((line) => line + "\n").join("") + prefix, context, kept_lines: kept.length, dropped_lines: scenario.lines.length - kept.length }; } const median = (values) => { values = values.filter((value) => value !== null && Number.isFinite(value)).sort((a, b) => a - b); const middle = Math.floor(values.length / 2); return values.length ? (values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2) : null; }; async function turn(scenario, mode, runId, measured) { // Stable turn-local draws across settings; retries never perturb later // repetitions. This is the JS runtime RNG, not Torch bit parity. let turnSeed = cfg.seed; for (const char of runId) turnSeed = Math.imul(turnSeed ^ char.charCodeAt(0), 16777619) >>> 0; if (!T.random?.seed) throw new Error("pinned runtime has no deterministic sampling RNG"); T.random.seed(turnSeed); const turnStart = absoluteNow(), generations = [], attempts = []; let firstToken = null, text = "", finalReason = "length", resolved = 0, unresolved = 0; if (measured) send("turn", { run_id: runId, start: turnStart, scenario: scenario.name, mode }); for (let attempt = 0; attempt < 3; attempt++) { text = ""; const attemptGenerations = []; for (let hop = 0; hop < 4; hop++) { const generationStart = absoluteNow(), fitted = fit(scenario, "Mochi:" + text); const inputs = tok(fitted.prompt, { add_special_tokens: false }); const promptTokens = inputs.input_ids.dims[1]; if (promptTokens + scenario.max_new > fitted.context) throw new Error("encoded prompt exceeds the recorded context budget"); const tokenizationEnd = absoluteNow(), state = { ids: [], times: [], text: "", reason: "length" }; class Stop extends T.StoppingCriteria { _call(inputIds) { const generated = inputIds[0].slice(promptTokens).map(Number); if (generated.length < state.ids.length) throw new Error("generation token history went backwards"); if (generated.length > state.ids.length) { if (generated.length !== state.ids.length + 1) throw new Error("batched token callbacks cannot establish individual decode timings"); const now = absoluteNow(); state.times.push(now); firstToken ??= now; state.ids = generated; state.text = tok.decode(generated, { skip_special_tokens: true }); if (measured) send("visible", { run_id: runId, start: turnStart, text: notCute.test(text + state.text) ? "" : clean((text + state.text) .replace(/\[[^\]\n]*$/, "").replace(/\{[^}\n]*$/, "")) }); } if (state.text.includes("\n")) { state.reason = "line"; return [true]; } if (lookupEnd.test(state.text)) { state.reason = "lookup"; return [true]; } return [false]; } } const output = await model.generate({ ...inputs, max_new_tokens: scenario.max_new, do_sample: mode === "sampled", ...(mode === "sampled" ? { temperature: 0.85, top_k: 40, top_p: 0.92 } : {}), stopping_criteria: [new Stop()] }); const generationEnd = absoluteNow(); const returned = Array.from(output.data).map(Number).slice(promptTokens); if (returned.length !== state.ids.length || returned.some((id, i) => id !== state.ids[i])) { throw new Error("runtime token callbacks and returned IDs disagree; decode timing would be inaccurate"); } if (state.reason === "length" && eosIds.has(state.ids.at(-1))) state.reason = "eos"; else if (state.reason === "length" && state.ids.length < scenario.max_new) state.reason = "runtime_stop"; let part = state.text; if (state.reason === "line") part = part.slice(0, part.indexOf("\n")); if (state.reason === "lookup") part = part.replace(/\]\[$/, "]"); text += part; const decodeTokens = Math.max(0, state.ids.length - 1); const decodeMs = decodeTokens ? state.times.at(-1) - state.times[0] : 0; const record = { attempt, hop, prompt: fitted.prompt, prompt_tokens: promptTokens, context: fitted.context, kept_lines: fitted.kept_lines, dropped_lines: fitted.dropped_lines, max_new_tokens: scenario.max_new, generated_tokens: state.ids.length, generated_ids: state.ids, text: part, stop_reason: state.reason, tokenization_ms: tokenizationEnd - generationStart, first_token_ms: state.times.length ? state.times[0] - generationStart : null, prefill_ms: state.times.length ? state.times[0] - tokenizationEnd : null, decode_tokens: decodeTokens, decode_ms: decodeMs, decode_tps: decodeTokens && decodeMs > 0 ? decodeTokens * 1000 / decodeMs : null, generation_ms: generationEnd - generationStart, token_times_ms: state.times.map((time) => time - generationStart), ...(cfg.profile_memory ? { cache_tensors: lastForward } : {}) }; generations.push(record); attemptGenerations.push(record); finalReason = state.reason; const match = state.reason === "lookup" && text.match(lookupTail); if (!match) break; if (hop === 3) { unresolved++; break; } const result = lookup(match[1], match[2]); record.app_written_lookup = { name: match[1], argument: match[2] || null, result }; text += `{${result}}`; resolved++; } const rejected = notCute.test(text); attempts.push({ attempt, rejected_by_text_filter: rejected, generations: attemptGenerations.length }); if (!rejected) break; text = ""; } text = text.trim().replace(/\s+/g, " "); const turnEnd = absoluteNow(), content = clean(text); if (measured) send("visible", { run_id: runId, start: turnStart, text: content }); const decodeTokens = generations.reduce((sum, generation) => sum + generation.decode_tokens, 0); const decodeMs = generations.reduce((sum, generation) => sum + generation.decode_ms, 0); return { run_id: runId, full_turn_ms: turnEnd - turnStart, first_token_ms: firstToken === null ? null : firstToken - turnStart, prefill_ms: generations[0]?.prefill_ms ?? null, decode_tokens: decodeTokens, decode_ms: decodeMs, decode_tps: decodeTokens && decodeMs > 0 ? decodeTokens * 1000 / decodeMs : null, generated_tokens: generations.reduce((sum, generation) => sum + generation.generated_tokens, 0), final_content_tokens: content ? tok.encode(content).length : 0, visible_characters: [...content].length, visible_utf8_bytes: new TextEncoder().encode(content).byteLength, silent: content.length === 0, text, visible_text: content, stop_reason: finalReason, lookups_resolved: resolved, lookups_unresolved: unresolved, completed: unresolved === 0, safety_attempts: attempts, generations }; } async function fixed(scenario, runId) { const generationStart = absoluteNow(), fitted = fit(scenario, "Mochi:"); const inputs = tok(fitted.prompt, { add_special_tokens: false }); const promptTokens = inputs.input_ids.dims[1], outputTokens = 32; if (promptTokens < 1 || promptTokens + outputTokens > fitted.context) { throw new Error("fixed workload exceeds the original fitted context budget"); } const tokenizationEnd = absoluteNow(), state = { ids: [], times: [] }; class Observe extends T.StoppingCriteria { _call(inputIds) { const generated = inputIds[0].slice(promptTokens).map(Number); if (generated.length < state.ids.length || generated.length > state.ids.length + 1) { throw new Error("fixed workload cannot establish individual decode timings"); } if (generated.length > state.ids.length) { state.times.push(absoluteNow()); state.ids = generated; } return [false]; } } // EOS is disabled per call, never on the shared model configuration. // No past_key_values are supplied: each repetition starts with empty KV, // while model/session and device-resident decode caches remain in use. const output = await model.generate({ ...inputs, max_new_tokens: outputTokens, do_sample: false, eos_token_id: null, forced_eos_token_id: null, stopping_criteria: [new Observe()] }); const generationEnd = absoluteNow(); const returned = Array.from(output.data).map(Number).slice(promptTokens); if (returned.length !== outputTokens || returned.length !== state.ids.length || returned.some((id, i) => id !== state.ids[i])) { throw new Error("fixed workload did not return exactly 32 observed tokens"); } const decodeTokens = state.ids.length - 1, decodeMs = state.times.at(-1) - state.times[0]; return { run_id: runId, prompt: fitted.prompt, prompt_tokens: promptTokens, context: fitted.context, kept_lines: fitted.kept_lines, dropped_lines: fitted.dropped_lines, max_new_tokens: outputTokens, generated_tokens: state.ids.length, generated_ids: state.ids, stop_reason: "fixed_length", tokenization_ms: tokenizationEnd - generationStart, first_token_ms: state.times[0] - generationStart, prefill_ms: state.times[0] - tokenizationEnd, decode_tokens: decodeTokens, decode_ms: decodeMs, decode_tps: decodeMs > 0 ? decodeTokens * 1000 / decodeMs : null, generation_ms: generationEnd - generationStart, total_ms: generationEnd - generationStart, full_turn_ms: generationEnd - generationStart, token_times_ms: state.times.map((time) => time - generationStart) }; } if (cfg.profile_memory) { const short = scenarios.find((scenario) => scenario.name === "short_idle"); const long = scenarios.find((scenario) => scenario.name === "long_react"); if (!short || !long) throw new Error("memory profiling requires both original pet scenarios"); const warmup = await turn(short, "sampled", "profile/warmup", false); await checkpoint("loaded_warmed", { cache_tensors: lastForward, result: warmup }); for (const [phase, scenario] of [["short_turn", short], ["long_turn", long]]) { const result = await turn(scenario, "sampled", `profile/${phase}`, false); await checkpoint(phase, { result }); } const rolling = { ...short, name: "normal_react", kind: "react", ctx: 1024, max_new: 64, lines: [...short.lines] }; for (let i = 1; i <= 50; i++) { const owner = ["hello little friend", "what can you see outside?", "I like drawing frogs", "how are you feeling?", "what should we draw?"][(i - 1) % 5]; rolling.lines.push(`Sam: ${owner}`); const result = await turn(rolling, "sampled", `profile/normal/${i}`, false); rolling.lines.push(`Mochi: ${result.text}`); send("private-turn", { result, normal_turn: i }); if ([1, 10, 50].includes(i)) await checkpoint(`after_${i}_normal_turns`, { result }); // Diagnostic only: allow a worker task boundary between exactly the // same seeded generations. This is not forced GC, an allocator fix, // or a production scheduling/default change. if (cfg.profile_yield_between_turns) await new Promise((resolve) => setTimeout(resolve, 0)); } send("done", { resource_timing: resources(), total_worker_ms: performance.now() }); return; } const worklist = { version: "pet-fixed-v1", fixed_output_tokens: 32, scenarios: scenarios.map((scenario) => ({ scenario: scenario.name, kind: scenario.kind, context: Math.min(scenario.ctx, maxContext), pet_max_new_tokens: scenario.max_new, fixed_prompt_fit_budget: scenario.max_new })), pet_modes: cfg.workload === "fixed" ? [] : modes, fixed_modes: cfg.workload === "pet" ? [] : ["greedy"], repeats: cfg.repeats, warmup: cfg.warmup, order: "all pet scenario/mode runs, then all fixed scenario runs; sequential" }; send("worklist", { worklist }); if (cfg.workload !== "fixed") for (const scenario of scenarios) for (const mode of modes) { send("phase", { phase: `warming ${scenario.name}/${mode}` }); for (let i = 0; i < cfg.warmup; i++) await turn(scenario, mode, `warmup/${scenario.name}/${mode}/${i}`, false); const measurements = []; for (let i = 0; i < cfg.repeats; i++) { send("phase", { phase: `${scenario.name}/${mode} repeat ${i + 1}/${cfg.repeats}` }); measurements.push(await turn(scenario, mode, `${scenario.name}/${mode}/${i}`, true)); } const metrics = ["full_turn_ms", "first_token_ms", "prefill_ms", "decode_ms", "decode_tps", "generated_tokens", "final_content_tokens", "visible_characters", "visible_utf8_bytes"]; send("result", { result: { scenario: scenario.name, mode, kind: scenario.kind, context: Math.min(scenario.ctx, maxContext), max_new_tokens: scenario.max_new, repeats: measurements, medians: Object.fromEntries(metrics.map((metric) => [metric, median(measurements.map((measurement) => measurement[metric]))])) } }); } if (cfg.workload !== "pet") for (const scenario of scenarios) { send("phase", { phase: `warming fixed/${scenario.name}/greedy` }); for (let i = 0; i < cfg.warmup; i++) await fixed(scenario, `warmup/fixed/${scenario.name}/${i}`); const measurements = []; for (let i = 0; i < cfg.repeats; i++) { send("phase", { phase: `fixed/${scenario.name} repeat ${i + 1}/${cfg.repeats}` }); measurements.push(await fixed(scenario, `fixed/${scenario.name}/${i}`)); } const metrics = ["prompt_tokens", "generated_tokens", "decode_tokens", "tokenization_ms", "first_token_ms", "prefill_ms", "decode_ms", "decode_tps", "generation_ms", "total_ms", "full_turn_ms"]; const first = measurements[0]; send("fixed_result", { result: { scenario: scenario.name, mode: "greedy", kind: scenario.kind, diagnostic_only: true, behavior_claim: false, prompt: first.prompt, prompt_tokens: first.prompt_tokens, context: first.context, kept_lines: first.kept_lines, dropped_lines: first.dropped_lines, max_new_tokens: 32, warmup: cfg.warmup, repeats: measurements, medians: Object.fromEntries(metrics.map((metric) => [metric, median(measurements.map((measurement) => measurement[metric]))])) } }); } send("done", { resource_timing: performance.getEntriesByType("resource").map((entry) => ({ name: entry.name, duration_ms: entry.duration, transfer_bytes: entry.transferSize, encoded_bytes: entry.encodedBodySize, decoded_bytes: entry.decodedBodySize })), total_worker_ms: performance.now() }); } catch (error) { send("error", { error: { message: error?.message || String(error), stack: error?.stack || null } }); } };}
try { const q = new URLSearchParams(location.search); function integer(name, fallback, min, max) { if (!q.has(name)) return fallback; const value = q.get(name); if (!/^\d+$/.test(value)) throw new Error(`${name} must be an explicit integer`); const number = Number(value); if (!Number.isSafeInteger(number) || number < min || number > max) throw new Error(`${name} must be ${min}..${max}`); return number; } function choice(name, fallback, choices) { const value = q.has(name) ? q.get(name) : fallback; if (!choices.includes(value)) throw new Error(`${name} must be ${choices.join("|")}`); return value; } const model = q.get("model") || "@MODEL_ID@"; if (!model.split("/").every((part) => /^[a-zA-Z0-9_.-]+$/.test(part) && part !== "." && part !== "..")) throw new Error("model must be an ID/path under the configured local models directory"); const cfg = { model, dtype: choice("dtype", @DTYPE@, ["q4", "q8", "fp32"]), device: choice("device", "wasm", ["wasm", "webgpu"]), threads: integer("threads", 0, 0, 256), qdq: integer("qdq", 0, 0, 1), prepack: integer("prepack", 0, 0, 1), repeats: integer("repeats", 3, 1, 100), warmup: integer("warmup", 1, 0, 20), seed: integer("seed", 0, 0, 4294967295), timeout_ms: integer("timeout", 120000, 1000, 3600000), prompts: choice("prompts", "both", ["short", "long", "both"]), mode: choice("mode", "both", ["greedy", "sampled", "both"]), workload: choice("workload", "pet", ["pet", "fixed", "both"]), profile_memory: q.get("profile-memory") === "1", profile_yield_between_turns: q.get("profile-memory") === "1" && q.get("profile-yield-between-turns") === "1", runtime: new URL("@RUNTIME@/", location.href).href, models: new URL("@MODELS@", location.href).href }; if (!cfg.models.endsWith("/")) throw new Error("configured models URL must end in /"); res.config = { ...cfg, benchmark_version: "pet-fixed-v1", fresh_page_per_setting: true, browser_cache: false, fixed_definition: "diagnostic compute only; 32 greedy tokens including EOS, ignoring all wire stops; original pet prompt fitting; no output usefulness/behavior claim", fixed_timing_definition: "prefill is tokenization-end to first sampled token; decode is first-to-last token for 31 decode tokens; total/full timing includes tokenization and generate return; fresh KV per repetition, resident session retained", sampling: { temperature: 0.85, top_k: 40, top_p: 0.92 }, decode_definition: "sum of per-generation last-token minus first-token intervals; excludes all prefills and app lookup work", visible_definition: "first nonempty cleaned pet text delivered to the page, excluding filtered unsafe text; first_visible_frame_ms is an after-frame upper bound", full_turn_definition: "tokenization + normal line/EOS/budget stopping + up to three lookup round trips + up to three text-filter attempts", fallback_policy: "no automatic device/thread fallback; WebGPU software adapters fail; runtime may partition shape/unsupported kernels" }; render(); const pageStart = performance.now(); if (cfg.profile_memory) { res.memory_observations = []; res.normal_turns = []; await new Promise((resolve) => { window.__subtokenlingMemory = { start: resolve, resume: () => worker?.postMessage({ type: "private-memory-continue" }), terminate: () => { worker?.terminate(); clearTimeout(timer); res.phase = "worker_terminated"; }, }; res.phase = "empty"; }); } timer = setTimeout(() => fail(new Error(`benchmark exceeded the ${cfg.timeout_ms} ms whole-page deadline`)), cfg.timeout_ms); workerUrl = URL.createObjectURL(new Blob(["(" + benchmarkWorker.toString() + ")();"], { type: "text/javascript" })); worker = new Worker(workerUrl, { type: "module" }); worker.onerror = (event) => fail(event.error || new Error(event.message || "benchmark worker failed")); worker.onmessageerror = () => fail(new Error("benchmark worker message could not be decoded")); worker.onmessage = ({ data }) => { if (res.status !== "running") return; if (data.type === "error") { fail(data.error); return; } if (data.type === "private-memory") { res.memory_observations.push(data); res.phase = data.phase; } if (data.type === "private-turn") res.normal_turns.push(data); if (data.type === "phase") res.phase = data.phase; if (data.type === "worklist") res.worklist = data.worklist; if (data.type === "fixed_result") res.fixed_results.push(data.result); if (data.type === "progress") res.progress = data.progress; if (data.type === "hardware") res.hardware.webgpu = data.gpu; if (data.type === "loaded") { res.load_ms = data.load_ms; res.config.effective_threads = data.effective_threads; res.model_config = data.model_config; res.provenance = data.provenance; res.hardware.webgpu = data.gpu; } if (data.type === "turn") { activeRun = data.run_id; shown.textContent = ""; visible.set(data.run_id, { first_visible_content_ms: null, first_visible_frame_ms: null }); } if (data.type === "visible") { shown.textContent = data.text; const metric = visible.get(data.run_id); if (data.text && metric && metric.first_visible_content_ms === null) { metric.first_visible_content_ms = performance.timeOrigin + performance.now() - data.start; requestAnimationFrame(() => { if (activeRun !== data.run_id || !shown.textContent) return; requestAnimationFrame(() => { if (activeRun !== data.run_id || !shown.textContent) return; metric.first_visible_frame_ms = performance.timeOrigin + performance.now() - data.start; }); }); } } if (data.type === "result") { const result = data.result; for (const measurement of result.repeats) Object.assign(measurement, visible.get(measurement.run_id)); for (const name of ["first_visible_content_ms", "first_visible_frame_ms"]) { const values = result.repeats.map((measurement) => measurement[name]).filter((value) => value !== null && Number.isFinite(value)).sort((a, b) => a - b); const middle = Math.floor(values.length / 2); result.medians[name] = values.length ? (values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2) : null; } res.results.push(result); } if (data.type === "done") { clearTimeout(timer); res.status = "success"; res.phase = "complete"; res.total_page_ms = performance.now() - pageStart; res.resource_timing = data.resource_timing; const median = (values) => { values = values.filter((value) => Number.isFinite(value) && value > 0).sort((a, b) => a - b); const middle = Math.floor(values.length / 2); return values.length ? (values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2) : null; }; res.summary = { sampled_turn_ms: median(res.results.filter((result) => result.mode === "sampled").flatMap((result) => result.repeats.map((measurement) => measurement.full_turn_ms))), greedy_turn_ms: median(res.results.filter((result) => result.mode === "greedy").flatMap((result) => result.repeats.map((measurement) => measurement.full_turn_ms))), }; // Deliberately leave the idle model worker alive for resident-memory // measurements. Closing this fresh benchmark page releases it. URL.revokeObjectURL(workerUrl); } if (data.type !== "visible" && data.type !== "progress") render(); }; worker.postMessage(cfg);} catch (error) { fail(error); }</script>