Something went wrong. Try again.
a tiny pet with a GPT-2 brain, living in your browser
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546<!doctype html><!-- Brain speed benchmark, included by site/build.sh at <base>/bench.html. No link is added to the app. Uses the release's model and runtime by default. Loads the model with transformers.js (WebAssembly), then reports greedy and sampled decoding speed for 32 tokens and the prefill time of a short pet prompt as JSON in the page and in window.res. Query: ?threads=N (default: ONNX Runtime's choice), ?dtype=q4, ?model=ID, ?qdq=1, ?prepack=1 Memory: window.mem() reports performance.measureUserAgentSpecificMemory().--><meta charset="utf-8"><title>tokenling bench</title><meta name="viewport" content="width=device-width, initial-scale=1"><link rel="icon" href="data:,"><pre id="out">loading…</pre><script type="module">const T = await import("@RUNTIME@/transformers.min.js");Object.assign(T.env, { allowRemoteModels: false, allowLocalModels: true, localModelPath: "@MODELS@", useBrowserCache: false });T.env.backends.onnx.wasm.wasmPaths = "@RUNTIME@/";const q = new URLSearchParams(location.search);if (q.get("threads")) T.env.backends.onnx.wasm.numThreads = +q.get("threads");const id = q.get("model") || "@MODEL_ID@", dtype = q.get("dtype") || "q4";const t0 = performance.now();const tok = await T.AutoTokenizer.from_pretrained(id);// Same session options as web/brain.ts; ?qdq=1 leaves ORT's QDQ fusion on (MatMulNBits),// ?prepack=1 allows weight prepacking.const session_options = q.get("qdq") ? {} : { extra: { session: { disable_quant_qdq: "1", ...(q.get("prepack") ? {} : { disable_prepacking: "1" }) } } };const model = await T.AutoModelForCausalLM.from_pretrained(id, { device: "wasm", dtype, session_options });const res = { model: id, dtype, load_ms: Math.round(performance.now() - t0), threads: T.env.backends.onnx.wasm.numThreads, cores: navigator.hardwareConcurrency, isolated: crossOriginIsolated };const prompt = "## tokenling chat\npet: Mochi · kid · day 3 · curious, foodie\nowner: Sam\nnow: 21:40 night, rainy\n" + "needs: tummy hungry, fun happy, energy sleepy, clean\n---\nSam: hi!\nMochi:";const inputs = tok(prompt);await model.generate({ ...inputs, max_new_tokens: 2, do_sample: false });for (const [name, opts] of [["greedy_tps", { do_sample: false }], ["sampled_tps", { do_sample: true, temperature: 0.85, top_k: 40, top_p: 0.92 }]]) { const t = performance.now(); await model.generate({ ...inputs, max_new_tokens: 32, min_new_tokens: 32, ...opts }); res[name] = +(32 / ((performance.now() - t) / 1000)).toFixed(1);}const t = performance.now();await model({ input_ids: inputs.input_ids, attention_mask: inputs.attention_mask });Object.assign(res, { prompt_tokens: inputs.input_ids.dims[1], prefill_ms: Math.round(performance.now() - t) });document.getElementById("out").textContent = JSON.stringify(res, null, 1);window.res = res;window.mem = async () => Math.round((await performance.measureUserAgentSpecificMemory()).bytes / 1e6);</script>