From 00d993a50ea0c9f5b38e4733cf8a08dcdf1b1db6 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Mon, 21 Sep 2026 14:51:57 -0700 Subject: [PATCH] Make computer-use benchmarks compact and trace verified drag batches --- slab/bin/computer-use-smoke.mjs | 13 +- slab/lib/computer-use-bench-report.mjs | 49 ++++++ slab/lib/native-drag-run.mjs | 30 +++- slab/mathplay/README.md | 19 ++ .../blueberry-drag-followup-2026-09-21.json | 149 ++++++++++++++++ .../blueberry-next-run-2026-09-21.json | 163 ++++++++++++++++++ slab/test/computer-use-bench-report.test.mjs | 32 ++++ slab/test/fixtures/frame-drag.html | 7 + 8 files changed, 450 insertions(+), 12 deletions(-) create mode 100644 slab/lib/computer-use-bench-report.mjs create mode 100644 slab/mathplay/results/blueberry-drag-followup-2026-09-21.json create mode 100644 slab/mathplay/results/blueberry-next-run-2026-09-21.json create mode 100644 slab/test/computer-use-bench-report.test.mjs diff --git a/slab/bin/computer-use-smoke.mjs b/slab/bin/computer-use-smoke.mjs index 07bd78626b..242ff31b4c 100644 --- a/slab/bin/computer-use-smoke.mjs +++ b/slab/bin/computer-use-smoke.mjs @@ -15,6 +15,7 @@ import { playWordplay } from "../lib/wordplay-run.mjs"; import { playMathplay } from "../lib/mathplay-run.mjs"; import { playNativeDrag } from '../lib/native-drag-run.mjs'; import { captureFrame as nativeCapture } from './frame.mjs'; +import { summarizeBenchmark } from '../lib/computer-use-bench-report.mjs'; const dragBench = process.argv.includes('--drag-bench'); const holdBench = process.argv.includes('--hold-bench'); @@ -36,6 +37,8 @@ if (wordplay && native) throw new Error('Use --wordplay for browser play or --na const root = resolve(import.meta.dirname, "../.."); const dir = await mkdtemp(join(tmpdir(), "computer-use-smoke-")); const report = { at: new Date().toISOString(), browser: [], native: [], jev: [] }; +const reportName = mathplay ? 'mathplay' : wordplay ? 'wordplay' : 'computer-use-smoke'; +const runName = `${reportName}-${report.at.replace(/[:.]/g, '-')}-${process.pid}`; report.installedFrame = installedFrame; if (nativeClickBench) { report.nativeInputMode = legacyInput ? 'legacy' : compactBench ? 'compact' : 'resident'; report.transport = process.env.SLAB_FRAME_TRANSPORT || 'socket'; } const children = []; @@ -277,7 +280,7 @@ try { } } // Only the generated fixture is retained; the browser profile is removed. - const artifact = join(tmpdir(), mathplay ? "mathplay-verified.png" : wordplay ? "wordplay-verified.png" : "computer-use-smoke-verified.png"); + const artifact = join(tmpdir(), `${runName}-verified.png`); await page.screenshot({ path: artifact }); report.screenshot = artifact; if (!wordplay && !mathplay && !dragBench) { const navigation = JSON.parse(text(await call("puppet_click", { ...browser, locator: { role: "link", name: "Second page" }, after: { locator: { text: "Navigation verified" } } }))); @@ -293,6 +296,10 @@ try { await context?.close(); if (site) await new Promise(resolve => site.close(resolve)); await rm(dir, { recursive: true, force: true }); - await writeFile(join(tmpdir(), mathplay ? "mathplay-report.json" : wordplay ? "wordplay-report.json" : "computer-use-smoke-report.json"), JSON.stringify(report, null, 2)); - console.log(JSON.stringify(report, null, 2)); + const reportPath = join(tmpdir(), `${runName}.json`); + const raw = JSON.stringify(report, null, 2); + await writeFile(reportPath, raw); + // Retain the historical latest-report path for existing local consumers. + await writeFile(join(tmpdir(), `${reportName}-report.json`), raw); + console.log(JSON.stringify(process.argv.includes('--full-report') ? report : summarizeBenchmark(report, reportPath), null, 2)); } diff --git a/slab/lib/computer-use-bench-report.mjs b/slab/lib/computer-use-bench-report.mjs new file mode 100644 index 0000000000..1d042e8acf --- /dev/null +++ b/slab/lib/computer-use-bench-report.mjs @@ -0,0 +1,49 @@ +// Keep receipts and event traces in the report file, not routine tool output. +function timings(values) { + const sorted = values.filter(Number.isFinite).sort((a, b) => a - b); + const n = sorted.length; + if (!n) return { samples: 0 }; + return { samples: n, + medianMs: +((sorted[Math.floor((n - 1) / 2)] + sorted[Math.floor(n / 2)]) / 2).toFixed(2), + p95Ms: sorted[Math.ceil(n * 0.95) - 1], maxMs: sorted.at(-1), + under100ms: sorted.filter(ms => ms < 100).length }; +} + +function groups(samples, key, verified) { + const grouped = new Map(); + for (const sample of samples || []) { + const name = key(sample); + if (!grouped.has(name)) grouped.set(name, []); + grouped.get(name).push(sample); + } + return Object.fromEntries([...grouped].map(([name, rows]) => [name, { + attempted: rows.length, + verified: rows.filter(verified).length, + // Failed operations have a timeout cost, not a successful action latency. + successful: timings(rows.filter(verified).map(row => row.ms)), + failed: timings(rows.filter(row => !verified(row)).map(row => row.ms)), + }])); +} + +export function summarizeBenchmark(report, reportPath) { + const { rounds, ...game } = report.game || {}; + return { + at: report.at, ok: report.ok, reportPath, + ...(report.error ? { error: report.error.slice(0, 600) } : {}), + scope: 'Tool action and result checks; excludes model inference and fixture startup.', + installedFrame: report.installedFrame, + ...(report.game ? { game } : {}), + ...(report.nativeClickLatency ? { clicks: groups(report.nativeClickLatency, + row => `hold=${row.holdMs ?? row.nativeInput?.holdMs ?? 'default'},settle=${row.settleMs}`, + row => Number.isInteger(row.verifiedCount) && row.nativeInput?.verification?.ok !== false) } : {}), + ...(report.dragSamples ? { dragBatches: report.dragBatches ?? 1, + drags: groups(report.dragSamples, + row => `hold=${row.holdMs},path=${row.durationMs},release=${row.releaseMs}`, + row => row.verified !== false && row.receipt?.verification?.ok === true && row.receipt.releasePosted === true && + /^Drops: [1-9]\d*$/.test(row.drops) && row.released === row.drops.replace('Drops:', 'Released:') && row.rejected === 'Rejected: 0') } : {}), + ...(report.clickLatency ? { browserClicks: (({timesMs, ...summary}) => summary)(report.clickLatency) } : {}), + checks: { nativeGuard: report.nativeGuard, compactRecovery: report.compactRecovery, + multiClick: report.multiClickChecks }, + screenshot: report.screenshot, + }; +} diff --git a/slab/lib/native-drag-run.mjs b/slab/lib/native-drag-run.mjs index 92ec6dbab6..f70638e813 100644 --- a/slab/lib/native-drag-run.mjs +++ b/slab/lib/native-drag-run.mjs @@ -9,6 +9,18 @@ export async function playNativeDrag(call,page,context,report){ await page.waitForFunction(()=>window.outerWidth===1000&&window.screenX===40,null,{timeout:3000}); await delay(200); } finally {await geometry.detach();} + const useDefaults=process.env.SLAB_DRAG_DEFAULTS==='1'; + const durations=(process.env.SLAB_DRAG_SWEEP||'128,64,32,16,8,0').split(',').map(Number); + const repeats=Number(process.env.SLAB_DRAG_REPEATS||25),holdMs=Number(process.env.SLAB_DRAG_HOLD||0),releaseMs=Number(process.env.SLAB_DRAG_RELEASE||0); + const batches=Number(process.env.SLAB_DRAG_BATCHES||1); + assert.ok(durations.length&&durations.every(n=>Number.isFinite(n)&&n>=0&&n<=2000)); + assert.ok(Number.isInteger(repeats)&&repeats>0&&repeats<=1000); + assert.ok(Number.isInteger(batches)&&batches>0&&batches<=20); + assert.ok(durations.length*repeats*batches<=1000,'At most 1000 drag attempts per run'); + report.dragSamples=[]; + report.dragBatches=batches; + for(let batch=1;batch<=batches;batch++){ + if(batch>1)await page.reload(); await page.bringToFront(); const first=text(await call('frame',{machine:'local',fast:false,visual:false},'native')); assert.match(first,/Frame drag check/); @@ -16,30 +28,30 @@ export async function playNativeDrag(call,page,context,report){ const target=first.match(/AXButton «Drop here» @\((-?\d+),(-?\d+)\)/); const release=first.match(/«Released: [0O]» @\((-?\d+),(-?\d+)\)/); assert.ok(source&&target&&release,'Frame must observe both drag endpoints and release counter: '+first); - let observation=JSON.parse(first.match(/^observation: (.+)$/m)[1]), expected=0; - const useDefaults=process.env.SLAB_DRAG_DEFAULTS==='1'; - const durations=(process.env.SLAB_DRAG_SWEEP||'128,64,32,16,8,0').split(',').map(Number); - const repeats=Number(process.env.SLAB_DRAG_REPEATS||25),holdMs=Number(process.env.SLAB_DRAG_HOLD||0),releaseMs=Number(process.env.SLAB_DRAG_RELEASE||0); - assert.ok(durations.length&&durations.every(n=>Number.isFinite(n)&&n>=0&&n<=2000)); - assert.ok(Number.isInteger(repeats)&&repeats>0&&repeats<=1000); - report.dragSamples=[]; + let observation=JSON.parse(first.match(/^observation: (.+)$/m)[1]),expected=0; for(const durationMs of durations)for(let i=0;iwindow.dragTrace)}; report.dragSamples.push(sample); assert.match(body,/capture: verified/,'Drag/release not verified; input not repeated'); assert.equal(receipt.kind,'drag');assert.equal(receipt.releasePosted,true); if(useDefaults){assert.equal(receipt.holdMs,0);assert.equal(receipt.durationMs,32);assert.equal(receipt.releaseMs,32);} assert.equal(sample.drops,`Drops: ${expected}`);assert.equal(sample.released,`Released: ${expected}`); assert.equal(sample.rejected,'Rejected: 0'); + assert.ok(sample.events.every(event=>event.trusted),'Fixture receipts must come from real input'); + for(const [kind,target] of [['dragstart','source'],['drop','target'],['dragend','source']]) + assert.equal(sample.events.filter(event=>event.kind===kind&&event.target===target).length,1,`One ${kind} per drag`); + sample.verified=true; observation=JSON.parse(body.match(/^observation: (.+)$/m)[1]); } await delay(100); assert.equal(await page.locator('#drops').textContent(),`Drops: ${expected}`); assert.equal(await page.locator('#releases').textContent(),`Released: ${expected}`); + } } diff --git a/slab/mathplay/README.md b/slab/mathplay/README.md index 5a4ba23857..12addda9e8 100644 --- a/slab/mathplay/README.md +++ b/slab/mathplay/README.md @@ -17,6 +17,11 @@ Every click must produce a new correct-round result and exactly one score increment. It uses no answer key, game-state access, DOM mutation, or Jev call. This measures the tool loop; it does not include language-model inference. +Benchmark stdout contains a compact JSON summary and `reportPath`. The unique +report file retains every sample, native receipt, and fixture event trace; +`--full-report` also prints it. The historical `*-report.json` file still holds +the latest run. Successful and failed timing samples are summarized separately. + Blueberry, September 21, 2026: two runs scored **60/60**. All clicks verified in under 100 ms. Run medians were **34.03 / 48.97 ms**, p95 **52.94 / 68.00 ms**, and maxima **55.56 / 92.10 ms**. The 30-round loops took **1.13 / 1.50 seconds** @@ -128,3 +133,17 @@ median / 206.67 ms p95**. Unverified and cross-window movement stays at 500 ms; this experiment does not establish Finder/file-transfer minimum timings. [Complete gesture evidence](results/blueberry-gesture-timing-2026-09-21.json). + +For repeated fresh-page checks, set `SLAB_DRAG_BATCHES`; repeat counts apply to +each batch. The run stops on the first unverified drag, with no replay. Keep the +mouse and keyboard idle; mark interrupted trials as inconclusive. + +```sh +SLAB_DRAG_DEFAULTS=1 SLAB_DRAG_SWEEP=32 SLAB_DRAG_REPEATS=20 SLAB_DRAG_BATCHES=5 node slab/bin/computer-use-smoke.mjs --drag-bench --installed-frame +``` + +The follow-up clean run passed 100/100 drops across five fresh pages at the +existing 32/32 ms timing: 140.20 ms median / 183.91 ms p95. A user reported +manual intervention during surrounding experiments, so those trials do not +justify changing the default. Native click verification remains the largest +measured click stage (30.55 ms median in the preceding 100-click run). diff --git a/slab/mathplay/results/blueberry-drag-followup-2026-09-21.json b/slab/mathplay/results/blueberry-drag-followup-2026-09-21.json new file mode 100644 index 0000000000..a19f269579 --- /dev/null +++ b/slab/mathplay/results/blueberry-drag-followup-2026-09-21.json @@ -0,0 +1,149 @@ +{ + "at": "2026-09-21T21:51:21.411Z", + "host": "blueberry", + "runtimeRevision": "77418028cd5d5f68c65b33e6fd99ef1461c4b875", + "defaultsChanged": false, + "interference": "User reported brief manual intervention during the timing experiments. Exact interval is unknown; exclude 32/64 and 64/32 follow-ups from default selection. The earlier failed 32/32 trial remains recorded, but its cause is unresolved.", + "decision": "Keep 32 ms movement / 32 ms destination dwell; clean rerun passed 100 drags across five fresh pages. No new execution-speed gain claimed.", + "output": { + "rawBytes": 254980, + "summaryBytes": 730, + "reductionPercent": 99.71 + }, + "runs": { + "clean32": { + "at": "2026-09-21T21:47:16.514Z", + "ok": true, + "reportPath": "/Users/jas/.local/share/slab/computer-use/benchmarks/blueberry-2026-09-21-followup/clean32.json", + "scope": "Tool action and result checks; excludes model inference and fixture startup.", + "installedFrame": true, + "dragBatches": 5, + "drags": { + "hold=0,path=32,release=32": { + "attempted": 100, + "verified": 100, + "successful": { + "samples": 100, + "medianMs": 140.19, + "p95Ms": 183.91, + "maxMs": 227.39, + "under100ms": 0 + }, + "failed": { + "samples": 0 + } + } + }, + "checks": {}, + "screenshot": "/var/folders/nl/5tj_95ln62358sk__f6kthjr0000gn/T/computer-use-smoke-verified.png" + }, + "interrupted32_64": { + "at": "2026-09-21T21:45:12.280Z", + "ok": false, + "reportPath": "/Users/jas/.local/share/slab/computer-use/benchmarks/blueberry-2026-09-21-followup/interrupted32_64.json", + "error": "Drag/release not verified; input not repeated", + "scope": "Tool action and result checks; excludes model inference and fixture startup.", + "installedFrame": true, + "dragBatches": 1, + "drags": { + "hold=0,path=32,release=64": { + "attempted": 69, + "verified": 68, + "successful": { + "samples": 68, + "medianMs": 179.53, + "p95Ms": 208.94, + "maxMs": 247.06, + "under100ms": 0 + }, + "failed": { + "samples": 1, + "medianMs": 726.62, + "p95Ms": 726.62, + "maxMs": 726.62, + "under100ms": 0 + } + } + }, + "checks": {} + }, + "interrupted64_32": { + "at": "2026-09-21T21:45:40.380Z", + "ok": true, + "reportPath": "/Users/jas/.local/share/slab/computer-use/benchmarks/blueberry-2026-09-21-followup/interrupted64_32.json", + "scope": "Tool action and result checks; excludes model inference and fixture startup.", + "installedFrame": true, + "dragBatches": 1, + "drags": { + "hold=0,path=64,release=32": { + "attempted": 100, + "verified": 100, + "successful": { + "samples": 100, + "medianMs": 175.81, + "p95Ms": 210.62, + "maxMs": 224.26, + "under100ms": 0 + }, + "failed": { + "samples": 0 + } + } + }, + "checks": {}, + "screenshot": "/var/folders/nl/5tj_95ln62358sk__f6kthjr0000gn/T/computer-use-smoke-verified.png" + }, + "integration": { + "at": "2026-09-21T21:49:55.809Z", + "ok": true, + "reportPath": "/Users/jas/.local/share/slab/computer-use/benchmarks/blueberry-2026-09-21-followup/integration.json", + "scope": "Tool action and result checks; excludes model inference and fixture startup.", + "installedFrame": true, + "dragBatches": 2, + "drags": { + "hold=0,path=32,release=32": { + "attempted": 10, + "verified": 10, + "successful": { + "samples": 10, + "medianMs": 130.42, + "p95Ms": 182.96, + "maxMs": 182.96, + "under100ms": 0 + }, + "failed": { + "samples": 0 + } + } + }, + "checks": {}, + "screenshot": "/var/folders/nl/5tj_95ln62358sk__f6kthjr0000gn/T/computer-use-smoke-verified.png" + }, + "math": { + "at": "2026-09-21T21:51:20.403Z", + "ok": true, + "reportPath": "/Users/jas/.local/share/slab/computer-use/benchmarks/blueberry-2026-09-21-followup/math.json", + "scope": "Tool action and result checks; excludes model inference and fixture startup.", + "installedFrame": false, + "game": { + "complete": true, + "correct": 30, + "initialObservationMs": 205.85, + "elapsedMs": 1028.85, + "medianClickMs": 33.63, + "p95ClickMs": 38.14, + "maxClickMs": 55.7, + "under100ms": 30, + "scope": "Local HTTP/MCP, rendered-question parsing, deterministic arithmetic, click, verification and next observation. Excludes language-model inference." + }, + "checks": {}, + "screenshot": "/var/folders/nl/5tj_95ln62358sk__f6kthjr0000gn/T/mathplay-2026-09-21T21-51-20-403Z-57460-verified.png" + } + }, + "validation": { + "summaryTests": 2, + "liveTraceChecks": 10, + "freshPageDragChecks": 100, + "mathAnswers": 30 + } +} diff --git a/slab/mathplay/results/blueberry-next-run-2026-09-21.json b/slab/mathplay/results/blueberry-next-run-2026-09-21.json new file mode 100644 index 0000000000..0ed71ad143 --- /dev/null +++ b/slab/mathplay/results/blueberry-next-run-2026-09-21.json @@ -0,0 +1,163 @@ +{ + "measuredAt": "2026-09-21T21:43:18.743Z", + "host": "blueberry", + "revision": "77418028cd5d5f68c65b33e6fd99ef1461c4b875", + "scope": "Warm local HTTP/MCP action loops; native input uses installed Frame and resident Swift. Puppet source hashes match installed release. Includes verification; excludes model inference and fixture startup. Math uses a headless disposable browser; clicks and drags use a visible disposable Chrome window.", + "clicks": { + "ok": true, + "samples": 100, + "medianMs": 44.56, + "p95Ms": 66.08, + "maxMs": 100.11, + "under100ms": 99, + "prior": { + "n": 25, + "medianMs": 46.87, + "p95Ms": 72.29, + "maxMs": 93.07, + "within24fpsBudget": 7 + }, + "stages": { + "guard": { + "samples": 100, + "medianMs": 4.32, + "p95Ms": 10.203208, + "maxMs": 26.722125, + "under100ms": 100 + }, + "input": { + "samples": 100, + "medianMs": 0.03, + "p95Ms": 0.066917, + "maxMs": 0.204417, + "under100ms": 100 + }, + "verification": { + "samples": 100, + "medianMs": 30.55, + "p95Ms": 46.314584, + "maxMs": 77.831833, + "under100ms": 100 + } + }, + "guard": { + "ms": 7.26, + "nativeMs": 5.850834, + "pixels": false, + "staleRejected": true, + "movedRejected": true, + "counterUnchanged": true + }, + "recovery": { + "alreadyTrueRejected": true, + "wrongRoleRejected": true, + "timeoutReturnedPixels": true, + "clickNotRepeated": true, + "consumedRejected": true + }, + "multiClick": [ + { + "count": 2, + "holdMs": 0, + "verifiedCount": 104 + }, + { + "count": 3, + "holdMs": 0, + "verifiedCount": 107 + } + ] + }, + "dragsDefault": { + "ok": false, + "attempted": 1, + "verified": 0, + "holdMs": 0, + "durationMs": 32, + "releaseMs": 32, + "error": "Drag/release not verified; input not repeated", + "receipt": { + "holdMs": 0, + "durationMs": 32, + "releaseMs": 32, + "ms": 685.47, + "receipt": { + "releaseMs": 32, + "status": "dispatched", + "holdMs": 0, + "releasePosted": true, + "durationMs": 32, + "verification": { + "ok": false, + "before": "Released: 0", + "attribute": "AXValue", + "expected": "Released: 1", + "role": "AXStaticText", + "ms": 527.222958, + "observed": "Released: 0" + }, + "inputMs": 77.4005, + "observationId": "BF8DB765-5BAA-4A29-9ECF-9518E3D02A69", + "kind": "drag", + "guardMs": 1.016667 + }, + "drops": "Drops: 0", + "released": "Released: 0", + "rejected": "Rejected: 1" + }, + "prior": { + "ok": true, + "timings": { + "hold=0,path=32,release=32": { + "n": 100, + "medianMs": 146.96, + "p95Ms": 206.67, + "maxMs": 283.14, + "within24fpsBudget": 0, + "verified": 100 + } + } + } + }, + "dragsConservative": { + "ok": true, + "verified": 100, + "holdMs": 0, + "durationMs": 64, + "releaseMs": 64, + "samples": 100, + "medianMs": 212, + "p95Ms": 242.28, + "maxMs": 265.19, + "under100ms": 0 + }, + "math": { + "complete": true, + "correct": 30, + "initialObservationMs": 190.89, + "elapsedMs": 1041.39, + "medianClickMs": 33.25, + "p95ClickMs": 47.39, + "maxClickMs": 59.9, + "under100ms": 30, + "scope": "Local HTTP/MCP, rendered-question parsing, deterministic arithmetic, click, verification and next observation. Excludes language-model inference.", + "prior": { + "medianClickMs": 48.97, + "p95ClickMs": 68, + "elapsedMs": 1497.36 + } + }, + "rawReports": "/Users/jas/.local/share/slab/computer-use/benchmarks/blueberry-2026-09-21-next", + "notes": [ + "100 zero-hold clicks passed; one took 100.11 ms.", + "Default drag failed on its first attempt; browser recorded Rejected: 1 and zero drops. No replay was sent.", + "Fresh 64 ms movement / 64 ms release fixture passed 100 drops and releases. This is not a guarantee for arbitrary apps or Finder transfers.", + "Defaults were not changed by this benchmark.", + "All fixture browser windows and daemons closed through harness cleanup.", + "Run-to-run differences are descriptive, not a controlled causal estimate." + ], + "followup": { + "report": "blueberry-drag-followup-2026-09-21.json", + "note": "User reported brief manual intervention during the timing experiments. Exact interval is unknown; exclude 32/64 and 64/32 follow-ups from default selection. The earlier failed 32/32 trial remains recorded, but its cause is unresolved." + } +} diff --git a/slab/test/computer-use-bench-report.test.mjs b/slab/test/computer-use-bench-report.test.mjs new file mode 100644 index 0000000000..7c91803f20 --- /dev/null +++ b/slab/test/computer-use-bench-report.test.mjs @@ -0,0 +1,32 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { summarizeBenchmark } from '../lib/computer-use-bench-report.mjs'; + +test('failed drags remain visible and do not improve successful latency', () => { + const row = { holdMs: 0, durationMs: 32, releaseMs: 32, ms: 120, + receipt: { verification: { ok: true }, releasePosted: true }, + drops: 'Drops: 1', released: 'Released: 1', rejected: 'Rejected: 0' }; + const report = summarizeBenchmark({ ok: false, error: 'Drag failed', dragSamples: [row, + { ...row, ms: 700, rejected: 'Rejected: 1' }, + { ...row, ms: 600, released: 'Released: 0' }, + { ...row, ms: 650, verified: false }], dragBatches: 5 }, '/tmp/result.json'); + const group = report.drags['hold=0,path=32,release=32']; + assert.equal(report.ok, false); + assert.equal(report.error, 'Drag failed'); + assert.equal(report.reportPath, '/tmp/result.json'); + assert.equal(report.dragBatches, 5); + assert.equal(group.attempted, 4); + assert.equal(group.verified, 1); + assert.equal(group.successful.medianMs, 120); + assert.equal(group.failed.medianMs, 650); +}); + +test('timing profiles stay separate and raw rounds stay in the file', () => { + const report = summarizeBenchmark({ ok: true, game: { correct: 30, rounds: [{ secretFixtureDetail: 'unused' }] }, + nativeClickLatency: [10, 20, 40, 50].map(ms => ({ ms, settleMs: 0, holdMs: 0, verifiedCount: 1 })) + .concat([{ ms: 500, settleMs: 180, holdMs: 40, verifiedCount: 2 }]) }, '/tmp/result.json'); + assert.equal(report.clicks['hold=0,settle=0'].successful.medianMs, 30); + assert.equal(report.clicks['hold=40,settle=180'].attempted, 1); + assert.deepEqual(report.game, { correct: 30 }); + assert.equal(JSON.stringify(report).includes('secretFixtureDetail'), false); +}); diff --git a/slab/test/fixtures/frame-drag.html b/slab/test/fixtures/frame-drag.html index 97d31875f7..fc86eee7bf 100644 --- a/slab/test/fixtures/frame-drag.html +++ b/slab/test/fixtures/frame-drag.html @@ -9,6 +9,13 @@ button {font:inherit;width:180px;height:120px;border:2px solid #245781;border-ra Drops: 0Released: 0Rejected: 0