diff --git a/captutor/lib/frame-client.mjs b/captutor/lib/frame-client.mjs index 44dd28f5c0..48237160e4 100644 --- a/captutor/lib/frame-client.mjs +++ b/captutor/lib/frame-client.mjs @@ -24,7 +24,7 @@ export function createFrameClient(path, { cli = false } = {}) { module ||= import(pathToFileURL(path).href); const { captureFrame } = await module; const result = await captureFrame("local", { - memory: true, noOCR: true, quietOverlay: true, screen, clearOverlays, + memory: true, noOCR: true, noVisual: true, quietOverlay: true, screen, clearOverlays, }); env = result.env; if (out && env.capture === "ok" && result.jpg?.length) { diff --git a/captutor/test/frame-client.test.mjs b/captutor/test/frame-client.test.mjs index c9298265cd..7879b37776 100644 --- a/captutor/test/frame-client.test.mjs +++ b/captutor/test/frame-client.test.mjs @@ -13,7 +13,7 @@ test("Captutor imports Frame once, yields during capture, and writes fresh audit let calls = 0; export async function captureFrame(machine, options) { await new Promise(resolve => setTimeout(resolve, 30)); - if (machine !== 'local' || !options.noOCR || !options.quietOverlay || !options.memory) + if (machine !== 'local' || !options.noOCR || !options.noVisual || !options.quietOverlay || !options.memory) throw new Error('incorrect capture options'); return { env: { capture: options.clearOverlays ? 'action' : 'ok', calls: ++calls }, jpg: Buffer.from('fresh') }; } diff --git a/slab/COMPUTER-USE-PERFORMANCE.md b/slab/COMPUTER-USE-PERFORMANCE.md index 3cdbee388b..fb82b8b0c3 100644 --- a/slab/COMPUTER-USE-PERFORMANCE.md +++ b/slab/COMPUTER-USE-PERFORMANCE.md @@ -54,10 +54,45 @@ counts, not tokenizer measurements. Earlier list compaction also covers fleet, prox, papers, calendar, and memory. All-day event end dates and memory ordering across years are preserved. -The JavaScript runtime from `e678ebb76` is installed on Blueberry and Neo, with -the prior pinned runtimes and launch-agent backups retained. The native app was -not rebuilt. All 32 focused local checks pass; all nine semantic-browser checks -also pass against Neo's installed runtime. +The first JavaScript runtime (`e678ebb76`) was installed on Blueberry and Neo, +with prior runtimes and launch-agent backups retained. That round passed 32 +focused local checks and nine semantic-browser checks against Neo's runtime. + +The second round adds `visual:false` to Frame capture, focus, reframe, and click +tools (`--no-visual` in the CLI). This skips supplemental contour detection while +preserving fresh pixels, accessibility data, and requested OCR. Internal target +guards, change probes, and Captutor's passive audits skip contours automatically. +Ordinary Frame captures retain contour detection by default. + +On Blueberry, six captures with pixels + AX and no contours took 58–218 ms. +Six with fast OCR and no contours took 61–132 ms. These were sequential mode +runs with substantial warm-up effects, not paired evidence of a fixed speedup. +Native action time additionally includes a fresh target guard, input, settling, +and the returned frame. The native app was rebuilt with the guarded installer +and installed on Blueberry and Neo; previous app bundles were retained. + +Contour results now reuse one exactly matching bounded analysis buffer. Only +normalized boxes are cached: global coordinates, scale, and pointer-distance +ranking are recalculated. Changed pixels or dimensions invalidate it. A static +light/dark fixture measured approximately 3 ms uncached versus 0.1 ms cached; +the animated Terminal produced **no cache hits**, so it showed no cache benefit. +The focused JavaScript checks now pass 33 tests. Standalone Swift fixtures cover +shape detection, invalidation, origin, scale, and focus ranking. + +An additional disposable website exercises the real Puppet HTTP/MCP path, +including verified clicks, typing, menu selection, a delayed update, and page +navigation. One complete run measured 16 ms for filling, 32–41 ms for ordinary +verified clicks, 67 ms for navigation, and 842 ms for the delayed update +(including its 300 ms application timer). Its first snapshot took 164 ms. +The native fixture click remains unverified: Blueberry's lock screen caused +Frame to return `permission_needed`, and the runner refused input. + +Optional Jev trials send only generated fixture button labels to the existing +OpenRouter decision adapter. The first three decisions selected the correct +button in 261–486 ms; a second set took 172–355 ms, also correct. This is six +simple selections, not an accuracy study. Jev adds decision latency to a known +locator; it can help only when replacing a slower reasoning step. No comparison +against a general-purpose model's end-to-end task time was performed. Reproduce browser timings (no user tabs or native input): @@ -65,6 +100,18 @@ Reproduce browser timings (no user tabs or native input): node slab/bin/computer-use-bench.mjs ``` +Reproduce the disposable website test (isolated browser profile and daemons): + +```sh +node slab/bin/computer-use-smoke.mjs +``` + +Add `--native` to open the fixture visibly and perform one guarded macOS click +on its observed button. An unlocked display is required. Add `--jev` to measure +three decisions using `OPENROUTER_API_KEY`; only generated fixture labels leave +the machine. The runner writes its JSON report and synthetic-page screenshot +to the system temporary directory, then removes its browser profile. + With an unlocked local display, add `--native` for five quiet captures per OCR mode. The report includes native stage timings and excludes failed captures; it stores no screen text or images. diff --git a/slab/bin/computer-use-bench.mjs b/slab/bin/computer-use-bench.mjs index 1f725a7544..f4f321f0fa 100644 --- a/slab/bin/computer-use-bench.mjs +++ b/slab/bin/computer-use-bench.mjs @@ -50,17 +50,20 @@ try { // Only successful captures count. Locked displays and missing grants must not // look like low-latency screenshots. This records no pixels or screen text. if (process.argv.includes("--native")) { - for (const [mode, options] of [["no-ocr", { noOCR: true }], ["fast-ocr", { fast: true }], ["accurate-ocr", {}]]) { + for (const [mode, options] of [["pixels-ax", { noOCR: true, noVisual: true }], ["no-ocr", { noOCR: true }], ["fast-ocr", { fast: true }], ["accurate-ocr", {}]]) { const times = [], stages = []; + let visualCacheHits = 0; let skipped; for (let i = 0; i < 5; i++) { const start = performance.now(); const { env, jpg } = await captureFrame("local", { ...options, memory: true, quietOverlay: true }); if (env.capture !== "ok" || !jpg?.length) { skipped = env.capture; break; } + if (options.noVisual && env.visual_suppressed !== "requested") { skipped = "native Frame needs update for noVisual"; break; } times.push(round(performance.now() - start)); stages.push(env.timings_ms); + if (env.visual_cache_hit) visualCacheHits++; } - report.native.push({ mode, ...summarize(times), ...(skipped ? { skipped } : {}), stages }); + report.native.push({ mode, ...summarize(times), visualCacheHits, ...(skipped ? { skipped } : {}), stages }); } } console.log(JSON.stringify(report, null, 2)); diff --git a/slab/bin/computer-use-smoke.mjs b/slab/bin/computer-use-smoke.mjs new file mode 100644 index 0000000000..868c2ab538 --- /dev/null +++ b/slab/bin/computer-use-smoke.mjs @@ -0,0 +1,145 @@ +#!/usr/bin/env node +// A disposable website/profile/daemon. Only --native sends macOS input, and +// only after Frame identifies the fixture window and its observed button. +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { mkdtemp, readFile, writeFile, rm, access } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { spawn } from "node:child_process"; +import { setTimeout as delay } from "node:timers/promises"; +import { chromium } from "playwright-core"; +import { createComputerUseClient } from "../lib/computer-use-client.mjs"; +import { chooseObservedTarget } from "../lib/jev-computer-use.mjs"; + +const native = process.argv.includes("--native"), jev = process.argv.includes("--jev"); +const root = resolve(import.meta.dirname, "../.."); +const dir = await mkdtemp(join(tmpdir(), "computer-use-smoke-")); +const report = { at: new Date().toISOString(), browser: [], native: [], jev: [] }; +const children = []; +let context, site; +const text = r => (r.content || []).filter(c => c.type === "text").map(c => c.text).join("\n"); +async function listen(server) { await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); return server.address().port; } +async function freePort() { const s = createServer(); const p = await listen(s); await new Promise(resolve => s.close(resolve)); return p; } +async function readyMCP(port) { + return (await fetch(`http://127.0.0.1:${port}/mcp`, { method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 0, method: "ping" }), signal: AbortSignal.timeout(1000) })).ok; +} +function start(script, args, env) { + const child = spawn(process.execPath, [join(root, script), ...args], { env, stdio: ["ignore", "pipe", "pipe"] }); + child.errors = ""; + child.stdout.on("data", () => {}); + child.stderr.on("data", chunk => { child.errors = (child.errors + chunk).slice(-2000); }); + children.push(child); return child; +} +async function waitReady(check) { + let last; + for (let i = 0; i < 60; i++) { + try { if (await check()) return; } catch (error) { last = error; } + if (children.some(c => c.exitCode !== null)) throw new Error(children.map(c => c.errors).join("\n")); + await delay(50); + } + throw last || new Error("Fixture startup timed out"); +} +try { + const html = await readFile(join(root, "slab/test/fixtures/computer-use.html")); + site = createServer((req, res) => { + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end(req.url === "/second" ? 'Second fixture page

Navigation verified

' : html); + }); + const sitePort = await listen(site); + context = await chromium.launchPersistentContext(join(dir, "profile"), { + channel: "chrome", headless: !native, viewport: native ? null : { width: 1000, height: 850 }, + args: ["--remote-debugging-port=0", "--window-size=1000,850", "--force-renderer-accessibility"], + }); + const browserPort = (await readFile(join(dir, "profile/DevToolsActivePort"), "utf8")).split("\n")[0]; + const page = context.pages()[0]; + await page.goto(`http://127.0.0.1:${sitePort}/`); + const cdp = await context.newCDPSession(page); + const { targetInfo } = await cdp.send("Target.getTargetInfo"); + await cdp.detach(); + const target = targetInfo.targetId; + const config = join(dir, "machines.json"), sock = join(dir, "puppet.sock"); + await writeFile(config, JSON.stringify({ machines: { fixture: { local: true, cdpUrl: `http://127.0.0.1:${browserPort}` } } })); + const env = { ...process.env, SLAB_PUPPET_CONFIG: config, SLAB_PUPPET_SOCK: sock, + SLAB_INPUT_LEASE_DIR: process.env.SLAB_INPUT_LEASE_DIR || join(process.env.HOME, ".local/share/slab/input-leases") }; + // The test daemon's status file belongs to the fixture; native input leases + // retain the real user's root so another controller still excludes us. + start("slab/bin/puppet.mjs", ["daemon"], { ...env, HOME: dir }); + await waitReady(async () => { await access(sock); return true; }); + const puppetPort = await freePort(), framePort = await freePort(); + start("slab/bin/puppet-mcp.mjs", ["--http", String(puppetPort)], env); + if (native) start("slab/bin/frame-mcp.mjs", ["--http", String(framePort)], env); + const servers = { puppet: `http://127.0.0.1:${puppetPort}/mcp`, ...(native ? { frame: `http://127.0.0.1:${framePort}/mcp` } : {}) }; + const tools = ["puppet_list", "puppet_snapshot", "puppet_click", "puppet_fill", "puppet_wait", "frame", "frame_click", "frame_reframe"]; + await waitReady(() => readyMCP(puppetPort)); + if (native) await waitReady(() => readyMCP(framePort)); + const client = createComputerUseClient({ servers, allowedTools: tools }); + await client.discover(); + await waitReady(async () => JSON.parse(text(await client.call("puppet_list", { full: true }))).fixture.connected); + async function call(name, args, lane = "browser") { + const start = performance.now(); + const result = await client.call(name, args); + assert.ok(!result.isError, text(result)); + report[lane].push({ tool: name, ...(args.locator ? { control: args.locator.name || args.locator.label } : {}), ms: Math.round(performance.now() - start) }); + return result; + } + const browser = { machine: "fixture", target }; + const snap = JSON.parse(text(await call("puppet_snapshot", browser))); + assert.match(snap.tree, /Add one/); + async function click(name, expected) { + const result = JSON.parse(text(await call("puppet_click", { ...browser, locator: { role: "button", name }, after: { locator: { text: expected } } }))); + assert.equal(result.performed, true); assert.equal(result.verification.ok, true); + } + await click("Add one", "Count: 1"); + await call("puppet_fill", { ...browser, locator: { label: "Name" }, value: "Fixture" }); + await click("Save name", "Saved: Fixture"); + await click("Open menu", "Choose blue"); + await click("Choose blue", "Selected: blue"); + await click("Start delayed update", "Ready"); + + if (jev) { + if (!process.env.OPENROUTER_API_KEY) report.jev.push({ skipped: "OPENROUTER_API_KEY unavailable" }); + else for (let i = 0; i < 3; i++) { + const start = performance.now(); + const candidates = await page.locator("button:visible").evaluateAll(buttons => buttons.map((b, i) => ({ id: `button_${i}`, label: b.textContent, role: "button", visible: true, disabled: b.disabled }))); + const decision = await chooseObservedTarget({ goal: "Increase the counter by one", observation: { id: crypto.randomUUID(), target, capturedAt: new Date().toISOString() }, candidates }); + const sample = { ms: Math.round(performance.now() - start), action: decision.action, label: decision.candidate?.label, probability: decision.probability, reason: decision.reason }; + report.jev.push(sample); + if (decision.action === "target") assert.equal(decision.candidate.label, "Add one"); + } + } + + if (native) { + await page.bringToFront(); + const observed = await call("frame", { machine: "local", fast: true, visual: false }, "native"); + const digest = text(observed); + assert.match(digest, /capture: ok/); + assert.match(digest, /frontmost: .*Chrome/); + assert.match(digest, /Computer-use check/); + const observation = JSON.parse(digest.match(/^observation: (.+)$/m)[1]); + const match = digest.match(/AXButton «Add one» @\((\d+),(\d+)\)/) || digest.match(/«Add one» @\((\d+),(\d+)\)/); + assert.ok(match, "Frame must observe the exact fixture control before native input"); + await call("frame_click", { machine: "local", observationId: observation.id, x: Number(match[1]), y: Number(match[2]), fast: true, visual: false }, "native"); + await page.getByText("Count: 2", { exact: true }).waitFor({ timeout: 3000 }); + report.native.push({ verified: "native click incremented counter exactly once" }); + const reframe = await call("frame_reframe", { machine: "local", fast: true, visual: false }, "native"); + assert.ok(reframe.content.length > 0); + } + // Only the generated fixture is retained; the browser profile is removed. + const artifact = join(tmpdir(), "computer-use-smoke-verified.png"); + await page.screenshot({ path: artifact }); report.screenshot = artifact; + const navigation = JSON.parse(text(await call("puppet_click", { ...browser, locator: { role: "link", name: "Second page" }, after: { locator: { text: "Navigation verified" } } }))); + assert.equal(navigation.verification.ok, true); + assert.equal(new URL(page.url()).pathname, "/second"); + report.ok = true; +} catch (error) { + report.ok = false; report.error = error.message; process.exitCode = 1; +} finally { + for (const child of children) child.kill(); + await context?.close(); + if (site) await new Promise(resolve => site.close(resolve)); + await rm(dir, { recursive: true, force: true }); + await writeFile(join(tmpdir(), "computer-use-smoke-report.json"), JSON.stringify(report, null, 2)); + console.log(JSON.stringify(report, null, 2)); +} diff --git a/slab/bin/frame-mcp.mjs b/slab/bin/frame-mcp.mjs index b4d6f33b20..d2915419c6 100755 --- a/slab/bin/frame-mcp.mjs +++ b/slab/bin/frame-mcp.mjs @@ -120,6 +120,7 @@ function digest(env) { const acts = (e.actions || []).join(","); L.push(` ${e.role} «${title}» @(${e.cx},${e.cy})${acts ? ` [${acts}]` : ""}`); } + if (env.visual_suppressed) L.push(`visual detection skipped: ${env.visual_suppressed}`); const visual = env.visual || []; L.push(`\nVISUAL (${visual.length} compact controls) — kind @(cx,cy), distance from hover:`); for (const v of visual) L.push(` ${v.kind} @(${v.cx},${v.cy}) d=${v.distance}`); @@ -169,7 +170,7 @@ const recentActionTrails = new FrameStateMap(); const recentFrames = new FrameStateMap(); const visionCache = new Map(); -async function captureFrame({ expectedTargetId, machine, ocr = true, fast = false, screen = false, cursor = true, cursorAt, targetAt, targetId, manualCheck, pressAt, pressCount = 1, pressTitle, actionOnly = false, clearTarget = false, clearOverlays = false, quietOverlay = false, overlays = false, crop, baseline = false, diff = false } = {}) { +async function captureFrame({ expectedTargetId, machine, ocr = true, visual = true, fast = false, screen = false, cursor = true, cursorAt, targetAt, targetId, manualCheck, pressAt, pressCount = 1, pressTitle, actionOnly = false, clearTarget = false, clearOverlays = false, quietOverlay = false, overlays = false, crop, baseline = false, diff = false } = {}) { if (!machine) throw new Error("`machine` is required (see frame_list)"); // Xbox uses synchronous curl/GPG helpers; keep that optional backend in a // child process so it cannot block fleet Mac requests in the shared server. @@ -189,7 +190,7 @@ async function captureFrame({ expectedTargetId, machine, ocr = true, fast = fals // through a temporary file for each MCP observation. const result = await captureNativeFrame(machine, { session: nativeFrameSession(), expectedTargetId, - memory: true, noOCR: !ocr, fast, screen, cursor, cursorAt, targetAt, + memory: true, noOCR: !ocr, noVisual: !visual, fast, screen, cursor, cursorAt, targetAt, targetId, manualCheck, pressAt, pressCount, pressTitle, actionOnly, clearTarget, clearOverlays, quietOverlay, overlays, crop, baseline, diff, }); @@ -224,7 +225,7 @@ async function toolInitialFrame(options = {}) { // REFRAME is change-driven: one silent full-window diff probe advances the // baseline, then OCR is run only inside the BSP-selected changed partition. -async function toolReframe({ machine, x, y, ocr = true, fast = true } = {}) { +async function toolReframe({ machine, x, y, ocr = true, visual = true, fast = true } = {}) { const focusX = Number(x); const focusY = Number(y); const hasX = Number.isFinite(focusX); @@ -234,6 +235,7 @@ async function toolReframe({ machine, x, y, ocr = true, fast = true } = {}) { const probe = await captureFrame({ machine, ocr: false, + visual: false, cursor: false, diff: true, baseline: true, @@ -251,6 +253,7 @@ async function toolReframe({ machine, x, y, ocr = true, fast = true } = {}) { const capture = await captureFrame({ machine, ocr, + visual, fast, cursor: false, crop, @@ -267,7 +270,7 @@ async function toolReframe({ machine, x, y, ocr = true, fast = true } = {}) { } // FOCUS is an intentional bounded read, independent of baseline/diff state. -async function toolFocus({ machine, x, y, width = 720, height = 520, ocr = true, fast = true } = {}) { +async function toolFocus({ machine, x, y, width = 720, height = 520, ocr = true, visual = true, fast = true } = {}) { x = Number(x); y = Number(y); width = Number(width); height = Number(height); if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) { throw new Error("x, y, width, and height must be finite; width/height must be positive"); @@ -281,6 +284,7 @@ async function toolFocus({ machine, x, y, width = 720, height = 520, ocr = true, const capture = await captureFrame({ machine, ocr, + visual, fast, cursorAt: [x, y], crop, @@ -741,7 +745,7 @@ async function recordActionTrail({ machine, label, baselineEnv, cursorAt, clearT // previous baseline before atomically advancing that baseline for the next // iteration. No OCR means no on-screen OCR overlay and low latency. const probe = await captureFrame({ - machine, ocr: false, fast: true, cursor: false, diff: true, baseline: true, + machine, ocr: false, visual: false, fast: true, cursor: false, diff: true, baseline: true, quietOverlay: true, clearTarget: clearTarget && samples.length === 0, }); @@ -888,15 +892,15 @@ async function verifyNativeTarget(machine, observationId, before = recentFrames. if (!before?.observation?.id || (observationId && observationId !== before.observation.id)) { throw new Error("Observation is missing or superseded; capture frame again in this session before acting"); } - const current = await captureFrame({ machine, ocr: false, cursor: false, quietOverlay: true }); + const current = await captureFrame({ machine, ocr: false, visual: false, cursor: false, quietOverlay: true }); assertFrameTarget(before, current.env, observationId); } -async function toolClick({ machine, observationId, x, y, count = 1, ocr = true, fast = true }) { +async function toolClick({ machine, observationId, x, y, count = 1, ocr = true, fast = true, visual = true }) { await verifyNativeTarget(machine, observationId); await clickPoint(machineSpec(machine), Number(x), Number(y), { count }); await settle(); - return toolFrame({ machine, ocr, fast, cursorAt: [Number(x), Number(y)] }); + return toolFrame({ machine, ocr, fast, visual, cursorAt: [Number(x), Number(y)] }); } async function toolStageClick({ machine, x, y, count = 1, label, ocr = true, fast = true }) { @@ -1061,7 +1065,8 @@ const TOOLS = [ properties: { machine: { type: "string", description: "Target name (e.g. xbox, neo, blueberry, local). See frame_list." }, screen: { type: "boolean", description: "Capture the complete display instead of the focused window (default false)." }, - ocr: { type: "boolean", description: "Run OCR (default true). Set false for a faster pixels+AX-only frame." }, + ocr: { type: "boolean", description: "Run OCR (default true). Set false to skip text recognition." }, + visual: { type: "boolean", description: "Detect compact visual controls (default true). Set false to skip contour detection while keeping fresh pixels and AX." }, fast: { type: "boolean", description: "Use Vision .fast OCR — lower latency, less accurate on small text (default false)." }, cursor: { type: "boolean", description: "Draw a high-contrast virtual cursor at the current mouse position (default true)." }, overlays: { type: "boolean", description: "Keep slab's own overlays — prompt rocks, the QR, the piece preview — in the shot instead of filtering them out, and widen the window crop to reach the menu bar. Use it when the overlays themselves are what you are designing (default false)." }, @@ -1080,6 +1085,7 @@ const TOOLS = [ x: { type: "number", description: "Optional global focus x; supply together with y." }, y: { type: "number", description: "Optional global focus y; supply together with x." }, ocr: { type: "boolean", description: "OCR only the changed crop (default true)." }, + visual: { type: "boolean", description: "Detect visual controls in the changed crop (default true). Diff probes always skip contours." }, fast: { type: "boolean", description: "Use fast OCR for the changed crop (default true)." }, }, required: ["machine"], @@ -1098,6 +1104,7 @@ const TOOLS = [ width: { type: "number", minimum: 64, description: "Crop width in screen points (default 720)." }, height: { type: "number", minimum: 64, description: "Crop height in screen points (default 520)." }, ocr: { type: "boolean", description: "OCR the focused crop (default true)." }, + visual: { type: "boolean", description: "Detect compact visual controls in the crop (default true)." }, fast: { type: "boolean", description: "Use fast OCR (default true)." }, }, required: ["machine", "x", "y"], @@ -1198,6 +1205,7 @@ const TOOLS = [ count: { type: "number", minimum: 1, maximum: 3, description: "Click count (default 1)." }, ocr: { type: "boolean", description: "Include OCR in the returned frame (default true)." }, fast: { type: "boolean", description: "Use fast OCR for the returned frame (default true)." }, + visual: { type: "boolean", description: "Detect visual controls in the returned frame (default true). Target guards always skip contours." }, }, required: ["machine", "x", "y"], }, diff --git a/slab/bin/frame.mjs b/slab/bin/frame.mjs index 0ba7b38c62..78985e0979 100755 --- a/slab/bin/frame.mjs +++ b/slab/bin/frame.mjs @@ -427,7 +427,7 @@ export async function captureFrame(name, options = {}) { () => captureFrameUnlocked(name, options)); } -async function captureFrameUnlocked(name, { memory = false, session, expectedTargetId, noOCR = false, fast = false, screen = false, cursor = false, cursorAt, targetAt, targetId, manualCheck, pressAt, pressCount = 1, pressTitle, actionOnly = false, clearTarget = false, clearOverlays = false, quietOverlay = false, overlays = false, crop, baseline = false, diff = false, out, json = false, direct = false, preview = false } = {}) { +async function captureFrameUnlocked(name, { memory = false, session, expectedTargetId, noOCR = false, noVisual = false, fast = false, screen = false, cursor = false, cursorAt, targetAt, targetId, manualCheck, pressAt, pressCount = 1, pressTitle, actionOnly = false, clearTarget = false, clearOverlays = false, quietOverlay = false, overlays = false, crop, baseline = false, diff = false, out, json = false, direct = false, preview = false } = {}) { if (name === XBOX_TARGET) { if (actionOnly || targetAt || targetId || manualCheck || pressAt || pressTitle || clearTarget || clearOverlays) { throw new Error("xbox is an observe-only Frame target; use the native gamepad/live-publish loop for control"); @@ -474,7 +474,7 @@ async function captureFrameUnlocked(name, { memory = false, session, expectedTar } if (session !== undefined && !/^[a-zA-Z0-9_-]{1,64}$/.test(session)) throw new Error("Invalid native frame session"); if (expectedTargetId !== undefined && !/^[a-zA-Z0-9_-]{1,64}$/.test(expectedTargetId)) throw new Error("Invalid staged target ID"); - const mode = [expectedTargetId ? `expect-target=${expectedTargetId}` : "", session ? `session=${session}` : "", screen ? "screen" : "window", noOCR ? "noocr" : "full", fast ? "fast" : "", cursorAt ? `cursor=${cursorAt[0]},${cursorAt[1]}` : cursor ? "cursor" : "", targetAt ? `target=${targetAt[0]},${targetAt[1]}` : "", targetId ? `target-id=${targetId}` : "", manualCheck ? `manual-check=${manualCheck}` : "", pressAt ? `press=${pressAt[0]},${pressAt[1]},${pressCount}` : "", pressTitle ? `press-title=${Buffer.from(pressTitle, "utf8").toString("base64")}` : "", actionOnly ? "action-only" : "", clearTarget ? "target-clear" : "", clearOverlays ? "overlay-clear" : "", quietOverlay ? "quiet-overlay" : "", overlays ? "overlays" : "", crop ? `crop=${crop.join(",")}` : "", baseline ? "baseline" : "", diff ? "diff" : ""] + const mode = [expectedTargetId ? `expect-target=${expectedTargetId}` : "", session ? `session=${session}` : "", screen ? "screen" : "window", noOCR ? "noocr" : "full", noVisual ? "novisual" : "", fast ? "fast" : "", cursorAt ? `cursor=${cursorAt[0]},${cursorAt[1]}` : cursor ? "cursor" : "", targetAt ? `target=${targetAt[0]},${targetAt[1]}` : "", targetId ? `target-id=${targetId}` : "", manualCheck ? `manual-check=${manualCheck}` : "", pressAt ? `press=${pressAt[0]},${pressAt[1]},${pressCount}` : "", pressTitle ? `press-title=${Buffer.from(pressTitle, "utf8").toString("base64")}` : "", actionOnly ? "action-only" : "", clearTarget ? "target-clear" : "", clearOverlays ? "overlay-clear" : "", quietOverlay ? "quiet-overlay" : "", overlays ? "overlays" : "", crop ? `crop=${crop.join(",")}` : "", baseline ? "baseline" : "", diff ? "diff" : ""] .filter(Boolean).join(" "); // Use the resident server only if already running (opt-in; see runServer); // otherwise a one-shot direct ssh. Both return an ACF1 {json, jpg} frame — @@ -638,7 +638,7 @@ const pointOpt = (f) => { if (!cmd || cmd === "-h" || cmd === "--help") { console.log( "frame — capture the focused window (pixels + OCR + AX + state) of a remote Mac\n\n" + - " frame [--screen] [--no-ocr] [--fast] [--cursor] [--target-at x,y] [--target-id id] [--manual-check id] [--press-at x,y] [--action-only] [--clear-target] [--clear-overlays] [--quiet-overlay] [--overlays] [--direct] [--preview] [--out file.jpg] [--json]\n" + + " frame [--screen] [--no-ocr] [--no-visual] [--fast] [--cursor] [--target-at x,y] [--target-id id] [--manual-check id] [--press-at x,y] [--action-only] [--clear-target] [--clear-overlays] [--quiet-overlay] [--overlays] [--direct] [--preview] [--out file.jpg] [--json]\n" + " --screen: capture the complete display instead of the focused window\n" + " --fast: Vision .fast OCR (lower latency, less accurate on small text)\n" + " --cursor: draw a high-contrast virtual marker at the mouse position\n" + @@ -678,7 +678,7 @@ else if (cmd === "view") { // Sugar for `frame --preview` so it reads like the slab-video / // slab-pdf "show me this" verbs. if (!argv[1]) { console.error("usage: frame view "); process.exit(1); } - await captureFrame(argv[1], { session: opt("--session"), noOCR: flag("--no-ocr"), fast: flag("--fast"), screen: flag("--screen"), cursor: flag("--cursor"), direct: flag("--direct"), out: opt("--out"), preview: true }); + await captureFrame(argv[1], { session: opt("--session"), noOCR: flag("--no-ocr"), noVisual: flag("--no-visual"), fast: flag("--fast"), screen: flag("--screen"), cursor: flag("--cursor"), direct: flag("--direct"), out: opt("--out"), preview: true }); } else if (cmd === "tape") { // `frame tape [secs] …` — the CLI face of frame-tape.mjs / frame_tape. // Recording (reel) + crop (ffmpeg) live in the shared lib so the MCP tool and @@ -700,7 +700,7 @@ else if (cmd === "view") { label: opt("--label"), }); console.log(JSON.stringify(r, null, 2)); -} else await captureFrame(cmd, { session: opt("--session"), noOCR: flag("--no-ocr"), fast: flag("--fast"), screen: flag("--screen"), cursor: flag("--cursor"), cursorAt: pointOpt("--cursor-at"), targetAt: pointOpt("--target-at"), targetId: opt("--target-id"), manualCheck: opt("--manual-check"), pressAt: pointOpt("--press-at"), pressCount: Number(opt("--press-count") || 1), pressTitle: opt("--press-title"), actionOnly: flag("--action-only"), clearTarget: flag("--clear-target"), clearOverlays: flag("--clear-overlays"), quietOverlay: flag("--quiet-overlay"), overlays: flag("--overlays"), crop: opt("--crop")?.split(",").map(Number), baseline: flag("--baseline"), diff: flag("--diff"), direct: flag("--direct"), out: opt("--out"), json: flag("--json"), preview: flag("--preview") }); +} else await captureFrame(cmd, { session: opt("--session"), noOCR: flag("--no-ocr"), noVisual: flag("--no-visual"), fast: flag("--fast"), screen: flag("--screen"), cursor: flag("--cursor"), cursorAt: pointOpt("--cursor-at"), targetAt: pointOpt("--target-at"), targetId: opt("--target-id"), manualCheck: opt("--manual-check"), pressAt: pointOpt("--press-at"), pressCount: Number(opt("--press-count") || 1), pressTitle: opt("--press-title"), actionOnly: flag("--action-only"), clearTarget: flag("--clear-target"), clearOverlays: flag("--clear-overlays"), quietOverlay: flag("--quiet-overlay"), overlays: flag("--overlays"), crop: opt("--crop")?.split(",").map(Number), baseline: flag("--baseline"), diff: flag("--diff"), direct: flag("--direct"), out: opt("--out"), json: flag("--json"), preview: flag("--preview") }); } if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { diff --git a/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift b/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift index bcbdec22bc..8d49ccd735 100644 --- a/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift +++ b/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift @@ -24,6 +24,7 @@ final class FrameCapture { static let shared = FrameCapture() private let queue = DispatchQueue(label: "computer.slab.frame", qos: .userInitiated) private var timer: DispatchSourceTimer? + private let visualDetector = FrameVisualControls() private let fm = FileManager.default // Transient overlay windows we draw (capture flash, OCR boxes). We exclude @@ -149,7 +150,7 @@ final class FrameCapture { } let session = mode.split(separator: " ").first(where: { $0.hasPrefix("session=") }) .map { String($0.dropFirst("session=".count)) } ?? "legacy" - produce(session: session, noOCR: flags.contains("noocr"), fast: flags.contains("fast"), + produce(session: session, noOCR: flags.contains("noocr"), noVisual: flags.contains("novisual"), fast: flags.contains("fast"), wholeScreen: flags.contains("screen"), virtualCursor: flags.contains("cursor"), cursorOverride: cursorOverride, crop: crop, saveBaseline: flags.contains("baseline"), includeDiff: flags.contains("diff"), @@ -856,55 +857,6 @@ final class FrameCapture { return out } - // Lightweight icon/control awareness for local reframes. OCR intentionally - // ignores drawn glyphs such as notification × buttons; contours recover - // compact, near-square controls without needing app-specific templates. - private func visualControls(_ cg: CGImage, scale: Double, origin: CGPoint, - focus: CGPoint?) -> [[String: Any]] { - // Contour cost grows sharply with dense desktop content. Analyze a - // bounded local buffer and carry its scale back to global coordinates; - // the returned frame pixels remain full quality. - var scan = cg - var scanScale = scale - if cg.width > 512 { - let w = 512, h = max(1, Int(Double(cg.height) * Double(w) / Double(cg.width))) - if let ctx = CGContext(data: nil, width: w, height: h, bitsPerComponent: 8, - bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(), - bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) { - ctx.interpolationQuality = .medium - ctx.draw(cg, in: CGRect(x: 0, y: 0, width: w, height: h)) - if let small = ctx.makeImage() { - scan = small - scanScale = scale * Double(w) / Double(cg.width) - } - } - } - let dark = VNDetectContoursRequest() - dark.contrastAdjustment = 1.5 - dark.detectsDarkOnLight = true - let light = VNDetectContoursRequest() - light.contrastAdjustment = 1.5 - light.detectsDarkOnLight = false - try? VNImageRequestHandler(cgImage: scan, options: [:]).perform([dark, light]) - let observations = [dark.results?.first, light.results?.first].compactMap { $0 } - let W = Double(scan.width) / scanScale, H = Double(scan.height) / scanScale - var out: [[String: Any]] = [] - for c in observations.flatMap({ $0.topLevelContours }) { - let b = c.normalizedPath.boundingBox - let x = origin.x + b.minX * W - let y = origin.y + (1 - b.maxY) * H - let w = b.width * W, h = b.height * H - let ratio = w / max(h, 0.1) - guard w >= 10, h >= 10, w <= 46, h <= 46, ratio >= 0.65, ratio <= 1.5 else { continue } - let cx = x + w / 2, cy = y + h / 2 - let distance = focus.map { hypot(cx - $0.x, cy - $0.y) } ?? 0 - let duplicate = out.contains { abs(($0["cx"] as? Int ?? 0) - Int(cx)) < 3 && abs(($0["cy"] as? Int ?? 0) - Int(cy)) < 3 } - if !duplicate { out.append(["kind": "compact-control", "cx": Int(cx), "cy": Int(cy), - "r": [Int(x), Int(y), Int(w), Int(h)], "distance": Int(distance)]) } - } - return out.sorted { ($0["distance"] as? Int ?? 0) < ($1["distance"] as? Int ?? 0) }.prefix(24).map { $0 } - } - // MARK: - Accessibility element tree of the frontmost app (trust already held) // Fetch several attributes in ONE IPC round-trip instead of one call each. @@ -1105,7 +1057,7 @@ final class FrameCapture { // MARK: - assemble + write the envelope - private func produce(session: String = "legacy", noOCR: Bool, fast: Bool = false, wholeScreen: Bool = false, + private func produce(session: String = "legacy", noOCR: Bool, noVisual: Bool = false, fast: Bool = false, wholeScreen: Bool = false, virtualCursor: Bool = false, cursorOverride: CGPoint? = nil, crop: CGRect? = nil, saveBaseline: Bool = false, includeDiff: Bool = false, showOverlay: Bool = true, @@ -1198,18 +1150,19 @@ final class FrameCapture { CGPoint(x: $0["x"] ?? 0, y: $0["y"] ?? 0) } let visualOrigin = region.origin - if reelActive { + if reelActive || noVisual { // VNDetectContours can overlap SCRecordingOutput's // WindowServer/CoreImage work and has repeatedly taken the // host app down mid-reel. It is supplemental to OCR + AX, so // omit only this pass while preserving the audit screenshot. env["visual"] = [] - env["visual_suppressed"] = "screen-recording" + env["visual_suppressed"] = reelActive ? "screen-recording" : "requested" tm["visual"] = 0 } else { - t = nowNs(); env["visual"] = visualControls(cg, scale: captureScale, + t = nowNs(); env["visual"] = visualDetector.controls(cg, scale: captureScale, origin: visualOrigin, focus: cursorOverride ?? cursorMeta) tm["visual"] = msSince(t) + env["visual_cache_hit"] = visualDetector.cacheHit } t = nowNs() let marker = virtualCursor ? (cursorOverride ?? cursorMeta) : nil diff --git a/slab/menubar-swift/Sources/SlabMenubar/FrameVisualControls.swift b/slab/menubar-swift/Sources/SlabMenubar/FrameVisualControls.swift new file mode 100644 index 0000000000..c4a26514d9 --- /dev/null +++ b/slab/menubar-swift/Sources/SlabMenubar/FrameVisualControls.swift @@ -0,0 +1,63 @@ +import CoreGraphics +import Foundation +import Vision + +// Used only on FrameCapture's serial queue. Cache one bounded analysis buffer, +// never the screenshot, OCR, AX tree, pointer position, or session baseline. +final class FrameVisualControls { + private var pixels: Data? + private var dimensions = CGSize.zero + private var boxes: [CGRect] = [] + private(set) var cacheHit = false + + func controls(_ cg: CGImage, scale: Double, origin: CGPoint, + focus: CGPoint?) -> [[String: Any]] { + cacheHit = false + let w = min(512, cg.width) + let h = max(1, Int(Double(cg.height) * Double(w) / Double(cg.width))) + guard let ctx = CGContext(data: nil, width: w, height: h, bitsPerComponent: 8, + bytesPerRow: w * 4, space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return [] } + ctx.interpolationQuality = .medium + ctx.draw(cg, in: CGRect(x: 0, y: 0, width: w, height: h)) + guard let data = ctx.data, let scan = ctx.makeImage() else { return [] } + let current = Data(bytes: data, count: ctx.bytesPerRow * h) + let size = CGSize(width: w, height: h) + // Exact byte equality, including image dimensions. Changed analysis pixels + // always invalidate; no age limit or approximate/perceptual matching. + if dimensions == size, pixels == current { + cacheHit = true + } else { + let dark = VNDetectContoursRequest() + dark.contrastAdjustment = 1.5 + dark.detectsDarkOnLight = true + let light = VNDetectContoursRequest() + light.contrastAdjustment = 1.5 + light.detectsDarkOnLight = false + do { try VNImageRequestHandler(cgImage: scan, options: [:]).perform([dark, light]) } + catch { pixels = nil; boxes = []; return [] } + guard let darkResult = dark.results?.first, let lightResult = light.results?.first + else { pixels = nil; boxes = []; return [] } + boxes = [darkResult, lightResult].flatMap { $0.topLevelContours } + .map { $0.normalizedPath.boundingBox } + pixels = current + dimensions = size + } + let scanScale = scale * Double(w) / Double(cg.width) + let W = Double(w) / scanScale, H = Double(h) / scanScale + var out: [[String: Any]] = [] + for b in boxes { + let x = origin.x + b.minX * W + let y = origin.y + (1 - b.maxY) * H + let bw = b.width * W, bh = b.height * H + let ratio = bw / max(bh, 0.1) + guard bw >= 10, bh >= 10, bw <= 46, bh <= 46, ratio >= 0.65, ratio <= 1.5 else { continue } + let cx = x + bw / 2, cy = y + bh / 2 + let distance = focus.map { hypot(cx - $0.x, cy - $0.y) } ?? 0 + let duplicate = out.contains { abs(($0["cx"] as? Int ?? 0) - Int(cx)) < 3 && abs(($0["cy"] as? Int ?? 0) - Int(cy)) < 3 } + if !duplicate { out.append(["kind": "compact-control", "cx": Int(cx), "cy": Int(cy), + "r": [Int(x), Int(y), Int(bw), Int(bh)], "distance": Int(distance)]) } + } + return out.sorted { ($0["distance"] as? Int ?? 0) < ($1["distance"] as? Int ?? 0) }.prefix(24).map { $0 } + } +} diff --git a/slab/test/FrameVisualControlsTests.swift b/slab/test/FrameVisualControlsTests.swift new file mode 100644 index 0000000000..fbdff7340b --- /dev/null +++ b/slab/test/FrameVisualControlsTests.swift @@ -0,0 +1,68 @@ +// Compile with FrameVisualControls.swift; no app, screen permissions, or input. +import CoreGraphics +import Foundation + +@main struct FrameVisualControlsTests { + static func check(_ ok: Bool, _ message: String) { + if !ok { fatalError(message) } + } + static func image(dark: Bool = false, extra: Bool = false, width: Int = 512) -> CGImage { + let ctx = CGContext(data: nil, width: width, height: 320, bitsPerComponent: 8, + bytesPerRow: width * 4, space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! + ctx.setFillColor(gray: dark ? 0 : 1, alpha: 1) + ctx.fill(CGRect(x: 0, y: 0, width: width, height: 320)) + ctx.setFillColor(gray: dark ? 1 : 0, alpha: 1) + ctx.fill(CGRect(x: 80, y: 80, width: 28, height: 28)) + ctx.fillEllipse(in: CGRect(x: 240, y: 160, width: 32, height: 32)) + if extra { ctx.fill(CGRect(x: 360, y: 60, width: 24, height: 24)) } + return ctx.makeImage()! + } + static func equal(_ a: [[String: Any]], _ b: [[String: Any]]) -> Bool { + NSDictionary(dictionary: ["controls": a]).isEqual(to: ["controls": b]) + } + static func main() { + for dark in [false, true] { + let detector = FrameVisualControls(), original = image(dark: dark) + let first = detector.controls(original, scale: 1, origin: .zero, focus: nil) + check(!detector.cacheHit && first.count >= 2, "must detect both fixture shapes on either background") + for (x, y) in [(94, 226), (256, 144)] { + check(first.contains { abs(($0["cx"] as! Int) - x) <= 2 && abs(($0["cy"] as! Int) - y) <= 2 }, + "shape centers must map from image coordinates to global top-left points") + } + let repeated = detector.controls(image(dark: dark), scale: 1, origin: .zero, focus: nil) + check(detector.cacheHit && equal(first, repeated), "identical pixels must reuse identical controls") + let moved = detector.controls(original, scale: 1, origin: CGPoint(x: 200, y: 400), focus: nil) + check(detector.cacheHit, "origin does not invalidate normalized shapes") + for (before, after) in zip(first, moved) { + check((after["cx"] as! Int) == (before["cx"] as! Int) + 200, "global x must move") + check((after["cy"] as! Int) == (before["cy"] as! Int) + 400, "global y must move") + } + let focus = CGPoint(x: first.last!["cx"] as! Int, y: first.last!["cy"] as! Int) + let ranked = detector.controls(original, scale: 1, origin: .zero, focus: focus) + check(detector.cacheHit && (ranked.first!["distance"] as! Int) == 0, "focus ranking must refresh on a cache hit") + let scaled = detector.controls(original, scale: 2, origin: .zero, focus: nil) + let freshScaled = FrameVisualControls().controls(original, scale: 2, origin: .zero, focus: nil) + check(detector.cacheHit && equal(scaled, freshScaled), "scale must be recomputed") + let changed = image(dark: dark, extra: true) + let next = detector.controls(changed, scale: 1, origin: .zero, focus: nil) + check(!detector.cacheHit && next.count > first.count, "a new control must invalidate cached pixels") + check(equal(next, FrameVisualControls().controls(changed, scale: 1, origin: .zero, focus: nil)), "changed pixels must match a fresh scan") + _ = detector.controls(image(dark: dark, width: 420), scale: 1, origin: .zero, focus: nil) + check(!detector.cacheHit, "new dimensions must invalidate") + } + let fixture = image(), detector = FrameVisualControls() + var cold: [Double] = [], warm: [Double] = [] + for _ in 0..<5 { + var start = Date() + _ = FrameVisualControls().controls(fixture, scale: 1, origin: .zero, focus: nil) + cold.append(Date().timeIntervalSince(start) * 1000) + _ = detector.controls(fixture, scale: 1, origin: .zero, focus: nil) + start = Date() + _ = detector.controls(fixture, scale: 1, origin: .zero, focus: nil) + warm.append(Date().timeIntervalSince(start) * 1000) + } + print("PASS: light/dark shapes, exact invalidation, origin, scale, focus, dimensions") + print("fixture median ms: uncached=\(cold.sorted()[2]), cached=\(warm.sorted()[2])") + } +} diff --git a/slab/test/computer-use-transport.test.mjs b/slab/test/computer-use-transport.test.mjs index 65ec54eb77..c89771a9a4 100644 --- a/slab/test/computer-use-transport.test.mjs +++ b/slab/test/computer-use-transport.test.mjs @@ -40,10 +40,10 @@ test('Frame memory transport preserves fragmented binary images, modes, and erro await rm(dir, { recursive: true, force: true }); }); const { captureFrame } = await import(`../bin/frame.mjs?test=${Date.now()}`); - const result = await captureFrame('fixture', { memory: true, noOCR: true, crop: [40,60,100,80], quietOverlay: true }); + const result = await captureFrame('fixture', { memory: true, noOCR: true, noVisual: true, crop: [40,60,100,80], quietOverlay: true }); assert.deepEqual(result.env, envelope); assert.deepEqual(result.jpg, jpg); - assert.equal(requests[0].mode, 'window noocr quiet-overlay crop=40,60,100,80'); + assert.equal(requests[0].mode, 'window noocr novisual quiet-overlay crop=40,60,100,80'); await assert.rejects(captureFrame('missing', { memory: true }), /unknown machine/); envelope = { error: 'fixture transport failure' }; await assert.rejects(captureFrame('fixture', { memory: true, pressAt: [1,2] }), /fixture transport failure/); @@ -82,6 +82,40 @@ test('native-only Puppet machines never enter the browser reconnect loop', async assert.equal(machine.info().lastError, null); await assert.rejects(machine.ensureConnected(), /no browser configured/); }); + +test('Frame MCP forwards explicit contour skipping and omits contours on reframe diff probes', async t => { + const dir = await mkdtemp(join(tmpdir(), 'frame-mcp-visual-')); + const socket = join(dir, 'frame.sock'), config = join(dir, 'machines.json'); + await writeFile(config, JSON.stringify({ machines: { fixture: { local: true } } })); + const modes = []; + const server = net.createServer(sock => sock.once('data', data => { + const { mode } = JSON.parse(String(data)); + modes.push(mode); + const env = { capture: 'ok', capture_scope: 'window', diff: [], diff_baseline: 'matched', + visual_suppressed: 'requested', observation: { session: mode.match(/session=([^ ]+)/)[1] } }; + const json = Buffer.from(JSON.stringify(env)), jpg = Buffer.from([255,216,255,217]); + sock.end(Buffer.concat([Buffer.from(`ACF1 ${json.length} ${jpg.length}\n`), json, jpg])); + })); + await new Promise(resolve => server.listen(socket, resolve)); + t.after(async () => { await new Promise(resolve => server.close(resolve)); await rm(dir, { recursive: true, force: true }); }); + for (const name of ['frame', 'frame_reframe']) { + const result = await new Promise((resolve, reject) => { + const child = execFile(process.execPath, ['slab/bin/frame-mcp.mjs'], { + env: { ...process.env, SLAB_FRAME_SOCK: socket, SLAB_PUPPET_CONFIG: config }, + }, (error, stdout) => error ? reject(error) : resolve(JSON.parse(stdout).result)); + child.stdin.end(JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { + name, arguments: { machine: 'fixture', ...(name === 'frame' ? { visual: false } : {}) }, + } }) + '\n'); + }); + assert.ok(!result.isError, JSON.stringify(result)); + assert.match(modes.at(-1), /\bnovisual\b/); + if (name === 'frame_reframe') { + assert.match(modes.at(-1), /\bdiff\b/); + assert.match(result.content[0].text, /unchanged/); + } + } + assert.equal(modes.length, 2, 'unchanged reframe must not request another capture'); +}); test('Puppet uses recent in-memory JPEG without a CDP screenshot', async () => { const {machine, calls} = browser(); machine.liveFrames.set('session', {data: 'live-image', at: Date.now()}); diff --git a/slab/test/fixtures/computer-use.html b/slab/test/fixtures/computer-use.html new file mode 100644 index 0000000000..496ea4996c --- /dev/null +++ b/slab/test/fixtures/computer-use.html @@ -0,0 +1,59 @@ + + + +Computer-use check + +
+

Computer-use check

+
+ Count: 0 + Idle +
+
+ + Unsaved +
+
+ + + No selection + Hover here Hover works + Second page +
+
+ +