#!/usr/bin/env node // frame.mjs — capture a rich "frame" of a remote Mac for fleet automation. // // A frame = pixels (a downscaled JPEG thumbnail) + OCR'd text with click // coordinates + the Accessibility element tree (roles/titles/AXPress targets) // + window/cursor/frontmost state, packed into one JSON envelope. By default // the pixels are isolated to the focused/frontmost window; `--screen` asks for // the complete display. It is the // native-capture complement to `puppet`: `frame` OBSERVES the focused window // (any native app, no DOM needed), `puppet` ACTS (trusted stroke/gesture/key). // Together they close an observe→act loop across the fleet. // // The capture is produced ON the target by the SlabMenubar app (see // FrameCapture.swift) — it already holds Accessibility trust and lives in the // GUI session, so it can reach the WindowServer that a plain ssh session // cannot. This CLI just drops a request file over SSH and reads the result. // // PERMISSIONS ARE LAZY: the app never prompts at launch. The first real frame // request is what triggers the Screen Recording grant on the target; until // granted, the envelope reports `capture: "permission_needed"` and `frame // setup ` walks you through the one-time toggle. // // This file ships in the PUBLIC aesthetic.computer repo, so it carries NO // machine names: the registry lives in the UNTRACKED config shared with // puppet at ~/.config/slab/puppet.json. A machine's name doubles as its ssh // host (the minis are ssh aliases); set "sshHost" per machine to override. import { execFileSync, spawn } from "node:child_process"; import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; import net from "node:net"; import { dirname, join, resolve } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; import { withMachineLease } from "../lib/computer-use-lease.mjs"; import { localFrame } from "../lib/frame-local.mjs"; const HOME = process.env.HOME; const CONFIG_PATH = process.env.SLAB_PUPPET_CONFIG || join(HOME, ".config", "slab", "puppet.json"); const FRAMES_DIR = join(HOME, ".local", "share", "slab", "frames"); const SOCK_PATH = process.env.SLAB_FRAME_SOCK || join(HOME, ".local", "share", "slab", "frame.sock"); // The menubar app on THIS (controller) Mac watches `state/open-frame` and // raises a preview window per machine, badged with the machine name (see // FramePreview.swift). Opt-in via `--preview` / `frame view ` so a // plain `frame` call never pops a window. SLAB_HOME mirrors the Swift side. const SLAB_HOME = process.env.SLAB_HOME || join(HOME, ".local", "share", "slab"); const FRAME_REQUEST_FILE = join(SLAB_HOME, "state", "open-frame"); const APP_BUNDLE = "computer.slab.menubar"; const XBOX_TARGET = "xbox"; const XBOX_VAULT_FILES = [ process.env.XBOX_DEVICE_PORTAL_ENV_GPG, join(HOME, "aesthetic-computer-vault", "xbox", "device-portal.env.gpg"), join(HOME, "aesthetic-computer", "aesthetic-computer-vault", "xbox", "device-portal.env.gpg"), ].filter(Boolean); function parseEnvText(source) { const result = {}; for (const raw of String(source || "").split(/\r?\n/)) { const match = raw.trim().match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/); if (!match) continue; let value = match[2].trim(); if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) value = value.slice(1, -1); result[match[1]] = value; } return result; } function xboxPortalConfig() { let vault = {}; const encrypted = XBOX_VAULT_FILES.find((candidate) => existsSync(candidate)); if (encrypted) { const plaintext = execFileSync("gpg", ["--batch", "--quiet", "--decrypt", encrypted], { encoding: "utf8", timeout: 15000, maxBuffer: 1024 * 1024, }); vault = parseEnvText(plaintext); } const config = { ...vault, ...process.env }; const host = config.XBOX_DEVICE_PORTAL_HOST; const port = config.XBOX_DEVICE_PORTAL_PORT || "11443"; const username = config.XBOX_DEVICE_PORTAL_USERNAME; const password = config.XBOX_DEVICE_PORTAL_PASSWORD; if (!host || !username || !password) { throw new Error( "xbox Frame target needs Device Portal credentials in the environment or xbox/device-portal.env.gpg", ); } return { base: `https://${host}:${port}`, username, password }; } function curlConfigValue(value) { return String(value).replace(/\\/g, "\\\\").replace(/"/g, '\\"'); } function xboxPortalGet(endpoint) { const { base, username, password } = xboxPortalConfig(); // Keep the Basic credential off the process argv; curl reads it from stdin. const config = [ "silent", "show-error", "insecure", `user = "${curlConfigValue(`auto-${username}:${password}`)}"`, `url = "${curlConfigValue(`${base}${endpoint}`)}"`, "", ].join("\n"); return execFileSync("/usr/bin/curl", ["--config", "-"], { input: config, timeout: 15000, maxBuffer: 32 * 1024 * 1024, }); } function captureXboxJpeg() { const png = xboxPortalGet("/ext/screenshot"); if (png.length < 8 || png[0] !== 0x89 || png.subarray(1, 4).toString("ascii") !== "PNG") { throw new Error("Xbox Device Portal screenshot was not a PNG image"); } const conversionDir = mkdtempSync(join(tmpdir(), "frame-xbox-")); const pngFile = join(conversionDir, "screen.png"); const jpegFile = join(conversionDir, "screen.jpg"); try { writeFileSync(pngFile, png); execFileSync("/usr/bin/sips", ["-s", "format", "jpeg", "-s", "formatOptions", "88", pngFile, "--out", jpegFile], { stdio: "ignore", timeout: 15000 }); return readFileSync(jpegFile); } finally { rmSync(conversionDir, { recursive: true, force: true }); } } // The remote read-loop agent: ONE per machine, held open by the server so each // frame is a stdin-write + stdout-read on an already-open ssh channel — no // channel setup (~100ms) and no node restart per call. `bash -s` reads its // SCRIPT from stdin, so the agent can't also read modes from stdin — instead we // drop this script on the target once and feed modes to `bash ` over the // persistent channel. Emits exactly one JSON line per mode (terminator-framed). // Emits a length-prefixed binary frame per request: `ACF1 \n` // then the JSON bytes then the raw JPEG bytes — no base64. One frame per mode // line read on stdin. // NOTE: this is a JS template literal — bash `${...}` would be interpreted as JS // interpolation, so the script uses only `$var` and `$(...)` (both literal here). const AGENT_SCRIPT = String.raw`#!/bin/bash d="$HOME/.local/share/slab/state"; mkdir -p "$d" emit() { local j="$d/frame.out.json" p="$d/frame.out.jpg" jl pl e el jl=$(wc -c < "$j" 2>/dev/null | tr -d ' '); [ -z "$jl" ] && jl=0 pl=$(wc -c < "$p" 2>/dev/null | tr -d ' '); [ -z "$pl" ] && pl=0 if [ "$jl" = 0 ]; then e='{"capture":"error","reason":"no-out"}' el=$(printf '%s' "$e" | wc -c | tr -d ' ') printf 'ACF1 %s 0\n%s' "$el" "$e"; return fi printf 'ACF1 %s %s\n' "$jl" "$pl"; cat "$j"; [ "$pl" != 0 ] && cat "$p" } while IFS= read -r mode; do [ -z "$mode" ] && mode=window rm -f "$d/frame.done"; printf '%s' "$mode" > "$d/frame.req" for i in $(seq 1 400); do [ -f "$d/frame.done" ] && break; sleep 0.01; done emit done`; const AGENT_REMOTE_PATH = "~/.local/share/slab/frame-agent.sh"; let localMachineName; function loadMachines() { let machines = {}; if (existsSync(CONFIG_PATH)) { machines = JSON.parse(readFileSync(CONFIG_PATH, "utf8")).machines || {}; } // The controller captures itself with no ssh (ssh-to-self is host-key // fragile). Expose it under its LocalHostName as a `local: true` machine — // synthetic, so it never has to live in puppet.json (which puppet's daemon // also reads and would try to CDP-connect). if (!localMachineName) { try { localMachineName = execFileSync("scutil", ["--get", "LocalHostName"], { encoding: "utf8" }).trim(); } catch {} localMachineName ||= "local"; } machines.local = { local: true }; if (!machines[localMachineName]) machines[localMachineName] = { local: true }; return machines; } function sshHostFor(name, machines) { return machines[name]?.sshHost || name; } // One ssh invocation with a warm multiplexed master so repeated frames are fast. // The command is fed to `bash -s` over stdin, NOT passed as an argument: the // remote runs the login shell, and some machines use fish (blueberry) which // can't parse the bash `for/$()/done` syntax. `bash -s` + stdin is shell- // agnostic and sidesteps all quoting. function ssh(host, remoteCmd, { timeoutMs = 15000 } = {}) { const cp = join(HOME, ".ssh", `cm-${host}`); return execFileSync( "ssh", [ "-o", "ControlMaster=auto", "-o", `ControlPath=${cp}`, "-o", "ControlPersist=300", "-o", "BatchMode=yes", host, "bash -s", ], { input: remoteCmd, encoding: "utf8", timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 }, ); } // Run a shell command on a machine — locally (no ssh) when it's flagged // "local" in the registry (e.g. neo, the controller, capturing its own screen; // `ssh localhost` is awkward/host-key-fragile), otherwise over the warm master. function runOn(name, machines, cmd, { timeoutMs = 15000 } = {}) { if (machines[name]?.local) { return execFileSync("bash", ["-c", cmd], { encoding: "utf8", timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024, }); } return ssh(sshHostFor(name, machines), cmd, { timeoutMs }); } function sshOpts(host) { return [ "-o", "ControlMaster=auto", "-o", `ControlPath=${join(HOME, ".ssh", `cm-${host}`)}`, "-o", "ControlPersist=300", "-o", "BatchMode=yes", ]; } // ─── ACF1 binary frame codec ────────────────────────────────────────────── // Wire format: `ACF1 \n` + + . No base64. Parses one frame off the front of a Buffer. // Returns {frame:{json,jpg}, rest} when complete, {skip, rest} for a non-ACF1 // line (ssh banner), or null when more bytes are needed. function parseFrame(buf) { const nl = buf.indexOf(0x0a); if (nl < 0) return null; const header = buf.subarray(0, nl).toString("ascii"); const m = header.match(/^ACF1 (\d+) (\d+)$/); if (!m) return { skip: true, rest: buf.subarray(nl + 1) }; const jl = +m[1], pl = +m[2]; const start = nl + 1, end = start + jl + pl; if (buf.length < end) return null; return { frame: { json: buf.subarray(start, start + jl).toString("utf8"), jpg: buf.subarray(start + jl, end) }, rest: buf.subarray(end), }; } function frameBytes(json, jpg) { const jb = Buffer.from(json, "utf8"); return Buffer.concat([Buffer.from(`ACF1 ${jb.length} ${jpg.length}\n`, "ascii"), jb, jpg]); } // Ask the local menubar app to open (or live-reload) a preview window for a // pulled frame. One `path\tmachine` line appended to `state/open-frame`; the // app's 2 s tick consumes it (FramePreview.consumeRequests). Re-asking the // same machine raises + reloads its window. Best-effort: a missing menubar // just leaves an unconsumed line, which is harmless. function requestPreview(machine, jpgPath) { mkdirSync(dirname(FRAME_REQUEST_FILE), { recursive: true }); appendFileSync(FRAME_REQUEST_FILE, `${jpgPath}\t${machine}\n`); } // ─── persistent server: one held-open agent per machine ─────────────────── // Removes the two per-call costs the transport audit found: node cold-start // (the server stays warm) and ssh channel setup (each agent is a single ssh // process kept open, fed modes on stdin). A frame then costs ~daemon work + one // RTT on the open channel instead of +~170ms of process/connection spin-up. const agents = new Map(); // name -> { proc, pending: [resolve], buf } function ensureAgentScript(name, machines) { if (machines[name]?.local) { const p = join(HOME, ".local", "share", "slab", "frame-agent.sh"); mkdirSync(dirname(p), { recursive: true }); writeFileSync(p, AGENT_SCRIPT); return p; } const host = sshHostFor(name, machines); execFileSync( "ssh", [...sshOpts(host), host, `mkdir -p ~/.local/share/slab && cat > ${AGENT_REMOTE_PATH}`], { input: AGENT_SCRIPT, encoding: "utf8", timeout: 15000 }, ); return AGENT_REMOTE_PATH; } function spawnAgent(name, machines) { const scriptPath = ensureAgentScript(name, machines); let proc; if (machines[name]?.local) { proc = spawn("bash", [scriptPath], { stdio: ["pipe", "pipe", "ignore"] }); } else { const host = sshHostFor(name, machines); proc = spawn("ssh", [...sshOpts(host), host, `bash ${AGENT_REMOTE_PATH}`], { stdio: ["pipe", "pipe", "ignore"], }); } const ag = { proc, pending: [], buf: Buffer.alloc(0) }; // buf is binary proc.stdout.on("data", (chunk) => { ag.buf = Buffer.concat([ag.buf, chunk]); for (;;) { const r = parseFrame(ag.buf); if (!r) break; ag.buf = r.rest; if (r.skip) continue; // ssh banner line const resolve = ag.pending.shift(); if (resolve) resolve(r.frame); } }); proc.on("exit", () => { agents.delete(name); const dead = { json: '{"capture":"error","reason":"agent-exit"}', jpg: Buffer.alloc(0) }; ag.pending.forEach((r) => r(dead)); }); agents.set(name, ag); return ag; } function agentRequest(name, machines, mode, timeoutMs = 15000) { return new Promise((resolve, reject) => { let ag; try { ag = agents.get(name) || spawnAgent(name, machines); } catch (e) { reject(e); return; } const timer = setTimeout(() => reject(new Error("agent timeout")), timeoutMs); ag.pending.push((frame) => { clearTimeout(timer); resolve(frame); }); ag.proc.stdin.write(mode + "\n"); }); } function runServer() { mkdirSync(dirname(SOCK_PATH), { recursive: true }); try { unlinkSync(SOCK_PATH); } catch {} const server = net.createServer((sock) => { // Request is a JSON line; response is an ACF1 binary frame (forwarded from // the held agent), so we frame the request side on '\n' but write binary. let buf = Buffer.alloc(0); sock.on("data", async (chunk) => { buf = Buffer.concat([buf, chunk]); let nl; while ((nl = buf.indexOf(0x0a)) >= 0) { const line = buf.subarray(0, nl).toString("utf8"); buf = buf.subarray(nl + 1); if (!line.trim()) continue; let req; try { req = JSON.parse(line); } catch { sock.write(frameBytes('{"error":"bad json"}', Buffer.alloc(0))); continue; } try { const frame = await agentRequest(req.machine, loadMachines(), req.mode || "window"); sock.write(frameBytes(frame.json, frame.jpg)); } catch (e) { sock.write(frameBytes(JSON.stringify({ error: String(e.message || e) }), Buffer.alloc(0))); } } }); sock.on("error", () => {}); }); server.on("error", (e) => { if (e.code === "EADDRINUSE") process.exit(0); // another server won the race throw e; }); server.listen(SOCK_PATH, () => console.log(`frame server listening on ${SOCK_PATH}`)); } function requestViaServer(machine, mode, timeoutMs = 15000) { return new Promise((resolve, reject) => { const sock = net.createConnection(SOCK_PATH); let buf = Buffer.alloc(0), received = false; const timer = setTimeout(() => { sock.destroy(); reject(new Error("server rpc timeout")); }, timeoutMs); sock.on("close", () => { clearTimeout(timer); if (!received) reject(new Error("frame server closed before returning a frame")); }); sock.on("error", (e) => { clearTimeout(timer); reject(e); }); sock.on("connect", () => sock.write(JSON.stringify({ machine, mode }) + "\n")); sock.on("data", (chunk) => { buf = Buffer.concat([buf, chunk]); const r = parseFrame(buf); if (r && r.frame) { received = true; clearTimeout(timer); sock.end(); resolve(r.frame); } }); }); } // One-shot direct path (no server): spawn ssh/local, feed the request, read one // ACF1 frame off stdout. Returns {json, jpg}. function directFrame(name, machines, mode, timeoutMs = 15000) { return new Promise((resolve, reject) => { const remote = `d="$HOME/.local/share/slab/state"; mkdir -p "$d"; rm -f "$d/frame.done"; ` + `printf '%s' '${mode}' > "$d/frame.req"; ` + `for i in $(seq 1 400); do [ -f "$d/frame.done" ] && break; sleep 0.01; done; ` + `j="$d/frame.out.json"; p="$d/frame.out.jpg"; ` + `jl=$(wc -c < "$j" 2>/dev/null | tr -d ' '); pl=$(wc -c < "$p" 2>/dev/null | tr -d ' '); ` + `[ -z "$jl" ] && jl=0; [ -z "$pl" ] && pl=0; ` + `printf 'ACF1 %s %s\\n' "$jl" "$pl"; cat "$j" 2>/dev/null; [ "$pl" != 0 ] && cat "$p" 2>/dev/null`; let proc; if (machines[name]?.local) { proc = spawn("bash", ["-c", remote], { stdio: ["ignore", "pipe", "ignore"] }); } else { const host = sshHostFor(name, machines); proc = spawn("ssh", [...sshOpts(host), host, "bash -s"], { stdio: ["pipe", "pipe", "ignore"] }); proc.stdin.end(remote); } let buf = Buffer.alloc(0), settled = false; const finish = (fn, arg) => { if (!settled) { settled = true; clearTimeout(timer); fn(arg); } }; const timer = setTimeout(() => { proc.kill(); finish(reject, new Error("direct timeout")); }, timeoutMs); proc.stdout.on("data", (chunk) => { buf = Buffer.concat([buf, chunk]); let r = parseFrame(buf); while (r && r.skip) { buf = r.rest; r = parseFrame(buf); } if (r && r.frame) { finish(resolve, r.frame); proc.kill(); } }); proc.on("error", (e) => finish(reject, e)); proc.on("exit", () => { const r = parseFrame(buf); if (r && r.frame) finish(resolve, r.frame); else finish(reject, new Error("no frame")); }); }); } export async function captureFrame(name, options = {}) { if (name === XBOX_TARGET) return captureFrameUnlocked(name, options); const spec = loadMachines()[name]; if (!spec) throw new Error(`unknown machine "${name}"`); return withMachineLease(spec.local ? spec : { sshHost: sshHostFor(name, loadMachines()) }, () => captureFrameUnlocked(name, options)); } async function captureFrameUnlocked(name, { memory = false, session, nativeGuard, nativeClick, nativeDrag, expectedTargetId, noOCR = false, noVisual = false, fast = false, screen = false, cursor = false, cursorAt, targetAt, targetId, manualCheck, pressAt, pressCount = 1, pressTitle, actionOnly = false, clearTarget = false, clearOverlays = false, quietOverlay = false, overlays = false, crop, baseline = false, diff = false, out, json = false, direct = false, preview = false } = {}) { if (name === XBOX_TARGET) { if (nativeGuard || nativeClick || nativeDrag || actionOnly || targetAt || targetId || manualCheck || pressAt || pressTitle || clearTarget || clearOverlays) { throw new Error("xbox is an observe-only Frame target; use the native gamepad/live-publish loop for control"); } const jpg = captureXboxJpeg(); const env = { capture: "ok", capture_scope: "screen", target_kind: "xbox-device-portal", meta: { frontmost: { app: "Xbox display", bundle: "Windows.Xbox" }, screen: { w: 1920, h: 1080, scale: 1 }, windows: [], }, ocr: [], ax: { trusted: false, elements: [] }, visual: [], diff: [], design_context: { viewing_mode: "10-foot television UI", coordinate_system: "1920x1080, origin top-left", review_priorities: [ "render representation and asset fidelity", "silhouette and line continuity", "focal scale and placement", "negative-space balance", "hierarchy and TV-distance readability", "contrast and color relationships", ], }, }; if (memory) return { env, jpg }; const outPath = out || join(FRAMES_DIR, "xbox.jpg"); mkdirSync(dirname(outPath), { recursive: true }); writeFileSync(outPath, jpg); if (preview) requestPreview(name, outPath); if (json) process.stdout.write(JSON.stringify(env)); else console.log(JSON.stringify({ ...env, thumb: outPath, ocr_count: 0, ax_count: 0 }, null, 2)); return; } const machines = loadMachines(); if (!machines[name]) { throw new Error(`unknown machine "${name}" — known: ${Object.keys(machines).join(", ") || "(none)"}`); } if (session !== undefined && !/^[a-zA-Z0-9_-]{1,64}$/.test(session)) throw new Error("Invalid native frame session"); if (expectedTargetId !== undefined && !/^[a-zA-Z0-9_-]{1,64}$/.test(expectedTargetId)) throw new Error("Invalid staged target ID"); if ([nativeGuard, nativeClick, nativeDrag].filter(Boolean).length > 1) throw new Error('Choose native guard or click or drag, not multiple operations'); if ((nativeGuard || nativeClick || nativeDrag) && (pressAt || targetAt || actionOnly || clearTarget || clearOverlays || manualCheck || screen || crop)) { throw new Error('Native input cannot be combined with another action or capture scope'); } const nativeMode = nativeGuard || nativeClick || nativeDrag ? (nativeDrag ? 'native-drag=' : nativeClick ? 'native-click=' : 'native-guard=') + Buffer.from(JSON.stringify(nativeDrag || nativeClick || nativeGuard)).toString('base64') : ''; const mode = [expectedTargetId ? `expect-target=${expectedTargetId}` : "", session ? `session=${session}` : "", screen ? "screen" : "window", noOCR ? "noocr" : "full", noVisual ? "novisual" : "", fast ? "fast" : "", cursorAt ? `cursor=${cursorAt[0]},${cursorAt[1]}` : cursor ? "cursor" : "", targetAt ? `target=${targetAt[0]},${targetAt[1]}` : "", targetId ? `target-id=${targetId}` : "", manualCheck ? `manual-check=${manualCheck}` : "", pressAt ? `press=${pressAt[0]},${pressAt[1]},${pressCount}` : "", pressTitle ? `press-title=${Buffer.from(pressTitle, "utf8").toString("base64")}` : "", actionOnly ? "action-only" : "", clearTarget ? "target-clear" : "", clearOverlays ? "overlay-clear" : "", quietOverlay ? "quiet-overlay" : "", overlays ? "overlays" : "", crop ? `crop=${crop.join(",")}` : "", baseline ? "baseline" : "", diff ? "diff" : ""] .concat(nativeMode).filter(Boolean).join(" "); // Use the resident server only if already running (opt-in; see runServer); // otherwise a one-shot direct ssh. Both return an ACF1 {json, jpg} frame — // the JPEG is raw bytes, never base64. let frame = null; if (!direct && existsSync(SOCK_PATH)) { try { frame = await requestViaServer(name, mode); } catch (error) { // Retry only a failed connection. A timeout may follow a delivered click. if (!["ENOENT", "ECONNREFUSED"].includes(error.code)) throw error; } } if (!frame) { try { frame = machines[name]?.local && !direct ? await localFrame(join(HOME, ".local", "share", "slab", "state"), mode) : await directFrame(name, machines, mode); } catch (e) { throw new Error(`${name} unreachable: ${String(e.message || e).split("\n")[0]}`); } } let env; try { env = JSON.parse(frame.json); } catch { throw new Error(`no frame from ${name} — is SlabMenubar running there? (frame doctor ${name})`); } if (env.error) throw new Error(env.error); if (session && (baseline || diff) && env.capture === "ok" && env.observation?.session !== session) { throw new Error("Native Frame does not support isolated sessions yet; update SlabMenubar on this machine"); } if (memory) return { env, jpg: frame.jpg?.length ? frame.jpg : undefined }; if (env.capture === "permission_needed") { console.error( `${name}: Screen Recording not granted to SlabMenubar yet.\n` + ` run: frame setup ${name}\n` + ` (AX + window/meta still captured; pixels + OCR are blocked until granted)`, ); } // Write the raw JPEG to disk (no base64 anywhere on the path). const outPath = out || join(FRAMES_DIR, `${name}.jpg`); const hasJpg = frame.jpg && frame.jpg.length > 0; if (hasJpg) { mkdirSync(dirname(outPath), { recursive: true }); writeFileSync(outPath, frame.jpg); } // Opt-in: pop (or live-reload) the local preview window, badged `name`. // Only with pixels in hand — a permission-blocked frame has nothing to show. if (preview && hasJpg) requestPreview(name, outPath); else if (preview && !hasJpg) { console.error(`(no pixels to preview for ${name} — capture blocked; run: frame setup ${name})`); } if (json) { process.stdout.write(frame.json); return; } const rest = { ...env }; rest.thumb = hasJpg ? outPath : null; rest.ocr_count = (env.ocr || []).length; rest.ax_count = (env.ax?.elements || []).length; delete rest.ocr; delete rest.ax; console.log(JSON.stringify(rest, null, 2)); } // Read a TCC grant state (system DB is world-readable here). function srGrant(name, machines) { try { const q = `sqlite3 "/Library/Application Support/com.apple.TCC/TCC.db" ` + `"select auth_value from access where service='kTCCServiceScreenCapture' ` + `and client='${APP_BUNDLE}';" 2>/dev/null`; return runOn(name, machines, q).trim(); } catch { return ""; } } function doctor(name) { const machines = loadMachines(); const names = name ? [name] : [...Object.keys(machines), XBOX_TARGET]; if (!names.length) { console.error(`no machines registered (${CONFIG_PATH})`); process.exit(1); } for (const n of names) { if (n === XBOX_TARGET) { try { const family = JSON.parse(xboxPortalGet("/api/os/devicefamily").toString("utf8")); console.log(`xbox: Device Portal reachable | ${family.DeviceType || "Windows.Xbox"} | screenshot /ext/screenshot`); } catch (error) { console.log(`xbox: UNREACHABLE (${String(error.message || error).split("\n")[0]})`); } continue; } let running = "?"; try { running = runOn(n, machines, "pgrep -x slab-menubar >/dev/null && echo yes || echo no").trim(); } catch (e) { console.log(`${n}: UNREACHABLE (${e.message.split("\n")[0]})`); continue; } const sr = srGrant(n, machines); const srLabel = sr === "2" ? "granted" : sr === "" ? "not listed (capture once to register)" : `denied (auth=${sr})`; console.log( `${n}: SlabMenubar ${running === "yes" ? "running" : "NOT running"} | ` + `Screen Recording ${srLabel} | Accessibility inherited from app trust`, ); } } async function setup(name) { if (name === XBOX_TARGET) { console.log("xbox: no Screen Recording grant is needed; Frame uses the authenticated Device Portal screenshot endpoint."); return; } const machines = loadMachines(); if (!machines[name]) { console.error(`unknown machine "${name}"`); process.exit(1); } console.log(`Triggering a capture on ${name} to surface the Screen Recording prompt…`); await captureFrame(name, { noOCR: true }); const sr = srGrant(name, machines); if (sr === "2") { console.log(`✓ ${name}: Screen Recording already granted — frames will include pixels + OCR.`); return; } console.log( `\nOn ${name}'s screen, grant Screen Recording to SlabMenubar:\n` + ` 1. A system prompt may already be showing — click "Allow".\n` + ` 2. Otherwise: System Settings → Privacy & Security → Screen & System\n` + ` Audio Recording → enable "SlabMenubar".\n` + ` 3. Re-run: frame ${name} (the app picks up the grant; no restart needed)\n` + `(Accessibility is already granted to the app, so the AX tree needs no toggle.)`, ); } function list() { const machines = loadMachines(); const names = Object.keys(machines); for (const n of names) console.log(`${n}\t-> ssh ${sshHostFor(n, machines)}`); console.log("xbox\t-> Device Portal /ext/screenshot (observe-only)"); } // ---- arg parse (imports must not run the CLI or write to MCP stdout) ---- async function main() { const argv = process.argv.slice(2); const cmd = argv[0]; const flag = (f) => argv.includes(f); const opt = (f) => { const i = argv.indexOf(f); return i >= 0 ? argv[i + 1] : undefined; }; const pointOpt = (f) => { const raw = opt(f); if (!raw) return undefined; const p = raw.split(",").map(Number); return p.length === 2 && p.every(Number.isFinite) ? p : undefined; }; if (!cmd || cmd === "-h" || cmd === "--help") { console.log( "frame — capture the focused window (pixels + OCR + AX + state) of a remote Mac\n\n" + " frame [--screen] [--no-ocr] [--no-visual] [--fast] [--cursor] [--target-at x,y] [--target-id id] [--manual-check id] [--press-at x,y] [--action-only] [--clear-target] [--clear-overlays] [--quiet-overlay] [--overlays] [--direct] [--preview] [--out file.jpg] [--json]\n" + " --screen: capture the complete display instead of the focused window\n" + " --fast: Vision .fast OCR (lower latency, less accurate on small text)\n" + " --cursor: draw a high-contrast virtual marker at the mouse position\n" + " --target-at: dim the display and outline a proposed click target\n" + " --target-id: correlate a direct human tap with the staged action\n" + " --manual-check: consume a matching direct-tap acknowledgement\n" + " --press-at: perform the approved AX action (physical click fallback)\n" + " --action-only: acknowledge an approved action without a redundant capture\n" + " --clear-target: clear the persistent proposed-click marker\n" + " --clear-overlays: retire all Frame-owned transient overlay windows\n" + " --quiet-overlay: perform OCR without drawing its on-screen boxes\n" + " --overlays: keep slab's own rocks and previews in the shot (for designing them)\n" + " --direct: bypass the resident server, do a one-shot ssh\n" + " --preview: pop a labeled preview window of the pulled frame on THIS Mac\n" + " frame view capture + open the badged preview window (= --preview)\n" + " frame tape [secs] [--crop x,y,w,h] [--fps n] [--cursor] [--out f.mp4] [--label l]\n" + " record a short HQ mp4 (whole display or a crop); prints its path + metadata\n" + " frame doctor [machine] per-machine daemon + permission status\n" + " frame setup trigger + guide the one-time Screen Recording grant\n" + " frame list registered machines\n" + " frame server run the resident server (holds a warm agent per\n" + " machine; auto-started on demand otherwise)\n" + " frame stop stop the resident server\n", ); process.exit(0); } if (cmd === "server") runServer(); else if (cmd === "stop") { try { unlinkSync(SOCK_PATH); } catch {} try { execFileSync("pkill", ["-f", "frame.mjs server"]); } catch {} console.log("frame server stopped"); } else if (cmd === "doctor") doctor(argv[1]); else if (cmd === "setup") await setup(argv[1]); else if (cmd === "list") list(); else if (cmd === "view") { // `frame view ` — capture + open the badged preview window. // Sugar for `frame --preview` so it reads like the slab-video / // slab-pdf "show me this" verbs. if (!argv[1]) { console.error("usage: frame view "); process.exit(1); } await captureFrame(argv[1], { session: opt("--session"), noOCR: flag("--no-ocr"), noVisual: flag("--no-visual"), fast: flag("--fast"), screen: flag("--screen"), cursor: flag("--cursor"), direct: flag("--direct"), out: opt("--out"), preview: true }); } else if (cmd === "tape") { // `frame tape [secs] …` — the CLI face of frame-tape.mjs / frame_tape. // Recording (reel) + crop (ffmpeg) live in the shared lib so the MCP tool and // this command stay one implementation. const machine = argv[1]; if (!machine) { console.error("usage: frame tape [secs] [--crop x,y,w,h] [--fps n] [--cursor] [--out file.mp4] [--label name]"); process.exit(1); } const { recordTape } = await import("../lib/frame-tape.mjs"); const secs = argv[2] && !argv[2].startsWith("--") ? Number(argv[2]) : (opt("--secs") ? Number(opt("--secs")) : undefined); const r = await recordTape({ machine, duration: secs, crop: opt("--crop")?.split(",").map(Number), fps: opt("--fps") ? Number(opt("--fps")) : undefined, cursor: flag("--cursor"), out: opt("--out"), label: opt("--label"), }); console.log(JSON.stringify(r, null, 2)); } else await captureFrame(cmd, { session: opt("--session"), noOCR: flag("--no-ocr"), noVisual: flag("--no-visual"), fast: flag("--fast"), screen: flag("--screen"), cursor: flag("--cursor"), cursorAt: pointOpt("--cursor-at"), targetAt: pointOpt("--target-at"), targetId: opt("--target-id"), manualCheck: opt("--manual-check"), pressAt: pointOpt("--press-at"), pressCount: Number(opt("--press-count") || 1), pressTitle: opt("--press-title"), actionOnly: flag("--action-only"), clearTarget: flag("--clear-target"), clearOverlays: flag("--clear-overlays"), quietOverlay: flag("--quiet-overlay"), overlays: flag("--overlays"), crop: opt("--crop")?.split(",").map(Number), baseline: flag("--baseline"), diff: flag("--diff"), direct: flag("--direct"), out: opt("--out"), json: flag("--json"), preview: flag("--preview") }); } if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { main().catch(error => { console.error(error.message); process.exitCode = 1; }); }