#!/usr/bin/env node // analysis-overlay.mjs — a realtime "analysis layer" viewer for slab. // // PROTOTYPE (uncommitted). It is the visual companion to `frame` (OBSERVE: // pixels + OCR + AX) and `puppet` (ACT: trusted stroke/gesture/key): it draws, // live over the captured screen, WHERE the machine "sees" text (OCR boxes), // WHERE it can act (AX click targets), and WHERE it just clicked (hit-scan // pulses) — the same analysis a driving agent reasons over, made glanceable. // // It serves an SVG overlay composited on top of the existing JPEG frame // streams, so it touches NEITHER the puppet daemon NOR the frame server — // it is a pure reader of files those tools already write: // // ~/.local/share/slab/frames/.jpg (frame --preview / loop) // ~/.local/share/slab/state/frame.out.json (latest local envelope) // ~/.local/share/slab/frames/.frame.json (optional per-machine envelope, see --emit) // ~/.local/share/slab/analysis/.hits.jsonl (click pulses; see hit logger below) // // Routes: // GET / dark grid, one analysis pane per machine // GET /m/ full-screen analysis pane for one machine // GET /frame/.jpg the raw JPEG (no-store) // GET /data/.json { screen, ocr[], ax[], hits[] } for the overlay // // The overlay (OCR boxes / AX targets / hit pulses) toggles per-layer in the // page with the keys O / A / H, or the on-page checkboxes. Boxes are drawn in // an scaled from screen-points to the displayed JPEG, so they track the // image at any window size. // // node slab/bin/analysis-overlay.mjs [port] # default 7778 (puppet-view owns 7777) // open http://localhost:7778 // // PUBLIC repo: no machine names here — they come from the frames on disk. import { readFileSync, readdirSync, existsSync, statSync } from "node:fs"; import http from "node:http"; import { homedir } from "node:os"; import { join } from "node:path"; const HOME = homedir(); const SLAB = process.env.SLAB_HOME || join(HOME, ".local", "share", "slab"); const FRAMES_DIR = join(SLAB, "frames"); const ANALYSIS_DIR = join(SLAB, "analysis"); const LOCAL_ENVELOPE = join(SLAB, "state", "frame.out.json"); const PORT = Number(process.argv[2] || 7778); // Machine names = the .jpg files frame --preview leaves on disk. function machineNames() { if (!existsSync(FRAMES_DIR)) return []; return readdirSync(FRAMES_DIR) .filter((f) => f.endsWith(".jpg") && !f.endsWith(".overlay.jpg")) .map((f) => f.slice(0, -4)); } function readJSON(path) { try { return JSON.parse(readFileSync(path, "utf8")); } catch { return null; } } // Find the freshest envelope for a machine. A per-machine sidecar // (.frame.json) wins if present (write it with `frame --json > // …`, or via the --emit helper); otherwise fall back to the single local // state envelope (only meaningful for the controller capturing itself). function envelopeFor(machine) { const sidecar = join(FRAMES_DIR, `${machine}.frame.json`); if (existsSync(sidecar)) return readJSON(sidecar); if (existsSync(LOCAL_ENVELOPE)) return readJSON(LOCAL_ENVELOPE); return null; } // Hit pulses: a JSON-lines log, one {x,y,kind,label,t} per click/stroke point. // Tail the last N within a recency window so the overlay shows recent activity. function hitsFor(machine, { windowMs = 8000, max = 200 } = {}) { const path = join(ANALYSIS_DIR, `${machine}.hits.jsonl`); if (!existsSync(path)) return []; let lines; try { lines = readFileSync(path, "utf8").trim().split("\n").slice(-max); } catch { return []; } const now = Date.now(); const out = []; for (const ln of lines) { if (!ln.trim()) continue; try { const h = JSON.parse(ln); if (typeof h.t === "number" && now - h.t > windowMs) continue; out.push(h); } catch {} } return out; } // The overlay payload: screen geometry + the boxes the agent reasons over. // OCR + AX rects are [x,y,w,h] in LOGICAL screen points (top-left origin) — // exactly what FrameCapture.swift emits. The page scales them to the image. function dataFor(machine) { const env = envelopeFor(machine) || {}; const screen = env.meta?.screen || { w: 0, h: 0, scale: 1 }; const ocr = (env.ocr || []).map((o) => ({ r: o.r, t: o.t, cx: o.cx, cy: o.cy })); const ax = (env.ax?.elements || []).map((e) => ({ r: e.r, role: e.role, title: e.title, cx: e.cx, cy: e.cy, })); let jpgMtime = 0; const jpg = join(FRAMES_DIR, `${machine}.jpg`); try { jpgMtime = statSync(jpg).mtimeMs; } catch {} return { machine, screen, ocr, ax, hits: hitsFor(machine), jpgMtime, frontmost: env.meta?.frontmost?.app || null, capture: env.capture || null, }; } // ─── page ────────────────────────────────────────────────────────────────── // The client polls /data/.json and rebuilds the SVG; the JPEG is reloaded // via cache-busting on mtime change. (MJPEG would be smoother but couples to // puppet-view's stream; this prototype stays decoupled — a poll loop is fine // for a few-Hz frame.) Layer toggles: O ocr, A ax, H hits. function panePage(machine) { return `analysis · ${machine}
${machine}  
`; } function gridPage() { const names = machineNames(); const tiles = names .map( (n) => `
${n}
`, ) .join(""); return `slab analysis layer
slab analysis layer — click a machine for the live OCR / AX / hit overlay
${tiles || '

no frames yet — pull one with: frame <machine> --preview

'}
`; } http .createServer((req, res) => { const url = req.url.split("?")[0]; if (url === "/") { res.writeHead(200, { "Content-Type": "text/html" }); return res.end(gridPage()); } let m; if ((m = url.match(/^\/m\/([\w.-]+)$/))) { res.writeHead(200, { "Content-Type": "text/html" }); return res.end(panePage(m[1])); } if ((m = url.match(/^\/data\/([\w.-]+)\.json$/))) { res.writeHead(200, { "Content-Type": "application/json", "Cache-Control": "no-store" }); return res.end(JSON.stringify(dataFor(m[1]))); } if ((m = url.match(/^\/frame\/([\w.-]+)\.jpg$/))) { const file = join(FRAMES_DIR, `${m[1]}.jpg`); if (!existsSync(file)) { res.writeHead(404); return res.end("no frame"); } res.writeHead(200, { "Content-Type": "image/jpeg", "Cache-Control": "no-store" }); return res.end(readFileSync(file)); } res.writeHead(404); res.end("not found"); }) .listen(PORT, () => console.log(`analysis-overlay: http://localhost:${PORT} (frames: ${FRAMES_DIR})`), );