From 4216311ed25eb35d0bd837a73d023c27bbd32ec1 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Mon, 22 Jun 2026 14:45:21 -0700 Subject: [PATCH] slab/puppet: observation scan overlay (see what the agent reads) - scanExpr(): boxes every readable text run (green) + interactive target (dashed blue) in viewport space, into a persistent #__ao_scan layer that auto-clears after a TTL. - puppet scan [--ttl]; shot auto-fires the scan (after capture) when analysis is on, so the live screen lights up with the agent's observation at every decision point without cluttering the capture itself. --- slab/bin/analysis-layer.mjs | 31 +++++++++++++++++++++++++++++++ slab/bin/puppet.mjs | 28 +++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/slab/bin/analysis-layer.mjs b/slab/bin/analysis-layer.mjs index dbb97fce09..550c0b02f4 100644 --- a/slab/bin/analysis-layer.mjs +++ b/slab/bin/analysis-layer.mjs @@ -108,6 +108,37 @@ export function overlayClearExpr() { return `(()=>{const s=document.getElementById("__analysis_overlay");if(s)s.remove();return true;})()`; } +// ─── observation scan (page-side "what I'm reading") ─────────────────────── +// A DOM analogue of OCR: box every readable text run (green) and every +// interactive target (dashed blue) currently on the page, drawn into a +// persistent #__ao_scan layer of the analysis overlay so a watcher SEES what +// the agent is observing. Pixel-accurate (browser viewport space, unlike +// whole-screen Vision OCR). `ttl` ms auto-clears the scan (0 = persist). +// Returns {text, targets} counts. +export function scanExpr(ttl = 2600) { + return `(()=>{ +const NS="http://www.w3.org/2000/svg"; +let s=document.getElementById("__analysis_overlay"); +if(!s){s=document.createElementNS(NS,"svg");s.id="__analysis_overlay"; + s.style.cssText="position:fixed;inset:0;width:100vw;height:100vh;z-index:2147483646;pointer-events:none"; + document.documentElement.appendChild(s);} +const mk=(t,a,txt)=>{const e=document.createElementNS(NS,t);for(const k in a)e.setAttribute(k,a[k]);if(txt!=null)e.textContent=txt;return e;}; +const old=s.querySelector("#__ao_scan");if(old)old.remove(); +const g=mk("g",{id:"__ao_scan"}); +const vw=innerWidth,vh=innerHeight,seen=new Set(); +const vis=el=>{const r=el.getBoundingClientRect();if(r.width<6||r.height<6||r.bottom<0||r.top>vh||r.right<0||r.left>vw)return null;const st=getComputedStyle(el);if(st.visibility==="hidden"||st.opacity==="0"||st.display==="none")return null;return r;}; +// interactive targets (dashed blue) — what I can act on +let nT=0;for(const el of document.querySelectorAll("button,a[href],input,textarea,select,[role=button],[role=option],[contenteditable=true]")){const r=vis(el);if(!r)continue;const k=Math.round(r.x)+","+Math.round(r.y)+","+Math.round(r.width);if(seen.has(k))continue;seen.add(k);nT++; + g.appendChild(mk("rect",{x:r.x,y:r.y,width:r.width,height:r.height,rx:4,fill:"rgba(92,168,255,.06)",stroke:"#5ca8ff","stroke-width":1.2,"stroke-dasharray":"4 3"}));} +// readable text runs (green) — what I'm reading +let nX=0;const walk=document.createTreeWalker(document.body,NodeFilter.SHOW_TEXT);let node;while((node=walk.nextNode())){const t=(node.textContent||"").trim();if(t.length<2||nX>180)continue;const el=node.parentElement;if(!el)continue;const r=vis(el);if(!r||r.width>vw*0.98)continue;const k="t"+Math.round(r.x)+","+Math.round(r.y);if(seen.has(k))continue;seen.add(k);nX++; + g.appendChild(mk("rect",{x:r.x-1,y:r.y-1,width:r.width+2,height:r.height+2,rx:2,fill:"none",stroke:"#39d98a","stroke-width":1,"stroke-opacity":.7}));} +s.appendChild(g); +if(${ttl}>0)setTimeout(()=>{const x=document.getElementById("__ao_scan");if(x)x.remove();const o=document.getElementById("__analysis_overlay");if(o&&!o.childNodes.length)o.remove();},${ttl}); +return JSON.stringify({text:nX,targets:nT}); +})()`; +} + // ─── CLI: emit a per-machine sidecar envelope for the JPEG viewer ────────── // `frame --json` prints the envelope to stdout; pipe it here to drop the // sidecar the viewer reads: frame panda --json | analysis-layer.mjs emit panda diff --git a/slab/bin/puppet.mjs b/slab/bin/puppet.mjs index d98e938772..9ab4f2bf92 100755 --- a/slab/bin/puppet.mjs +++ b/slab/bin/puppet.mjs @@ -40,7 +40,7 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from " import net from "node:net"; import { homedir } from "node:os"; import { dirname, join } from "node:path"; -import { logHit, overlayDrawExpr, overlayClearExpr } from "./analysis-layer.mjs"; +import { logHit, overlayDrawExpr, overlayClearExpr, scanExpr } from "./analysis-layer.mjs"; const HOME = homedir(); const CONFIG_PATH = @@ -341,15 +341,27 @@ class Machine { if (live && opts.format === "jpeg" && !opts.fresh) { try { const fs = await import("node:fs"); - return fs.readFileSync(live).toString("base64"); // 0ms: latest screencast frame + const data = fs.readFileSync(live).toString("base64"); // 0ms: latest screencast frame + this.analysisScan(sessionId); + return data; } catch {} } const params = { format: opts.format ?? "png" }; if (params.format === "jpeg") params.quality = opts.quality ?? 80; const r = await this.call("Page.captureScreenshot", params, sessionId); + // After the capture (so the shot stays clean), light up the observation + // overlay on the LIVE page — a watcher sees what was just looked at. + this.analysisScan(sessionId); return r.data; // base64 } + // Fire the observation scan (text + interactive boxes) on the page, fire- + // and-forget. No-op unless analysis is on for this machine. + analysisScan(sessionId) { + if (!this.analysisOn) return; + this.callNoWait("Runtime.evaluate", { expression: scanExpr() }, sessionId); + } + // Trusted mouse drag through points, virtual cursor overlay riding along. async stroke(points, filter) { const sessionId = await this.session(filter); @@ -698,6 +710,10 @@ async function handleRequest(req, sock) { // analysis on|off [machine|all] — toggle the realtime hit-scan overlay + // hit logging per machine. Works on disconnected machines too (just sets // the flag); turning off clears any live page-side overlay. + // scan: draw the observation overlay (text + interactive boxes) on the + // page so a watcher sees what the agent is reading. Returns {text,targets}. + case "scan": + return one(machine).eval(scanExpr(args.ttl ?? 2600), args.target); case "analysis": { const want = args.on !== false && args.on !== "off"; const names = !machine || machine === "all" ? [...machines.keys()] : [machine]; @@ -1008,6 +1024,12 @@ async function main() { }); console.log("cursor placed"); return; + // puppet scan [--ttl=2600] — draw the observation overlay + case "scan": + console.log( + await rpc({ cmd: "scan", machine: args[0], args: { ttl: flags.ttl ? Number(flags.ttl) : undefined, target: flags.target } }), + ); + return; // puppet analysis on|off [machine|all] — realtime hit-scan overlay layer case "analysis": { const on = (args[0] || "on").toLowerCase() !== "off"; @@ -1034,7 +1056,7 @@ async function main() { return; default: console.log( - "usage: puppet daemon|config|status|list|eval|evalall|batch|broadcast|fanout|nav|newtab|close|reload|shot|watch|unwatch|stroke|gesture|key|cursor|analysis|tail (see header)", + "usage: puppet daemon|config|status|list|eval|evalall|batch|broadcast|fanout|nav|newtab|close|reload|shot|watch|unwatch|stroke|gesture|key|cursor|scan|analysis|tail (see header)", ); } } -- 2.51.2