From f06e33f671f9c43fd1f7760806d2133d9a23623d Mon Sep 17 00:00:00 2001 From: prompt.ac/@jeffrey Date: Thu, 16 Jul 2026 00:43:39 +0000 Subject: [PATCH] feat(slab): add binary frame difference maps --- slab/bin/frame-mcp.mjs | 12 +++++++++--- slab/bin/frame.mjs | 6 +++--- slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 file(s) changed, 87 insertion(s)(+), 8 deletion(s)(-) diff --git a/slab/bin/frame-mcp.mjs b/slab/bin/frame-mcp.mjs --- a/slab/bin/frame-mcp.mjs +++ b/slab/bin/frame-mcp.mjs @@ -86,11 +86,14 @@ } const visual = env.visual || []; L.push(`\nVISUAL (${visual.length} compact controls) — kind @(cx,cy), distance from hover:`); for (const v of visual) L.push(` ${v.kind} @(${v.cx},${v.cy}) d=${v.distance}`); + const changes = env.diff || []; + L.push(`\nDIFF (${changes.length} changed regions) — @(cx,cy), changed grid cells:`); + for (const d of changes) L.push(` change @(${d.cx},${d.cy}) cells=${d.cells}`); return L.join("\n"); } // ── the capture tool: frame a machine, return image + digest ──────────────── -async function toolFrame({ machine, ocr = true, fast = false, cursor = true, cursorAt, crop } = {}) { +async function toolFrame({ machine, ocr = true, fast = false, cursor = true, cursorAt, crop, baseline = false, diff = false } = {}) { if (!machine) throw new Error("`machine` is required (see frame_list)"); const out = join(tmpdir(), `frame-mcp-${machine}-${process.pid}.jpg`); const args = [machine, "--json", "--out", out]; @@ -99,6 +102,8 @@ if (fast) args.push("--fast"); if (cursorAt) args.push("--cursor-at", `${cursorAt[0]},${cursorAt[1]}`); else if (cursor) args.push("--cursor"); if (crop) args.push("--crop", crop.join(",")); + if (baseline) args.push("--baseline"); + if (diff) args.push("--diff"); const { stdout } = await runFrame(args); let env; @@ -128,10 +133,11 @@ } async function toolHover({ machine, x, y, width = 720, height = 520, ocr = true, fast = true }) { x = Number(x); y = Number(y); + const crop = [Math.round(x - width / 2), Math.round(y - height / 2), Math.round(width), Math.round(height)]; + await toolFrame({ machine, ocr: false, cursor: false, crop, baseline: true }); hoverPoint(machineSpec(machine), x, y); await settle(350); - const crop = [Math.round(x - width / 2), Math.round(y - height / 2), Math.round(width), Math.round(height)]; - return toolFrame({ machine, ocr, fast, cursorAt: [x, y], crop }); + return toolFrame({ machine, ocr, fast, cursorAt: [x, y], crop, diff: true }); } // Native exploration primitives return the post-action frame in the SAME MCP diff --git a/slab/bin/frame.mjs b/slab/bin/frame.mjs --- a/slab/bin/frame.mjs +++ b/slab/bin/frame.mjs @@ -326,13 +326,13 @@ }); }); } -async function captureFrame(name, { noOCR = false, fast = false, cursor = false, cursorAt, crop, out, json = false, direct = false, preview = false } = {}) { +async function captureFrame(name, { noOCR = false, fast = false, cursor = false, cursorAt, crop, baseline = false, diff = false, out, json = false, direct = false, preview = false } = {}) { const machines = loadMachines(); if (!machines[name]) { console.error(`unknown machine "${name}" — known: ${Object.keys(machines).join(", ") || "(none)"}`); process.exit(1); } - const mode = [noOCR ? "noocr" : "full", fast ? "fast" : "", cursorAt ? `cursor=${cursorAt[0]},${cursorAt[1]}` : cursor ? "cursor" : "", crop ? `crop=${crop.join(",")}` : ""] + const mode = [noOCR ? "noocr" : "full", fast ? "fast" : "", cursorAt ? `cursor=${cursorAt[0]},${cursorAt[1]}` : cursor ? "cursor" : "", crop ? `crop=${crop.join(",")}` : "", baseline ? "baseline" : "", diff ? "diff" : ""] .filter(Boolean).join(" "); // Use the resident server only if already running (opt-in; see runServer); // otherwise a one-shot direct ssh. Both return an ACF1 {json, jpg} frame — @@ -504,4 +504,4 @@ // Sugar for `frame --preview` so it reads like the slab-video / // slab-pdf "show me this" verbs. if (!argv[1]) { console.error("usage: frame view "); process.exit(1); } await captureFrame(argv[1], { noOCR: flag("--no-ocr"), fast: flag("--fast"), cursor: flag("--cursor"), direct: flag("--direct"), out: opt("--out"), preview: true }); -} else await captureFrame(cmd, { noOCR: flag("--no-ocr"), fast: flag("--fast"), cursor: flag("--cursor"), cursorAt: pointOpt("--cursor-at"), crop: opt("--crop")?.split(",").map(Number), direct: flag("--direct"), out: opt("--out"), json: flag("--json"), preview: flag("--preview") }); +} else await captureFrame(cmd, { noOCR: flag("--no-ocr"), fast: flag("--fast"), cursor: flag("--cursor"), cursorAt: pointOpt("--cursor-at"), crop: opt("--crop")?.split(",").map(Number), baseline: flag("--baseline"), diff: flag("--diff"), direct: flag("--direct"), out: opt("--out"), json: flag("--json"), preview: flag("--preview") }); diff --git a/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift b/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift --- a/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift +++ b/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift @@ -37,6 +37,7 @@ private let captureScale: Double = 1.0 private let overlayLock = NSLock() private var overlayWindowIDs = Set() private var ocrOverlayWindow: NSWindow? + private var diffBaseline: (rect: CGRect, image: CGImage)? private func registerOverlay(_ w: NSWindow) { overlayLock.lock(); overlayWindowIDs.insert(w.windowNumber); overlayLock.unlock() } @@ -71,7 +72,8 @@ let v = token.dropFirst("crop=".count).split(separator: ",").compactMap { Double($0) } if v.count == 4 { crop = CGRect(x: v[0], y: v[1], width: v[2], height: v[3]) } } produce(noOCR: mode.contains("noocr"), fast: mode.contains("fast"), - virtualCursor: mode.contains("cursor"), cursorOverride: cursorOverride, crop: crop) + virtualCursor: mode.contains("cursor"), cursorOverride: cursorOverride, crop: crop, + saveBaseline: mode.contains("baseline"), includeDiff: mode.contains("diff")) fm.createFile(atPath: Paths.frameDone, contents: nil) } @@ -391,11 +393,76 @@ NSGraphicsContext.restoreGraphicsState() return rep.representation(using: .jpeg, properties: [.compressionFactor: 0.6]) } + // Binary difference map for assured hover exploration. Work on a 4×4 + // occupancy grid: unchanged cells vanish; adjacent changed cells collapse + // into global-coordinate regions that can be grepped locally or promoted + // as tiny crops to a visual model only when meaning remains ambiguous. + private func differenceMap(_ before: CGImage, _ after: CGImage, + origin: CGPoint, scale: Double) -> [[String: Any]] { + guard before.width == after.width, before.height == after.height else { return [] } + let w = after.width, h = after.height, stride = w * 4 + func pixels(_ image: CGImage) -> [UInt8] { + var data = [UInt8](repeating: 0, count: h * stride) + data.withUnsafeMutableBytes { raw in + if let ctx = CGContext(data: raw.baseAddress, width: w, height: h, + bitsPerComponent: 8, bytesPerRow: stride, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) { + ctx.draw(image, in: CGRect(x: 0, y: 0, width: w, height: h)) + } + } + return data + } + let a = pixels(before), b = pixels(after), cell = 4 + let gw = (w + cell - 1) / cell, gh = (h + cell - 1) / cell + var changed = [Bool](repeating: false, count: gw * gh) + for gy in 0..= 24 { hits += 1 } + } + } + changed[gy * gw + gx] = hits >= 2 + } } + var seen = [Bool](repeating: false, count: changed.count) + var out: [[String: Any]] = [] + for sy in 0..= 2 else { continue } + let px = minX * cell, pyTop = h - min(h, (maxY + 1) * cell) + let pw = min(w, (maxX + 1) * cell) - px + let ph = min(h, (maxY + 1) * cell) - minY * cell + let x = origin.x + Double(px) / scale, y = origin.y + Double(pyTop) / scale + let rw = Double(pw) / scale, rh = Double(ph) / scale + out.append(["kind": "change", "cx": Int(x + rw / 2), "cy": Int(y + rh / 2), + "r": [Int(x), Int(y), Int(rw), Int(rh)], "cells": q.count]) + } } + return out.sorted { ($0["cells"] as? Int ?? 0) > ($1["cells"] as? Int ?? 0) }.prefix(32).map { $0 } + } + // MARK: - assemble + write the envelope private func produce(noOCR: Bool, fast: Bool = false, virtualCursor: Bool = false, cursorOverride: CGPoint? = nil, - crop: CGRect? = nil) { + crop: CGRect? = nil, saveBaseline: Bool = false, + includeDiff: Bool = false) { func nowNs() -> UInt64 { DispatchTime.now().uptimeNanoseconds } func msSince(_ t: UInt64) -> Double { (Double(nowNs() - t) / 1e6 * 10).rounded() / 10 } var env: [String: Any] = [:] @@ -428,6 +495,12 @@ // encode/decode CPU. The transport length-prefixes the two. Written // BEFORE the JSON + done marker so a reader that sees `done` has both. var jpgBytes = 0 if let cg = cg { + let region = boundedCrop ?? CGRect(x: 0, y: 0, width: cg.width, height: cg.height) + if includeDiff, let baseline = diffBaseline, baseline.rect.equalTo(region) { + t = nowNs(); env["diff"] = differenceMap(baseline.image, cg, + origin: region.origin, scale: captureScale); tm["diff"] = msSince(t) + } else { env["diff"] = [] } + if saveBaseline { diffBaseline = (region, cg) } if noOCR { env["ocr"] = [] } else { -- tangled.sh