diff --git a/slab/bin/frame-mcp.mjs b/slab/bin/frame-mcp.mjs index 8b14f94189..57645f5699 100644 --- a/slab/bin/frame-mcp.mjs +++ b/slab/bin/frame-mcp.mjs @@ -21,7 +21,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { tmpdir } from "node:os"; import { httpPort, serveHttp, serveStdio } from "../../toolchain/mcp/http-front.mjs"; -import { clickPoint, sendKeys } from "./macos.mjs"; +import { clickPoint, hoverPoint, sendKeys } from "./macos.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); const FRAME = join(HERE, "frame.mjs"); @@ -87,7 +87,7 @@ function digest(env) { } // ── the capture tool: frame a machine, return image + digest ──────────────── -async function toolFrame({ machine, ocr = true, fast = false, cursor = true, cursorAt } = {}) { +async function toolFrame({ machine, ocr = true, fast = false, cursor = true, cursorAt, crop } = {}) { if (!machine) throw new Error("`machine` is required (see frame_list)"); const out = join(tmpdir(), `frame-mcp-${machine}-${process.pid}.jpg`); const args = [machine, "--json", "--out", out]; @@ -95,6 +95,7 @@ async function toolFrame({ machine, ocr = true, fast = false, cursor = true, cur if (fast) args.push("--fast"); if (cursorAt) args.push("--cursor-at", `${cursorAt[0]},${cursorAt[1]}`); else if (cursor) args.push("--cursor"); + if (crop) args.push("--crop", crop.join(",")); const { stdout } = await runFrame(args); let env; @@ -122,6 +123,14 @@ async function toolFrame({ machine, ocr = true, fast = false, cursor = true, cur return content; } +async function toolHover({ machine, x, y, width = 720, height = 520, ocr = true, fast = true }) { + x = Number(x); y = Number(y); + hoverPoint(machineSpec(machine), x, y); + await settle(350); + const crop = [Math.round(x - width / 2), Math.round(y - height / 2), Math.round(width), Math.round(height)]; + return toolFrame({ machine, ocr, fast, cursorAt: [x, y], crop }); +} + // Native exploration primitives return the post-action frame in the SAME MCP // response. Agents need one tool round-trip, not act → wait → call frame again. async function toolClick({ machine, x, y, count = 1, ocr = true, fast = true }) { @@ -169,6 +178,11 @@ const TOOLS = [ required: ["machine"], }, }, + { + name: "frame_hover", + description: "OBSERVES CONTEXT: move the real pointer without clicking, wait for hover-only controls/tooltips, then return a cheaper cropped reframe around that point. Lesson 1: when an element may reveal options, hover and reframe before clicking. Coordinates remain global and click-ready.", + inputSchema: { type: "object", properties: { machine: { type: "string" }, x: { type: "number" }, y: { type: "number" }, width: { type: "number", description: "Crop width (default 720)." }, height: { type: "number", description: "Crop height (default 520)." }, ocr: { type: "boolean" }, fast: { type: "boolean" } }, required: ["machine", "x", "y"] }, + }, { name: "frame_click", description: "ACTS + OBSERVES: click a native macOS screen coordinate from frame OCR/AX, then immediately return a fresh frame with a virtual marker at the click. Use for low-risk UI exploration; inspect labels and avoid destructive controls.", @@ -223,6 +237,7 @@ const TOOLS = [ async function callTool(name, args) { switch (name) { case "frame": return toolFrame(args || {}); + case "frame_hover": return toolHover(args || {}); case "frame_click": return toolClick(args || {}); case "frame_key": return toolKey(args || {}); case "frame_list": return toolList(); diff --git a/slab/bin/frame.mjs b/slab/bin/frame.mjs index ecf74ae23e..284105e97e 100755 --- a/slab/bin/frame.mjs +++ b/slab/bin/frame.mjs @@ -326,13 +326,13 @@ function directFrame(name, machines, mode, timeoutMs = 15000) { }); } -async function captureFrame(name, { noOCR = false, fast = false, cursor = false, cursorAt, out, json = false, direct = false, preview = false } = {}) { +async function captureFrame(name, { noOCR = false, fast = false, cursor = false, cursorAt, crop, out, json = false, direct = false, preview = false } = {}) { const machines = loadMachines(); if (!machines[name]) { console.error(`unknown machine "${name}" — known: ${Object.keys(machines).join(", ") || "(none)"}`); process.exit(1); } - const mode = [noOCR ? "noocr" : "full", fast ? "fast" : "", cursorAt ? `cursor=${cursorAt[0]},${cursorAt[1]}` : cursor ? "cursor" : ""] + const mode = [noOCR ? "noocr" : "full", fast ? "fast" : "", cursorAt ? `cursor=${cursorAt[0]},${cursorAt[1]}` : cursor ? "cursor" : "", crop ? `crop=${crop.join(",")}` : ""] .filter(Boolean).join(" "); // Use the resident server only if already running (opt-in; see runServer); // otherwise a one-shot direct ssh. Both return an ACF1 {json, jpg} frame — @@ -504,4 +504,4 @@ else if (cmd === "view") { // slab-pdf "show me this" verbs. if (!argv[1]) { console.error("usage: frame view "); process.exit(1); } await captureFrame(argv[1], { noOCR: flag("--no-ocr"), fast: flag("--fast"), cursor: flag("--cursor"), direct: flag("--direct"), out: opt("--out"), preview: true }); -} else await captureFrame(cmd, { noOCR: flag("--no-ocr"), fast: flag("--fast"), cursor: flag("--cursor"), cursorAt: pointOpt("--cursor-at"), direct: flag("--direct"), out: opt("--out"), json: flag("--json"), preview: flag("--preview") }); +} else await captureFrame(cmd, { noOCR: flag("--no-ocr"), fast: flag("--fast"), cursor: flag("--cursor"), cursorAt: pointOpt("--cursor-at"), crop: opt("--crop")?.split(",").map(Number), direct: flag("--direct"), out: opt("--out"), json: flag("--json"), preview: flag("--preview") }); diff --git a/slab/bin/macos.mjs b/slab/bin/macos.mjs index 4e516d8e00..5bed09361d 100644 --- a/slab/bin/macos.mjs +++ b/slab/bin/macos.mjs @@ -72,6 +72,20 @@ export function osa(spec, script) { return sh(spec, "osascript -", { stdin: script }); } +export function jxa(spec, script) { + return sh(spec, "osascript -l JavaScript -", { stdin: script }); +} + +// Post a genuine no-click mouse move. A simple cursor warp is ignored from +// some SSH sessions; the HID event reliably activates contextual hover UI. +export function hoverPoint(spec, x, y) { + if (!Number.isFinite(x) || !Number.isFinite(y)) throw new Error("hover coordinates must be numbers"); + return jxa(spec, `ObjC.import("CoreGraphics"); +const p = $.CGPointMake(${Math.round(x)}, ${Math.round(y)}); +const e = $.CGEventCreateMouseEvent(null, $.kCGEventMouseMoved, p, $.kCGMouseButtonLeft); +$.CGEventPost($.kCGHIDEventTap, e);`); +} + // AppleScript string literal: quote it, escape backslash + quote. function aslit(s) { return '"' + String(s).replace(/\\/g, "\\\\").replace(/"/g, '\\"') + '"'; diff --git a/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift b/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift index f15d99ae08..411bc503bd 100644 --- a/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift +++ b/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift @@ -65,8 +65,13 @@ final class FrameCapture { let xy = token.dropFirst("cursor=".count).split(separator: ",").compactMap { Double($0) } if xy.count == 2 { cursorOverride = CGPoint(x: xy[0], y: xy[1]) } } + var crop: CGRect? + if let token = mode.split(separator: " ").first(where: { $0.hasPrefix("crop=") }) { + let v = token.dropFirst("crop=".count).split(separator: ",").compactMap { Double($0) } + if v.count == 4 { crop = CGRect(x: v[0], y: v[1], width: v[2], height: v[3]) } + } produce(noOCR: mode.contains("noocr"), fast: mode.contains("fast"), - virtualCursor: mode.contains("cursor"), cursorOverride: cursorOverride) + virtualCursor: mode.contains("cursor"), cursorOverride: cursorOverride, crop: crop) fm.createFile(atPath: Paths.frameDone, contents: nil) } @@ -160,7 +165,7 @@ final class FrameCapture { // MARK: - capture (in-process; no screencapture subprocess → no launchd throttle) - private func captureDisplay() -> CGImage? { + private func captureDisplay(crop: CGRect? = nil) -> CGImage? { guard #available(macOS 14.0, *) else { return nil } let sem = DispatchSemaphore(value: 0) var img: CGImage? @@ -182,8 +187,13 @@ final class FrameCapture { } let filter = SCContentFilter(display: display, excludingWindows: exclude) let cfg = SCStreamConfiguration() - cfg.width = Int(Double(display.width) * self.captureScale) - cfg.height = Int(Double(display.height) * self.captureScale) + var region = CGRect(x: 0, y: 0, width: display.width, height: display.height) + if let crop = crop { + region = crop.intersection(CGRect(x: 0, y: 0, width: display.width, height: display.height)) + cfg.sourceRect = region + } + cfg.width = Int(Double(region.width) * self.captureScale) + cfg.height = Int(Double(region.height) * self.captureScale) cfg.showsCursor = true img = try? await SCScreenshotManager.captureImage(contentFilter: filter, configuration: cfg) } @@ -193,7 +203,8 @@ final class FrameCapture { // MARK: - OCR (text + click-center coords, in logical points) - private func ocr(_ cg: CGImage, scale: Double, fast: Bool = false) -> [[String: Any]] { + private func ocr(_ cg: CGImage, scale: Double, fast: Bool = false, + origin: CGPoint = .zero) -> [[String: Any]] { let req = VNRecognizeTextRequest() req.recognitionLevel = fast ? .fast : .accurate req.usesLanguageCorrection = false @@ -206,7 +217,8 @@ final class FrameCapture { for obs in (req.results ?? []) { guard let c = obs.topCandidates(1).first else { continue } let bb = obs.boundingBox - let x = bb.minX * W / scale, y = (1 - bb.maxY) * H / scale + let x = origin.x + bb.minX * W / scale + let y = origin.y + (1 - bb.maxY) * H / scale let w = bb.width * W / scale, h = bb.height * H / scale out.append(["t": c.string, "cx": Int(x + w / 2), "cy": Int(y + h / 2), "r": [Int(x), Int(y), Int(w), Int(h)]]) @@ -312,9 +324,11 @@ final class FrameCapture { // MARK: - thumbnail (downscale to 1568px — what a vision model downsamples to anyway) - private func thumbJPEG(_ cg: CGImage, maxWidth: Int, cursor: CGPoint? = nil) -> Data? { - let s = Double(maxWidth) / Double(cg.width) - let w = maxWidth, h = max(1, Int(Double(cg.height) * s)) + private func thumbJPEG(_ cg: CGImage, maxWidth: Int, cursor: CGPoint? = nil, + crop: CGRect? = nil) -> Data? { + let w = min(maxWidth, cg.width) + let s = Double(w) / Double(cg.width) + let h = max(1, Int(Double(cg.height) * s)) guard let rep = NSBitmapImageRep(bitmapDataPlanes: nil, pixelsWide: w, pixelsHigh: h, bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false, colorSpaceName: .deviceRGB, bytesPerRow: 0, bitsPerPixel: 0), @@ -324,8 +338,9 @@ final class FrameCapture { ctx.cgContext.interpolationQuality = .high ctx.cgContext.draw(cg, in: CGRect(x: 0, y: 0, width: w, height: h)) if let cursor = cursor, let screen = NSScreen.main { - let x = cursor.x * CGFloat(w) / screen.frame.width - let y = CGFloat(h) - cursor.y * CGFloat(h) / screen.frame.height + let region = crop ?? screen.frame + let x = (cursor.x - region.minX) * CGFloat(w) / region.width + let y = CGFloat(h) - (cursor.y - region.minY) * CGFloat(h) / region.height let radius = max(9.0, CGFloat(w) / 110.0) let marker = CGRect(x: x - radius, y: y - radius, width: radius * 2, height: radius * 2) @@ -348,7 +363,8 @@ final class FrameCapture { // MARK: - assemble + write the envelope private func produce(noOCR: Bool, fast: Bool = false, - virtualCursor: Bool = false, cursorOverride: CGPoint? = nil) { + virtualCursor: Bool = false, cursorOverride: CGPoint? = nil, + crop: CGRect? = nil) { func nowNs() -> UInt64 { DispatchTime.now().uptimeNanoseconds } func msSince(_ t: UInt64) -> Double { (Double(nowNs() - t) / 1e6 * 10).rounded() / 10 } var env: [String: Any] = [:] @@ -371,7 +387,9 @@ final class FrameCapture { var t = nowNs(); let mt = meta(); tm["meta"] = msSince(t) env["meta"] = mt - t = nowNs(); let cg = captureDisplay(); tm["capture"] = msSince(t) + let boundedCrop = crop.flatMap { c in NSScreen.main.map { c.intersection($0.frame) } } + if let c = boundedCrop { env["crop"] = ["x": Int(c.minX), "y": Int(c.minY), "w": Int(c.width), "h": Int(c.height)] } + t = nowNs(); let cg = captureDisplay(crop: boundedCrop); tm["capture"] = msSince(t) if cg != nil { flashCaptureIndicator() } // subtle post-capture awareness flash // The JPEG ships as RAW BYTES in a sidecar file (frame.out.jpg), not // base64 in the JSON — base64 inflates the payload +33% and burns @@ -382,7 +400,8 @@ final class FrameCapture { if noOCR { env["ocr"] = [] } else { - t = nowNs(); let boxes = ocr(cg, scale: captureScale, fast: fast); tm["ocr"] = msSince(t) + t = nowNs(); let boxes = ocr(cg, scale: captureScale, fast: fast, + origin: boundedCrop?.origin ?? .zero); tm["ocr"] = msSince(t) env["ocr"] = boxes showOcrOverlay(boxes) } @@ -391,7 +410,7 @@ final class FrameCapture { CGPoint(x: $0["x"] ?? 0, y: $0["y"] ?? 0) } let marker = virtualCursor ? (cursorOverride ?? cursorMeta) : nil - let jpg = thumbJPEG(cg, maxWidth: 1568, cursor: marker) ?? Data() + let jpg = thumbJPEG(cg, maxWidth: 1568, cursor: marker, crop: boundedCrop) ?? Data() try? jpg.write(to: URL(fileURLWithPath: Paths.frameOutJpg)) jpgBytes = jpg.count tm["thumb"] = msSince(t)