From da0eb4c2f782d8c253928e16101032d9b0e3a401 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Tue, 23 Jun 2026 18:21:31 -0700 Subject: [PATCH] slab: capture flash + screen-wide OCR overlay, kept out of frames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit frame.mjs captures now fire a subtle whole-display flash and draw the whole-screen OCR boxes (random translucent colors) as a ~7s overlay, so a watcher sees what was read across the entire display — not just inside a browser window. Both are excluded from the capture itself (every window the app owns is filtered out of SCContentFilter), so they never bleed into a frame. OCR keeps small/dense text (minimumTextHeight 0 + Vision rev3); the puppet scan overlay TTL goes 2600ms -> 7000ms. --- slab/bin/analysis-layer.mjs | 2 +- slab/bin/puppet.mjs | 2 +- .../Sources/SlabMenubar/FrameCapture.swift | 119 +++++++++++++++++- 3 files changed, 119 insertions(+), 4 deletions(-) diff --git a/slab/bin/analysis-layer.mjs b/slab/bin/analysis-layer.mjs index 550c0b02f..377bef3d0 100644 --- a/slab/bin/analysis-layer.mjs +++ b/slab/bin/analysis-layer.mjs @@ -115,7 +115,7 @@ export function overlayClearExpr() { // the agent is observing. Pixel-accurate (browser viewport space, unlike // whole-screen Vision OCR). `ttl` ms auto-clears the scan (0 = persist). // Returns {text, targets} counts. -export function scanExpr(ttl = 2600) { +export function scanExpr(ttl = 7000) { return `(()=>{ const NS="http://www.w3.org/2000/svg"; let s=document.getElementById("__analysis_overlay"); diff --git a/slab/bin/puppet.mjs b/slab/bin/puppet.mjs index 3d2f79985..7cce35610 100755 --- a/slab/bin/puppet.mjs +++ b/slab/bin/puppet.mjs @@ -716,7 +716,7 @@ async function handleRequest(req, sock) { // scan: draw the observation overlay (text + interactive boxes) on the // page so a watcher sees what the agent is reading. Returns {text,targets}. case "scan": - return one(machine).eval(scanExpr(args.ttl ?? 2600), args.target); + return one(machine).eval(scanExpr(args.ttl ?? 7000), args.target); case "analysis": { const want = args.on !== false && args.on !== "off"; const names = !machine || machine === "all" ? [...machines.keys()] : [machine]; diff --git a/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift b/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift index 034d00610..e5d210bdd 100644 --- a/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift +++ b/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift @@ -26,6 +26,19 @@ final class FrameCapture { private var timer: DispatchSourceTimer? private let fm = FileManager.default + // Transient overlay windows we draw (capture flash, OCR boxes). We exclude + // them from the screen capture by windowID so they never appear in a frame + // — that's the "doesn't interfere" guarantee. (The badge etc. still show.) + private let overlayLock = NSLock() + private var overlayWindowIDs = Set() + private var ocrOverlayWindow: NSWindow? + private func registerOverlay(_ w: NSWindow) { + overlayLock.lock(); overlayWindowIDs.insert(w.windowNumber); overlayLock.unlock() + } + private func unregisterOverlay(_ w: NSWindow) { + overlayLock.lock(); overlayWindowIDs.remove(w.windowNumber); overlayLock.unlock() + } + func start() { let dir = (Paths.frameReq as NSString).deletingLastPathComponent try? fm.createDirectory(atPath: dir, withIntermediateDirectories: true) @@ -46,6 +59,92 @@ final class FrameCapture { fm.createFile(atPath: Paths.frameDone, contents: nil) } + // A subtle whole-display flash, fired AFTER the pixels are grabbed so it + // never lands in the capture — just end-user awareness that a frame was + // snapped. Runs on the main thread, click-through, brief and low-alpha; the + // capture/OCR pipeline keeps going on its own queue meanwhile. + private func flashCaptureIndicator() { + DispatchQueue.main.async { + for screen in NSScreen.screens { + let win = NSWindow(contentRect: screen.frame, styleMask: .borderless, + backing: .buffered, defer: false) + win.isOpaque = false + win.backgroundColor = .clear + win.level = .screenSaver + win.ignoresMouseEvents = true + win.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle, .fullScreenAuxiliary] + let view = NSView(frame: NSRect(origin: .zero, size: screen.frame.size)) + view.wantsLayer = true + view.layer?.backgroundColor = NSColor.white.cgColor + win.contentView = view + win.alphaValue = 0.0 + win.orderFrontRegardless() + self.registerOverlay(win) + NSAnimationContext.runAnimationGroup({ ctx in + ctx.duration = 0.06 + win.animator().alphaValue = 0.22 + }, completionHandler: { + NSAnimationContext.runAnimationGroup({ ctx in + ctx.duration = 0.34 + win.animator().alphaValue = 0.0 + }, completionHandler: { self.unregisterOverlay(win); win.orderOut(nil) }) + }) + } + } + } + + // Draw the whole-screen OCR boxes as a brief screen-wide overlay, so a + // watcher sees what was read across the ENTIRE display — not just inside a + // browser window (puppet's page-side scan). Click-through, excluded from + // captures by windowID, holds ~7s then fades. Boxes are points/top-left + // (from ocr()); CALayer is bottom-left, so Y flips against screen height. + private func showOcrOverlay(_ boxes: [[String: Any]]) { + guard !boxes.isEmpty else { return } + DispatchQueue.main.async { + guard let screen = NSScreen.main else { return } + if let old = self.ocrOverlayWindow { self.unregisterOverlay(old); old.orderOut(nil) } + let H = screen.frame.height + let win = NSWindow(contentRect: screen.frame, styleMask: .borderless, + backing: .buffered, defer: false) + win.isOpaque = false + win.backgroundColor = .clear + win.level = .screenSaver + win.ignoresMouseEvents = true + win.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle, .fullScreenAuxiliary] + let view = NSView(frame: NSRect(origin: .zero, size: screen.frame.size)) + view.wantsLayer = true + if let root = view.layer { + for b in boxes { + guard let r = b["r"] as? [Int], r.count == 4 else { continue } + let box = CALayer() + box.frame = CGRect(x: CGFloat(r[0]), y: H - CGFloat(r[1]) - CGFloat(r[3]), + width: CGFloat(r[2]), height: CGFloat(r[3])) + // A random hue per box, semi-transparent fill — so the whole + // read is vivid and every box stands out against the others. + let c = NSColor(hue: .random(in: 0...1), saturation: 0.8, brightness: 1.0, alpha: 1.0) + box.backgroundColor = c.withAlphaComponent(0.28).cgColor + box.borderColor = c.withAlphaComponent(0.95).cgColor + box.borderWidth = 1.2 + box.cornerRadius = 2 + root.addSublayer(box) + } + } + win.contentView = view + win.orderFrontRegardless() + self.registerOverlay(win) + self.ocrOverlayWindow = win + DispatchQueue.main.asyncAfter(deadline: .now() + 7.0) { + guard self.ocrOverlayWindow === win else { return } + NSAnimationContext.runAnimationGroup({ ctx in + ctx.duration = 0.4 + win.animator().alphaValue = 0.0 + }, completionHandler: { + self.unregisterOverlay(win); win.orderOut(nil); self.ocrOverlayWindow = nil + }) + } + } + } + // MARK: - capture (in-process; no screencapture subprocess → no launchd throttle) private func captureDisplay() -> CGImage? { @@ -57,7 +156,18 @@ final class FrameCapture { guard let content = try? await SCShareableContent.excludingDesktopWindows( false, onScreenWindowsOnly: true), let display = content.displays.first else { return } - let filter = SCContentFilter(display: display, excludingWindows: []) + // GUARANTEE we capture UNDER everything this app draws. Belt: any + // window we own (flash, OCR overlay, badge, previews) by bundle id. + // Suspenders: the explicitly-tracked overlay window ids, in case a + // window's owning app is momentarily unresolved. A frame is always + // the machine's real content beneath our overlays — never them. + let myBundle = Bundle.main.bundleIdentifier + let exclude = content.windows.filter { w in + if w.owningApplication?.bundleIdentifier == myBundle { return true } + self.overlayLock.lock(); defer { self.overlayLock.unlock() } + return self.overlayWindowIDs.contains(Int(w.windowID)) + } + let filter = SCContentFilter(display: display, excludingWindows: exclude) let cfg = SCStreamConfiguration() cfg.width = display.width cfg.height = display.height @@ -75,6 +185,8 @@ final class FrameCapture { req.recognitionLevel = fast ? .fast : .accurate req.usesLanguageCorrection = false req.recognitionLanguages = ["en-US"] + req.minimumTextHeight = 0 // don't skip small/dense text (e.g. terminal monospace) + if #available(macOS 13.0, *) { req.revision = VNRecognizeTextRequestRevision3 } try? VNImageRequestHandler(cgImage: cg, options: [:]).perform([req]) let W = Double(cg.width), H = Double(cg.height) var out: [[String: Any]] = [] @@ -230,6 +342,7 @@ final class FrameCapture { env["meta"] = mt let scale = ((mt["screen"] as? [String: Any])?["scale"] as? CGFloat).map(Double.init) ?? 1.0 t = nowNs(); let cg = captureDisplay(); tm["capture"] = msSince(t) + if cg != nil { flashCaptureIndicator() } // subtle post-capture awareness flash // The JPEG ships as RAW BYTES in a sidecar file (frame.out.jpg), not // base64 in the JSON — base64 inflates the payload +33% and burns // encode/decode CPU. The transport length-prefixes the two. Written @@ -239,7 +352,9 @@ final class FrameCapture { if noOCR { env["ocr"] = [] } else { - t = nowNs(); env["ocr"] = ocr(cg, scale: scale, fast: fast); tm["ocr"] = msSince(t) + t = nowNs(); let boxes = ocr(cg, scale: scale, fast: fast); tm["ocr"] = msSince(t) + env["ocr"] = boxes + showOcrOverlay(boxes) } t = nowNs() let jpg = thumbJPEG(cg, maxWidth: 1568) ?? Data() -- 2.51.2