diff --git a/slab/bin/analysis-layer.mjs b/slab/bin/analysis-layer.mjs --- a/slab/bin/analysis-layer.mjs +++ b/slab/bin/analysis-layer.mjs @@ -115,7 +115,7 @@ // persistent #__ao_scan layer of the analysis overlay so a watcher SEES what // the agent is observing. Pixel-accurate (browser viewport space, unlike // whole-screen Vision OCR). `ttl` ms auto-clears the scan (0 = persist). // Returns {text, targets} counts. -export function scanExpr(ttl = 2600) { +export function scanExpr(ttl = 7000) { return `(()=>{ const NS="http://www.w3.org/2000/svg"; let s=document.getElementById("__analysis_overlay"); diff --git a/slab/bin/puppet.mjs b/slab/bin/puppet.mjs --- a/slab/bin/puppet.mjs +++ b/slab/bin/puppet.mjs @@ -716,7 +716,7 @@ // the flag); turning off clears any live page-side overlay. // scan: draw the observation overlay (text + interactive boxes) on the // page so a watcher sees what the agent is reading. Returns {text,targets}. case "scan": - return one(machine).eval(scanExpr(args.ttl ?? 2600), args.target); + return one(machine).eval(scanExpr(args.ttl ?? 7000), args.target); case "analysis": { const want = args.on !== false && args.on !== "off"; const names = !machine || machine === "all" ? [...machines.keys()] : [machine]; diff --git a/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift b/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift --- a/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift +++ b/slab/menubar-swift/Sources/SlabMenubar/FrameCapture.swift @@ -26,6 +26,19 @@ private let queue = DispatchQueue(label: "computer.slab.frame", qos: .userInitiated) private var timer: DispatchSourceTimer? private let fm = FileManager.default + // Transient overlay windows we draw (capture flash, OCR boxes). We exclude + // them from the screen capture by windowID so they never appear in a frame + // — that's the "doesn't interfere" guarantee. (The badge etc. still show.) + private let overlayLock = NSLock() + private var overlayWindowIDs = Set() + private var ocrOverlayWindow: NSWindow? + private func registerOverlay(_ w: NSWindow) { + overlayLock.lock(); overlayWindowIDs.insert(w.windowNumber); overlayLock.unlock() + } + private func unregisterOverlay(_ w: NSWindow) { + overlayLock.lock(); overlayWindowIDs.remove(w.windowNumber); overlayLock.unlock() + } + func start() { let dir = (Paths.frameReq as NSString).deletingLastPathComponent try? fm.createDirectory(atPath: dir, withIntermediateDirectories: true) @@ -46,6 +59,92 @@ produce(noOCR: mode.contains("noocr"), fast: mode.contains("fast")) fm.createFile(atPath: Paths.frameDone, contents: nil) } + // A subtle whole-display flash, fired AFTER the pixels are grabbed so it + // never lands in the capture — just end-user awareness that a frame was + // snapped. Runs on the main thread, click-through, brief and low-alpha; the + // capture/OCR pipeline keeps going on its own queue meanwhile. + private func flashCaptureIndicator() { + DispatchQueue.main.async { + for screen in NSScreen.screens { + let win = NSWindow(contentRect: screen.frame, styleMask: .borderless, + backing: .buffered, defer: false) + win.isOpaque = false + win.backgroundColor = .clear + win.level = .screenSaver + win.ignoresMouseEvents = true + win.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle, .fullScreenAuxiliary] + let view = NSView(frame: NSRect(origin: .zero, size: screen.frame.size)) + view.wantsLayer = true + view.layer?.backgroundColor = NSColor.white.cgColor + win.contentView = view + win.alphaValue = 0.0 + win.orderFrontRegardless() + self.registerOverlay(win) + NSAnimationContext.runAnimationGroup({ ctx in + ctx.duration = 0.06 + win.animator().alphaValue = 0.22 + }, completionHandler: { + NSAnimationContext.runAnimationGroup({ ctx in + ctx.duration = 0.34 + win.animator().alphaValue = 0.0 + }, completionHandler: { self.unregisterOverlay(win); win.orderOut(nil) }) + }) + } + } + } + + // Draw the whole-screen OCR boxes as a brief screen-wide overlay, so a + // watcher sees what was read across the ENTIRE display — not just inside a + // browser window (puppet's page-side scan). Click-through, excluded from + // captures by windowID, holds ~7s then fades. Boxes are points/top-left + // (from ocr()); CALayer is bottom-left, so Y flips against screen height. + private func showOcrOverlay(_ boxes: [[String: Any]]) { + guard !boxes.isEmpty else { return } + DispatchQueue.main.async { + guard let screen = NSScreen.main else { return } + if let old = self.ocrOverlayWindow { self.unregisterOverlay(old); old.orderOut(nil) } + let H = screen.frame.height + let win = NSWindow(contentRect: screen.frame, styleMask: .borderless, + backing: .buffered, defer: false) + win.isOpaque = false + win.backgroundColor = .clear + win.level = .screenSaver + win.ignoresMouseEvents = true + win.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle, .fullScreenAuxiliary] + let view = NSView(frame: NSRect(origin: .zero, size: screen.frame.size)) + view.wantsLayer = true + if let root = view.layer { + for b in boxes { + guard let r = b["r"] as? [Int], r.count == 4 else { continue } + let box = CALayer() + box.frame = CGRect(x: CGFloat(r[0]), y: H - CGFloat(r[1]) - CGFloat(r[3]), + width: CGFloat(r[2]), height: CGFloat(r[3])) + // A random hue per box, semi-transparent fill — so the whole + // read is vivid and every box stands out against the others. + let c = NSColor(hue: .random(in: 0...1), saturation: 0.8, brightness: 1.0, alpha: 1.0) + box.backgroundColor = c.withAlphaComponent(0.28).cgColor + box.borderColor = c.withAlphaComponent(0.95).cgColor + box.borderWidth = 1.2 + box.cornerRadius = 2 + root.addSublayer(box) + } + } + win.contentView = view + win.orderFrontRegardless() + self.registerOverlay(win) + self.ocrOverlayWindow = win + DispatchQueue.main.asyncAfter(deadline: .now() + 7.0) { + guard self.ocrOverlayWindow === win else { return } + NSAnimationContext.runAnimationGroup({ ctx in + ctx.duration = 0.4 + win.animator().alphaValue = 0.0 + }, completionHandler: { + self.unregisterOverlay(win); win.orderOut(nil); self.ocrOverlayWindow = nil + }) + } + } + } + // MARK: - capture (in-process; no screencapture subprocess → no launchd throttle) private func captureDisplay() -> CGImage? { @@ -57,7 +156,18 @@ defer { sem.signal() } guard let content = try? await SCShareableContent.excludingDesktopWindows( false, onScreenWindowsOnly: true), let display = content.displays.first else { return } - let filter = SCContentFilter(display: display, excludingWindows: []) + // GUARANTEE we capture UNDER everything this app draws. Belt: any + // window we own (flash, OCR overlay, badge, previews) by bundle id. + // Suspenders: the explicitly-tracked overlay window ids, in case a + // window's owning app is momentarily unresolved. A frame is always + // the machine's real content beneath our overlays — never them. + let myBundle = Bundle.main.bundleIdentifier + let exclude = content.windows.filter { w in + if w.owningApplication?.bundleIdentifier == myBundle { return true } + self.overlayLock.lock(); defer { self.overlayLock.unlock() } + return self.overlayWindowIDs.contains(Int(w.windowID)) + } + let filter = SCContentFilter(display: display, excludingWindows: exclude) let cfg = SCStreamConfiguration() cfg.width = display.width cfg.height = display.height @@ -75,6 +185,8 @@ let req = VNRecognizeTextRequest() req.recognitionLevel = fast ? .fast : .accurate req.usesLanguageCorrection = false req.recognitionLanguages = ["en-US"] + req.minimumTextHeight = 0 // don't skip small/dense text (e.g. terminal monospace) + if #available(macOS 13.0, *) { req.revision = VNRecognizeTextRequestRevision3 } try? VNImageRequestHandler(cgImage: cg, options: [:]).perform([req]) let W = Double(cg.width), H = Double(cg.height) var out: [[String: Any]] = [] @@ -230,6 +342,7 @@ var t = nowNs(); let mt = meta(); tm["meta"] = msSince(t) env["meta"] = mt let scale = ((mt["screen"] as? [String: Any])?["scale"] as? CGFloat).map(Double.init) ?? 1.0 t = nowNs(); let cg = captureDisplay(); tm["capture"] = msSince(t) + if cg != nil { flashCaptureIndicator() } // subtle post-capture awareness flash // The JPEG ships as RAW BYTES in a sidecar file (frame.out.jpg), not // base64 in the JSON — base64 inflates the payload +33% and burns // encode/decode CPU. The transport length-prefixes the two. Written @@ -239,7 +352,9 @@ if let cg = cg { if noOCR { env["ocr"] = [] } else { - t = nowNs(); env["ocr"] = ocr(cg, scale: scale, fast: fast); tm["ocr"] = msSince(t) + t = nowNs(); let boxes = ocr(cg, scale: scale, fast: fast); tm["ocr"] = msSince(t) + env["ocr"] = boxes + showOcrOverlay(boxes) } t = nowNs() let jpg = thumbJPEG(cg, maxWidth: 1568) ?? Data()