From 0c02d57d166dfd38360c13516828ba208351c3b0 Mon Sep 17 00:00:00 2001 From: prompt.ac/@jeffrey Date: Tue, 28 Jul 2026 21:32:43 +0000 Subject: [PATCH] slab: guard and deliver rich Messages media --- slab/bin/dm-mcp.mjs | 147 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------- slab/bin/imsg.mjs | 387 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------ slab/bin/visual-evidence.mjs | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ slab/lib/imessage-send-routing.mjs | 34 ++++++++++++++++++++++++++++++++++ slab/test/imessage-send-routing.test.mjs | 35 +++++++++++++++++++++++++++++++++++ 5 file(s) changed, 769 insertion(s)(+), 32 deletion(s)(-) diff --git a/slab/bin/dm-mcp.mjs b/slab/bin/dm-mcp.mjs --- a/slab/bin/dm-mcp.mjs +++ b/slab/bin/dm-mcp.mjs @@ -29,16 +29,18 @@ // // SAFETY: dm_send NEVER sends on the first call. It resolves + echoes the target // and the message and asks for `confirm: true` — outward, hard-to-unsend, often // to NDA contacts. This is the send guardrail baked in, not bolted on. +import { createHash } from "node:crypto"; import { execFile } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync, statSync } from "node:fs"; import { homedir } from "node:os"; -import { dirname, join } from "node:path"; +import { dirname, isAbsolute, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { httpPort, serveHttp, serveStdio } from "../../toolchain/mcp/http-front.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); const SIGNAL = join(HERE, "signal.mjs"); const IMSG = join(HERE, "imsg.mjs"); +const VISUAL_EVIDENCE = join(HERE, "visual-evidence.mjs"); const PUPPET_JSON = join(homedir(), ".config", "slab", "puppet.json"); // ── shell helpers ─────────────────────────────────────────────────────────── @@ -234,21 +236,51 @@ } // Send — two-step by design. First call previews the resolved target + message; // only `confirm: true` actually sends. -async function toolSend({ channel, to, text: body, confirm, machine } = {}) { +function inspectOutgoingFiles(values = []) { + return values.map((value) => { + const path = isAbsolute(String(value)) ? String(value) : resolve(String(value)); + if (!existsSync(path)) throw new Error(`attachment does not exist: ${path}`); + const info = statSync(path); + if (!info.isFile()) throw new Error(`attachment is not a file: ${path}`); + if (info.size > 20 * 1024 * 1024) throw new Error(`attachment exceeds the 20 MB backend limit: ${path}`); + return { + path, + bytes: info.size, + sha256: createHash("sha256").update(readFileSync(path)).digest("hex"), + }; + }); +} + +async function toolSend({ + channel, to, text: body = "", image, attachments = [], linkPreview, + visibleTitle, confirm, machine, +} = {}) { const ch = (channel || "").toLowerCase(); - if (!body) throw new Error("`text` (the message) is required"); + const requestedFiles = [image, ...(Array.isArray(attachments) ? attachments : [attachments])].filter(Boolean); + const files = inspectOutgoingFiles(requestedFiles); + const message = String(body || ""); + if (!message && !files.length && !linkPreview) { + throw new Error("provide `text`, `image`/`attachments`, or `linkPreview`"); + } if (ch === "signal") { + if (linkPreview) throw new Error("`linkPreview` is Messages-only; put the URL in `text` for Signal"); const rcpt = await resolveSignalRecipient(to, machine); if (!confirm) { return text( `PREVIEW — not sent. Re-call with confirm:true to send.\n` + `channel: Signal machine: ${isLocal(machine) ? "local" : machine}\n` + - `to: ${rcpt.label} [${rcpt.how}]\n--- message ---\n${body}`, + `to: ${rcpt.label} [${rcpt.how}]\n` + + `${files.map((file) => `attachment: ${file.path} (${file.bytes} bytes, sha256 ${file.sha256})`).join("\n")}` + + `${files.length && message ? "\n" : ""}${message ? `--- message ---\n${message}` : ""}`, ); } const acct = await signalAccount(machine); - const { stdout } = await runSignalCli(["-a", acct, "send", "-m", body, rcpt.id], machine, { timeoutMs: 60000 }); + const sendArgs = ["-a", acct, "send"]; + if (message) sendArgs.push("-m", message); + for (const file of files) sendArgs.push("-a", file.path); + sendArgs.push(rcpt.id); + const { stdout } = await runSignalCli(sendArgs, machine, { timeoutMs: 60000 }); // Best effort: if this is the conversation Slab watches, replying is also // an acknowledgement. Other conversations keep independent cursors. await runBridge(SIGNAL, ["ack", "--to", String(to)], machine).catch(() => {}); @@ -265,30 +297,72 @@ const toArgs = ["--to", String(to)]; const { stdout: resolved } = await runBridge(IMSG, ["resolve", ...toArgs], machine, { timeoutMs: 30000 }); const rcpt = JSON.parse(resolved); if (!confirm) { + const previewLines = [ + "PREVIEW — not sent. Re-call with confirm:true to send.", + `channel: Messages (iMessage/RCS/SMS) machine: ${isLocal(machine) ? "local" : machine}`, + `to: ${rcpt.displayName} [requested: "${to}"]`, + `visible recipient guard: ${visibleTitle || rcpt.displayName}`, + ...files.map((file) => `attachment: ${file.path} (${file.bytes} bytes, sha256 ${file.sha256})`), + ...(linkPreview ? [`rich link preview: ${linkPreview}`] : []), + ...(message ? ["--- message ---", message] : []), + ]; return text( - `PREVIEW — not sent. Re-call with confirm:true to send.\n` + - `channel: Messages (iMessage/RCS/SMS) machine: ${isLocal(machine) ? "local" : machine}\n` + - `to: ${rcpt.displayName} [requested: "${to}"]\n` + - `--- message ---\n${body}`, + previewLines.join("\n"), + ); + } + const guardArgs = ["--expected-title", String(visibleTitle || rcpt.displayName)]; + const receipts = []; + for (const file of files) { + const { stdout } = await runBridge( + IMSG, + ["send", "--media", file.path, ...guardArgs, ...toArgs], + machine, + { timeoutMs: 60000 }, + ); + receipts.push({ kind: "attachment", ...JSON.parse(stdout.trim()) }); + } + if (linkPreview) { + const { stdout } = await runBridge( + IMSG, + ["send", "--link-preview", String(linkPreview), ...guardArgs, ...toArgs], + machine, + { timeoutMs: 60000 }, ); + receipts.push({ kind: "link-preview", ...JSON.parse(stdout.trim()) }); + } + // If text is exactly the rich-preview URL, the URL balloon already carries + // it; do not emit a duplicate plain bubble. + if (message && message.trim() !== String(linkPreview || "").trim()) { + const { stdout } = await runBridge(IMSG, ["send", message, ...toArgs], machine, { timeoutMs: 30000 }); + receipts.push({ kind: "text", ...JSON.parse(stdout.trim()) }); } - const { stdout } = await runBridge(IMSG, ["send", body, ...toArgs], machine, { timeoutMs: 30000 }); - let delivery = null; - try { delivery = JSON.parse(stdout.trim()); } catch {} - if (delivery?.status) { - if (delivery.status === "failed") { - throw new Error(`Messages rejected the send to ${rcpt.displayName}`); - } - const verb = delivery.status === "delivered" ? "delivered" : "sent"; - const service = delivery.service || "Messages"; - return text(`✅ ${verb} to ${rcpt.displayName} via ${service}`); + for (const receipt of receipts) { + if (receipt.status === "failed") throw new Error(`Messages rejected ${receipt.kind} to ${rcpt.displayName}`); } - return text(`✅ Messages send completed for ${rcpt.displayName}${stdout.trim() ? `: ${stdout.trim()}` : ""}`); + return text(JSON.stringify({ ok: true, to: rcpt.displayName, receipts }, null, 2)); } throw new Error(`unknown channel "${channel}" (use "signal" or "imessage")`); } +async function toolVisualCapture({ kind = "url", url, selector, label, machine, region, out } = {}) { + const mode = String(kind).toLowerCase(); + const args = [mode]; + if (mode === "url") { + if (!url || !selector) throw new Error("URL evidence requires `url` and a CSS `selector`"); + args.push("--url", String(url), "--selector", String(selector)); + if (label) args.push("--label", String(label)); + } else if (mode === "frame") { + args.push("--machine", String(machine || "local")); + if (region) args.push("--region", String(region)); + } else { + throw new Error('`kind` must be "url" or "frame"'); + } + if (out) args.push("--out", String(out)); + const { stdout } = await run(process.execPath, [VISUAL_EVIDENCE, ...args], { timeoutMs: 90000 }); + return text(stdout.trim()); +} + const TAPBACK_LABELS = new Map([ ["heart", "heart"], ["love", "heart"], ["thumbs-up", "thumbs up"], ["thumbsup", "thumbs up"], ["like", "thumbs up"], @@ -404,17 +478,37 @@ }, }, { name: "dm_send", - description: "Send a DM. TWO-STEP AND SAFE: the first call PREVIEWS the resolved recipient + message and does NOT send; call again with confirm:true to actually send. `to` is required. Signal accepts an ACI, +E164, or name. The imessage channel uses Messages.app, selects the conversation's current iMessage/RCS/SMS transport, verifies sent/delivered state, and accepts a named contact from imsg.json or a raw handle.", + description: "Send text, image attachments, or a real rich link preview. TWO-STEP AND SAFE: the first call PREVIEWS the resolved recipient and exact payload and does NOT send; call again with confirm:true. Messages media sends bind the visible conversation title, preserve existing drafts, capture a pre-send evidence frame, and verify the recipient-scoped database receipt. Use linkPreview (not plain text) when an Apple URL preview card is required.", inputSchema: { type: "object", properties: { channel: { type: "string", enum: ["signal", "imessage"], description: "Which channel." }, to: { type: "string", description: "Required recipient. Signal: ACI / +number / name. iMessage: named imsg.json contact or raw +number/email." }, - text: { type: "string", description: "The message body (multi-line ok)." }, + text: { type: "string", description: "Optional message body (multi-line ok)." }, + image: { type: "string", description: "Optional absolute/local image path; alias for one attachments entry." }, + attachments: { type: "array", items: { type: "string" }, description: "Optional file paths. Messages currently accepts supported images; Signal passes files to signal-cli." }, + linkPreview: { type: "string", description: "Messages only: send this http(s) URL as an Apple rich URL balloon with metadata." }, + visibleTitle: { type: "string", description: "Optional exact/contained Messages conversation title for the recipient guard; defaults to the resolved contact display name." }, confirm: { type: "boolean", description: "Must be true to actually send. Omit/false = preview only." }, machine: { type: "string", description: "Machine (default local; signal-cli sends route over ssh for remote)." }, }, - required: ["channel", "text"], + required: ["channel", "to"], + }, + }, + { + name: "dm_visual_capture", + description: "Create send-ready visual evidence without raw browser control. URL mode loads one http(s) page, applies a Captutor spotlight to one ordinary CSS selector, labels it, and crops until the evidence dominates. Frame mode captures the focused window from a named fleet Mac and can crop an explicit x,y,width,height region. Returns path, hash, bounds, and source metadata; it never sends.", + inputSchema: { + type: "object", + properties: { + kind: { type: "string", enum: ["url", "frame"], description: "Capture source (default url)." }, + url: { type: "string", description: "URL mode: http(s) page to capture." }, + selector: { type: "string", description: "URL mode: ordinary CSS selector to spotlight; js=/text= selectors are refused." }, + label: { type: "string", description: "URL mode: short visible evidence label (max 80 characters)." }, + machine: { type: "string", description: "Frame mode: fleet machine (default local)." }, + region: { type: "string", description: "Frame mode: optional x,y,width,height crop in image pixels." }, + out: { type: "string", description: "Optional output path; defaults under ~/.local/share/slab/visual-evidence/." }, + }, }, }, { @@ -443,6 +537,7 @@ case "dm_groups": return toolGroups(args || {}); case "dm_attachments": return toolAttachments(args || {}); case "dm_contacts": return toolContacts(args || {}); case "dm_send": return toolSend(args || {}); + case "dm_visual_capture": return toolVisualCapture(args || {}); case "dm_react": return toolReact(args || {}); default: throw new Error(`Unknown tool: ${name}`); } @@ -458,7 +553,7 @@ jsonrpc: "2.0", id, result: { protocolVersion: "2024-11-05", capabilities: { tools: {} }, - serverInfo: { name: "dm-mcp", version: "1.1.0" }, + serverInfo: { name: "dm-mcp", version: "1.2.0" }, }, }; case "initialized": @@ -485,4 +580,4 @@ } const port = httpPort(process.argv, 7771); if (port) serveHttp({ handleMessage, port, banner: "✉️ dm-mcp shared daemon" }); -else serveStdio({ handleMessage, banner: "✉️ dm-mcp server started (dm_inbox, dm_chats, dm_read, dm_search, dm_groups, dm_attachments, dm_contacts, dm_send, dm_react)" }); +else serveStdio({ handleMessage, banner: "✉️ dm-mcp server started (dm_inbox, dm_chats, dm_read, dm_search, dm_groups, dm_attachments, dm_contacts, dm_send, dm_visual_capture, dm_react)" }); diff --git a/slab/bin/imsg.mjs b/slab/bin/imsg.mjs --- a/slab/bin/imsg.mjs +++ b/slab/bin/imsg.mjs @@ -33,16 +33,23 @@ import { chmodSync, existsSync, mkdirSync, + mkdtempSync, readFileSync, rmSync, + statSync, writeFileSync, } from "node:fs"; -import { homedir } from "node:os"; -import { join, dirname } from "node:path"; +import { homedir, tmpdir } from "node:os"; +import { join, dirname, extname, resolve } from "node:path"; +import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; import { formatRichText } from "../lib/imessage-rich-text.mjs"; import { chooseMessagesRoute, + classifyMessagesAttachment, classifyMessagesDelivery, + conversationTitleMatches, + isMessagesComposerEmpty, shouldRetryViaSms, } from "../lib/imessage-send-routing.mjs"; @@ -59,6 +66,7 @@ const INDEX_PATH = join(STATE_DIR, "index.sqlite"); const CHAT_DB = join(HOME, "Library", "Messages", "chat.db"); const SQLITE3 = "/usr/bin/sqlite3"; const DEFAULT_INDEX_DAYS = 730; +const FRAME = join(dirname(fileURLToPath(import.meta.url)), "frame.mjs"); // ─── config ────────────────────────────────────────────────────────────── @@ -667,6 +675,327 @@ } return result; } +const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".heic", ".heif", ".webp"]); + +function inspectImage(path) { + const absolute = resolve(String(path || "")); + if (!existsSync(absolute)) throw new Error(`image not found: ${absolute}`); + const stat = statSync(absolute); + if (!stat.isFile()) throw new Error(`image is not a file: ${absolute}`); + const extension = extname(absolute).toLowerCase(); + if (!IMAGE_EXTENSIONS.has(extension)) { + throw new Error(`unsupported Messages image type "${extension || "(none)"}" — use jpg, png, heic, or webp`); + } + if (stat.size <= 0) throw new Error("image is empty"); + if (stat.size > 20 * 1024 * 1024) throw new Error("image exceeds the 20 MB guarded send limit"); + return { + path: absolute, + bytes: stat.size, + sha256: createHash("sha256").update(readFileSync(absolute)).digest("hex"), + }; +} + +function normalizeImageForClipboard(info) { + const dir = mkdtempSync(join(tmpdir(), "slab-imsg-media-")); + const output = join(dir, "image.jpg"); + const converted = spawnSync( + "/usr/bin/sips", + ["-s", "format", "jpeg", "-s", "formatOptions", "88", info.path, "--out", output], + { encoding: "utf8" }, + ); + if (converted.status !== 0 || !existsSync(output)) { + rmSync(dir, { recursive: true, force: true }); + throw new Error((converted.stderr || "could not normalize image for Messages").trim()); + } + return { dir, output }; +} + +function outgoingMediaAfter(handles, baseline) { + const ids = (handles || []).map(sqlString).join(","); + return sqlite( + `SELECT m.ROWID AS rowid, m.service AS service, m.error AS error, + m.is_sent AS is_sent, m.is_delivered AS is_delivered, + a.ROWID AS attachment_rowid, a.transfer_state AS transfer_state, + a.mime_type AS mime_type, a.total_bytes AS total_bytes + FROM message m + JOIN handle h ON h.ROWID=m.handle_id + JOIN message_attachment_join maj ON maj.message_id=m.ROWID + JOIN attachment a ON a.ROWID=maj.attachment_id + WHERE h.id IN (${ids}) AND m.is_from_me=1 + AND m.ROWID > ${Number(baseline) || 0} + ORDER BY m.ROWID DESC, a.ROWID DESC LIMIT 1;`, + )[0] || null; +} + +function localFrameMachine() { + if (process.env.SLAB_FRAME_MACHINE) return process.env.SLAB_FRAME_MACHINE; + const result = spawnSync("/usr/sbin/scutil", ["--get", "LocalHostName"], { encoding: "utf8" }); + return (result.stdout || "").trim() || "local"; +} + +function visibleMessagesState(outPath = null) { + const args = [FRAME, localFrameMachine(), "--no-ocr", "--quiet-overlay", "--json"]; + if (outPath) args.push("--out", outPath); + const framed = spawnSync( + process.execPath, + args, + { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }, + ); + if (framed.status !== 0) { + throw new Error((framed.stderr || "could not verify the visible Messages conversation").trim()); + } + const snapshot = JSON.parse((framed.stdout || "{}").trim()); + if (snapshot?.meta?.frontmost?.bundle !== "com.apple.MobileSMS") { + return { title: "", composer: "" }; + } + const fields = (snapshot?.ax?.elements || []) + .filter((element) => element.role === "AXTextField") + .sort((a, b) => (Number(b.cy) || 0) - (Number(a.cy) || 0)); + return { + title: String((snapshot?.meta?.windows || []).find((window) => window.app === "Messages")?.title || "").trim(), + composer: String(fields[0]?.title || "").trim(), + }; +} + +async function selectVerifiedConversation(handle, expectedTitle, timeoutMs = 10000) { + const opened = spawnSync( + "/usr/bin/open", + [`im:${encodeURIComponent(String(handle))}`], + { encoding: "utf8" }, + ); + if (opened.status !== 0) { + throw new Error((opened.stderr || "could not open Messages conversation").trim()); + } + const deadline = Date.now() + timeoutMs; + let visible = { title: "", composer: "" }; + while (Date.now() < deadline) { + await wait(450); + visible = visibleMessagesState(); + if (!conversationTitleMatches(visible.title, expectedTitle)) continue; + if (!isMessagesComposerEmpty(visible.composer)) { + throw new Error( + `recipient guard preserved an existing draft in "${visible.title}"; clear or send it before attaching media`, + ); + } + return visible.title; + } + throw new Error( + `recipient guard refused image paste: requested "${expectedTitle}" but Messages showed "${visible.title || "no verified conversation"}"`, + ); +} + +function pasteImageIntoConversation(imagePath) { + const script = ` +on run argv + set imageFile to POSIX file (item 1 of argv) + set savedClipboard to the clipboard as record + try + set the clipboard to (read imageFile as JPEG picture) + tell application "Messages" to activate + delay 0.2 + tell application "System Events" + tell process "Messages" + keystroke "v" using command down + end tell + end tell + delay 0.4 + on error errorMessage number errorNumber + set the clipboard to savedClipboard + error errorMessage number errorNumber + end try + set the clipboard to savedClipboard +end run`; + const pasted = spawnSync("/usr/bin/osascript", ["-e", script, imagePath], { encoding: "utf8" }); + if (pasted.status !== 0) throw new Error((pasted.stderr || "Messages image paste failed").trim()); +} + +function pasteTextIntoConversation(value) { + const script = ` +on run argv + set savedClipboard to the clipboard as record + try + set the clipboard to item 1 of argv + tell application "Messages" to activate + delay 0.2 + tell application "System Events" to tell process "Messages" to keystroke "v" using command down + delay 0.4 + on error errorMessage number errorNumber + set the clipboard to savedClipboard + error errorMessage number errorNumber + end try + set the clipboard to savedClipboard +end run`; + const pasted = spawnSync("/usr/bin/osascript", ["-e", script, value], { encoding: "utf8" }); + if (pasted.status !== 0) throw new Error((pasted.stderr || "Messages text paste failed").trim()); +} + +function pressMessagesReturn() { + const pressed = spawnSync( + "/usr/bin/osascript", + ["-e", 'tell application "Messages" to activate', "-e", "delay 0.2", "-e", 'tell application "System Events" to tell process "Messages" to key code 36'], + { encoding: "utf8" }, + ); + if (pressed.status !== 0) throw new Error((pressed.stderr || "Messages send key failed").trim()); +} + +function clearOwnedComposer(expectedTitle, ownedText = "") { + const state = visibleMessagesState(); + const ownsDraft = state.composer.includes("\uFFFC") || state.composer.trim() === String(ownedText).trim(); + if (!conversationTitleMatches(state.title, expectedTitle) || !ownsDraft) return false; + const cleared = spawnSync( + "/usr/bin/osascript", + [ + "-e", 'tell application "Messages" to activate', + "-e", "delay 0.2", + "-e", 'tell application "System Events" to tell process "Messages" to keystroke "a" using command down', + "-e", 'tell application "System Events" to tell process "Messages" to key code 51', + ], + { encoding: "utf8" }, + ); + return cleared.status === 0; +} + +async function waitForRichDraft(expectedTitle, timeoutMs = 12000, settleMs = 2400) { + const deadline = Date.now() + timeoutMs; + let state = { title: "", composer: "" }; + while (Date.now() < deadline) { + await wait(500); + state = visibleMessagesState(); + if (!conversationTitleMatches(state.title, expectedTitle)) { + throw new Error(`recipient changed while building preview: expected "${expectedTitle}", saw "${state.title || "none"}"`); + } + if (state.composer.includes("\uFFFC")) { + // The attachment object appears before Messages finishes fetching and + // painting link metadata. Hold the route, then verify it a second time + // so the evidence frame and Return key cannot race a loading card. + await wait(settleMs); + const settled = visibleMessagesState(); + if (!conversationTitleMatches(settled.title, expectedTitle)) { + throw new Error( + `recipient changed while preview settled: expected "${expectedTitle}", saw "${settled.title || "none"}"`, + ); + } + if (!settled.composer.includes("\uFFFC")) { + throw new Error("Messages preview disappeared before it became ready"); + } + return settled; + } + } + throw new Error(`Messages did not build a rich preview in ${timeoutMs / 1000}s`); +} + +function captureSendEvidence(prefix, expectedTitle) { + const dir = join(STATE_DIR, "evidence"); + mkdirSync(dir, { recursive: true }); + const path = join(dir, `${prefix}-${Date.now()}.jpg`); + const state = visibleMessagesState(path); + if (!conversationTitleMatches(state.title, expectedTitle) || !state.composer.includes("\uFFFC")) { + rmSync(path, { force: true }); + throw new Error("preview changed before visual evidence could be captured"); + } + return path; +} + +function outgoingLinkAfter(handles, baseline) { + const ids = (handles || []).map(sqlString).join(","); + return sqlite( + `SELECT m.ROWID AS rowid, m.service AS service, m.error AS error, + m.is_sent AS is_sent, m.is_delivered AS is_delivered, + m.balloon_bundle_id AS balloon_bundle_id, + length(m.payload_data) AS payload_bytes + FROM message m JOIN handle h ON h.ROWID=m.handle_id + WHERE h.id IN (${ids}) AND m.is_from_me=1 + AND m.ROWID > ${Number(baseline) || 0} + AND m.balloon_bundle_id='com.apple.messages.URLBalloonProvider' + ORDER BY m.ROWID DESC LIMIT 1;`, + )[0] || null; +} + +async function sendLinkPreview(handles, url, expectedTitle, timeoutMs = 20000) { + let parsed; + try { parsed = new URL(String(url)); } catch { throw new Error("link preview requires a valid URL"); } + if (!new Set(["http:", "https:"]).has(parsed.protocol)) throw new Error("link preview URL must use http or https"); + const route = chooseMessagesRoute(handles, latestSuccessfulRoute(handles)); + const baseline = latestRecipientRowid(handles); + let submitted = false; + try { + const visibleConversation = await selectVerifiedConversation(route.handle, expectedTitle); + pasteTextIntoConversation(parsed.href); + await waitForRichDraft(expectedTitle); + const evidencePath = captureSendEvidence("link-preview", expectedTitle); + pressMessagesReturn(); + submitted = true; + const deadline = Date.now() + timeoutMs; + let lastRow = null; + while (Date.now() < deadline) { + lastRow = outgoingLinkAfter(handles, baseline); + const state = classifyMessagesDelivery(lastRow); + if (state.status === "failed") throw new Error(`Messages rejected rich-link row ${lastRow?.rowid || "unknown"}`); + if (state.status === "sent" || state.status === "delivered") { + return { + ...state, + rowid: Number(lastRow.rowid), + balloonBundleId: lastRow.balloon_bundle_id, + payloadBytes: Number(lastRow.payload_bytes) || 0, + evidencePath, + visibleConversation, + url: parsed.href, + }; + } + await wait(250); + } + throw new Error(`Messages rich-link send did not confirm within ${timeoutMs / 1000}s`); + } finally { + if (!submitted) clearOwnedComposer(expectedTitle, parsed.href); + } +} + +async function sendImage(handles, path, expectedTitle, timeoutMs = 20000) { + const info = inspectImage(path); + const normalized = normalizeImageForClipboard(info); + const route = chooseMessagesRoute(handles, latestSuccessfulRoute(handles)); + const baseline = latestRecipientRowid(handles); + let submitted = false; + try { + const visibleConversation = await selectVerifiedConversation(route.handle, expectedTitle); + pasteImageIntoConversation(normalized.output); + await waitForRichDraft(expectedTitle); + const evidencePath = captureSendEvidence("image", expectedTitle); + pressMessagesReturn(); + submitted = true; + const deadline = Date.now() + timeoutMs; + let lastRow = null; + while (Date.now() < deadline) { + lastRow = outgoingMediaAfter(handles, baseline); + const state = classifyMessagesAttachment(lastRow); + if (state.status === "failed") { + throw new Error( + `Messages rejected image row ${lastRow?.rowid || "unknown"} ` + + `(message error ${state.error}, transfer state ${state.transferState || 0})`, + ); + } + if (state.status === "sent" || state.status === "delivered") { + return { + ...state, + rowid: Number(lastRow.rowid), + attachmentRowid: Number(lastRow.attachment_rowid), + mimeType: lastRow.mime_type || "image/jpeg", + bytes: Number(lastRow.total_bytes) || info.bytes, + sourceBytes: info.bytes, + sourceSha256: info.sha256, + visibleConversation, + evidencePath, + }; + } + await wait(250); + } + throw new Error(`Messages image paste did not confirm within ${timeoutMs / 1000}s`); + } finally { + if (!submitted) clearOwnedComposer(expectedTitle); + rmSync(normalized.dir, { recursive: true, force: true }); + } +} + const TAPBACKS = new Map([ ["heart", { key: "1", label: "heart" }], ["love", { key: "1", label: "heart" }], @@ -1086,6 +1415,9 @@ } // Optional `--to ` selects a contact; default otherwise. const args = [...rest]; let toArg = null; + let mediaPath = null; + let linkPreview = null; + let expectedTitle = null; const richIndex = args.indexOf("--rich"); const rich = richIndex >= 0; if (rich) args.splice(richIndex, 1); @@ -1099,19 +1431,62 @@ } toArg = candidate; args.splice(ti, 2); } + const mi = args.indexOf("--media"); + if (mi >= 0) { + const candidate = args[mi + 1]; + if (!candidate || candidate.startsWith("--")) { + console.error("imsg send: --media requires an image path"); + process.exit(1); + } + mediaPath = candidate; + args.splice(mi, 2); + } + const li = args.indexOf("--link-preview"); + if (li >= 0) { + const candidate = args[li + 1]; + if (!candidate || candidate.startsWith("--")) { + console.error("imsg send: --link-preview requires a URL"); + process.exit(1); + } + linkPreview = candidate; + args.splice(li, 2); + } + const ei = args.indexOf("--expected-title"); + if (ei >= 0) { + const candidate = args[ei + 1]; + if (!candidate || candidate.startsWith("--")) { + console.error("imsg send: --expected-title requires the visible Messages title"); + process.exit(1); + } + expectedTitle = candidate; + args.splice(ei, 2); + } const source = args.join(" ").trim(); const body = rich ? formatRichText(source) : source; - if (!body) { - console.error("usage: imsg send [--rich] [--to ]"); + if (!body && !mediaPath && !linkPreview) { + console.error("usage: imsg send [--rich] [--media image | --link-preview URL] [text] [--to ]"); process.exit(1); } + if (mediaPath && linkPreview) throw new Error("send one guarded rich attachment at a time"); const rcpt = resolveRecipient(cfg, toArg); - const delivery = await sendMessage(rcpt.handles, body); + const guardedTitle = expectedTitle || rcpt.displayName; + const mediaDelivery = mediaPath + ? await sendImage(rcpt.handles, mediaPath, guardedTitle) + : null; + const linkDelivery = linkPreview + ? await sendLinkPreview(rcpt.handles, linkPreview, guardedTitle) + : null; + const delivery = body ? await sendMessage(rcpt.handles, body) : null; const watched = defaultContact(cfg); if (watched && rcpt.handles.some((h) => watched.handles.includes(h))) { acknowledge(cfg); } - print({ displayName: rcpt.displayName, ...delivery }); + print({ + displayName: rcpt.displayName, + ...(delivery || linkDelivery || mediaDelivery), + ...(mediaDelivery ? { media: mediaDelivery } : {}), + ...(linkDelivery ? { linkPreview: linkDelivery } : {}), + }); break; } case "react": { diff --git a/slab/bin/visual-evidence.mjs b/slab/bin/visual-evidence.mjs new file mode 100644 --- /dev/null +++ b/slab/bin/visual-evidence.mjs @@ -0,0 +1,198 @@ +#!/usr/bin/env node +// Bounded visual evidence for Loopboy/DM workflows. +// URL captures accept one ordinary CSS selector and apply Captutor's spotlight. +// Frame captures remain read-only and may be cropped to an explicit rectangle. + +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { dirname, extname, isAbsolute, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import puppeteer from "puppeteer"; +import { spotlight } from "../../captutor/lib/effects.mjs"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const FRAME = join(HERE, "frame.mjs"); +const STATE_DIR = join(homedir(), ".local", "share", "slab", "visual-evidence"); + +function option(args, name, fallback = null) { + const index = args.indexOf(name); + return index >= 0 ? args[index + 1] : fallback; +} + +function numberOption(args, name, fallback) { + const value = Number(option(args, name, fallback)); + if (!Number.isFinite(value) || value <= 0) throw new Error(`${name} must be a positive number`); + return value; +} + +function outputPath(requested, prefix, extension = ".jpg") { + const path = requested + ? (isAbsolute(requested) ? requested : resolve(requested)) + : join(STATE_DIR, `${prefix}-${Date.now()}${extension}`); + mkdirSync(dirname(path), { recursive: true }); + return path; +} + +function receipt(path, extra = {}) { + const bytes = readFileSync(path); + return { + ...extra, + path, + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }; +} + +function boundedClip(rects, viewport, margin = 44) { + const valid = rects.filter((r) => r && r.width > 0 && r.height > 0); + if (!valid.length) throw new Error("capture target has no visible bounds"); + let left = Math.min(...valid.map((r) => r.x)) - margin; + let top = Math.min(...valid.map((r) => r.y)) - margin; + let right = Math.max(...valid.map((r) => r.x + r.width)) + margin; + let bottom = Math.max(...valid.map((r) => r.y + r.height)) + margin; + const minimumWidth = Math.min(640, viewport.width); + const minimumHeight = Math.min(420, viewport.height); + if (right - left < minimumWidth) { + const grow = (minimumWidth - (right - left)) / 2; + left -= grow; right += grow; + } + if (bottom - top < minimumHeight) { + const grow = (minimumHeight - (bottom - top)) / 2; + top -= grow; bottom += grow; + } + left = Math.max(0, Math.min(left, viewport.width - minimumWidth)); + top = Math.max(0, Math.min(top, viewport.height - minimumHeight)); + right = Math.min(viewport.width, Math.max(right, left + minimumWidth)); + bottom = Math.min(viewport.height, Math.max(bottom, top + minimumHeight)); + return { + x: Math.round(left), y: Math.round(top), + width: Math.round(right - left), height: Math.round(bottom - top), + }; +} + +async function captureUrl(args) { + const rawUrl = option(args, "--url"); + const selector = option(args, "--selector"); + const label = String(option(args, "--label", "Look here")).trim().slice(0, 80); + if (!rawUrl || !selector) throw new Error("url capture requires --url and --selector"); + if (/^(js|text)=/i.test(selector)) throw new Error("only an ordinary CSS selector is allowed"); + let url; + try { url = new URL(rawUrl); } catch { throw new Error("--url must be a valid URL"); } + if (!new Set(["http:", "https:"]).has(url.protocol)) throw new Error("--url must use http or https"); + const width = Math.min(1920, Math.round(numberOption(args, "--width", 1280))); + const height = Math.min(1200, Math.round(numberOption(args, "--height", 900))); + const path = outputPath(option(args, "--out"), "url", ".jpg"); + const executablePath = existsSync("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome") + ? "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" + : undefined; + const browser = await puppeteer.launch({ headless: true, executablePath }); + try { + const page = await browser.newPage(); + await page.setViewport({ width, height, deviceScaleFactor: 1 }); + await page.goto(url.href, { waitUntil: "networkidle2", timeout: 45000 }); + await page.waitForSelector(selector, { visible: true, timeout: 15000 }); + const cdp = { eval: (expression) => page.evaluate(expression) }; + await spotlight(cdp, selector, { + label, + durationMs: 0, + dim: 0.58, + padding: 14, + scrollIntoView: true, + }); + await new Promise((resolveWait) => setTimeout(resolveWait, 850)); + const bounds = await page.evaluate((targetSelector) => { + const box = (node) => { + if (!node) return null; + const r = node.getBoundingClientRect(); + return { x: r.x, y: r.y, width: r.width, height: r.height }; + }; + return { + target: box(document.querySelector(targetSelector)), + label: box(document.querySelector("#__captutor_fx")?.shadowRoot?.querySelector(".label")), + title: document.title, + }; + }, selector); + const clip = boundedClip([bounds.target, bounds.label], { width, height }); + // Chrome's clipped screenshot path can drop fixed shadow-DOM overlays + // after scrollIntoView. Capture the viewport first, then crop the pixels; + // this guarantees the Captutor ring and label survive in the artifact. + const temporary = mkdtempSync(join(tmpdir(), "visual-url-")); + try { + const viewportPath = join(temporary, "viewport.jpg"); + await page.screenshot({ path: viewportPath, type: "jpeg", quality: 91 }); + execFileSync("/usr/bin/sips", [ + "-c", String(clip.height), String(clip.width), + "--cropOffset", String(clip.y), String(clip.x), + viewportPath, "--out", path, + ], { encoding: "utf8" }); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + return receipt(path, { + kind: "url", + url: url.href, + selector, + label, + pageTitle: bounds.title, + target: bounds.target, + labelBounds: bounds.label, + clip, + }); + } finally { + await browser.close(); + } +} + +function parseRegion(value) { + if (!value) return null; + const parts = String(value).split(",").map(Number); + if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n)) || parts.some((n) => n < 0) || parts[2] <= 0 || parts[3] <= 0) { + throw new Error("--region must be x,y,width,height in image pixels"); + } + return parts.map(Math.round); +} + +function captureFrame(args) { + const machine = option(args, "--machine", "local"); + const region = parseRegion(option(args, "--region")); + const path = outputPath(option(args, "--out"), "frame", ".jpg"); + const temporary = region ? mkdtempSync(join(tmpdir(), "visual-frame-")) : null; + const rawPath = temporary ? join(temporary, `raw${extname(path) || ".jpg"}`) : path; + try { + const raw = execFileSync(process.execPath, [ + FRAME, machine, "--no-ocr", "--quiet-overlay", "--json", "--out", rawPath, + ], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }); + const frame = JSON.parse(raw.trim() || "{}"); + if (region) { + const [x, y, width, height] = region; + execFileSync("/usr/bin/sips", [ + "-c", String(height), String(width), "--cropOffset", String(y), String(x), + rawPath, "--out", path, + ], { encoding: "utf8" }); + } + return receipt(path, { + kind: "frame", + machine, + region, + frontmost: frame?.meta?.frontmost || null, + window: (frame?.meta?.windows || [])[0] || null, + }); + } finally { + if (temporary) rmSync(temporary, { recursive: true, force: true }); + } +} + +const [mode, ...args] = process.argv.slice(2); +try { + const result = mode === "url" + ? await captureUrl(args) + : mode === "frame" + ? captureFrame(args) + : (() => { throw new Error("usage: visual-evidence.mjs [options]"); })(); + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); +} catch (error) { + process.stderr.write(`${error.message || error}\n`); + process.exit(1); +} diff --git a/slab/lib/imessage-send-routing.mjs b/slab/lib/imessage-send-routing.mjs --- a/slab/lib/imessage-send-routing.mjs +++ b/slab/lib/imessage-send-routing.mjs @@ -26,6 +26,40 @@ if (Number(row.is_sent)) return { status: "sent", service, error: 0 }; return { status: "pending", service, error: 0 }; } +export function classifyMessagesAttachment(row) { + const message = classifyMessagesDelivery(row); + if (message.status === "failed") return message; + if (!row) return message; + + const transferState = Number(row.transfer_state) || 0; + if (transferState === 6) { + return { status: "failed", service: message.service, error: message.error, transferState }; + } + if (transferState !== 5) { + return { status: "pending", service: message.service, error: message.error, transferState }; + } + return { ...message, transferState }; +} + +export function conversationTitleMatches(title, expected) { + const clean = (value) => String(value || "") + .toLowerCase() + .replace(/[^a-z0-9@+]+/g, " ") + .trim(); + const actual = clean(title); + const wanted = clean(expected); + const actualWords = ` ${actual} `; + const wantedWords = ` ${wanted} `; + return actual.length >= 3 && wanted.length >= 3 && ( + actual === wanted || actualWords.includes(wantedWords) || wantedWords.includes(actualWords) + ); +} + +export function isMessagesComposerEmpty(value) { + const text = String(value ?? "").trim(); + return text === "" || text === "Message" || text === "iMessage" || text === "Text Message"; +} + export function shouldRetryViaSms(route, delivery) { return delivery?.status === "failed" && route?.appleService === "iMessage" && diff --git a/slab/test/imessage-send-routing.test.mjs b/slab/test/imessage-send-routing.test.mjs --- a/slab/test/imessage-send-routing.test.mjs +++ b/slab/test/imessage-send-routing.test.mjs @@ -2,6 +2,9 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + classifyMessagesAttachment, + conversationTitleMatches, + isMessagesComposerEmpty, chooseMessagesRoute, classifyMessagesDelivery, shouldRetryViaSms, @@ -65,6 +68,38 @@ assert.deepEqual( classifyMessagesDelivery({ service: "SMS", error: 0, is_sent: 1, is_delivered: 0 }), { status: "sent", service: "SMS", error: 0 }, ); +}); + +test("requires an attachment transfer before confirming media delivery", () => { + assert.deepEqual(classifyMessagesAttachment(null), { + status: "pending", service: null, error: 0, + }); + assert.deepEqual( + classifyMessagesAttachment({ service: "iMessage", error: 0, is_sent: 1, transfer_state: 1 }), + { status: "pending", service: "iMessage", error: 0, transferState: 1 }, + ); + assert.deepEqual( + classifyMessagesAttachment({ service: "iMessage", error: 0, is_sent: 0, transfer_state: 6 }), + { status: "failed", service: "iMessage", error: 0, transferState: 6 }, + ); + assert.deepEqual( + classifyMessagesAttachment({ service: "iMessage", error: 0, is_sent: 1, is_delivered: 1, transfer_state: 5 }), + { status: "delivered", service: "iMessage", error: 0, transferState: 5 }, + ); +}); + +test("refuses a focused Messages conversation that is not the recipient", () => { + assert.equal(conversationTitleMatches("Alex Freundlich", "Alex"), true); + assert.equal(conversationTitleMatches("Alexis", "Alex"), false); + assert.equal(conversationTitleMatches("Amy Lynn", "Alex"), false); + assert.equal(conversationTitleMatches("Maybe: jeffrey", "me@jas.life"), false); + assert.equal(conversationTitleMatches("", "Maybe: jeffrey"), false); +}); + +test("preserves an existing Messages draft", () => { + assert.equal(isMessagesComposerEmpty("Message"), true); + assert.equal(isMessagesComposerEmpty("iMessage"), true); + assert.equal(isMessagesComposerEmpty(" design"), false); }); test("retries only an explicitly failed phone-number iMessage via SMS", () => { -- tangled.sh