From 5449fc15c7bb27e891d4396169b07e95c55584cd Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Wed, 23 Sep 2026 19:54:04 -0700 Subject: [PATCH] =?UTF-8?q?aesel=20pro:=20mime=20awareness=20=E2=80=94=20t?= =?UTF-8?q?he=20file=20the=20tools=20touch=20on=20the=20Slab=20card,=20nam?= =?UTF-8?q?ed,=20playable,=20draggable;=20tenths=20on=20the=20timer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- easel/layouts/pro.json | 2 +- easel/package.json | 2 +- easel/src/claude-server.mjs | 7 +- easel/src/layout.mjs | 4 +- easel/src/media.mjs | 71 +++++++++++++++++++ easel/src/render.mjs | 14 ++-- easel/src/transcript.mjs | 2 + easel/src/tui.mjs | 25 ++++++- easel/test/media.test.mjs | 58 +++++++++++++++ .../SlabMenubar/LocalArtifactPreview.swift | 64 +++++++++++++++-- .../Sources/SlabMenubar/PromptPreview.swift | 63 ++++++++++++++-- .../tests/local-artifact-preview.sh | 17 +++++ 12 files changed, 303 insertions(+), 26 deletions(-) create mode 100644 easel/src/media.mjs create mode 100644 easel/test/media.test.mjs diff --git a/easel/layouts/pro.json b/easel/layouts/pro.json index 16dcab8ae..6306a19a3 100644 --- a/easel/layouts/pro.json +++ b/easel/layouts/pro.json @@ -1,6 +1,6 @@ { "bottom": ["bar", "gap", "status"], - "status": ["handle", "workspace", "model", "activity"], + "status": ["handle", "workspace", "media", "model", "activity"], "bar": [95, 70, 135], "prompt": "", "separator": " · " diff --git a/easel/package.json b/easel/package.json index bf44fd55f..0c49b6eee 100644 --- a/easel/package.json +++ b/easel/package.json @@ -1,6 +1,6 @@ { "name": "aesel", - "version": "0.7.39", + "version": "0.7.40", "private": true, "type": "module", "scripts": { diff --git a/easel/src/claude-server.mjs b/easel/src/claude-server.mjs index 581660d92..42fb6e516 100644 --- a/easel/src/claude-server.mjs +++ b/easel/src/claude-server.mjs @@ -513,13 +513,14 @@ export class ClaudeServer extends EventEmitter { const field = FILE_TOOLS.get(block.name); if (field) { const path = block.input?.[field]; - return { id: block.id, type: "fileChange", changes: path ? [{ path }] : [] }; + return { id: block.id, type: "fileChange", changes: path ? [{ path }] : [], input: block.input }; } if (COMMAND_TOOLS.has(block.name)) { - return { id: block.id, type: "commandExecution", command: block.input?.command || block.name }; + return { id: block.id, type: "commandExecution", command: block.input?.command || block.name, input: block.input }; } const detail = toolDetail(block.input); - return { id: block.id, type: "dynamicToolCall", tool: detail ? `${block.name} · ${detail}` : block.name }; + // The whole input rides along: the interface reads media paths out of it. + return { id: block.id, type: "dynamicToolCall", tool: detail ? `${block.name} · ${detail}` : block.name, input: block.input }; } #toolResults(message) { diff --git a/easel/src/layout.mjs b/easel/src/layout.mjs index 068d4946e..867ffda6f 100644 --- a/easel/src/layout.mjs +++ b/easel/src/layout.mjs @@ -27,7 +27,7 @@ import { promisify } from "node:util"; const run = promisify(execFile); export const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); export const BOTTOM_ROWS = ["gap", "bar", "status", "rule", "help", "header", "path"]; -export const STATUS_FACTS = ["handle", "workspace", "model", "engine", "mode", "activity", "inbox"]; +export const STATUS_FACTS = ["handle", "workspace", "media", "model", "engine", "mode", "activity", "inbox"]; // What a saved shape may say, and nothing else. Unknown keys are dropped and // unknown tokens skipped, so a typo in the file costs a row, not the frame. @@ -85,7 +85,7 @@ export class Layout extends EventEmitter { const base = normalize(readJson(this.baked) || {}); const over = normalize(readJson(this.file) || {}); this.override = over; - this.spec = { bottom: ["bar", "gap", "status"], status: ["handle", "workspace", "model", "activity"], bar: [95, 70, 135], prompt: "", separator: " · ", mouse: true, lines: false, ...base, ...over }; + this.spec = { bottom: ["bar", "gap", "status"], status: ["handle", "workspace", "media", "model", "activity"], bar: [95, 70, 135], prompt: "", separator: " · ", mouse: true, lines: false, ...base, ...over }; return this.spec; } diff --git a/easel/src/media.mjs b/easel/src/media.mjs new file mode 100644 index 000000000..3b7d66bed --- /dev/null +++ b/easel/src/media.mjs @@ -0,0 +1,71 @@ +// Mime awareness: which media file a session has its hands on. +// +// A pro session has no piece, so nothing tells the Slab card what to show. +// What it has instead is a stream of tool calls, and the paths in them say +// what is being made: the png a brush just wrote, the mp4 ffmpeg just +// finished. This reads those paths out of a tool's input and output, keeps +// the ones that are real files of a kind a card can show, and puts the +// freshest first. +import { statSync } from "node:fs"; +import { homedir } from "node:os"; +import path from "node:path"; + +export const MEDIA_TYPES = new Map([ + ["png", { kind: "picture", mime: "image/png", glyph: "🖼" }], + ["jpg", { kind: "picture", mime: "image/jpeg", glyph: "🖼" }], + ["jpeg", { kind: "picture", mime: "image/jpeg", glyph: "🖼" }], + ["webp", { kind: "picture", mime: "image/webp", glyph: "🖼" }], + ["wav", { kind: "sound", mime: "audio/wav", glyph: "🔊" }], + ["mp3", { kind: "sound", mime: "audio/mpeg", glyph: "🔊" }], + ["pdf", { kind: "paper", mime: "application/pdf", glyph: "📄" }], + ["mp4", { kind: "video", mime: "video/mp4", glyph: "🎬" }], + ["m4v", { kind: "video", mime: "video/mp4", glyph: "🎬" }], + ["mov", { kind: "video", mime: "video/quicktime", glyph: "🎬" }], + ["webm", { kind: "video", mime: "video/webm", glyph: "🎬" }], +]); + +// Paths end where shell syntax, quotes, JSON punctuation or whitespace begin. +// A colon is a separator too: `out.png:12` is a location, not a file. +const BREAKS = /[\s"'`<>|;&(),=:\[\]{}]+/; + +// Every media file named in the text that exists, freshest first. +export function mediaPaths(text, cwd, { home = homedir() } = {}) { + const found = []; + const seen = new Set(); + for (const raw of String(text || "").split(BREAKS)) { + const token = raw.replace(/[.!?]+$/, ""); + const type = MEDIA_TYPES.get(path.extname(token).slice(1).toLowerCase()); + if (!type) continue; + const expanded = token.startsWith("~/") ? path.join(home, token.slice(2)) : token; + const absolute = path.resolve(cwd, expanded); + if (seen.has(absolute)) continue; + seen.add(absolute); + let info; + try { info = statSync(absolute); } catch { continue; } + if (!info.isFile() || info.size === 0) continue; + found.push({ path: absolute, name: path.basename(absolute), ...type, mtimeMs: info.mtimeMs, size: info.size }); + } + return found.sort((a, b) => b.mtimeMs - a.mtimeMs); +} + +// The text of a tool item worth reading for paths: what it was told, what it +// touched, and the tail of what it said back. +export function itemText(item) { + if (!item) return ""; + return [ + item.command, + item.tool, + item.path, + ...(item.changes || []).map((change) => change?.path), + item.input ? JSON.stringify(item.input) : "", + typeof item.aggregatedOutput === "string" ? item.aggregatedOutput.slice(-4000) : "", + ].filter(Boolean).join("\n"); +} + +// Whether the next sighting replaces the current one: a different file, or +// the same file written again since. +export function mediaChanged(current, next) { + if (!next) return false; + if (!current) return true; + return current.path !== next.path || current.mtimeMs !== next.mtimeMs; +} diff --git a/easel/src/render.mjs b/easel/src/render.mjs index f94cb8bef..789eb9eaf 100644 --- a/easel/src/render.mjs +++ b/easel/src/render.mjs @@ -688,7 +688,7 @@ export function renderFrame(state, columns = 80, rows = 24, useColor = true) { const pro = state.profile?.name === "pro" && !(state.desktop || state.desktopProsePrompt); // The shape is data — see layout.mjs — so the rows under the transcript are // whatever the layout says, in the order it says them. - const shape = { bottom: ["bar", "gap", "status"], status: ["handle", "workspace", "model", "activity"], bar: [95, 70, 135], prompt: "", separator: " · ", ...(state.layout || {}) }; + const shape = { bottom: ["bar", "gap", "status"], status: ["handle", "workspace", "media", "model", "activity"], bar: [95, 70, 135], prompt: "", separator: " · ", ...(state.layout || {}) }; const transcriptRows = pro ? height - shape.bottom.length : height - 5; // The QR code keeps its own column on the right, so the transcript is // narrowed rather than overdrawn. A code is an image, not text: it needs its @@ -913,7 +913,8 @@ export function fishPath(path, home = process.env.HOME || "") { export function windowTitle(state) { const provider = providerLabel(state.providerSettings?.backend); const doing = state.approval ? "◉ approval" : state.busy ? "● working" : state.status === "connecting" ? "◌ connecting" : state.status === "offline" ? "○ offline" : ""; - return ["🫏 aesel", fishPath(state.workspace), provider, doing].filter(Boolean).join(" · "); + const media = state.media ? `${state.media.glyph} ${state.media.name}` : ""; + return ["🫏 aesel", fishPath(state.workspace), media, provider, doing].filter(Boolean).join(" · "); } // A drop-down: a short list standing on the status line's fact that opened @@ -1077,7 +1078,8 @@ function ruleRow(row, width) { // `28s…`, and `· quiet 20s` once nothing has arrived for a while. export function workingTimer(state, now = Date.now()) { const started = state.requestStartedAt || now; - const seconds = Math.max(0, Math.floor((now - started) / 1000)); + // Tenths, so the count is visibly running rather than visibly waiting. + const seconds = (Math.max(0, now - started) / 1000).toFixed(1); const quiet = Math.max(0, Math.floor((now - (state.lastRequestEventAt || started)) / 1000)); return `${seconds}s…${quiet >= 15 && state.status !== "approval" ? ` · quiet ${quiet}s` : ""}`; } @@ -1093,7 +1095,7 @@ export function breathingHandle(account, state) { // The status line under the bar, and where each fact on it starts, so the // frame can paint it and a click can find the model on it. export function proStatus(state, width, useColor, shape = state.layout || {}) { - const status = shape.status || ["handle", "workspace", "model", "activity"]; + const status = shape.status || ["handle", "workspace", "media", "model", "activity"]; const separator = shape.separator ?? " · "; const account = state.account || ""; const model = state.modelLabel || state.model || state.providerSettings?.model || ""; @@ -1105,6 +1107,8 @@ export function proStatus(state, width, useColor, shape = state.layout || {}) { model, engine: providerLabel(engine), mode: state.mode === "local" ? "local" : "remote", + // The media file the session has its hands on, by its own name. + media: state.media ? `${state.media.glyph} ${clipText(state.media.name, Math.max(12, Math.floor(width / 4)))}` : "", // The little guy dances on the line while the machine has the floor, and // beside him the seconds, the way Claude Code counts them. activity: state.busy ? `${mascotRow(state.mascotMs ?? 0, true)} ${workingTimer(state)}${state.toolNow ? ` · ${clipText(state.toolNow.replace(/\s+/g, " "), Math.max(12, Math.floor(width / 3)))}` : ""}` : state.selection && !state.selection.active ? "selected · Enter copies · Esc clears" : state.flash && state.flash.until > Date.now() ? state.flash.text : state.status === "connecting" ? "connecting…" : state.status === "offline" ? "offline" : state.scrollOffset ? `${state.scrollOffset} lines above · End latest` : "", @@ -1116,7 +1120,7 @@ export function proStatus(state, width, useColor, shape = state.layout || {}) { // the machine is doing is the last thing to be cut. const keep = new Set(status); const measure = () => [...keep].reduce((n, name) => (plain[name] ? n + textWidth(plain[name]) + (n ? textWidth(separator) : 0) : n), 1); - for (const name of ["engine", "workspace", "model", "handle", "mode", "inbox"]) { + for (const name of ["engine", "media", "workspace", "model", "handle", "mode", "inbox"]) { if (measure() <= width - 1) break; keep.delete(name); } diff --git a/easel/src/transcript.mjs b/easel/src/transcript.mjs index 8cad5e642..2e90e71c3 100644 --- a/easel/src/transcript.mjs +++ b/easel/src/transcript.mjs @@ -28,6 +28,8 @@ export const SUMMARY_MAX = 500; export const KINDS = [ "user", "inbox", "assistant", "tool_call", "tool_result", "approval", "notice", "turn", "engine", + // The media file the session took up: a path, a mime and a kind. + "media", ]; export const defaultRoot = () => diff --git a/easel/src/tui.mjs b/easel/src/tui.mjs index 3b41a5efe..6035cdfb0 100755 --- a/easel/src/tui.mjs +++ b/easel/src/tui.mjs @@ -58,6 +58,7 @@ import { cleanText, clipText, color, aeselInk, renderBoot, renderFrame, renderGe import { mascotNextFrameIn, mascotRowNextFrameIn } from "./mascot.mjs"; import { DEFAULT_RUNTIME, runtimeMenu } from "./runtimes.mjs"; import { SlabSession } from "./slab-session.mjs"; +import { mediaPaths, itemText, mediaChanged } from "./media.mjs"; import { Artifacts, MEDIA } from './artifacts.mjs'; import { desktopSnapshot, readDesktopSession, writeDesktopSession, restoreDesktopEngine, writeDesktopControl, readDesktopIntent } from "./desktop-session.mjs"; import { archiveThread, replaceWork } from './new-work.mjs'; @@ -897,9 +898,9 @@ function danceTick() { const next = mascotRowNextFrameIn(state.mascotMs, state.busy); if (next === null) return; redraw(); - // Pro breathes and counts on this clock, so it ticks often enough to be - // fluid; the frame diff makes each tick cheap. - danceTimer = setTimeout(danceTick, pro && state.busy ? Math.min(next, 120) : next); + // Pro breathes and counts tenths on this clock, so it ticks ten times a + // second; the frame diff makes each tick cheap. + danceTimer = setTimeout(danceTick, pro && state.busy ? Math.min(next, 100) : next); danceTimer.unref?.(); } function startDance() { @@ -1269,6 +1270,22 @@ function itemSummary(item) { return null; } +// Mime awareness. A pro session has no piece to put on the Slab card, so the +// card shows the media the tools are touching instead: the png a brush just +// wrote, the mp4 ffmpeg just finished, the filename under it. Every tool +// call is read twice, when it starts and when it ends, because the file a +// command is about to write only exists once it has run. A private session +// keeps the file to itself; the status line still names it. +function noteMedia(item) { + if (!pro) return; + const next = mediaPaths(itemText(item), cwd)[0]; + if (!mediaChanged(state.media, next)) return; + state.media = { ...next, version: (state.media?.version || 0) + 1 }; + transcript.event("media", { path: next.path, mime: next.mime, kind: next.kind }); + if (profile.private) return; + slabSession.artifact(next.kind, { path: next.path, mime: next.mime, name: next.name, version: state.media.version, artifactId: slabSession.sessionId }); +} + function restoreThread(thread) { const restored = []; for (const turn of thread?.turns || []) { @@ -1340,6 +1357,7 @@ function handleNotification({ method, params = {} }) { break; case "item/started": { observeToolActivity(state, method, params.item); + noteMedia(params.item); if (process.env.EASEL_DESKTOP && /(?:^|__)ac_frame(?:$|\s)/.test(String(params.item?.tool || ""))) process.stdout.write('\x1b]777;easel-camera:request\x07'); state.status = "tool"; if (params.item?.type === "fileChange") state.status = "writing"; @@ -1354,6 +1372,7 @@ function handleNotification({ method, params = {} }) { break; } case "item/completed": { + noteMedia(params.item); const item = params.item; observeToolActivity(state, method, item); if (item?.type === "agentMessage") { updateEntry(item.id, "assistant", item.text); transcriptCompleted.add(item.id);if (!turnAssistant.includes(item.id)) turnAssistant.push(item.id);const entry=state.entries.find(e=>e.id===item.id);if(entry){entry.activityOnly=state.busy;state.activityMessageId=entry.id;state.activityText=entry.text;} } diff --git a/easel/test/media.test.mjs b/easel/test/media.test.mjs new file mode 100644 index 000000000..6d382f9c6 --- /dev/null +++ b/easel/test/media.test.mjs @@ -0,0 +1,58 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync, utimesSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { mediaPaths, itemText, mediaChanged, MEDIA_TYPES } from "../src/media.mjs"; + +const root = mkdtempSync(path.join(tmpdir(), "easel-media-")); +const png = path.join(root, "frame.png"); +const mov = path.join(root, "reel.mov"); +const wav = path.join(root, "voice.wav"); +mkdirSync(path.join(root, "out")); +writeFileSync(png, Buffer.from([0x89, 0x50])); +writeFileSync(mov, Buffer.from([0, 0, 0, 1])); +writeFileSync(wav, Buffer.from("RIFF")); +writeFileSync(path.join(root, "empty.png"), ""); +utimesSync(png, new Date(2000), new Date(2000)); +utimesSync(mov, new Date(9000), new Date(9000)); +utimesSync(wav, new Date(5000), new Date(5000)); + +test("finds the media files a tool names, freshest first, whatever the quoting", () => { + const text = `ffmpeg -i "${png}" '${wav}' -o ${mov}; echo {"file_path":"${png}"} frame.png:12 https://example.com/a.png`; + const found = mediaPaths(text, root); + assert.deepEqual(found.map((f) => f.name), ["reel.mov", "voice.wav", "frame.png"]); + assert.equal(found[0].kind, "video"); + assert.equal(found[0].mime, "video/quicktime"); + assert.equal(found[2].mime, "image/png"); +}); + +test("resolves relative and home paths against the workspace, and skips what is not a file", () => { + const found = mediaPaths("open ./frame.png and out/missing.png and empty.png and ~/nowhere.png", root, { home: root }); + assert.deepEqual(found.map((f) => f.name), ["frame.png"]); + assert.equal(found[0].path, png); + assert.deepEqual(mediaPaths("nothing here", root), []); + assert.deepEqual(mediaPaths("", root), []); +}); + +test("reads a tool item's command, changes, input and output", () => { + const text = itemText({ command: "ls", tool: "Read · a.png", changes: [{ path: "b.mov" }], input: { file_path: "c.wav" }, aggregatedOutput: "wrote d.pdf" }); + for (const name of ["ls", "a.png", "b.mov", "c.wav", "d.pdf"]) assert.ok(text.includes(name), name); + assert.equal(itemText(null), ""); +}); + +test("a sighting replaces the current one only when the file or its write time differs", () => { + const a = { path: png, mtimeMs: 1 }; + assert.equal(mediaChanged(null, a), true); + assert.equal(mediaChanged(a, { path: png, mtimeMs: 1 }), false); + assert.equal(mediaChanged(a, { path: png, mtimeMs: 2 }), true); + assert.equal(mediaChanged(a, { path: mov, mtimeMs: 1 }), true); + assert.equal(mediaChanged(a, undefined), false); +}); + +test("every kind the card can show has a glyph and a mime", () => { + for (const [ext, type] of MEDIA_TYPES) { + assert.ok(["picture", "sound", "paper", "video"].includes(type.kind), ext); + assert.ok(type.mime.includes("/") && type.glyph, ext); + } +}); diff --git a/slab/menubar-swift/Sources/SlabMenubar/LocalArtifactPreview.swift b/slab/menubar-swift/Sources/SlabMenubar/LocalArtifactPreview.swift index aa1d8e04e..c846c742a 100644 --- a/slab/menubar-swift/Sources/SlabMenubar/LocalArtifactPreview.swift +++ b/slab/menubar-swift/Sources/SlabMenubar/LocalArtifactPreview.swift @@ -1,5 +1,6 @@ import Foundation import AppKit +import AVFoundation import PDFKit /// A local session marker may nominate one output, never a directory or remote URL. @@ -13,7 +14,7 @@ struct LocalArtifactPreview: Equatable { let kind: String init?(marker: [String: Any], kind: String) { - guard ["picture", "sound", "paper"].contains(kind), + guard ["picture", "sound", "paper", "video"].contains(kind), let path = marker["path"] as? String, path.hasPrefix("/"), let mime = marker["mime"] as? String, let version = marker["version"] as? Int, version > 0, @@ -22,6 +23,7 @@ struct LocalArtifactPreview: Equatable { "picture": ["image/png", "image/jpeg", "image/webp"], "sound": ["audio/wav", "audio/x-wav", "audio/mpeg"], "paper": ["application/pdf", "text/plain", "text/markdown", "text/x-tex", "application/x-tex"], + "video": ["video/mp4", "video/quicktime", "video/webm"], ] guard allowed[kind]?.contains(mime) == true else { return nil } self.path = path; self.mime = mime; self.version = version @@ -30,15 +32,54 @@ struct LocalArtifactPreview: Equatable { var key: String { "\(kind):\(artifactID):\(version):\(path)" } - func readValidatedFile() throws -> Data { + /// The file's own name, drawn under it on the card. + var name: String { (path as NSString).lastPathComponent } + + /// Pictures, sounds and papers are read whole and re-written into the + /// staging directory. A video is linked there instead of copied, so it may + /// be as large as a render actually is. + var sizeLimit: Int { kind == "video" ? 4 * 1024 * 1024 * 1024 : 64 * 1024 * 1024 } + + /// The nominated file, if it is one regular file of a size this card will + /// take. A symbolic link is refused: the marker names the file, not a + /// pointer that could be repointed after the fact. + func validatedFile() throws -> URL { let url = URL(fileURLWithPath: path) let info = try url.resourceValues(forKeys: [.isRegularFileKey, .isSymbolicLinkKey, .fileSizeKey]) guard info.isRegularFile == true, info.isSymbolicLink != true, - let size = info.fileSize, size > 0, size <= 64 * 1024 * 1024 else { + let size = info.fileSize, size > 0, size <= sizeLimit else { throw NSError(domain: "EaselPreview", code: 1, - userInfo: [NSLocalizedDescriptionKey: "Preview must be a regular file under 64 MB."]) + userInfo: [NSLocalizedDescriptionKey: "Preview must be a regular file under \(sizeLimit / (1024 * 1024)) MB."]) + } + return url + } + + func readValidatedFile() throws -> Data { + try Data(contentsOf: try validatedFile(), options: .mappedIfSafe) + } + + /// Stage the file into the directory WebKit may read, as a hard link where + /// the volume allows one and a copy where it does not. The link shares the + /// file's bytes and nothing else: the grant still covers this directory + /// alone. + func stageFile(into directory: URL) throws -> URL { + let source = try validatedFile() + let target = directory.appendingPathComponent("artifact") + do { try FileManager.default.linkItem(at: source, to: target) } + catch { try FileManager.default.copyItem(at: source, to: target) } + return target + } + + /// A video's frame size, the way it is meant to be shown (a phone + /// recording carries its rotation as a transform, not in its pixels). + func videoDimensions() -> CGSize { + let asset = AVURLAsset(url: URL(fileURLWithPath: path)) + if let track = asset.tracks(withMediaType: .video).first { + let size = track.naturalSize.applying(track.preferredTransform) + let width = abs(size.width), height = abs(size.height) + if width > 0 && height > 0 { return CGSize(width: width, height: height) } } - return try Data(contentsOf: url, options: .mappedIfSafe) + return CGSize(width: 768, height: 432) } /// Pixel dimensions for pictures; PDF points for the first page. UI resizing @@ -72,16 +113,25 @@ struct LocalArtifactPreview: Equatable { } else if kind == "sound" { content = (waveform ?? "") + "" ready = "const a=document.getElementById('artifact'); a.onloadedmetadata=ready; a.onerror=failed; if(a.readyState>=1)ready();" + } else if kind == "video" { + // A render plays itself, silently, the way a clip does on a desk: + // the sound is one click away and never ambushes the room. + content = "" + ready = "const a=document.getElementById('artifact'); a.onloadedmetadata=ready; a.onerror=failed; if(a.readyState>=1)ready();" } else { content = "
\(Self.escaped(text ?? ""))
" ready = "requestAnimationFrame(ready);" } + // The card scales this whole page down to rest size, so the caption is + // sized against the viewport rather than in points: it has to still be + // a name at a sixth of its size. + let caption = "
\(Self.escaped(name))
" return """ - - \(content) + + \(content)\(caption) """ } diff --git a/slab/menubar-swift/Sources/SlabMenubar/PromptPreview.swift b/slab/menubar-swift/Sources/SlabMenubar/PromptPreview.swift index a063aa923..50566b843 100644 --- a/slab/menubar-swift/Sources/SlabMenubar/PromptPreview.swift +++ b/slab/menubar-swift/Sources/SlabMenubar/PromptPreview.swift @@ -177,7 +177,7 @@ final class PromptPreview { private static let openDuration: TimeInterval = 0.16 private let window: PromptPreviewWindow - private let webView: WKWebView + private let webView: PromptPreviewWebView private let refreshBridge = PromptPreviewRefreshBridge() private var lastRefresh = Date.distantPast /// Scale through AppKit's frame/bounds mapping, not a layer transform. @@ -349,6 +349,7 @@ final class PromptPreview { publicationToken = publication requestedArtifact = nil readyArtifact = nil + webView.dragFile = nil artifactLoading = false artifactFailed = false refreshBridge.localDirectory = nil @@ -389,8 +390,10 @@ final class PromptPreview { let nonce = UUID().uuidString var stagedDirectory: URL? do { - let bytes = try artifact.readValidatedFile() - let dimensions = artifact.dimensions(bytes) + // A video is never read into memory here: it is linked into the + // staging directory and WebKit streams it from there. + let bytes = artifact.kind == "video" ? Data() : try artifact.readValidatedFile() + let dimensions = artifact.kind == "video" ? artifact.videoDimensions() : artifact.dimensions(bytes) if artifact.kind == "picture", NSImage(data: bytes) == nil { throw CocoaError(.fileReadCorruptFile) } if artifact.mime == "application/pdf", (PDFDocument(data: bytes)?.pageCount ?? 0) == 0 { throw CocoaError(.fileReadCorruptFile) } let directory = FileManager.default.temporaryDirectory.appendingPathComponent("slab-artifact-\(UUID().uuidString)", isDirectory: true) @@ -406,6 +409,9 @@ final class PromptPreview { if artifact.kind == "paper" { guard bytes.count <= 2 * 1024 * 1024, let decoded = String(data: bytes, encoding: .utf8) else { throw CocoaError(.fileReadCorruptFile) } text = decoded + } else if artifact.kind == "video" { + text = nil + _ = try artifact.stageFile(into: directory) } else { text = nil try bytes.write(to: directory.appendingPathComponent("artifact"), options: .atomic) @@ -425,6 +431,8 @@ final class PromptPreview { self.artifactDirectory = directory self.refreshBridge.localDirectory = directory self.refreshBridge.nonce = nonce + // The file on the card can be dragged off it, as itself. + self.webView.dragFile = URL(fileURLWithPath: artifact.path) self.artifactNavigation = self.webView.loadFileURL(target, allowingReadAccessTo: directory) } coverTimer?.invalidate() @@ -868,6 +876,7 @@ final class PromptPreview { var isOnScreen: Bool { window.isVisible } func close() { + webView.dragFile = nil frameCaptureID = nil frameContext = "" setHovered(false) @@ -950,6 +959,52 @@ final class PromptPreviewWindow: NSWindow { } /// The first left-click both focuses and reaches the piece's actual controls. -private final class PromptPreviewWebView: WKWebView { +/// +/// When the card shows a file, a drag lifts a copy of that file off the card +/// — into a message, a folder, an editor — and a click still reaches the +/// media's own controls. The click is held back until the pointer has said +/// which of the two it is: a press that lets go where it landed is replayed +/// to the page as the click it was. +private final class PromptPreviewWebView: WKWebView, NSDraggingSource { + /// The file behind the page, when the page is a file. Nil for a live piece. + var dragFile: URL? + private static let dragSlop: CGFloat = 5 + override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true } + + override func mouseDown(with event: NSEvent) { + guard let file = dragFile, let window else { super.mouseDown(with: event); return } + let start = event.locationInWindow + while true { + guard let next = window.nextEvent(matching: [.leftMouseDragged, .leftMouseUp]) else { continue } + if next.type == .leftMouseUp { + super.mouseDown(with: event) + super.mouseUp(with: next) + return + } + if hypot(next.locationInWindow.x - start.x, next.locationInWindow.y - start.y) >= Self.dragSlop { + beginFileDrag(file, from: event) + return + } + } + } + + private func beginFileDrag(_ file: URL, from event: NSEvent) { + let item = NSDraggingItem(pasteboardWriter: file as NSURL) + // A picture drags as itself; anything else drags as its Finder icon. + let image = NSImage(contentsOf: file) ?? NSWorkspace.shared.icon(forFile: file.path) + var size = image.size + if size.width > 0, size.height > 0 { + let scale = min(bounds.width / size.width, bounds.height / size.height, 1) + size = NSSize(width: size.width * scale, height: size.height * scale) + } else { + size = NSSize(width: 64, height: 64) + } + let at = convert(event.locationInWindow, from: nil) + item.setDraggingFrame(NSRect(x: at.x - size.width / 2, y: at.y - size.height / 2, width: size.width, height: size.height), contents: image) + beginDraggingSession(with: [item], event: event, source: self) + } + + func draggingSession(_ session: NSDraggingSession, sourceOperationMaskFor context: NSDraggingContext) -> NSDragOperation { .copy } + func ignoreModifierKeys(for session: NSDraggingSession) -> Bool { true } } diff --git a/slab/menubar-swift/tests/local-artifact-preview.sh b/slab/menubar-swift/tests/local-artifact-preview.sh index e10852920..09a9bd7b1 100755 --- a/slab/menubar-swift/tests/local-artifact-preview.sh +++ b/slab/menubar-swift/tests/local-artifact-preview.sh @@ -22,6 +22,16 @@ let audioHTML = sound.html(nonce:"test") assert(audioHTML.contains("controls preload='metadata'")) assert(!audioHTML.contains("autoplay")) assert(audioHTML.contains("onloadedmetadata=ready")) +assert(audioHTML.contains("
sound.wav
")) +let reelMarker: [String: Any] = ["path":"/tmp/reel one.mov", "mime":"video/quicktime", "version":3,"artifactId":"video-one"] +let reel = LocalArtifactPreview(marker: reelMarker, kind:"video")! +assert(reel.name == "reel one.mov") +assert(reel.sizeLimit > 64 * 1024 * 1024) +let videoHTML = reel.html(nonce:"test") +assert(videoHTML.contains("