From fb6fd396cc6e2299ebc6c097ce36ab4958f45f7b Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Mon, 5 Oct 2026 20:22:29 -0700 Subject: [PATCH] Slab sound card as a spectrogram; Aesel media list and reply chips The Slab card shows a sound as its own spectrogram, measured off the main thread with AVFoundation and an FFT: dim ahead, lit where it has played, the playhead blooming the moment's bands. Click plays, drag scrubs. File artifacts no longer carry the turn's "working" chip or a version. In Aesel the media name on the status line opens a list of the files the card has shown, with pin, open, reveal and copy. Picking one makes it the subject of the next message: a chip on the prompt, its path sent along. dev.sh swaps, re-signs and kickstarts instead of a full install, skips relaunching for an identical binary, and waits out the performance guard. Co-Authored-By: Claude Opus 5.5 --- aesel/src/desktop-session.mjs | 2 +- aesel/src/render.mjs | 19 ++-- aesel/src/tui.mjs | 82 +++++++++++++- aesel/test/render.test.mjs | 29 +++++ .../SlabMenubar/LocalArtifactPreview.swift | 69 +++++++++++- .../Sources/SlabMenubar/PromptPreview.swift | 42 ++++++- .../Sources/SlabMenubar/SoundSketch.swift | 103 ++++++++++++++++++ slab/menubar-swift/dev.sh | 85 ++++++++++++++- 8 files changed, 406 insertions(+), 25 deletions(-) create mode 100644 slab/menubar-swift/Sources/SlabMenubar/SoundSketch.swift diff --git a/aesel/src/desktop-session.mjs b/aesel/src/desktop-session.mjs index e355023f85..7b413cd65c 100644 --- a/aesel/src/desktop-session.mjs +++ b/aesel/src/desktop-session.mjs @@ -8,7 +8,7 @@ const LIMIT = 32 * 1024 * 1024; export function desktopSnapshot({ cwd, backend, model, effort = "", live, state, options, engine, handoff = "", archivedConversation = [] }) { return clone({ schema: 1, cwd: resolve(cwd), savedAt: new Date().toISOString(), backend, model, effort, live: { file: live.file, runtime: live.runtime?.id || live.runtime, channel: live.fallbackChannel || live.channel, genre: live.genre?.id || "piece" }, - ui: Object.fromEntries(["entries", "input", "cursor", "history", "historyIndex", "queued", "medium", "livePaused", "showQr", "autoAllow", "scrollOffset", "pieceSlug", "media", "spend"].map((key) => [key, state[key]]).filter(([, value]) => value !== undefined)), + ui: Object.fromEntries(["entries", "input", "cursor", "history", "historyIndex", "queued", "medium", "livePaused", "showQr", "autoAllow", "scrollOffset", "pieceSlug", "media", "mediaRecent", "subject", "spend"].map((key) => [key, state[key]]).filter(([, value]) => value !== undefined)), options: { autopublish: options.autopublish, mouseEnabled: options.mouseEnabled }, engine: { threadId: engine.threadId || "", ...(Number.isSafeInteger(engine.turns) ? {turns: engine.turns} : {}), ...(["ac", "open"].includes(backend) ? { messages: engine.messages || [], turns: engine.turns || 0 } : {}) }, handoff, archivedConversation }); } diff --git a/aesel/src/render.mjs b/aesel/src/render.mjs index aa4bedde8d..49f7b50d78 100644 --- a/aesel/src/render.mjs +++ b/aesel/src/render.mjs @@ -830,7 +830,10 @@ export function renderFrame(state, columns = 80, rows = 24, useColor = true) { } else { const input = Array.from(cleanText(state.input || "")); const cursor = Math.max(0, Math.min(state.cursor ?? input.length, input.length)); - const room = Math.max(1, width - 2); + // The file the next message is about, as a chip at the head of the + // line: picked from the media list, sent with the message, then gone. + const chip = state.subject ? `${state.subject.glyph} ${clipText(state.subject.name, Math.max(8, Math.floor(width / 4)))} › ` : ""; + const room = Math.max(1, width - 2 - textWidth(chip)); const start = Math.max(0, cursor - room + 1); const shown = input.slice(start, start + room); const at = cursor - start; @@ -839,8 +842,8 @@ export function renderFrame(state, columns = 80, rows = 24, useColor = true) { // ground after it, and the bar has to run to the edge. // No painted block: the terminal's own cursor stands here and blinks // the way it does everywhere else. The frame says where it goes. - const lead = `${shape.prompt ? `${shape.prompt} ` : ""}${start > 0 ? "‹" : ""}`; - inner = `${shape.prompt ? `${ink(palette.prompt)}${shape.prompt}${ink(palette.text)} ` : ""}${start > 0 ? "‹" : ""}${shown.slice(0, at).join("")}${under === " " && at >= shown.length ? "" : under}${shown.slice(at + 1).join("")}`; + const lead = `${chip}${shape.prompt ? `${shape.prompt} ` : ""}${start > 0 ? "‹" : ""}`; + inner = `${chip ? `${ink(palette.highlight)}${chip}${ink(palette.text)}` : ""}${shape.prompt ? `${ink(palette.prompt)}${shape.prompt}${ink(palette.text)} ` : ""}${start > 0 ? "‹" : ""}${shown.slice(0, at).join("")}${under === " " && at >= shown.length ? "" : under}${shown.slice(at + 1).join("")}`; state.cursorCell = { row: height - shape.bottom.length + shape.bottom.indexOf("bar") + 1, col: 1 + textWidth(lead) + textWidth(shown.slice(0, at).join("")) }; } // The words start on the bar's first cell: the purple is the margin. @@ -996,7 +999,7 @@ export function dropdownGeometry(state, width, height, shape = state.layout || { // The anchor row is the status line; without one, the bar. const anchor = height - bottom.length + (statusRow >= 0 ? statusRow : Math.max(0, bottom.lastIndexOf("bar"))); const facts = proStatus(state, width, false, shape).spans; - const span = facts.find((s) => s.name === "model") || facts.find((s) => s.name === "engine"); + const span = drop.kind === "media" ? facts.find((s) => s.name === "media") : facts.find((s) => s.name === "model") || facts.find((s) => s.name === "engine"); const items = drop.loading ? [{ id: "", label: "loading…", detail: "" }] : drop.items.length ? drop.items : [{ id: "", label: "nothing to choose", detail: "" }]; const count = Math.max(1, Math.min(items.length, 10, anchor - 2)); const index = Math.max(0, Math.min(drop.index || 0, items.length - 1)); @@ -1276,8 +1279,8 @@ export function proStatus(state, width, useColor, shape = state.layout || {}) { x += textWidth(separator); } spans.push({ name, x, width: textWidth(text) }); - // The model underlines under the mouse: it is the one fact that is a control. - const hovered = (name === "model" && state.hover === "model") || (name === "engine" && state.hover === "provider"); + // The controls underline under the mouse: the model, and the media file. + const hovered = (name === "model" && state.hover === "model") || (name === "engine" && state.hover === "provider") || (name === "media" && state.hover === "media"); // While the machine works the handle breathes: bright, then muted, on the // dance clock — a pulse you can see from across the room. let fact; @@ -1357,8 +1360,8 @@ export function headerAction(state, columns, rows, x, y) { if (banner && y - 1 >= banner.top && y - 1 < banner.top + banner.height && x - 1 >= banner.x && x - 1 < banner.x + banner.inner) return "update"; const row = shape.bottom.lastIndexOf("status"); if (row < 0 || y !== rows - (shape.bottom.length - 1 - row)) return ""; - const hit = proStatus(state, Math.max(32, columns), false, shape).spans.find((span) => (span.name === "model" || span.name === "engine") && x >= span.x + 1 && x <= span.x + span.width); - return hit ? (hit.name === "engine" ? "provider" : "model") : ""; + const hit = proStatus(state, Math.max(32, columns), false, shape).spans.find((span) => ["model", "engine", "media"].includes(span.name) && x >= span.x + 1 && x <= span.x + span.width); + return hit ? (hit.name === "engine" ? "provider" : hit.name) : ""; } if(y===rows-2){const hit=modelControls(state,columns).find(c=>x>=c.x&&x {}; function pinMedia(media) { unwatchMedia(); state.media = { ...media, source: "pinned", version: (state.media?.version || 0) + 1 }; + rememberMedia(media); const show = () => { if (profile.private) return; const { kind, path, mime, name, version } = state.media; @@ -1619,6 +1622,65 @@ function pinMedia(media) { redraw(); }); } +// The files this session's card has shown, newest first, for the list the +// media name opens on the status line. +function rememberMedia(media) { + const recent = (state.mediaRecent || []).filter((path) => path !== media.path); + state.mediaRecent = [media.path, ...recent].slice(0, 8); +} +// Click the media name: the files the card has shown, to put one back on it +// and keep it there, then what can be done with the one showing now. +function openMediaDropdown() { + const current = state.media?.path; + const paths = [...new Set([current, ...(state.mediaRecent || [])].filter(Boolean))]; + const files = paths.map((file) => mediaFile(file, cwd)).filter(Boolean); + const items = files.map((media) => ({ + file: media.path, + label: `${media.path === current ? "● " : " "}${media.glyph} ${media.name}`, + detail: fishPath(path.dirname(media.path)), + })); + if (state.subject) items.push({ act: "unsubject", label: " ✕ not about it", detail: `the next message drops ${state.subject.name}` }); + if (current && mediaFile(current, cwd)) { + if (state.media.source === "pinned") items.push({ act: "unpin", label: " ⊘ unpin", detail: "the card follows the tools again" }); + else items.push({ act: "pin", file: current, label: " ⦿ pin", detail: "hold it on the card, reload on every write" }); + items.push( + { act: "open", label: " ↗ open", detail: "in its own app" }, + { act: "reveal", label: " ⌕ reveal", detail: "in Finder" }, + { act: "copy", label: " ⧉ copy path", detail: "" }, + ); + } + state.dropdown = { kind: "media", title: "media", items, index: Math.max(0, items.findIndex((item) => item.file === current && !item.act)), loading: false }; + redraw(); +} +function chooseMedia(item) { + state.dropdown = null; + const current = state.media?.path; + if (item.file && (!item.act || item.act === "pin")) { + const media = mediaFile(item.file, cwd); + if (!media) { addEntry("error", `${path.basename(item.file)} is gone`); return redraw(); } + slabSession.preview(""); + pinMedia(media); + // Picking a file is also saying what the next message is about, so the + // message can be short: "louder in the lift". + if (!item.act) state.subject = { path: media.path, name: media.name, glyph: media.glyph, mime: media.mime }; + flash(item.act ? `${media.name} pinned · reloads on every write` : `next message is about ${media.name} · backspace drops it`); + } else if (item.act === "unsubject") { + state.subject = null; + } else if (item.act === "unpin") { + unpinMedia(); + flash("unpinned"); + } else if (current && (item.act === "open" || item.act === "reveal")) { + const child = spawn("open", item.act === "reveal" ? ["-R", current] : [current], { stdio: "ignore", detached: true }); + child.on("error", (error) => { addEntry("error", errorText(error)); redraw(); }); + child.unref(); + } else if (current && item.act === "copy") { + const child = spawn("pbcopy", [], { stdio: ["pipe", "ignore", "ignore"] }); + child.on("error", (error) => { addEntry("error", errorText(error)); redraw(); }); + child.stdin.end(current); + flash("path copied"); + } + return redraw(); +} function unpinMedia() { unwatchMedia(); unwatchMedia = () => {}; @@ -2828,8 +2890,13 @@ async function startTurn(text, { from = "", recovery = false } = {}) { state.queued = []; return finish(); } + // A typed message carries the file it is about; a retry carries it again. + const about = recovery ? recoveryRequest?.about : !from ? state.subject : null; if (!from && !recovery) { - transcript.event("user", { text }); + state.subject = null; + if (about) flash(`sent about ${about.name}`); + if (recoveryRequest) recoveryRequest.about = about; + transcript.event("user", { text, ...(about ? { about: about.path } : {}) }); // The first thing asked is what the session was about. if (!subject) { subject = text.slice(0, 140); @@ -2857,7 +2924,7 @@ async function startTurn(text, { from = "", recovery = false } = {}) { state.activityStage='';redraw(); } if(recoveryRequest)recoveryRequest.submitted=true; - await engine.startTurn(text+runtimeFeedbackContext(observed)+pixels.context,{images:pixels.images}); + await engine.startTurn(text+aboutContext(about)+runtimeFeedbackContext(observed)+pixels.context,{images:pixels.images}); } catch (error) { state.busy = false; state.status = "failed"; @@ -2871,6 +2938,11 @@ async function startTurn(text, { from = "", recovery = false } = {}) { } } +// What a chip on the prompt adds to the message: which file "this" is. +const aboutContext = (about) => about + ? `\n\n[This message is about ${about.path} (${about.mime}), the file picked from the media list and showing on the Slab card.]` + : ""; + function replaceInput(value) { state.input = value; state.cursor = Array.from(value).length; @@ -3069,7 +3141,8 @@ function handleKey(input) { } else if (input === "\x1b[D") state.cursor = Math.max(0, state.cursor - 1); else if (input === "\x1b[C") state.cursor = Math.min(Array.from(state.input).length, state.cursor + 1); else if (input === "\x7f" || input === "\b") { - if (state.cursor > 0) { + if (state.cursor === 0 && state.subject) state.subject = null; + else if (state.cursor > 0) { const characters = Array.from(state.input); characters.splice(--state.cursor, 1); state.input = characters.join(""); @@ -3154,6 +3227,7 @@ function decodeKeys(buffer) { if (mouse.click && action === "profile") openProfile(); if (mouse.click && action === "update") { void submitInput("/update"); return; } if (mouse.click && (action === "model" || action === "provider")) { if (pro) openDropdown(); else openSettings(); } + if (mouse.click && action === "media") { if (state.dropdown?.kind === "media") closeDropdown(); else openMediaDropdown(); } if (mouse.click && action === "dismiss") closeDropdown(); if (action.startsWith("pick:") && state.dropdown) { const index = Number(action.slice(5)); if (mouse.click) void chooseDropdown(index); else if (state.dropdown.index !== index) { state.dropdown.index = index; redraw(); } } if(mouse.click&&action.startsWith('settings:')){ diff --git a/aesel/test/render.test.mjs b/aesel/test/render.test.mjs index 2aa7c5f3d7..dd3b02c1af 100644 --- a/aesel/test/render.test.mjs +++ b/aesel/test/render.test.mjs @@ -481,6 +481,35 @@ test("a drop-down stands on the fact that opened it, and a click on one of its r assert.match(loading, /loading…/); }); +test("the media name is a control: a click opens its list, standing on the name", async () => { + const { dropdownGeometry, headerAction, proStatus } = await import("../src/render.mjs"); + const base = { + workspace: "/client", mode: "remote", status: "ready", busy: false, input: "", + account: "@tester", model: "claude-sonnet-5", providerSettings: { backend: "claude", model: "claude-sonnet-5" }, + profile: { name: "pro" }, entries: [{ id: "u", kind: "user", text: "hi" }], + media: { glyph: "🔊", name: "climbalift-now.mp3", path: "/w/out/climbalift-now.mp3" }, + }; + const span = proStatus(base, 100, false).spans.find((s) => s.name === "media"); + assert.ok(span, "the media file is on the line"); + const row = 24 - 2 + 2; // the status line, one-based + assert.equal(headerAction(base, 100, 24, span.x + 2, row), "media", "clicking the name opens its list"); + const items = [ + { file: "/w/out/climbalift-now.mp3", label: "● 🔊 climbalift-now.mp3", detail: "/w/out" }, + { act: "open", label: " ↗ open", detail: "in its own app" }, + ]; + const state = { ...base, hover: "media", dropdown: { kind: "media", title: "media", items, index: 0, loading: false } }; + const g = dropdownGeometry(state, 100, 24); + assert.equal(g.x, proStatus(state, 100, false).spans.find((s) => s.name === "media").x, "it stands on the media name"); + const frame = renderFrame(state, 100, 24, false).split("\n"); + assert.match(frame[g.top], /▾ media/); + assert.match(frame[g.top + 1], /climbalift-now\.mp3 {2,}\/w\/out/); + assert.equal(headerAction(state, 100, 24, g.x + 2, g.top + 3), "pick:1"); + const typed = { ...base, input: "louder in the lift", cursor: 18, subject: { glyph: "🔊", name: "climbalift-now.mp3" } }; + const bar = renderFrame(typed, 100, 24, false).split("\n")[24 - 3]; + assert.match(bar, /^🔊 climbalift-now\.mp3 › louder in the lift/, "the file the message is about leads the line"); + assert.equal(typed.cursorCell.col, 1 + "🔊 climbalift-now.mp3 › ".length + 18, "the cursor sits after the chip and the words"); +}); + test("a reply's markdown is read, not shown, and pro's page is flush left", async () => { const { markdown } = await import("../src/render.mjs"); const read = markdown("## Plan\n- **Airtable** needs `auth` first\n- see [docs](https://example.com/x)"); diff --git a/slab/menubar-swift/Sources/SlabMenubar/LocalArtifactPreview.swift b/slab/menubar-swift/Sources/SlabMenubar/LocalArtifactPreview.swift index c846c742af..7885765e5c 100644 --- a/slab/menubar-swift/Sources/SlabMenubar/LocalArtifactPreview.swift +++ b/slab/menubar-swift/Sources/SlabMenubar/LocalArtifactPreview.swift @@ -104,10 +104,13 @@ struct LocalArtifactPreview: Equatable { } /// No project content is executable. Only the fixed readiness script runs. - func html(text: String? = nil, waveform: String? = nil, nonce: String) -> String { + func html(text: String? = nil, waveform: String? = nil, sketch: SoundSketch? = nil, nonce: String) -> String { let content: String let ready: String - if kind == "picture" { + if kind == "sound", let sketch { + content = Self.soundContent + ready = Self.soundScript(sketch) + } else if kind == "picture" { content = "Picture preview" ready = "const a=document.getElementById('artifact'); a.onload=ready; a.onerror=failed; if(a.complete&&a.naturalWidth)ready();" } else if kind == "sound" { @@ -135,6 +138,68 @@ struct LocalArtifactPreview: Equatable { """ } + /// The sound card is the record, not a player: its whole spectrogram, + /// lows at the bottom, lit where it has played and dim ahead, with the + /// playhead blooming the moment's bands. No button, no transport bar — + /// click plays or pauses, drag scrubs, space toggles. + static let soundContent = """ + +
+ + """ + + static func soundScript(_ s: SoundSketch) -> String { + """ + const a=document.getElementById('artifact'),deck=document.getElementById('deck'),c=document.getElementById('c'),g=c.getContext('2d'); + const q=t=>Array.from(t,ch=>parseInt(ch,36)/35),SP=q('\(s.spectrum)'),PK=q('\(s.peaks)'),B=\(SoundSketch.bands),F=\(SoundSketch.fps),D=\(s.duration),FR=SP.length/B; + // Contrast: the quiet two-fifths of the range go to black, the rest curves up. + const lv=v=>Math.pow(Math.max(0,(v-.4)/.6),1.5); + const col=(k,v,lit)=>`hsl(${330-k*(150/(B-1))},${lit?90:14}%,${lit?4+62*v:5+44*v}%)`; + let lit,dim,raf=0,down=null,moved=false; + const fmt=t=>{t=Math.max(0,Math.floor(t));return Math.floor(t/60)+':'+String(t%60).padStart(2,'0');}; + // The spectrogram, drawn once per size: each pixel column the loudest + // frame it covers, each band a row, quieter cells darker. + function paint(W,H,on){ + const o=document.createElement('canvas');o.width=W;o.height=H;const x=o.getContext('2d'),bh=H/B; + for(let px=0;px0||!a.paused){ + const fi=Math.min(FR-1,Math.floor(t*F)),w=Math.max(3,W*.012); + for(let k=0;k0||!a.paused?fmt(t)+' / ':'')+fmt(D),tw=g.measureText(label).width; + g.fillRect(W-tw-fs*1.1,fs*.4,tw+fs*.8,fs*1.3);g.fillStyle=a.paused?'rgba(255,255,255,.75)':'#f27cad';g.fillText(label,W-fs*.7,fs*.55); + } + function size(){const r=c.getBoundingClientRect(),d=window.devicePixelRatio||1,W=Math.max(1,Math.round(r.width*d)),H=Math.max(1,Math.round(r.height*d)); + if(W===c.width&&H===c.height&&lit)return draw();c.width=W;c.height=H;lit=paint(W,H,true);dim=paint(W,H,false);draw();} + function loop(){draw();raf=a.paused?0:requestAnimationFrame(loop);} + const toggle=()=>{a.paused?a.play():a.pause();}; + const seek=e=>{const r=deck.getBoundingClientRect();a.currentTime=Math.max(0,Math.min(1,(e.clientX-r.left)/r.width))*D;draw();}; + deck.addEventListener('pointerdown',e=>{down=e.clientX;moved=false;deck.setPointerCapture(e.pointerId);}); + deck.addEventListener('pointermove',e=>{if(down===null)return;if(Math.abs(e.clientX-down)>4)moved=true;if(moved)seek(e);}); + deck.addEventListener('pointerup',e=>{if(down!==null&&!moved)toggle();down=null;}); + document.addEventListener('keydown',e=>{if(e.code==='Space'){e.preventDefault();toggle();}}); + a.addEventListener('play',()=>{if(!raf)loop();}); + a.addEventListener('pause',draw);a.addEventListener('ended',draw);a.addEventListener('seeked',draw); + new ResizeObserver(size).observe(deck); + a.onloadedmetadata=()=>{ready();size();};a.onerror=failed;if(a.readyState>=1){ready();size();} + """ + } + /// Static waveform of the actual Easel PCM WAV; unsupported formats simply /// retain their native audio controls. Never draw invented progress. static func waveform(_ data: Data) -> String? { diff --git a/slab/menubar-swift/Sources/SlabMenubar/PromptPreview.swift b/slab/menubar-swift/Sources/SlabMenubar/PromptPreview.swift index 5f28523a63..4d823d76cd 100644 --- a/slab/menubar-swift/Sources/SlabMenubar/PromptPreview.swift +++ b/slab/menubar-swift/Sources/SlabMenubar/PromptPreview.swift @@ -425,8 +425,34 @@ final class PromptPreview { /// Stage only the nominated output. HTML is generated here, never loaded /// from an authored project, and the WebKit read grant covers this copy only. + /// The latest sound's measurements, by artifact key; nil inside means the + /// file could not be read and the card falls back to plain controls. + private var sketches: [String: SoundSketch?] = [:] + private var sketching: String? + func loadArtifact(_ artifact: LocalArtifactPreview) { guard artifact.key != loadedURL else { return } + // A sound is measured before its page is written (decode and FFT, about + // a second for a whole song in a debug build), off the main thread. + // The card says it is loading meanwhile and comes back here when done. + if artifact.kind == "sound", sketches[artifact.key] == nil { + requestedArtifact = artifact + artifactFailed = false + artifactLoading = true + guard sketching != artifact.key else { return } + sketching = artifact.key + let url = URL(fileURLWithPath: artifact.path) + DispatchQueue.global(qos: .userInitiated).async { [weak self] in + let sketch = SoundSketch(url: url) + DispatchQueue.main.async { + guard let self else { return } + if self.sketching == artifact.key { self.sketching = nil } + self.sketches = [artifact.key: sketch] + if self.requestedArtifact == artifact { self.loadArtifact(artifact) } + } + } + return + } loadedURL = artifact.key requestedArtifact = artifact artifactFailed = false @@ -461,8 +487,9 @@ final class PromptPreview { try bytes.write(to: directory.appendingPathComponent("artifact"), options: .atomic) } target = directory.appendingPathComponent("index.html") - let waveform = artifact.kind == "sound" ? LocalArtifactPreview.waveform(bytes) : nil - try artifact.html(text: text, waveform: waveform, nonce: nonce).write(to: target, atomically: true, encoding: .utf8) + let sketch = artifact.kind == "sound" ? (sketches[artifact.key] ?? nil) : nil + let waveform = artifact.kind == "sound" && sketch == nil ? LocalArtifactPreview.waveform(bytes) : nil + try artifact.html(text: text, waveform: waveform, sketch: sketch, nonce: nonce).write(to: target, atomically: true, encoding: .utf8) } let start = { [weak self] in guard let self, self.requestedArtifact == artifact else { @@ -502,7 +529,8 @@ final class PromptPreview { artifactFailed = false cover.isHidden = true cover.image = nil - state.version = artifact.version + state.version = 0 + state.working = false state.piece = artifact.kind.capitalized state.previewError = false celebrateRefresh() @@ -519,9 +547,13 @@ final class PromptPreview { func setState(_ next: PromptPreviewState) { var resolved = next if requestedArtifact != nil { - resolved.version = readyArtifact?.version ?? 0 + // A file on the card is not the turn's work in progress: the agent + // being busy changes nothing about an mp3 already playing, and the + // version is only how many times it was written. The chip speaks + // for the file alone — while it loads, or when it fails. + resolved.version = 0 resolved.piece = (readyArtifact ?? requestedArtifact)?.kind.capitalized ?? next.piece - resolved.working = next.working || artifactLoading + resolved.working = artifactLoading resolved.previewError = artifactFailed } state = resolved diff --git a/slab/menubar-swift/Sources/SlabMenubar/SoundSketch.swift b/slab/menubar-swift/Sources/SlabMenubar/SoundSketch.swift new file mode 100644 index 0000000000..12fd511dec --- /dev/null +++ b/slab/menubar-swift/Sources/SlabMenubar/SoundSketch.swift @@ -0,0 +1,103 @@ +import Foundation +import AVFoundation +import Accelerate + +/// What the sound card draws, measured from the file itself: an overview of +/// its loudness in columns, and the spectrum in sixteen bands twenty times a +/// second, so the bars that move while it plays are the record's own and +/// never an animation standing in for it. Any format AVFoundation reads +/// (mp3, wav, aiff, m4a). Values are quantized to one base-36 character each, +/// which keeps a three-minute song's sketch near sixty kilobytes of page. +struct SoundSketch { + static let columns = 480 + static let bands = 16 + static let fps = 20 + + let duration: Double + /// `columns` characters of peak level, then `columns` of RMS. + let peaks: String + let rms: String + /// `frames × bands` characters, frame-major. + let spectrum: String + let frames: Int + + private static let digits = Array("0123456789abcdefghijklmnopqrstuvwxyz") + private static func q(_ x: Float) -> Character { digits[max(0, min(35, Int((x * 35).rounded())))] } + + init?(url: URL, maxSeconds: Double = 20 * 60) { + guard let file = try? AVAudioFile(forReading: url) else { return nil } + let sr = file.processingFormat.sampleRate, total = Int(file.length) + guard sr > 0, total > 0, Double(total) / sr <= maxSeconds else { return nil } + // One mono line: the channels averaged, read in blocks. + var mono = [Float](repeating: 0, count: total) + let block: AVAudioFrameCount = 1 << 16 + guard let buffer = AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: block) else { return nil } + var at = 0 + while at < total { + do { try file.read(into: buffer, frameCount: min(block, AVAudioFrameCount(total - at))) } catch { break } + let n = Int(buffer.frameLength), ch = Int(buffer.format.channelCount) + guard n > 0, let data = buffer.floatChannelData else { break } + for c in 0.. 0 else { return nil } + duration = Double(count) / sr + + // The overview: peak and RMS per column, against the loudest column. + var peakCol = [Float](repeating: 0, count: Self.columns), rmsCol = peakCol + mono.withUnsafeBufferPointer { p in + for x in 0..= 0 && s < count ? mono[s] * window[j] : 0 } + re.withUnsafeMutableBufferPointer { r in im.withUnsafeMutableBufferPointer { m in + var split = DSPSplitComplex(realp: r.baseAddress!, imagp: m.baseAddress!) + frame.withUnsafeBufferPointer { f in + f.baseAddress!.withMemoryRebound(to: DSPComplex.self, capacity: n / 2) { vDSP_ctoz($0, 2, &split, 1, vDSP_Length(n / 2)) } + } + vDSP_fft_zrip(setup, &split, 1, log2n, FFTDirection(FFT_FORWARD)) + vDSP_zvmags(&split, 1, &mag, 1, vDSP_Length(n / 2)) + } } + mag.withUnsafeBufferPointer { m in + for k in 0../dev/null 2>&1 || return 1 + hash="$(security find-certificate -Z -c "Slab Menubar Self-Signed" "${KC}" 2>/dev/null | awk '/SHA-1 hash:/{print $3; exit}')" + [[ -n "${hash}" ]] || return 1 + security unlock-keychain -p "${SLAB_SIGN_KC_PASS:-slab-signing}" "${KC}" 2>/dev/null || true + # A fresh inode, not an overwrite: kickstart against the old inode's cached + # signature is the "Code Signature Invalid" launch failure install.sh + # documents. Then sign the bundle around it. + cp "${bin}" "${APP_BIN}.next" + mv -f "${APP_BIN}.next" "${APP_BIN}" + codesign --force --sign "${hash}" --identifier computer.slab.menubar "${APP}" >/dev/null 2>&1 || return 1 + old="$(pgrep -f "SlabMenubar.app/Contents/MacOS/slab-menubar" | head -1)" + launchctl kickstart -k "${LABEL}" || return 1 + for i in $(seq 1 16); do + pid="$(pgrep -f "SlabMenubar.app/Contents/MacOS/slab-menubar" | head -1)" + [[ -n "${pid}" && "${pid}" != "${old}" ]] && return 0 + sleep 0.25 + done + return 1 +} + +# The debug binary, without asking SwiftPM (a second package load, ~1.3 s): +# the newest of the two layouts SwiftPM uses, the ask only as a fallback. +debug_bin() { + local a="${DIR}/.build/out/Products/Debug/slab-menubar-swift" b="${DIR}/.build/debug/slab-menubar-swift" + if [[ -x "$a" && ( ! -x "$b" || "$a" -nt "$b" ) ]]; then echo "$a" + elif [[ -x "$b" ]]; then echo "$b" + else echo "$(cd "${DIR}" && /usr/bin/swift build -c debug --show-bin-path)/slab-menubar-swift"; fi +} + +# 0 relaunched · 1 failed · 75 the host's performance guard deferred the build once() { - local t0=$SECONDS bin + local t0=$SECONDS bin status sum # build-dev.sh and install.sh each take the host build lock, so they run # one after the other, never nested. - if ! "${DIR}/build-dev.sh" >"${LOG}" 2>&1; then + "${DIR}/build-dev.sh" >"${LOG}" 2>&1; status=$? + if (( status == 75 )); then + printf '… deferred: %s\n' "$(grep -m1 -o 'deferred.*' "${LOG}" | cut -c1-110)" + return 75 + elif (( status != 0 )); then grep -E 'error:' "${LOG}" | sort -u | head -20 printf '✗ build failed (%ss) · full log %s\n' "$((SECONDS - t0))" "${LOG}" return 1 fi - bin="$(cd "${DIR}" && swift build -c debug --show-bin-path)/slab-menubar-swift" + bin="$(debug_bin)" + # The same binary as last time (a save that compiled to nothing new): no + # relaunch, so the menubar does not blink for a comment. + sum="$(shasum -a 256 "${bin}" | cut -c1-64)" + if [[ -f "${DIR}/.build/.dev-swapped" && "$(cat "${DIR}/.build/.dev-swapped")" == "${sum}" ]] \ + && pgrep -f "SlabMenubar.app/Contents/MacOS/slab-menubar" >/dev/null; then + printf '= unchanged binary, still running (%ss)\n' "$((SECONDS - t0))" + return 0 + fi + if swap "${bin}"; then + echo "${sum}" > "${DIR}/.build/.dev-swapped" + printf '✓ slab relaunched on the debug build (%ss)\n' "$((SECONDS - t0))" + return 0 + fi + echo "• full install (first run, bundle changed, or the swap did not come up)" if ! SLAB_PREBUILT="${bin}" "${DIR}/install.sh" >"${LOG}.install" 2>&1; then tail -8 "${LOG}.install" printf '✗ install failed (%ss)\n' "$((SECONDS - t0))" @@ -27,6 +90,7 @@ once() { fi grep -E 'identical|signed with|running \(pid|recovered|! ' "${LOG}.install" | tail -3 if pgrep -f "SlabMenubar.app/Contents/MacOS/slab-menubar" >/dev/null; then + echo "${sum}" > "${DIR}/.build/.dev-swapped" printf '✓ slab relaunched on the debug build (%ss)\n' "$((SECONDS - t0))" else printf '✗ slab is not running after install (%ss) · /tmp/slab-menubar.err\n' "$((SECONDS - t0))" @@ -34,11 +98,22 @@ once() { fi } +# Until it is not deferred: the guard admits the build once load falls. +until_built() { + local status + while :; do + once; status=$? + (( status == 75 )) || return "${status}" + sleep 5 + done +} + if [[ "${1:-}" != "--watch" ]]; then once; exit $?; fi +# Watch mode waits out the performance guard instead of giving up. command -v fswatch >/dev/null || { echo "--watch needs fswatch: brew install fswatch"; exit 1; } -once +until_built echo "• watching ${DIR}/Sources (ctrl-c to stop)" # -o: one line per batch of events; -l: let an editor's burst of writes settle fswatch -o -l 0.4 -e '.*' -i '\.swift$' "${DIR}/Sources" | while read -r _; do - echo "• change"; once + echo "• change"; until_built done -- 2.51.2