From 648e73a46e5dca91312d57cee02fd88240834e6e Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Tue, 8 Sep 2026 09:19:52 -0400 Subject: [PATCH] =?UTF-8?q?chat:=20prutti's=20words=20learn=20to=20speak?= =?UTF-8?q?=20=E2=80=94=20a=20vox=20chip=20on=20every=20message?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pruttivox reaches the chat: an amber chip after each @prutti message plays it in his consented voice clone, word lighting up as it is spoken. /api/pruttivox verifies authorship in Mongo, renders through ElevenLabs with-timestamps, and caches mp3 + word timings on the CDN so each message costs one render ever. CLI lane + consent gates in marketing/klokkentales (SCORE.md). Claude-Session: https://claude.ai/code/session_01GmNKmbpGxYo3NqtqyFosjY --- marketing/klokkentales/SCORE.md | 18 ++ marketing/klokkentales/bin/pruttivox.mjs | 107 +++++++++ system/netlify.toml | 25 ++- system/netlify/functions/pruttivox.mjs | 208 ++++++++++++++++++ .../public/aesthetic.computer/disks/chat.mjs | 174 ++++++++++++++- 5 files changed, 520 insertions(+), 12 deletions(-) create mode 100644 marketing/klokkentales/bin/pruttivox.mjs create mode 100644 system/netlify/functions/pruttivox.mjs diff --git a/marketing/klokkentales/SCORE.md b/marketing/klokkentales/SCORE.md index a123d285eb..e9a3653b31 100644 --- a/marketing/klokkentales/SCORE.md +++ b/marketing/klokkentales/SCORE.md @@ -69,8 +69,26 @@ node bin/feed.mjs # Review first; public release always requires the explicit second command. node bin/buzzsprout.mjs summer-so-far-2026 --private node bin/buzzsprout.mjs publish summer-so-far-2026 + +# Pruttivox: one-off read-aloud of a community text in the Prutti IVC. +node bin/pruttivox.mjs "teksten her" --from @snakes --publish ``` +## Pruttivox + +Community members send a text; Prutti's clone reads it. Every clip ends with a +spoken "Pruttivox. Syntetisk stemme." tag, and each render logs its text and +requester to the vault. Listen to the whole clip before sharing. Never render a +text that has the voice make real-world commitments — payments, meetings, +endorsements, claims about other people; the clone reads performances, it does +not speak for Prutti. Clips are shared in the clock channel where Prutti +participates and can veto any clip. + +The chat lane (`/api/pruttivox` + the "vox" chip in chat/laklok) reads only +messages @prutti himself typed — the text comes from the database by message +id, never from the caller — and caches each render on the CDN with word +timings for the karaoke highlight. + ## Release gates 1. Confirm the date window and evidence snapshot. diff --git a/marketing/klokkentales/bin/pruttivox.mjs b/marketing/klokkentales/bin/pruttivox.mjs new file mode 100644 index 0000000000..79ee5fc6d7 --- /dev/null +++ b/marketing/klokkentales/bin/pruttivox.mjs @@ -0,0 +1,107 @@ +#!/usr/bin/env node + +// Pruttivox: read a community text aloud in the consented Prutti IVC. +// Every clip ends with a spoken synthetic-voice tag and is logged to the +// vault with its requester, so provenance survives the disposable out/ dir. +// +// node bin/pruttivox.mjs "Hej klokken, god torsdag" --from @snakes +// node bin/pruttivox.mjs "..." --slug god-torsdag --publish + +import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { klokkentalesVault, loadKlokkentalesEnv } from "../lib/env.mjs"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const outRoot = resolve(HERE, "..", "out", "pruttivox"); +const logPath = resolve(klokkentalesVault, "pruttivox", "log.jsonl"); +const tagPath = resolve(outRoot, "tag.mp3"); +const tagText = "Pruttivox. Syntetisk stemme."; +const publishPrefix = "klokkentales/pruttivox"; + +const env = loadKlokkentalesEnv(); +const apiBase = env.ELEVENLABS_API_BASE || "https://api.elevenlabs.io"; +const receiptPath = resolve(klokkentalesVault, "voices", "prutti", "voice.json"); +const voiceId = env.PRUTTI_ELEVENLABS_VOICE_ID || (existsSync(receiptPath) + ? JSON.parse(readFileSync(receiptPath, "utf8")).voiceId + : null); + +const argv = process.argv.slice(2); +const text = argv.find((arg) => !arg.startsWith("--")); +const flag = (name) => { + const index = argv.indexOf(`--${name}`); + return index >= 0 ? argv[index + 1] : null; +}; + +if (!voiceId) throw new Error("Prutti voice has not been created; see voice.mjs"); +if (!env.ELEVENLABS_API_KEY) throw new Error("ELEVENLABS_API_KEY is missing"); +if (!text || !text.trim()) { + throw new Error('usage: pruttivox.mjs "the text" [--from @handle] [--slug name] [--publish] [--force]'); +} +if (text.length > 1200) { + throw new Error(`text is ${text.length} characters; keep pruttivox clips under 1200`); +} + +const slug = (flag("slug") || text.trim().toLowerCase() + .replace(/[^\p{L}\p{N}\s-]/gu, "").split(/\s+/).slice(0, 5).join("-")) + .replace(/[^a-z0-9æøå-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, ""); +if (!slug) throw new Error("could not derive a slug from the text; pass --slug"); +const from = flag("from") || ""; +const output = resolve(outRoot, `${slug}.mp3`); +const publish = argv.includes("--publish"); + +const log = existsSync(logPath) + ? readFileSync(logPath, "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)) + : []; +if (publish && !argv.includes("--force") && log.some((entry) => entry.slug === slug && entry.published)) { + throw new Error(`"${slug}" was already published; a republish sits stale on the CDN for ~1h — pick a new slug or pass --force`); +} + +async function speak(line, destination) { + const response = await fetch(`${apiBase}/v1/text-to-speech/${encodeURIComponent(voiceId)}`, { + method: "POST", + headers: { "xi-api-key": env.ELEVENLABS_API_KEY, "Content-Type": "application/json" }, + body: JSON.stringify({ + text: line, + model_id: "eleven_multilingual_v2", + voice_settings: { stability: 0.38, similarity_boost: 0.9, style: 0.48, use_speaker_boost: true, speed: 0.98 }, + }), + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) throw new Error(`ElevenLabs ${response.status}: ${(await response.text()).slice(0, 500)}`); + writeFileSync(destination, Buffer.from(await response.arrayBuffer()), { mode: 0o600 }); +} + +mkdirSync(outRoot, { recursive: true }); +if (!existsSync(tagPath)) await speak(tagText, tagPath); +const bodyPath = resolve(outRoot, `${slug}.body.mp3`); +await speak(text, bodyPath); + +execFileSync("ffmpeg", [ + "-y", "-i", bodyPath, "-i", tagPath, + "-filter_complex", "[0:a]apad=pad_dur=0.4[a0];[a0][1:a]concat=n=2:v=0:a=1", + "-ar", "44100", "-b:a", "192k", + "-metadata", "artist=Pruttivox (syntetisk stemme / synthetic voice)", + "-metadata", "album=Klokkentales", + "-metadata", `title=${slug}`, + "-metadata", "comment=Consented ElevenLabs IVC — aesthetic.computer/klokkentales", + output, +], { stdio: "ignore" }); + +const url = `https://assets.aesthetic.computer/${publishPrefix}/${slug}.mp3`; +if (publish) { + execFileSync("aws", [ + "s3", "cp", output, `s3://assets-aesthetic-computer/${publishPrefix}/${slug}.mp3`, + "--endpoint-url", "https://sfo3.digitaloceanspaces.com", "--acl", "public-read", + ], { stdio: "inherit" }); +} + +mkdirSync(dirname(logPath), { recursive: true }); +appendFileSync(logPath, JSON.stringify({ + slug, text, from, published: publish, url: publish ? url : undefined, + at: new Date().toISOString(), +}) + "\n", { mode: 0o600 }); + +console.log(output); +if (publish) console.log(url); diff --git a/system/netlify.toml b/system/netlify.toml index a9f95ec345..2ddcc349b0 100644 --- a/system/netlify.toml +++ b/system/netlify.toml @@ -336,10 +336,10 @@ included_env_vars = ["CONTEXT", "ADMIN_SUB", "AUTH0_M2M_CLIENT_ID", "AUTH0_M2M_S external_node_modules = ["mongodb", "mongodb-connection-string-url"] included_files = ["backend/**/*.mjs"] included_env_vars = ["CONTEXT", "ADMIN_SUB", "AUTH0_M2M_CLIENT_ID", "AUTH0_M2M_SECRET", "MONGODB_CONNECTION_STRING", "MONGODB_NAME"] -[functions.nom-scores] -external_node_modules = ["mongodb", "mongodb-connection-string-url"] -included_files = ["backend/**/*.mjs", "public/aesthetic.computer/lib/nom-score.mjs"] -included_env_vars = ["CONTEXT", "ADMIN_SUB", "AUTH0_M2M_CLIENT_ID", "AUTH0_M2M_SECRET", "MONGODB_CONNECTION_STRING", "MONGODB_NAME"] +[functions.nom-scores] +external_node_modules = ["mongodb", "mongodb-connection-string-url"] +included_files = ["backend/**/*.mjs", "public/aesthetic.computer/lib/nom-score.mjs"] +included_env_vars = ["CONTEXT", "ADMIN_SUB", "AUTH0_M2M_CLIENT_ID", "AUTH0_M2M_SECRET", "MONGODB_CONNECTION_STRING", "MONGODB_NAME"] [functions.track-media] external_node_modules = ["sharp", "mongodb", "mongodb-connection-string-url", "adm-zip", "@atproto/api"] included_files = ["backend/**/*.mjs"] @@ -401,6 +401,11 @@ included_env_vars = ["CONTEXT", "AUTH0_M2M_CLIENT_ID", "AUTH0_M2M_SECRET", "MONG external_node_modules = ["mongodb", "mongodb-connection-string-url"] included_files = ["backend/**/*.mjs"] included_env_vars = ["MONGODB_CONNECTION_STRING", "MONGODB_NAME"] +[functions.pruttivox] +# Speak a @prutti chat message in his consented voice clone (word-timed) +external_node_modules = ["mongodb", "mongodb-connection-string-url"] +included_files = ["backend/**/*.mjs"] +included_env_vars = ["MONGODB_CONNECTION_STRING", "MONGODB_NAME", "ART_ENDPOINT", "ART_KEY", "ART_SECRET", "ELEVENLABS_API_KEY", "PRUTTI_ELEVENLABS_VOICE_ID"] [functions.metrics] external_node_modules = ["mongodb", "mongodb-connection-string-url"] included_files = ["backend/**/*.mjs"] @@ -1829,10 +1834,10 @@ from = "/api/piece-fans" to = "/.netlify/functions/piece-fans" status = 200 [[redirects]] -from = "/api/nom-scores" -to = "/.netlify/functions/nom-scores" -status = 200 -[[redirects]] +from = "/api/nom-scores" +to = "/.netlify/functions/nom-scores" +status = 200 +[[redirects]] from = "/api/ticket/*" to = "/.netlify/functions/ticket" status = 200 @@ -1937,6 +1942,10 @@ from = "/api/boot-log" to = "/.netlify/functions/boot-log" status = 200 [[redirects]] +from = "/api/pruttivox" +to = "/.netlify/functions/pruttivox" +status = 200 +[[redirects]] from = "/api/paper-hit" to = "/.netlify/functions/paper-hit" status = 200 diff --git a/system/netlify/functions/pruttivox.mjs b/system/netlify/functions/pruttivox.mjs new file mode 100644 index 0000000000..4d12833f47 --- /dev/null +++ b/system/netlify/functions/pruttivox.mjs @@ -0,0 +1,208 @@ +// Pruttivox — speak a @prutti chat message aloud in his consented voice +// clone, with per-word timing for the karaoke highlight in chat/laklok. +// +// GET /api/pruttivox?id= +// → { audio, duration, words: [{ i, s, e }], text } +// +// The message id must belong to @prutti — the text is read from the +// database, never from the caller, so this can't become an open TTS +// proxy. Renders cache to Spaces keyed by voice + spoken text, so each +// message costs ElevenLabs once. Voice consent + veto gates live in +// marketing/klokkentales/SCORE.md. + +import crypto from "crypto"; +import { + S3Client, + GetObjectCommand, + PutObjectCommand, +} from "@aws-sdk/client-s3"; +// ObjectId must come from the same mongodb copy the backend client uses — +// a second copy's BSON types fail serialization (BSONVersionError). +import { connect, ObjectId } from "../../backend/database.mjs"; +import { respond } from "../../backend/http.mjs"; + +const BUCKET = "assets-aesthetic-computer"; +const CDN = "https://assets.aesthetic.computer"; +const PREFIX = "klokkentales/pruttivox/chat/"; +const VOICE_HANDLE = "prutti"; + +const s3 = new S3Client({ + endpoint: `https://${process.env.ART_ENDPOINT}`, + region: "us-east-1", + credentials: { + accessKeyId: process.env.ART_KEY, + secretAccessKey: process.env.ART_SECRET, + }, +}); + +// Group the character alignment from `/with-timestamps` into word times. +// A word here is any non-whitespace run — the same split the client uses +// on the displayed text, so word i lines up with displayed token i. +function wordsFromAlignment(alignment) { + const chars = alignment.characters; + const starts = alignment.character_start_times_seconds; + const ends = alignment.character_end_times_seconds; + const words = []; + let current = null; + for (let i = 0; i < chars.length; i += 1) { + if (/\s/.test(chars[i])) { + current = null; + continue; + } + if (!current) { + current = { s: starts[i], e: ends[i] }; + words.push(current); + } else { + current.e = ends[i]; + } + } + return words.map((w, i) => ({ + i, + s: Number(w.s.toFixed(3)), + e: Number(w.e.toFixed(3)), + })); +} + +export async function handler(event) { + if (event.httpMethod === "OPTIONS") return respond(204, null); + if (event.httpMethod !== "GET") + return respond(405, { message: "Method Not Allowed" }); + + const id = event.queryStringParameters?.id; + if (!id || !ObjectId.isValid(id)) + return respond(400, { message: "Invalid message id." }); + + const voiceId = process.env.PRUTTI_ELEVENLABS_VOICE_ID; + if (!voiceId || !process.env.ELEVENLABS_API_KEY) + return respond(503, { message: "Pruttivox is not configured." }); + + const database = await connect(); + try { + let msg = null; + for (const collection of ["chat-clock", "chat-system"]) { + msg = await database.db + .collection(collection) + .findOne({ _id: new ObjectId(id) }); + if (msg) break; + } + if (!msg || msg.deleted || !msg.user || !msg.text) + return respond(404, { message: "No such message." }); + + // One handle can carry several sister auth-provider ids — match them all. + const speakers = await database.db + .collection("@handles") + .find({ handle: { $regex: `^${VOICE_HANDLE}$`, $options: "i" } }) + .toArray(); + if (!speakers.some((s) => s._id === msg.user)) + return respond(403, { message: `Only @${VOICE_HANDLE} can be voxed.` }); + + // Spoken text: the displayed tokens with unreadable ones swapped out, + // 1:1 by index so the client can highlight the displayed word. + const tokens = [...msg.text.matchAll(/\S+/g)].map((m) => m[0]); + if (!tokens.length) return respond(422, { message: "Nothing to say." }); + const spoken = tokens + .map((t) => (/^(https?:\/\/|www\.)/i.test(t) ? "link" : t)) + .join(" "); + + const hash = crypto + .createHash("sha256") + .update(`pruttivox:${voiceId}:${spoken}`) + .digest("hex"); + const audioKey = `${PREFIX}${hash}.mp3`; + const wordsKey = `${PREFIX}${hash}.json`; + + try { + const cached = await s3.send( + new GetObjectCommand({ Bucket: BUCKET, Key: wordsKey }), + ); + const payload = JSON.parse(await cached.Body.transformToString()); + return respond(200, payload); + } catch (err) { + if (err.name !== "NoSuchKey" && err.$metadata?.httpStatusCode !== 404) { + console.error("Pruttivox cache read error:", err); + } + } + + const response = await fetch( + `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/with-timestamps`, + { + method: "POST", + headers: { + "xi-api-key": process.env.ELEVENLABS_API_KEY, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + text: spoken, + model_id: "eleven_multilingual_v2", + voice_settings: { + stability: 0.38, + similarity_boost: 0.9, + style: 0.48, + use_speaker_boost: true, + speed: 0.98, + }, + }), + }, + ); + if (!response.ok) { + const detail = (await response.text()).slice(0, 300); + console.error(`Pruttivox ElevenLabs error ${response.status}:`, detail); + return respond(502, { message: "Voice synthesis failed." }); + } + const generated = await response.json(); + const audio = Buffer.from(generated.audio_base64, "base64"); + const words = wordsFromAlignment(generated.alignment); + + const payload = { + audio: `${CDN}/${audioKey}`, + duration: words.length ? words[words.length - 1].e : 0, + words, + text: msg.text, + voice: "prutti-ivc", + }; + + await s3.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: audioKey, + Body: audio, + ContentType: "audio/mpeg", + ACL: "public-read", + CacheControl: "public, max-age=31536000", + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: wordsKey, + Body: JSON.stringify(payload), + ContentType: "application/json", + ACL: "public-read", + CacheControl: "public, max-age=31536000", + }), + ); + + // Same ledger the `say` endpoint writes — one row per fresh utterance. + try { + await database.db.collection("sayings").insertOne({ + text: spoken, + provider: "pruttivox", + voice: "prutti-ivc", + cacheKey: audioKey, + url: payload.audio, + messageId: id, + cached: false, + when: new Date(), + }); + } catch (err) { + console.error("⚠️ sayings log failed:", err?.message || err); + } + + return respond(200, payload); + } catch (error) { + console.error("Pruttivox failed:", error); + return respond(500, { message: "An error has occurred." }); + } finally { + await database.disconnect(); + } +} diff --git a/system/public/aesthetic.computer/disks/chat.mjs b/system/public/aesthetic.computer/disks/chat.mjs index 5ef0343f8c..c673af2295 100644 --- a/system/public/aesthetic.computer/disks/chat.mjs +++ b/system/public/aesthetic.computer/disks/chat.mjs @@ -257,6 +257,93 @@ let youtubePreviewCache = new Map(); // Store loaded YouTube thumbnails let youtubeLoadQueue = new Set(); // Track which videos are being loaded let youtubeModalOpen = false; // Track if YouTube modal is open let youtubeModalVideoId = null; // Current video in modal + +// 🗣️ Pruttivox — a "vox" chip on @prutti's messages speaks them in his +// consented voice clone (server: /api/pruttivox, gates in +// marketing/klokkentales/SCORE.md). One message plays at a time; the word +// being spoken lights up. Word i in the server's timing table is displayed +// token i, so the highlight is an index lookup, not a text match. +const VOX_HANDLE = "prutti"; +let vox = null; // { messageId, phase: "loading"|"playing", words, tokens, +// prefix, playing, wordIndex, duration, awaitingProgress } +let voxSimTick = 0; + +function voxable(message) { + return ( + !message.deleted && + message.id && + (message.from || "").replace(/^@/, "").toLowerCase() === VOX_HANDLE + ); +} + +// Whitespace tokens with char offsets — same split as the server tokenizer. +function voxTokenize(text) { + const tokens = []; + const re = /\S+/g; + let m; + while ((m = re.exec(text))) { + tokens.push({ start: m.index, end: m.index + m[0].length }); + } + return tokens; +} + +function voxStop() { + vox?.playing?.kill?.(0.05); + vox = null; +} + +async function voxToggle(message, api) { + if (vox && vox.messageId === message.id) { + voxStop(); + return; + } + voxStop(); + const my = (vox = { + messageId: message.id, + phase: "loading", + wordIndex: -1, + prefix: (message.from || "").length + 1, // fullMessage = from + " " + text + }); + try { + const res = await fetch(`/api/pruttivox?id=${encodeURIComponent(message.id)}`); + if (!res.ok) throw new Error(`pruttivox ${res.status}`); + const data = await res.json(); + if (vox !== my) return; // Superseded while loading. + my.words = data.words || []; + my.tokens = voxTokenize(message.text); + my.duration = data.duration || 0; + const sfx = await api.net.preload(data.audio); + if (vox !== my) return; + my.playing = api.sound.play(sfx, undefined, { + kill: () => { + if (vox === my) vox = null; + }, + }); + my.phase = "playing"; + } catch (err) { + console.warn("🗣️ Vox failed:", err); + if (vox === my) vox = null; + } +} + +// The active word as a synthetic paint element, or null. +function voxWordElementFor(message) { + if ( + !vox || + vox.phase !== "playing" || + vox.messageId !== message.id || + vox.wordIndex < 0 + ) { + return null; + } + const token = vox.tokens?.[vox.wordIndex]; + if (!token) return null; + return { + type: "voxword", + start: token.start + vox.prefix, + end: token.end + vox.prefix, + }; +} let domApi = null; // Store dom API reference for modal let jumpApi = null; // Store jump reference for iOS external-link fallback let sendApi = null; // Store send reference (used by receive() for tape callbacks) @@ -1197,6 +1284,14 @@ function paint( let hoverKey = ""; for (const h of hoveredElements) hoverKey += h.start + ":" + h.end + ","; + // 🗣️ Vox playback recolors this message (chip state + karaoke word), so + // its phase and word index join the cache key — the cache rebuilds as + // the spoken word advances and again when playback ends. + const voxWordEl = voxWordElementFor(message); + if (vox && vox.messageId === message.id) { + hoverKey += "vox:" + vox.phase + ":" + vox.wordIndex + ","; + } + let charPos = 0; // Track position in the full message let lastLineRenderedWidthForThisMessage = 0; // Track actual rendered width of last line for THIS message @@ -1239,6 +1334,18 @@ function paint( // Cache color-coded + shadow lines per message (invalidated by hover state) const needsRebuild = !message._colorLineCache || message._colorLineHoverKey !== hoverKey; if (needsRebuild) { + // The karaoke word joins the element list unless it overlaps a parsed + // element (a spoken URL, say) — two splices on one range corrupt the + // line, so the link keeps its color and the highlight skips that word. + let paintElements = parsedElements; + if ( + voxWordEl && + !parsedElements.some( + (el) => el.start < voxWordEl.end && el.end > voxWordEl.start, + ) + ) { + paintElements = parsedElements.concat(voxWordEl); + } const cachedLines = []; let tempCharPos = charPos; const mt = Array.isArray(theme.messageText) ? theme.messageText : [200, 200, 200]; @@ -1260,7 +1367,7 @@ function paint( let colorCodedLine = lineEscaped; // Find elements that overlap with this line and apply colors (in reverse order) - const lineElements = parsedElements + const lineElements = paintElements .filter(el => el.start < lineEnd && el.end > lineStart) .sort((a, b) => b.start - a.start); @@ -1305,6 +1412,14 @@ function paint( } } else if (element.type === "ytlink") { color = isHovered ? [255, 130, 130] : [255, 70, 70]; // YouTube red chip + } else if (element.type === "voxlink") { + // 🗣️ Amber at rest, pale while the render loads, lime while speaking. + const voxActive = vox && vox.messageId === message.id; + if (voxActive && vox.phase === "loading") color = [255, 235, 180]; + else if (voxActive) color = [190, 255, 80]; + else color = isHovered ? [255, 220, 120] : [255, 170, 60]; + } else if (element.type === "voxword") { + color = [190, 255, 80]; // The word being spoken right now. } else if (element.type === "email") { color = isHovered ? theme.emailHover : theme.email; } else if (element.type === "url") { @@ -3658,6 +3773,11 @@ function act( action: () => jump("out:" + element.text) }; break; + } else if (element.type === "voxlink") { + beep(); + // 🗣️ Speak this message in prutti's voice (tap again to stop). + voxToggle(message, api); + break; } else if (element.type === "painting") { beep(); // Show confirmation modal for painting @@ -4251,6 +4371,41 @@ function act( } function sim({ api, num, send, net, store }) { + // 🗣️ Advance the vox karaoke while a message is being spoken. Progress + // polls round-trip to BIOS, so sample every few sim ticks and never + // stack a second request on an unanswered one. + voxSimTick += 1; + if (vox) api.needsPaint?.(); // Chip + karaoke tints animate while active. + if ( + vox?.phase === "playing" && + vox.playing && + !vox.awaitingProgress && + voxSimTick % 5 === 0 + ) { + const my = vox; + my.awaitingProgress = true; + my.playing + .progress() + .then((p) => { + if (vox !== my) return; + my.awaitingProgress = false; + if (my.playing.killed || (p?.progress ?? 0) >= 0.999) { + vox = null; // Finished — the cache key change clears the tints. + return; + } + const t = (p?.progress || 0) * (p?.duration || my.duration || 0); + let index = -1; + for (let i = 0; i < my.words.length; i += 1) { + if (t >= my.words[i].s) index = i; + else break; + } + my.wordIndex = index; + }) + .catch(() => { + if (vox === my) my.awaitingProgress = false; + }); + } + // ✏️ Closing the composer without submitting cancels a pending re-edit. // `opened` waits out the frames between tapping "edit" and the keyboard // actually coming up, so the edit doesn't self-cancel on arrival. @@ -5035,8 +5190,11 @@ function computeMessagesHeight({ text, screen, typeface }, chat, defaultTypeface // broadcast's popout live chat (message.link, sent by the bridge). const ytSuffix = message.via === "youtube" ? " yt" : ""; + // 🗣️ @prutti's messages carry a "vox" chip that speaks them aloud. + const voxSuffix = voxable(message) ? " vox" : ""; + // Use plain handle for layout (colors applied during rendering) - const fullMessage = message.from + " " + message.text + countSuffix + ytSuffix; + const fullMessage = message.from + " " + message.text + countSuffix + ytSuffix + voxSuffix; const tb = text.box( fullMessage, { x: leftMargin, y: 0 }, @@ -5051,13 +5209,21 @@ function computeMessagesHeight({ text, screen, typeface }, chat, defaultTypeface // AI assistant messages are markdown, not AC chat syntax — skip painting/handle parsing message._parsedElements = message.from === "aa" ? [] : parseMessageElements(fullMessage); if (ytSuffix) { + const ytEnd = fullMessage.length - voxSuffix.length; message._parsedElements.push({ type: "ytlink", - start: fullMessage.length - 2, - end: fullMessage.length, + start: ytEnd - 2, + end: ytEnd, text: message.link || "https://www.youtube.com/@aesthetic.computer/streams", }); } + if (voxSuffix) { + message._parsedElements.push({ + type: "voxlink", + start: fullMessage.length - 3, + end: fullMessage.length, + }); + } // Add height for all lines in the message // Each line is msgRowHeight tall (per-message font height) // Plus add lineGap between lines within the message and after the message -- 2.51.2