From 4b7227fa5f72c29d93443b496e297a67e5a453e0 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Wed, 30 Sep 2026 23:54:25 -0700 Subject: [PATCH] Add opt-in speech captions and local word boundary events --- aesel/context/pieces.md | 15 +++++++ system/public/aesthetic.computer/lib/disk.mjs | 6 +++ .../aesthetic.computer/lib/speech-caption.mjs | 20 ++++++++++ .../public/aesthetic.computer/lib/speech.mjs | 39 ++++++++++++++----- tests/speech/captions.test.cjs | 36 +++++++++++++++++ 5 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 system/public/aesthetic.computer/lib/speech-caption.mjs create mode 100644 tests/speech/captions.test.cjs diff --git a/aesel/context/pieces.md b/aesel/context/pieces.md index d83afa6a53..60ad3d8063 100644 --- a/aesel/context/pieces.md +++ b/aesel/context/pieces.md @@ -287,3 +287,18 @@ source piece-name # Fork existing piece `piece-name:param1:param2`, user pieces via `@handle/piece-name`, QR sharing via `share piece-name` - Leaves (pieces) stay small and can be loop-generated — see `HAND.md` + +### Spoken narration and captions + +Use the top-level `speak` API for a spoken voice; `speaker` exposes audio analysis and is not text-to-speech. A request for a narrator requires audible speech, not just drawn text. + +```js +export function act({event, speak}) { + if (event.is("touch")) speak("A small bird sings from its branch.", "male", "local", {captions: true, lang: "en-US", rate: 0.9}); + // event.is("speech:completed") signals when another line can begin. +} +``` + +Call once from an interaction or on `speech:completed`, never every frame. Local mode uses device/browser voices without a network round trip. `speech:started` supplies `event.content.text`; `speech:word` supplies `{text, word, charIndex, charLength, elapsed}` for synchronized animation. Word boundaries depend on browser/voice support; captions still show the full utterance without them. `speech:error` reports failures. + +`{captions: true}` enables the shared caption overlay. Local voices highlight actual reported word boundaries. Cloud mode can show the utterance but currently has no word alignment; do not invent timing. Cloud voices use `speak(text, "male:18", "cloud", {captions: true})` and require the hosted voice service. Generated narration is synthesized voice, distinct from Walkieware's original microphone playback. Do not invent factual claims about a depicted species; use scene-grounded narration unless facts were supplied or verified. diff --git a/system/public/aesthetic.computer/lib/disk.mjs b/system/public/aesthetic.computer/lib/disk.mjs index cee224f8c9..cff7571ac2 100644 --- a/system/public/aesthetic.computer/lib/disk.mjs +++ b/system/public/aesthetic.computer/lib/disk.mjs @@ -11299,6 +11299,12 @@ async function makeFrame({ data: { type, content } }) { return; } + // Speech boundaries let pieces synchronize their own captions or animation. + if (type === "speech:started" || type === "speech:word") { + actAlerts.push({ name: type, content }); + return; + } + // 🗣️ An act that fires when an utterance has ended in the Web Speech API. if (type === "speech:completed") { actAlerts.push("speech:completed"); diff --git a/system/public/aesthetic.computer/lib/speech-caption.mjs b/system/public/aesthetic.computer/lib/speech-caption.mjs new file mode 100644 index 0000000000..126ee5ade8 --- /dev/null +++ b/system/public/aesthetic.computer/lib/speech-caption.mjs @@ -0,0 +1,20 @@ +// Optional speech captions. Rendering uses text nodes, never interpreted markup. +let active; +export function speechCaption(words, enabled = false) { + if (!enabled) return {boundary(){},end(){}}; + active?.remove(); + const caption=document.createElement('div'); + caption.dataset.acSpeechCaption=''; + caption.setAttribute('aria-hidden','true'); // The same words are already audible. + Object.assign(caption.style,{position:'fixed',left:'5%',right:'5%',bottom:'max(12px, env(safe-area-inset-bottom))',padding:'10px 14px',background:'rgba(0,0,0,.8)',color:'white',font:'600 clamp(18px, 4vw, 28px) monospace',textAlign:'center',lineHeight:'1.3',pointerEvents:'none',zIndex:'2147483646',maxHeight:'35vh',overflow:'hidden'}); + caption.textContent=words;document.body.append(caption);active=caption; + return { + boundary(index,length){ + if(active!==caption||!Number.isInteger(index)||index<0||index>=words.length)return; + const end=Math.min(words.length,index+Math.max(0,length||0)); + const word=document.createElement('span');word.style.color='#fff176';word.textContent=words.slice(index,end); + caption.replaceChildren(document.createTextNode(words.slice(0,index)),word,document.createTextNode(words.slice(end))); + }, + end(){caption.remove();if(active===caption)active=null;} + }; +} diff --git a/system/public/aesthetic.computer/lib/speech.mjs b/system/public/aesthetic.computer/lib/speech.mjs index 9007d63d7d..348f133360 100644 --- a/system/public/aesthetic.computer/lib/speech.mjs +++ b/system/public/aesthetic.computer/lib/speech.mjs @@ -1,3 +1,4 @@ +import { speechCaption } from "./speech-caption.mjs"; // Speech, 23.08.09.15.50 // A thin API over the web speech synthesis API, // with cloud support. @@ -39,7 +40,11 @@ if (synth?.onvoiceschanged !== undefined) { // and returns a mp3 file. function speak(words, voice, mode = "local", opts = {}) { - if (!synth) console.warn("No speech is supported on this platform."); + voice ||= "female"; + if (mode === "local" && !synth) { + window.acSEND?.({type:"speech:error",content:{status:0,provider:"local",message:"Speech synthesis unavailable"}}); + return; + } if (mode === "local") { if (synth.speaking) { console.error("🗣️ Already speaking..."); @@ -52,17 +57,30 @@ function speak(words, voice, mode = "local", opts = {}) { if (voice.startsWith("female")) voiceIndex = 9; if (voice.startsWith("male")) voiceIndex = 10; - utterance.voice = voices[voiceIndex]; + utterance.voice = voices.find(v => v.name === voice) || voices[voiceIndex] || voices.find(v => v.default) || voices[0]; + if(opts.lang)utterance.lang=opts.lang; + if(Number.isFinite(opts.rate))utterance.rate=Math.max(0.1,Math.min(10,opts.rate)); + let caption; + utterance.onstart = () => { + caption=speechCaption(words,opts.captions===true); + window.acSEND?.({type:"speech:started",content:{text:words,voice,mode}}); + }; + utterance.onboundary = event => { + if(event.name!=="word")return; + const index=event.charIndex; + const length=event.charLength || words.slice(index).match(/^\S+/)?.[0].length || 0; + caption?.boundary(index,length); + window.acSEND?.({type:"speech:word",content:{text:words,word:words.slice(index,index+length),charIndex:index,charLength:length,elapsed: event.elapsedTime}}); + }; + utterance.onend = () => { + caption?.end(); + if(!opts.skipCompleted)window.acSEND?.({type:"speech:completed"}); + }; // console.log("Speaking:", words, utterance.voice); - if (!opts.skipCompleted) { - utterance.onend = function (event) { - // console.log("🗣️ Speech completed:", event); - window.acSEND({ type: "speech:completed" }); // Send to piece. - }; - } - utterance.onerror = function (event) { + caption?.end(); + window.acSEND?.({type:"speech:error",content:{status:0,provider:"local",message:event.error}}); console.error("🗣️ Speech failure:", event); }; @@ -106,6 +124,8 @@ function speak(words, voice, mode = "local", opts = {}) { return; } + const caption=speechCaption(words,opts.captions===true); + window.acSEND?.({type:"speech:started",content:{text:words,voice,mode}}); speakAPI.playSfx( id, label, @@ -117,6 +137,7 @@ function speak(words, voice, mode = "local", opts = {}) { targetDuration: opts.targetDuration, // Time stretch to target duration, then pitch shift }, () => { + caption.end(); if (!opts.skipCompleted) window.acSEND({ type: "speech:completed" }); }, ); diff --git a/tests/speech/captions.test.cjs b/tests/speech/captions.test.cjs new file mode 100644 index 0000000000..91920bb149 --- /dev/null +++ b/tests/speech/captions.test.cjs @@ -0,0 +1,36 @@ +const test=require('node:test'); +const assert=require('node:assert/strict'); +const {readFile}=require('node:fs/promises'); +const {resolve}=require('node:path'); +const puppeteer=require('puppeteer'); +test('local speech captions follow actual boundaries, report events, and clean up',async()=>{ + const browser=await puppeteer.launch({headless:true,executablePath:'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'}); + try { + const page=await browser.newPage(); + await page.setRequestInterception(true); + page.on('request',async r=>{ + const path=new URL(r.url()).pathname; + if(path==='/')return r.respond({status:200,contentType:'text/html',body:''}); + if(path==='/helpers.mjs')return r.respond({status:200,contentType:'text/javascript',body:'export const utf8ToBase64 = btoa;'}); + if(!['/speech.mjs','/speech-caption.mjs'].includes(path))return r.abort(); + return r.respond({status:200,contentType:'text/javascript',body:await readFile(resolve('system/public/aesthetic.computer/lib','.'+path),'utf8')}); + }); + await page.goto('http://speech.test/'); + const result=await page.evaluate(async()=>{ + window.events=[];window.acSEND=e=>events.push(e); + Object.defineProperty(window,'speechSynthesis',{value:{getVoices:()=>[],speaking:false,speak:u=>window.last=u},configurable:true}); + window.SpeechSynthesisUtterance=class {constructor(text){this.text=text;}}; + const {speak}=await import('/speech.mjs'); + speak('Go, go! bird','male','local',{captions:true}); + last.onstart();last.onboundary({name:'word',charIndex:4,charLength:2,elapsedTime:0.5}); + const node=document.querySelector('[data-ac-speech-caption]'); + const during={text:node.textContent,highlight:node.querySelector('span').textContent,noMarkup:!node.querySelector('b'),events:[...events]}; + last.onend();const cleaned=!document.querySelector('[data-ac-speech-caption]'); + speak('again',undefined,'local',{captions:true});last.onstart();last.onerror({error:'interrupted'}); + return {during,cleaned,afterError:!document.querySelector('[data-ac-speech-caption]'),events}; + }); + assert.equal(result.during.text,'Go, go! bird');assert.equal(result.during.highlight,'go');assert.ok(result.during.noMarkup); + assert.equal(result.during.events[1].content.charIndex,4);assert.ok(result.cleaned&&result.afterError); + assert.ok(result.events.some(e=>e.type==='speech:completed'));assert.ok(result.events.some(e=>e.type==='speech:error')); + } finally {await browser.close();} +}); -- 2.51.2