From e880c059f44554972e70bd55fb025e9e46a055b1 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Sat, 3 Oct 2026 10:43:20 -0700 Subject: [PATCH] Add Whistlegraph pixel size, checked edits, and MP4 export progress --- apple/whistlegraph/.gitignore | 1 + apple/whistlegraph/Info.plist | 1 + apple/whistlegraph/README.md | 24 +- apple/whistlegraph/Resources/Web/engine.mjs | 66 +++--- .../whistlegraph/Resources/Web/story-tape.js | 38 +++ .../Sources/NativeScreenFixture.swift | 9 +- .../whistlegraph/Sources/StoryControls.swift | 82 +++++++ apple/whistlegraph/Sources/StoryExport.swift | 218 ++++++++++++++++++ .../Sources/StoryMovieSheet.swift | 50 ++++ apple/whistlegraph/Sources/StoryTape.swift | 12 + .../Sources/VersionNarrator.swift | 158 ++++++++----- .../Sources/WhistlegraphAccount.swift | 15 ++ .../Sources/WhistlegraphApp.swift | 26 ++- .../Sources/WhistlegraphHeader.swift | 21 +- .../Sources/WhistlegraphScreen.swift | 83 +++++-- apple/whistlegraph/Tests/canvas-tape.test.mjs | 31 +++ .../whistlegraph/Tests/native-bridge.test.cjs | 24 +- apple/whistlegraph/Tests/pixel-size.test.mjs | 35 +++ .../Tests/wwDabev-investigation.json | 16 ++ .../UITests/HeaderSheetsTests.swift | 22 ++ .../UITests/StoryCardsTests.swift | 80 +++++++ .../Whistlegraph.xcodeproj/project.pbxproj | 24 +- apple/whistlegraph/bundle.mjs | 2 + apple/whistlegraph/project.yml | 2 +- .../aesthetic.computer/lib/canvas-tape.mjs | 15 ++ 25 files changed, 903 insertions(+), 152 deletions(-) create mode 100644 apple/whistlegraph/Resources/Web/story-tape.js create mode 100644 apple/whistlegraph/Sources/StoryControls.swift create mode 100644 apple/whistlegraph/Sources/StoryExport.swift create mode 100644 apple/whistlegraph/Sources/StoryMovieSheet.swift create mode 100644 apple/whistlegraph/Sources/StoryTape.swift create mode 100644 apple/whistlegraph/Tests/canvas-tape.test.mjs create mode 100644 apple/whistlegraph/Tests/pixel-size.test.mjs create mode 100644 apple/whistlegraph/Tests/wwDabev-investigation.json create mode 100644 apple/whistlegraph/UITests/StoryCardsTests.swift create mode 100644 system/public/aesthetic.computer/lib/canvas-tape.mjs diff --git a/apple/whistlegraph/.gitignore b/apple/whistlegraph/.gitignore index d7ff1e2b55..51e4fd304b 100644 --- a/apple/whistlegraph/.gitignore +++ b/apple/whistlegraph/.gitignore @@ -10,3 +10,4 @@ Tests/threads/ Tests/native-shell/ Tests/scenes/ Tests/spaces/ +Resources/Web/canvas-tape.mjs diff --git a/apple/whistlegraph/Info.plist b/apple/whistlegraph/Info.plist index 4f48683f4b..9b4c8e34ef 100644 --- a/apple/whistlegraph/Info.plist +++ b/apple/whistlegraph/Info.plist @@ -1,5 +1,6 @@ CFBundleDevelopmentRegionenCFBundleExecutable$(EXECUTABLE_NAME)CFBundleIdentifier$(PRODUCT_BUNDLE_IDENTIFIER)CFBundleNameWhistlegraphCFBundleDisplayNameWhistlegraphCFBundlePackageTypeAPPLCFBundleShortVersionString$(MARKETING_VERSION)CFBundleVersion$(CURRENT_PROJECT_VERSION)UILaunchScreen +NSPhotoLibraryAddUsageDescriptionSave the story videos you export to your camera roll. NSMicrophoneUsageDescriptionHold the talk button to tell Whistlegraph what to make. Recording stops when you release it. NSSpeechRecognitionUsageDescriptionWhistlegraph turns your spoken request into text. Speech recognition runs on this iPhone. Recordings stay on this phone for playback. UISupportedInterfaceOrientationsUIInterfaceOrientationPortraitUIUserInterfaceStyleDarkITSAppUsesNonExemptEncryption diff --git a/apple/whistlegraph/README.md b/apple/whistlegraph/README.md index 5316e1343c..265801767d 100644 --- a/apple/whistlegraph/README.md +++ b/apple/whistlegraph/README.md @@ -6,6 +6,14 @@ From this directory, run `./run.sh device` to bundle, build, install, and open t Typing uses AC's `compkey` sample and QWERTY pitch mapping. Enter sends the prompt, pasted line breaks become spaces, and the limit is 96 characters. Account settings control key and button sounds together. +## Story cards + +The stacked-cards button opens the selected version's ancestry as a vertical story. Pause, previous, next, and close leave the editing selection intact. The picture keeps its 4:3 aspect; its version and spoken caption sit directly below it with space reserved around them for social-app overlays. + +Export uses the shared AC `canvas-tape.mjs` hardware encoder, also used by BIOS HD tapes. An internal 1080×1920 canvas composes only program pixels and captions; native AVFoundation adds the original local utterances or file-rendered speech. No screen or microphone recording is used. Export runs through the story in the foreground, with a ten-minute/256 MB limit, cancellation, and a local MP4 preview offering Save video (Photos add-only permission) and Share. Program-generated synth audio is not yet mixed into this story export. + +`StoryCardsTests` checks branch navigation, caption placement, selection restoration, and the on-device MP4 flow. The `story` debug fixture uses separate local storage and no cloud thread; it writes `Documents/story-export-test.mp4` for frame/audio inspection. + ## Update compatibility Whistlegraph updates the existing Walkieware installation. Keep these persisted and deployed contracts until an explicit migration replaces them: @@ -29,13 +37,19 @@ node apple/whistlegraph/Tests/native-bridge.test.cjs --native-shell The browser check uses Puppeteer and Chrome with mock inference; it makes no model requests. Native UI tests live in the `WhistlegraphUITests` scheme. -## Checked edits (experimental) +## Edit checks + +Every generated edit runs syntax/API checks and waits for matching-source runtime feedback before the visual review. Client errors trigger at most one repair across code and visual checks. A candidate that still fails restores the previous saved version; its failure remains visible after the restored frame paints. Missing or stale evidence cannot trigger a paid repair or save a version. There is no opt-out setting. + +The initial generation allows four tool rounds and one output continuation, with two repair rounds and no repair continuation. Provider calls use a 4,096-token output limit. Code generation/checking has a 75-second deadline; visual review has a separate 90-second deadline. These bound work, not cost. A painted frame is execution evidence, not visual acceptance. + +## Pixel size -Account settings → **Check edits (experimental)** enables a deterministic edit contract containing the selected branch requests, caption, and known API failures. It defaults off; the existing DeepSeek path with thinking disabled remains the default. Debug launches can set `WALKIE_COMPILED_TASK=1`. +Tap the piece name → Pixel size: **1×, 2×, 3×, 4×**. The default is **2×**, matching `bios.mjs`. Bigger values make bigger pixels. Changes resize the live preview without creating a version or restarting the piece; the preference survives app launches. -The experiment checks JavaScript syntax, direct unsupported HSL drawing calls, invalid `ink.box`-style calls, and matching-source runtime feedback. Each render carries a SHA-256 source hash and render ID; stale feedback is ignored. An actionable failure after a completed generation permits one repair. A failed or unverified result restores the previous piece without committing a version. A painted frame is execution evidence, not visual or semantic acceptance; animation timing still needs observation. +## Story export -Limits: four initial tool rounds with one output continuation, two repair rounds without continuations, 4,096 output tokens per provider call, and a 75-second generation/validation deadline. This allows at most seven provider calls per attempt; it is not a monetary cap. A resumed request is a new attempt. Existing deterministic local edits still bypass inference. +The story share button shows preparation, card-rendering progress, audio preparation, and the encoder's MP4 percentage. Cancel stops the export and returns to the story. The completed video opens with Save video and Share; saving to Photos displays activity until it finishes. Video capture runs at the story's playback speed. ## Attempt receipts @@ -53,7 +67,7 @@ node apple/whistlegraph/Tests/native-bridge.test.cjs --native-shell --checked-ed node apple/whistlegraph/Tests/native-bridge.test.cjs --native-shell --checked-edits --repair-fails ``` -Both experiment browser checks use mock inference. They verify the contract, repair limit, rollback, render identity, and receipt fields without spending braincells. +The browser checks use mock inference. They verify the contract, repair limit, rollback, render identity, and receipt fields without spending braincells. ## Chalk with words and sound diff --git a/apple/whistlegraph/Resources/Web/engine.mjs b/apple/whistlegraph/Resources/Web/engine.mjs index 3c25a364cb..8e86d9d798 100644 --- a/apple/whistlegraph/Resources/Web/engine.mjs +++ b/apple/whistlegraph/Resources/Web/engine.mjs @@ -25,9 +25,9 @@ import * as vfs from '/easel/phone/shim/fs.mjs'; const post = body => window.webkit.messageHandlers.walkie.postMessage({id:'engine',...body}); const benchmark=(event,fields={})=>{post({action:'benchmark',event,fields});window.__walkiewareSequenceEvent?.(event,fields);}; const file = '/piece/walkieware.mjs'; -const storageKey=window.__walkiewareSpace?'walkieware-space-source':window.__walkiewareLocalSequence?'walkieware-local-source':window.__walkiewareSequence?'walkieware-sequence-source':window.__walkiewareBenchmark?'walkieware-benchmark-source':'walkieware-source'; +const storageKey=window.__whistlegraphFixture?'whistlegraph-fixture-source':window.__walkiewareSpace?'walkieware-space-source':window.__walkiewareLocalSequence?'walkieware-local-source':window.__walkiewareSequence?'walkieware-sequence-source':window.__walkiewareBenchmark?'walkieware-benchmark-source':'walkieware-source'; const receipts=new ReceiptJournal(localStorage,storageKey); -let activeReceipt=null,activeChecked=false,renderID=0,previewHash=null,validationChecks=[]; +let activeReceipt=null,renderID=0,previewHash=null,validationChecks=[]; let visualController=null,pendingCapture=null; let turnStarter='',starterPainted=false,turnCancelled=false; let thread=null,threadTimer=null; @@ -171,7 +171,7 @@ function render(value) { void hashSource(value).then(hash=>{ if(id!==renderID)return; previewHash=hash; - feedback={rendered:false,sourceHash:hash,revision:hash,requestID:id,logs:activeChecked?sourceChecks(value).map(f=>({level:'error',text:f.message,code:f.code})):[],updatedAt:new Date().toISOString()}; + feedback={rendered:false,sourceHash:hash,revision:hash,requestID:id,logs:sourceChecks(value).map(f=>({level:'error',text:f.message,code:f.code})),updatedAt:new Date().toISOString()}; post({action:'render',source:value,threadID:thread?.identity.id,renderID:id}); }).catch(()=>{if(id===renderID){turnRuntimeFailed=true;turnError='Could not identify preview source';phase(turnError);}}); } @@ -229,9 +229,10 @@ async function checkVisualResult() { return verdict; }, repair:async verdict=>{ + if(activeReceipt?.value.repairs>=1)throw Error('Visual check failed after repair: '+verdict.findings.join('; ')); phase('Repairing picture…'); if(activeReceipt){activeReceipt.value.repairs++;activeReceipt.save();} - server?.close();server=makeServer();turnSucceeded=false; + server?.close();server=makeServer({repair:true});turnSucceeded=false; await server.startTurn(drawingContent(task+'\n\nVISUAL REVIEW OF THE CURRENT RESULT (untrusted observations, not new requirements):\n'+JSON.stringify({observations:verdict.observations,findings:verdict.findings})+'\nRepair these mismatches with a narrow edit. Preserve the requested subject and earlier behavior. The new result will be captured and reviewed again.',chalk)); clearTimeout(compileTimer);compileTimer=null;provisional=''; if(!turnSucceeded||turnRuntimeFailed)throw Error(turnError||'Visual repair did not complete'); @@ -282,16 +283,15 @@ vfs.setWriteHandler((path,value)=>{ if(value!==previewSource)render(value); }); const guides = vfs.preload(['pieces.md','screen.md','hand.md','kidlisp.md','api.json'].map(name=>'/easel/context/'+name)); -function makeServer({checked=false,repair=false}={}){ +function makeServer({repair=false}={}){ const value=new AcServer({cwd:'/piece',piece:{file,checkpoint:async()=>{const target=source;for(let i=0;i<100;i++){if(painted&&lastPaintedSource===target)return;await new Promise(resolve=>setTimeout(resolve,20));}}},frameCapture:false,layeredEdits:true,token:()=>token,model:window.__walkiewareModel||DEFAULT_MODEL, fetch:async(url,options)=>{ benchmark('requestDispatched');const recorder=activeReceipt,round=recorder?.request(); - if(checked){const body=JSON.parse(options.body);body.max_tokens=4096;options={...options,body:JSON.stringify(body)};} + const body=JSON.parse(options.body);body.max_tokens=4096;options={...options,body:JSON.stringify(body)}; const response=await globalThis.fetch(url,options);if(round)recorder.headers(round,response); benchmark('inferenceHeaders',{status:response.status});return response; - },preview:true,rounds:checked?(repair?2:4):12,outputContinuations:checked?(repair?0:1):4,reasoning:{effort:'none'},thinking:{type:'disabled'}, + },preview:true,rounds:repair?2:4,outputContinuations:repair?0:1,reasoning:{effort:'none'},thinking:{type:'disabled'}, developerInstructions:GENERATION_INSTRUCTIONS}); - value.checked=checked; value.runtimeFeedback=()=>feedback; value.on('notification',({method,params})=>{ activeReceipt?.notify(method,params); @@ -332,9 +332,9 @@ async function ask(text,displayText=text,advice=null,starter=null,localText=text document.body.classList.add('live-mode');phase('Sending…');log('Submitted'); timer=setInterval(()=>$('live-time').textContent=((performance.now()-started)/1000).toFixed(1)+'s',100); let noChange=false; - activeChecked=window.__whistlegraphCheckedEdits===true;validationChecks=[];runtimeErrors.length=0; + validationChecks=[];runtimeErrors.length=0; try{ - activeReceipt=new AttemptReceipt({requestID:activeAttempt.id,parent:turnParent,parentHash:await hashSource(previous),path:activeChecked?'compiled':'current',model:window.__walkiewareModel||DEFAULT_MODEL,journal:receipts}); + activeReceipt=new AttemptReceipt({requestID:activeAttempt.id,parent:turnParent,parentHash:await hashSource(previous),path:'compiled',model:window.__walkiewareModel||DEFAULT_MODEL,journal:receipts}); const drawing=inputData(text)?.drawing; const chalkImage=drawing?drawingImage(drawing):null; text=inferenceRequest(text); @@ -375,34 +375,27 @@ async function ask(text,displayText=text,advice=null,starter=null,localText=text const missing=await guides;if(missing.length)throw Error('Bundled piece guides are unavailable: '+missing.join(', ')); benchmark('guidesReady'); vfs.mount(file,source||'export function paint({wipe}) { wipe("black"); }'); - if(activeChecked){ - const prompt=compileEditContract({request:text,...selectedBranch(versions.value),source}); - const deadline=setTimeout(()=>{turnCancelled=true;turnError='Edit check timed out';server?.interrupt();},75000); - try{ - const result=await runEditExperiment({prompt,cancelled:()=>turnCancelled, - onRepair:()=>{activeReceipt.value.repairs=1;activeReceipt.save();phase('Repairing…');}, - generate:async(task,repair)=>{server?.close();server=makeServer({checked:true,repair});turnSucceeded=false;await server.startTurn(drawingContent(task,chalkImage));return turnSucceeded;}, - inspect:async()=>{ - const target=source,hash=await hashSource(target); - for(let i=0;i<100&&!turnCancelled&&(!painted||lastPaintedSource!==target);i++)await new Promise(resolve=>setTimeout(resolve,20)); - return validateCandidate(target,feedback,hash); - }}); - if(result.validation){ - validationChecks=result.validation.findings.map(f=>({code:f.code,sourceHash:result.validation.sourceHash})); - if(!result.validation.passed){turnSucceeded=false;turnRuntimeFailed=true;turnError='Edit check failed: '+result.validation.findings.map(f=>f.code).join(', ');} - } - }finally{clearTimeout(deadline);} - }else{ - // Branch context already carries history. Retire visual conversations at - // turn boundaries so an old sketch cannot masquerade as the current one. - if(server?.checked||server?.hasChalk||chalkImage){server?.close();server=null;} - server??=makeServer();server.hasChalk=!!chalkImage; - await server.startTurn(drawingContent(contextualRequest(versions.value,text),chalkImage)); - } + const prompt=compileEditContract({request:text,...selectedBranch(versions.value),source}); + const deadline=setTimeout(()=>{turnCancelled=true;turnError='Edit check timed out';server?.interrupt();},75000); + try{ + const result=await runEditExperiment({prompt,cancelled:()=>turnCancelled, + onRepair:()=>{activeReceipt.value.repairs=1;activeReceipt.save();phase('Repairing…');}, + generate:async(task,repair)=>{server?.close();server=makeServer({repair});turnSucceeded=false;await server.startTurn(drawingContent(task,chalkImage));return turnSucceeded;}, + inspect:async()=>{ + const target=source,hash=await hashSource(target); + for(let i=0;i<100&&!turnCancelled&&(!painted||lastPaintedSource!==target);i++)await new Promise(resolve=>setTimeout(resolve,20)); + return validateCandidate(target,feedback,hash); + }}); + if(result.validation){ + validationChecks=result.validation.findings.map(f=>({code:f.code,sourceHash:result.validation.sourceHash})); + if(!result.validation.passed){turnSucceeded=false;turnRuntimeFailed=true;turnError='Edit check failed: '+result.validation.findings.map(f=>f.code).join(', ');} + } + }finally{clearTimeout(deadline);} + }catch(error){turnError=error.message;phase('Could not start');log(error.message);} finally{ if(noChange){await finishReceipt('unchanged');localStorage.removeItem(storageKey+'-inflight');activeAttempt=null;lastAttempt={...lastAttempt,status:'unchanged'};end();phase('Already there');benchmark('localEditUnchanged');return;} - if(!activeChecked&&!turnSucceeded&&!turnCancelled&&turnStarter&&starterPainted){ + if(!turnSucceeded&&!turnRuntimeFailed&&!turnCancelled&&turnStarter&&starterPainted){ // A failed refinement must not erase the useful, verified first drawing. source=turnStarter;vfs.mount(file,source); if(previewSource!==source)render(source); @@ -447,14 +440,13 @@ window.walkiewareEngineEvent=event=>{ if(pendingCapture?.id===event.captureID)pendingCapture.finish(event.error?Error(event.error):null,event); return; } - if(event.kind==='inferenceSettings')window.__whistlegraphCheckedEdits=event.checkedEdits===true; if(event.kind==='account') {token=event.token;if(token){musicalSocket.resume();thread?.resume();}else{musicalSocket.suspend();thread?.suspend();}window.walkiewareAccountReady=!!token;accountIdentity(token);if(token&&pending)void ask(pending);else void resumeAttempt();} if(event.kind==='error'){phase('Sign-in needed');log(event.text);window.walkiewareWorkFinished?.();} if(event.kind==='previewReady'){ready=true;log('AC runtime ready');} if(event.kind==='previewEvent'){ if(event.event?.sourceHash!==previewHash||event.event?.requestID!==renderID)return; activeReceipt?.observe(event.event); - if(event.event.kind==='painted'){if(turnStarter&&previewSource===turnStarter&&!starterPainted){starterPainted=true;benchmark('starterPainted');}if(previewSource.trimEnd()!==previous.trimEnd())window.__walkiewareSequenceEvent?.('painted');painted=true;lastPaintedSource=previewSource;if(busy&&activeAttempt&&source===previewSource&&!turnRuntimeFailed){activeAttempt={...activeAttempt,checkpoint:source};try{saveAttempt(localStorage,storageKey,activeAttempt);}catch(error){log('Could not persist checkpoint: '+error.message);}}feedback={...feedback,rendered:true,updatedAt:new Date().toISOString()};if(activeReceipt&&previewSource.trimEnd()!==previous.trimEnd())activeReceipt.painted();log('Checkpoint painted');phase(busy?'Building…':'Ready to play');if(narrationPending!==null){post({action:'narrationReady',version:narrationPending});narrationPending=null;}void resumeAttempt();} + if(event.event.kind==='painted'){if(turnStarter&&previewSource===turnStarter&&!starterPainted){starterPainted=true;benchmark('starterPainted');}if(previewSource.trimEnd()!==previous.trimEnd())window.__walkiewareSequenceEvent?.('painted');painted=true;lastPaintedSource=previewSource;if(busy&&activeAttempt&&source===previewSource&&!turnRuntimeFailed){activeAttempt={...activeAttempt,checkpoint:source};try{saveAttempt(localStorage,storageKey,activeAttempt);}catch(error){log('Could not persist checkpoint: '+error.message);}}feedback={...feedback,rendered:true,updatedAt:new Date().toISOString()};if(activeReceipt&&previewSource.trimEnd()!==previous.trimEnd())activeReceipt.painted();log('Checkpoint painted');phase(busy?'Building…':lastAttempt?.status==='failed'?'Could not finish · previous version restored':'Ready to play');if(narrationPending!==null){post({action:'narrationReady',version:narrationPending});narrationPending=null;}void resumeAttempt();} if(event.event.kind==='invalidated'){turnRuntimeFailed=true;window.__walkiewareSequenceEvent?.('runtimeError',{message:'Preview invalidated'});painted=false;feedback={...feedback,rendered:false,logs:[...(feedback?.logs||[]),{level:'error',text:'Preview invalidated'}],updatedAt:new Date().toISOString()};log('Preview failed; inspect activity');phase('Preview error');if(lastPaintedSource && lastPaintedSource!==previewSource){render(lastPaintedSource);log('Restored last painted checkpoint');}} if(event.event.kind==='console'&&['error','warn'].includes(event.event.event?.level)){ const entry={level:event.event.event.level,text:event.event.event.message||'Runtime error'}; diff --git a/apple/whistlegraph/Resources/Web/story-tape.js b/apple/whistlegraph/Resources/Web/story-tape.js new file mode 100644 index 0000000000..8297201925 --- /dev/null +++ b/apple/whistlegraph/Resources/Web/story-tape.js @@ -0,0 +1,38 @@ +// Injected into the piece frame with the shared AC canvas-tape encoder. +(() => { + if (window === window.top || location.origin !== 'https://aesthetic.computer') return; + let recorder, timer, session, state={}, delivery=Promise.resolve(), failure=null; + const post = body => window.webkit.messageHandlers.walkie.postMessage({action:'storyTape',session,...body}); + const canvas=document.createElement('canvas'); canvas.width=1080;canvas.height=1920; + const ctx=canvas.getContext('2d'); + function draw() { + ctx.fillStyle='black';ctx.fillRect(0,0,1080,1920);ctx.imageSmoothingEnabled=false; + const layers=[document.querySelector('canvas[data-ac-visit-canvas]'), ...document.querySelectorAll('canvas[data-type="webgl-composite"],canvas[data-type="webgpu"],canvas[data-type="hd"]')]; + let drew=false; + for(const layer of layers) if(layer?.width && getComputedStyle(layer).display!=='none') { ctx.drawImage(layer,76,270,928,696);drew=true; } + if(!drew)throw Error('The piece canvas is unavailable.'); + ctx.fillStyle='#aaa';ctx.font='40px monospace';ctx.fillText('v'+(state.version??''),76,1030); + ctx.fillStyle='white';ctx.font='58px WhistlegraphComic, sans-serif'; + const words=String(state.caption||'').split(/\s+/);let line='',y=1120; + for(const word of words){const next=line?line+' '+word:word;if(ctx.measureText(next).width>928&&line){ctx.fillText(line,76,y);y+=72;line=word;if(y>1320)break;}else line=next;} + if(y<=1320)ctx.fillText(line,76,y); + } + window.whistlegraphStoryTape={ + update(value){state=value;}, + async start(id){ + await window.whistlegraphStoryFont; + if(recorder)throw Error('A tape is already exporting.');session=id;failure=null;delivery=Promise.resolve(); + draw();recorder=createCanvasTapeRecorder(canvas,{fps:30,mp4Only:true}); + recorder.ondataavailable=e=>{if(!e.data.size)return;delivery=delivery.then(async()=>{ + const bytes=new Uint8Array(await e.data.arrayBuffer()); + for(let i=0;i{failure=error;});}; + recorder.onerror=e=>{failure=e.error||Error('Canvas tape encoding failed');}; + recorder.onstop=()=>{clearInterval(timer);delivery.then(()=>post(failure?{kind:'error',error:failure.message}:{kind:'done'}));recorder=null;}; + timer=setInterval(()=>{try{draw();}catch(error){failure=error;clearInterval(timer);if(recorder?.state!=='inactive')recorder.stop();}},1000/30); + recorder.start(1000); + }, + stop(){if(recorder?.state!=='inactive')recorder?.stop();}, + cancel(){clearInterval(timer);if(recorder){recorder.ondataavailable=null;recorder.onstop=null;if(recorder.state!=='inactive')recorder.stop();recorder=null;}session=null;} + }; +})(); diff --git a/apple/whistlegraph/Sources/NativeScreenFixture.swift b/apple/whistlegraph/Sources/NativeScreenFixture.swift index ad9d06c923..8c6f372c35 100644 --- a/apple/whistlegraph/Sources/NativeScreenFixture.swift +++ b/apple/whistlegraph/Sources/NativeScreenFixture.swift @@ -1,14 +1,15 @@ #if DEBUG import Foundation -// Simulator-only screen fixtures. Never connect a fixture to a cloud thread. +// Isolated screen fixtures; only the story export fixture runs on a phone. +// Never connect a fixture to a cloud thread. enum NativeScreenFixture { static var mode: String { ProcessInfo.processInfo.environment["WALKIE_NATIVE_SCREEN_FIXTURE"] ?? "" } static var enabled: Bool { #if targetEnvironment(simulator) - return ["history", "recording", "gestures", "working"].contains(mode) + return ["history", "recording", "gestures", "working", "story"].contains(mode) #else - return false + return mode == "story" #endif } static var script: String { @@ -26,7 +27,7 @@ enum NativeScreenFixture { let ledger: [String: Any] = ["format": 1, "head": 3, "versions": versions] let json = String(data: try! JSONSerialization.data(withJSONObject: ledger), encoding: .utf8)! let busy = mode == "working" ? "window.__walkiewareFixtureBusy='Someone is eating them';" : "" - return busy + "window.__walkiewareDisableThread=true;localStorage.setItem('walkieware-source-versions',JSON.stringify(\(json)));localStorage.setItem('walkieware-source',\(json).versions[3].source);" + return busy + "window.__whistlegraphFixture=true;window.__walkiewareDisableThread=true;localStorage.setItem('whistlegraph-fixture-source-versions',JSON.stringify(\(json)));localStorage.setItem('whistlegraph-fixture-source',\(json).versions[3].source);" } } #endif diff --git a/apple/whistlegraph/Sources/StoryControls.swift b/apple/whistlegraph/Sources/StoryControls.swift new file mode 100644 index 0000000000..d806767402 --- /dev/null +++ b/apple/whistlegraph/Sources/StoryControls.swift @@ -0,0 +1,82 @@ +import SwiftUI + +struct StoryExportProgress: View { + @ObservedObject var narrator: VersionNarrator + @ObservedObject var exporter: StoryExport + let cancel: () -> Void + private var title: String { + switch exporter.stage { + case .preparing: return "Preparing video…" + case .rendering: return "Rendering card \(narrator.index + 1) of \(narrator.count)" + case .finishingVideo: return "Finishing video…" + case .narration: return "Adding audio…" + case .encoding: return "Encoding MP4" + } + } + private var progress: Double? { + switch exporter.stage { + case .rendering: return min(1, Double(narrator.index) / Double(max(1, narrator.count)) + narrator.progress / Double(max(1, narrator.count))) + case .narration, .encoding: return exporter.progress + default: return nil + } + } + var body: some View { + VStack(spacing: 12) { + HStack { + Text(title).font(.headline).accessibilityIdentifier("story-export-stage") + Spacer() + if exporter.stage == .encoding { Text(exporter.progress, format: .percent.precision(.fractionLength(0))).monospacedDigit() } + } + if let progress { + ProgressView(value: progress).tint(.white) + .accessibilityLabel(title).accessibilityIdentifier("story-export-bar") + } else { ProgressView().tint(.white).frame(maxWidth: .infinity) } + Button("Cancel export", action: cancel).padding(.vertical, 8) + .accessibilityIdentifier("story-export-cancel") + } + .padding(20).foregroundStyle(.white) + .background(.black.opacity(0.9), in: RoundedRectangle(cornerRadius: 20)) + .padding(20).accessibilityElement(children: .contain).accessibilityIdentifier("story-export-progress") + } +} + +struct StoryControls: View { + @ObservedObject var narrator: VersionNarrator + @ObservedObject var exporter: StoryExport + let export: () -> Void + let close: () -> Void + var body: some View { + VStack(spacing: 14) { + HStack(spacing: 4) { + ForEach(0..)] = [] + private var deadline: Task? + + func begin(session: WhistlegraphSession, started: @escaping () -> Void) { + guard !busy else { return } + self.session = session; operation = UUID(); let expected = operation + stage = .preparing; progress = 0 + busy = true; error = ""; bytes = 0; narration = []; tapeStarted = false + previousIdleTimer = UIApplication.shared.isIdleTimerDisabled; UIApplication.shared.isIdleTimerDisabled = true + Task { + do { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("story-canvas-\(expected).mp4") + FileManager.default.createFile(atPath: url.path, contents: nil) + raw = url; file = try FileHandle(forWritingTo: url) + session.storyTapeEvent = { [weak self] event in self?.receive(event) } + guard operation == expected else { return } + self.started = Date(); recording = true; started() + deadline = Task { [weak self] in + try? await Task.sleep(for: .seconds(600)) + guard !Task.isCancelled, let self, self.operation == expected else { return } + self.fail("Story export exceeded ten minutes. Export a shorter branch.") + } + } catch { if operation == expected { fail(error.localizedDescription) } } + } + } + func update(version: Int, caption: String) { + guard busy else { return } + Task { try? await session?.storyTape("update", arguments: ["value": ["version": version, "caption": caption]]) } + } + func addNarration(_ row: PieceRevision) async { + guard recording else { return } + do { + try await session?.storyTape("update", arguments: ["value": ["version": row.id, "caption": row.utterance]]) + if !tapeStarted { + try await session?.storyTape("start", arguments: ["id": operation.uuidString]) + tapeStarted = true; started = Date(); stage = .rendering + } + } catch { fail(error.localizedDescription); return } + guard !row.utterance.isEmpty || row.recordingID != nil else { return } + let at = Date().timeIntervalSince(started) + let task = Task<(URL, Double, Bool), Error> { + if let id = row.recordingID, let url = UtteranceRecording.url(id), FileManager.default.fileExists(atPath: url.path) { + return (url, try RecordingTrim.read(url).start, false) + } + return (try await StorySpeech.render(row.utterance), 0, true) + } + narration.append((at, task)) + } + func finish() { + guard recording else { return }; recording = false + stage = .finishingVideo + let expected = operation + Task { do { try await session?.storyTape("stop") } catch { if operation == expected { fail(error.localizedDescription) } } } + } + private func receive(_ event: [String: Any]) { + guard busy, event["session"] as? String == operation.uuidString else { return } + do { + switch event["kind"] as? String { + case "chunk": + guard let value = event["data"] as? String, value.count <= 300_000, let data = Data(base64Encoded: value) else { throw ExportError.invalidChunk } + bytes += data.count; guard bytes <= 256_000_000 else { throw ExportError.tooLarge } + try file?.write(contentsOf: data) + case "done": + try file?.close(); file = nil + let expected = operation + Task { + do { + guard let raw else { throw ExportError.noVideo } + let output = try await mux(raw, operation: expected) + guard operation == expected else { try? FileManager.default.removeItem(at: output); return } + #if DEBUG + if NativeScreenFixture.enabled && NativeScreenFixture.mode == "story" { + let evidence = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0].appendingPathComponent("story-export-test.mp4") + try? FileManager.default.removeItem(at: evidence); try FileManager.default.copyItem(at: output, to: evidence) + } + #endif + sharedURL = output; movie = StoryMovie(url: output); cleanup(); busy = false; recording = false + } catch { if operation == expected { fail(error.localizedDescription) } } + } + case "error": fail(event["error"] as? String ?? "Canvas recording failed.") + default: break + } + } catch { fail(error.localizedDescription) } + } + private func cleanup() { + if let previousIdleTimer { UIApplication.shared.isIdleTimerDisabled = previousIdleTimer }; previousIdleTimer = nil + deadline?.cancel(); try? file?.close(); file = nil + if let raw { try? FileManager.default.removeItem(at: raw) }; raw = nil + session?.storyTapeEvent = nil + for item in narration { Task { if let (url, _, temporary) = try? await item.task.value, temporary { try? FileManager.default.removeItem(at: url) } } } + narration = [] + } + private func fail(_ message: String) { cancel(); error = message } + func cancel() { + operation = UUID(); exportSession?.cancelExport(); exportSession = nil + let current = session + Task { try? await current?.storyTape("cancel") } + cleanup(); busy = false; recording = false + } + func clearMovie() { if let sharedURL { try? FileManager.default.removeItem(at: sharedURL) }; sharedURL = nil; movie = nil } + private func checkOperation(_ expected: UUID) throws { + guard busy, operation == expected else { throw CancellationError() } + } + private func mux(_ raw: URL, operation expected: UUID) async throws -> URL { + try checkOperation(expected) + stage = .narration; progress = 0 + let asset = AVURLAsset(url: raw), mix = AVMutableComposition() + let duration = try await asset.load(.duration) + try checkOperation(expected) + guard let video = try await asset.loadTracks(withMediaType: .video).first, + let destination = mix.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid) else { throw ExportError.noVideo } + try destination.insertTimeRange(CMTimeRange(start: .zero, duration: duration), of: video, at: .zero) + destination.preferredTransform = try await video.load(.preferredTransform) + let clips = narration + for (index, clip) in clips.enumerated() { + let (url, trim, _) = try await clip.task.value + try checkOperation(expected) + let audio = AVURLAsset(url: url) + guard let track = try await audio.loadTracks(withMediaType: .audio).first, + let target = mix.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) else { continue } + let length = try await audio.load(.duration).seconds - trim + let end = index+1 < clips.count ? clips[index+1].at : duration.seconds + let seconds = min(length, end-clip.at, duration.seconds-clip.at) + if seconds > 0 { try target.insertTimeRange(CMTimeRange(start: CMTime(seconds: trim, preferredTimescale: 600), duration: CMTime(seconds: seconds, preferredTimescale: 600)), of: track, at: CMTime(seconds: clip.at, preferredTimescale: 600)) } + progress = Double(index + 1) / Double(clips.count) + } + try checkOperation(expected) + guard let exporter = AVAssetExportSession(asset: mix, presetName: AVAssetExportPresetHighestQuality) else { throw ExportError.noVideo } + let output = FileManager.default.temporaryDirectory.appendingPathComponent("Whistlegraph-\(UUID()).mp4") + exporter.outputURL = output; exporter.outputFileType = .mp4; exporter.shouldOptimizeForNetworkUse = true; exportSession = exporter + stage = .encoding; progress = 0 + let monitor = Task { [weak self] in + while !Task.isCancelled { + guard let self, self.operation == expected, self.busy else { return } + self.progress = Double(exporter.progress) + try? await Task.sleep(for: .milliseconds(150)) + } + } + defer { monitor.cancel() } + await exporter.export() + guard operation == expected else { try? FileManager.default.removeItem(at: output); throw CancellationError() } + exportSession = nil + guard exporter.status == .completed else { try? FileManager.default.removeItem(at: output); throw exporter.error ?? ExportError.noVideo } + progress = 1 + return output + } + private enum ExportError: LocalizedError { + case noVideo, invalidChunk, tooLarge + var errorDescription: String? { + switch self { case .noVideo: return "Could not create the MP4."; case .invalidChunk: return "Invalid canvas tape data."; case .tooLarge: return "This story is too large. Export a shorter branch." } + } + } +} + +// Render fallback narration to an audio file, never through a microphone. +@MainActor private final class StorySpeech { + private let voice = AVSpeechSynthesizer() + private var file: AVAudioFile? + private var continuation: CheckedContinuation? + private var timeout: Task? + private let url = FileManager.default.temporaryDirectory.appendingPathComponent("story-voice-\(UUID()).caf") + static func render(_ text: String) async throws -> URL { + let renderer = StorySpeech() + return try await renderer.render(text) + } + private func render(_ text: String) async throws -> URL { + try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + self.timeout = Task { + try? await Task.sleep(for: .seconds(45)) + guard !Task.isCancelled, let pending = self.continuation else { return } + self.continuation = nil; self.voice.stopSpeaking(at: .immediate) + pending.resume(throwing: NSError(domain: "StorySpeech", code: 1, userInfo: [NSLocalizedDescriptionKey: "Narration rendering timed out."])) + } + let line = AVSpeechUtterance(string: text.isEmpty ? " " : text) + line.rate = AVSpeechUtteranceDefaultSpeechRate; line.voice = AVSpeechSynthesisVoice(language: "en-US") + voice.write(line) { buffer in + guard let pcm = buffer as? AVAudioPCMBuffer else { return } + Task { @MainActor in + guard let continuation = self.continuation else { return } + do { + if pcm.frameLength == 0 { self.timeout?.cancel(); self.continuation = nil; self.file = nil; continuation.resume(returning: self.url); return } + if self.file == nil { self.file = try AVAudioFile(forWriting: self.url, settings: pcm.format.settings) } + try self.file?.write(from: pcm) + } catch { self.timeout?.cancel(); self.continuation = nil; continuation.resume(throwing: error) } + } + } + } + } +} +struct StoryShareSheet: UIViewControllerRepresentable { + let url: URL + func makeUIViewController(context: Context) -> UIActivityViewController { UIActivityViewController(activityItems: [url], applicationActivities: nil) } + func updateUIViewController(_ controller: UIActivityViewController, context: Context) {} +} diff --git a/apple/whistlegraph/Sources/StoryMovieSheet.swift b/apple/whistlegraph/Sources/StoryMovieSheet.swift new file mode 100644 index 0000000000..cb8997983e --- /dev/null +++ b/apple/whistlegraph/Sources/StoryMovieSheet.swift @@ -0,0 +1,50 @@ +import SwiftUI +import AVKit +import Photos + +struct StoryMovieSheet: View { + let url: URL + @Environment(\.dismiss) private var dismiss + @State private var player: AVPlayer? + @State private var sharing = false + @State private var saving = false + @State private var saved = false + @State private var error = "" + var body: some View { + NavigationStack { + VStack(spacing: 18) { + VideoPlayer(player: player).aspectRatio(9.0 / 16.0, contentMode: .fit) + HStack(spacing: 16) { + Button { save() } label: { + HStack { + if saving { ProgressView() } + Label(saved ? "Saved to Photos" : saving ? "Saving…" : "Save video", systemImage: saved ? "checkmark" : "square.and.arrow.down") + } + .frame(maxWidth: .infinity) + }.buttonStyle(.borderedProminent).disabled(saved || saving).accessibilityIdentifier("story-save-video") + Button { sharing = true } label: { Label("Share", systemImage: "square.and.arrow.up") }.buttonStyle(.bordered) + } + }.padding().navigationTitle("Your video").navigationBarTitleDisplayMode(.inline) + .toolbar { ToolbarItem(placement: .confirmationAction) { Button("Done") { dismiss() }.disabled(saving) } } + } + .interactiveDismissDisabled(saving) + .onAppear { player = AVPlayer(url: url) } + .onDisappear { player?.pause() } + .sheet(isPresented: $sharing) { StoryShareSheet(url: url) } + .alert("Could not save video", isPresented: Binding(get: { !error.isEmpty }, set: { if !$0 { error = "" } })) { + Button("OK") { error = "" } + } message: { Text(error) } + } + private func save() { + saving = true + Task { @MainActor in + defer { saving = false } + let status = await PHPhotoLibrary.requestAuthorization(for: .addOnly) + guard status == .authorized || status == .limited else { error = "Allow Whistlegraph to add videos in Settings → Privacy & Security → Photos."; return } + do { + try await PHPhotoLibrary.shared().performChanges { PHAssetChangeRequest.creationRequestForAssetFromVideo(atFileURL: url) } + saved = true + } catch { self.error = error.localizedDescription } + } + } +} diff --git a/apple/whistlegraph/Sources/StoryTape.swift b/apple/whistlegraph/Sources/StoryTape.swift new file mode 100644 index 0000000000..ddff6d24db --- /dev/null +++ b/apple/whistlegraph/Sources/StoryTape.swift @@ -0,0 +1,12 @@ +import Foundation + +struct StoryTape { + static var script: String { + guard let root = Bundle.main.url(forResource: "Web", withExtension: nil), + let encoder = try? String(contentsOf: root.appendingPathComponent("canvas-tape.mjs")), + let story = try? String(contentsOf: root.appendingPathComponent("story-tape.js")) else { return "" } + let font = (try? Data(contentsOf: root.appendingPathComponent("ComicRelief-Regular.ttf")))?.base64EncodedString() ?? "" + let loadFont = "window.whistlegraphStoryFont = new FontFace('WhistlegraphComic', 'url(data:font/ttf;base64,\(font))').load().then(font => document.fonts.add(font));\n" + return loadFont + encoder.replacingOccurrences(of: "export function", with: "function") + "\n" + story + } +} diff --git a/apple/whistlegraph/Sources/VersionNarrator.swift b/apple/whistlegraph/Sources/VersionNarrator.swift index 96ab5faf83..cc8e4af604 100644 --- a/apple/whistlegraph/Sources/VersionNarrator.swift +++ b/apple/whistlegraph/Sources/VersionNarrator.swift @@ -3,43 +3,83 @@ import AVFoundation @MainActor final class VersionNarrator: NSObject, ObservableObject, AVSpeechSynthesizerDelegate, AVAudioPlayerDelegate { @Published var isPlaying = false + @Published var isPaused = false @Published var utterance = "" @Published var currentWord = "" @Published var spokenRange: NSRange? - private var words: [PlaybackWord] = [] - private var wordClock: Task? @Published var error = "" + @Published private(set) var index = 0 + @Published private(set) var count = 0 + @Published private(set) var progress = 0.0 + var onComplete: (() -> Void)? + var onNarration: ((PieceRevision) async -> Void)? + private var words: [PlaybackWord] = [] + private var clock: Task? private var recordingID: String? private var player: AVAudioPlayer? private var playbackEnd = Double.infinity + private var playbackStart = 0.0 + private var elapsed = 0.0 + private var duration = 4.0 + private var tail: Double? private let voice = AVSpeechSynthesizer() private weak var session: WhistlegraphSession? private var versions: [PieceRevision] = [] - private var index = 0 private var waiting: Int? private var spoken: AVSpeechUtterance? - private var run = UUID() private var timer: Task? + private var run = UUID() + private var completed = false override init() { super.init(); voice.delegate = self } func play(_ session: WhistlegraphSession) { stop() self.session = session - versions = session.snapshot.versions.filter { $0.id > 0 }.sorted { $0.id < $1.id } + // Follow the selected branch; discarded alternatives are not story beats. + let rows = Dictionary(uniqueKeysWithValues: session.snapshot.versions.map { ($0.id, $0) }) + var chain: [PieceRevision] = [], visited = Set(), cursor: Int? = session.snapshot.head + while let id = cursor, let row = rows[id], visited.insert(id).inserted { + if row.id > 0 { chain.append(row) }; cursor = row.parent + } + versions = chain.reversed(); count = versions.count guard !versions.isEmpty else { return } - isPlaying = true; index = 0; show(versions[0], presentation: true) + isPlaying = true; index = 0; show() } func select(_ version: Int, session: WhistlegraphSession) { stop(); self.session = session guard session.snapshot.versions.contains(where: { $0.id == version }) else { return } session.command("checkout", version: version) } - private func show(_ row: PieceRevision, presentation: Bool) { - wordClock?.cancel(); currentWord = ""; spokenRange = nil; words = row.words ?? [] - waiting = row.id; recordingID = row.recordingID; utterance = row.utterance; error = "" - session?.command(presentation ? "presentVersion" : "checkout", version: row.id) + func next() { + guard isPlaying else { return } + if index + 1 < count { index += 1; show() } + else { setPaused(true); progress = 1 } + } + func previous() { guard isPlaying else { return }; index = max(0, index - 1); show() } + func restart() { guard isPlaying else { return }; index = 0; isPaused = false; show() } + func setPaused(_ paused: Bool) { + guard isPlaying else { return } + isPaused = paused + if paused { player?.pause(); voice.pauseSpeaking(at: .immediate) } + else if completed { restart() } + else { + player?.play() + if voice.isPaused { voice.continueSpeaking() } + else if !voice.isSpeaking, let spoken { voice.speak(spoken) } + } + } + private func clearPlayback() { + clock?.cancel(); timer?.cancel(); run = UUID(); waiting = nil + spoken = nil; voice.stopSpeaking(at: .immediate) + player?.stop(); player = nil; currentWord = ""; spokenRange = nil + } + private func show() { + clearPlayback(); completed = false; progress = 0; elapsed = 0; tail = nil + let row = versions[index] + words = row.words ?? []; waiting = row.id; recordingID = row.recordingID + utterance = row.utterance; error = "" + session?.command("presentVersion", version: row.id) let expected = run - timer?.cancel() timer = Task { [weak self] in try? await Task.sleep(for: .seconds(10)) guard !Task.isCancelled, let self, self.run == expected, self.waiting != nil else { return } @@ -49,75 +89,81 @@ import AVFoundation func painted(_ version: Int?) { guard let version, waiting == version else { return } waiting = nil; timer?.cancel() + let expected = run, row = versions[index] + Task { + await onNarration?(row) + guard run == expected, isPlaying else { return } + startNarration() + } + } + private func startNarration() { + duration = max(4, Double(utterance.split(separator: " ").count) / 2.5 + 1.5) do { let audio = AVAudioSession.sharedInstance() - try audio.setCategory(.playback, mode: .default) - try audio.setActive(true) + try audio.setCategory(.playback, mode: .default); try audio.setActive(true) if let recordingID, let url = UtteranceRecording.url(recordingID), FileManager.default.fileExists(atPath: url.path) { - let original = try AVAudioPlayer(contentsOf: url) - let trim = try RecordingTrim.read(url) - original.currentTime = trim.start; playbackEnd = trim.end - original.delegate = self; player = original - if original.play() { followWords(original); return } + let original = try AVAudioPlayer(contentsOf: url), trim = try RecordingTrim.read(url) + original.currentTime = trim.start; playbackStart = trim.start; playbackEnd = trim.end + duration = max(4, trim.end - trim.start + 1.5) + original.delegate = self; player = original; original.prepareToPlay() + if !isPaused { original.play() } } } catch { self.error = "Original recording unavailable; reading the transcript." } - guard !utterance.isEmpty else { advanceAfterPause(); return } - let line = AVSpeechUtterance(string: utterance) - line.rate = AVSpeechUtteranceDefaultSpeechRate - line.voice = AVSpeechSynthesisVoice(language: "en-US") - spoken = line; voice.speak(line) - } - private func followWords(_ audio: AVAudioPlayer) { - wordClock?.cancel() - wordClock = Task { [weak self, weak audio] in + if player == nil { + if utterance.isEmpty { tail = 4 } + else { + let line = AVSpeechUtterance(string: utterance) + line.rate = AVSpeechUtteranceDefaultSpeechRate; line.voice = AVSpeechSynthesisVoice(language: "en-US") + spoken = line + if !isPaused { voice.speak(line) } + } + } + let expected = run + clock = Task { [weak self] in + var last = Date() while !Task.isCancelled { - guard let self, let audio, self.player === audio, audio.isPlaying else { return } - if audio.currentTime >= self.playbackEnd { - audio.stop(); self.player = nil; self.advanceAfterPause(); return + try? await Task.sleep(for: .milliseconds(33)) + guard !Task.isCancelled, let self, self.run == expected, self.isPlaying else { return } + let now = Date(), delta = min(0.1, now.timeIntervalSince(last)); last = now + guard !self.isPaused else { continue } + self.elapsed += delta + if let audio = self.player { + self.currentWord = PlaybackWord.at(audio.currentTime * 1000, in: self.words) + self.spokenRange = PlaybackWord.range(at: audio.currentTime * 1000, in: self.words, text: self.utterance) + if audio.currentTime >= self.playbackEnd { audio.stop(); self.player = nil; self.narrationFinished() } + } + if let remaining = self.tail { + self.tail = remaining - delta + if remaining <= 0 { self.finishCard(); return } } - self.currentWord = PlaybackWord.at(audio.currentTime * 1000, in: self.words) - self.spokenRange = PlaybackWord.range(at: audio.currentTime * 1000, in: self.words, text: self.utterance) - try? await Task.sleep(for: .milliseconds(30)) + self.progress = min(0.98, self.elapsed / self.duration) } } } - nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance: AVSpeechUtterance) { - let text = (utterance.speechString as NSString).substring(with: characterRange) + private func narrationFinished() { currentWord = ""; spokenRange = nil; tail = max(1.5, 4 - elapsed); duration = elapsed + (tail ?? 1.5) } + private func finishCard() { + if index + 1 < count { index += 1; show() } + else { completed = true; isPaused = true; progress = 1; onComplete?() } + } + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString range: NSRange, utterance: AVSpeechUtterance) { Task { @MainActor [weak self] in guard let self, self.spoken === utterance else { return } - self.currentWord = text; self.spokenRange = characterRange + self.currentWord = (utterance.speechString as NSString).substring(with: range); self.spokenRange = range } } nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { Task { @MainActor [weak self] in - guard let self, self.spoken === utterance else { return } - self.spoken = nil; self.advanceAfterPause() + guard let self, self.spoken === utterance else { return }; self.spoken = nil; self.narrationFinished() } } nonisolated func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) { Task { @MainActor [weak self] in - guard let self, self.player === player else { return } - self.player = nil; self.advanceAfterPause() - } - } - private func advanceAfterPause() { - wordClock?.cancel(); currentWord = ""; spokenRange = nil - guard isPlaying else { return } - let expected = run - timer = Task { [weak self] in - try? await Task.sleep(for: .seconds(1)) - guard !Task.isCancelled, let self, self.run == expected, self.isPlaying else { return } - self.index += 1 - if self.index < self.versions.count { self.show(self.versions[self.index], presentation: true) } - else { self.stop() } + guard let self, self.player === player else { return }; self.player = nil; self.narrationFinished() } } func stop() { - wordClock?.cancel(); wordClock = nil; currentWord = "" let wasPlaying = isPlaying - run = UUID(); timer?.cancel(); timer = nil; waiting = nil; spoken = nil - player?.stop(); player = nil; recordingID = nil - voice.stopSpeaking(at: .immediate); isPlaying = false + clearPlayback(); isPlaying = false; isPaused = false; recordingID = nil if wasPlaying { session?.command("endPresentation") } } } diff --git a/apple/whistlegraph/Sources/WhistlegraphAccount.swift b/apple/whistlegraph/Sources/WhistlegraphAccount.swift index cf1decc523..2328c09459 100644 --- a/apple/whistlegraph/Sources/WhistlegraphAccount.swift +++ b/apple/whistlegraph/Sources/WhistlegraphAccount.swift @@ -102,10 +102,25 @@ final class WhistlegraphBundle: NSObject, WKURLSchemeHandler { } struct WhistlegraphPreview { + static let pixelSizeKey = "walkieware-pixel-size" + // bios.mjs defaults to two CSS points per framebuffer pixel on iPhone. + static var savedPixelSize: Int { + let saved = UserDefaults.standard.integer(forKey: pixelSizeKey) + return (1...4).contains(saved) ? saved : 2 + } static let script = """ (() => { if (window === window.top || location.origin !== 'https://aesthetic.computer') return; window.acFORCE_NOGAP = true; + window.walkiewareSetPixelSize = size => { + if (!Number.isInteger(size) || size < 1 || size > 4) return; + window.__walkiewarePixelSize = size; + window.acPACK_DENSITY = size; + window.acAutoDensityOverride = true; + try { localStorage.setItem('ac-density', String(size)); } catch {} + window.postMessage({type:'ac-density-change', density:size}, location.origin); + }; + window.walkiewareSetPixelSize(window.__walkiewarePixelSize ?? 2); let ready = false, revision = 0, paintedRevision = 0, sessionID = ''; const post = body => window.webkit.messageHandlers.walkie.postMessage(body); window.walkiewareRender = async (source, threadID, renderID) => { diff --git a/apple/whistlegraph/Sources/WhistlegraphApp.swift b/apple/whistlegraph/Sources/WhistlegraphApp.swift index b525b69c00..3b676dda2a 100644 --- a/apple/whistlegraph/Sources/WhistlegraphApp.swift +++ b/apple/whistlegraph/Sources/WhistlegraphApp.swift @@ -43,11 +43,6 @@ struct Workspace: UIViewRepresentable { func makeUIView(context: Context) -> WKWebView { let config = WKWebViewConfiguration() config.userContentController.addUserScript(WKUserScript(source: "window.__walkiewareNativeShell = true;", injectionTime: .atDocumentStart, forMainFrameOnly: true)) - var checkedEdits = UserDefaults.standard.bool(forKey: "whistlegraph-checked-edits") - #if DEBUG - if ProcessInfo.processInfo.environment["WALKIE_COMPILED_TASK"] == "1" { checkedEdits = true } - #endif - config.userContentController.addUserScript(WKUserScript(source: "window.__whistlegraphCheckedEdits = \(checkedEdits);", injectionTime: .atDocumentStart, forMainFrameOnly: true)) config.userContentController.add(context.coordinator, name: "walkie") config.setURLSchemeHandler(WhistlegraphBundle(), forURLScheme: "walkieware") // Custom-scheme fetch responses have status 0 on device. Seed the @@ -62,6 +57,8 @@ struct Workspace: UIViewRepresentable { if let data = try? JSONSerialization.data(withJSONObject: guides), let json = String(data: data, encoding: .utf8) { config.userContentController.addUserScript(WKUserScript(source: "window.__aeselGuides = \(json);", injectionTime: .atDocumentStart, forMainFrameOnly: true)) } + config.userContentController.addUserScript(WKUserScript(source: StoryTape.script, injectionTime: .atDocumentStart, forMainFrameOnly: false)) + config.userContentController.addUserScript(WKUserScript(source: "window.__walkiewarePixelSize = \(voice.pixelSize);", injectionTime: .atDocumentStart, forMainFrameOnly: false)) config.userContentController.addUserScript(WKUserScript(source: WhistlegraphPreview.script, injectionTime: .atDocumentStart, forMainFrameOnly: false)) #if DEBUG if NativeScreenFixture.enabled { config.userContentController.addUserScript(WKUserScript(source: NativeScreenFixture.script, injectionTime: .atDocumentStart, forMainFrameOnly: true)) } @@ -182,9 +179,18 @@ final class WhistlegraphSession: NSObject, ObservableObject, WKScriptMessageHand var narratedVersion: Int? @Published var workspaceReady = false @Published var pieces: [PieceSummary] = [] + @Published private(set) var pixelSize = WhistlegraphPreview.savedPixelSize let drawing = DrawingDraft() private var speechStartedAt: TimeInterval? + func setPixelSize(_ size: Int) { + guard (1...4).contains(size), !snapshot.busy, capturePhase == .idle else { return } + pixelSize = size + UserDefaults.standard.set(size, forKey: WhistlegraphPreview.pixelSizeKey) + guard let frame = previewFrame else { return } + Task { _ = try? await webView?.callAsyncJavaScript("window.walkiewareSetPixelSize?.(size);", arguments: ["size": size], in: frame, contentWorld: .page) } + } + func command(_ action: String, version: Int? = nil, text: String? = nil, piece: String? = nil) { guard ["checkout", "newPiece", "openPiece", "stop", "signIn", "ask", "retry", "presentVersion", "endPresentation"].contains(action) else { return } if action == "newPiece" || action == "openPiece" { @@ -265,6 +271,11 @@ final class WhistlegraphSession: NSObject, ObservableObject, WKScriptMessageHand capture.onLevel = { [weak self] rms in guard let self else { return }; self.microphoneLevels = Array(self.microphoneLevels.dropFirst()) + [rms] } capture.onReplayRelease = { [weak self] in self?.webView?.evaluateJavaScript("voiceEnd()", completionHandler: nil) } } + var storyTapeEvent: (([String: Any]) -> Void)? + func storyTape(_ action: String, arguments: [String: Any] = [:]) async throws { + guard let frame = previewFrame, let webView else { throw NSError(domain: "StoryTape", code: 1, userInfo: [NSLocalizedDescriptionKey: "The piece is not ready to export."]) } + _ = try await webView.callAsyncJavaScript("if (!window.whistlegraphStoryTape) throw Error('Canvas tape is unavailable'); await window.whistlegraphStoryTape[action](value ?? id);", arguments: ["action": action, "value": arguments["value"] ?? NSNull(), "id": arguments["id"] ?? NSNull()], in: frame, contentWorld: .page) + } private var previewFrame: WKFrameInfo? private var previewSource = "" private var previewRequestID = 0 @@ -277,6 +288,7 @@ final class WhistlegraphSession: NSObject, ObservableObject, WKScriptMessageHand message.frameInfo.request.url?.host == "aesthetic.computer", message.frameInfo.request.url?.scheme == "https", let body = message.body as? [String: Any] { + if body["action"] as? String == "storyTape" { storyTapeEvent?(body); return } if body["action"] as? String == "previewReady" { #if DEBUG print("[walkieware] preview runtime ready") @@ -432,10 +444,10 @@ final class WhistlegraphSession: NSObject, ObservableObject, WKScriptMessageHand let source = previewSource let threadID = previewThreadID let renderID = previewRequestID - Task { _ = try? await webView?.callAsyncJavaScript("window.walkiewareRender?.(source, threadID, renderID);", arguments: ["source": source, "threadID": threadID, "renderID": renderID], in: frame, contentWorld: .page) } + let size = pixelSize + Task { _ = try? await webView?.callAsyncJavaScript("window.walkiewareSetPixelSize?.(size); window.walkiewareRender?.(source, threadID, renderID);", arguments: ["source": source, "threadID": threadID, "renderID": renderID, "size": size], in: frame, contentWorld: .page) } } - func setCheckedEdits(_ enabled: Bool) { emitEngine(["kind": "inferenceSettings", "checkedEdits": enabled]) } func emitEngine(_ event: [String: Any]) { guard let data = try? JSONSerialization.data(withJSONObject: event), diff --git a/apple/whistlegraph/Sources/WhistlegraphHeader.swift b/apple/whistlegraph/Sources/WhistlegraphHeader.swift index e3e1159c9a..4695a52d62 100644 --- a/apple/whistlegraph/Sources/WhistlegraphHeader.swift +++ b/apple/whistlegraph/Sources/WhistlegraphHeader.swift @@ -22,6 +22,7 @@ struct PieceSummary: Decodable, Identifiable { struct PiecesSheet: View { let pieces: [PieceSummary] let colors: [[Double]] + @Binding var pixelSize: Int let disabled: Bool let open: (String) -> Void let newPiece: () -> Void @@ -35,6 +36,16 @@ struct PiecesSheet: View { .font(.custom("ComicRelief-Bold", size: 20, relativeTo: .title3)) }.disabled(disabled).accessibilityIdentifier("pieces-new") } + Section("Pixel size") { + Picker("Pixel size", selection: $pixelSize) { + ForEach(1...4, id: \.self) { size in + Text("\(size)×").tag(size) + } + } + .pickerStyle(.segmented) + .disabled(disabled) + .accessibilityIdentifier("pieces-pixel-size") + } Section(pieces.count == 1 ? "Your piece" : "Your pieces") { ForEach(pieces) { piece in Button { @@ -73,8 +84,6 @@ struct PiecesSheet: View { // Tapping the handle: who is signed in, and the door out. Settings grow here. struct AccountSheet: View { - var setCheckedEdits: (Bool) -> Void = { _ in } - @AppStorage("whistlegraph-checked-edits") private var checkedEdits = false let handle: String let colors: [[Double]] @Binding var appearance: String @@ -108,11 +117,6 @@ struct AccountSheet: View { Toggle("Interface sounds", isOn: $sounds).accessibilityIdentifier("account-sounds") .onChange(of: sounds) { _, on in if on { ButtonSounds.play(.tick) } } } footer: { Text("Keys and buttons respect silent mode. Haptics stay on.") } - Section { - Toggle("Check edits (experimental)", isOn: $checkedEdits) - .accessibilityIdentifier("account-checked-edits") - .onChange(of: checkedEdits) { _, enabled in setCheckedEdits(enabled) } - } footer: { Text("Checks generated code and may try one repair. May use more braincells.") } if !handle.isEmpty { Section { Button(role: .destructive) { confirmingSignOut = true } label: { Label("Sign out", systemImage: "rectangle.portrait.and.arrow.right") } @@ -160,12 +164,13 @@ struct IdentityHeader: View { } .sheet(isPresented: $showingPieces) { PiecesSheet(pieces: session.pieces, colors: session.snapshot.colors, + pixelSize: Binding(get: { session.pixelSize }, set: { session.setPixelSize($0) }), disabled: session.snapshot.busy || session.capturePhase != .idle, open: { session.command("openPiece", piece: $0) }, newPiece: { session.command("newPiece") }) } .sheet(isPresented: $showingAccount) { - AccountSheet(setCheckedEdits: { session.setCheckedEdits($0) }, handle: session.snapshot.handle, colors: session.snapshot.colors, appearance: $appearance, + AccountSheet(handle: session.snapshot.handle, colors: session.snapshot.colors, appearance: $appearance, signIn: { session.command("signIn") }, signOut: { session.signOut() }) } } diff --git a/apple/whistlegraph/Sources/WhistlegraphScreen.swift b/apple/whistlegraph/Sources/WhistlegraphScreen.swift index 99c7d183df..cb92111be3 100644 --- a/apple/whistlegraph/Sources/WhistlegraphScreen.swift +++ b/apple/whistlegraph/Sources/WhistlegraphScreen.swift @@ -63,6 +63,7 @@ struct WhistlegraphScreen: View { @ObservedObject private var drawing: DrawingDraft init(session: WhistlegraphSession) { self.session = session; self.drawing = session.drawing } @StateObject private var narrator = VersionNarrator() + @StateObject private var exporter = StoryExport() @Environment(\.scenePhase) private var scenePhase @State private var held = false @State private var showComposer = false @@ -81,7 +82,7 @@ struct WhistlegraphScreen: View { private var accent: Color { theme.accent } private var chalkActive: Bool { drawing.enabled || held || session.capturePhase == .opening || session.capturePhase == .recording } var body: some View { - VStack(spacing: session.layout.spacing) { + VStack(spacing: narrator.isPlaying ? 0 : session.layout.spacing) { HStack { IdentityHeader(session: session, appearance: $appearance, size: session.layout.historySize) { narrator.stop(); showComposer = false } Spacer() @@ -89,12 +90,12 @@ struct WhistlegraphScreen: View { showComposer = false ButtonSounds.play(.play); narrator.play(session) } label: { - Image(systemName: "play.fill").font(.system(size: 28, weight: .bold)).frame(width: 44, height: 44) + Image(systemName: "rectangle.stack.fill").font(.system(size: 28, weight: .bold)).frame(width: 44, height: 44) } .disabled(session.snapshot.busy || session.capturePhase != .idle || !session.engineReady || !session.snapshot.versions.contains(where: { $0.id > 0 })) - .accessibilityLabel("Play version story").accessibilityIdentifier("play-versions") + .accessibilityLabel("Open story cards").accessibilityIdentifier("play-versions") }.frame(height: narrator.isPlaying ? 0 : nil).clipped().accessibilityHidden(narrator.isPlaying) - VStack(spacing: 4) { + VStack(spacing: narrator.isPlaying ? 12 : 4) { // This representable never changes identity when cards or versions change. ZStack { Workspace(voice: session) @@ -111,13 +112,17 @@ struct WhistlegraphScreen: View { VStack(spacing: 12) { Text(failure); Button("Reload") { session.reloadWorkspace() } }.padding().background(.black.opacity(0.85)) } } - .aspectRatio(narrator.isPlaying ? nil : 4 / 3, contentMode: .fit) - .frame(maxHeight: narrator.isPlaying ? .infinity : nil) + .aspectRatio(4.0 / 3.0, contentMode: .fit) + .frame(maxWidth: .infinity) + .accessibilityIdentifier("story-picture") .overlay { - if narrator.isPlaying { Rectangle().strokeBorder(paper.opacity(0.8), lineWidth: 2).allowsHitTesting(false) } - } - .overlay(alignment: .topTrailing) { - if narrator.isPlaying { Button { narrator.stop() } label: { Image(systemName: "xmark.circle.fill").font(.largeTitle).padding(16).background(.black.opacity(0.5), in: Circle()) }.accessibilityLabel("Close version story") } + if narrator.isPlaying && !exporter.busy { + HStack { + Color.clear.contentShape(Rectangle()).frame(width: 28).onTapGesture { narrator.previous() } + Spacer().allowsHitTesting(false) + Color.clear.contentShape(Rectangle()).frame(width: 28).onTapGesture { narrator.next() } + }.accessibilityHidden(true) + } } if !narrator.isPlaying { HStack(spacing: 18) { @@ -145,21 +150,28 @@ struct WhistlegraphScreen: View { .frame(maxWidth: .infinity, alignment: .leading).allowsHitTesting(false) } } - if narrator.isPlaying && (session.snapshot.hasPreview || session.snapshot.hasHistory) { - HStack(alignment: .bottom) { - ComicTitle(text: "v\(session.snapshot.head)", size: session.layout.historySize) + if narrator.isPlaying { + VStack(alignment: .leading, spacing: 6) { + Text("v\(session.snapshot.head)") + .font(.system(size: 15, weight: .semibold, design: .monospaced)).foregroundStyle(.white.opacity(0.65)) .accessibilityLabel("Running version \(session.snapshot.head)") - Spacer(minLength: 0) - PlaybackCaption(text: narrator.utterance, spokenRange: narrator.spokenRange, accent: accent) - .font(.custom("ComicRelief-Regular", size: session.layout.historySize, relativeTo: .title3)) - .multilineTextAlignment(.trailing) + .accessibilityIdentifier("story-version") + PlaybackCaption(text: narrator.utterance, spokenRange: narrator.spokenRange, accent: .yellow) + .font(.custom("ComicRelief-Regular", size: 22, relativeTo: .title3)) + .lineLimit(3).multilineTextAlignment(.leading) .accessibilityIdentifier("spoken-word") - - } + }.frame(maxWidth: .infinity, minHeight: 82, alignment: .topLeading).foregroundStyle(.white) + Spacer(minLength: 0) } } - if !narrator.error.isEmpty { Text(narrator.error).foregroundStyle(.orange) } - if let failure = session.captureError { Text(failure).font(.body).foregroundStyle(.orange).frame(maxWidth: .infinity, alignment: .leading) } + .padding(.horizontal, narrator.isPlaying ? 28 : 0) + .padding(.top, narrator.isPlaying ? 100 : 0) + .padding(.bottom, narrator.isPlaying ? 150 : 0) + .frame(maxWidth: .infinity) + .frame(height: narrator.isPlaying ? UIScreen.main.bounds.width * 16 / 9 : nil) + .background(narrator.isPlaying ? Color.black : Color.clear) + if !narrator.error.isEmpty && !narrator.isPlaying { Text(narrator.error).foregroundStyle(.orange) } + if let failure = session.captureError, !narrator.isPlaying { Text(failure).font(.body).foregroundStyle(.orange).frame(maxWidth: .infinity, alignment: .leading) } if !narrator.isPlaying && (session.snapshot.hasPiece || session.snapshot.hasHistory || session.snapshot.busy || session.snapshot.attempt?.status == "failed") { VersionFeed(snapshot: session.snapshot, foreground: paper, selectionColor: paper, textSize: session.layout.historySize, disabled: session.snapshot.busy || session.capturePhase != .idle, holdSelection: drawing.hasInk, stop: { session.command("stop") }, retry: { session.command("retry") }) { narrator.select($0, session: session) } } else if !narrator.isPlaying { Spacer(minLength: 0) } @@ -185,17 +197,38 @@ struct WhistlegraphScreen: View { } } }.padding(.horizontal, session.layout.pageInset).foregroundStyle(paper) - .background(theme.background) + .background(narrator.isPlaying ? Color.black : theme.background) } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .overlay { + if narrator.isPlaying { + StoryControls(narrator: narrator, exporter: exporter, export: exportStory, close: { exporter.cancel(); narrator.stop() }) + .opacity(exporter.busy ? 0 : 1).allowsHitTesting(!exporter.busy) + if exporter.busy { + VStack { Spacer(); StoryExportProgress(narrator: narrator, exporter: exporter) { exporter.cancel(); narrator.setPaused(true) } } + } + } + } + .alert("Could not export", isPresented: Binding(get: { !exporter.error.isEmpty }, set: { if !$0 { exporter.error = "" } })) { + Button("OK") { exporter.error = "" } + } message: { Text(exporter.error) } + .sheet(item: $exporter.movie, onDismiss: { exporter.clearMovie() }) { movie in StoryMovieSheet(url: movie.url) } .statusBarHidden(narrator.isPlaying) + .onChange(of: narrator.isPlaying) { _, playing in if !playing && exporter.busy { exporter.cancel() } } .onChange(of: session.narratedFrame) { _, _ in narrator.painted(session.narratedVersion) } - .onChange(of: scenePhase) { _, value in if value != .active { narrator.stop() } } + .onChange(of: scenePhase) { _, value in if value == .background { exporter.cancel(); narrator.stop() } else if value == .inactive && !exporter.busy { narrator.setPaused(true) } } .onChange(of: session.capturePhase) { _, value in if value != .idle { narrator.stop() } } - .onDisappear { narrator.stop() } - .background(theme.background.ignoresSafeArea()) + .onDisappear { exporter.cancel(); narrator.stop() } + .background((narrator.isPlaying ? Color.black : theme.background).ignoresSafeArea()) .tint(paper) .animation(reduceMotion ? nil : .easeInOut(duration: 0.3), value: themePhase) } + private func exportStory() { + narrator.setPaused(true) + narrator.onComplete = { exporter.finish() } + narrator.onNarration = { await exporter.addNarration($0) } + exporter.begin(session: session) { narrator.restart() } + } private var canTalk: Bool { session.workspaceReady && session.engineReady && !session.snapshot.busy && session.capturePhase != .processing } private var talkControl: some View { TimelineView(.animation(minimumInterval: 1 / 30, paused: session.capturePhase != .recording)) { context in diff --git a/apple/whistlegraph/Tests/canvas-tape.test.mjs b/apple/whistlegraph/Tests/canvas-tape.test.mjs new file mode 100644 index 0000000000..ebb491d199 --- /dev/null +++ b/apple/whistlegraph/Tests/canvas-tape.test.mjs @@ -0,0 +1,31 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {createCanvasTapeRecorder} from '../../../system/public/aesthetic.computer/lib/canvas-tape.mjs'; + +function fixture({supported=['video/mp4'], fail=false}={}) { + const stopped=[], tracks=[]; + const video={stop:()=>stopped.push('video')},audio={stop:()=>stopped.push('audio')}; + const stream={getVideoTracks:()=>[video],addTrack:t=>tracks.push(t)}; + const canvas={captureStream:()=>stream}; + class Recorder extends EventTarget { + static isTypeSupported(type){return supported.includes(type);} + constructor(stream,options){super();if(fail)throw Error('encoder failed');this.mimeType=options.mimeType;} + } + return {Recorder,canvas,audio,stopped,tracks}; +} + +test('finishing a tape releases its canvas track, preserving shared app audio',()=>{ + const f=fixture();globalThis.MediaRecorder=f.Recorder; + const tape=createCanvasTapeRecorder(f.canvas,{audioTracks:[f.audio]}); + assert.equal(tape.mimeType,'video/mp4');assert.deepEqual(f.tracks,[f.audio]); + tape.dispatchEvent(new Event('stop'));assert.deepEqual(f.stopped,['video']); +}); +test('MP4-only export fails before capture when only WebM is supported',()=>{ + const f=fixture({supported:['video/webm']});globalThis.MediaRecorder=f.Recorder; + f.canvas.captureStream=()=>{throw Error('must not capture');}; + assert.throws(()=>createCanvasTapeRecorder(f.canvas,{mp4Only:true}),/cannot encode an MP4/); +}); +test('an encoder failure releases the acquired video track',()=>{ + const f=fixture({fail:true});globalThis.MediaRecorder=f.Recorder; + assert.throws(()=>createCanvasTapeRecorder(f.canvas),/encoder failed/);assert.deepEqual(f.stopped,['video']); +}); diff --git a/apple/whistlegraph/Tests/native-bridge.test.cjs b/apple/whistlegraph/Tests/native-bridge.test.cjs index 1d04a1c6f7..68adfa8674 100644 --- a/apple/whistlegraph/Tests/native-bridge.test.cjs +++ b/apple/whistlegraph/Tests/native-bridge.test.cjs @@ -17,7 +17,7 @@ const server = http.createServer(async (req, res) => { if(visual) { visualRequests++; assert.equal(parsed.messages[0].content.filter(b=>b.type==='image').length>=4,true); - const failed=process.argv.includes('--visual-fails'); + const failed=process.argv.includes('--visual-fails')||process.argv.includes('--review-fails-after-repair'); send({type:'content_block_delta',delta:{type:'text_delta',text:JSON.stringify({passed:!failed,observations:'Fixture-only visual evidence; not a physical capture.',findings:failed?['Requested subject is missing.']:[]})}}); send({type:'message_delta',delta:{stop_reason:'end_turn'}});res.end();return; } @@ -25,9 +25,9 @@ const server = http.createServer(async (req, res) => { send({type:'content_block_start',index:0,content_block:{type:'tool_use',id:'visual-candidate-'+requests,name:'write_piece'}}); send({type:'content_block_delta',index:0,delta:{type:'input_json_delta',partial_json:JSON.stringify({source:'export function paint({wipe}) {wipe('+requests+');}'})}}); send({type:'content_block_stop',index:0});send({type:'message_delta',delta:{stop_reason:'end_turn'}}); - } else if(process.argv.includes('--checked-edits')) { - const invalid=requests===1||process.argv.includes('--repair-fails'); - const source=invalid?'export const caption="Cats"; export function paint({wipe,ink}) {wipe(0);ink(`hsl(30,100%,50%)`).circle(30,30,10);}':'export const caption="Bouncing cats"; export function paint({wipe,ink}) {wipe(0);ink(255,128,0).circle(30,30,10);}'; + } else if(process.argv.includes('--checked-edits')||process.argv.includes('--runtime-errors')) { + const invalid=!process.argv.includes('--runtime-errors')&&(requests===1||process.argv.includes('--repair-fails')); + const source=invalid?'export const caption="Cats"; export function paint({wipe,ink}) {wipe(0);ink(`hsl(30,100%,50%)`).circle(30,30,10);}':'export const caption="Bouncing cats"; export function paint({wipe,ink}) {wipe(0);ink(255,128,0).circle(30,30,'+ (10+requests) +');}'; send({type:'message_start',message:{id:'provider-'+requests,model:'fixture/reported',usage:{input_tokens:10}}}); send({type:'content_block_start',index:0,content_block:{type:'tool_use',id:'checked-'+requests,name:'write_piece'}}); send({type:'content_block_delta',index:0,delta:{type:'input_json_delta',partial_json:JSON.stringify({source})}}); @@ -96,6 +96,8 @@ const server = http.createServer(async (req, res) => { assert.equal(visualRequests,process.argv.includes('--visual-fails')?2:0); assert.equal(requests,process.argv.includes('--visual-fails')?2:1,'at most one repair; stale capture buys none'); assert.equal(await page.evaluate(()=>walkiewareIsBusy()),false); + await page.waitForFunction(()=>__nativeMessages.some(m=>m.action==='snapshot'&&m.snapshot.phase==='Could not finish · previous version restored')); + assert.match(await page.evaluate(()=>document.getElementById('live-phase').textContent),/Could not finish/,'rollback paint must retain the failure status'); console.log('PASS: visual rejection preserves the ledger; repair is bounded and rechecked; stale captures fail before inference. Mock capture and provider.');return; } if(process.argv.includes('--drawing')) { @@ -162,8 +164,13 @@ const server = http.createServer(async (req, res) => { if(process.env.CHALK_RABBIT_OUT)await require('node:fs/promises').writeFile(process.env.CHALK_RABBIT_OUT,Buffer.from(dots.block.source.data,'base64')); assert.deepEqual(errors,[]);console.log('PASS: typed and spoken gesture input, aligned sound, per-version attachment, and failed-draft retention');return; } - if(process.argv.includes('--checked-edits')) { - await page.evaluate(()=>{walkiewareEngineEvent({kind:'inferenceSettings',checkedEdits:true});walkiewareEngineEvent({kind:'account',token:'fixture-only'});walkiewareEngineEvent({kind:'previewReady'});}); + if(process.argv.includes('--checked-edits')||process.argv.includes('--runtime-errors')) { + await page.evaluate(runtimeErrors=>{ + if(runtimeErrors){ + const post=window.webkit.messageHandlers.walkie.postMessage;let candidates=0; + window.webkit.messageHandlers.walkie.postMessage=m=>{post(m);if(m.action==='render'&&m.source.includes('Bouncing cats')&&++candidates===1)setTimeout(async()=>{const d=await crypto.subtle.digest('SHA-256',new TextEncoder().encode(m.source));const hash=[...new Uint8Array(d)].map(v=>v.toString(16).padStart(2,'0')).join('');walkiewareEngineEvent({kind:'previewEvent',event:{kind:'console',sourceHash:hash,requestID:m.renderID,event:{level:'error',message:'Paint failure: fixture runtime error'}}});},25);}; + } + walkiewareEngineEvent({kind:'inferenceSettings',checkedEdits:false});walkiewareEngineEvent({kind:'account',token:'fixture-only'});walkiewareEngineEvent({kind:'previewReady'});},process.argv.includes('--runtime-errors')); const before=await page.evaluate(()=>localStorage.getItem('walkieware-source-versions')); const checkedSketch={schema:'whistlegraph-drawing/v1',id:'33333333-3333-4333-8333-333333333333',revision:1,aspect:4/3,strokes:[[[100,500,0],[200,100,500],[300,500,1000]]]}; await page.evaluate(drawing=>walkiewareAskDrawing('Make the cats bounce',drawing),checkedSketch); @@ -179,7 +186,10 @@ const server = http.createServer(async (req, res) => { assert.equal(receipt.rounds[0].providerRequestID,'provider-1');assert.equal(receipt.rounds[1].usage.costUSD,.001); assert.ok(receipt.observations.every(o=>o.sourceHash.length===64));assert.equal(receipt.acceptance,'unreviewed'); assert.ok(!JSON.stringify(receipt).includes('Make the cats bounce'),'receipt has no prompt text'); - if(process.argv.includes('--repair-fails')){ + if(process.argv.includes('--review-fails-after-repair')){ + assert.equal(receipt.status,'failed');assert.deepEqual(receipt.checks.map(c=>c.code),['visual-fail']); + assert.equal(await page.evaluate(()=>localStorage.getItem('walkieware-source-versions')),before,'visual failure after a code repair cannot save or purchase another repair'); + }else if(process.argv.includes('--repair-fails')){ assert.equal(receipt.status,'failed');assert.equal(receipt.checks[0].code,'unsupported-hsl'); assert.equal(await page.evaluate(()=>localStorage.getItem('walkieware-source-versions')),before,'failed repair cannot commit the invalid candidate'); }else{ diff --git a/apple/whistlegraph/Tests/pixel-size.test.mjs b/apple/whistlegraph/Tests/pixel-size.test.mjs new file mode 100644 index 0000000000..6a8e6a7d33 --- /dev/null +++ b/apple/whistlegraph/Tests/pixel-size.test.mjs @@ -0,0 +1,35 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFile} from 'node:fs/promises'; +import {runInNewContext} from 'node:vm'; + +const swift = await readFile(new URL('../Sources/WhistlegraphAccount.swift', import.meta.url), 'utf8'); +const script = swift.match(/static let script = """\n([\s\S]*?)\n """/)[1]; +function preview(size, stored = '4') { + const messages = [], storage = new Map([['ac-density', stored]]); + const window = {top: {}, __walkiewarePixelSize: size, postMessage: m => messages.push(m), addEventListener() {}}; + runInNewContext(script, {window, location: {origin: 'https://aesthetic.computer'}, + localStorage: {setItem: (key, value) => storage.set(key, value)}, setInterval() {}}); + return {window, storage, messages}; +} + +test('native default wins over stale preview density before boot', () => { + const p = preview(undefined); + assert.equal(p.storage.get('ac-density'), '2'); + assert.equal(p.window.acPACK_DENSITY, 2); +}); + +test('all pixel sizes resize the running preview without reloading its source', () => { + const p = preview(3); + p.window.acSEND = () => assert.fail('A density change must not reload the piece'); + for (const size of [1, 4, 2, 3]) { + p.window.walkiewareSetPixelSize(size); + assert.equal(p.storage.get('ac-density'), String(size)); + assert.equal(p.window.acAutoDensityOverride, true); + assert.equal(p.messages.at(-1).type, 'ac-density-change'); + assert.equal(p.messages.at(-1).density, size); + } + const before = p.messages.length; + for (const invalid of [0, 5, 1.5, '2', NaN, null]) p.window.walkiewareSetPixelSize(invalid); + assert.equal(p.messages.length, before); +}); diff --git a/apple/whistlegraph/Tests/wwDabev-investigation.json b/apple/whistlegraph/Tests/wwDabev-investigation.json new file mode 100644 index 0000000000..69bc41ec91 --- /dev/null +++ b/apple/whistlegraph/Tests/wwDabev-investigation.json @@ -0,0 +1,16 @@ +{ + "thread": "wwDabev", + "savedVersion": 9, + "attemptStartedAt": "2026-10-03T16:20:27.267Z", + "elapsedMs": 40787, + "request": "Add 300 more smiley faces", + "result": "Visual review rejected initial candidate and one repair; restored saved v9.", + "retainedCandidate": { + "smileys": 19, + "stars": 7, + "placement": "inside the leaf" + }, + "evidence": "Authenticated cloud diagnostics and paired iPhone local receipt/checkpoint; candidate boot executed locally to count initialized objects.", + "statusBug": "A delayed rollback painted event overwrote the failed phase with Ready to play.", + "limits": "No new model generation or change to saved piece history." +} diff --git a/apple/whistlegraph/UITests/HeaderSheetsTests.swift b/apple/whistlegraph/UITests/HeaderSheetsTests.swift index 5cfb0d9ad6..ae842ec613 100644 --- a/apple/whistlegraph/UITests/HeaderSheetsTests.swift +++ b/apple/whistlegraph/UITests/HeaderSheetsTests.swift @@ -38,4 +38,26 @@ final class HeaderSheetsTests: XCTestCase { app.buttons["Done"].tap() XCTAssertTrue(waitForDisappearance(of: app.buttons["account-sign-out"]), "Done closes the account sheet") } + + func testPixelSizePersistsAcrossLaunches() { + let app = launch() + let code = app.buttons["workspace-settings"] + XCTAssertTrue(code.waitForExistence(timeout: 20)) + code.tap() + let picker = app.segmentedControls["pieces-pixel-size"] + XCTAssertTrue(picker.waitForExistence(timeout: 10)) + for size in [1, 2, 3, 4] { + picker.buttons["\(size)×"].tap() + XCTAssertTrue(picker.buttons["\(size)×"].isSelected) + } + let image = XCTAttachment(screenshot: app.screenshot()) + image.name = "Pixel size in the piece menu"; image.lifetime = .keepAlways; add(image) + app.terminate() + app.launch() + XCTAssertTrue(code.waitForExistence(timeout: 20)) + code.tap() + XCTAssertTrue(picker.waitForExistence(timeout: 10)) + XCTAssertTrue(picker.buttons["4×"].isSelected) + picker.buttons["2×"].tap() + } } diff --git a/apple/whistlegraph/UITests/StoryCardsTests.swift b/apple/whistlegraph/UITests/StoryCardsTests.swift new file mode 100644 index 0000000000..62f59141a3 --- /dev/null +++ b/apple/whistlegraph/UITests/StoryCardsTests.swift @@ -0,0 +1,80 @@ +import XCTest + +final class StoryCardsTests: XCTestCase { + func testExportStoryMP4OnPhone() throws { + let app = XCUIApplication() + app.launchEnvironment["WALKIE_NATIVE_SCREEN_FIXTURE"] = "story" + app.launch() + let cards = app.buttons["play-versions"] + XCTAssertTrue(cards.waitForExistence(timeout: 30)) + expectation(for: NSPredicate(format: "enabled == true"), evaluatedWith: cards) + waitForExpectations(timeout: 60) + cards.tap() + let export = app.buttons["story-export"] + XCTAssertTrue(export.waitForExistence(timeout: 15)) + let image = XCTAttachment(screenshot: app.screenshot()); image.name = "Phone story card"; image.lifetime = .keepAlways; add(image) + export.tap() + let progress = app.otherElements["story-export-progress"] + XCTAssertTrue(progress.waitForExistence(timeout: 10), "Export shows progress immediately") + XCTAssertTrue(app.buttons["story-export-cancel"].exists) + let rendering = XCTAttachment(screenshot: app.screenshot()); rendering.name = "MP4 rendering progress"; rendering.lifetime = .keepAlways; add(rendering) + let copy = app.buttons["story-save-video"] + XCTAssertTrue(copy.waitForExistence(timeout: 90), app.debugDescription) + let shared = XCTAttachment(screenshot: app.screenshot()); shared.name = "MP4 ready to save"; shared.lifetime = .keepAlways; add(shared) + app.terminate() + } + + func testCancelExportRestoresStoryControls() { + let app = XCUIApplication() + app.launchEnvironment["WALKIE_NATIVE_SCREEN_FIXTURE"] = "story" + app.launch() + let cards = app.buttons["play-versions"] + XCTAssertTrue(cards.waitForExistence(timeout: 30)) + expectation(for: NSPredicate(format: "enabled == true"), evaluatedWith: cards) + waitForExpectations(timeout: 60) + cards.tap() + let export = app.buttons["story-export"] + XCTAssertTrue(export.waitForExistence(timeout: 15)); export.tap() + let cancel = app.buttons["story-export-cancel"] + XCTAssertTrue(cancel.waitForExistence(timeout: 10)); cancel.tap() + XCTAssertTrue(app.buttons["story-pause"].waitForExistence(timeout: 10)) + XCTAssertTrue(export.isEnabled) + XCTAssertFalse(app.buttons["story-save-video"].exists) + app.terminate() + } + + func testCardsKeepCaptionsBelowPictureAndRestoreSelection() { + let app = XCUIApplication() + app.launchEnvironment["WALKIE_NATIVE_SCREEN_FIXTURE"] = "history" + app.launch() + let cards = app.buttons["play-versions"] + XCTAssertTrue(cards.waitForExistence(timeout: 20)) + expectation(for: NSPredicate(format: "enabled == true"), evaluatedWith: cards) + waitForExpectations(timeout: 60) + XCTAssertEqual(cards.label, "Open story cards") + cards.tap() + let pause = app.buttons["story-pause"] + XCTAssertTrue(pause.waitForExistence(timeout: 10)); pause.tap() + XCTAssertEqual(pause.label, "Resume story") + XCTAssertTrue(app.buttons["story-export"].exists) + XCTAssertTrue(app.staticTexts["story-version"].waitForExistence(timeout: 10)) + let caption = app.staticTexts["spoken-word"] + XCTAssertTrue(caption.exists) + let picture = app.webViews["story-picture"] + if picture.exists { XCTAssertGreaterThanOrEqual(app.staticTexts["story-version"].frame.minY, picture.frame.maxY) } + XCTAssertLessThan(caption.frame.maxY, app.frame.height * 0.8) + app.buttons["Next card"].tap() + let version = app.staticTexts["story-version"] + expectation(for: NSPredicate(format: "label == %@", "Running version 3"), evaluatedWith: version) + waitForExpectations(timeout: 10) + // v2 is a sibling branch and must not appear in this story. + XCTAssertTrue(app.otherElements.matching(NSPredicate(format: "label == %@", "Card 2 of 2")).firstMatch.exists) + let image = XCTAttachment(screenshot: app.screenshot()); image.name = "Story card with caption"; image.lifetime = .keepAlways; add(image) + app.buttons["Previous card"].tap() + expectation(for: NSPredicate(format: "label == %@", "Running version 1"), evaluatedWith: version) + waitForExpectations(timeout: 10) + app.buttons["Close version story"].tap() + let selected = app.buttons["version-3"] + XCTAssertTrue(selected.waitForExistence(timeout: 10)); XCTAssertEqual(selected.value as? String, "Current version") + } +} diff --git a/apple/whistlegraph/Whistlegraph.xcodeproj/project.pbxproj b/apple/whistlegraph/Whistlegraph.xcodeproj/project.pbxproj index 3b60173ae1..91bb0fdf7e 100644 --- a/apple/whistlegraph/Whistlegraph.xcodeproj/project.pbxproj +++ b/apple/whistlegraph/Whistlegraph.xcodeproj/project.pbxproj @@ -8,6 +8,7 @@ /* Begin PBXBuildFile section */ 0300D7A47360F4B79AB5B31F /* SwipeToTypeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 212791F2E33DED2E1C368651 /* SwipeToTypeTests.swift */; }; + 03C5B25FC742430AFD02AD1E /* StoryMovieSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = 250E0B1999C7810299ACB7A9 /* StoryMovieSheet.swift */; }; 10C0A16D4803EC60EAC29531 /* RecordingTrim.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC1EFF8BAE7678E320E3DCA0 /* RecordingTrim.swift */; }; 123228FADA6696AB354306C7 /* InputButtonLabels.swift in Sources */ = {isa = PBXBuildFile; fileRef = 35643029A3E82EEDEAC8DED6 /* InputButtonLabels.swift */; }; 13604BE92157CDBFDD32AA19 /* WorkspaceCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 147EAC245AD250B9B652F903 /* WorkspaceCoordinator.swift */; }; @@ -17,7 +18,9 @@ 275555C74507A6F62B0F037B /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = FF5F023E15041FF61F9FD358 /* Assets.xcassets */; }; 2F01E350A7FF55359B002344 /* ButtonSounds.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5ABFC6C11146A1540B635EE6 /* ButtonSounds.swift */; }; 2F15D5A580A90A18E92D90F1 /* AudioBenchmark.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1CC14C186FE1CB5B19AD1D72 /* AudioBenchmark.swift */; }; + 3CF6178F7BA1702ADFA457B7 /* StoryControls.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8C42750EF4F7E96146B03585 /* StoryControls.swift */; }; 41FC24FA4CF0D6C4D0C9E7C3 /* WhistlegraphTheme.swift in Sources */ = {isa = PBXBuildFile; fileRef = D25F85E540016CEBC3A6BC09 /* WhistlegraphTheme.swift */; }; + 47DADA794ABD0CD203CFF48C /* StoryExport.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88B494AD8CD86FCC5DE0E4CC /* StoryExport.swift */; }; 49D6E6ED3128002039D4F931 /* UtteranceRecording.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8F9B92C3457F07429C8C2600 /* UtteranceRecording.swift */; }; 4B35B066F49BDD9BD8B90B5A /* WhistlegraphScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = E66B3ADC4D6D837FABC53B8F /* WhistlegraphScreen.swift */; }; 5646BD41AA620CC47BF89361 /* Web in Resources */ = {isa = PBXBuildFile; fileRef = 4BA75145769FC4401A11BBE8 /* Web */; }; @@ -25,6 +28,7 @@ 749AC25AE02AAE4BCDA5683A /* VisualCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5DF0A171A5771A7A1FD56852 /* VisualCapture.swift */; }; 77291EEE768914D5F5EDB84B /* PlaybackWord.swift in Sources */ = {isa = PBXBuildFile; fileRef = E97104E13862EE30444D07C9 /* PlaybackWord.swift */; }; 77EC5FCFB7F9BF4A6BD4F94B /* NativeSignIn.swift in Sources */ = {isa = PBXBuildFile; fileRef = 21D4A924CCE7B4BEC62E98A9 /* NativeSignIn.swift */; }; + 87EDDAE34B992D308236F1A9 /* StoryCardsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8B4655A00B6DC810F7203D1 /* StoryCardsTests.swift */; }; 8A9F944BA95C971D70ADDF6B /* VersionNarrator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6D137824491A01D4C19417B8 /* VersionNarrator.swift */; }; 9DBA1AF2AD7E3EDF69C8132A /* CodeTicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 10E2AAA0A3F2BA1A5AEBED58 /* CodeTicker.swift */; }; A028C6EA0D6572E890932C2D /* PromptKeySounds.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4F9A906C6B2232BD07BEFE5 /* PromptKeySounds.swift */; }; @@ -36,6 +40,7 @@ D3D16A0333764D0E5C3B620D /* SpeechCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = B7A25F803FAC121BFECC1AD7 /* SpeechCapture.swift */; }; D8E8FFEA41DF3B0073DC5373 /* TalkHoldInput.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4728CB634CB3E4AED4F4D353 /* TalkHoldInput.swift */; }; D9E54D2F66B9FC35483A8F45 /* PlaybackCaption.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4D3B156A3435E646EE0E6EEA /* PlaybackCaption.swift */; }; + E55F422280707B98E1D3E842 /* StoryTape.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03AEEA7D7E12C650CB618757 /* StoryTape.swift */; }; FAC2911EFFCA887527D2415F /* WhistlegraphDrawing.swift in Sources */ = {isa = PBXBuildFile; fileRef = 99D9A8C801D3B9D1192D1143 /* WhistlegraphDrawing.swift */; }; /* End PBXBuildFile section */ @@ -50,6 +55,7 @@ /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ + 03AEEA7D7E12C650CB618757 /* StoryTape.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StoryTape.swift; sourceTree = ""; }; 060870DF76C6FC078A48A276 /* WhistlegraphAccount.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WhistlegraphAccount.swift; sourceTree = ""; }; 10E2AAA0A3F2BA1A5AEBED58 /* CodeTicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CodeTicker.swift; sourceTree = ""; }; 147EAC245AD250B9B652F903 /* WorkspaceCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceCoordinator.swift; sourceTree = ""; }; @@ -57,6 +63,7 @@ 1CC14C186FE1CB5B19AD1D72 /* AudioBenchmark.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioBenchmark.swift; sourceTree = ""; }; 212791F2E33DED2E1C368651 /* SwipeToTypeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwipeToTypeTests.swift; sourceTree = ""; }; 21D4A924CCE7B4BEC62E98A9 /* NativeSignIn.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NativeSignIn.swift; sourceTree = ""; }; + 250E0B1999C7810299ACB7A9 /* StoryMovieSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StoryMovieSheet.swift; sourceTree = ""; }; 35643029A3E82EEDEAC8DED6 /* InputButtonLabels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InputButtonLabels.swift; sourceTree = ""; }; 3F98F7D3CE551FC6458D1F37 /* HeaderSheetsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HeaderSheetsTests.swift; sourceTree = ""; }; 4728CB634CB3E4AED4F4D353 /* TalkHoldInput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TalkHoldInput.swift; sourceTree = ""; }; @@ -68,6 +75,8 @@ 5D233DBB6F87184A91C13D41 /* MusicalInput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MusicalInput.swift; sourceTree = ""; }; 5DF0A171A5771A7A1FD56852 /* VisualCapture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VisualCapture.swift; sourceTree = ""; }; 6D137824491A01D4C19417B8 /* VersionNarrator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VersionNarrator.swift; sourceTree = ""; }; + 88B494AD8CD86FCC5DE0E4CC /* StoryExport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StoryExport.swift; sourceTree = ""; }; + 8C42750EF4F7E96146B03585 /* StoryControls.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StoryControls.swift; sourceTree = ""; }; 8F9B92C3457F07429C8C2600 /* UtteranceRecording.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UtteranceRecording.swift; sourceTree = ""; }; 99D9A8C801D3B9D1192D1143 /* WhistlegraphDrawing.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WhistlegraphDrawing.swift; sourceTree = ""; }; A542DD3D25782F331540474E /* Whistlegraph.app */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.application; path = Whistlegraph.app; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -75,6 +84,7 @@ B4F9A906C6B2232BD07BEFE5 /* PromptKeySounds.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PromptKeySounds.swift; sourceTree = ""; }; B55FB4AA387E716F9FEE0F83 /* SequenceBenchmark.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SequenceBenchmark.swift; sourceTree = ""; }; B7A25F803FAC121BFECC1AD7 /* SpeechCapture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpeechCapture.swift; sourceTree = ""; }; + B8B4655A00B6DC810F7203D1 /* StoryCardsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StoryCardsTests.swift; sourceTree = ""; }; BA5F188938A81E704BB7F9F9 /* WhistlegraphApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WhistlegraphApp.swift; sourceTree = ""; }; CC1EFF8BAE7678E320E3DCA0 /* RecordingTrim.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RecordingTrim.swift; sourceTree = ""; }; D25F85E540016CEBC3A6BC09 /* WhistlegraphTheme.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WhistlegraphTheme.swift; sourceTree = ""; }; @@ -122,6 +132,10 @@ CC1EFF8BAE7678E320E3DCA0 /* RecordingTrim.swift */, B55FB4AA387E716F9FEE0F83 /* SequenceBenchmark.swift */, B7A25F803FAC121BFECC1AD7 /* SpeechCapture.swift */, + 8C42750EF4F7E96146B03585 /* StoryControls.swift */, + 88B494AD8CD86FCC5DE0E4CC /* StoryExport.swift */, + 250E0B1999C7810299ACB7A9 /* StoryMovieSheet.swift */, + 03AEEA7D7E12C650CB618757 /* StoryTape.swift */, 4728CB634CB3E4AED4F4D353 /* TalkHoldInput.swift */, 8F9B92C3457F07429C8C2600 /* UtteranceRecording.swift */, 6D137824491A01D4C19417B8 /* VersionNarrator.swift */, @@ -141,6 +155,7 @@ isa = PBXGroup; children = ( 3F98F7D3CE551FC6458D1F37 /* HeaderSheetsTests.swift */, + B8B4655A00B6DC810F7203D1 /* StoryCardsTests.swift */, 212791F2E33DED2E1C368651 /* SwipeToTypeTests.swift */, ); path = UITests; @@ -275,6 +290,10 @@ 10C0A16D4803EC60EAC29531 /* RecordingTrim.swift in Sources */, C4390906B03E8E8666BC54A3 /* SequenceBenchmark.swift in Sources */, D3D16A0333764D0E5C3B620D /* SpeechCapture.swift in Sources */, + 3CF6178F7BA1702ADFA457B7 /* StoryControls.swift in Sources */, + 47DADA794ABD0CD203CFF48C /* StoryExport.swift in Sources */, + 03C5B25FC742430AFD02AD1E /* StoryMovieSheet.swift in Sources */, + E55F422280707B98E1D3E842 /* StoryTape.swift in Sources */, D8E8FFEA41DF3B0073DC5373 /* TalkHoldInput.swift in Sources */, 49D6E6ED3128002039D4F931 /* UtteranceRecording.swift in Sources */, 8A9F944BA95C971D70ADDF6B /* VersionNarrator.swift in Sources */, @@ -294,6 +313,7 @@ buildActionMask = 2147483647; files = ( 19717448D7598ADAABBF739E /* HeaderSheetsTests.swift in Sources */, + 87EDDAE34B992D308236F1A9 /* StoryCardsTests.swift in Sources */, 0300D7A47360F4B79AB5B31F /* SwipeToTypeTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -396,7 +416,7 @@ CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_STYLE = Automatic; COPY_PHASE_STRIP = NO; - CURRENT_PROJECT_VERSION = 92; + CURRENT_PROJECT_VERSION = 95; DEBUG_INFORMATION_FORMAT = dwarf; DEVELOPMENT_TEAM = FB5948YR3S; ENABLE_STRICT_OBJC_MSGSEND = YES; @@ -481,7 +501,7 @@ CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_STYLE = Automatic; COPY_PHASE_STRIP = NO; - CURRENT_PROJECT_VERSION = 92; + CURRENT_PROJECT_VERSION = 95; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; DEVELOPMENT_TEAM = FB5948YR3S; ENABLE_NS_ASSERTIONS = NO; diff --git a/apple/whistlegraph/bundle.mjs b/apple/whistlegraph/bundle.mjs index d6f164506f..2f50a08db8 100644 --- a/apple/whistlegraph/bundle.mjs +++ b/apple/whistlegraph/bundle.mjs @@ -14,3 +14,5 @@ for(const tree of ["src","phone","context"]) { } await copyFile(new URL('apple/aesel/Resources/ComicRelief-Bold.ttf',root),new URL('Resources/Web/ComicRelief-Bold.ttf',base)); + +await copyFile(new URL('system/public/aesthetic.computer/lib/canvas-tape.mjs',root),new URL('Resources/Web/canvas-tape.mjs',base)); diff --git a/apple/whistlegraph/project.yml b/apple/whistlegraph/project.yml index 6cd93d2fdc..95973417cb 100644 --- a/apple/whistlegraph/project.yml +++ b/apple/whistlegraph/project.yml @@ -9,7 +9,7 @@ settings: CODE_SIGN_STYLE: Automatic SWIFT_VERSION: "5.0" MARKETING_VERSION: "0.1.0" - CURRENT_PROJECT_VERSION: "92" + CURRENT_PROJECT_VERSION: "95" targets: Whistlegraph: type: application diff --git a/system/public/aesthetic.computer/lib/canvas-tape.mjs b/system/public/aesthetic.computer/lib/canvas-tape.mjs new file mode 100644 index 0000000000..12d281f831 --- /dev/null +++ b/system/public/aesthetic.computer/lib/canvas-tape.mjs @@ -0,0 +1,15 @@ +// Shared hardware encoder for AC's HD tapes and Whistlegraph story exports. +// Only canvas pixels and explicitly supplied audio tracks enter the recording. +export function createCanvasTapeRecorder(canvas, {fps = 30, audioTracks = [], videoBitsPerSecond = 8e6, mp4Only = false} = {}) { + const types = ['video/mp4;codecs=avc1.640028,mp4a.40.2', 'video/mp4', ...(!mp4Only ? ['video/webm;codecs=vp9,opus', 'video/webm'] : [])]; + const mimeType = types.find(type => MediaRecorder.isTypeSupported(type)); + if (!mimeType) throw Error('This device cannot encode an MP4 canvas tape.'); + const stream = canvas.captureStream(fps); + const ownedTracks = stream.getVideoTracks(); + audioTracks.forEach(track => stream.addTrack(track)); + try { + const recorder = new MediaRecorder(stream, {mimeType, videoBitsPerSecond}); + recorder.addEventListener('stop', () => ownedTracks.forEach(track => track.stop()), {once:true}); + return recorder; + } catch (error) { ownedTracks.forEach(track => track.stop()); throw error; } +} -- 2.51.2