diff --git a/apple/whistlegraph/README.md b/apple/whistlegraph/README.md index d9a21dc6dc..8f478bf98d 100644 --- a/apple/whistlegraph/README.md +++ b/apple/whistlegraph/README.md @@ -49,7 +49,7 @@ The browser check uses Puppeteer and Chrome with mock inference; it makes no mod Every generated edit runs syntax/API checks and waits for matching-source runtime feedback before the visual review. Client errors trigger at most one repair across code and visual checks. A candidate that still fails restores the previous saved version; its failure remains visible after the restored frame paints. Missing or stale evidence cannot trigger a paid repair or save a version. There is no opt-out setting. -The initial generation uses DeepSeek V4.1 Flash with four tool rounds and one output continuation. The single repair uses DeepSeek V4 Pro with a 1,024-token thinking budget, two tool rounds and no continuation. Provider calls retain the 4,096-token output limit. Remote requests and remote edit descriptions use the same 96-grapheme, single-line limit as phone typing; oversized requests are rejected before generation. Diagnostic context and source code are separate from the short user request. Code generation/checking has a 75-second deadline; visual review has a separate 90-second deadline. These bound work, not cost. A painted frame is execution evidence, not visual acceptance. +The initial generation uses DeepSeek V4.1 Flash with four tool rounds and one output continuation. The single repair uses DeepSeek V4 Pro (or V4.1 Flash when the request carries chalk, since V4 Pro accepts no images) with a 1,024-token thinking budget, two tool rounds and no continuation. Provider calls retain the 4,096-token output limit. Remote requests and remote edit descriptions use the same 96-grapheme, single-line limit as phone typing; oversized requests are rejected before generation. Diagnostic context and source code are separate from the short user request. Code generation/checking has a 75-second deadline; visual review has a separate 90-second deadline. These bound work, not cost. A painted frame is execution evidence, not visual acceptance. ## Pixel size diff --git a/apple/whistlegraph/Resources/Web/engine.mjs b/apple/whistlegraph/Resources/Web/engine.mjs index 6554e4bc3f..2c7ba48f5b 100644 --- a/apple/whistlegraph/Resources/Web/engine.mjs +++ b/apple/whistlegraph/Resources/Web/engine.mjs @@ -66,7 +66,7 @@ let accountToken='',accountHandle='',accountPalette=[],accountVerification=Promi let personalAccess=null,turnPersonalAccess=false; let turnHandle='',turnModel='',activeModel='',braincells=null,braincellsError='',creditsRequest=0; function selectedModel(handle=accountHandle){try{return localStorage.getItem('whistlegraph-model-'+handle)||'';}catch{return '';}} -function profile(repair=false){return generationProfile(busy?turnHandle:accountHandle,{repair,personalAccess:busy?turnPersonalAccess:hasPersonalAccess(personalAccess),model:busy?turnModel:selectedModel()});} +function profile(repair=false){return generationProfile(busy?turnHandle:accountHandle,{repair,image:busy&&!!inputData(turnRequest)?.drawing,personalAccess:busy?turnPersonalAccess:hasPersonalAccess(personalAccess),model:busy?turnModel:selectedModel()});} async function refreshBraincells(){ benchmark('braincellsRequest'); const currentToken=token,request=++creditsRequest; @@ -239,6 +239,13 @@ function updateFeed() { $('live-time').hidden=!busy;nativeSnapshot(); } +// A phone loading the piece in the production frame over cellular can take +// seconds to paint. Two seconds turned late paints into unverified-render +// rollbacks (wgZuhus, 2026-10-08), so wait longer but stop on a runtime error. +async function waitForPaint(target, stopped, ms=20000) { + const until=Date.now()+ms; + while(Date.now()setTimeout(resolve,50)); +} function render(value) { previewSource=value;painted=false;feedback=null;previewHash=null; const id=++renderID; @@ -283,7 +290,7 @@ async function checkVisualResult() { inspect:async()=>{ phase('Checking picture…'); const target=source,hash=await hashSource(target),id=renderID; - for(let i=0;i<100&&!signal.aborted&&(!painted||lastPaintedSource!==target);i++)await new Promise(resolve=>setTimeout(resolve,20)); + await waitForPaint(target,()=>signal.aborted); signal.throwIfAborted(); const runtime=validateCandidate(target,feedback,hash); if(!runtime.passed)throw Error('Visual check needs a working current preview: '+runtime.findings.map(f=>f.code).join(', ')); @@ -464,7 +471,7 @@ async function ask(text,displayText=text,advice=null,starter=null,localText=text generate:async(task,repair)=>{server?.close();server=makeServer({repair});turnSucceeded=false;await server.startTurn(drawingContent(task,chalkImage));return turnSucceeded;}, inspect:async()=>{ const target=source,hash=await hashSource(target); - for(let i=0;i<100&&!turnCancelled&&(!painted||lastPaintedSource!==target);i++)await new Promise(resolve=>setTimeout(resolve,20)); + await waitForPaint(target,()=>turnCancelled); return validateCandidate(target,feedback,hash); }}); if(result.validation){ diff --git a/apple/whistlegraph/Resources/Web/generation-policy.mjs b/apple/whistlegraph/Resources/Web/generation-policy.mjs index ec2feee434..e06befcd43 100644 --- a/apple/whistlegraph/Resources/Web/generation-policy.mjs +++ b/apple/whistlegraph/Resources/Web/generation-policy.mjs @@ -10,11 +10,13 @@ export function modelChoices(handle, {personalAccess=false}={}) { return Object.entries(MODEL_LABELS).filter(([id])=>(handle==='jeffrey'||personalAccess)||(!id.startsWith('anthropic/')&&!id.startsWith('openai/'))).map(([id,label])=>({id,label})); } // The handle comes from verifyAccount, never from a preference or command. -export function generationProfile(handle, {repair=false,model='',personalAccess=false}={}) { +// DeepSeek V4 Pro is text-only on OpenRouter (404 "No endpoints found that support +// image input"), so a repair that carries chalk stays on the image-capable default. +export function generationProfile(handle, {repair=false,model='',personalAccess=false,image=false}={}) { const personal=handle==='jeffrey'||personalAccess; const selected=modelChoices(handle,{personalAccess}).some(option=>option.id===model)?model:''; const chosen=selected||(personal?'anthropic/claude-opus-5':DEFAULT_MODEL); - const resolved=repair&&chosen===DEFAULT_MODEL?REPAIR_MODEL:chosen; + const resolved=repair&&!image&&chosen===DEFAULT_MODEL?REPAIR_MODEL:chosen; if(personal)return {personalRelay:resolved.startsWith('anthropic/')||resolved.startsWith('openai/'),model:resolved,maxTokens:16384,rounds:repair?4:12,outputContinuations:repair?1:2,reasoning:{max_tokens:4096},thinking:{type:'enabled',budget_tokens:4096}}; return {model:resolved,maxTokens:4096,rounds:repair?2:4,outputContinuations:repair?0:1,reasoning:repair?{max_tokens:1024}:{effort:'none'},thinking:repair?{type:'enabled',budget_tokens:1024}:{type:'disabled'}}; } diff --git a/apple/whistlegraph/Sources/WhistlegraphScreen.swift b/apple/whistlegraph/Sources/WhistlegraphScreen.swift index d8facc19ba..90fcb448aa 100644 --- a/apple/whistlegraph/Sources/WhistlegraphScreen.swift +++ b/apple/whistlegraph/Sources/WhistlegraphScreen.swift @@ -96,6 +96,13 @@ struct WhistlegraphScreen: View { session.previewFormat.fit(width: UIScreen.main.bounds.width - session.layout.pageInset * 2 - 12, height: min(380, UIScreen.main.bounds.height * 0.4)) } + // With the keyboard and composer up, the full-size preview pushes the text + // field under the keyboard. Scale the framed preview (not the runtime, which + // would resize the piece) so the sketch stays visible above the composer. + private var composeScale: CGFloat { + guard showComposer, !narrator.isPlaying else { return 1 } + return min(1, UIScreen.main.bounds.height * 0.2 / (previewSize.height + 12)) + } var body: some View { VStack(spacing: narrator.isPlaying ? 0 : session.layout.spacing) { HStack { @@ -115,7 +122,7 @@ struct WhistlegraphScreen: View { .accessibilityLabel("Open story cards").accessibilityIdentifier("play-versions") } }.frame(height: narrator.isPlaying ? 0 : nil).clipped().accessibilityHidden(narrator.isPlaying) - if !narrator.isPlaying { + if !narrator.isPlaying && !showComposer { HStack { WarePicker(session: session) { narrator.stop(); showComposer = false } Spacer(minLength: 0) @@ -148,6 +155,9 @@ struct WhistlegraphScreen: View { .overlay { WhistlegraphPreviewInset() } .padding(6) .background(WhistlegraphWoodFrame()) + .scaleEffect(composeScale, anchor: .top) + .frame(height: composeScale < 1 ? (previewSize.height + 12) * composeScale : nil, alignment: .top) + .animation(reduceMotion ? nil : .easeInOut(duration: 0.25), value: showComposer) .accessibilityIdentifier("story-picture") .overlay { if narrator.isPlaying && !exporter.requested { diff --git a/apple/whistlegraph/Tests/generation-policy.test.mjs b/apple/whistlegraph/Tests/generation-policy.test.mjs index d43a3d38b9..dd926b2e73 100644 --- a/apple/whistlegraph/Tests/generation-policy.test.mjs +++ b/apple/whistlegraph/Tests/generation-policy.test.mjs @@ -5,6 +5,8 @@ for(const handle of ['', 'fixture', 'Jeffrey', 'jeffrey-other']) { assert.equal(generationProfile(handle).rounds,4); assert.equal(generationProfile(handle).maxTokens,4096); assert.equal(generationProfile(handle,{repair:true}).model,REPAIR_MODEL); + // The text-only repair model cannot take the chalk PNG. + assert.equal(generationProfile(handle,{repair:true,image:true}).model,DEFAULT_MODEL); } assert.equal(generationProfile('jeffrey').model,'anthropic/claude-opus-5'); assert.equal(generationProfile('jeffrey').rounds,12); diff --git a/apple/whistlegraph/Tests/native-bridge.test.cjs b/apple/whistlegraph/Tests/native-bridge.test.cjs index c199b4c4cf..193b6ea00e 100644 --- a/apple/whistlegraph/Tests/native-bridge.test.cjs +++ b/apple/whistlegraph/Tests/native-bridge.test.cjs @@ -182,8 +182,8 @@ const server = http.createServer(async (req, res) => { // Exercise the actual microphone bridge: drawing and sound arrive in one final event. await page.evaluate(drawing=>{ voiceStart();const id=__nativeMessages.filter(m=>m.action==='start').at(-1).id; - walkieNativeEvent({id,kind:'listening'}); - walkieNativeEvent({id,kind:'mixedFinal',drawing,text:JSON.stringify({transcript:'follow this sweep',words:[{text:'sweep',atMs:700,durationMs:250}],sound:{schema:'whistlegraph-sound/v1',durationMs:1200,audibleMs:1000,frames:[{atMs:700,rms:.8,pitchHz:440}],onsetsMs:[700],recordingID:'22222222-2222-4222-8222-222222222222'}})}); + whistlegraphNativeEvent({id,kind:'listening'}); + whistlegraphNativeEvent({id,kind:'mixedFinal',drawing,text:JSON.stringify({transcript:'follow this sweep',words:[{text:'sweep',atMs:700,durationMs:250}],sound:{schema:'whistlegraph-sound/v1',durationMs:1200,audibleMs:1000,frames:[{atMs:700,rms:.8,pitchHz:440}],onsetsMs:[700],recordingID:'22222222-2222-4222-8222-222222222222'}})}); },{...sketch,revision:5}); await page.waitForFunction(()=>JSON.parse(localStorage.getItem('whistlegraph-source-versions')).head===2&&!whistlegraphIsBusy()); assert.equal(requests,2);const mixed=JSON.stringify(inferenceBodies[1]); @@ -246,7 +246,8 @@ const server = http.createServer(async (req, res) => { assert.ok(inferenceBodies.every(b=>b.max_tokens===4096)); assert.equal(inferenceBodies[0].model,'deepseek/deepseek-v4.1-flash'); assert.equal(inferenceBodies[0].thinking.type,'disabled'); - assert.equal(inferenceBodies[1].model,'deepseek/deepseek-v4-pro'); + // V4 Pro is text-only on OpenRouter; a chalk repair stays on the image-capable model. + assert.equal(inferenceBodies[1].model,'deepseek/deepseek-v4.1-flash'); assert.equal(inferenceBodies[1].thinking.budget_tokens,1024); const receipt=await page.evaluate(()=>JSON.parse(localStorage.getItem('whistlegraph-source-receipts')).at(-1).receipt); assert.equal(receipt.repairs,1);assert.equal(receipt.rounds.length,process.argv.includes('--repair-fails')?2:3);assert.equal(receipt.rounds[0].reportedModel,'fixture/reported'); diff --git a/apple/whistlegraph/Tests/personal-relay-live.mjs b/apple/whistlegraph/Tests/personal-relay-live.mjs index aba6d20f02..d089569b7e 100644 --- a/apple/whistlegraph/Tests/personal-relay-live.mjs +++ b/apple/whistlegraph/Tests/personal-relay-live.mjs @@ -14,9 +14,11 @@ const server=createServer(async(req,res)=>{try{const file=resolve(root,'.'+new U await new Promise(r=>server.listen(0,'127.0.0.1',r)); const browser=await puppeteer.launch({headless:true,executablePath:'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'}); try{ - const page=await browser.newPage();await page.setViewport({width:430,height:800}); + const page=await browser.newPage(); + // WHISTLEGRAPH_LIVE_SLOW=1 approximates a phone on cellular: 4x CPU, 300 ms latency. + if(process.env.WHISTLEGRAPH_LIVE_SLOW){const cdp=await page.createCDPSession();await cdp.send('Emulation.setCPUThrottlingRate',{rate:4});await cdp.send('Network.enable');await cdp.send('Network.emulateNetworkConditions',{offline:false,latency:300,downloadThroughput:1.5e6,uploadThroughput:7.5e5});}await page.setViewport({width:430,height:800}); let ready=false,lastRender,snapshot,reviewFrames=0;const errors=[],inference=[]; - page.on('pageerror',e=>errors.push(e.message));page.on('request',r=>{if(r.url().includes('/api/easel-inference')||r.url().includes('/api/aesel/'))inference.push(new URL(r.url()).pathname);}); + page.on('pageerror',e=>errors.push(e.message));page.on('console',m=>{if(m.type()==='error')console.log('console',m.text().slice(0,200));});page.on('request',r=>{if(r.url().includes('/api/easel-inference')||r.url().includes('/api/aesel/'))inference.push(new URL(r.url()).pathname);}); const runtime=()=>page.frames().find(f=>f.url().startsWith('https://aesthetic.computer/')); const render=async m=>{lastRender=m;if(ready)await runtime().evaluate(m=>window.whistlegraphRender(m.source,'relay-live-test',m.renderID),m);}; await page.exposeFunction('__nativeBridge',async m=>{ @@ -24,7 +26,7 @@ try{ if(m.action==='snapshot')snapshot=m.snapshot; if(m.action==='render')await render(m); if(m.action==='previewReady'){ready=true;if(lastRender)await render(lastRender);await page.evaluate(()=>window.whistlegraphEngineEvent({kind:'previewReady'}));} - if(m.action==='previewEvent')await page.evaluate(e=>window.whistlegraphEngineEvent({kind:'previewEvent',event:e}),m.event); + if(m.action==='previewEvent')console.log('previewEvent',JSON.stringify(m.event).slice(0,200));if(m.action==='render')console.log('render',m.renderID,m.source.length);if(m.action==='previewEvent')await page.evaluate(e=>window.whistlegraphEngineEvent({kind:'previewEvent',event:e}),m.event); if(m.action==='visualCapture'){ const frames=[],started=Date.now(); for(let i=0;i<4;i++){ @@ -49,10 +51,15 @@ try{ for(let i=0;i<60 && (!ready||snapshot?.handle!=='jeffrey');i++)await new Promise(r=>setTimeout(r,500)); if(!ready||snapshot?.handle!=='jeffrey'){console.log(JSON.stringify({ready,handle:snapshot?.handle,errors,frames:await Promise.all(page.frames().map(async f=>({url:f.url(),state:await f.evaluate(()=>({title:document.title,preloaded:!!window.preloaded,send:typeof window.acSEND,render:typeof window.whistlegraphRender,text:document.body.innerText.slice(0,100)})).catch(()=>null)})))}));await page.screenshot({path:out+'/startup.png'});} if(!ready||snapshot?.handle!=='jeffrey')throw Error('Runtime or signed-in account not ready'); + // Build 108+ requires first-use AI consent bound to the verified handle and token. + await page.evaluate(token=>window.whistlegraphSetAIConsent({creation:true,handle:'jeffrey',token}),token); console.log('Real browser ready; generating through personal relay'); - await page.evaluate(()=>window.whistlegraphAsk('Make a pink circle centered on black. Keep it static.')); - await new Promise(r=>setTimeout(r,500)); - const stored=await page.evaluate(()=>({attempt:JSON.parse(localStorage.getItem('walkieware-source-attempt')),source:localStorage.getItem('walkieware-source'),receipts:JSON.parse(localStorage.getItem('walkieware-source-receipts'))})); + // WHISTLEGRAPH_LIVE_CHALK=1 sends a chalk sketch with the request, as the phone does with Chalk on. + if(process.env.WHISTLEGRAPH_LIVE_CHALK){const sketch=JSON.parse(await readFile('apple/whistlegraph/Tests/fixtures/chalk-sailboat.json','utf8'));await page.evaluate(d=>window.whistlegraphAskDrawing('anime palm tree',d),sketch);} + else await page.evaluate(()=>window.whistlegraphAsk('Make a pink circle centered on black. Keep it static.')); + // Wait for the whole turn (generation, checks, review), not a fixed delay. + for(let i=0;i<400;i++){await new Promise(r=>setTimeout(r,1000));if(!(await page.evaluate(()=>window.whistlegraphIsBusy())))break;} + const stored=await page.evaluate(()=>({attempt:JSON.parse(localStorage.getItem('whistlegraph-source-attempt')),source:localStorage.getItem('whistlegraph-source'),receipts:JSON.parse(localStorage.getItem('whistlegraph-source-receipts'))})); const report={provider:snapshot?.inference?.provider,attempt:stored.attempt,source:stored.source,reviewFrames,apiCalls:inference,errors,receipt:stored.receipts?.at(-1)?.receipt}; await writeFile(out+'/live-report.json',JSON.stringify(report,null,2));console.log(JSON.stringify({provider:report.provider,attempt:report.attempt,reviewFrames,errors})); if(inference.some(path=>path==='/api/easel-inference')||reviewFrames<4||stored.attempt?.status!=='completed')throw Error('Personal generation/review did not pass'); diff --git a/apple/whistlegraph/UITests/SwipeToTypeTests.swift b/apple/whistlegraph/UITests/SwipeToTypeTests.swift index d747ea7dfa..19a6e0e05a 100644 --- a/apple/whistlegraph/UITests/SwipeToTypeTests.swift +++ b/apple/whistlegraph/UITests/SwipeToTypeTests.swift @@ -59,6 +59,9 @@ final class SwipeToTypeTests: XCTestCase { let preview = app.webViews.firstMatch XCTAssertTrue(preview.exists) XCTAssertLessThanOrEqual(preview.frame.maxY, app.keyboards.firstMatch.frame.minY) + // The field and its Send button must clear the keyboard, not just the preview. + XCTAssertLessThanOrEqual(editor.frame.maxY, app.keyboards.firstMatch.frame.minY) + XCTAssertLessThanOrEqual(app.buttons["request-send"].frame.maxY, app.keyboards.firstMatch.frame.minY) editor.tap() editor.typeText(String(repeating: "a", count: 110)) XCTAssertEqual((editor.value as? String)?.count, 96)