diff --git a/aesel/src/app-server.mjs b/aesel/src/app-server.mjs index ad614656c3..13fc99a206 100644 --- a/aesel/src/app-server.mjs +++ b/aesel/src/app-server.mjs @@ -23,7 +23,7 @@ export class AppServer extends EventEmitter { super(); this.cwd = cwd; this.command = command; - this.args = args.includes("app-server") ? [...args, ...codexMcpArgs(cwd,environment)] : args; + this.args = args.includes("app-server") && !dynamicTools ? [...args, ...codexMcpArgs(cwd,environment)] : args; this.environment = environment; this.resumeThreadId = resumeThreadId; this.developerInstructions = developerInstructions; diff --git a/apple/whistlegraph/Resources/Web/engine.mjs b/apple/whistlegraph/Resources/Web/engine.mjs index a64424ddd8..bb1a5830d1 100644 --- a/apple/whistlegraph/Resources/Web/engine.mjs +++ b/apple/whistlegraph/Resources/Web/engine.mjs @@ -1,3 +1,4 @@ +import {fetchPersonalAccess,hasPersonalAccess} from './personal-access.mjs'; import {createAIConsentGate} from './ai-consent.mjs'; import {SourceEditor} from './source-editor.mjs'; import {inferenceError} from './inference-error.mjs'; @@ -59,9 +60,10 @@ let outputStream='',reasoningStream='',code = '', codeItem = '', firstDelta = fa const events = []; const signIn=document.createElement('button');signIn.id='connect-ac';signIn.textContent='Account';signIn.onclick=()=>post({action:'signIn'});const identity=document.createElement('div');identity.id='walkieware-identity';const codeLabel=document.createElement('span');codeLabel.id='walkieware-thread';identity.append(signIn,codeLabel);document.body.append(identity); let accountToken='',accountHandle='',accountPalette=[],accountVerification=Promise.resolve(); +let personalAccess=null,turnPersonalAccess=false; let turnHandle='',turnModel='',activeModel='',braincells=null,braincellsError='',creditsRequest=0; function selectedModel(handle=accountHandle){try{return localStorage.getItem('whistlegraph-model-'+handle)||'';}catch{return '';}} -function profile(repair=false){return generationProfile(busy?turnHandle:accountHandle,{repair,model:busy?turnModel:selectedModel()});} +function profile(repair=false){return generationProfile(busy?turnHandle:accountHandle,{repair,personalAccess:busy?turnPersonalAccess:hasPersonalAccess(personalAccess),model:busy?turnModel:selectedModel()});} async function refreshBraincells(){ benchmark('braincellsRequest'); const currentToken=token,request=++creditsRequest; @@ -80,7 +82,7 @@ async function refreshBraincells(){ } function inferenceSnapshot(){ const model=activeModel||profile().model,receipt=activeReceipt?.value||receipts.rows.at(-1)?.receipt; - return {model,label:MODEL_LABELS[model]||model,provider:profile().personalRelay?(model.startsWith('openai/')?'Personal Codex':'Personal Claude'):'OpenRouter',selection:profile().model,models:modelChoices(accountHandle),braincells,braincellsError,threadCost:receipts.cost.snapshot(), + return {model,label:MODEL_LABELS[model]||model,provider:profile().personalRelay?(model.startsWith('openai/')?'Personal Codex':'Personal Claude'):'OpenRouter',selection:profile().model,models:modelChoices(accountHandle,{personalAccess:hasPersonalAccess(personalAccess)}),braincells,braincellsError,threadCost:receipts.cost.snapshot(), usage:receipt?{inputTokens:receipt.rounds.reduce((n,r)=>n+(r.usage?.inputTokens||0),0),outputTokens:receipt.rounds.reduce((n,r)=>n+(r.usage?.outputTokens||0),0),rounds:receipt.rounds.length,repairs:receipt.repairs,status:receipt.status,cost:{usd:receipt.rounds.reduce((n,r)=>n+(r.usage?.costUSD||0),0),partial:receipt.rounds.some(r=>r.usage?.costUSD==null && (r.httpStatus==null || r.httpStatus<400)),estimated:receipt.rounds.some(r=>r.usage?.estimated)}}:null}; } function paintHandle(handle,colors=handleCharacterColors('@'+handle)){ @@ -88,8 +90,8 @@ function paintHandle(handle,colors=handleCharacterColors('@'+handle)){ signIn.replaceChildren(...Array.from('@'+handle,(character,index)=>{const span=document.createElement('span');span.textContent=character;span.style.color='rgb('+colors[index].join(',')+')';return span;}));nativeSnapshot(); } function accountIdentity(value){ - if(value===accountToken)return;accountToken=value;accountHandle='';syncAIConsent();braincells=null;braincellsError='';signIn.textContent=value?'…':'Sign in'; - accountVerification=value?verifyAccount(value).then(account=>{if(accountToken!==value)return;accountHandle=account.handle;syncAIConsent();if(!accountHandle){signIn.textContent='Set handle';return;}paintHandle(accountHandle);void refreshBraincells();const handle=accountHandle;void fetchHandleColors('@'+handle).then(colors=>{if(accountToken===value&&accountHandle===handle)paintHandle(handle,colors);}).catch(()=>{});}).catch(()=>{if(accountToken===value)signIn.textContent='Retry sign-in';}):Promise.resolve(); + if(value===accountToken)return;accountToken=value;accountHandle='';personalAccess=null;syncAIConsent();braincells=null;braincellsError='';signIn.textContent=value?'…':'Sign in'; + accountVerification=value?verifyAccount(value).then(async account=>{if(accountToken!==value)return;accountHandle=account.handle;syncAIConsent();if(!accountHandle){signIn.textContent='Set handle';return;}const access=await fetchPersonalAccess(value);if(accountToken!==value)return;personalAccess=access;paintHandle(accountHandle);void refreshBraincells();const handle=accountHandle;void fetchHandleColors('@'+handle).then(colors=>{if(accountToken===value&&accountHandle===handle)paintHandle(handle,colors);}).catch(()=>{});}).catch(()=>{if(accountToken===value)signIn.textContent='Retry sign-in';}):Promise.resolve(); } const $ = id => document.getElementById(id); const ui = document.createElement('section'); ui.id = 'live-work'; ui.hidden = true; @@ -176,7 +178,7 @@ window.walkiewareNativeCommand=command=>{ if(command.action==='ask'&&Array.from(new Intl.Segmenter(undefined,{granularity:'grapheme'}).segment(command.text)).length>96)return {accepted:false,reason:'inputTooLong'}; } if(command.action==='refreshBraincells')void refreshBraincells(); - if(command.action==='setModel'&&!busy&&accountHandle&&modelChoices(accountHandle).some(m=>m.id===command.text)){ + if(command.action==='setModel'&&!busy&&accountHandle&&modelChoices(accountHandle,{personalAccess:hasPersonalAccess(personalAccess)}).some(m=>m.id===command.text)){ try{localStorage.setItem('whistlegraph-model-'+accountHandle,command.text);}catch{return;} server?.close();server=null;activeModel='';nativeSnapshot(); } @@ -390,7 +392,7 @@ async function ask(text,displayText=text,advice=null,starter=null,localText=text let noChange=false; validationChecks=[];runtimeErrors.length=0; try{ - await accountVerification;if(turnCancelled)throw Error('Stopped');turnHandle=accountToken===token?accountHandle:'';turnModel=selectedModel(turnHandle); + await accountVerification;if(turnCancelled)throw Error('Stopped');turnHandle=accountToken===token?accountHandle:'';turnPersonalAccess=!!turnHandle&&hasPersonalAccess(personalAccess);turnModel=selectedModel(turnHandle); activeReceipt=new AttemptReceipt({requestID:activeAttempt.id,parent:turnParent,parentHash:await hashSource(previous),path:'compiled',model:window.__walkiewareModel||profile().model,journal:receipts}); const drawing=inputData(text)?.drawing; const chalkImage=drawing?drawingImage(drawing):null; diff --git a/apple/whistlegraph/Resources/Web/generation-policy.mjs b/apple/whistlegraph/Resources/Web/generation-policy.mjs index a84d79242e..ec2feee434 100644 --- a/apple/whistlegraph/Resources/Web/generation-policy.mjs +++ b/apple/whistlegraph/Resources/Web/generation-policy.mjs @@ -6,13 +6,13 @@ export const MODEL_LABELS = { 'anthropic/claude-opus-5':'Claude Opus 5 · personal', 'openai/gpt-6-astra':'GPT-6 Astra · personal', }; -export function modelChoices(handle) { - return Object.entries(MODEL_LABELS).filter(([id])=>handle==='jeffrey'||(!id.startsWith('anthropic/')&&!id.startsWith('openai/'))).map(([id,label])=>({id,label})); +export function modelChoices(handle, {personalAccess=false}={}) { + return Object.entries(MODEL_LABELS).filter(([id])=>(handle==='jeffrey'||personalAccess)||(!id.startsWith('anthropic/')&&!id.startsWith('openai/'))).map(([id,label])=>({id,label})); } // The handle comes from verifyAccount, never from a preference or command. -export function generationProfile(handle, {repair=false,model=''}={}) { - const personal=handle==='jeffrey'; - const selected=modelChoices(handle).some(option=>option.id===model)?model:''; +export function generationProfile(handle, {repair=false,model='',personalAccess=false}={}) { + const personal=handle==='jeffrey'||personalAccess; + const selected=modelChoices(handle,{personalAccess}).some(option=>option.id===model)?model:''; const chosen=selected||(personal?'anthropic/claude-opus-5':DEFAULT_MODEL); const resolved=repair&&chosen===DEFAULT_MODEL?REPAIR_MODEL:chosen; if(personal)return {personalRelay:resolved.startsWith('anthropic/')||resolved.startsWith('openai/'),model:resolved,maxTokens:16384,rounds:repair?4:12,outputContinuations:repair?1:2,reasoning:{max_tokens:4096},thinking:{type:'enabled',budget_tokens:4096}}; diff --git a/apple/whistlegraph/Resources/Web/personal-access.mjs b/apple/whistlegraph/Resources/Web/personal-access.mjs new file mode 100644 index 0000000000..0e083e49ba --- /dev/null +++ b/apple/whistlegraph/Resources/Web/personal-access.mjs @@ -0,0 +1,15 @@ +// A capability from the authenticated relay, never a saved handle preference. +export function hasPersonalAccess(access, now=Date.now()) { + return access?.personal===true && Array.isArray(access.providers) && + ['claude','codex'].every(provider=>access.providers.includes(provider)) && + (access.expiresAt===null || Date.parse(access.expiresAt)>now); +} +export async function fetchPersonalAccess(token, {fetch=globalThis.fetch}={}) { + if(!token)return null; + try { + const response=await fetch('https://help.aesthetic.computer/api/aesel/access',{ + headers:{Authorization:'Bearer '+token},signal:AbortSignal.timeout(8000)}); + if(!response.ok)return null; + const access=await response.json();return hasPersonalAccess(access)?access:null; + } catch {return null;} +} diff --git a/apple/whistlegraph/Sources/AIConsentRecord.swift b/apple/whistlegraph/Sources/AIConsentRecord.swift index 8ec5a5c9e7..5a3eefa166 100644 --- a/apple/whistlegraph/Sources/AIConsentRecord.swift +++ b/apple/whistlegraph/Sources/AIConsentRecord.swift @@ -6,6 +6,7 @@ struct AIConsentRecord: Codable, Equatable { var version = Self.version var creation = false var cloudSpeech = false + var cloudSpeechChoice: Bool? = nil var cloudNarration = false var updatedAt = Date() static func key(subject: String) -> String { diff --git a/apple/whistlegraph/Sources/RecordedTranscription.swift b/apple/whistlegraph/Sources/RecordedTranscription.swift index 0b088184b2..8c12d0c1fc 100644 --- a/apple/whistlegraph/Sources/RecordedTranscription.swift +++ b/apple/whistlegraph/Sources/RecordedTranscription.swift @@ -3,23 +3,43 @@ import AVFoundation struct TimedTranscript: Decodable { struct Word: Decodable { let text: String; let atMs: Double; let durationMs: Double } + struct Billing: Decodable { let braincells: Int; let durationMs: Double } + let billing: Billing? let transcript: String let words: [Word] var timeline: [[String: Any]] { words.map { ["text": $0.text, "atMs": $0.atMs, "durationMs": $0.durationMs] } } } +struct SpeechFailure: LocalizedError { + let status: Int + var errorDescription: String? { + switch status { + case 402: return "Not enough braincells for Whisper. Using device speech." + case 401, 403: return "Sign in again for Whisper. Using device speech." + case 429: return "Whisper is busy. Using device speech." + default: return "Whisper is unavailable. Using device speech." + } + } +} + enum RecordedTranscription { static func recover(_ id: String, token: String) async throws -> TimedTranscript { let audio = try await Task.detached(priority: .utility) { try wav(id) }.value try Task.checkCancellation() - var request = URLRequest(url: URL(string: "https://help.aesthetic.computer/api/aesel/transcribe")!) - request.httpMethod = "POST"; request.timeoutInterval = 15 + var request = URLRequest(url: URL(string: "https://aesthetic.computer/api/whistlegraph-transcribe")!) + request.httpMethod = "POST"; request.timeoutInterval = 25 request.setValue("Bearer " + token, forHTTPHeaderField: "Authorization") request.setValue("application/json", forHTTPHeaderField: "Content-Type") - request.httpBody = try JSONSerialization.data(withJSONObject: ["audio": audio.base64EncodedString()]) + request.httpBody = try JSONSerialization.data(withJSONObject: ["requestId": id, "audio": audio.base64EncodedString()]) + DeviceActionLog.shared.record(.speech, .started, control: .cloudSpeech) let (data, response) = try await URLSession.shared.data(for: request) - guard (response as? HTTPURLResponse)?.statusCode == 200 else { throw URLError(.badServerResponse) } - return try JSONDecoder().decode(TimedTranscript.self, from: data) + try Task.checkCancellation() + let status = (response as? HTTPURLResponse)?.statusCode ?? 0 + DeviceActionLog.shared.record(.speech, status == 200 ? .succeeded : .httpError, control: .cloudSpeech, [.status: status]) + guard status == 200 else { throw SpeechFailure(status: status) } + let result = try JSONDecoder().decode(TimedTranscript.self, from: data) + if let cost = result.billing { DeviceActionLog.shared.record(.speech, .committed, control: .cloudSpeech, [.used: cost.braincells, .durationMs: Int(cost.durationMs)]) } + return result } static func wav(_ id: String) throws -> Data { guard let url = UtteranceRecording.url(id) else { throw URLError(.badURL) } diff --git a/apple/whistlegraph/Sources/SpeechCapture.swift b/apple/whistlegraph/Sources/SpeechCapture.swift index c427894252..f7cc121d9e 100644 --- a/apple/whistlegraph/Sources/SpeechCapture.swift +++ b/apple/whistlegraph/Sources/SpeechCapture.swift @@ -11,6 +11,9 @@ import Speech @MainActor final class SpeechCapture { /// listening · partial · sound · musicalObservation · processing · mixedFinal · error var hasVisualInput: () -> Bool = { false } + var cloudSpeechEnabled: () -> Bool = { false } + var onSpeechNotice: (String) -> Void = { _ in } + var onSpeechCharge: () -> Void = {} var speechToken: () async throws -> String? = { nil } var onEvent: (_ kind: String, _ text: String, _ id: String) -> Void = { _, _, _ in } /// Latest microphone energy, for the live waveform. @@ -51,8 +54,7 @@ import Speech private var replayTask: Task? private var finishing: Task? private var deadline: Task? - private var live: LiveTranscription? - private var liveText = "", liveFinal = false, recognitionGraceExpired = false + private var recognitionGraceExpired = false private var finalSound: [String: Any]? private var alignment: Task? private var alignmentPending = false @@ -70,27 +72,6 @@ import Speech if NativeScreenFixture.enabled && NativeScreenFixture.mode == "gestures" { emit("listening"); return } #endif musicalInput = MusicalInput() - if !AudioBenchmark.enabled { - let stream = LiveTranscription(); live = stream - stream.onText = { [weak self] text, final in - guard let self, self.turn == id else { return } - self.liveText = text; self.liveFinal = final - self.emit("partial", text: text) - if final && self.audioEnded { self.deliver() } - } - stream.onFailure = { [weak self] in - guard let self, self.turn == id else { return } - self.liveFinal = true - if !self.latest.isEmpty { self.emit("partial", text: self.latest) } - self.deliver() - } - // This starts before microphone permission/session setup. Initial - // chunks queue until the short-lived socket is ready. - stream.start(token: speechToken) - musicalInput.onPCM = { [weak stream] data in - DispatchQueue.main.async { stream?.append(data) } - } - } musicalInput.onUpdate = { [weak self] pitch, rms in Task { @MainActor in guard let self, self.turn == id, self.held else { return } @@ -113,12 +94,13 @@ import Speech SFSpeechRecognizer.requestAuthorization { continuation.resume(returning: $0 == .authorized) } } guard self.turn == id, self.held else { return } - guard speech else { self.fail("Speech permission is off. Enable it in Settings."); return } + guard speech || self.cloudSpeechEnabled() else { self.fail("Speech permission is off. Enable it in Settings."); return } let microphone = await AVCaptureDevice.requestAccess(for: .audio) guard self.turn == id, self.held else { return } guard microphone else { self.fail("Microphone permission is off."); return } - guard let recognizer = self.recognizer, recognizer.isAvailable, - recognizer.supportsOnDeviceRecognition else { + let recognizer = self.recognizer + let localSpeech = speech && recognizer?.isAvailable == true && recognizer?.supportsOnDeviceRecognition == true + guard localSpeech || self.cloudSpeechEnabled() else { self.fail("On-device English speech is unavailable."); return } do { @@ -144,7 +126,7 @@ import Speech try self.engine.start() } self.emit("listening") - self.task = recognizer.recognitionTask(with: request) { [weak self] result, error in + self.task = localSpeech ? recognizer?.recognitionTask(with: request) { [weak self] result, error in let text = result?.bestTranscription.formattedString let segments = result?.bestTranscription.segments.map { ["text": $0.substring, "atMs": $0.timestamp * 1000, "durationMs": $0.duration * 1000, "confidence": Double($0.confidence)] as [String: Any] } let isFinal = result?.isFinal ?? false @@ -152,7 +134,7 @@ import Speech Task { @MainActor in guard let self, self.turn == id else { return } if let segments { self.wordSegments = Array(segments.prefix(256)) } - if let text { self.latest = text; if self.liveText.isEmpty { self.emit("partial", text: text) } } + if let text { self.latest = text; self.emit("partial", text: text) } if isFinal { self.recognitionFinal = true; AudioBenchmark.mark("recognitionFinal") } if self.audioEnded && isFinal { self.deliver() } else if self.audioEnded && message != nil && self.latest.isEmpty { @@ -160,7 +142,7 @@ import Speech self.deliver() // Sound remains useful when speech finds no words. } } - } + } : nil if AudioBenchmark.enabled { let sound = self.musicalInput self.replayTask = Task { [weak self] in @@ -203,22 +185,20 @@ import Speech AudioBenchmark.mark("audioDrained") let sound = await withCheckedContinuation { continuation in self.musicalInput.finish { continuation.resume(returning: $0) } } guard !Task.isCancelled, self.turn == id else { return } - // MusicalInput's serial queue has drained all PCM callbacks before - // this continuation, so commit follows the last recorded sample. - await withCheckedContinuation { continuation in - DispatchQueue.main.async { self.live?.finish(); continuation.resume() } - } - guard !Task.isCancelled, self.turn == id else { return } self.finalSound = sound - if !AudioBenchmark.enabled, let recording = sound["recordingID"] as? String, - self.maxDuration > 8 || self.latest.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + if !AudioBenchmark.enabled, self.cloudSpeechEnabled(), let recording = sound["recordingID"] as? String { self.alignmentPending = true self.alignment = Task { [weak self] in guard let self else { return } var result: TimedTranscript? do { if let token = try await self.speechToken() { result = try await RecordedTranscription.recover(recording, token: token) } } - catch { /* Live/on-device words and the original recording survive a network failure. */ } + catch { + guard !Task.isCancelled, self.turn == id else { return } + DeviceActionLog.shared.recordError(.speech, error) + self.onSpeechNotice((error as? SpeechFailure)?.errorDescription ?? "Whisper is unavailable. Using device speech.") + } guard !Task.isCancelled, self.turn == id else { return } + if result?.billing != nil { self.onSpeechCharge() } self.aligned = result; self.alignmentPending = false; self.deliver() } } @@ -236,10 +216,10 @@ import Speech private func deliver() { guard !delivering, !turn.isEmpty, audioEnded, let sound = finalSound, !alignmentPending, - aligned != nil || liveFinal || recognitionFinal || recognitionGraceExpired else { return } + aligned != nil || recognitionFinal || recognitionGraceExpired else { return } delivering = true let recovered = aligned?.transcript.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" - let text = !recovered.isEmpty ? recovered : (!liveText.isEmpty ? liveText : latest).trimmingCharacters(in: .whitespacesAndNewlines) + let text = !recovered.isEmpty ? recovered : latest.trimmingCharacters(in: .whitespacesAndNewlines) // Stream arrival times are not word timestamps. Only use measured // alignment from the same transcript; never invent timing for deltas. let words = !recovered.isEmpty ? aligned!.timeline : (text == latest.trimmingCharacters(in: .whitespacesAndNewlines) ? wordSegments : []) @@ -264,8 +244,8 @@ import Speech } func cancel() { - live?.close(); live = nil; alignment?.cancel(); alignment = nil - liveText = ""; liveFinal = false; finalSound = nil; aligned = nil; alignmentPending = false; recognitionGraceExpired = false + alignment?.cancel(); alignment = nil + finalSound = nil; aligned = nil; alignmentPending = false; recognitionGraceExpired = false turn = ""; held = false; listeningSince = nil; maxDuration = 8 replayTask?.cancel(); replayTask = nil finishing?.cancel(); finishing = nil diff --git a/apple/whistlegraph/Sources/WhistlegraphApp.swift b/apple/whistlegraph/Sources/WhistlegraphApp.swift index ba0accd73c..32ee97f28e 100644 --- a/apple/whistlegraph/Sources/WhistlegraphApp.swift +++ b/apple/whistlegraph/Sources/WhistlegraphApp.swift @@ -49,6 +49,9 @@ struct WhistlegraphApp: App { WhistlegraphAIConsentSheet(canAllow: voice.canAllowAIConsent, allow: voice.allowAIConsent, decline: voice.declineAIConsent) } + .sheet(isPresented: $voice.showingSpeechConsent) { + WhistlegraphSpeechConsentSheet(choose: voice.chooseCloudSpeech) + } .preferredColorScheme(appearance == "light" ? .light : appearance == "dark" ? .dark : nil) .onChange(of: phase) { _, value in DeviceActionLog.shared.record(.lifecycle, value == .active ? .active : value == .background ? .background : .inactive) @@ -216,6 +219,9 @@ final class WhistlegraphSession: NSObject, ObservableObject, WKScriptMessageHand @Published var layout = NativeLayout() @Published var snapshot = PieceSnapshot() @Published var showingAIConsent = false + @Published var showingSpeechConsent = false + @Published var speechNotice: String? + private var speechConsentAccount: (subject: String, handle: String, generation: Int)? @Published private(set) var verifyingAIAccount = false @Published var actionError: String? @Published var typedDraft = "" @@ -373,6 +379,13 @@ final class WhistlegraphSession: NSObject, ObservableObject, WKScriptMessageHand // Permission does not start a microphone after the finger has lifted. guard aiConsent.creation else { requestAIConsent(); return } if isConsentFixture { return } + if !aiConsent.cloudSpeech && aiConsent.record.cloudSpeechChoice == nil, let subject = aiConsent.subject { + speechConsentAccount = (subject, snapshot.handle, account.generation) + showingSpeechConsent = true + DeviceActionLog.shared.record(.consent, .presented, control: .cloudSpeech) + return + } + speechNotice = nil performanceTurn = false captureError = nil; transcript = ""; speechStartedAt = nil; capturePhase = .opening webView?.evaluateJavaScript("voiceStart()") { [weak self] _, error in @@ -382,6 +395,13 @@ final class WhistlegraphSession: NSObject, ObservableObject, WKScriptMessageHand self?.reportActionFailure("Recording could not start. Try holding Talk again.", reason: .failed) } } + func chooseCloudSpeech(_ enabled: Bool) { + defer { showingSpeechConsent = false; speechConsentAccount = nil } + guard let pending = speechConsentAccount, pending.subject == aiConsent.subject, + pending.handle == snapshot.handle, pending.generation == account.generation else { return } + aiConsent.set(\.cloudSpeech, enabled) + // A choice never starts recording after the finger has lifted. + } func latchPerformance() { DeviceActionLog.shared.record(.talkLatch, .requested) guard capturePhase == .opening || capturePhase == .recording else { return } @@ -477,6 +497,8 @@ final class WhistlegraphSession: NSObject, ObservableObject, WKScriptMessageHand } } private func applyAIConsent() { + if showingSpeechConsent, let pending = speechConsentAccount, + pending.subject != aiConsent.subject || pending.handle != snapshot.handle || pending.generation != account.generation { showingSpeechConsent = false; speechConsentAccount = nil } if showingAIConsent && !canAllowAIConsent { declineAIConsent() } if !aiConsent.creation { command("stop"); cancelHold() } if !aiConsent.cloudSpeech { cancelHold() } @@ -589,8 +611,11 @@ final class WhistlegraphSession: NSObject, ObservableObject, WKScriptMessageHand #endif aiConsent.changed = { [weak self] in self?.applyAIConsent() } capture.hasVisualInput = { [weak self] in self?.drawing.hasInk == true } + capture.cloudSpeechEnabled = { [weak self] in self?.aiConsent.cloudSpeech == true } + capture.onSpeechNotice = { [weak self] text in self?.speechNotice = text } + capture.onSpeechCharge = { [weak self] in self?.command("refreshBraincells") } capture.speechToken = { [weak self] in - guard let self, self.snapshot.handle == "jeffrey", self.aiConsent.cloudSpeech else { return nil } + guard let self, self.aiConsent.cloudSpeech else { return nil } let generation = self.account.generation let token = try await self.account.token() guard !Task.isCancelled, generation == self.account.generation, self.aiConsent.cloudSpeech else { return nil } diff --git a/apple/whistlegraph/Sources/WhistlegraphPrivacy.swift b/apple/whistlegraph/Sources/WhistlegraphPrivacy.swift index 9a5b4bce8c..95e55c6db1 100644 --- a/apple/whistlegraph/Sources/WhistlegraphPrivacy.swift +++ b/apple/whistlegraph/Sources/WhistlegraphPrivacy.swift @@ -21,6 +21,7 @@ import SwiftUI guard let subject, signedIn else { return } DeviceActionLog.shared.record(.setting, value ? .enabled : .disabled, control: key == \.creation ? .creation : key == \.cloudSpeech ? .cloudSpeech : .cloudNarration) + if key == \.cloudSpeech { record.cloudSpeechChoice = value } record[keyPath: key] = value; record.updatedAt = Date() record.save(subject: subject); changed() } @@ -102,7 +103,7 @@ struct WhistlegraphPrivacySheet: View { .disabled(!consent.signedIn).accessibilityIdentifier("privacy-ai-creation") } Section("Cloud speech") { - Text("Optional OpenAI transcription sends microphone audio through AC or directly to OpenAI during recording and for word timing. It currently requires an eligible personal account. With this off, speech recognition stays on the device.") + Text("Whisper sends each finished recording through AC to OpenAI for transcription and word timing. AC pays OpenAI and charges 40 braincells per recorded second, using your daily allowance first, then purchased braincells. Failed transcriptions are refunded. AC keeps billing receipts, not audio or transcripts; a retry result may remain in memory for one minute. With this off, speech recognition stays on the device.") Toggle("Allow audio to OpenAI", isOn: Binding(get: { consent.cloudSpeech }, set: { consent.set(\.cloudSpeech, $0) })) .disabled(!consent.signedIn).accessibilityIdentifier("privacy-cloud-speech") } @@ -125,3 +126,21 @@ struct WhistlegraphPrivacySheet: View { .toolbar { ToolbarItem(placement: .confirmationAction) { Button("Done") { dismiss() } } } } } + +struct WhistlegraphSpeechConsentSheet: View { + let choose: (Bool) -> Void + var body: some View { + ScrollView { + VStack(alignment: .leading, spacing: 20) { + Text("Use Whisper for speech?").font(.title.bold()).accessibilityAddTraits(.isHeader) + Text("After you release Talk, AC sends the recording to OpenAI for transcription and word timing. Live captions stay on your device.") + Text("40 braincells per recorded second — 320 for eight seconds. Your daily allowance is used first, then purchased braincells. Failed transcriptions are refunded.") + Text("AC keeps billing receipts, not recordings or transcripts. Retry results stay in memory for up to one minute. Change this in Brain → AI & privacy.") + Button("Allow Whisper") { choose(true) }.buttonStyle(.borderedProminent).accessibilityIdentifier("speech-consent-allow") + Button("Use device speech") { choose(false) }.buttonStyle(.bordered).accessibilityIdentifier("speech-consent-device") + Link("Privacy policy", destination: URL(string: "https://aesthetic.computer/privacy-policy.html")!) + }.font(.body).padding(24) + } + .presentationDetents([.large]).presentationDragIndicator(.visible) + } +} diff --git a/apple/whistlegraph/Sources/WhistlegraphScreen.swift b/apple/whistlegraph/Sources/WhistlegraphScreen.swift index 434f247aa0..1cadd3a0cf 100644 --- a/apple/whistlegraph/Sources/WhistlegraphScreen.swift +++ b/apple/whistlegraph/Sources/WhistlegraphScreen.swift @@ -202,6 +202,7 @@ struct WhistlegraphScreen: View { .frame(height: narrator.isPlaying ? UIScreen.main.bounds.width * 16 / 9 : nil) .background(narrator.isPlaying ? storyBackground : Color.clear) if !narrator.error.isEmpty && !narrator.isPlaying { Text(narrator.error).foregroundStyle(.orange) } + if let notice = session.speechNotice { Text(notice).font(.body).foregroundStyle(.orange).frame(maxWidth: .infinity, alignment: .leading).accessibilityIdentifier("speech-fallback-notice") } if let failure = session.captureError, !narrator.isPlaying { Text(failure).font(.body).foregroundStyle(.orange).frame(maxWidth: .infinity, alignment: .leading) } if !session.snapshot.error.isEmpty && !narrator.isPlaying { Text(session.snapshot.error).font(.body).foregroundStyle(.orange).accessibilityIdentifier("workspace-error") } if session.verifyingAIAccount { ProgressView("Checking your account…").accessibilityIdentifier("account-verifying") } diff --git a/apple/whistlegraph/Tests/AIConsentRecordCheck.swift b/apple/whistlegraph/Tests/AIConsentRecordCheck.swift index b73dff6713..71ebb94e98 100644 --- a/apple/whistlegraph/Tests/AIConsentRecordCheck.swift +++ b/apple/whistlegraph/Tests/AIConsentRecordCheck.swift @@ -20,6 +20,13 @@ import Foundation precondition(allOff(AIConsentRecord.read(subject: "alice", defaults: defaults)), "Changed disclosure needs fresh permission") defaults.set(Data("invalid".utf8), forKey: AIConsentRecord.key(subject: "alice")) precondition(allOff(AIConsentRecord.read(subject: "alice", defaults: defaults))) - print("AIConsentRecordCheck passed (7 cases)") + record = AIConsentRecord(); record.creation = true; record.cloudSpeechChoice = false + record.save(subject: "alice", defaults: defaults) + precondition(AIConsentRecord.read(subject: "alice", defaults: defaults).cloudSpeechChoice == false, "Device speech choice persists") + precondition(AIConsentRecord.read(subject: "bob", defaults: defaults).cloudSpeechChoice == nil, "Speech choice stays with the account") + let old = #"{"version":1,"creation":true,"cloudSpeech":false,"cloudNarration":false,"updatedAt":0}"# + defaults.set(Data(old.utf8), forKey: AIConsentRecord.key(subject: "legacy")) + precondition(AIConsentRecord.read(subject: "legacy", defaults: defaults).creation, "Existing creation consent survives the optional speech choice") + print("AIConsentRecordCheck passed (10 cases)") } } diff --git a/apple/whistlegraph/Tests/generation-policy.test.mjs b/apple/whistlegraph/Tests/generation-policy.test.mjs index dfa2e46a28..d43a3d38b9 100644 --- a/apple/whistlegraph/Tests/generation-policy.test.mjs +++ b/apple/whistlegraph/Tests/generation-policy.test.mjs @@ -22,3 +22,15 @@ assert.equal(generationProfile('jeffrey',{model:DEFAULT_MODEL}).personalRelay,fa assert.equal(generationProfile('jeffrey',{model:'openai/gpt-6-astra'}).personalRelay,true); assert.equal(generationProfile('jeffrey',{model:'openai/gpt-6-astra'}).model,'openai/gpt-6-astra'); for (const handle of ['', 'fixture', 'Jeffrey', 'jeffrey-other']) assert.equal(generationProfile(handle,{model:'openai/gpt-6-astra'}).model,DEFAULT_MODEL); +const trial={personalAccess:true}; +assert.equal(generationProfile('fifi',trial).model,'anthropic/claude-opus-5'); +assert.equal(generationProfile('fifi',{...trial,model:'openai/gpt-6-astra'}).personalRelay,true); +assert.equal(generationProfile('fifi',{...trial,model:DEFAULT_MODEL}).personalRelay,false); +assert.equal(generationProfile('fifi',{model:'anthropic/claude-opus-5'}).model,DEFAULT_MODEL); +const {hasPersonalAccess,fetchPersonalAccess}=await import('../Resources/Web/personal-access.mjs'); +const access={personal:true,providers:['claude','codex'],expiresAt:new Date(2000).toISOString()}; +assert.equal(hasPersonalAccess(access,1999),true);assert.equal(hasPersonalAccess(access,2000),false); +assert.equal(hasPersonalAccess({personal:true}),false); +assert.equal(await fetchPersonalAccess('x',{fetch:async()=>({ok:false})}),null); +assert.equal(await fetchPersonalAccess('x',{fetch:async()=>{throw Error('offline')}}),null); +console.log('PASS verified trial capability, expiry and fail-closed discovery.'); diff --git a/apple/whistlegraph/Tests/native-bridge.test.cjs b/apple/whistlegraph/Tests/native-bridge.test.cjs index 42f44ebe53..f2a2cabfe4 100644 --- a/apple/whistlegraph/Tests/native-bridge.test.cjs +++ b/apple/whistlegraph/Tests/native-bridge.test.cjs @@ -97,7 +97,7 @@ const server = http.createServer(async (req, res) => { window.__walkiewareDisableThread=true; window.__guideFetches=0; const fetchOriginal=window.fetch.bind(window); - window.fetch=(url,init)=>{if(String(url).includes('/api/easel-credits'))return Promise.resolve(Response.json({remaining:180000,used:20000,limit:200000,purchased:50000}));if(String(url).includes('/api/handle-colors'))return Promise.resolve(Response.json({colors:[{r:200,g:100,b:255}]}));if(String(url).includes('/userinfo'))return Promise.resolve(Response.json({sub:'fixture-user'}));if(String(url).includes('/handle?for='))return Promise.resolve(Response.json({handle:personal?'jeffrey':'fixture'}));if(String(url).includes('/api/easel-musical-jev')){const b=JSON.parse(init.body);return Promise.resolve(Response.json({schema:'walkieware-decision/v1',sessionId:b.sessionId,sequence:b.sequence,choice:'follow_speech',confidence:.95}));}if(String(url).startsWith('/easel/context/')){window.__guideFetches++;return Promise.resolve({ok:false,status:0});}return fetchOriginal(String(url).includes('/api/easel-inference')?'/mock-inference':url,init);}; + window.fetch=(url,init)=>{if(String(url).includes('/api/aesel/access'))return Promise.resolve(Response.json({personal:personal,providers:personal?['claude','codex']:[],expiresAt:null}));if(String(url).includes('/api/easel-credits'))return Promise.resolve(Response.json({remaining:180000,used:20000,limit:200000,purchased:50000}));if(String(url).includes('/api/handle-colors'))return Promise.resolve(Response.json({colors:[{r:200,g:100,b:255}]}));if(String(url).includes('/userinfo'))return Promise.resolve(Response.json({sub:'fixture-user'}));if(String(url).includes('/handle?for='))return Promise.resolve(Response.json({handle:personal?'jeffrey':'fixture'}));if(String(url).includes('/api/easel-musical-jev')){const b=JSON.parse(init.body);return Promise.resolve(Response.json({schema:'walkieware-decision/v1',sessionId:b.sessionId,sequence:b.sequence,choice:'follow_speech',confidence:.95}));}if(String(url).startsWith('/easel/context/')){window.__guideFetches++;return Promise.resolve({ok:false,status:0});}return fetchOriginal(String(url).includes('/api/easel-inference')?'/mock-inference':url,init);}; window.__nativeMessages=[]; window.webkit={messageHandlers:{walkie:{postMessage:m=>{window.__nativeMessages.push(m);if(m.action==='visualCapture'){const canvas=document.createElement('canvas');canvas.width=32;canvas.height=24;canvas.getContext('2d').fillRect(0,0,32,24);setTimeout(()=>window.walkiewareEngineEvent({kind:'visualCapture',captureID:m.captureID,sourceHash:window.__staleVisual?'old':m.sourceHash,renderID:m.renderID,frames:[0,800,1600,2400].map(atMs=>({atMs,width:32,height:24,png:canvas.toDataURL('image/png').split(',')[1]}))}),10);}if(m.action==='render')setTimeout(async()=>{const digest=await crypto.subtle.digest('SHA-256',new TextEncoder().encode(m.source));const sourceHash=[...new Uint8Array(digest)].map(b=>b.toString(16).padStart(2,'0')).join('');window.walkiewareEngineEvent({kind:'previewEvent',event:{kind:'painted',sourceHash,requestID:m.renderID}});},20);}}}}; },guideSeed,process.argv.includes('--native-shell'),process.argv.includes('--jeffrey')); diff --git a/apple/whistlegraph/Whistlegraph.xcodeproj/project.pbxproj b/apple/whistlegraph/Whistlegraph.xcodeproj/project.pbxproj index e0490abbbd..24bda40db0 100644 --- a/apple/whistlegraph/Whistlegraph.xcodeproj/project.pbxproj +++ b/apple/whistlegraph/Whistlegraph.xcodeproj/project.pbxproj @@ -598,7 +598,7 @@ CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_STYLE = Automatic; COPY_PHASE_STRIP = NO; - CURRENT_PROJECT_VERSION = 109; + CURRENT_PROJECT_VERSION = 110; DEBUG_INFORMATION_FORMAT = dwarf; DEVELOPMENT_TEAM = FB5948YR3S; ENABLE_STRICT_OBJC_MSGSEND = YES; @@ -683,7 +683,7 @@ CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_STYLE = Automatic; COPY_PHASE_STRIP = NO; - CURRENT_PROJECT_VERSION = 109; + CURRENT_PROJECT_VERSION = 110; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; DEVELOPMENT_TEAM = FB5948YR3S; ENABLE_NS_ASSERTIONS = NO; @@ -743,7 +743,7 @@ CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_STYLE = Automatic; COPY_PHASE_STRIP = NO; - CURRENT_PROJECT_VERSION = 109; + CURRENT_PROJECT_VERSION = 110; DEBUG_INFORMATION_FORMAT = dwarf; DEVELOPMENT_TEAM = FB5948YR3S; ENABLE_STRICT_OBJC_MSGSEND = YES; diff --git a/apple/whistlegraph/project.yml b/apple/whistlegraph/project.yml index fc2d384517..3665f497be 100644 --- a/apple/whistlegraph/project.yml +++ b/apple/whistlegraph/project.yml @@ -16,7 +16,7 @@ settings: CODE_SIGN_STYLE: Automatic SWIFT_VERSION: "5.0" MARKETING_VERSION: "0.1.0" - CURRENT_PROJECT_VERSION: "109" + CURRENT_PROJECT_VERSION: "110" targets: Whistlegraph: type: application