diff --git a/lith/server.mjs b/lith/server.mjs index 9c87877287..15eef6c0f9 100644 --- a/lith/server.mjs +++ b/lith/server.mjs @@ -1501,6 +1501,9 @@ if (!DEV) { reconcilingCredits=true; try { const paid=await import(pathToFileURL(join(SYSTEM,"backend","easel-paid-credits.mjs")).href); + const {connect}=await import(pathToFileURL(join(SYSTEM,"backend","database.mjs")).href); + const {speechBilling}=await import(pathToFileURL(join(SYSTEM,"backend","whistlegraph-speech-billing.mjs")).href); + await speechBilling((await connect()).db).reconcile(); const count=await paid.withWallets(w=>paid.reconcilePaidHolds(w)); if(count)console.log("[lith] recovered credit holds:",count); } catch { console.error("[lith] credit reconciliation failed; will retry"); } diff --git a/system/backend/account-deletion.mjs b/system/backend/account-deletion.mjs index 77266134ea..9e18e2f284 100644 --- a/system/backend/account-deletion.mjs +++ b/system/backend/account-deletion.mjs @@ -141,6 +141,7 @@ const EXPORT = [ // tokens, signed payloads or mint capabilities. An allowlist also keeps // future provider credentials out of this download. const PRIVATE_EXPORT = [ + ["whistlegraph-speech-requests", "user", ["_id", "braincells", "charged", "free", "paid", "status", "startedAt", "finishedAt"]], ["whistlegraph-iap-accounts", "_id", ["createdAt"]], ["whistlegraph-iap-purchases", "user", [ "_id", "transactionId", "productId", "credits", "environment", "createdAt", @@ -331,6 +332,7 @@ export async function handleQuarantined(db, handle, now = new Date()) { // Records keyed to the account that are deleted outright. // [collection, query(sub, ctx)] const DELETE = [ + ["whistlegraph-speech-requests", (sub) => ({ user: sub })], ["paintings", (sub) => ({ user: sub })], ["pieces", (sub) => ({ user: sub })], ["moods", (sub) => ({ user: sub })], diff --git a/system/backend/whistlegraph-speech-billing.mjs b/system/backend/whistlegraph-speech-billing.mjs new file mode 100644 index 0000000000..ea8d439120 --- /dev/null +++ b/system/backend/whistlegraph-speech-billing.mjs @@ -0,0 +1,60 @@ +import {createHash,randomUUID} from 'node:crypto'; +import {DAILY_TOKEN_BUDGET,dayKey} from './ai-budget.mjs'; +import {BRAINCELLS_PER_USD,INFERENCE_MARKUP,reserve,settle} from './easel-paid-credits.mjs'; +// https://developers.openai.com/api/docs/models/whisper-1 — $0.006/minute. +export const WHISPER_USD_PER_MINUTE=0.006; +export const speechCost=ms=>Math.ceil(Math.round(ms/60000*WHISPER_USD_PER_MINUTE*BRAINCELLS_PER_USD*INFERENCE_MARKUP*1e6)/1e6); +const fail=(status,message)=>Object.assign(Error(message),{status}); +const uuid=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +export const speechRequestKey=(user,id)=>createHash('sha256').update(user+'\0'+id.toLowerCase()).digest('hex'); +const walletIn=(db,session)=>({ + updateOne:(filter,update,options={})=>db.collection('ac-credit-wallets').updateOne(filter,update,{...options,session}), +}); +export function speechBilling(db,{now=()=>new Date(),limit=DAILY_TOKEN_BUDGET}={}) { + const receipts=db.collection('whistlegraph-speech-requests'),usage=db.collection('ai-usage'); + async function transaction(fn) { + const session=db.client.startSession(); + try{return await session.withTransaction(()=>fn(session),{readConcern:{level:'snapshot'},writeConcern:{w:'majority'}});} + finally{await session.endSession();} + } + async function finish(id,success) { + return transaction(async session=>{ + const r=await receipts.findOne({_id:id},{session}); + if(!r||r.status!=='pending')return false; + if(r.hold)await settle(r.hold,success?r.paid:0,walletIn(db,session),{now:now()}); + if(!success&&r.free)await usage.updateOne({_id:r.usageId},{$inc:{tokens:-r.free}},{session}); + if(success)await usage.updateOne({_id:r.usageId},{$inc:{asks:1},$set:{last:now(),lastModel:'openai/whisper-1'}},{session}); + await receipts.updateOne({_id:id,status:'pending'},{$set:{status:success?'complete':'failed',charged:success?r.braincells:0,finishedAt:now()}},{session}); + return true; + }); + } + async function reconcile(user) { + const query={status:'pending',expiresAt:{$lte:now()},...(user?{user}:{})}; + let count=0;for await(const r of receipts.find(query,{projection:{_id:1}}))if(await finish(r._id,false))count++; + return count; + } + async function begin({user,handle,requestId,audio,durationMs}) { + if(!uuid.test(requestId||''))throw fail(400,'Recording ID must be a UUID'); + const id=speechRequestKey(user,requestId),hash=createHash('sha256').update(audio).digest('hex'); + const braincells=speechCost(durationMs); + if(!Number.isSafeInteger(braincells)||braincells<1||durationMs>46000)throw fail(400,'Invalid recording duration'); + await reconcile(user); + const at=now(),day=dayKey(at),usageId=handle+':'+day,holdId=randomUUID(); + return transaction(async session=>{ + const previous=await receipts.findOne({_id:id},{session}); + if(previous) { + if(previous.hash!==hash)throw fail(409,'Recording ID was already used for different audio'); + throw fail(409,previous.status==='pending'?'Transcription is still processing':'Recording already processed; use the saved or device transcript'); + } + await usage.updateOne({_id:usageId},{$setOnInsert:{handle,day,first:at,tokens:0}},{upsert:true,session}); + const spent=await usage.findOne({_id:usageId},{session}); + const free=Math.min(braincells,Math.max(0,limit-(Number(spent.tokens)||0))),paid=braincells-free; + const hold=paid?await reserve(user,paid,walletIn(db,session),{id:holdId,now:at}):null; + if(paid&&!hold)throw fail(402,'Not enough braincells for OpenAI speech; using device speech'); + if(free)await usage.updateOne({_id:usageId},{$inc:{tokens:free}},{session}); + await receipts.insertOne({_id:id,user,hash,usageId,day,free,paid,hold,braincells,status:'pending',startedAt:at,expiresAt:new Date(+at+120000)},{session}); + return {id,braincells,free,paid,durationMs}; + }); + } + return {begin,finish,reconcile}; +} diff --git a/system/backend/whistlegraph-whisper.mjs b/system/backend/whistlegraph-whisper.mjs new file mode 100644 index 0000000000..a5bc4159bf --- /dev/null +++ b/system/backend/whistlegraph-whisper.mjs @@ -0,0 +1,39 @@ +// Bounded Whisper upload; audio and transcripts are never written to disk. +const fail=(status,message)=>Object.assign(Error(message),{status}); +export function wavInput(input) { + if(typeof input?.audio!=='string'||input.audio.length>2_000_000||input.audio.length%4||!/^[A-Za-z0-9+/]*={0,2}$/.test(input.audio))throw fail(400,'Invalid recording'); + const b=Buffer.from(input.audio,'base64'); + if(b.length<46||b.toString('ascii',0,4)!=='RIFF'||b.toString('ascii',8,12)!=='WAVE'|| + b.toString('ascii',12,16)!=='fmt '||b.readUInt32LE(16)!==16||b.readUInt16LE(20)!==1|| + b.readUInt16LE(22)!==1||b.readUInt32LE(24)!==16000||b.readUInt32LE(28)!==32000|| + b.readUInt16LE(32)!==2||b.readUInt16LE(34)!==16||b.toString('ascii',36,40)!=='data'|| + b.readUInt32LE(4)!==b.length-8||b.readUInt32LE(40)!==b.length-44||(b.length-44)%2)throw fail(400,'Expected mono 16 kHz PCM WAV'); + const durationMs=(b.length-44)/32; + if(durationMs<100||durationMs>46000)throw fail(400,'Recording must be under 46 seconds'); + return {audio:b,durationMs}; +} +export function createTranscriber({apiKey,fetch=globalThis.fetch}={}) { + let active=0; + return async input=>{ + if(!apiKey)throw fail(503,'Speech transcription is unavailable'); + const {audio,durationMs}=wavInput(input); + if(active>=2)throw fail(429,'Speech transcription is busy'); + active++; + try { + const form=new FormData();form.set('file',new Blob([audio],{type:'audio/wav'}),'recording.wav'); + form.set('model','whisper-1');form.set('language','en');form.set('response_format','verbose_json'); + form.append('timestamp_granularities[]','word'); + const start=performance.now(); + const response=await fetch('https://api.openai.com/v1/audio/transcriptions',{method:'POST',headers:{Authorization:'Bearer '+apiKey},body:form,signal:AbortSignal.timeout(12000)}); + if(!response.ok)throw fail(502,'Speech transcription failed'); + const result=await response.json(); + if(typeof result.text!=='string'||result.text.length>12000||!Array.isArray(result.words)||result.words.length>256)throw fail(502,'Invalid speech response'); + const words=result.words.map(w=>{ + if(typeof w.word!=='string'||w.word.length>2000||!Number.isFinite(w.start)||!Number.isFinite(w.end)||w.start<0||w.enddurationMs+250)throw fail(502,'Invalid word timing'); + return {text:w.word,atMs:Math.round(w.start*1000),durationMs:Math.round((w.end-w.start)*1000)}; + }); + return {transcript:result.text.trim(),words,provider:'openai',model:'whisper-1',elapsedMs:Math.round(performance.now()-start)}; + }catch(error){if(error.status)throw error;throw fail(502,'Speech transcription unavailable');} + finally{active--;} + }; +} diff --git a/system/netlify/functions/whistlegraph-transcribe.mjs b/system/netlify/functions/whistlegraph-transcribe.mjs new file mode 100644 index 0000000000..5db863258d --- /dev/null +++ b/system/netlify/functions/whistlegraph-transcribe.mjs @@ -0,0 +1,52 @@ +import {createHash} from 'node:crypto'; +import {wavInput,createTranscriber} from '../../backend/whistlegraph-whisper.mjs'; +import {speechBilling,speechRequestKey} from '../../backend/whistlegraph-speech-billing.mjs'; +const reply=(statusCode,value)=>({statusCode,headers:{'Content-Type':'application/json','Cache-Control':'private, no-store','Access-Control-Allow-Origin':'*','Access-Control-Allow-Methods':'POST, OPTIONS','Access-Control-Allow-Headers':'Authorization,Content-Type'},body:JSON.stringify(value)}); +export function createHandler({authorize,getHandleOrEmail,billing,transcribe,enabled=true,now=Date.now}) { + // Retry results live only in memory for one minute. Durable receipts contain + // cost and an audio digest, never recordings, transcripts or bearer tokens. + const retry=new Map();let active=0; + return async event=>{ + if(event.httpMethod==='OPTIONS')return reply(204,null); + if(event.httpMethod!=='POST')return reply(405,{error:'POST only'}); + if(!event.headers?.authorization)return reply(401,{error:'Sign in for OpenAI speech'}); + try { + const user=await authorize(event.headers); + if(!user?.sub)return reply(401,{error:'Sign in again for OpenAI speech'}); + if(user.email_verified!==true)return reply(403,{error:'Verify your email for OpenAI speech'}); + if(!enabled)return reply(503,{error:'OpenAI speech is unavailable'}); + const handle=await getHandleOrEmail(user.sub); + if(typeof handle!=='string'||!handle.startsWith('@'))return reply(403,{error:'Choose an AC handle first'}); + if(typeof event.body!=='string'||event.body.length>2_001_000)return reply(413,{error:'Recording too large'}); + let input;try {input=JSON.parse(event.body);}catch{return reply(400,{error:'Invalid recording'});} + if(!input||typeof input!=='object'||Array.isArray(input)||typeof input.requestId!=='string'||!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(input.requestId))return reply(400,{error:'Recording ID must be a UUID'}); + const {audio,durationMs}=wavInput(input),id=speechRequestKey(user.sub,input.requestId); + const hash=createHash('sha256').update(audio).digest('hex'); + for(const [key,item] of retry)if(item.until<=now())retry.delete(key); + const cached=retry.get(id);if(cached){if(cached.hash!==hash)return reply(409,{error:'Recording ID already used'});return reply(200,cached.value);} + if(active>=2)return reply(429,{error:'OpenAI speech is busy; using device speech'}); + active++; + try { + const receipt=await billing.begin({user:user.sub,handle:handle.slice(1),requestId:input.requestId,audio,durationMs}); + let result; + try {result=await transcribe(input);} + catch(error){await billing.finish(receipt.id,false);throw error;} + // If accounting cannot complete, recovery refunds the hold. Do not + // retry the provider or pretend the user received a successful result. + if(!await billing.finish(receipt.id,true))throw Object.assign(Error('Speech expired; using device speech'),{status:503}); + const value={...result,billing:{braincells:receipt.braincells,free:receipt.free,paid:receipt.paid,durationMs}}; + if(retry.size>=100)retry.delete(retry.keys().next().value); + retry.set(id,{hash,value,until:now()+60000});return reply(200,value); + } finally {active--;} + } catch(error){return reply(error.status||503,{error:error.status?error.message:'OpenAI speech unavailable; using device speech'});} + }; +} +let live; +export async function handler(event) { + if(!live)live=(async()=>{ + const [{authorize,getHandleOrEmail},{connect}]=await Promise.all([import('../../backend/authorization.mjs'),import('../../backend/database.mjs')]); + const {db}=await connect(),key=process.env.WHISTLEGRAPH_TRANSCRIPTION_KEY; + return createHandler({authorize,getHandleOrEmail,billing:speechBilling(db),transcribe:createTranscriber({apiKey:key}),enabled:!!key}); + })().catch(error=>{live=null;throw error;}); + try{return await(await live)(event);}catch{return reply(503,{error:'OpenAI speech unavailable'});} +} diff --git a/system/public/privacy-policy.html b/system/public/privacy-policy.html index 24c316f382..5adff18f63 100644 --- a/system/public/privacy-policy.html +++ b/system/public/privacy-policy.html @@ -59,7 +59,7 @@ AI creation requires your permission. Requests, relevant source and version history, drawings and their stroke timing, generated-preview images used for visual review, and sound measurements needed for a request are sent through Aesthetic Computer to OpenRouter and your selected model provider. Available providers include DeepSeek, Moonshot AI, Alibaba/Qwen, MiniMax, and Z.ai. Eligible personal-model accounts can also use Anthropic or OpenAI through our relay.

- Cloud speech and cloud narration each require separate permission. Cloud speech sends microphone audio to OpenAI for transcription, directly or through our relay. Cloud narration sends caption text through Aesthetic Computer to ElevenLabs. Without those permissions, speech recognition and narration use the device when available. Typed input remains available. + Cloud speech and cloud narration each require separate permission. Whisper cloud speech sends each finished recording through AC to OpenAI for transcription and word timing. It uses 40 braincells per recorded second, drawn from the daily allowance first and then purchased credits. Failed transcriptions are refunded. AC stores billing receipts and an audio digest to prevent duplicate charges, but does not store the recording or transcript for this service. A retry result stays in server memory for up to one minute. Earlier personal builds may also use live OpenAI transcription. Cloud narration sends caption text through Aesthetic Computer to ElevenLabs. Without those permissions, speech recognition and narration use the device when available. Typed input remains available.

You can withdraw these permissions in Whistlegraph's AI & privacy settings. This stops future transfers for the disabled feature; it cannot recall data already sent. Account deletion removes the drafts and history we store for your account. Provider processing and retention follow their own policies, including OpenRouter, OpenAI, Anthropic, and ElevenLabs. diff --git a/system/tests/whistlegraph-speech.test.mjs b/system/tests/whistlegraph-speech.test.mjs new file mode 100644 index 0000000000..d74494fceb --- /dev/null +++ b/system/tests/whistlegraph-speech.test.mjs @@ -0,0 +1,65 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {randomUUID} from 'node:crypto'; +import {speechCost,speechBilling} from '../backend/whistlegraph-speech-billing.mjs'; +import {wavInput,createTranscriber} from '../backend/whistlegraph-whisper.mjs'; +import {createHandler} from '../netlify/functions/whistlegraph-transcribe.mjs'; +function wav(ms=1000) { + const b=Buffer.alloc(44+32*ms);b.write('RIFF');b.writeUInt32LE(b.length-8,4);b.write('WAVEfmt ',8);b.writeUInt32LE(16,16);b.writeUInt16LE(1,20);b.writeUInt16LE(1,22);b.writeUInt32LE(16000,24);b.writeUInt32LE(32000,28);b.writeUInt16LE(2,32);b.writeUInt16LE(16,34);b.write('data',36);b.writeUInt32LE(b.length-44,40);return b; +} +const event=(id=randomUUID())=>({httpMethod:'POST',headers:{authorization:'Bearer test'},body:JSON.stringify({requestId:id,audio:wav().toString('base64')})}); +test('Whisper uses actual bounded PCM duration and the published cost conversion',()=>{ + assert.equal(speechCost(1000),40);assert.equal(speechCost(8000),320);assert.equal(speechCost(45000),1800); + assert.equal(wavInput({audio:wav(1100).toString('base64')}).durationMs,1100); + assert.throws(()=>wavInput({audio:wav(47000).toString('base64')}),{status:400}); + const bad=wav();bad.writeUInt32LE(48000,24);assert.throws(()=>wavInput({audio:bad.toString('base64')}),{status:400}); +}); +test('authenticated replay returns the same result without another reservation or provider call',async()=>{ + let starts=0,calls=0,finishes=0; + const handler=createHandler({authorize:async()=>({sub:'u',email_verified:true}),getHandleOrEmail:async()=> '@fifi',billing:{begin:async()=>{starts++;return {id:'r',braincells:40,free:40,paid:0}},finish:async(_,ok)=>{assert.equal(ok,true);finishes++;return true;}},transcribe:async()=>{calls++;return {transcript:'hello',words:[]}}}); + const input=event();assert.equal((await handler(input)).statusCode,200);assert.equal((await handler(input)).statusCode,200); + assert.deepEqual([starts,calls,finishes],[1,1,1]); + const changed={...input,body:JSON.stringify({...JSON.parse(input.body),audio:wav(2000).toString('base64')})};assert.equal((await handler(changed)).statusCode,409); + assert.equal((await handler({...input,headers:{}})).statusCode,401); +}); +test('provider failure refunds and unverified users never reserve or upload audio',async()=>{ + let verified=true,refunded=0,calls=0; + const handler=createHandler({authorize:async()=>({sub:'u',email_verified:verified}),getHandleOrEmail:async()=> '@fifi',billing:{begin:async()=>({id:'r'}),finish:async(_,ok)=>{assert.equal(ok,false);refunded++;return true}},transcribe:async()=>{calls++;throw Object.assign(Error('Provider failed'),{status:502});}}); + assert.equal((await handler(event())).statusCode,502);assert.equal(refunded,1); + verified=false;assert.equal((await handler(event())).statusCode,403);assert.equal(calls,1); +}); +test('Whisper request preserves measured word timing',async()=>{ + const transcribe=createTranscriber({apiKey:'test',fetch:async(url,options)=>{ + assert.equal(options.body.get('model'),'whisper-1');assert.equal(options.body.get('timestamp_granularities[]'),'word'); + return {ok:true,json:async()=>({text:'Hello',words:[{word:'Hello',start:0.1,end:0.5}]})}; + }}); + assert.deepEqual((await transcribe({audio:wav().toString('base64')})).words,[{text:'Hello',atMs:100,durationMs:400}]); +}); +// Explicit integration mode, using only fresh, randomly named collections. +// Never read or mutate a real account, wallet or usage document. +test('Mongo transactions fence concurrent spending, replay, failure and crash recovery',{skip:process.env.WHISPER_BILLING_INTEGRATION!=='1'},async()=>{ + const {connect,closePool}=await import('../backend/database.mjs');const {db}=await connect(); + const prefix='test-whisper-'+randomUUID()+'-',names=new Set(); + const isolated={client:db.client,collection(name){names.add(prefix+name);return db.collection(prefix+name);}}; + let clock=new Date('2026-10-05T12:00:00Z');const billing=speechBilling(isolated,{now:()=>clock,limit:60}); + const request=(user='test-user')=>({user,handle:user,requestId:randomUUID(),audio:wav(),durationMs:1000}); + try { + // Materialize the three isolated collections before transactional DDL. + for(const name of ['ai-usage','ac-credit-wallets','whistlegraph-speech-requests'])await isolated.collection(name).insertOne({_id:'fixture'}); + const wallets=isolated.collection('ac-credit-wallets'),usage=isolated.collection('ai-usage'),receipts=isolated.collection('whistlegraph-speech-requests'); + await wallets.insertOne({_id:'test-user',balance:100}); + const a=request(),first=await billing.begin(a);assert.equal(first.free,40);assert.equal(first.paid,0); + await assert.rejects(billing.begin(a),{status:409});await billing.finish(first.id,true);assert.equal(await billing.finish(first.id,false),false); + const second=await billing.begin(request());assert.equal(second.free,20);assert.equal(second.paid,20); + assert.equal((await wallets.findOne({_id:'test-user'})).balance,80);await billing.finish(second.id,false); + assert.equal((await wallets.findOne({_id:'test-user'})).balance,100);assert.equal((await usage.findOne({_id:'test-user:2026-10-05'})).tokens,40); + const third=await billing.begin(request());await billing.finish(third.id,true); + assert.equal((await wallets.findOne({_id:'test-user'})).balance,80); + const stale=await billing.begin(request());clock=new Date(+clock+121000);assert.equal(await billing.reconcile(),1); + assert.equal((await wallets.findOne({_id:'test-user'})).balance,80);assert.equal(await billing.finish(stale.id,true),false); + const concurrent=await Promise.allSettled([billing.begin(request('race')),billing.begin(request('race'))]); + assert.equal(concurrent.filter(v=>v.status==='fulfilled').length,1);assert.equal(concurrent.find(v=>v.status==='rejected').reason.status,402); + const doc=await receipts.findOne({_id:first.id});assert.equal(doc.charged,40);assert.ok(!('audio' in doc));assert.ok(!('transcript' in doc)); + const overspend=request('no-money');await billing.begin(overspend);await assert.rejects(billing.begin(request('no-money')),{status:402}); + } finally {for(const name of names)await db.collection(name).drop();await closePool();} +});