diff --git a/aesel/src/musical-decisions.mjs b/aesel/src/musical-decisions.mjs new file mode 100644 index 0000000000..7589f4476b --- /dev/null +++ b/aesel/src/musical-decisions.mjs @@ -0,0 +1,37 @@ +// Shared phone/server vocabulary. No transcript, PCM, source, or identity goes to Jev. +export const MUSICAL_CHOICES = Object.freeze({ + follow_speech:'Follow the spoken request; use sound only as supporting expression.', + trace_pitch:'Use the measured pitch contour to shape visual movement.', + pulse_onsets:'Use measured sound attacks as a rhythm for visual events; do not assert a tempo.', + sustain:'Use a sustained sound to shape a continuous visual behavior.', + observe:'Evidence is insufficient for a musical mapping; keep the ordinary creation flow.' +}); +export function normalizeMusicalFeatures(value) { + const fields=['hasSpeech','hasTonalSound','soundAfterSpeech','contour','attacks','rhythm','energy']; + if(!value||Object.keys(value).some(k=>!fields.includes(k)))throw Error('Unexpected musical feature'); + for(const k of fields.slice(0,3))if(typeof value[k]!=='boolean')throw Error('Invalid musical flag'); + if(!['none','steady','rising','falling','varied'].includes(value.contour)||!['unknown','regular','irregular'].includes(value.rhythm)||!['quiet','steady','changing'].includes(value.energy)||!Number.isInteger(value.attacks)||value.attacks<0||value.attacks>16)throw Error('Invalid musical feature'); + return Object.fromEntries(fields.map(k=>[k,value[k]])); +} +export function summarizeMusicalInput(input) { + const hasSpeech=Boolean(input.transcript?.trim()); + const end=Math.max(0,...(input.words||[]).map(w=>w.atMs+w.durationMs)); + const all=input.sound?.frames||[], tail=hasSpeech?all.filter(f=>f.atMs>end+80):all; + const pitches=tail.filter(f=>f.pitchHz>0).map(f=>f.pitchHz); + let contour='none'; + if(pitches.length>=3){ + const third=Math.max(1,Math.floor(pitches.length/3)); + const avg=a=>a.reduce((s,v)=>s+Math.log2(v),0)/a.length; + const change=12*(avg(pitches.slice(-third))-avg(pitches.slice(0,third))); + const span=12*Math.log2(Math.max(...pitches)/Math.min(...pitches)); + contour=change>1?'rising':change< -1?'falling':span>2?'varied':'steady'; + } + const onsets=(input.sound?.onsetsMs||[]).filter(t=>!hasSpeech||t>end+80); + const intervals=onsets.slice(1).map((v,i)=>v-onsets[i]);let rhythm='unknown'; + if(intervals.length>=2){const mean=intervals.reduce((s,v)=>s+v,0)/intervals.length;rhythm=mean>0&&Math.sqrt(intervals.reduce((s,v)=>s+(v-mean)**2,0)/intervals.length)/mean<.2?'regular':'irregular';} + const amplitudes=tail.map(f=>f.rms).filter(v=>v>.012); + return normalizeMusicalFeatures({hasSpeech,hasTonalSound:pitches.length>=3,soundAfterSpeech:hasSpeech&&litudes.length>=3,contour,attacks:Math.min(16,onsets.length),rhythm,energy:amplitudes.length<3?'quiet':Math.max(...amplitudes)/Math.min(...amplitudes)>2?'changing':'steady'}); +} +export function musicalDecisionRequest(features) { + return {state:normalizeMusicalFeatures(features),questions:{mapping:{type:'choice',criteria:MUSICAL_CHOICES,instructions:'Choose a useful creative mapping for this musical input. Spoken instructions take priority: favor follow_speech when hasSpeech. Features after speech are only supporting expression. Repeated clear attacks favor pulse_onsets; a rising/falling tonal contour favors trace_pitch; a steady tone favors sustain. Quiet or insufficient evidence favors observe. These are creative suggestions, never claims of user intent or successful tests.'}}}; +} diff --git a/aesel/src/musical-input-advisor.mjs b/aesel/src/musical-input-advisor.mjs new file mode 100644 index 0000000000..98dabd8fdd --- /dev/null +++ b/aesel/src/musical-input-advisor.mjs @@ -0,0 +1,41 @@ +import {summarizeMusicalInput,MUSICAL_CHOICES} from './musical-decisions.mjs'; +// Incremental observations are replaceable snapshots, never a queue of stale work. +export class MusicalInputAdvisor { + constructor({fetchImpl=globalThis.fetch,token,endpoint='https://aesthetic.computer/api/easel-musical-jev',onEvent=()=>{},timeoutMs=900}={}){Object.assign(this,{fetchImpl,token,endpoint,onEvent,timeoutMs});this.reset();} + reset(){this.controller?.abort();this.sessionId=globalThis.crypto.randomUUID();this.sequence=0;this.calls=0;this.cache=null;this.pending=null;this.lastSent=0;} + observe(input){ + const features=summarizeMusicalInput(input),key=JSON.stringify(features); + if(this.cache?.key===key||this.pending||this.calls>=3||Date.now()-this.lastSent<500)return; + void this.request(features,key); + } + async request(features,key){ + const bearer=this.token?.();if(!bearer)return null; + const sessionId=this.sessionId,sequence=++this.sequence;this.calls++;this.lastSent=Date.now(); + const controller=new AbortController();this.controller=controller; + const started=performance.now(); + const work=(async()=>{ + let timer; + try{ + await Promise.resolve(); + const deadline=new Promise((_,reject)=>{timer=setTimeout(()=>{controller.abort();reject(Error('deadline'));},this.timeoutMs);}); + const response=await Promise.race([deadline,this.fetchImpl(this.endpoint,{method:'POST',signal:controller.signal,headers:{'Content-Type':'application/json',Authorization:`Bearer ${bearer}`},body:JSON.stringify({schema:'walkieware-input/v1',sessionId,sequence,features})}).then(async r=>{if(!r.ok)throw Error('unavailable');return r.json();})]); + if(this.sessionId!==sessionId||controller.signal.aborted||response.schema!=='walkieware-decision/v1'||response.sessionId!==sessionId||response.sequence!==sequence||!Object.hasOwn(MUSICAL_CHOICES,response.choice)||!Number.isFinite(response.confidence)||response.confidence<.8||response.confidence>1)return null; + const value={key,choice:response.choice,cue:MUSICAL_CHOICES[response.choice],elapsedMs:Math.round(performance.now()-started)}; + this.cache=value;this.onEvent('jevDecision',{choice:value.choice,elapsedMs:value.elapsedMs,sequence});return value; + }catch{if(this.sessionId===sessionId)this.onEvent('jevFallback',{});return null;} + finally{clearTimeout(timer);if(this.sessionId===sessionId)this.pending=null;} + })(); + this.pending={key,work};return work; + } + async finish(input){ + const features=summarizeMusicalInput(input),key=JSON.stringify(features); + if(this.cache?.key===key){this.onEvent('jevCacheHit',{});return this.cache;} + if(this.pending?.key===key)return this.pending.work; + this.controller?.abort();this.pending=null; + // An older observation must never overwrite this final result. + this.sessionId=globalThis.crypto.randomUUID();this.cache=null; + if(this.calls>=4)return null; + return this.request(features,key); + } + cancel(){this.reset();} +} diff --git a/aesel/test/musical-jev.test.mjs b/aesel/test/musical-jev.test.mjs new file mode 100644 index 0000000000..8e53d53661 --- /dev/null +++ b/aesel/test/musical-jev.test.mjs @@ -0,0 +1,36 @@ +import test from 'node:test';import assert from 'node:assert/strict'; +import {summarizeMusicalInput} from '../src/musical-decisions.mjs'; +import {MusicalInputAdvisor} from '../src/musical-input-advisor.mjs'; +import {createMusicalHandler,musicalBudget} from '../../system/backend/easel-musical-jev.mjs'; +const input={transcript:'PRIVATE WORDS',words:[{atMs:0,durationMs:200}],sound:{frames:[{atMs:400,rms:.2,pitchHz:440},{atMs:600,rms:.2,pitchHz:660},{atMs:800,rms:.2,pitchHz:880}],onsetsMs:[400]}}; +const features=summarizeMusicalInput(input); +const event={httpMethod:'POST',headers:{authorization:'test'},body:JSON.stringify({schema:'walkieware-input/v1',sessionId:crypto.randomUUID(),sequence:1,features})}; +const answer={answers:{mapping:{choice:'follow_speech',probabilities:{follow_speech:.94}}}}; +test('server minimizes evidence and echoes observation identity',async()=>{ + const handler=createMusicalHandler({authenticate:async()=> 'subject',budget:{consume:async()=>true},evaluate:async request=>{assert.doesNotMatch(JSON.stringify(request),/PRIVATE|subject/);return answer;}}); + const r=await handler(event);assert.equal(r.statusCode,200);assert.equal(JSON.parse(r.body).sessionId,JSON.parse(event.body).sessionId); + assert.equal((await handler({...event,headers:{}})).statusCode,401); + assert.equal((await handler({...event,body:JSON.stringify({...JSON.parse(event.body),prompt:'anything'})})).statusCode,400); + assert.equal((await handler({...event,body:JSON.stringify({...JSON.parse(event.body),features:{...features,transcript:'bad'}})})).statusCode,400); +}); +test('budget failure, provider failure and invalid account never fabricate advice',async()=>{ + let calls=0;const base={authenticate:async()=> 'subject',budget:{consume:async()=>false},evaluate:async()=>{calls++;return answer;}}; + assert.equal((await createMusicalHandler(base)(event)).statusCode,429);assert.equal(calls,0); + assert.equal((await createMusicalHandler({...base,authenticate:async()=>null})(event)).statusCode,401); + assert.equal((await createMusicalHandler({...base,budget:{consume:async()=>true},evaluate:async()=>{throw Error('secret');}})(event)).statusCode,503); +}); +test('streaming advice is cached for matching musical state without sending words',async()=>{ + let calls=0;const advisor=new MusicalInputAdvisor({token:()=> 'test',fetchImpl:async(_,options)=>{calls++;assert.doesNotMatch(options.body,/PRIVATE|transcript|pitchHz/);const b=JSON.parse(options.body);return Response.json({...b,schema:'walkieware-decision/v1',choice:'follow_speech',confidence:.94});}}); + advisor.observe(input);await advisor.pending.work;const r=await advisor.finish(input);assert.equal(r.choice,'follow_speech');assert.equal(calls,1); +}); +test('late responses, low confidence and timeouts cannot steer',async()=>{ + let resolve;const a=new MusicalInputAdvisor({token:()=> 'test',timeoutMs:15,fetchImpl:()=>new Promise(r=>resolve=r)}); + const p=a.finish(input);await Promise.resolve();a.cancel();resolve(Response.json({choice:'sustain',confidence:1}));assert.equal(await p,null);assert.equal(a.cache,null); + const stalled=new MusicalInputAdvisor({token:()=> 'test',timeoutMs:15,fetchImpl:()=>new Promise(()=>{})});assert.equal(await stalled.finish(input),null); + const uncertain=new MusicalInputAdvisor({token:()=> 'test',fetchImpl:async(_,o)=>Response.json({...JSON.parse(o.body),schema:'walkieware-decision/v1',choice:'sustain',confidence:.2})});assert.equal(await uncertain.finish(input),null); +}); +test('feature summaries separate speech interval from later sound',()=>{assert.equal(features.hasSpeech,true);assert.equal(features.soundAfterSpeech,true);assert.equal(features.contour,'rising');assert.equal(features.attacks,1);}); +test('atomic quota reservation refuses a full counter',async()=>{ + let increments=0;const budget=musicalBudget({updateOne:async()=>{},findOneAndUpdate:async()=>{increments++;return null;}}); + assert.equal(await budget.consume('subject',Date.now()),false);assert.equal(increments,1); +}); diff --git a/system/backend/easel-musical-jev.mjs b/system/backend/easel-musical-jev.mjs new file mode 100644 index 0000000000..bbcaf24311 --- /dev/null +++ b/system/backend/easel-musical-jev.mjs @@ -0,0 +1,40 @@ +import {createHash} from 'node:crypto'; +import {musicalDecisionRequest,MUSICAL_CHOICES} from '../../aesel/src/musical-decisions.mjs'; +import {evaluateChoices} from '../../aesel/src/jev-decisions.mjs'; +export function musicalBudget(collection) { + async function reserve(id,limit,expiresAt){ + try{await collection.updateOne({_id:id},{$setOnInsert:{count:0,expiresAt}},{upsert:true});}catch(e){if(e.code!==11000)throw e;} + const r=await collection.findOneAndUpdate({_id:id,count:{$lt:limit}},{$inc:{count:1}},{returnDocument:'after'}); + return Boolean((r?.value??r)?.count); + } + return {async consume(subject,now){ + const day=new Date(now).toISOString().slice(0,10), user=createHash('sha256').update(subject).digest('hex'); + const expires=new Date(now+2*86400000); + return await reserve(`minute:${Math.floor(now/60000)}:${user}`,30,expires)&&await reserve(`user:${day}:${user}`,500,expires)&&await reserve(`global:${day}`,20000,expires); + }}; +} +export function createMusicalHandler({authenticate,budget,evaluate=evaluateChoices,now=Date.now}={}) { + const busy=new Set(); + const reply=(statusCode,value)=>({statusCode,headers:{'Content-Type':'application/json','Cache-Control':'no-store','Access-Control-Allow-Origin':'*','Access-Control-Allow-Headers':'Content-Type, Authorization','Access-Control-Allow-Methods':'POST, OPTIONS'},body:JSON.stringify(value)}); + return async event=>{ + if(event.httpMethod==='OPTIONS')return reply(200,{}); + if(event.httpMethod!=='POST')return reply(405,{error:'POST only'}); + if(!event.headers?.authorization)return reply(401,{error:'Sign in first'}); + if(typeof event.body!=='string'||event.body.length>2048)return reply(400,{error:'Invalid observation'}); + let body,request; + try{body=JSON.parse(event.body);if(body.schema!=='walkieware-input/v1'||! /^[a-f0-9-]{36}$/i.test(body.sessionId)||!Number.isInteger(body.sequence)||body.sequence<1||body.sequence>1000||Object.keys(body).some(k=>!['schema','sessionId','sequence','features'].includes(k)))throw Error();request=musicalDecisionRequest(body.features);}catch{return reply(400,{error:'Invalid observation'});} + let subject;try{subject=await authenticate(event.headers);}catch{return reply(503,{error:'Account check unavailable'});} + if(!subject)return reply(401,{error:'A valid account with a handle is required'}); + if(busy.has(subject)||busy.size>=8)return reply(429,{error:'Decision already running'}); + busy.add(subject); + try{ + if(!await budget.consume(subject,now()))return reply(429,{error:'Decision allowance reached'}); + const started=performance.now(); + const result=await evaluate(request,{signal:AbortSignal.timeout(1000)}); + const answer=result.answers?.mapping, confidence=answer?.probabilities?.[answer.choice]; + if(!Object.hasOwn(MUSICAL_CHOICES,answer?.choice)||!Number.isFinite(confidence)||confidence<0||confidence>1)throw Error('Invalid answer'); + return reply(200,{schema:'walkieware-decision/v1',sessionId:body.sessionId,sequence:body.sequence,choice:answer.choice,confidence,elapsedMs:Math.round(performance.now()-started)}); + }catch{return reply(503,{error:'Musical advice unavailable; continue without it'});} + finally{busy.delete(subject);} + }; +} diff --git a/system/netlify/functions/easel-musical-jev.mjs b/system/netlify/functions/easel-musical-jev.mjs new file mode 100644 index 0000000000..43f7b64870 --- /dev/null +++ b/system/netlify/functions/easel-musical-jev.mjs @@ -0,0 +1,10 @@ +import {connect} from '../../backend/database.mjs'; +import {authorize,getHandleOrEmail} from '../../backend/authorization.mjs'; +import {createMusicalHandler,musicalBudget} from '../../backend/easel-musical-jev.mjs'; +let storage; +async function budget(){return storage??= (async()=>{const {db}=await connect();const c=db.collection('easel-musical-jev-budget');await c.createIndex({expiresAt:1},{expireAfterSeconds:0});return musicalBudget(c);})().catch(e=>{storage=null;throw e;});} +const handlerImpl=createMusicalHandler({ + authenticate:async headers=>{const user=await authorize(headers);if(!user?.sub)return null;const handle=await getHandleOrEmail(user.sub);return typeof handle==='string'&&handle.startsWith('@')?user.sub:null;}, + budget:{consume:async(...args)=>(await budget()).consume(...args)} +}); +export const handler=event=>handlerImpl(event);