diff --git a/aesel/src/whistlegraph-thread.mjs b/aesel/src/whistlegraph-thread.mjs index 42025c234a..d31133c868 100644 --- a/aesel/src/whistlegraph-thread.mjs +++ b/aesel/src/whistlegraph-thread.mjs @@ -1,4 +1,15 @@ // Stable local identity; the server atomically reserves its pronounceable code. +// What the phone remembers of the last cloud ledger it saw: a mark of its +// text, not the text. Keeping the whole ledger a second time doubled the +// biggest record in a 5 MB store (a 16-version piece is 500k characters +// each way, 2026-10-10) and left no room to park a piece. +export function ledgerMark(ledger) { + const text = typeof ledger === 'string' ? ledger : ledgerText(ledger); + if (text.startsWith('mark:')) return text; + let a = 0x811c9dc5, b = 0x01000193 ^ 0x5bd1e995; + for (let i = 0; i < text.length; i++) { const c = text.charCodeAt(i); a = Math.imul(a ^ c, 0x01000193); b = Math.imul(b ^ c, 0x5bd1e995) ^ (b >>> 13); } + return 'mark:' + (a >>> 0).toString(16).padStart(8, '0') + (b >>> 0).toString(16).padStart(8, '0') + text.length.toString(16); +} export const ledgerText = ledger => ledger?JSON.stringify({format:ledger.format,head:ledger.head,versions:ledger.versions.map(v=>({id:v.id,parent:v.parent,source:v.source,request:v.request??null,createdAt:v.createdAt||'',layers:Number.isInteger(v.layers)?v.layers:0}))}):'null'; export async function verifyThreadRevision(command,state) { const before=state();if(before.busy)throw Error('Device busy'); @@ -16,7 +27,7 @@ export class WhistlegraphThread { constructor({storage,key,token,ledger,state,onStatus,onCommand,onAdopt=null,onTurn=null,receipts=null,WebSocketImpl=globalThis.WebSocket,url='wss://aesthetic.computer/api/whistlegraph-stream',heartbeatMs=15000,maxIdleMs=45000,reconnectMs=3000}) { this.receipts=receipts; Object.assign(this,{storage,key,token,ledger,state,onStatus,onCommand,onAdopt,onTurn,WebSocketImpl,url,heartbeatMs,maxIdleMs,reconnectMs}); - this.identity=threadIdentity(storage,key);this.revision=Number(storage.getItem(key+'-cloud-revision')||0);this.last=storage.getItem(key+'-cloud-ledger')||''; + this.identity=threadIdentity(storage,key);this.revision=Number(storage.getItem(key+'-cloud-revision')||0);this.last=storage.getItem(key+'-cloud-ledger')||'';if(this.last&&!this.last.startsWith('mark:')){try{this.last=ledgerMark(JSON.parse(this.last));storage.setItem(key+'-cloud-ledger',this.last);}catch{this.last='';}} this.active=false;this.sending=false;this.ready=false;this.connectURL=url; } async resume() { @@ -33,7 +44,7 @@ export class WhistlegraphThread { if(m.type==='ready') { this.receiptSupport=m.capabilities?.includes('attempt-receipts-v1')===true; this.identity.code=m.thread.code;this.storage.setItem(this.key+'-thread',JSON.stringify(this.identity)); - const cloud=ledgerText(m.thread.ledger),local=ledgerText(this.ledger()); + const cloud=ledgerMark(m.thread.ledger),local=ledgerMark(this.ledger()); // Never silently replace local work with another device's history. A // device with nothing unsynced (its local ledger is the last cloud // ledger it saw) follows the server forward: that is a turn that ran @@ -49,13 +60,13 @@ export class WhistlegraphThread { if(m.type==='turn'&&m.turn){try{await this.onTurn?.(m.turn);}catch{}} if(m.type==='updated'&&m.thread?.ledger) { if(this.sending)return; - const cloud=ledgerText(m.thread.ledger),local=ledgerText(this.ledger()); + const cloud=ledgerMark(m.thread.ledger),local=ledgerMark(this.ledger()); if(cloud===local){this.revision=m.thread.revision;this.last=cloud;this.storage.setItem(this.key+'-cloud-revision',String(this.revision));this.storage.setItem(this.key+'-cloud-ledger',cloud);return;} if(this.canFollow(local,m.thread)){await this.follow(m.thread);this.update();} else this.onStatus(this.identity.code,'History conflict'); } if(m.type==='saved') { - this.revision=m.thread.revision;this.last=ledgerText(m.thread.ledger);this.sending=false; + this.revision=m.thread.revision;this.last=ledgerMark(m.thread.ledger);this.sending=false; this.storage.setItem(this.key+'-cloud-revision',String(this.revision));this.storage.setItem(this.key+'-cloud-ledger',this.last);this.sync(); } if(m.type==='receiptSaved'&&m.id===this.receiptSending) { @@ -84,19 +95,19 @@ export class WhistlegraphThread { send(value){if(this.ws?.readyState===1)this.ws.send(JSON.stringify(value));} canFollow(local,thread){return !!this.onAdopt&&local===this.last&&Number.isSafeInteger(thread.revision)&&thread.revision>this.revision;} async follow(thread){ - this.revision=thread.revision;this.last=ledgerText(thread.ledger); + this.revision=thread.revision;this.last=ledgerMark(thread.ledger); this.storage.setItem(this.key+'-cloud-revision',String(this.revision));this.storage.setItem(this.key+'-cloud-ledger',this.last); await this.onAdopt(thread.ledger,thread); this.onStatus(this.identity.code,'Updated'); } - sync(){if(!this.ready||this.sending)return;const ledger=this.ledger(),next=ledgerText(ledger);if(next===this.last)return;this.sending=true;this.send({type:'sync',revision:this.revision,ledger});} + sync(){if(!this.ready||this.sending)return;const ledger=this.ledger(),next=ledgerMark(ledger);if(next===this.last)return;this.sending=true;this.send({type:'sync',revision:this.revision,ledger});} update(){if(this.ready)this.send({type:'state',state:this.state()});} flushReceipts(){ if(!this.ready||!this.receiptSupport||this.receiptSending)return; const receipt=this.receipts?.pending();if(!receipt)return; this.receiptSending=receipt.id;this.send({type:'receipt',receipt}); } - async flush(){for(let i=0;i<100;i++){if(!this.ready)throw Error('Connection lost; inspect history before retrying');if(!this.sending&&this.last===ledgerText(this.ledger()))return;await new Promise(r=>setTimeout(r,100));}throw Error('Version sync pending; inspect before retrying');} + async flush(){for(let i=0;i<100;i++){if(!this.ready)throw Error('Connection lost; inspect history before retrying');if(!this.sending&&this.last===ledgerMark(this.ledger()))return;await new Promise(r=>setTimeout(r,100));}throw Error('Version sync pending; inspect before retrying');} disconnect(ws){if(this.ws!==ws)return;this.ws=null;this.ready=false;this.sending=false;this.receiptSending=null;clearTimeout(this.receiptTimer);clearInterval(this.heartbeat);this.onStatus(this.identity.code,'Offline');try{ws.close();}catch{}if(this.active)this.timer=setTimeout(()=>this.resume(),this.reconnectMs);} suspend(){this.active=false;clearTimeout(this.timer);if(this.ws)this.disconnect(this.ws);} } diff --git a/apple/whistlegraph/Resources/Web/engine.mjs b/apple/whistlegraph/Resources/Web/engine.mjs index e0b65b6df5..2f01f76dac 100644 --- a/apple/whistlegraph/Resources/Web/engine.mjs +++ b/apple/whistlegraph/Resources/Web/engine.mjs @@ -16,7 +16,7 @@ import {initializeBasePiece,isBasePiece} from './base-piece.mjs'; import {handleCharacterColors,fetchHandleColors} from '/easel/src/handle-colors.mjs'; import {verifyAccount} from '/easel/src/account-access.mjs'; import {localEdit} from './local-scene.mjs'; -import {WhistlegraphThread,verifyThreadRevision,ledgerText} from '/easel/src/whistlegraph-thread.mjs'; +import {WhistlegraphThread,verifyThreadRevision,ledgerMark} from '/easel/src/whistlegraph-thread.mjs'; import {instantPiece} from './instant-piece.mjs'; import {MusicalInputSocket} from '/easel/src/musical-input-socket.mjs'; import {MusicalInputAdvisor} from '/easel/src/musical-input-advisor.mjs'; @@ -199,7 +199,7 @@ function nativeSnapshot(){ const displayVersion=presentedVersion===null?versions?.head:versions?.value.versions.find(v=>v.id===presentedVersion); if(captionSource!==displayVersion?.source){captionSource=displayVersion?.source;caption=pieceCaption(captionSource||'');} const phaseText=$('live-phase').textContent; - const snapshot={ware:'piece',inference:inferenceSnapshot(),output:outputStream,caption,code:thread?.identity.code||'',handle:accountHandle,colors:accountPalette,head:displayVersion?.id||0,hasPiece:!!source.trim(),hasPreview:!!source.trim()||!!provisional.trim(),busy:busy||!!remoteJob,phase:phaseText,error:!busy&&/error|unavailable|could not|sign.in|loading|no piece/i.test(phaseText)?phaseText:'',attempt:lastAttempt?{request:lastAttempt.request.slice(0,1000),status:lastAttempt.status,error:lastAttempt.error||''}:null,progress:(busy||remoteJob)?(remoteJob?turnProgress:{phase:phaseText,round:0,worker:'this phone',startedAt:lastAttempt?.startedAt||'',steps:turnSteps.slice(-16),said:''}):null,draft:(d=>d?{request:d.request.slice(0,1000),error:d.error||'',createdAt:d.createdAt||'',characters:d.source.length}:null)(readDraft())}; + const snapshot={ware:'piece',inference:inferenceSnapshot(),output:outputStream,caption,code:thread?.identity.code||'',handle:accountHandle,colors:accountPalette,head:displayVersion?.id||0,hasPiece:!!source.trim(),hasPreview:!!source.trim()||!!provisional.trim(),busy:busy||!!remoteJob,phase:phaseText,error:!busy&&/error|unavailable|could not|sign.in|loading|no piece|no room|not on the knot/i.test(phaseText)?phaseText:'',attempt:lastAttempt?{request:lastAttempt.request.slice(0,1000),status:lastAttempt.status,error:lastAttempt.error||''}:null,progress:(busy||remoteJob)?(remoteJob?turnProgress:{phase:phaseText,round:0,worker:'this phone',startedAt:lastAttempt?.startedAt||'',steps:turnSteps.slice(-16),said:''}):null,draft:(d=>d?{request:d.request.slice(0,1000),error:d.error||'',createdAt:d.createdAt||'',characters:d.source.length}:null)(readDraft())}; const serialized=JSON.stringify(snapshot);if(serialized===nativeLast&&!historyChanged)return;nativeLast=serialized; if(historyChanged)snapshot.versions=nativeRevisions; post({action:'snapshot',snapshot}); @@ -988,7 +988,7 @@ if(versions&&!window.__whistlegraphSequence&&!window.__whistlegraphBenchmark&&!w localStorage.setItem(storageKey+'-versions',JSON.stringify(cloud.ledger)); localStorage.setItem(storageKey+'-thread',JSON.stringify({id:cloud.id,code:cloud.code})); localStorage.setItem(storageKey+'-cloud-revision',String(cloud.revision)); - localStorage.setItem(storageKey+'-cloud-ledger',ledgerText(cloud.ledger)); + localStorage.setItem(storageKey+'-cloud-ledger',ledgerMark(cloud.ledger)); }catch{phase('This phone has no room to open that piece');location.reload();return;} location.reload(); } diff --git a/apple/whistlegraph/Resources/Web/legacy-storage.mjs b/apple/whistlegraph/Resources/Web/legacy-storage.mjs index 9b6727da14..db8b597ac7 100644 --- a/apple/whistlegraph/Resources/Web/legacy-storage.mjs +++ b/apple/whistlegraph/Resources/Web/legacy-storage.mjs @@ -29,3 +29,25 @@ export function migrateLegacyStorage(storage) { clear() { storage.clear(); }, }; } + +// The last cloud ledger used to be kept whole beside the ledger itself, in +// the open piece and inside every parked piece's extras. Each copy becomes +// a mark. On a phone with 5 MB this is the difference between parking a +// piece and "no room" (2026-10-10: 5.3 MB used, half of it copies). +export function compactLedgerCopies(storage, mark) { + let freed = 0; + for (const key of Array.from({length: storage.length}, (_, i) => storage.key(i))) { + if (!key) continue; + try { + if (key.endsWith('-cloud-ledger')) { + const value = storage.getItem(key); + if (value && !value.startsWith('mark:')) { const next = mark(JSON.parse(value)); storage.setItem(key, next); freed += value.length - next.length; } + } else if (key.startsWith('whistlegraph-archive-')) { + const value = storage.getItem(key); const saved = JSON.parse(value); + const copy = saved?.extras?.['-cloud-ledger']; + if (typeof copy === 'string' && !copy.startsWith('mark:')) { saved.extras['-cloud-ledger'] = mark(JSON.parse(copy)); const next = JSON.stringify(saved); storage.setItem(key, next); freed += value.length - next.length; } + } + } catch {} + } + return freed; +} diff --git a/apple/whistlegraph/Resources/Web/ware-engine.mjs b/apple/whistlegraph/Resources/Web/ware-engine.mjs index 083d525fde..7dbe518332 100644 --- a/apple/whistlegraph/Resources/Web/ware-engine.mjs +++ b/apple/whistlegraph/Resources/Web/ware-engine.mjs @@ -1,6 +1,8 @@ import {currentWare,selectWare} from './wares.mjs'; -import {migrateLegacyStorage} from './legacy-storage.mjs'; +import {migrateLegacyStorage,compactLedgerCopies} from './legacy-storage.mjs'; +import {ledgerMark} from '/easel/src/whistlegraph-thread.mjs'; Object.defineProperty(window, 'localStorage', {value:migrateLegacyStorage(window.localStorage), configurable:true}); +try{const freed=compactLedgerCopies(localStorage,ledgerMark);if(freed)console.log('[whistlegraph] compacted ledger copies, freed '+freed.toLocaleString('en-US')+' characters');}catch{} window.whistlegraphSelectWare=id=>{ if(window.whistlegraphIsBusy?.()||window.whistlegraphRecording?.())return false; try { diff --git a/apple/whistlegraph/Tests/ledger-mark.test.mjs b/apple/whistlegraph/Tests/ledger-mark.test.mjs new file mode 100644 index 0000000000..4b334cc55d --- /dev/null +++ b/apple/whistlegraph/Tests/ledger-mark.test.mjs @@ -0,0 +1,36 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {ledgerMark, ledgerText} from '../../../aesel/src/whistlegraph-thread.mjs'; +import {compactLedgerCopies} from '../Resources/Web/legacy-storage.mjs'; + +const ledger = (n, extra = '') => ({format: 1, head: n, versions: Array.from({length: n + 1}, (_, id) => ({id, parent: id ? id - 1 : null, source: 'x'.repeat(2000) + extra + id, request: id ? 'r' + id : null, createdAt: '', layers: 0}))}); +const memory = () => { const m = new Map(); return {get length() { return m.size; }, key: i => [...m.keys()][i] ?? null, getItem: k => m.get(k) ?? null, setItem: (k, v) => m.set(k, String(v)), removeItem: k => m.delete(k), clear: () => m.clear()}; }; + +test('a ledger mark is short, stable, and tells ledgers apart', () => { + const a = ledgerMark(ledger(3)), b = ledgerMark(ledger(3)), c = ledgerMark(ledger(3, 'y')), d = ledgerMark(ledger(4)); + assert.equal(a, b); + assert.notEqual(a, c); assert.notEqual(a, d); + assert.ok(a.startsWith('mark:') && a.length < 40, a); + assert.equal(ledgerMark(ledgerText(ledger(3))), a, 'marking the text equals marking the ledger'); + assert.equal(ledgerMark(a), a, 'a mark marks to itself'); +}); + +test('compaction turns whole cloud-ledger copies into marks, live and parked', () => { + const storage = memory(); + const open = ledger(5), parked = ledger(4); + storage.setItem('whistlegraph-source-cloud-ledger', ledgerText(open)); + storage.setItem('whistlegraph-source-versions', JSON.stringify(open)); + storage.setItem('whistlegraph-archive-abc', JSON.stringify({identity: {id: 'abc', code: 'wgTest'}, ledger: parked, source: 'x', extras: {'-cloud-ledger': ledgerText(parked), '-cloud-revision': '5'}})); + storage.setItem('whistlegraph-archive-def', JSON.stringify({identity: {id: 'def'}, ledger: parked, source: 'x', extras: {}})); + storage.setItem('unrelated', 'keep'); + const before = [...Array(storage.length)].reduce((n, _, i) => n + storage.getItem(storage.key(i)).length, 0); + const freed = compactLedgerCopies(storage, ledgerMark); + const after = [...Array(storage.length)].reduce((n, _, i) => n + storage.getItem(storage.key(i)).length, 0); + assert.equal(storage.getItem('whistlegraph-source-cloud-ledger'), ledgerMark(open)); + assert.equal(JSON.parse(storage.getItem('whistlegraph-archive-abc')).extras['-cloud-ledger'], ledgerMark(parked)); + assert.equal(JSON.parse(storage.getItem('whistlegraph-archive-abc')).extras['-cloud-revision'], '5'); + assert.equal(storage.getItem('unrelated'), 'keep'); + assert.equal(before - after, freed); + assert.ok(freed > 20000, 'freed ' + freed); + assert.equal(compactLedgerCopies(storage, ledgerMark), 0, 'a second pass frees nothing'); +});