From 8aa7ea51022c53803be433e38da66dc13c0d73b9 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Sat, 10 Oct 2026 01:16:19 -0700 Subject: [PATCH] Whistlegraph: the last cloud ledger is remembered as a mark, not a second copy Opening a knot piece failed silently: the phone's store was at 5.3 MB of its 5 MB, so parking the open piece could not write. Half of that was the cloud ledger kept whole beside the ledger itself, in the open piece and in every parked piece's extras. The thread client now keeps a short mark of the ledger text and compares marks; a stored whole copy becomes a mark on first read, and a compaction pass at boot rewrites every copy, live and parked. A no-room or missing-thread failure now shows as a note on the phone instead of a phase only the page could see. --- aesel/src/whistlegraph-thread.mjs | 25 +++++++++---- apple/whistlegraph/Resources/Web/engine.mjs | 6 ++-- .../Resources/Web/legacy-storage.mjs | 22 ++++++++++++ .../Resources/Web/ware-engine.mjs | 4 ++- apple/whistlegraph/Tests/ledger-mark.test.mjs | 36 +++++++++++++++++++ 5 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 apple/whistlegraph/Tests/ledger-mark.test.mjs diff --git a/aesel/src/whistlegraph-thread.mjs b/aesel/src/whistlegraph-thread.mjs index 42025c234a..d31133c868 100644 --- a/aesel/src/whistlegraph-thread.mjs +++ b/aesel/src/whistlegraph-thread.mjs @@ -1,4 +1,15 @@ // Stable local identity; the server atomically reserves its pronounceable code. +// What the phone remembers of the last cloud ledger it saw: a mark of its +// text, not the text. Keeping the whole ledger a second time doubled the +// biggest record in a 5 MB store (a 16-version piece is 500k characters +// each way, 2026-10-10) and left no room to park a piece. +export function ledgerMark(ledger) { + const text = typeof ledger === 'string' ? ledger : ledgerText(ledger); + if (text.startsWith('mark:')) return text; + let a = 0x811c9dc5, b = 0x01000193 ^ 0x5bd1e995; + for (let i = 0; i < text.length; i++) { const c = text.charCodeAt(i); a = Math.imul(a ^ c, 0x01000193); b = Math.imul(b ^ c, 0x5bd1e995) ^ (b >>> 13); } + return 'mark:' + (a >>> 0).toString(16).padStart(8, '0') + (b >>> 0).toString(16).padStart(8, '0') + text.length.toString(16); +} export const ledgerText = ledger => ledger?JSON.stringify({format:ledger.format,head:ledger.head,versions:ledger.versions.map(v=>({id:v.id,parent:v.parent,source:v.source,request:v.request??null,createdAt:v.createdAt||'',layers:Number.isInteger(v.layers)?v.layers:0}))}):'null'; export async function verifyThreadRevision(command,state) { const before=state();if(before.busy)throw Error('Device busy'); @@ -16,7 +27,7 @@ export class WhistlegraphThread { constructor({storage,key,token,ledger,state,onStatus,onCommand,onAdopt=null,onTurn=null,receipts=null,WebSocketImpl=globalThis.WebSocket,url='wss://aesthetic.computer/api/whistlegraph-stream',heartbeatMs=15000,maxIdleMs=45000,reconnectMs=3000}) { this.receipts=receipts; Object.assign(this,{storage,key,token,ledger,state,onStatus,onCommand,onAdopt,onTurn,WebSocketImpl,url,heartbeatMs,maxIdleMs,reconnectMs}); - this.identity=threadIdentity(storage,key);this.revision=Number(storage.getItem(key+'-cloud-revision')||0);this.last=storage.getItem(key+'-cloud-ledger')||''; + this.identity=threadIdentity(storage,key);this.revision=Number(storage.getItem(key+'-cloud-revision')||0);this.last=storage.getItem(key+'-cloud-ledger')||'';if(this.last&&!this.last.startsWith('mark:')){try{this.last=ledgerMark(JSON.parse(this.last));storage.setItem(key+'-cloud-ledger',this.last);}catch{this.last='';}} this.active=false;this.sending=false;this.ready=false;this.connectURL=url; } async resume() { @@ -33,7 +44,7 @@ export class WhistlegraphThread { if(m.type==='ready') { this.receiptSupport=m.capabilities?.includes('attempt-receipts-v1')===true; this.identity.code=m.thread.code;this.storage.setItem(this.key+'-thread',JSON.stringify(this.identity)); - const cloud=ledgerText(m.thread.ledger),local=ledgerText(this.ledger()); + const cloud=ledgerMark(m.thread.ledger),local=ledgerMark(this.ledger()); // Never silently replace local work with another device's history. A // device with nothing unsynced (its local ledger is the last cloud // ledger it saw) follows the server forward: that is a turn that ran @@ -49,13 +60,13 @@ export class WhistlegraphThread { if(m.type==='turn'&&m.turn){try{await this.onTurn?.(m.turn);}catch{}} if(m.type==='updated'&&m.thread?.ledger) { if(this.sending)return; - const cloud=ledgerText(m.thread.ledger),local=ledgerText(this.ledger()); + const cloud=ledgerMark(m.thread.ledger),local=ledgerMark(this.ledger()); if(cloud===local){this.revision=m.thread.revision;this.last=cloud;this.storage.setItem(this.key+'-cloud-revision',String(this.revision));this.storage.setItem(this.key+'-cloud-ledger',cloud);return;} if(this.canFollow(local,m.thread)){await this.follow(m.thread);this.update();} else this.onStatus(this.identity.code,'History conflict'); } if(m.type==='saved') { - this.revision=m.thread.revision;this.last=ledgerText(m.thread.ledger);this.sending=false; + this.revision=m.thread.revision;this.last=ledgerMark(m.thread.ledger);this.sending=false; this.storage.setItem(this.key+'-cloud-revision',String(this.revision));this.storage.setItem(this.key+'-cloud-ledger',this.last);this.sync(); } if(m.type==='receiptSaved'&&m.id===this.receiptSending) { @@ -84,19 +95,19 @@ export class WhistlegraphThread { send(value){if(this.ws?.readyState===1)this.ws.send(JSON.stringify(value));} canFollow(local,thread){return !!this.onAdopt&&local===this.last&&Number.isSafeInteger(thread.revision)&&thread.revision>this.revision;} async follow(thread){ - this.revision=thread.revision;this.last=ledgerText(thread.ledger); + this.revision=thread.revision;this.last=ledgerMark(thread.ledger); this.storage.setItem(this.key+'-cloud-revision',String(this.revision));this.storage.setItem(this.key+'-cloud-ledger',this.last); await this.onAdopt(thread.ledger,thread); this.onStatus(this.identity.code,'Updated'); } - sync(){if(!this.ready||this.sending)return;const ledger=this.ledger(),next=ledgerText(ledger);if(next===this.last)return;this.sending=true;this.send({type:'sync',revision:this.revision,ledger});} + sync(){if(!this.ready||this.sending)return;const ledger=this.ledger(),next=ledgerMark(ledger);if(next===this.last)return;this.sending=true;this.send({type:'sync',revision:this.revision,ledger});} update(){if(this.ready)this.send({type:'state',state:this.state()});} flushReceipts(){ if(!this.ready||!this.receiptSupport||this.receiptSending)return; const receipt=this.receipts?.pending();if(!receipt)return; this.receiptSending=receipt.id;this.send({type:'receipt',receipt}); } - async flush(){for(let i=0;i<100;i++){if(!this.ready)throw Error('Connection lost; inspect history before retrying');if(!this.sending&&this.last===ledgerText(this.ledger()))return;await new Promise(r=>setTimeout(r,100));}throw Error('Version sync pending; inspect before retrying');} + async flush(){for(let i=0;i<100;i++){if(!this.ready)throw Error('Connection lost; inspect history before retrying');if(!this.sending&&this.last===ledgerMark(this.ledger()))return;await new Promise(r=>setTimeout(r,100));}throw Error('Version sync pending; inspect before retrying');} disconnect(ws){if(this.ws!==ws)return;this.ws=null;this.ready=false;this.sending=false;this.receiptSending=null;clearTimeout(this.receiptTimer);clearInterval(this.heartbeat);this.onStatus(this.identity.code,'Offline');try{ws.close();}catch{}if(this.active)this.timer=setTimeout(()=>this.resume(),this.reconnectMs);} suspend(){this.active=false;clearTimeout(this.timer);if(this.ws)this.disconnect(this.ws);} } diff --git a/apple/whistlegraph/Resources/Web/engine.mjs b/apple/whistlegraph/Resources/Web/engine.mjs index e0b65b6df5..2f01f76dac 100644 --- a/apple/whistlegraph/Resources/Web/engine.mjs +++ b/apple/whistlegraph/Resources/Web/engine.mjs @@ -16,7 +16,7 @@ import {initializeBasePiece,isBasePiece} from './base-piece.mjs'; import {handleCharacterColors,fetchHandleColors} from '/easel/src/handle-colors.mjs'; import {verifyAccount} from '/easel/src/account-access.mjs'; import {localEdit} from './local-scene.mjs'; -import {WhistlegraphThread,verifyThreadRevision,ledgerText} from '/easel/src/whistlegraph-thread.mjs'; +import {WhistlegraphThread,verifyThreadRevision,ledgerMark} from '/easel/src/whistlegraph-thread.mjs'; import {instantPiece} from './instant-piece.mjs'; import {MusicalInputSocket} from '/easel/src/musical-input-socket.mjs'; import {MusicalInputAdvisor} from '/easel/src/musical-input-advisor.mjs'; @@ -199,7 +199,7 @@ function nativeSnapshot(){ const displayVersion=presentedVersion===null?versions?.head:versions?.value.versions.find(v=>v.id===presentedVersion); if(captionSource!==displayVersion?.source){captionSource=displayVersion?.source;caption=pieceCaption(captionSource||'');} const phaseText=$('live-phase').textContent; - const snapshot={ware:'piece',inference:inferenceSnapshot(),output:outputStream,caption,code:thread?.identity.code||'',handle:accountHandle,colors:accountPalette,head:displayVersion?.id||0,hasPiece:!!source.trim(),hasPreview:!!source.trim()||!!provisional.trim(),busy:busy||!!remoteJob,phase:phaseText,error:!busy&&/error|unavailable|could not|sign.in|loading|no piece/i.test(phaseText)?phaseText:'',attempt:lastAttempt?{request:lastAttempt.request.slice(0,1000),status:lastAttempt.status,error:lastAttempt.error||''}:null,progress:(busy||remoteJob)?(remoteJob?turnProgress:{phase:phaseText,round:0,worker:'this phone',startedAt:lastAttempt?.startedAt||'',steps:turnSteps.slice(-16),said:''}):null,draft:(d=>d?{request:d.request.slice(0,1000),error:d.error||'',createdAt:d.createdAt||'',characters:d.source.length}:null)(readDraft())}; + const snapshot={ware:'piece',inference:inferenceSnapshot(),output:outputStream,caption,code:thread?.identity.code||'',handle:accountHandle,colors:accountPalette,head:displayVersion?.id||0,hasPiece:!!source.trim(),hasPreview:!!source.trim()||!!provisional.trim(),busy:busy||!!remoteJob,phase:phaseText,error:!busy&&/error|unavailable|could not|sign.in|loading|no piece|no room|not on the knot/i.test(phaseText)?phaseText:'',attempt:lastAttempt?{request:lastAttempt.request.slice(0,1000),status:lastAttempt.status,error:lastAttempt.error||''}:null,progress:(busy||remoteJob)?(remoteJob?turnProgress:{phase:phaseText,round:0,worker:'this phone',startedAt:lastAttempt?.startedAt||'',steps:turnSteps.slice(-16),said:''}):null,draft:(d=>d?{request:d.request.slice(0,1000),error:d.error||'',createdAt:d.createdAt||'',characters:d.source.length}:null)(readDraft())}; const serialized=JSON.stringify(snapshot);if(serialized===nativeLast&&!historyChanged)return;nativeLast=serialized; if(historyChanged)snapshot.versions=nativeRevisions; post({action:'snapshot',snapshot}); @@ -988,7 +988,7 @@ if(versions&&!window.__whistlegraphSequence&&!window.__whistlegraphBenchmark&&!w localStorage.setItem(storageKey+'-versions',JSON.stringify(cloud.ledger)); localStorage.setItem(storageKey+'-thread',JSON.stringify({id:cloud.id,code:cloud.code})); localStorage.setItem(storageKey+'-cloud-revision',String(cloud.revision)); - localStorage.setItem(storageKey+'-cloud-ledger',ledgerText(cloud.ledger)); + localStorage.setItem(storageKey+'-cloud-ledger',ledgerMark(cloud.ledger)); }catch{phase('This phone has no room to open that piece');location.reload();return;} location.reload(); } diff --git a/apple/whistlegraph/Resources/Web/legacy-storage.mjs b/apple/whistlegraph/Resources/Web/legacy-storage.mjs index 9b6727da14..db8b597ac7 100644 --- a/apple/whistlegraph/Resources/Web/legacy-storage.mjs +++ b/apple/whistlegraph/Resources/Web/legacy-storage.mjs @@ -29,3 +29,25 @@ export function migrateLegacyStorage(storage) { clear() { storage.clear(); }, }; } + +// The last cloud ledger used to be kept whole beside the ledger itself, in +// the open piece and inside every parked piece's extras. Each copy becomes +// a mark. On a phone with 5 MB this is the difference between parking a +// piece and "no room" (2026-10-10: 5.3 MB used, half of it copies). +export function compactLedgerCopies(storage, mark) { + let freed = 0; + for (const key of Array.from({length: storage.length}, (_, i) => storage.key(i))) { + if (!key) continue; + try { + if (key.endsWith('-cloud-ledger')) { + const value = storage.getItem(key); + if (value && !value.startsWith('mark:')) { const next = mark(JSON.parse(value)); storage.setItem(key, next); freed += value.length - next.length; } + } else if (key.startsWith('whistlegraph-archive-')) { + const value = storage.getItem(key); const saved = JSON.parse(value); + const copy = saved?.extras?.['-cloud-ledger']; + if (typeof copy === 'string' && !copy.startsWith('mark:')) { saved.extras['-cloud-ledger'] = mark(JSON.parse(copy)); const next = JSON.stringify(saved); storage.setItem(key, next); freed += value.length - next.length; } + } + } catch {} + } + return freed; +} diff --git a/apple/whistlegraph/Resources/Web/ware-engine.mjs b/apple/whistlegraph/Resources/Web/ware-engine.mjs index 083d525fde..7dbe518332 100644 --- a/apple/whistlegraph/Resources/Web/ware-engine.mjs +++ b/apple/whistlegraph/Resources/Web/ware-engine.mjs @@ -1,6 +1,8 @@ import {currentWare,selectWare} from './wares.mjs'; -import {migrateLegacyStorage} from './legacy-storage.mjs'; +import {migrateLegacyStorage,compactLedgerCopies} from './legacy-storage.mjs'; +import {ledgerMark} from '/easel/src/whistlegraph-thread.mjs'; Object.defineProperty(window, 'localStorage', {value:migrateLegacyStorage(window.localStorage), configurable:true}); +try{const freed=compactLedgerCopies(localStorage,ledgerMark);if(freed)console.log('[whistlegraph] compacted ledger copies, freed '+freed.toLocaleString('en-US')+' characters');}catch{} window.whistlegraphSelectWare=id=>{ if(window.whistlegraphIsBusy?.()||window.whistlegraphRecording?.())return false; try { diff --git a/apple/whistlegraph/Tests/ledger-mark.test.mjs b/apple/whistlegraph/Tests/ledger-mark.test.mjs new file mode 100644 index 0000000000..4b334cc55d --- /dev/null +++ b/apple/whistlegraph/Tests/ledger-mark.test.mjs @@ -0,0 +1,36 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {ledgerMark, ledgerText} from '../../../aesel/src/whistlegraph-thread.mjs'; +import {compactLedgerCopies} from '../Resources/Web/legacy-storage.mjs'; + +const ledger = (n, extra = '') => ({format: 1, head: n, versions: Array.from({length: n + 1}, (_, id) => ({id, parent: id ? id - 1 : null, source: 'x'.repeat(2000) + extra + id, request: id ? 'r' + id : null, createdAt: '', layers: 0}))}); +const memory = () => { const m = new Map(); return {get length() { return m.size; }, key: i => [...m.keys()][i] ?? null, getItem: k => m.get(k) ?? null, setItem: (k, v) => m.set(k, String(v)), removeItem: k => m.delete(k), clear: () => m.clear()}; }; + +test('a ledger mark is short, stable, and tells ledgers apart', () => { + const a = ledgerMark(ledger(3)), b = ledgerMark(ledger(3)), c = ledgerMark(ledger(3, 'y')), d = ledgerMark(ledger(4)); + assert.equal(a, b); + assert.notEqual(a, c); assert.notEqual(a, d); + assert.ok(a.startsWith('mark:') && a.length < 40, a); + assert.equal(ledgerMark(ledgerText(ledger(3))), a, 'marking the text equals marking the ledger'); + assert.equal(ledgerMark(a), a, 'a mark marks to itself'); +}); + +test('compaction turns whole cloud-ledger copies into marks, live and parked', () => { + const storage = memory(); + const open = ledger(5), parked = ledger(4); + storage.setItem('whistlegraph-source-cloud-ledger', ledgerText(open)); + storage.setItem('whistlegraph-source-versions', JSON.stringify(open)); + storage.setItem('whistlegraph-archive-abc', JSON.stringify({identity: {id: 'abc', code: 'wgTest'}, ledger: parked, source: 'x', extras: {'-cloud-ledger': ledgerText(parked), '-cloud-revision': '5'}})); + storage.setItem('whistlegraph-archive-def', JSON.stringify({identity: {id: 'def'}, ledger: parked, source: 'x', extras: {}})); + storage.setItem('unrelated', 'keep'); + const before = [...Array(storage.length)].reduce((n, _, i) => n + storage.getItem(storage.key(i)).length, 0); + const freed = compactLedgerCopies(storage, ledgerMark); + const after = [...Array(storage.length)].reduce((n, _, i) => n + storage.getItem(storage.key(i)).length, 0); + assert.equal(storage.getItem('whistlegraph-source-cloud-ledger'), ledgerMark(open)); + assert.equal(JSON.parse(storage.getItem('whistlegraph-archive-abc')).extras['-cloud-ledger'], ledgerMark(parked)); + assert.equal(JSON.parse(storage.getItem('whistlegraph-archive-abc')).extras['-cloud-revision'], '5'); + assert.equal(storage.getItem('unrelated'), 'keep'); + assert.equal(before - after, freed); + assert.ok(freed > 20000, 'freed ' + freed); + assert.equal(compactLedgerCopies(storage, ledgerMark), 0, 'a second pass frees nothing'); +}); -- 2.51.2