A music player that connects to your cloud/distributed storage. diffuse.sh
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611/** * Migration projection engine: applying authored lens documents to records. * * Lens documents (see `~/common/lens-registry.js`) describe transitions between * lexicon NSIDs as schema-independent steps. In keeping with atproto's lexicon * model, the NSID is the schema identity: compatible evolution keeps the same * NSID, while a breaking change is a NEW NSID with an authored lens from the old * NSID to the new one. This engine projects records from one NSID's shape to * another without the panproto WASM engine for the common, reversible cases * (renames, additive fields). Use `migrate()` for that projection. * * Wild/disruptive changes (e.g. dropping a field where the discarded value must * be written back) are handled losslessly via the panproto complement machinery * (see the design doc, open item #1); the pure-step projection here covers the * additive/rename migrations diffuse actually ships. * * @import {LensDocument} from "~/common/lens-registry.js" */
import { collectionSchema, unwrap, wrap } from "./self-describing.js";import { resolve } from "./lens-registry.js";import { put as panprotoPut, parseLexicon } from "./panproto.js";import { migrateLegacyFacet } from "./tiles.js";
/** * Project a collection of records from one NSID's shape to another using a lens * document's steps. * * Supports the DSL document steps diffuse authors: `rename_field`, `add_field`, * `remove_field`. Records are plain objects; unknown steps are left as-is. * * @param {unknown[]} records * @param {LensDocument} lens * @returns {unknown[]} * * @example Renames a field and adds a defaulted field * ```ts * import { project } from "~/common/lens.js"; * * const out = project( * [{ $type: "sh.diffuse.output.facet", id: "a", name: "x", favourite: true }], * { * id: "f", source: "sh.diffuse.output.facet", target: "sh.diffuse.output.facet2", * steps: [ * { rename_field: { old: "favourite", new: "starred" } }, * { add_field: { parent: "sh.diffuse.output.facet2:body", name: "description", kind: "string" } }, * ], * }, * ); * const rec = out[0] as Record<string, unknown>; * if (!("starred" in rec)) throw new Error("expected renamed field"); * if ("favourite" in rec) throw new Error("old field should be gone"); * if (!("description" in rec)) throw new Error("expected added field"); * ``` * * @example Remove a field * ```ts * import { project } from "~/common/lens.js"; * * const out = project([{ id: "a", dropped: true }], { * id: "f", source: "s", target: "t", * steps: [{ remove_field: { name: "dropped" } }], * }); * if ("dropped" in (out[0] as Record<string, unknown>)) throw new Error("expected removed"); * ``` */export function project(records, lens) { return records.map((record) => { if (typeof record !== "object" || record === null || Array.isArray(record)) { return record; } /** @type {Record<string, unknown>} */ const rec = { .../** @type {Record<string, unknown>} */ (record) };
for (const step of lens.steps) { const s = /** @type {any} */ (step); if (s?.rename_field) { const oldName = /** @type {string} */ (s.rename_field.old); const newName = /** @type {string} */ (s.rename_field.new); if (oldName in rec) { rec[newName] = rec[oldName]; delete rec[oldName]; } } else if (s?.remove_field) { const name = /** @type {string} */ (s.remove_field.name); delete rec[name]; } else if (s?.add_field) { const name = /** @type {string} */ (s.add_field.name); if (!(name in rec)) rec[name] = defaultValue(/** @type {string} */(s.add_field.kind)); } }
return rec; });}
/** * Migrate a stored envelope to the current lexicon NSID if it is stale, returning * the migrated data and the updated `$schemaHistory`. When the stored schema * already matches, it is returned unchanged. * * @template {unknown[]} T * @template {unknown} L * @param {{ data: T; envelope: import("./self-describing.js").SelfDescribing<T, L> | null }} stored * @param {string} current - The current lexicon NSID * @param {import("./lens-registry.js").resolve} resolveLens * @returns {{ data: T; history: import("./self-describing.js").HistoryEntry<LensDocument | null>[] }} * * @example Migrates a stale envelope (old NSID) to the current NSID * ```ts * import { wrap } from "~/common/self-describing.js"; * import { register, resolve } from "~/common/lens-registry.js"; * import { migrate } from "~/common/lens.js"; * * register({ * id: "f-old-new", source: "sh.diffuse.output.facet", target: "sh.diffuse.output.facet2", * steps: [{ rename_field: { old: "favourite", new: "starred" } }], * }); * const envelope = wrap([{ id: "a", favourite: true }], { schema: "sh.diffuse.output.facet" }); * const out = migrate( * { data: envelope.data, envelope }, * "sh.diffuse.output.facet2", * resolve, * ); * if (out.history.length !== 1) throw new Error("expected one history entry"); * if (!("starred" in (out.data[0] as object))) throw new Error("expected migrated record"); * ``` */export function migrate(stored, current, resolveLens) { const env = stored.envelope; /** @type {import("./self-describing.js").HistoryEntry<LensDocument | null>[]} */ const storedHistory = /** @type {any} */ (env?.$schemaHistory ?? []); if (!env || env.$schema === current) { return { data: stored.data, history: storedHistory }; }
const lens = resolveLens(env.$schema, current, { history: storedHistory }); if (!lens) { // No lens available for the transition; leave data as-is rather than guess. return { data: stored.data, history: storedHistory }; }
const projected = /** @type {T} */ (project(envelopeToArray(env), lens)); return { data: projected, history: [ ...storedHistory, { from: env.$schema, to: current, lens, complement: null, }, ], };}
/** * Read a stored value for a collection, migrating it to the collection's current * lexicon NSID if it is stale. Returns the data to expose and the (possibly * updated) envelope. * * @template {unknown[]} T * @param {unknown} value * @param {import("./self-describing.js").CollectionName} name * @param {import("./lens-registry.js").resolve} resolveLens * @returns {{ data: T; envelope: import("./self-describing.js").SelfDescribing<T, LensDocument | null> | null }} * * @example Reads a stale envelope and migrates it to the current NSID * ```ts * import { wrap } from "~/common/self-describing.js"; * import { register, resolve } from "~/common/lens-registry.js"; * import { migrateEnvelope } from "~/common/lens.js"; * * register({ * id: "f-old-current", source: "sh.diffuse.output.facetOld", target: "sh.diffuse.output.facet", * steps: [{ rename_field: { old: "favourite", new: "starred" } }], * }); * // A payload stored under an OLDER lexicon NSID; the current collection NSID is * // sh.diffuse.output.facet, so this migrates. * const envelope = wrap([{ id: "a", favourite: true }], { schema: "sh.diffuse.output.facetOld" }); * const out = migrateEnvelope(envelope, "facets", resolve); * if (!("starred" in (out.data[0] as object))) throw new Error("expected migrated record"); * if (!out.envelope || out.envelope.$schema !== "sh.diffuse.output.facet") throw new Error("expected migrated envelope NSID"); * ``` * * @example Lifts a legacy html facet to the tile shape on read * ```ts * import { wrap } from "~/common/self-describing.js"; * import { resolve } from "~/common/lens-registry.js"; * import { migrateEnvelope } from "~/common/lens.js"; * * // A facet stored before the tile schema carries a plain `html` string and no `resources`. * const envelope = wrap([{ id: "a", name: "x", html: "<p>hi</p>" }], { schema: "sh.diffuse.output.facet" }); * const out = migrateEnvelope(envelope, "facets", resolve); * const rec = out.data[0] as Record<string, unknown>; * if ("html" in rec) throw new Error("legacy html should be dropped"); * if (!rec.resources || !rec.blocks) throw new Error("should be lifted into a tile"); * if (!out.envelope) throw new Error("expected envelope"); * const envRec = out.envelope.data[0] as Record<string, unknown>; * if (envRec.resources === undefined) throw new Error("envelope data should be lifted"); * ``` */export function migrateEnvelope(value, name, resolveLens) { const current = collectionSchema(name); const { data, envelope } = unwrap(value, { $schema: current });
/** @type {import("./self-describing.js").SelfDescribing<T, LensDocument | null> | null} */ const env = /** @type {any} */ (envelope);
if (!env || env.$schema === current) { const dataLifted = (name === "facets" ? liftLegacyFacets(data) : data); return { data: /** @type {T} */ (dataLifted), envelope: env ? { ...env, data: /** @type {T} */ (dataLifted) } : env, }; }
const migrated = migrate({ data: /** @type {T} */ (data), envelope: env }, current, resolveLens); const dataLifted = name === "facets" ? liftLegacyFacets(migrated.data) : migrated.data;
/** @type {import("./self-describing.js").SelfDescribing<T, LensDocument | null>} */ const migratedEnvelope = { ...env, $schema: current, $schemaHistory: migrated.history, data: /** @type {T} */ (dataLifted), }; return { data: /** @type {T} */ (dataLifted), envelope: migratedEnvelope };}
/** * Lifts any legacy `facets` records (raw `html`, pre-tile shape) to the current * tile shape synchronously, leaving already-tile records untouched. Non-array * stored data (a single non-collection value) is returned unchanged. * * @param {unknown} data * @returns {unknown} */function liftLegacyFacets(data) { if (!Array.isArray(data)) return data; return data.map((record) => record && typeof record === "object" && !Array.isArray(record) ? migrateLegacyFacet(/** @type {Record<string, unknown>} */ (record)) : record, );}
/** * Wrap a collection into a self-describing envelope stamped with the collection's * current NSID, ready to be serialized by an encoder. * * @template {unknown[]} T * @param {T} data * @param {import("./self-describing.js").CollectionName} name * @returns {import("./self-describing.js").SelfDescribing<T, LensDocument | null>} * * @example Wraps records for a collection * ```ts * import { encodeCollection } from "~/common/lens.js"; * * const envelope = encodeCollection([{ id: "a" }], "facets"); * if (envelope.$schema !== "sh.diffuse.output.facet") throw new Error("expected facet NSID"); * if (envelope.data[0].id !== "a") throw new Error("expected data"); * ``` */export function encodeCollection(data, name) { return wrap(data, { schema: collectionSchema(name) });}
/** * Decode a stored value (envelope or legacy) into a collection, migrating it to * the collection's current NSID if stale. Returns `null` when `value` is `null` * or `undefined`, so encoders can map that to an empty collection. * * @template {unknown[]} T * @param {unknown} value * @param {import("./self-describing.js").CollectionName} name * @returns {T | null} */export function decodeCollection(value, name) { if (value === null || value === undefined) return null; const { data } = migrateEnvelope(value, name, resolve); return /** @type {T} */ (data);}
/** * Encode a collection as self-describing JSON, either as a string or as bytes. * * @param {unknown[]} data * @param {import("./self-describing.js").CollectionName} name * @param {boolean} [asBytes] * @returns {string | Uint8Array} * * @example Encodes a collection as a JSON string * ```ts * import { encodeJsonCollection } from "~/common/lens.js"; * * const out = encodeJsonCollection([{ id: "a" }], "tracks") as string; * const parsed = JSON.parse(out) as { $schema: string }; * if (parsed.$schema !== "sh.diffuse.output.track") throw new Error("expected track NSID"); * ``` * * @example Encodes a collection as JSON bytes * ```ts * import { encodeJsonCollection } from "~/common/lens.js"; * * const out = encodeJsonCollection([{ id: "a" }], "tracks", true); * if (!(out instanceof Uint8Array)) throw new Error("expected bytes"); * ``` */export function encodeJsonCollection(data, name, asBytes = false) { const json = JSON.stringify(encodeCollection(data, name)); return asBytes ? new TextEncoder().encode(json) : json;}
/** * Decode a JSON-encoded collection (string, bytes, or an already-parsed object — * an envelope or legacy array), migrating stale payloads. `undefined`/`null` * yields an empty collection. * * @template {unknown[]} T * @param {Uint8Array | string | object | null | undefined} raw * @param {import("./self-describing.js").CollectionName} name * @returns {T} * * @example Round-trips through encodeJsonCollection * ```ts * import { encodeJsonCollection, decodeJsonCollection } from "~/common/lens.js"; * * const bytes = encodeJsonCollection([{ id: "a" }], "tracks", true) as Uint8Array; * const out = decodeJsonCollection(bytes, "tracks") as Array<{ id: string }>; * if (out[0].id !== "a") throw new Error("expected record"); * ``` * * @example Empty for undefined input * ```ts * import { decodeJsonCollection } from "~/common/lens.js"; * * if (decodeJsonCollection(undefined, "tracks").length !== 0) throw new Error("expected empty"); * ``` */export function decodeJsonCollection(raw, name) { try { let parsed; if (raw instanceof Uint8Array) { parsed = JSON.parse(new TextDecoder().decode(raw)); } else if (raw === undefined || raw === null) { return /** @type {T} */ (/** @type {unknown} */ ([])); } else if (typeof raw === "string") { parsed = JSON.parse(raw); } else { // Already-parsed value (e.g. a stored envelope object). parsed = raw; } return normalizeCollection(decodeCollection(parsed, name)); } catch (err) { console.error(err); return /** @type {T} */ (/** @type {unknown} */ ([])); }}
/** * Guarantee a collection is always returned as an array. * * @template {unknown[]} T * @param {T | null | undefined | unknown} value * @returns {T} */function normalizeCollection(value) { if (Array.isArray(value)) return /** @type {T} */ (value); return /** @type {T} */ (/** @type {unknown} */ ([]));}
/** * Write back an edited record into the collection's current shape, losslessly. * * This is the panproto write-back path and the ONLY place the migration flow * loads `@panproto/core` (its WASM). It is never called on a plain read — it runs * only when an app edits a record in an older shape and needs to store it back in * the current shape, using the lens + complement recorded in `$schemaHistory`. * * When a complement is available (captured when the data was migrated forward), it * is used to reconstruct the discarded fields so the old app's edit is preserved * losslessly; otherwise the forward projection is pure-JS and panproto is not * loaded. * * @param {unknown} editedRecord - the record as the (older) app edited it * @param {{ lens: LensDocument; toLexicon?: object; complement?: Uint8Array | string | null }} opts * @returns {Promise<unknown>} the record in the current shape, ready to store * * @example Runs the write-back for an empty lens (loads panproto lazily) * ```ts * import { writeBack } from "~/common/lens.js"; * * const out = await writeBack({ $type: "sh.diffuse.output.facet", id: "a", name: "x", favourite: true }, { * lens: { id: "l", source: "sh.diffuse.output.facet", target: "sh.diffuse.output.facet2", steps: [] }, * toLexicon: { * lexicon: 1, id: "sh.diffuse.output.facet2", * defs: { main: { type: "record", record: { type: "object", properties: { * $type: { type: "string" }, id: { type: "string" }, name: { type: "string" }, * } } } }, * }, * }); * if (typeof out !== "object") throw new Error("expected a record back"); * ``` */export async function writeBack(editedRecord, { lens, toLexicon, complement }) { // Only load panproto (WASM) when a complement is actually used for lossless // write-back and we have the target lexicon to instantiate it; otherwise the // common additive/rename path stays pure-JS via `project`. if (!complement || !toLexicon) { return project([editedRecord], lens)[0]; }
const schema = await parseLexicon(toLexicon); const comp = typeof complement === "string" ? base64ToBytes(complement) : /** @type {Uint8Array} */ (complement); return panprotoPut(lens, schema, editedRecord, comp);}
/** * Write a collection back into its stored (possibly newer) shape on save. * * Called by encoders before wrapping a save into the envelope. If the stored * envelope's `$schemaHistory` ends at a non-null `complement` for the transition * to the current NSID, each record is written back (loading panproto lazily to * preserve discarded fields); otherwise the records pass through unchanged and no * WASM is loaded. * * @template {unknown[]} T * @param {T} items * @param {import("./self-describing.js").CollectionName} name * @param {import("./self-describing.js").SelfDescribing<T, LensDocument | null> | null} storedEnvelope * @param {object} [toLexicon] * @returns {Promise<T>} * * @example Passes records through when the stored envelope already matches (no panproto) * ```ts * import { writeBackCollection, encodeCollection } from "~/common/lens.js"; * * const items = [{ id: "a" }]; * const envelope = encodeCollection(items, "tracks"); * const out = await writeBackCollection(items, "tracks", envelope); * if (out[0].id !== "a") throw new Error("expected unchanged records"); * ``` * * @example Passes records through when no lexicon is available (no panproto) * ```ts * import { writeBackCollection } from "~/common/lens.js"; * import { wrap } from "~/common/self-describing.js"; * * const items = [{ id: "a" }]; * const envelope = wrap([{ id: "a" }], { schema: "sh.diffuse.output.track" }) as { * $schema: string; $schemaHistory: any[]; data: unknown[]; * }; * // stored under an older NSID with a complement-carrying history, but no lexicon: * const out = await writeBackCollection(items, "tracks", { * ...envelope, $schema: "sh.diffuse.output.trackOld", * $schemaHistory: [{ from: "sh.diffuse.output.trackOld", to: "sh.diffuse.output.track", lens: { id: "l", source: "s", target: "t", steps: [] }, complement: new Uint8Array(1) }], * }) as Array<{ id: string }>; * if (out[0].id !== "a") throw new Error("expected unchanged records without a lexicon"); * ``` */export async function writeBackCollection(items, name, storedEnvelope, toLexicon) { if (!storedEnvelope || storedEnvelope.$schema === collectionSchema(name)) { // Nothing stale to write back — the stored envelope is already this build's // shape, or there is no history. Pass through (no panproto load). return items; }
const entry = storedEnvelope.$schemaHistory[storedEnvelope.$schemaHistory.length - 1]; const lens = entry?.lens; const complement = entry?.complement; if (!lens || !complement || !toLexicon) { // No complement recorded, or no lexicon to instantiate against — nothing // panproto could losslessly write back; keep the records as-is. return items; }
const written = await Promise.all( /** @type {unknown[]} */ (items).map((item) => writeBack(item, { lens, toLexicon, complement }), ), ); return /** @type {T} */ (written);}
/** * Reconstruct the stored self-describing envelope of a collection from its raw * encoded value (a JSON string/bytes, or a stored envelope object). Returns * `null` when the value is absent/legacy (no envelope). * * @template {unknown[]} T * @param {Uint8Array | string | unknown} raw - the raw stored payload for the collection * @param {import("./self-describing.js").CollectionName} name * @returns {import("./self-describing.js").SelfDescribing<T, LensDocument | null> | null} * * @example Reads the envelope out of a stored JSON string * ```ts * import { readStoredEnvelope, encodeJsonCollection } from "~/common/lens.js"; * * const stored = encodeJsonCollection([{ id: "a" }], "tracks", true); * const env = readStoredEnvelope(stored, "tracks"); * if (!env || env.$schema !== "sh.diffuse.output.track") throw new Error("expected stored envelope"); * ``` * * @example Returns null for absent/legacy (bare array) data * ```ts * import { readStoredEnvelope } from "~/common/lens.js"; * * if (readStoredEnvelope(null, "tracks") !== null) throw new Error("expected null for absent"); * if (readStoredEnvelope([{ id: "a" }], "tracks") !== null) throw new Error("expected null for legacy array"); * ``` */export function readStoredEnvelope(raw, name) { if (raw === null || raw === undefined) return null; let parsed = raw; if (raw instanceof Uint8Array) { parsed = JSON.parse(new TextDecoder().decode(raw)); } else if (typeof raw === "string") { parsed = JSON.parse(raw); } const { envelope } = unwrap(parsed, { $schema: collectionSchema(name) }); return /** @type {import("./self-describing.js").SelfDescribing<T, LensDocument | null> | null} */ ( /** @type {any} */ (envelope) );}
/** * The write path for a JSON-encoded collection: write back any stale records to * the stored shape (guarded no-op), then encode. Used by the JSON encoders' save * so the write-back path is wired on save without loading panproto unless a * cross-NSID complement + lexicon make it necessary. * * @param {unknown[]} items * @param {import("./self-describing.js").CollectionName} name * @param {Uint8Array | string | null | undefined} stored - the raw stored payload * @param {boolean} [asBytes] * @returns {Promise<string | Uint8Array>} * * @example Encodes a collection through the wired save path (guarded no-op) * ```ts * import { saveJsonCollection, decodeJsonCollection } from "~/common/lens.js"; * * const out = await saveJsonCollection([{ id: "a" }], "tracks", null) as string; * const back = decodeJsonCollection(out, "tracks") as Array<{ id: string }>; * if (back[0].id !== "a") throw new Error("expected encoded records"); * ``` * * @example Produces bytes when asBytes is set * ```ts * import { saveJsonCollection } from "~/common/lens.js"; * * const out = await saveJsonCollection([{ id: "a" }], "tracks", null, true); * if (!(out instanceof Uint8Array)) throw new Error("expected bytes"); * ``` */export async function saveJsonCollection(items, name, stored, asBytes = false) { const storedEnvelope = readStoredEnvelope(stored, name); const written = await writeBackCollection(items, name, storedEnvelope, undefined); return encodeJsonCollection(written, name, asBytes);}
/** * @param {string} base64 * @returns {Uint8Array} */function base64ToBytes(base64) { const bin = atob(base64); const bytes = new Uint8Array(bin.length); for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); return bytes;}
/** * @template {unknown[]} T * @template {unknown} L * @param {import("./self-describing.js").SelfDescribing<T, L>} envelope */function envelopeToArray(envelope) { return /** @type {unknown[]} */ (envelope.data);}
/** * A sensible default for an `add_field` kind. * * @param {string} kind * @returns {unknown} */function defaultValue(kind) { switch (kind) { case "boolean": return false; case "integer": case "float": case "number": return 0; case "array": return []; case "object": return {}; case "null": return null; default: return ""; }}