From 25b6414a67241bb0d3234924ed8e2cc1ac3ff93b Mon Sep 17 00:00:00 2001 From: Steven Vandevelde Date: Thu, 26 Jun 2025 14:45:18 +0200 Subject: [PATCH] refactor: more web workers + bit of clean up --- src/pages/configurator/input/_applet.astro | 40 +++---- .../orchestrator/queue-audio/_applet.astro | 4 +- .../orchestrator/queue-tracks/_applet.astro | 1 - src/pages/processor/search/_applet.astro | 107 ++---------------- .../pilot/audio/_applet.astro | 0 .../pilot/audio/_manifest.json | 0 .../pilot/audio/index.astro | 0 .../pilot/audio/types.d.ts | 0 src/pages/theme/pilot/index.astro | 2 +- src/scripts/configurator/input/constants.ts | 5 + src/scripts/configurator/input/types.d.ts | 3 + src/scripts/configurator/input/worker.ts | 25 ++++ src/scripts/processor/search/constants.ts | 17 +++ .../processor/search/types.d.ts | 0 src/scripts/processor/search/worker.ts | 99 ++++++++++++++++ src/scripts/theme/pilot/index.ts | 4 +- 16 files changed, 178 insertions(+), 129 deletions(-) rename src/pages/{constituent => theme}/pilot/audio/_applet.astro (100%) rename src/pages/{constituent => theme}/pilot/audio/_manifest.json (100%) rename src/pages/{constituent => theme}/pilot/audio/index.astro (100%) rename src/pages/{constituent => theme}/pilot/audio/types.d.ts (100%) create mode 100644 src/scripts/configurator/input/constants.ts create mode 100644 src/scripts/configurator/input/types.d.ts create mode 100644 src/scripts/configurator/input/worker.ts create mode 100644 src/scripts/processor/search/constants.ts rename src/{pages => scripts}/processor/search/types.d.ts (100%) create mode 100644 src/scripts/processor/search/worker.ts diff --git a/src/pages/configurator/input/_applet.astro b/src/pages/configurator/input/_applet.astro index 7663149..8a714a9 100644 --- a/src/pages/configurator/input/_applet.astro +++ b/src/pages/configurator/input/_applet.astro @@ -35,26 +35,28 @@ import type { Applet } from "@web-applets/sdk"; import * as URI from "uri-js"; - import type { Consult, ConsultGrouping, GroupConsult, Track } from "@applets/core/types.d.ts"; + import type { Consult, GroupConsult, Track } from "@applets/core/types.d.ts"; + import type { Actions } from "@scripts/configurator/input/worker"; + import type { Scheme } from "@scripts/configurator/input/types"; + import { CONNECTIONS } from "@scripts/configurator/input/constants"; import { applet, register } from "@scripts/applet/common"; - import { groupTracksPerScheme } from "@scripts/common"; + import { endpoint, groupTracksPerScheme } from "@scripts/common"; //////////////////////////////////////////// // SETUP //////////////////////////////////////////// + const worker = endpoint( + new SharedWorker(new URL("../../../scripts/configurator/input/worker", import.meta.url), { + type: "module", + }).port, + ); + + // Register applet const context = register(); //////////////////////////////////////////// // CONNECTIONS ////////////////////////////////////////////// Applet connections - const CONNECTIONS = { - "file+local": "/input/native-fs", - opensubsonic: "/input/opensubsonic", - s3: "/input/s3", - }; - - type Scheme = keyof typeof CONNECTIONS; - const connections: Record | undefined> = { "file+local": undefined, opensubsonic: undefined, @@ -89,7 +91,7 @@ }; const contextualize = async (tracks: Track[]) => { - const groups = await groupTracks(tracks); + const groups = await worker.call.groupTracks(tracks); const promises = Object.entries(groups).map( async ([scheme, tracksGroup]: [string, Track[]]) => { if (!isSupportedScheme(scheme) || tracksGroup.length === 0) return; @@ -122,7 +124,7 @@ }; const list = async (cachedTracks: Track[] = []) => { - const groups = await groupTracks(cachedTracks); + const groups = await worker.call.groupTracks(cachedTracks); const promises = Object.entries(groups).map( async ([scheme, cachedTracksGroup]: [string, Track[]]) => { @@ -157,18 +159,4 @@ context.setActionHandler("groupConsult", groupConsult); context.setActionHandler("list", list); context.setActionHandler("resolve", resolve); - - //////////////////////////////////////////// - // 🛠️ - //////////////////////////////////////////// - async function groupTracks(tracks: Track[]) { - return groupTracksPerScheme( - tracks, - Object.fromEntries( - Object.entries(CONNECTIONS).map(([k, _v]) => { - return [k, []]; - }), - ), - ); - } diff --git a/src/pages/orchestrator/queue-audio/_applet.astro b/src/pages/orchestrator/queue-audio/_applet.astro index 8a3d85d..3521ad8 100644 --- a/src/pages/orchestrator/queue-audio/_applet.astro +++ b/src/pages/orchestrator/queue-audio/_applet.astro @@ -1,5 +1,4 @@ diff --git a/src/pages/constituent/pilot/audio/_applet.astro b/src/pages/theme/pilot/audio/_applet.astro similarity index 100% rename from src/pages/constituent/pilot/audio/_applet.astro rename to src/pages/theme/pilot/audio/_applet.astro diff --git a/src/pages/constituent/pilot/audio/_manifest.json b/src/pages/theme/pilot/audio/_manifest.json similarity index 100% rename from src/pages/constituent/pilot/audio/_manifest.json rename to src/pages/theme/pilot/audio/_manifest.json diff --git a/src/pages/constituent/pilot/audio/index.astro b/src/pages/theme/pilot/audio/index.astro similarity index 100% rename from src/pages/constituent/pilot/audio/index.astro rename to src/pages/theme/pilot/audio/index.astro diff --git a/src/pages/constituent/pilot/audio/types.d.ts b/src/pages/theme/pilot/audio/types.d.ts similarity index 100% rename from src/pages/constituent/pilot/audio/types.d.ts rename to src/pages/theme/pilot/audio/types.d.ts diff --git a/src/pages/theme/pilot/index.astro b/src/pages/theme/pilot/index.astro index 6909697..6b9bf22 100644 --- a/src/pages/theme/pilot/index.astro +++ b/src/pages/theme/pilot/index.astro @@ -10,5 +10,5 @@ import "@styles/theme/pilot/index.css";
- + diff --git a/src/scripts/configurator/input/constants.ts b/src/scripts/configurator/input/constants.ts new file mode 100644 index 0000000..7bf9fcd --- /dev/null +++ b/src/scripts/configurator/input/constants.ts @@ -0,0 +1,5 @@ +export const CONNECTIONS = { + "file+local": "/input/native-fs", + opensubsonic: "/input/opensubsonic", + s3: "/input/s3", +}; diff --git a/src/scripts/configurator/input/types.d.ts b/src/scripts/configurator/input/types.d.ts new file mode 100644 index 0000000..2a7aeee --- /dev/null +++ b/src/scripts/configurator/input/types.d.ts @@ -0,0 +1,3 @@ +import type { CONNECTIONS } from "./constants"; + +export type Scheme = keyof typeof CONNECTIONS; diff --git a/src/scripts/configurator/input/worker.ts b/src/scripts/configurator/input/worker.ts new file mode 100644 index 0000000..cfcd0f8 --- /dev/null +++ b/src/scripts/configurator/input/worker.ts @@ -0,0 +1,25 @@ +import { expose, groupTracksPerScheme } from "@scripts/common"; +import { CONNECTIONS } from "./constants"; +import type { Track } from "@applets/core/types"; + +//////////////////////////////////////////// +// ACTIONS +//////////////////////////////////////////// +const actions = expose({ + groupTracks, +}); + +export type Actions = typeof actions; + +// Actions + +function groupTracks(tracks: Track[]) { + return groupTracksPerScheme( + tracks, + Object.fromEntries( + Object.entries(CONNECTIONS).map(([k, _v]) => { + return [k, []]; + }), + ), + ); +} diff --git a/src/scripts/processor/search/constants.ts b/src/scripts/processor/search/constants.ts new file mode 100644 index 0000000..439df87 --- /dev/null +++ b/src/scripts/processor/search/constants.ts @@ -0,0 +1,17 @@ +export const SCHEMA = { + id: "string" as const, + kind: "string" as const, + tags: { + album: "string" as const, + artist: "string" as const, + genre: "string" as const, + title: "string" as const, + year: "number" as const, + }, + + // TODO: + // isFavorite: "boolean" as const, + // inPlaylists: [ ... ], + + embeddings: "vector[512]" as const, +}; diff --git a/src/pages/processor/search/types.d.ts b/src/scripts/processor/search/types.d.ts similarity index 100% rename from src/pages/processor/search/types.d.ts rename to src/scripts/processor/search/types.d.ts diff --git a/src/scripts/processor/search/worker.ts b/src/scripts/processor/search/worker.ts new file mode 100644 index 0000000..7c8ad8f --- /dev/null +++ b/src/scripts/processor/search/worker.ts @@ -0,0 +1,99 @@ +import * as Orama from "@orama/orama"; +// import { pluginQPS } from "@orama/plugin-qps"; + +import type { Track } from "@applets/core/types"; +import { expose } from "@scripts/common"; +import { SCHEMA } from "./constants"; +import type { State } from "./types"; + +//////////////////////////////////////////// +// SETUP +//////////////////////////////////////////// + +let state: State = { + inserted: new Set(), +}; + +// TODO: Generate embeddings plugin +// +// I tried this and getting some bundler/vite errors about a default import. +// +// const plugin = await pluginEmbeddings({ +// embeddings: { +// defaultProperty: "embeddings", +// onInsert: { +// generate: true, +// // Properties to use for generating embeddings at insert time. +// // These properties will be concatenated and used to generate embeddings. +// properties: ["album", "artist", "title", "year", "kind", "genre"], +// // verbose: true, +// }, +// }, +// }); +// +// TODO: +// +// Does not work either. +// `TypeError: a is undefined` +// +// pluginQPS() + +const PLUGINS: Orama.OramaPlugin[] = []; + +// Search through tracks +const db = Orama.create({ + schema: SCHEMA, + plugins: PLUGINS, + + // components: { + // TODO: + // https://docs.orama.com/open-source/usage/insert#remote-document-storing + // documentStore: { ... } + // }, +}); + +//////////////////////////////////////////// +// ACTIONS +//////////////////////////////////////////// +const actions = expose({ + search, + supply, +}); + +export type Actions = typeof actions; + +// Actions + +async function search(term: string): Promise { + const results = await Orama.search(db, { + // mode: "hybrid", + term, + }); + + return results.hits.map((hit) => hit.document as unknown as Track); +} + +async function supply(tracks: Track[]) { + // TODO: Generate a hash based on the track itself, + // so we can detect changes to tags or other data. + + const ids = []; + const tracksMap: Record = {}; + + tracks.forEach((track) => { + ids.push(track.id); + tracksMap[track.id] = track; + }); + + const currentSet = state.inserted; + const newSet = new Set(tracks.map((t) => t.id)); + + const removedIds = currentSet.difference(newSet); + const newIds = newSet.difference(currentSet); + const newTracks = Array.from(newIds).map((id) => tracksMap[id]); + + await Orama.removeMultiple(db, Array.from(removedIds)); + await Orama.insertMultiple(db, newTracks); + + state.inserted = newSet; +} diff --git a/src/scripts/theme/pilot/index.ts b/src/scripts/theme/pilot/index.ts index 1f1bc26..e9df3d5 100644 --- a/src/scripts/theme/pilot/index.ts +++ b/src/scripts/theme/pilot/index.ts @@ -6,7 +6,7 @@ import { applet, reactive } from "@scripts/applet/common"; import type * as AudioEngine from "@applets/engine/audio/types.d.ts"; import type * as QueueEngine from "@applets/engine/queue/types.d.ts"; -import type * as AudioUI from "@applets/constituent/pilot/audio/types"; +import type * as AudioUI from "@applets/theme/pilot/audio/types"; const engine = { audio: await applet("/engine/audio"), @@ -20,7 +20,7 @@ const _orchestrator = { }; const ui = { - audio: await applet("/constituent/pilot/audio/", { setHeight: true }), + audio: await applet("/theme/pilot/audio/", { setHeight: true }), }; //////////////////////////////////////////// -- 2.51.2