From 002824c748bf75949cece2f4450e8dc7ed84ff03 Mon Sep 17 00:00:00 2001 From: Steven Vandevelde Date: Wed, 28 May 2025 00:39:23 +0200 Subject: [PATCH] feat: add s3 input --- deno.lock | 5 +- package-lock.json | 44 +-- package.json | 1 + src/pages/configurator/input/_applet.astro | 18 +- src/pages/configurator/input/_manifest.json | 11 +- src/pages/index.astro | 2 + src/pages/input/native-fs/_applet.astro | 7 +- src/pages/input/native-fs/_manifest.json | 11 +- src/pages/input/s3/_applet.astro | 362 ++++++++++++++++++ src/pages/input/s3/_manifest.json | 55 +++ src/pages/input/s3/index.astro | 9 + .../orchestrator/input-cache/_applet.astro | 17 +- .../output-management/_applet.astro | 2 +- .../orchestrator/single-queue/_applet.astro | 8 +- .../processor/metadata-fetcher/_applet.astro | 9 +- src/scripts/theme.ts | 11 +- src/scripts/themes/webamp/index.ts | 10 +- 17 files changed, 523 insertions(+), 59 deletions(-) create mode 100644 src/pages/input/s3/_applet.astro create mode 100644 src/pages/input/s3/_manifest.json create mode 100644 src/pages/input/s3/index.astro diff --git a/deno.lock b/deno.lock index f1dd6a5..746e906 100644 --- a/deno.lock +++ b/deno.lock @@ -21,8 +21,8 @@ ], "packageJson": { "dependencies": [ - "npm:98.css@~0.1.21", - "npm:@atcute/cid@^2.2.2", + "npm:@jsr/bradenmacdonald__s3-lite-client@0.9", + "npm:@jsr/std__media-types@^1.1.0", "npm:@picocss/pico@^2.1.1", "npm:@types/throttle-debounce@^5.0.2", "npm:astro-purgecss@^5.2.2", @@ -32,6 +32,7 @@ "npm:idb-keyval@^6.2.1", "npm:music-metadata@^11.2.3", "npm:native-file-system-adapter@^3.0.1", + "npm:node-s3-url-encode@^0.0.4", "npm:purgecss@^7.0.2", "npm:query-string@^9.1.2", "npm:sass@^1.87.0", diff --git a/package-lock.json b/package-lock.json index dd5f11a..1f16ad4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,15 +5,15 @@ "packages": { "": { "dependencies": { - "@atcute/cid": "^2.2.2", "@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@^0.9.0", "@picocss/pico": "^2.1.1", + "@std/media-types": "npm:@jsr/std__media-types@^1.1.0", "@web-applets/sdk": "https://gitpkg.vercel.app/unternet-co/web-applets/sdk?tokono.ma/experiment&scripts.postinstall=npm%20i%20%40types%2Fnode%20%26%26%20npx%20tsc", - "98.css": "^0.1.21", "iconoir": "^7.11.0", "idb-keyval": "^6.2.1", "music-metadata": "^11.2.3", "native-file-system-adapter": "^3.0.1", + "node-s3-url-encode": "^0.0.4", "query-string": "^9.1.2", "spellcaster": "^6.0.0", "throttle-debounce": "^5.0.2", @@ -107,28 +107,6 @@ "node": "^18.17.1 || ^20.3.0 || >=22.0.0" } }, - "node_modules/@atcute/cid": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/@atcute/cid/-/cid-2.2.2.tgz", - "integrity": "sha512-deAGMqLAyplt7eIukhkjlsGubvrcMrtXkDKlUYZDo4WUdL7hSjBywtPXf6SbMK+Mjvst7l2+83OqTcY5AuuxtA==", - "dependencies": { - "@atcute/multibase": "^1.1.3", - "@atcute/uint8array": "^1.0.1" - } - }, - "node_modules/@atcute/multibase": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@atcute/multibase/-/multibase-1.1.3.tgz", - "integrity": "sha512-vQQO0tDuQPguBvHdgV3ryn7R8U6beQ50KA/juYm+dCeT/3hOK2stMbX+IaW8JEuwkT5lJsU8wDIOicQT4mB7Ag==", - "dependencies": { - "@atcute/uint8array": "^1.0.1" - } - }, - "node_modules/@atcute/uint8array": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@atcute/uint8array/-/uint8array-1.0.1.tgz", - "integrity": "sha512-AAnlFKyfDRgb9GNZJbhQ6OuMhbmNPirQyapb8KnmcEhxQZ3+tt+4NcwqekEegY4MpNqSTYeeTdyxq0wGZv1JHg==" - }, "node_modules/@babel/helper-string-parser": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", @@ -1710,6 +1688,12 @@ "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", "dev": true }, + "node_modules/@std/media-types": { + "name": "@jsr/std__media-types", + "version": "1.1.0", + "resolved": "https://npm.jsr.io/~/11/@jsr/std__media-types/1.1.0.tgz", + "integrity": "sha512-dHvaxHL7ENWnltgL653uo3KnKFse3ZbopZop2gqsT7yrscx7irZEClu5Cba7gMPPRk4Lg1FbriNcaBViM2RSBw==" + }, "node_modules/@swc/helpers": { "version": "0.5.17", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.17.tgz", @@ -1848,12 +1832,6 @@ "hasInstallScript": true, "license": "MIT" }, - "node_modules/98.css": { - "version": "0.1.21", - "resolved": "https://registry.npmjs.org/98.css/-/98.css-0.1.21.tgz", - "integrity": "sha512-ddk5qtUWyapM0Bzd5jwGExoE5fdSEGrP+F5VbYjyZLf2c9UVmn6w2NPTvCsoD4BWdGsjdLjlkQGhWwWTJcYQJQ==", - "license": "MIT" - }, "node_modules/acorn": { "version": "8.14.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", @@ -5039,6 +5017,12 @@ "integrity": "sha512-0uGYQ1WQL1M5kKvGRXWQ3uZCHtLTO8hln3oBjIusM75WoesZ909uQJs/Hb946i2SS+Gsrhkaa6iAO17jRIv6DQ==", "dev": true }, + "node_modules/node-s3-url-encode": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/node-s3-url-encode/-/node-s3-url-encode-0.0.4.tgz", + "integrity": "sha512-l0IizfnxE1hb9dadzYBpA27syfL9LFkPzCKH6YWrssv2sPLjVuCent67A8GPe4isdj4bEsbgdPWLTcV4gxEg9w==", + "license": "MIT" + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", diff --git a/package.json b/package.json index d2827b6..2f5bf39 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "dependencies": { "@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@^0.9.0", "@picocss/pico": "^2.1.1", + "@std/media-types": "npm:@jsr/std__media-types@^1.1.0", "@web-applets/sdk": "https://gitpkg.vercel.app/unternet-co/web-applets/sdk?tokono.ma/experiment&scripts.postinstall=npm%20i%20%40types%2Fnode%20%26%26%20npx%20tsc", "iconoir": "^7.11.0", "idb-keyval": "^6.2.1", diff --git a/src/pages/configurator/input/_applet.astro b/src/pages/configurator/input/_applet.astro index 9a7e582..b62283e 100644 --- a/src/pages/configurator/input/_applet.astro +++ b/src/pages/configurator/input/_applet.astro @@ -33,7 +33,7 @@ import { applets } from "@web-applets/sdk"; import type { Track } from "@applets/core/types.d.ts"; - import { applet, waitUntilAppletIsReady } from "@scripts/theme"; + import { applet } from "@scripts/theme"; //////////////////////////////////////////// // SETUP @@ -51,6 +51,7 @@ // Applet connections const input = { nativeFs: await applet("../../input/native-fs", { container }), + s3: await applet("../../input/s3", { container }), }; //////////////////////////////////////////// @@ -65,6 +66,7 @@ }, { [input.nativeFs.manifest.input_properties.scheme]: [], + [input.s3.manifest.input_properties.scheme]: [], }, ); @@ -76,6 +78,11 @@ timeoutDuration: 60000 * 60 * 24, }); + case input.s3.manifest.input_properties.scheme: + return await input.s3.sendAction("list", cachedTracksGroup, { + timeoutDuration: 60000 * 60 * 24, + }); + default: return cachedTracks; } @@ -88,12 +95,15 @@ return tracks; }; - const resolve = async (fileUri: string) => { - const scheme = fileUri.split(":", 1)[0]; + const resolve = async (args: { method: string; uri: string }) => { + const scheme = args.uri.split(":", 1)[0]; switch (scheme) { case input.nativeFs.manifest.input_properties.scheme: - return await input.nativeFs.sendAction("resolve", fileUri); + return await input.nativeFs.sendAction("resolve", args); + + case input.s3.manifest.input_properties.scheme: + return await input.s3.sendAction("resolve", args); default: return undefined; diff --git a/src/pages/configurator/input/_manifest.json b/src/pages/configurator/input/_manifest.json index e9e1e4a..ce72dae 100644 --- a/src/pages/configurator/input/_manifest.json +++ b/src/pages/configurator/input/_manifest.json @@ -18,8 +18,15 @@ "title": "Resolve", "description": "Potentially translates a track uri with a matching scheme into a URL pointing at the audio bytes. If it can be resolved that is, otherwise you'll get `undefined`.", "params_schema": { - "type": "string", - "description": "The uri to resolve" + "type": "object", + "properties": { + "method": { + "type": "string", + "description": "The HTTP method that is going to be used on the resolved URI." + }, + "uri": { "type": "string", "description": "The URI to resolve." } + }, + "required": ["method", "uri"] } } } diff --git a/src/pages/index.astro b/src/pages/index.astro index ee1ab54..3060418 100644 --- a/src/pages/index.astro +++ b/src/pages/index.astro @@ -48,6 +48,8 @@ const orchestrators = [ const output = [ { url: "output/indexed-db/", title: "IndexedDB" }, { url: "output/native-fs/", title: "Native File System" }, + { url: "output/todo/", title: "(TODO) Keyhive/Beelay" }, + { url: "output/todo/", title: "(TODO) Some local-first sync engine" }, ]; const processors = [ diff --git a/src/pages/input/native-fs/_applet.astro b/src/pages/input/native-fs/_applet.astro index 12a6ab0..053e884 100644 --- a/src/pages/input/native-fs/_applet.astro +++ b/src/pages/input/native-fs/_applet.astro @@ -16,12 +16,11 @@ diff --git a/src/pages/input/s3/_manifest.json b/src/pages/input/s3/_manifest.json new file mode 100644 index 0000000..d3af312 --- /dev/null +++ b/src/pages/input/s3/_manifest.json @@ -0,0 +1,55 @@ +{ + "name": "diffuse/input/s3", + "title": "Diffuse Input | S3", + "entrypoint": "index.html", + "input_properties": { + "scheme": "s3" + }, + "actions": { + "consult": { + "title": "Consult", + "params_schema": { + "type": "string", + "description": "The uri to check the availability of." + } + }, + "list": { + "title": "List", + "description": "List tracks.", + "params_schema": { + "type": "array", + "description": "A list of (cached) tracks with an uri matching the scheme", + "items": { + "type": "object" + } + } + }, + "resolve": { + "title": "Resolve", + "description": "Potentially translates a track uri with a matching scheme into a URL pointing at the audio bytes. If it can be resolved that is, otherwise you'll get `undefined`. Use the `consult` action to get a more detailed answer.", + "params_schema": { + "type": "object", + "properties": { + "method": { + "type": "string", + "description": "The HTTP method that is going to be used on the resolved URI." + }, + "uri": { "type": "string", "description": "The URI to resolve." } + }, + "required": ["method", "uri"] + } + }, + "mount": { + "title": "Mount", + "description": "Prepare for usage." + }, + "unmount": { + "title": "Unmount", + "description": "Callback after usage.", + "params_schema": { + "type": "string", + "description": "The handle id to unmount" + } + } + } +} diff --git a/src/pages/input/s3/index.astro b/src/pages/input/s3/index.astro new file mode 100644 index 0000000..f6d0270 --- /dev/null +++ b/src/pages/input/s3/index.astro @@ -0,0 +1,9 @@ +--- +import Layout from "@layouts/applet-pico-ui.astro"; +import Applet from "./_applet.astro"; +import { title } from "./_manifest.json"; +--- + + + + diff --git a/src/pages/orchestrator/input-cache/_applet.astro b/src/pages/orchestrator/input-cache/_applet.astro index 7d4dcc5..535c6ab 100644 --- a/src/pages/orchestrator/input-cache/_applet.astro +++ b/src/pages/orchestrator/input-cache/_applet.astro @@ -16,7 +16,7 @@ // Applet connections const configurator = { - input: await applet("../../configurator/input", { context: self.parent }), + input: await applet("../../configurator/input", { context: self.top || self.parent }), }; const orchestrator = { @@ -52,13 +52,18 @@ if (track.tags) return [...acc, track]; - const url = await configurator.input.sendAction("resolve", track.uri, { - timeoutDuration: 60000, - }); + const getURL = await configurator.input.sendAction( + "resolve", + { method: "GET", uri: track.uri }, + { + timeoutDuration: 60000, + }, + ); - if (!url) return acc; + if (!getURL) return acc; - const meta = await processor.metadataFetcher.sendAction("extract", url, { + // TODO: Do we need to pass the HEAD URL too? + const meta = await processor.metadataFetcher.sendAction("extract", getURL, { timeoutDuration: 60000, }); diff --git a/src/pages/orchestrator/output-management/_applet.astro b/src/pages/orchestrator/output-management/_applet.astro index 0c1bd91..27091ee 100644 --- a/src/pages/orchestrator/output-management/_applet.astro +++ b/src/pages/orchestrator/output-management/_applet.astro @@ -20,7 +20,7 @@ // Applet connections const configurator = { - output: await applet("../../configurator/output", { context: self.parent }), + output: await applet("../../configurator/output", { context: self.top || self.parent }), }; // Load tracks diff --git a/src/pages/orchestrator/single-queue/_applet.astro b/src/pages/orchestrator/single-queue/_applet.astro index 4519ace..97cd4e9 100644 --- a/src/pages/orchestrator/single-queue/_applet.astro +++ b/src/pages/orchestrator/single-queue/_applet.astro @@ -15,8 +15,12 @@ // Applet connections const engine = { - audio: await applet("../../engine/audio", { context: self.parent }), - queue: await applet("../../engine/queue", { context: self.parent }), + audio: await applet("../../engine/audio", { + context: self.top || self.parent, + }), + queue: await applet("../../engine/queue", { + context: self.top || self.parent, + }), }; const orchestrator = { diff --git a/src/pages/processor/metadata-fetcher/_applet.astro b/src/pages/processor/metadata-fetcher/_applet.astro index 68b9268..259999c 100644 --- a/src/pages/processor/metadata-fetcher/_applet.astro +++ b/src/pages/processor/metadata-fetcher/_applet.astro @@ -1,6 +1,8 @@