From a00f94f9b1361c9e310047b7ccb8798e5eb679ee Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 22 Aug 2025 18:34:47 +0000 Subject: [PATCH 01/22] feat: Implement caching backend * Two-tier caching * primary is always in-memory LRU * secondary supports file and valkey * Different caches for metadata and scrobbles * no file support for metadata (too big) --- .gitignore | 1 + package-lock.json | 201 ++++++++++++++++++-- package.json | 4 + src/backend/common/Cache.ts | 166 ++++++++++++++++ src/backend/common/infrastructure/Atomic.ts | 4 + 5 files changed, 357 insertions(+), 19 deletions(-) create mode 100644 src/backend/common/Cache.ts diff --git a/.gitignore b/.gitignore index b724b2a3..8fb4a0a7 100644 --- a/.gitignore +++ b/.gitignore @@ -118,6 +118,7 @@ dist .pnp.* config/*.json +config/mscache config/yti-* *.txt .idea/ diff --git a/package-lock.json b/package-lock.json index 42cb2270..5468d6cc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "@gr2m/net-interceptor": "^1.0.0", "@jellyfin/sdk": "^0.11.0", "@kenyip/backoff-strategies": "^1.0.4", + "@keyv/valkey": "^1.0.8", "@lukehagar/plexjs": "^0.39.0", "@react-nano/use-event-source": "^0.13.0", "@reduxjs/toolkit": "^1.9.5", @@ -37,6 +38,7 @@ "avahi-browse": "^1.1.4", "better-sse": "^0.8.0", "body-parser": "^1.19.0", + "cacheable": "^1.10.4", "castv2": "^0.1.10", "clsx": "^2.0.0", "common-tags": "^1.8.2", @@ -50,6 +52,7 @@ "express-session": "^1.17.2", "fast-deep-equal": "^3.1.3", "fixed-size-list": "^0.3.0", + "flat-cache": "^6.1.13", "formidable": "^3.5", "glob": "^11.0.0", "google-auth-library": "^9.15.0", @@ -59,6 +62,7 @@ "iso-websocket": "^0.3.0", "iti": "^0.6.0", "json5": "^2.2.3", + "keyv": "^5.5.0", "kodi-api": "^0.2.1", "lastfm-node-client": "^2.2.0", "mopidy": "^1.3.0", @@ -1317,6 +1321,12 @@ "@types/node": ">=18" } }, + "node_modules/@iovalkey/commands": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@iovalkey/commands/-/commands-0.1.0.tgz", + "integrity": "sha512-/B9W4qKSSITDii5nkBCHyPkIkAi+ealUtr1oqBJsLxjSRLka4pxun2VvMNSmcwgAMxgXtQfl0qRv7TE+udPJzg==", + "license": "MIT" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1545,6 +1555,24 @@ "resolved": "https://registry.npmjs.org/@kenyip/backoff-strategies/-/backoff-strategies-1.0.4.tgz", "integrity": "sha512-vduQZw2ctS3kIuSnCSSRiE4J90Y8WShR9xVG+e1lvFWksU2aTxjdkArcQqJ+XLm22JS380OZmrIPY1U06TAsng==" }, + "node_modules/@keyv/serialize": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.0.tgz", + "integrity": "sha512-RlDgexML7Z63Q8BSaqhXdCYNBy/JQnqYIwxofUrNLGCblOMHp+xux2Q8nLMLlPpgHQPoU0Do8Z6btCpRBEqZ8g==", + "license": "MIT" + }, + "node_modules/@keyv/valkey": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@keyv/valkey/-/valkey-1.0.8.tgz", + "integrity": "sha512-V2GziCVL5xpeVKrOo0EztKErQ+N9ZQ+a1ROPoxCU7AQ/aSIa7oRaQZzN2kOOFG6upTKLYeiUFz2q5vjacMEeIg==", + "license": "MIT", + "dependencies": { + "iovalkey": "^0.3.3" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/@lukehagar/plexjs": { "version": "0.39.0", "resolved": "https://registry.npmjs.org/@lukehagar/plexjs/-/plexjs-0.39.0.tgz", @@ -3803,6 +3831,16 @@ "node": ">= 0.8" } }, + "node_modules/cacheable": { + "version": "1.10.4", + "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-1.10.4.tgz", + "integrity": "sha512-Gd7ccIUkZ9TE2odLQVS+PDjIvQCdJKUlLdJRVvZu0aipj07Qfx+XIej7hhDrKGGoIxV5m5fT/kOJNJPQhQneRg==", + "license": "MIT", + "dependencies": { + "hookified": "^1.11.0", + "keyv": "^5.5.0" + } + }, "node_modules/cacheable-lookup": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-2.0.1.tgz", @@ -3815,6 +3853,15 @@ "node": ">=10" } }, + "node_modules/cacheable-lookup/node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, "node_modules/cacheable-request": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.4.tgz", @@ -3832,6 +3879,15 @@ "node": ">=8" } }, + "node_modules/cacheable-request/node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, "node_modules/cacheable-request/node_modules/normalize-url": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", @@ -4161,6 +4217,15 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/codec-parser": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/codec-parser/-/codec-parser-2.5.0.tgz", @@ -4556,6 +4621,15 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -5336,6 +5410,31 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-entry-cache/node_modules/flat-cache": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/file-entry-cache/node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, "node_modules/file-type": { "version": "16.5.4", "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", @@ -5453,24 +5552,21 @@ } }, "node_modules/flat-cache": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", - "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", - "dev": true, + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-6.1.13.tgz", + "integrity": "sha512-gmtS2PaUjSPa4zjObEIn4WWliKyZzYljgxODBfxugpK6q6HU9ClXzgCJ+nlcPKY9Bt090ypTOLIFWkV0jbKFjw==", + "license": "MIT", "dependencies": { - "flatted": "^3.2.9", - "keyv": "^4.5.3", - "rimraf": "^3.0.2" - }, - "engines": { - "node": "^10.12.0 || >=12.0.0" + "cacheable": "^1.10.4", + "flatted": "^3.3.3", + "hookified": "^1.11.0" } }, "node_modules/flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", - "dev": true + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", + "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "license": "ISC" }, "node_modules/follow-redirects": { "version": "1.15.9", @@ -6107,6 +6203,12 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" }, + "node_modules/hookified": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.12.0.tgz", + "integrity": "sha512-hMr1Y9TCLshScrBbV2QxJ9BROddxZ12MX9KsCtuGGy/3SmmN5H1PllKerrVlSotur9dlE8hmUKAOSa3WDzsZmQ==", + "license": "MIT" + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -6319,6 +6421,26 @@ "node": ">= 0.4" } }, + "node_modules/iovalkey": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/iovalkey/-/iovalkey-0.3.3.tgz", + "integrity": "sha512-4rTJX6Q5wTYEvxboXi8DsEiUo+OvqJGtLYOSGm37KpdRXsG5XJjbVtYKGJpPSWP+QT7rWscA4vsrdmzbEbenpw==", + "license": "MIT", + "dependencies": { + "@iovalkey/commands": "^0.1.0", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=18.12.0" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -7048,7 +7170,8 @@ "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "license": "MIT" }, "node_modules/json-schema-traverse": { "version": "1.0.0", @@ -7164,11 +7287,12 @@ } }, "node_modules/keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.5.0.tgz", + "integrity": "sha512-QG7qR2tijh1ftOvClut4YKKg1iW6cx3GZsKoGyJPxHkGWK9oJhG9P3j5deP0QQOGDowBMVQFaP+Vm4NpGYvmIQ==", + "license": "MIT", "dependencies": { - "json-buffer": "3.0.1" + "@keyv/serialize": "^1.1.0" } }, "node_modules/klaw-sync": { @@ -7252,6 +7376,12 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, "node_modules/lodash.flattendeep": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", @@ -7259,6 +7389,12 @@ "dev": true, "peer": true }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -9538,6 +9674,27 @@ "node": ">=8" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/redux": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/redux/-/redux-4.2.1.tgz", @@ -10203,6 +10360,12 @@ "dev": true, "peer": true }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", diff --git a/package.json b/package.json index d51be5b5..d6693ace 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "@gr2m/net-interceptor": "^1.0.0", "@jellyfin/sdk": "^0.11.0", "@kenyip/backoff-strategies": "^1.0.4", + "@keyv/valkey": "^1.0.8", "@lukehagar/plexjs": "^0.39.0", "@react-nano/use-event-source": "^0.13.0", "@reduxjs/toolkit": "^1.9.5", @@ -69,6 +70,7 @@ "avahi-browse": "^1.1.4", "better-sse": "^0.8.0", "body-parser": "^1.19.0", + "cacheable": "^1.10.4", "castv2": "^0.1.10", "clsx": "^2.0.0", "common-tags": "^1.8.2", @@ -82,6 +84,7 @@ "express-session": "^1.17.2", "fast-deep-equal": "^3.1.3", "fixed-size-list": "^0.3.0", + "flat-cache": "^6.1.13", "formidable": "^3.5", "glob": "^11.0.0", "google-auth-library": "^9.15.0", @@ -91,6 +94,7 @@ "iso-websocket": "^0.3.0", "iti": "^0.6.0", "json5": "^2.2.3", + "keyv": "^5.5.0", "kodi-api": "^0.2.1", "lastfm-node-client": "^2.2.0", "mopidy": "^1.3.0", diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts new file mode 100644 index 00000000..6d3c4d01 --- /dev/null +++ b/src/backend/common/Cache.ts @@ -0,0 +1,166 @@ +import { Cacheable, CacheableMemory, Keyv, KeyvStoreAdapter } from 'cacheable'; +import { FlatCache } from 'flat-cache'; +import KeyvValkey from '@keyv/valkey'; +import { childLogger, Logger } from '@foxxmd/logging'; +import { projectDir } from './index.js'; +import path from 'path'; +import { fileOrDirectoryIsWriteable } from '../utils.js'; + +export type CacheProvider = 'memory' | 'valkey' | 'file'; + +interface CacheConfig { + provider: T + connection?: string +} + +export type CacheMetadaProvider = Exclude; +export type CacheMetadataConfig = CacheConfig + +const asCacheMetadataProvider = (val: string): val is CacheScrobbleProvider => { + return ['memory', 'valkey'].includes(val); +} + +export type CacheScrobbleProvider = CacheProvider; +export type CacheScrobbleConfig = CacheConfig; + +const asCacheScrobbleProvider = (val: string): val is CacheScrobbleProvider => { + return ['memory', 'valkey', 'file'].includes(val); +} + +export interface CacheConfigOptions { + metadata?: CacheMetadataConfig + scrobble?: CacheScrobbleConfig +} + +const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); + +export class MSCache { + + config: Required + + cacheMetadata: Cacheable; + cacheScrobble: Cacheable; + + logger: Logger; + + constructor(logger: Logger, config: CacheConfigOptions) { + this.logger = childLogger(logger, 'Cache'); + + const { + metadata: { + provider: mProvider = 'memory', + ...restMetadata + } = {}, + scrobble: { + provider: sProvider = 'memory', + connection = configDir, + ...restScrobble + } = {}, + } = config; + + this.config = { + metadata: { + provider: mProvider, + ...restMetadata, + }, + scrobble: { + provider: sProvider, + connection, + ...restScrobble + } + }; + } + + init = async () => { + await this.initMetadataCache(); + await this.initScrobbleCache(); + } + + protected initCacheable = async (config: CacheConfig, cacheFor: string) => { + + let logger = childLogger(this.logger, cacheFor); + + const ns = `ms-${cacheFor.toLocaleLowerCase()}`; + + const memoryCache = new CacheableMemory({ + ttl: '1h', + useClone: true, + lruSize: 200 + }); + const primaryCache = new Keyv({ store: memoryCache, namespace: ns }); + + let secondaryCache: Keyv | KeyvStoreAdapter | undefined; + + if (config.provider === 'valkey') { + logger.debug('Building valkey cache...'); + const valkey = new KeyvValkey(config.connection, {maxRetriesPerRequest: 5, connectTimeout: 1100}); + const metadataKv = new Keyv({ store: valkey, throwOnErrors: true, namespace: ns}); + try { + await metadataKv.get('test'); + secondaryCache = metadataKv; + logger.debug('valkey cache connected'); + } catch (e) { + this.logger.warn(new Error(`Unable to connect to cache ${config.connection}`, { cause: e })); + } + } else if(config.provider === 'file') { + logger.debug('Building file cache...'); + try { + fileOrDirectoryIsWriteable(config.connection); + } catch (e) { + logger.warn(new Error(`Unable to use path for file cache at ${config.connection}`, {cause: e})); + } + const flatCache = new FlatCache({ + ttl: 0, + lruSize: 500, + cacheDir: config.connection, + cacheId: 'mscache', + persistInterval: 1 * 1000 * 60, + expirationInterval: 1 * 1000 * 60, // 1 minute + }); + + let loadError: Error; + + const onlySaveError = (e: Error) => { + loadError = e; + } + flatCache.on('error', onlySaveError); + try { + logger.debug('Loading cache from file...'); + flatCache.load('mscache'); + if(loadError !== undefined) { + throw loadError; + } + logger.debug('File cache loaded.'); + flatCache.off('error', onlySaveError); + flatCache.on('error', (e) => { + logger.warn(e); + }) + } catch (e) { + logger.warn(new Error(`Unable to use file cache at ${path.join(config.connection, 'mscache')}`, {cause: e})); + } + secondaryCache = new Keyv({store: flatCache, throwOnErrors: true}); + } + return new Cacheable({ primary: primaryCache, secondary: secondaryCache }); + + } + + initScrobbleCache = async () => { + if (this.cacheScrobble === undefined) { + if (!asCacheScrobbleProvider(this.config.scrobble.provider)) { + throw new Error(`Cache Scrobble provider '${this.config.scrobble.provider}' must be one of: memory, valkey, file`); + } + + this.cacheScrobble = await this.initCacheable(this.config.scrobble, 'Scrobble'); + } + } + + initMetadataCache = async () => { + if (this.cacheMetadata === undefined) { + if (!asCacheMetadataProvider(this.config.metadata.provider)) { + throw new Error(`Cache Metadata provider '${this.config.metadata.provider}' must be one of: memory, valkey`); + } + + this.cacheMetadata = await this.initCacheable(this.config.metadata, 'Metadata'); + } + } +} \ No newline at end of file diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 90902dcb..80be609f 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -345,3 +345,7 @@ export type WhenParts = PlayTransformPartsAtomic; export type WhenConditions = WhenParts[]; export type WhenConditionsConfig = WhenConditions; + +export type WithRequiredProperty = Type & { + [Property in Key]-?: Type[Property]; +}; \ No newline at end of file -- 2.51.2 From 56f5431dd91de2c9a4b92ab4c8d7e2029f667587 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 26 Aug 2025 21:06:22 +0000 Subject: [PATCH 02/22] chore: Update iti usage * Update package to improve typing support/esm * Remove dependency on sources/client/notifiers to prevent circular dependency --- package-lock.json | 9 +++++---- package.json | 2 +- src/backend/index.ts | 14 ++++++++++---- src/backend/ioc.ts | 14 ++++++-------- src/backend/server/api.ts | 14 ++++++-------- src/backend/server/index.ts | 8 +++++--- src/core/Atomic.ts | 28 ++++++++++++++++++++++++++-- 7 files changed, 59 insertions(+), 30 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5468d6cc..a2a4fd33 100644 --- a/package-lock.json +++ b/package-lock.json @@ -60,7 +60,7 @@ "http-cookie-agent": "^7.0.1", "icecast-metadata-stats": "^0.1.12", "iso-websocket": "^0.3.0", - "iti": "^0.6.0", + "iti": "^0.7.0", "json5": "^2.2.3", "keyv": "^5.5.0", "kodi-api": "^0.2.1", @@ -6988,9 +6988,10 @@ } }, "node_modules/iti": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/iti/-/iti-0.6.0.tgz", - "integrity": "sha512-JqujcnAIF3pmzitjbT3acc0LkordU6oHBDvWeT6a25wvEVBddFX3DFx/p6YBwGX1TTFsyLgVZtwhGOknthC96A==", + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iti/-/iti-0.7.0.tgz", + "integrity": "sha512-ERAkSZdhgrYCKF7zqX7PEGjWpjvotP858ACtpRhXssOo+vdYPD97UUn+/Tzif7oKiabdefQ4NzDETXGHZhZ7dA==", + "license": "MIT", "dependencies": { "utility-types": "^3.10.0" }, diff --git a/package.json b/package.json index d6693ace..af071362 100644 --- a/package.json +++ b/package.json @@ -92,7 +92,7 @@ "http-cookie-agent": "^7.0.1", "icecast-metadata-stats": "^0.1.12", "iso-websocket": "^0.3.0", - "iti": "^0.6.0", + "iti": "^0.7.0", "json5": "^2.2.3", "keyv": "^5.5.0", "kodi-api": "^0.2.1", diff --git a/src/backend/index.ts b/src/backend/index.ts index d6abdb3a..3e47477a 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -20,6 +20,7 @@ import { isDebugMode, parseBool, readJson, retry, sleep } from "./utils.js"; import { createVegaGenerator } from './utils/SchemaUtils.js'; import ScrobbleClients from './scrobblers/ScrobbleClients.js'; import ScrobbleSources from './sources/ScrobbleSources.js'; +import { Notifiers } from './notifier/Notifiers.js'; dayjs.extend(utc) dayjs.extend(isBetween); @@ -95,7 +96,14 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) createVegaGenerator() initLogger.info('Schema definitions generated'); - initServer(logger, appLoggerStream, output); + const scrobbleClients = new ScrobbleClients(root.get('clientEmitter'), root.get('sourceEmitter'), root.get('localUrl'), root.get('configDir'), root.get('logger')); + const scrobbleSources = new ScrobbleSources(root.get('sourceEmitter'), { + localUrl: root.get('localUrl'), + configDir: root.get('configDir'), + version: root.get('version') + }, root.get('logger')); + + initServer(logger, appLoggerStream, output, scrobbleSources, scrobbleClients); if(process.env.IS_LOCAL === 'true') { logger.info('multi-scrobbler can be run as a background service! See: https://foxxmd.github.io/multi-scrobbler/docs/installation/service'); @@ -106,18 +114,16 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) logger.warn(appConfigFail); } - const notifiers = root.get('notifiers'); + const notifiers = new Notifiers(root.get('notifierEmitter'), root.get('clientEmitter'), root.get('sourceEmitter'), root.get('logger')); //root.get('notifiers'); await notifiers.buildWebhooks(webhooks); /* * setup clients * */ - const scrobbleClients = root.get('clients') as ScrobbleClients; await scrobbleClients.buildClientsFromConfig(notifiers); /* * setup sources * */ - const scrobbleSources = root.get('sources') as ScrobbleSources; await scrobbleSources.buildSourcesFromConfig([]); // check ambiguous client/source types like this for now diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index 3536f922..ff412106 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -5,9 +5,6 @@ import { createContainer } from "iti"; import path from "path"; import { projectDir } from "./common/index.js"; import { WildcardEmitter } from "./common/WildcardEmitter.js"; -import { Notifiers } from "./notifier/Notifiers.js"; -import ScrobbleClients from "./scrobblers/ScrobbleClients.js"; -import ScrobbleSources from "./sources/ScrobbleSources.js"; import { generateBaseURL } from "./utils/NetworkUtils.js"; import { PassThrough } from "stream"; @@ -22,7 +19,7 @@ let root: ReturnType; export interface RootOptions { baseUrl?: string, - port?: string | number + port?: number logger: Logger disableWeb?: boolean loggerStream?: PassThrough @@ -51,23 +48,24 @@ const createRoot = (options?: RootOptions) => { const f = e; }); + const portVal: number | string = process.env.PORT ?? port; + return createContainer().add({ version, configDir: configDir, isProd: process.env.NODE_ENV !== undefined && (process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'prod'), - port: process.env.PORT ?? port, + // @ts-ignore + port: (Number.isInteger(portVal) ? portVal : Number.parseInt(portVal)) as number, disableWeb, clientEmitter: () => cEmitter, sourceEmitter: () => sEmitter, notifierEmitter: () => new EventEmitter(), loggerStream, loggingConfig, + logger: options.logger }).add((items) => { const localUrl = generateBaseURL(baseUrl, items.port) return { - clients: () => new ScrobbleClients(items.clientEmitter, items.sourceEmitter, localUrl, items.configDir, options.logger), - sources: () => new ScrobbleSources(items.sourceEmitter, { localUrl, configDir: items.configDir, version }, options.logger), - notifiers: () => new Notifiers(items.notifierEmitter, items.clientEmitter, items.sourceEmitter, options.logger), localUrl, hasDefinedBaseUrl: baseUrl !== undefined, isSubPath: localUrl.pathname !== '/' && localUrl.pathname.length > 0 diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index cace9c89..0612c18e 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -12,6 +12,7 @@ import { LogOutputConfig, PlayObject, SOURCE_SOT, + SOURCE_SOT_TYPES, SourceStatusData, } from "../../core/Atomic.js"; import { capitalize } from "../../core/StringUtils.js"; @@ -30,6 +31,8 @@ import { makeClientCheckMiddle, makeSourceCheckMiddle } from "./middleware.js"; import { setupPlexRoutes } from "./plexRoutes.js"; import { setupTautulliRoutes } from "./tautulliRoutes.js"; import { setupWebscrobblerRoutes } from "./webscrobblerRoutes.js"; +import ScrobbleSources from "../sources/ScrobbleSources.js"; +import ScrobbleClients from "../scrobblers/ScrobbleClients.js"; const maxBufferSize = 300; const output: Record> = {}; @@ -51,7 +54,7 @@ const getLogs = (minLevel: number, limit: number = maxBufferSize, sort: 'asc' | return allLogs.flat(1).sort((a, b) => a.time - b.time).slice(0, limit); } -export const setupApi = (app: ExpressWithAsync, logger: Logger, appLoggerStream: PassThrough, initialLogOutput: LogDataPretty[] = []) => { +export const setupApi = (app: ExpressWithAsync, logger: Logger, appLoggerStream: PassThrough, initialLogOutput: LogDataPretty[] = [], scrobbleSources: ScrobbleSources, scrobbleClients: ScrobbleClients) => { for(const level of Object.keys(logger.levels.labels)) { output[level] = new FixedSizeList(maxBufferSize); } @@ -90,9 +93,6 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, appLoggerStream: } }); - const scrobbleSources = root.get('sources'); - const scrobbleClients = root.get('clients'); - const clientMiddleFunc = makeClientCheckMiddle(scrobbleClients); const sourceMiddleFunc = makeSourceCheckMiddle(scrobbleSources); @@ -169,9 +169,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, appLoggerStream: app.getAsync('/api/status', async (req, res, next) => { - const ss = root.get('sources'); - - const sourceData = ss.sources.map((x) => { + const sourceData = scrobbleSources.sources.map((x) => { const { type, tracksDiscovered = 0, @@ -193,7 +191,7 @@ export const setupApi = (app: ExpressWithAsync, logger: Logger, appLoggerStream: hasAuthInteraction: requiresAuthInteraction, authed, players: 'players' in x ? (x as MemorySource).playersToObject() : {}, - sot: ('playerSourceOfTruth' in x) ? x.playerSourceOfTruth : SOURCE_SOT.HISTORY, + sot: ('playerSourceOfTruth' in x) ? x.playerSourceOfTruth as SOURCE_SOT_TYPES : SOURCE_SOT.HISTORY, supportsUpstreamRecentlyPlayed: x.supportsUpstreamRecentlyPlayed, supportsManualListening: x.supportsManualListening, manualListening: x.manualListening, diff --git a/src/backend/server/index.ts b/src/backend/server/index.ts index 872df819..6de124a6 100644 --- a/src/backend/server/index.ts +++ b/src/backend/server/index.ts @@ -13,11 +13,13 @@ import { getRoot } from "../ioc.js"; import { parseBool } from "../utils.js"; import { getAddress } from "../utils/NetworkUtils.js"; import { setupApi } from "./api.js"; +import ScrobbleSources from '../sources/ScrobbleSources.js'; +import ScrobbleClients from '../scrobblers/ScrobbleClients.js'; const app = addAsync(express()); const router = Router(); -export const initServer = async (parentLogger: Logger, appLoggerStream: PassThrough, initialOutput: LogDataPretty[] = []) => { +export const initServer = async (parentLogger: Logger, appLoggerStream: PassThrough, initialOutput: LogDataPretty[] = [], sources: ScrobbleSources, clients: ScrobbleClients) => { const logger = childLogger(parentLogger, 'API'); // parentLogger.child({labels: ['API']}, mergeArr); @@ -48,7 +50,7 @@ export const initServer = async (parentLogger: Logger, appLoggerStream: PassThro const local = root.get('localUrl'); const localDefined = root.get('hasDefinedBaseUrl'); - setupApi(app, logger, appLoggerStream, initialOutput); + setupApi(app, logger, appLoggerStream, initialOutput, sources, clients); const addy = getAddress(); const addresses: string[] = []; @@ -78,7 +80,7 @@ export const initServer = async (parentLogger: Logger, appLoggerStream: PassThro app.use('/docs', express.static(path.resolve(projectDir, `./docsite/build`))); if(process.env.USE_HASH_ROUTER === undefined) { - process.env.USE_HASH_ROUTER = root.get('isSubPath'); + process.env.USE_HASH_ROUTER = root.get('isSubPath').toString(); } ViteExpress.config({ diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 19078bcf..e406133a 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -5,7 +5,31 @@ import { AdditionalTrackInfoResponse } from "../backend/common/vendor/listenbrai export interface SourceStatusData { status: string; - type: "spotify" | "plex" | "tautulli" | "subsonic" | "jellyfin" | "lastfm" | "deezer" | "ytmusic" | "mpris" | "mopidy" | "listenbrainz" | "jriver" | "kodi" | 'webscrobbler' | 'chromecast'; + type: 'spotify' + | 'plex' + | 'tautulli' + | 'subsonic' + | 'jellyfin' + | 'lastfm' + | 'deezer' + | 'endpointlz' + | 'endpointlfm' + | 'ytmusic' + | 'mpris' + | 'mopidy' + | 'musiccast' + | 'listenbrainz' + | 'jriver' + | 'kodi' + | 'webscrobbler' + | 'chromecast' + | 'maloja' + | 'musikcube' + | 'mpd' + | 'vlc' + | 'icecast' + | 'azuracast' + | 'koito'; display: string; tracksDiscovered: number; name: string; @@ -23,7 +47,7 @@ export interface SourceStatusData { export interface ClientStatusData { status: string; - type: "maloja" | "lastfm" | "listenbrainz"; + type: "maloja" | "lastfm" | "listenbrainz" | "koito"; display: string; scrobbled: number; deadLetterScrobbles: number -- 2.51.2 From 4eee6d4322335a9d48fca97fed0b66afc4e8cafc Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 26 Aug 2025 22:09:44 +0000 Subject: [PATCH 03/22] somewhat working scrobble caching --- src/backend/common/AbstractComponent.ts | 43 ++++++++++++++++++- src/backend/common/Cache.ts | 37 +++------------- src/backend/common/errors/MSErrors.ts | 6 ++- src/backend/common/infrastructure/Atomic.ts | 22 +++++++++- .../common/infrastructure/config/aioConfig.ts | 4 +- src/backend/index.ts | 2 + src/backend/ioc.ts | 8 +++- .../scrobblers/AbstractScrobbleClient.ts | 24 +++++++++++ .../tests/scrobbler/scrobblers.test.ts | 4 ++ 9 files changed, 114 insertions(+), 36 deletions(-) diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 7c940ebc..0661a176 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -24,7 +24,7 @@ import { CommonClientConfig } from "./infrastructure/config/client/index.js"; import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; import play = Simulate.play; import { WebhookPayload } from "./infrastructure/config/health/webhooks.js"; -import { AuthCheckError, BuildDataError, ConnectionCheckError, PostInitError, TransformRulesError } from "./errors/MSErrors.js"; +import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, TransformRulesError } from "./errors/MSErrors.js"; import { messageWithCauses, messageWithCausesTruncatedDefault } from "../utils/ErrorUtils.js"; export default abstract class AbstractComponent { @@ -35,6 +35,7 @@ export default abstract class AbstractComponent { buildOK?: boolean | null; connectionOK?: boolean | null; + cacheOK?: boolean | null; initializing: boolean = false; @@ -65,6 +66,7 @@ export default abstract class AbstractComponent { await this.buildComponentLogger(); } await this.buildInitData(force); + await this.parseCache(force); this.buildTransformRules(); await this.checkConnection(force); await this.testAuth(force); @@ -105,6 +107,45 @@ export default abstract class AbstractComponent { } } + public async parseCache(force: boolean = false) { + if(this.cacheOK) { + if(!force) { + return; + } + this.logger.debug('Cache OK but step was forced'); + } + try { + const res = await this.doParseCache(); + if(res === undefined) { + this.cacheOK = null; + this.logger.debug('No cache to parse.'); + return; + } + if (res === true) { + this.logger.verbose('Parsing caching succeeded'); + } else if (typeof res === 'string') { + this.logger.verbose(`Parsing caching succeeded => ${res}`); + } + this.cacheOK = true; + } catch (e) { + this.cacheOK = false; + throw new ParseCacheError('Parsing cache for initialization failed', {cause: e}); + } + } + + /** + * Build or parse any cache required for this Component + * + * * Return undefined if not possible or not required + * * Return TRUE if build succeeded + * * Return string if build succeeded and should log result + * * Throw error on failure + * */ + protected async doParseCache(): Promise { + return; + } + + public async buildInitData(force: boolean = false) { if(this.buildOK) { if(!force) { diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index 6d3c4d01..e46c1667 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -5,32 +5,7 @@ import { childLogger, Logger } from '@foxxmd/logging'; import { projectDir } from './index.js'; import path from 'path'; import { fileOrDirectoryIsWriteable } from '../utils.js'; - -export type CacheProvider = 'memory' | 'valkey' | 'file'; - -interface CacheConfig { - provider: T - connection?: string -} - -export type CacheMetadaProvider = Exclude; -export type CacheMetadataConfig = CacheConfig - -const asCacheMetadataProvider = (val: string): val is CacheScrobbleProvider => { - return ['memory', 'valkey'].includes(val); -} - -export type CacheScrobbleProvider = CacheProvider; -export type CacheScrobbleConfig = CacheConfig; - -const asCacheScrobbleProvider = (val: string): val is CacheScrobbleProvider => { - return ['memory', 'valkey', 'file'].includes(val); -} - -export interface CacheConfigOptions { - metadata?: CacheMetadataConfig - scrobble?: CacheScrobbleConfig -} +import { asCacheMetadataProvider, asCacheScrobbleProvider, CacheConfig, CacheConfigOptions, CacheMetadaProvider, CacheProvider } from './infrastructure/Atomic.js'; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); @@ -43,17 +18,18 @@ export class MSCache { logger: Logger; - constructor(logger: Logger, config: CacheConfigOptions) { + constructor(logger: Logger, config: CacheConfigOptions = {}) { this.logger = childLogger(logger, 'Cache'); const { metadata: { - provider: mProvider = 'memory', + provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadaProvider | undefined) ?? 'memory'), + connection: mConn = process.env.CACHE_METADATA_CONN, ...restMetadata } = {}, scrobble: { - provider: sProvider = 'memory', - connection = configDir, + provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheProvider | undefined) ?? 'file'), + connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir), ...restScrobble } = {}, } = config; @@ -61,6 +37,7 @@ export class MSCache { this.config = { metadata: { provider: mProvider, + connection: mConn, ...restMetadata, }, scrobble: { diff --git a/src/backend/common/errors/MSErrors.ts b/src/backend/common/errors/MSErrors.ts index 3ada4ca2..5060f497 100644 --- a/src/backend/common/errors/MSErrors.ts +++ b/src/backend/common/errors/MSErrors.ts @@ -8,12 +8,16 @@ export class BuildDataError extends StageError { name = 'Init Build Data'; } +export class ParseCacheError extends StageError { + name = 'Init Parse Cache'; +} + export class TransformRulesError extends StageError { name = 'Transform Rules'; } export class ConnectionCheckError extends StageError { - name = 'Conenction Check'; + name = 'Connection Check'; } export class AuthCheckError extends StageError { diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 80be609f..2c7f3dc4 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -348,4 +348,24 @@ export type WhenConditionsConfig = WhenConditions; export type WithRequiredProperty = Type & { [Property in Key]-?: Type[Property]; -}; \ No newline at end of file +}; +export type CacheProvider = 'memory' | 'valkey' | 'file'; +export interface CacheConfig { + provider: T; + connection?: string; +} +export type CacheMetadaProvider = Exclude; +export type CacheMetadataConfig = CacheConfig; +export const asCacheMetadataProvider = (val: string): val is CacheScrobbleProvider => { + return ['memory', 'valkey'].includes(val); +}; +export type CacheScrobbleProvider = CacheProvider; +export type CacheScrobbleConfig = CacheConfig; +export const asCacheScrobbleProvider = (val: string): val is CacheScrobbleProvider => { + return ['memory', 'valkey', 'file'].includes(val); +}; +export interface CacheConfigOptions { + metadata?: CacheMetadataConfig; + scrobble?: CacheScrobbleConfig; +} + diff --git a/src/backend/common/infrastructure/config/aioConfig.ts b/src/backend/common/infrastructure/config/aioConfig.ts index cfe6134d..95d5fdce 100644 --- a/src/backend/common/infrastructure/config/aioConfig.ts +++ b/src/backend/common/infrastructure/config/aioConfig.ts @@ -5,7 +5,7 @@ import { RequestRetryOptions } from "./common.js"; import { WebhookConfig } from "./health/webhooks.js"; import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js"; import { SourceAIOConfig } from "./source/sources.js"; -import { ClientType, SourceType } from "../Atomic.js"; +import { CacheConfigOptions, ClientType, SourceType } from "../Atomic.js"; export interface SourceDefaults extends CommonSourceOptions { @@ -64,6 +64,8 @@ export interface AIOConfig { * @examples [false] * */ debugMode?: boolean + + cache?: CacheConfigOptions } export interface AIOClientConfig { diff --git a/src/backend/index.ts b/src/backend/index.ts index 3e47477a..c2a0369d 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -103,6 +103,8 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) version: root.get('version') }, root.get('logger')); + await root.get('cache').init(); + initServer(logger, appLoggerStream, output, scrobbleSources, scrobbleClients); if(process.env.IS_LOCAL === 'true') { diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index ff412106..5cb149f8 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -8,6 +8,8 @@ import { WildcardEmitter } from "./common/WildcardEmitter.js"; import { generateBaseURL } from "./utils/NetworkUtils.js"; import { PassThrough } from "stream"; +import { CacheConfigOptions } from "./common/infrastructure/Atomic.js"; +import { MSCache } from "./common/Cache.js"; export let version: string = 'unknown'; @@ -24,6 +26,7 @@ export interface RootOptions { disableWeb?: boolean loggerStream?: PassThrough loggingConfig?: LogOptions + cache?: CacheConfigOptions } const createRoot = (options?: RootOptions) => { @@ -32,7 +35,7 @@ const createRoot = (options?: RootOptions) => { baseUrl = process.env.BASE_URL, disableWeb: dw, loggerStream, - loggingConfig + loggingConfig, } = options || {}; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); let disableWeb = dw; @@ -62,7 +65,8 @@ const createRoot = (options?: RootOptions) => { notifierEmitter: () => new EventEmitter(), loggerStream, loggingConfig, - logger: options.logger + logger: options.logger, + cache: new MSCache(options.logger, options.cache) }).add((items) => { const localUrl = generateBaseURL(baseUrl, items.port) return { diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 27ca1c2b..dd77811c 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -52,6 +52,8 @@ import { } from "../utils/TimeUtils.js"; import { WebhookPayload } from "../common/infrastructure/config/health/webhooks.js"; import { AsyncTask, SimpleIntervalJob, Task, ToadScheduler } from "toad-scheduler"; +import { MSCache } from "../common/Cache.js"; +import { getRoot } from "../ioc.js"; type PlatformMappedPlays = Map; type NowPlayingQueue = Map; @@ -97,6 +99,8 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i nowPlayingTaskInterval: number = 5000; npLogger: Logger; + cache: MSCache; + declare config: CommonClientConfig; notifier: Notifiers; @@ -110,6 +114,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.npLogger = childLogger(this.logger, 'Now Playing'); this.notifier = notifier; this.emitter = emitter; + this.cache = getRoot().get('cache'); this.scrobbledPlayObjs = new FixedSizeList(this.MAX_STORED_SCROBBLES); @@ -169,6 +174,9 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i protected getIdentifier() { return `${capitalize(this.type)} - ${this.name}` } + protected getMachineId() { + return `${this.type}-${this.name}`; + } public notify = async (payload: WebhookPayload) => { this.emitEvent('notify', payload); @@ -295,6 +303,18 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } } + protected async doParseCache(): Promise { + const cachedQueue = (await this.cache.cacheScrobble.get(`${this.getMachineId()}-queue`) as QueuedScrobble[] ?? []); + const cachedQLength = cachedQueue.length; + this.queuedScrobbles = cachedQueue; + + const cachedDead = (await this.cache.cacheScrobble.get(`${this.getMachineId()}-dead`) as DeadLetterScrobble[] ?? []); + const cachedDLength = cachedDead.length; + this.deadLetterScrobbles = cachedDead; + + return `Scrobbled from Cache: ${cachedQLength} Queue | ${cachedDLength} Dead Letter`; + } + protected async postInitialize(): Promise { const { options: { @@ -887,10 +907,12 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); } this.logger.info(`Removed scrobble ${buildTrackString(this.deadLetterScrobbles[index].play)} from queue`, {leaf: 'Dead Letter'}); this.deadLetterScrobbles.splice(index, 1); + this.cache.cacheScrobble.set(`${this.getMachineId()}-dead`, this.deadLetterScrobbles); } removeDeadLetterScrobbles = () => { this.deadLetterScrobbles = []; + this.cache.cacheScrobble.set(`${this.getMachineId()}-dead`, []); this.logger.info('Removed all scrobbles from queue', {leaf: 'Dead Letter'}); } @@ -910,6 +932,7 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); this.queuedScrobbles.push(queuedPlay); } this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); + this.cache.cacheScrobble.set(`${this.getMachineId()}-queue`, this.queuedScrobbles); } protected addDeadLetterScrobble = (data: QueuedScrobble, error: (Error | string) = 'Unspecified error') => { @@ -923,6 +946,7 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); this.deadLetterScrobbles.push(deadData); this.deadLetterScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); this.emitEvent('deadLetter', {dead: deadData}); + this.cache.cacheScrobble.set(`${this.getMachineId()}-dead`, this.deadLetterScrobbles); } queuePlayingNow = (data: PlayObject, source: SourceIdentifier) => { diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index be5c4a95..4755595f 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -16,6 +16,8 @@ import MockDate from 'mockdate'; import { NowPlayingScrobbler, TestAuthScrobbler, TestScrobbler } from "./TestScrobbler.js"; import { PlayPlatformId } from '../../common/infrastructure/Atomic.js'; +import { getRoot } from '../../ioc.js'; +import { loggerTest } from '@foxxmd/logging'; chai.use(asPromised); @@ -29,6 +31,8 @@ const normalizedWithMixedDur = normalizePlays(mixedDurPlays, {initialDate: first const normalizedWithMixedDurOlder = normalizePlays(mixedDurPlays, {initialDate: olderFirstPlayDate}); +getRoot({cache: {scrobble: {provider: 'memory'}}, logger: loggerTest}); + const generateTestScrobbler = () => { const testScrobbler = new TestScrobbler(); testScrobbler.verboseOptions = { -- 2.51.2 From e2c8158aec23e81eb08a9dd9d1ac02246a0baf23 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 27 Aug 2025 00:49:46 +0000 Subject: [PATCH 04/22] feat: Log file cache save --- src/backend/common/Cache.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index e46c1667..4da97cf0 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -111,7 +111,10 @@ export class MSCache { flatCache.off('error', onlySaveError); flatCache.on('error', (e) => { logger.warn(e); - }) + }); + flatCache.on('save', () => { + logger.debug('Saved cache to file'); + }); } catch (e) { logger.warn(new Error(`Unable to use file cache at ${path.join(config.connection, 'mscache')}`, {cause: e})); } -- 2.51.2 From 05a9ca15a9f33d254cce95e9facb9ce802955e62 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 27 Aug 2025 13:06:40 +0000 Subject: [PATCH 05/22] refactor(cache): Update file cache name and gitignore --- .gitignore | 1 + src/backend/common/Cache.ts | 6 +++--- src/backend/scrobblers/AbstractScrobbleClient.ts | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 8fb4a0a7..afffe313 100644 --- a/.gitignore +++ b/.gitignore @@ -120,6 +120,7 @@ dist config/*.json config/mscache config/yti-* +config/*.cache *.txt .idea/ diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index 4da97cf0..c6faf4ea 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -90,7 +90,7 @@ export class MSCache { ttl: 0, lruSize: 500, cacheDir: config.connection, - cacheId: 'mscache', + cacheId: 'scrobble.cache', persistInterval: 1 * 1000 * 60, expirationInterval: 1 * 1000 * 60, // 1 minute }); @@ -103,7 +103,7 @@ export class MSCache { flatCache.on('error', onlySaveError); try { logger.debug('Loading cache from file...'); - flatCache.load('mscache'); + flatCache.load('scrobble.cache'); if(loadError !== undefined) { throw loadError; } @@ -116,7 +116,7 @@ export class MSCache { logger.debug('Saved cache to file'); }); } catch (e) { - logger.warn(new Error(`Unable to use file cache at ${path.join(config.connection, 'mscache')}`, {cause: e})); + logger.warn(new Error(`Unable to use file cache at ${path.join(config.connection, 'scrobble.cache')}`, {cause: e})); } secondaryCache = new Keyv({store: flatCache, throwOnErrors: true}); } diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index dd77811c..1e3aa3dd 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -312,7 +312,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i const cachedDLength = cachedDead.length; this.deadLetterScrobbles = cachedDead; - return `Scrobbled from Cache: ${cachedQLength} Queue | ${cachedDLength} Dead Letter`; + return `Scrobbles from Cache: ${cachedQLength} Queue | ${cachedDLength} Dead Letter`; } protected async postInitialize(): Promise { -- 2.51.2 From 78f4767dd2db8b32124b7e91b82fdf13ba48756d Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 27 Aug 2025 16:52:27 +0000 Subject: [PATCH 06/22] refactor: Refactor di cache to allow singleton or transient --- src/backend/index.ts | 2 +- src/backend/ioc.ts | 24 +++++++++++++++---- .../scrobblers/AbstractScrobbleClient.ts | 2 +- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/backend/index.ts b/src/backend/index.ts index c2a0369d..aa85c734 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -103,7 +103,7 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) version: root.get('version') }, root.get('logger')); - await root.get('cache').init(); + await root.items.cache().init(); initServer(logger, appLoggerStream, output, scrobbleSources, scrobbleClients); diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index 5cb149f8..f1a14c45 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -1,5 +1,5 @@ import { getVersion } from "@foxxmd/get-version"; -import { Logger, LogOptions } from "@foxxmd/logging"; +import { Logger, loggerDebug, LogOptions } from "@foxxmd/logging"; import { EventEmitter } from "events"; import { createContainer } from "iti"; import path from "path"; @@ -26,16 +26,18 @@ export interface RootOptions { disableWeb?: boolean loggerStream?: PassThrough loggingConfig?: LogOptions - cache?: CacheConfigOptions + cache?: CacheConfigOptions | MSCache | (() => MSCache) } -const createRoot = (options?: RootOptions) => { +const createRoot = (options: RootOptions = {logger: loggerDebug}) => { const { port = 9078, baseUrl = process.env.BASE_URL, disableWeb: dw, loggerStream, loggingConfig, + logger, + cache } = options || {}; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); let disableWeb = dw; @@ -43,6 +45,18 @@ const createRoot = (options?: RootOptions) => { disableWeb = process.env.DISABLE_WEB === 'true'; } + let cacheFunc: () => MSCache; + let maybeSingletonCache: MSCache; + + if(cache instanceof MSCache) { + maybeSingletonCache = cache; + } else if(typeof cache === 'function') { + cacheFunc = cache; + } else { + maybeSingletonCache = new MSCache(logger, cache); + } + + const cEmitter = new WildcardEmitter(); // do nothing, just catch cEmitter.on('error', (e) => null); @@ -65,8 +79,8 @@ const createRoot = (options?: RootOptions) => { notifierEmitter: () => new EventEmitter(), loggerStream, loggingConfig, - logger: options.logger, - cache: new MSCache(options.logger, options.cache) + logger: logger, + cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc }).add((items) => { const localUrl = generateBaseURL(baseUrl, items.port) return { diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 1e3aa3dd..01383025 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -114,7 +114,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.npLogger = childLogger(this.logger, 'Now Playing'); this.notifier = notifier; this.emitter = emitter; - this.cache = getRoot().get('cache'); + this.cache = getRoot().items.cache(); this.scrobbledPlayObjs = new FixedSizeList(this.MAX_STORED_SCROBBLES); -- 2.51.2 From 32e2aa69742261b83f768b994da6adb435a6da5d Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 27 Aug 2025 16:52:41 +0000 Subject: [PATCH 07/22] test: Setup transient di cache --- .mocharc.json | 3 +++ src/backend/tests/scrobbler/TestScrobbler.ts | 5 +++++ src/backend/tests/scrobbler/scrobblers.test.ts | 10 +++------- src/backend/tests/setup.ts | 8 ++++++++ 4 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 src/backend/tests/setup.ts diff --git a/.mocharc.json b/.mocharc.json index 15d99f31..c22a525b 100644 --- a/.mocharc.json +++ b/.mocharc.json @@ -3,5 +3,8 @@ "extension": "ts", "import": "tsx/esm", "spec": "./src/backend/tests/**/*.test.ts", + "file": [ + "./src/backend/tests/setup.ts" + ], "exit": true } diff --git a/src/backend/tests/scrobbler/TestScrobbler.ts b/src/backend/tests/scrobbler/TestScrobbler.ts index 93279fab..f1bae249 100644 --- a/src/backend/tests/scrobbler/TestScrobbler.ts +++ b/src/backend/tests/scrobbler/TestScrobbler.ts @@ -28,6 +28,11 @@ export class TestScrobbler extends AbstractScrobbleClient { return (await this.existingScrobble(playObj)) !== undefined; } + protected async doParseCache() { + await this.cache.init(); + return super.doParseCache(); + } + playToClientPayload(playObject: PlayObject): object { diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index 4755595f..f3bf87c7 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -8,16 +8,14 @@ import { http, HttpResponse } from 'msw'; import pEvent from 'p-event'; import { PlayObject } from "../../../core/Atomic.js"; import { genGroupIdStr, sleep } from "../../utils.js"; -import mixedDuration from '../plays/mixedDuration.json'; -import withDuration from '../plays/withDuration.json'; +import mixedDuration from '../plays/mixedDuration.json' with { type: 'json' }; +import withDuration from '../plays/withDuration.json' with { type: 'json' }; import { MockNetworkError, withRequestInterception } from "../utils/networking.js"; import { asPlays, generatePlay, generatePlayPlatformId, generatePlays, normalizePlays } from "../utils/PlayTestUtils.js"; import MockDate from 'mockdate'; import { NowPlayingScrobbler, TestAuthScrobbler, TestScrobbler } from "./TestScrobbler.js"; import { PlayPlatformId } from '../../common/infrastructure/Atomic.js'; -import { getRoot } from '../../ioc.js'; -import { loggerTest } from '@foxxmd/logging'; chai.use(asPromised); @@ -31,8 +29,6 @@ const normalizedWithMixedDur = normalizePlays(mixedDurPlays, {initialDate: first const normalizedWithMixedDurOlder = normalizePlays(mixedDurPlays, {initialDate: olderFirstPlayDate}); -getRoot({cache: {scrobble: {provider: 'memory'}}, logger: loggerTest}); - const generateTestScrobbler = () => { const testScrobbler = new TestScrobbler(); testScrobbler.verboseOptions = { @@ -665,7 +661,7 @@ describe('Scrobble client uses transform plays correctly', function() { describe('Manages scrobble queue', function() { - before(async function() { + beforeEach(async function() { testScrobbler = generateTestScrobbler(); await testScrobbler.initialize(); testScrobbler.recentScrobbles = normalizedWithMixedDur; diff --git a/src/backend/tests/setup.ts b/src/backend/tests/setup.ts new file mode 100644 index 00000000..c41a7fff --- /dev/null +++ b/src/backend/tests/setup.ts @@ -0,0 +1,8 @@ +import { loggerTest } from '@foxxmd/logging'; +import { getRoot } from "../ioc.js"; +import { MSCache } from '../common/Cache.js'; + +const transientCache = () => new MSCache(loggerTest, {scrobble: {provider: 'memory'}}); + +const root = getRoot({cache: transientCache, logger: loggerTest}); +root.items.cache().init(); \ No newline at end of file -- 2.51.2 From 63b6012488992ed3637b7fe06ce3d61a4b3668ad Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 27 Aug 2025 17:35:44 +0000 Subject: [PATCH 08/22] feat(cache): Rehydrate dayjs data on cached scrobble parsing --- src/backend/common/Cache.ts | 19 ++++++++ .../scrobblers/AbstractScrobbleClient.ts | 5 +- src/backend/utils/CacheUtils.ts | 46 +++++++++++++++++++ src/core/Atomic.ts | 15 ++++-- 4 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 src/backend/utils/CacheUtils.ts diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index c6faf4ea..4c8d4f1b 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -1,6 +1,14 @@ import { Cacheable, CacheableMemory, Keyv, KeyvStoreAdapter } from 'cacheable'; import { FlatCache } from 'flat-cache'; +import {parse} from 'flatted'; import KeyvValkey from '@keyv/valkey'; +import dayjs from 'dayjs'; +import duration from 'dayjs/plugin/duration.js'; +import isBetween from 'dayjs/plugin/isBetween.js'; +import relativeTime from 'dayjs/plugin/relativeTime.js'; +import isToday from 'dayjs/plugin/isToday.js'; +import timezone from 'dayjs/plugin/timezone.js'; +import utc from 'dayjs/plugin/utc.js'; import { childLogger, Logger } from '@foxxmd/logging'; import { projectDir } from './index.js'; import path from 'path'; @@ -9,6 +17,13 @@ import { asCacheMetadataProvider, asCacheScrobbleProvider, CacheConfig, CacheCon const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); +dayjs.extend(utc) +dayjs.extend(isBetween); +dayjs.extend(relativeTime); +dayjs.extend(duration); +dayjs.extend(timezone); +dayjs.extend(isToday); + export class MSCache { config: Required @@ -93,6 +108,10 @@ export class MSCache { cacheId: 'scrobble.cache', persistInterval: 1 * 1000 * 60, expirationInterval: 1 * 1000 * 60, // 1 minute + // deserialize: (str) => { + // const data = parse(str) + // return data; + // } }); let loadError: Error; diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 01383025..0abfea83 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -54,6 +54,7 @@ import { WebhookPayload } from "../common/infrastructure/config/health/webhooks. import { AsyncTask, SimpleIntervalJob, Task, ToadScheduler } from "toad-scheduler"; import { MSCache } from "../common/Cache.js"; import { getRoot } from "../ioc.js"; +import { rehydratePlay } from "../utils/CacheUtils.js"; type PlatformMappedPlays = Map; type NowPlayingQueue = Map; @@ -306,11 +307,11 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i protected async doParseCache(): Promise { const cachedQueue = (await this.cache.cacheScrobble.get(`${this.getMachineId()}-queue`) as QueuedScrobble[] ?? []); const cachedQLength = cachedQueue.length; - this.queuedScrobbles = cachedQueue; + this.queuedScrobbles = cachedQueue.map(x => ({...x, play: rehydratePlay(x.play)})); const cachedDead = (await this.cache.cacheScrobble.get(`${this.getMachineId()}-dead`) as DeadLetterScrobble[] ?? []); const cachedDLength = cachedDead.length; - this.deadLetterScrobbles = cachedDead; + this.deadLetterScrobbles = cachedDead.map(x => ({...x, play: rehydratePlay(x.play), lastRetry: x.lastRetry !== undefined ? dayjs(x.lastRetry) : undefined})); return `Scrobbles from Cache: ${cachedQLength} Queue | ${cachedDLength} Dead Letter`; } diff --git a/src/backend/utils/CacheUtils.ts b/src/backend/utils/CacheUtils.ts new file mode 100644 index 00000000..60bb4faf --- /dev/null +++ b/src/backend/utils/CacheUtils.ts @@ -0,0 +1,46 @@ +import dayjs from 'dayjs'; +import duration from 'dayjs/plugin/duration.js'; +import isBetween from 'dayjs/plugin/isBetween.js'; +import relativeTime from 'dayjs/plugin/relativeTime.js'; +import isToday from 'dayjs/plugin/isToday.js'; +import timezone from 'dayjs/plugin/timezone.js'; +import utc from 'dayjs/plugin/utc.js'; +import { AmbPlayObject, ListenRangeData, ListenRangeDataAmb, PlayObject, PlayProgress, PlayProgressAmb } from '../../core/Atomic.js'; +import objectHash from 'object-hash'; + +dayjs.extend(utc) +dayjs.extend(isBetween); +dayjs.extend(relativeTime); +dayjs.extend(duration); +dayjs.extend(timezone); +dayjs.extend(isToday); + +export const rehydratePlay = (obj: AmbPlayObject): PlayObject => { + if(obj.data?.playDate !== undefined && typeof obj.data.playDate === 'string') { + obj.data.playDate = dayjs(obj.data.playDate); + if(obj.data.playDateCompleted !== undefined) { + obj.data.playDateCompleted = dayjs(obj.data.playDateCompleted); + } + + if(obj.data.listenRanges !== undefined) { + obj.data.listenRanges = obj.data.listenRanges.map(rehydrateListenRangeData); + } + } + return obj as PlayObject; +} + +// this may become problematic since we aren't re-instantiating Progress class, just implementing interface +// but that may only be an issue if rehydrating source data which isn't in scope so far +export const rehydrateListenRangeData = (obj: ListenRangeDataAmb): ListenRangeData => { + return { + start: rehydratePlayProgress(obj.start), + end: rehydratePlayProgress(obj.end) + } as ListenRangeData; +} + +export const rehydratePlayProgress = (obj: PlayProgressAmb): PlayProgress => { + if(typeof obj.timestamp === 'string') { + obj.timestamp = dayjs(obj.timestamp); + } + return obj as PlayProgress; +} \ No newline at end of file diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index e406133a..dd86b12c 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -76,17 +76,26 @@ export interface TrackStringOptions { } } -export interface PlayProgress { - timestamp: Dayjs +export interface PlayProgressAmb { + timestamp: string | Dayjs position?: number positionPercent?: number } +export interface PlayProgress extends PlayProgressAmb { + timestamp: Dayjs +} + export interface PlayProgressPositional extends PlayProgress { position: number } -export interface ListenRangeData { +export interface ListenRangeDataAmb { + start: PlayProgressAmb + end: PlayProgressAmb +} + +export interface ListenRangeData extends ListenRangeDataAmb { start: ListenProgress end: ListenProgress } -- 2.51.2 From 39bc9cd4107fdf03c4bb49d16bcab641fe71c278 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 28 Aug 2025 13:58:38 +0000 Subject: [PATCH 09/22] build(vscode): Update devcontainer to use compose stack with valkey for caching integration --- .devcontainer/compose.yml | 25 +++++++++++++++++++++++++ .devcontainer/devcontainer.json | 8 +++++++- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/compose.yml diff --git a/.devcontainer/compose.yml b/.devcontainer/compose.yml new file mode 100644 index 00000000..6e244406 --- /dev/null +++ b/.devcontainer/compose.yml @@ -0,0 +1,25 @@ +version: '3.7' +services: + # Update this to the name of the service you want to work with in your docker-compose.yml file + app: + # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer + # folder. Note that the path of the Dockerfile and context is relative to the *primary* + # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile" + # array). The sample below assumes your primary file is in the root of your project. + # + # build: + # context: . + # dockerfile: .devcontainer/Dockerfile + + image: mcr.microsoft.com/devcontainers/javascript-node:1-20-bookworm + + volumes: + # Update this to wherever you want VS Code to mount the folder of your project + - ..:/workspaces:cached + command: sleep infinity + + valkey: + image: valkey/valkey + env_file: + - path: ./.env + required: false \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c5a30676..23fc0a31 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,13 @@ { "name": "Node.js", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "mcr.microsoft.com/devcontainers/javascript-node:1-20-bookworm", + "dockerComposeFile": [ + "compose.yml" + ], + + "service": "app", + + "workspaceFolder": "/workspaces", // Features to add to the dev container. More info: https://containers.dev/features. // "features": {}, -- 2.51.2 From 652fa5a14cb4856491887e6589257ab788e30682 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 28 Aug 2025 17:24:11 +0000 Subject: [PATCH 10/22] feat(cache): Improve data marshalling and cache init * Use typeson for stringified data marshalling * Use clone for in-memory serializing to preserve dayjs * Refactor cache backend init into indepedent, testable functions * Add `false` as cache provider type to allow disabling secondary cache --- package-lock.json | 69 +++++- package.json | 2 + src/backend/common/Cache.ts | 244 ++++++++++++++------ src/backend/common/infrastructure/Atomic.ts | 18 +- src/backend/tests/cache/cache.test.ts | 171 ++++++++++++++ 5 files changed, 421 insertions(+), 83 deletions(-) create mode 100644 src/backend/tests/cache/cache.test.ts diff --git a/package-lock.json b/package-lock.json index a2a4fd33..2edba090 100644 --- a/package-lock.json +++ b/package-lock.json @@ -88,6 +88,8 @@ "tough-cookie": "^5.1.2", "ts-json-schema-generator": "^2.3.0", "tsx": "^4.7.0", + "typeson": "^9.0.4", + "typeson-registry": "^11.1.1", "vite-express": "^0.16.0", "vlc-client": "^1.1.1", "xml2js": "0.6.1", @@ -3642,6 +3644,15 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-arraybuffer-es6": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/base64-arraybuffer-es6/-/base64-arraybuffer-es6-3.1.0.tgz", + "integrity": "sha512-QKKtftiSrKjilihGNLXxnrb9LJj7rnEdB1cYAqVpekFy0tisDklAf1RAgvpm0HsGYx9sv7FUbgpsrfwTyCPVLg==", + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -9392,7 +9403,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, "engines": { "node": ">=6" } @@ -11232,6 +11242,63 @@ "resolved": "https://registry.npmjs.org/typescript-event-target/-/typescript-event-target-1.1.1.tgz", "integrity": "sha512-dFSOFBKV6uwaloBCCUhxlD3Pr/P1a/tJdcmPrTXCHlEFD3faj0mztjcGn6VBAhQ0/Bdy8K3VWrrqwbt/ffsYsg==" }, + "node_modules/typeson": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/typeson/-/typeson-9.0.4.tgz", + "integrity": "sha512-umRYLe37m4fTu0AlgFaqu1C+N+i3LqAehCzbvNx9w7alyN1wpPyS6FbBEsvWD2FppS2yJ9gk0FkFBkZ/wJE13Q==", + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/typeson-registry": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/typeson-registry/-/typeson-registry-11.1.1.tgz", + "integrity": "sha512-WF8meNtURfGXXY2RitEMgZrMEclHU8IdxxN4YTcnXTQUHvisPYsKrxxPFhc4DKTCAc6sVvWVtKelOXFDtw/OQQ==", + "license": "MIT", + "dependencies": { + "base64-arraybuffer-es6": "^3.1.0", + "typeson": "^9.0.3", + "whatwg-url": "^14.0.0" + }, + "engines": { + "node": ">=18.14.0" + } + }, + "node_modules/typeson-registry/node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typeson-registry/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/typeson-registry/node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/uid-safe": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", diff --git a/package.json b/package.json index af071362..19f2778e 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,8 @@ "tough-cookie": "^5.1.2", "ts-json-schema-generator": "^2.3.0", "tsx": "^4.7.0", + "typeson": "^9.0.4", + "typeson-registry": "^11.1.1", "vite-express": "^0.16.0", "vlc-client": "^1.1.1", "xml2js": "0.6.1", diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index 4c8d4f1b..e82ee946 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -1,20 +1,23 @@ -import { Cacheable, CacheableMemory, Keyv, KeyvStoreAdapter } from 'cacheable'; -import { FlatCache } from 'flat-cache'; -import {parse} from 'flatted'; +import { Cacheable, createKeyv, Keyv, KeyvStoreAdapter, KeyvOptions, CacheableOptions } from 'cacheable'; +import { FlatCache, FlatCacheOptions } from 'flat-cache'; import KeyvValkey from '@keyv/valkey'; -import dayjs from 'dayjs'; +import dayjs, { Dayjs } from 'dayjs'; import duration from 'dayjs/plugin/duration.js'; import isBetween from 'dayjs/plugin/isBetween.js'; import relativeTime from 'dayjs/plugin/relativeTime.js'; import isToday from 'dayjs/plugin/isToday.js'; import timezone from 'dayjs/plugin/timezone.js'; import utc from 'dayjs/plugin/utc.js'; +import clone from 'clone'; import { childLogger, Logger } from '@foxxmd/logging'; import { projectDir } from './index.js'; import path from 'path'; import { fileOrDirectoryIsWriteable } from '../utils.js'; -import { asCacheMetadataProvider, asCacheScrobbleProvider, CacheConfig, CacheConfigOptions, CacheMetadaProvider, CacheProvider } from './infrastructure/Atomic.js'; - +import { asCacheMetadataProvider, asCacheScrobbleProvider, CacheConfig, CacheConfigOptions, CacheMetadaProvider, CacheProvider, CacheScrobbleProvider } from './infrastructure/Atomic.js'; +import { Typeson } from 'typeson'; +import { builtin } from 'typeson-registry'; +import { MaybeLogger } from './logging.js'; +import { ListenProgressPositional, ListenProgressTS } from '../sources/PlayerState/ListenProgress.js'; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); dayjs.extend(utc) @@ -24,6 +27,19 @@ dayjs.extend(duration); dayjs.extend(timezone); dayjs.extend(isToday); +const typeson = new Typeson().register([ + builtin, +]); +typeson.register({ + Dayjs: [ + (x) => dayjs.isDayjs(x), + (d: Dayjs) => d.toJSON(), + (date) => dayjs(date) + ], + ListenProgressTS, + ListenProgressPositional +}); + export class MSCache { config: Required @@ -43,7 +59,7 @@ export class MSCache { ...restMetadata } = {}, scrobble: { - provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheProvider | undefined) ?? 'file'), + provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'), connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir), ...restScrobble } = {}, @@ -64,82 +80,51 @@ export class MSCache { } init = async () => { - await this.initMetadataCache(); + // disabled for now + //await this.initMetadataCache(); await this.initScrobbleCache(); } protected initCacheable = async (config: CacheConfig, cacheFor: string) => { let logger = childLogger(this.logger, cacheFor); + const providerHints = ['In-Memory (Primary)']; + if(config.provider !== false) { + providerHints.push(`${config.provider} (Secondary)`) + } + logger.verbose(`Cache Providers: ${providerHints.join(' | ')}`) const ns = `ms-${cacheFor.toLocaleLowerCase()}`; - - const memoryCache = new CacheableMemory({ - ttl: '1h', - useClone: true, - lruSize: 200 - }); - const primaryCache = new Keyv({ store: memoryCache, namespace: ns }); - - let secondaryCache: Keyv | KeyvStoreAdapter | undefined; - - if (config.provider === 'valkey') { - logger.debug('Building valkey cache...'); - const valkey = new KeyvValkey(config.connection, {maxRetriesPerRequest: 5, connectTimeout: 1100}); - const metadataKv = new Keyv({ store: valkey, throwOnErrors: true, namespace: ns}); - try { - await metadataKv.get('test'); - secondaryCache = metadataKv; - logger.debug('valkey cache connected'); - } catch (e) { - this.logger.warn(new Error(`Unable to connect to cache ${config.connection}`, { cause: e })); - } - } else if(config.provider === 'file') { - logger.debug('Building file cache...'); - try { - fileOrDirectoryIsWriteable(config.connection); - } catch (e) { - logger.warn(new Error(`Unable to use path for file cache at ${config.connection}`, {cause: e})); - } - const flatCache = new FlatCache({ - ttl: 0, - lruSize: 500, - cacheDir: config.connection, - cacheId: 'scrobble.cache', - persistInterval: 1 * 1000 * 60, - expirationInterval: 1 * 1000 * 60, // 1 minute - // deserialize: (str) => { - // const data = parse(str) - // return data; - // } - }); - - let loadError: Error; - - const onlySaveError = (e: Error) => { - loadError = e; - } - flatCache.on('error', onlySaveError); - try { - logger.debug('Loading cache from file...'); - flatCache.load('scrobble.cache'); - if(loadError !== undefined) { - throw loadError; - } - logger.debug('File cache loaded.'); - flatCache.off('error', onlySaveError); - flatCache.on('error', (e) => { - logger.warn(e); - }); - flatCache.on('save', () => { - logger.debug('Saved cache to file'); - }); - } catch (e) { - logger.warn(new Error(`Unable to use file cache at ${path.join(config.connection, 'scrobble.cache')}`, {cause: e})); - } - secondaryCache = new Keyv({store: flatCache, throwOnErrors: true}); + + const cacheOpts: CacheableOptions = { + primary: initMemoryCache({ namespace: ns }) + } + + let secondaryCache: Keyv | KeyvStoreAdapter | undefined; + + if (config.provider === 'valkey') { + logger.debug(`Building valkey cache from ${config.connection}`); + try { + secondaryCache = await initValkeyCache(ns, config.connection); + logger.debug('valkey cache connected'); + } catch (e) { + this.logger.warn(e); + } + } else if (config.provider === 'file') { + logger.debug(`Building file cache from ${path.join(config.connection, `${ns}.cache`)}`); + + try { + const [keyvFile] = initFileCache({ cacheDir: config.connection, cacheId: `${ns}.cache` }, logger); + secondaryCache = keyvFile; + } catch (e) { + logger.warn(e); } - return new Cacheable({ primary: primaryCache, secondary: secondaryCache }); + } + + if(secondaryCache !== undefined) { + cacheOpts.secondary = secondaryCache; + } + return new Cacheable(cacheOpts); } @@ -162,4 +147,115 @@ export class MSCache { this.cacheMetadata = await this.initCacheable(this.config.metadata, 'Metadata'); } } +} + + +export const initMemoryCache = (opts: Parameters[0] = {}): Keyv | KeyvStoreAdapter => { + const memory = createKeyv({ + ttl: '1h', + lruSize: 200, + ...opts, + useClone: false, + }); + // structuredClone does not work well with dayjs https://github.com/iamkun/dayjs/issues/2236 + // but deep cloning is fine so disable useClone and provide our own cloning function + memory.serialize = (data) => { + return clone(data) as string; + } + return memory; +} + +export const flatCacheCreate = (opts: FlatCacheOptions) => { + return new FlatCache({ + ttl: 0, + lruSize: 500, + cacheDir: opts.cacheDir ?? configDir, + cacheId: opts.cacheId ?? 'scrobble.cache', + persistInterval: 1 * 1000 * 60, + expirationInterval: 1 * 1000 * 60, // 1 minute + ...opts + }); +} + +export const flatCacheLoad = (flatCache: FlatCache, logger: MaybeLogger): void => { + + const cachePath = path.join(flatCache.cacheDir, flatCache.cacheId); + try { + fileOrDirectoryIsWriteable(cachePath); + } catch (e) { + new Error(`Unable to use path for file cache at ${cachePath}`, { cause: e }) + } + + let loadError: Error; + + const onlySaveError = (e: Error) => { + loadError = e; + } + flatCache.on('error', onlySaveError); + try { + logger.debug('Loading cache from file...'); + flatCache.load(); + if (loadError !== undefined) { + throw loadError; + } + logger.debug(`File cache loaded`); + flatCache.off('error', onlySaveError); + } catch (e) { + throw new Error(`Unable to use file cache at ${cachePath}`, { cause: e }); + } +} + +export const initFileCache = (opts: FlatCacheOptions = {}, logger: MaybeLogger = new MaybeLogger()): [Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined] => { + const flatCache = flatCacheCreate(opts); + try { + flatCacheLoad(flatCache, logger); + flatCache.on('error', (e) => { + logger.warn(e); + }); + flatCache.on('save', () => { + logger.debug('Saved cache to file'); + }); + + const cache = new Keyv({ + store: flatCache, + throwOnErrors: true, + ...typesonMarshalling + }); + return [cache, flatCache]; + } catch (e) { + throw e; + } +} + +export const valkeyCacheCreate = (ns: string, ...args: ConstructorParameters): Keyv => { + const [connection, valkeyOpts = {}] = args; + const valkey = new KeyvValkey(connection, { maxRetriesPerRequest: 5, connectTimeout: 1100, ...valkeyOpts }); + const kv = new Keyv({ + store: valkey, + throwOnErrors: true, + namespace: ns, + ...typesonMarshalling + }); + return kv; +} + +export const initValkeyCache = async (ns: string, ...args: ConstructorParameters): Promise => { + const kv = valkeyCacheCreate(ns, ...args); + try { + await kv.get('test'); + return kv; + } catch (e) { + throw new Error(`Unable to connect to cache ${args[0]}`, { cause: e }) + } +} + +const typesonMarshalling: Pick = { + serialize: (data) => { + const str = typeson.stringifySync(data); + return str; + }, + deserialize: (str) => { + const data = typeson.parseSync(str); + return data; + } } \ No newline at end of file diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 2c7f3dc4..e73818fa 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -349,21 +349,23 @@ export type WhenConditionsConfig = WhenConditions; export type WithRequiredProperty = Type & { [Property in Key]-?: Type[Property]; }; -export type CacheProvider = 'memory' | 'valkey' | 'file'; +export type CacheProvider = 'memory' | 'valkey' | 'file' | false; export interface CacheConfig { provider: T; connection?: string; } -export type CacheMetadaProvider = Exclude; +export type CacheMetadaProvider = CacheProvider;//Exclude; export type CacheMetadataConfig = CacheConfig; -export const asCacheMetadataProvider = (val: string): val is CacheScrobbleProvider => { - return ['memory', 'valkey'].includes(val); -}; +export const asCacheProvider = (val: boolean | string): val is CacheProvider => { + if(typeof val === 'string') { + return ['memory', 'valkey', 'file'].includes(val); + } + return val === false; +} +export const asCacheMetadataProvider = (val: any): val is CacheScrobbleProvider => asCacheProvider(val); export type CacheScrobbleProvider = CacheProvider; export type CacheScrobbleConfig = CacheConfig; -export const asCacheScrobbleProvider = (val: string): val is CacheScrobbleProvider => { - return ['memory', 'valkey', 'file'].includes(val); -}; +export const asCacheScrobbleProvider = (val: any): val is CacheScrobbleProvider => asCacheProvider(val); export interface CacheConfigOptions { metadata?: CacheMetadataConfig; scrobble?: CacheScrobbleConfig; diff --git a/src/backend/tests/cache/cache.test.ts b/src/backend/tests/cache/cache.test.ts new file mode 100644 index 00000000..580be240 --- /dev/null +++ b/src/backend/tests/cache/cache.test.ts @@ -0,0 +1,171 @@ +import { loggerTest, loggerDebug } from "@foxxmd/logging"; +import chai, { assert, expect } from 'chai'; +import asPromised from 'chai-as-promised'; +import { after, before, describe, it } from 'mocha'; +import dayjs from "dayjs"; +import withLocalTmpDir from 'with-local-tmp-dir'; +import path from 'path'; +import { initFileCache, initMemoryCache, initValkeyCache, MSCache } from "../../common/Cache.js"; +import { generatePlay, generatePlayerStateData, generatePlays, normalizePlays } from "../utils/PlayTestUtils.js"; +import { ListenProgressPositional, ListenProgressTS } from "../../sources/PlayerState/ListenProgress.js"; +import { isPortReachableConnect } from "../../utils/NetworkUtils.js"; + +chai.use(asPromised); + +describe('#Caching', function () { + + describe('#MemoryCaching', function () { + + it('Memory cache preserves dayjs', async function () { + + const cache = initMemoryCache(); + + const now = dayjs(); + + await cache.set('foo', now); + + const time = await cache.get('foo'); + + expect(time).to.not.be.undefined; + expect(dayjs.isDayjs(time)).is.true; + expect(now.toJSON()).eq((time as any).toJSON()); + }); + + it('Memory cache preserves ListenProgress', async function () { + + const cache = initMemoryCache(); + + const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 }); + + await cache.set('foo', prog); + + const cachedProg = await cache.get('foo') as ListenProgressTS; + + expect(cachedProg).to.not.be.undefined; + expect(cachedProg instanceof ListenProgressTS).is.true; + expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON()); + }); + + }); + + describe('#FileCaching', function () { + + it('File cache serializes and deserializes dayjs', async function () { + + withLocalTmpDir(async () => { + + const [keyv, flat] = initFileCache({ cacheDir: process.cwd() }); + + const now = dayjs(); + + await keyv.set('foo', now); + flat.save(); + + const [cleanKeyv, cleanFlat] = initFileCache({ cacheDir: process.cwd() }); + + const time = await cleanKeyv.get('foo'); + + expect(time).to.not.be.undefined; + expect(time instanceof dayjs).is.true; + expect(now.toJSON()).eq((time as any).toJSON()); + + }, { unsafeCleanup: true }); + }); + + it('File cache serializes and deserializes ListenProgress', async function () { + + withLocalTmpDir(async () => { + + const [keyv, flat] = initFileCache({ cacheDir: process.cwd() }); + + const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 }); + + await keyv.set('foo', prog); + await flat.save(); + + const [cleanKeyv, cleanFlat] = initFileCache({ cacheDir: process.cwd() }); + + const cachedProg = await cleanKeyv.get('foo'); + + expect(cachedProg).to.not.be.undefined; + expect(cachedProg instanceof ListenProgressTS).is.true; + expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON()); + + }, { unsafeCleanup: true }); + }); + }); + + describe('#ValkeyCaching', function () { + before(async function () { + try { + await isPortReachableConnect(6379, { host: 'valkey' }); + } catch (e) { + // don't run valkey tests if valkey isn't present + this.skip(); + } + }); + + it('Valkey cache serializes and deserializes dayjs', async function () { + + const keyv = await initValkeyCache('test', 'redis://valkey:6379'); + await keyv.clear(); + + const now = dayjs(); + + await keyv.set('foo', now); + + const time = await keyv.get('foo'); + + expect(time).to.not.be.undefined; + expect(time instanceof dayjs).is.true; + expect(now.toJSON()).eq((time as any).toJSON()); + + }); + + it('Valkey cache serializes and deserializes ListenProgress', async function () { + + const keyv = await initValkeyCache('test', 'redis://valkey:6379'); + await keyv.clear(); + + const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 }); + + await keyv.set('foo', prog); + + const cachedProg = await keyv.get('foo'); + + expect(cachedProg).to.not.be.undefined; + expect(cachedProg instanceof ListenProgressTS).is.true; + expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON()); + + }); + }); + + + + + // it('File backend serializes and deserializes non-primitives', async function() { + + // withLocalTmpDir(async () => { + + // const cache = new MSCache(loggerTest, {scrobble: {provider: 'file', connection: process.cwd()}}); + // await cache.init(); + + // const now = dayjs(); + + // await cache.cacheScrobble.set('foo', now); + // await cache.cacheScrobble.secondary.store.save(); + // await cache.cacheScrobble.disconnect() + + // const cleanCache = new MSCache(loggerTest, {scrobble: {provider: 'file', connection: process.cwd()}}); + // await cleanCache.init(); + + // const time = await cleanCache.cacheScrobble.get('foo'); + + // expect(time).to.not.be.undefined; + // expect(time instanceof dayjs).is.true; + // expect(now.toJSON).eq((time as any).toJSON()); + + // }, {unsafeCleanup: true}); + // }); + +}); -- 2.51.2 From 814e609efcde350d55a38a3e0f9bac3602c2e943 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 28 Aug 2025 17:27:11 +0000 Subject: [PATCH 11/22] fix: Add clone as runtime dependency --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2edba090..afdc1298 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,6 +40,7 @@ "body-parser": "^1.19.0", "cacheable": "^1.10.4", "castv2": "^0.1.10", + "clone": "^2.1.2", "clsx": "^2.0.0", "common-tags": "^1.8.2", "compare-versions": "^4.1.2", @@ -128,7 +129,6 @@ "@vitejs/plugin-react": "^4.2.1", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", - "clone": "^2.1.2", "eslint": "^8.56.0", "eslint-plugin-prefer-arrow-functions": "^3.2.4", "mocha": "^10.3.0", @@ -4196,7 +4196,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", - "dev": true, + "license": "MIT", "engines": { "node": ">=0.8" } diff --git a/package.json b/package.json index 19f2778e..c228fa94 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "body-parser": "^1.19.0", "cacheable": "^1.10.4", "castv2": "^0.1.10", + "clone": "^2.1.2", "clsx": "^2.0.0", "common-tags": "^1.8.2", "compare-versions": "^4.1.2", @@ -160,7 +161,6 @@ "@vitejs/plugin-react": "^4.2.1", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", - "clone": "^2.1.2", "eslint": "^8.56.0", "eslint-plugin-prefer-arrow-functions": "^3.2.4", "mocha": "^10.3.0", -- 2.51.2 From 682d5076833a26798caac7d3e863a96f81b50632 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 2 Sep 2025 12:09:05 +0000 Subject: [PATCH 12/22] build: move net-interceptor to dev packages --- package-lock.json | 3 ++- package.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index afdc1298..95f9f431 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,6 @@ "@foxxmd/redact-string": "^0.1.2", "@foxxmd/regex-buddy-core": "^0.1.2", "@foxxmd/string-sameness": "^0.4.0", - "@gr2m/net-interceptor": "^1.0.0", "@jellyfin/sdk": "^0.11.0", "@kenyip/backoff-strategies": "^1.0.4", "@keyv/valkey": "^1.0.8", @@ -100,6 +99,7 @@ "@dbus-types/notifications": "^0.0.5", "@eslint/js": "^8.56.0", "@faker-js/faker": "^9.0.1", + "@gr2m/net-interceptor": "^1.0.0", "@istanbuljs/nyc-config-typescript": "^1.0.2", "@testing-library/jest-dom": "^5.17.0", "@testing-library/react": "^13.4.0", @@ -1180,6 +1180,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/@gr2m/net-interceptor/-/net-interceptor-1.0.0.tgz", "integrity": "sha512-qLTp3npmB5SDiEIh1Jg1v8a09dlnfYV1kcSyLmsyucvZBT89nM+6L1QdlrCC/W1cTvdTpvI+Vleit94UVX7/Cg==", + "dev": true, "engines": { "node": ">= 14" } diff --git a/package.json b/package.json index c228fa94..e9bfa12d 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,6 @@ "@foxxmd/redact-string": "^0.1.2", "@foxxmd/regex-buddy-core": "^0.1.2", "@foxxmd/string-sameness": "^0.4.0", - "@gr2m/net-interceptor": "^1.0.0", "@jellyfin/sdk": "^0.11.0", "@kenyip/backoff-strategies": "^1.0.4", "@keyv/valkey": "^1.0.8", @@ -132,6 +131,7 @@ "@dbus-types/notifications": "^0.0.5", "@eslint/js": "^8.56.0", "@faker-js/faker": "^9.0.1", + "@gr2m/net-interceptor": "^1.0.0", "@istanbuljs/nyc-config-typescript": "^1.0.2", "@testing-library/jest-dom": "^5.17.0", "@testing-library/react": "^13.4.0", -- 2.51.2 From 48467b08d3de23038bcea195976680997e310b09 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Sep 2025 20:10:18 +0000 Subject: [PATCH 13/22] chore: Bump with-local-tmp-dir version tmp dependency has been removed so previous buggy behavior(?) is gone --- package-lock.json | 105 ++++++++++++++++++++++++++++++++-------------- package.json | 7 +--- 2 files changed, 74 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index 95f9f431..22ef30b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -139,7 +139,7 @@ "typescript": "5.5.4", "typescript-eslint": "^7.0.1", "vite": "^5.2.12", - "with-local-tmp-dir": "^5.1.1" + "with-local-tmp-dir": "^6.0.0" }, "engines": { "node": ">=18.19.1", @@ -561,12 +561,13 @@ "integrity": "sha512-lp316jRb+OCCm8KbzidppWCKteISwgpWU6sxzzDI+PJtflfX87h/AHVQdcqY1f3/za2S869KliJbtJZuan1H2Q==" }, "node_modules/@dword-design/chdir": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@dword-design/chdir/-/chdir-2.1.4.tgz", - "integrity": "sha512-cHpMTx4XepHQi3hmDnKma5YXU4pm01k9xKI1k8Wwq25cN3j0FCWbiji11hdrRYr6+uDC9mrb/M5Mo0okYiNRzA==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@dword-design/chdir/-/chdir-4.0.0.tgz", + "integrity": "sha512-Lw0SDZbeMnsGwBwZNeazDh5RL9oVXsEVD9M3S2hOrgQTNq3LwCs7ndxO72exOGqC+OVQBt8OFQ08gsUOX7mdCQ==", "dev": true, + "license": "MIT", "engines": { - "node": ">=12" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/dword-design" @@ -2779,6 +2780,17 @@ "@types/node": "*" } }, + "node_modules/@types/fs-extra": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz", + "integrity": "sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/jsonfile": "*", + "@types/node": "*" + } + }, "node_modules/@types/hoist-non-react-statics": { "version": "3.3.5", "resolved": "https://registry.npmjs.org/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.5.tgz", @@ -2824,6 +2836,16 @@ "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" }, + "node_modules/@types/jsonfile": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.4.tgz", + "integrity": "sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/keyv": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz", @@ -2933,6 +2955,13 @@ "@types/node": "*" } }, + "node_modules/@types/retry": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz", + "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/send": { "version": "0.17.4", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", @@ -6626,6 +6655,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-network-error": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.1.0.tgz", + "integrity": "sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-node-process": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", @@ -8593,6 +8635,24 @@ "node": ">=8" } }, + "node_modules/p-retry": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", + "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.2", + "is-network-error": "^1.0.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-timeout": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", @@ -10880,27 +10940,6 @@ "node": ">=0.6.0" } }, - "node_modules/tmp-promise": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", - "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", - "dev": true, - "dependencies": { - "tmp": "^0.2.0" - } - }, - "node_modules/tmp-promise/node_modules/tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dev": true, - "dependencies": { - "rimraf": "^3.0.0" - }, - "engines": { - "node": ">=8.17.0" - } - }, "node_modules/to-readable-stream": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-2.1.0.tgz", @@ -12029,16 +12068,18 @@ } }, "node_modules/with-local-tmp-dir": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/with-local-tmp-dir/-/with-local-tmp-dir-5.1.1.tgz", - "integrity": "sha512-7HZx3lC2BNMqrkob/29+GxBZtt0J2j5qyYn77/FivEAKS2Q90Tlc6KaqjBAPnVo0B3OfHDtCBZG3i8S6kD8N8A==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/with-local-tmp-dir/-/with-local-tmp-dir-6.0.0.tgz", + "integrity": "sha512-UqVBYzOaK6gXybyO3tYGrATBfQ9itasWQVpO5DPi7/YitEMS0u7wqgJHLpD9CUYSHyIgeKmLw/HN/iIm3dGxCA==", "dev": true, + "license": "MIT", "dependencies": { - "@dword-design/chdir": "^2.0.0", - "tmp-promise": "^3.0.0" + "@dword-design/chdir": "^4.0.0", + "@types/fs-extra": "^11.0.4", + "p-retry": "^6.2.1" }, "engines": { - "node": ">=14" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/dword-design" diff --git a/package.json b/package.json index e9bfa12d..05a1fac8 100644 --- a/package.json +++ b/package.json @@ -171,7 +171,7 @@ "typescript": "5.5.4", "typescript-eslint": "^7.0.1", "vite": "^5.2.12", - "with-local-tmp-dir": "^5.1.1" + "with-local-tmp-dir": "^6.0.0" }, "browserslist": { "production": [ @@ -188,11 +188,6 @@ "overrides": { "spotify-web-api-node": { "superagent": "$superagent" - }, - "with-local-tmp-dir": { - "tmp-promise": { - "tmp": "0.2.1" - } } } } -- 2.51.2 From b4154bb0d554ba6229891c0452c3e04831860da9 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Sep 2025 20:11:02 +0000 Subject: [PATCH 14/22] refactor: Move transient cache to exportable function --- src/backend/tests/setup.ts | 4 +--- src/backend/tests/utils/CacheTestUtils.ts | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 src/backend/tests/utils/CacheTestUtils.ts diff --git a/src/backend/tests/setup.ts b/src/backend/tests/setup.ts index c41a7fff..ecc3656a 100644 --- a/src/backend/tests/setup.ts +++ b/src/backend/tests/setup.ts @@ -1,8 +1,6 @@ import { loggerTest } from '@foxxmd/logging'; import { getRoot } from "../ioc.js"; -import { MSCache } from '../common/Cache.js'; - -const transientCache = () => new MSCache(loggerTest, {scrobble: {provider: 'memory'}}); +import { transientCache } from './utils/CacheTestUtils.js'; const root = getRoot({cache: transientCache, logger: loggerTest}); root.items.cache().init(); \ No newline at end of file diff --git a/src/backend/tests/utils/CacheTestUtils.ts b/src/backend/tests/utils/CacheTestUtils.ts new file mode 100644 index 00000000..96b4621b --- /dev/null +++ b/src/backend/tests/utils/CacheTestUtils.ts @@ -0,0 +1,4 @@ +import { loggerTest } from "@foxxmd/logging"; +import { MSCache } from "../../common/Cache.js"; + +export const transientCache = () => new MSCache(loggerTest, {scrobble: {provider: 'memory'}}); \ No newline at end of file -- 2.51.2 From d78bc48e6f0f6ed9edc8d621b033a85fd8ea5888 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Sep 2025 20:11:23 +0000 Subject: [PATCH 15/22] feat: Allow passing file cache options other than connection --- src/backend/common/Cache.ts | 2 +- src/backend/common/infrastructure/Atomic.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index e82ee946..c488b230 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -114,7 +114,7 @@ export class MSCache { logger.debug(`Building file cache from ${path.join(config.connection, `${ns}.cache`)}`); try { - const [keyvFile] = initFileCache({ cacheDir: config.connection, cacheId: `${ns}.cache` }, logger); + const [keyvFile] = initFileCache({ ...config, cacheDir: config.connection, cacheId: `${ns}.cache` }, logger); secondaryCache = keyvFile; } catch (e) { logger.warn(e); diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index e73818fa..dccad631 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -353,6 +353,7 @@ export type CacheProvider = 'memory' | 'valkey' | 'file' | false; export interface CacheConfig { provider: T; connection?: string; + [key: string]: any } export type CacheMetadaProvider = CacheProvider;//Exclude; export type CacheMetadataConfig = CacheConfig; -- 2.51.2 From ffb41b4e303d4eac3e8523fe392148d25c63d235 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Sep 2025 20:12:08 +0000 Subject: [PATCH 16/22] test: Add scrobble client caching test --- src/backend/tests/cache/cache.test.ts | 54 +++++++++++++++++++-------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/src/backend/tests/cache/cache.test.ts b/src/backend/tests/cache/cache.test.ts index 580be240..2474896b 100644 --- a/src/backend/tests/cache/cache.test.ts +++ b/src/backend/tests/cache/cache.test.ts @@ -9,6 +9,11 @@ import { initFileCache, initMemoryCache, initValkeyCache, MSCache } from "../../ import { generatePlay, generatePlayerStateData, generatePlays, normalizePlays } from "../utils/PlayTestUtils.js"; import { ListenProgressPositional, ListenProgressTS } from "../../sources/PlayerState/ListenProgress.js"; import { isPortReachableConnect } from "../../utils/NetworkUtils.js"; +import { getRoot } from "../../ioc.js"; +import { transientCache } from "../utils/CacheTestUtils.js"; +import { TestScrobbler } from "../scrobbler/TestScrobbler.js"; +import { sleep } from "../../utils.js"; +import {promises} from 'node:fs'; chai.use(asPromised); @@ -68,6 +73,7 @@ describe('#Caching', function () { expect(time).to.not.be.undefined; expect(time instanceof dayjs).is.true; expect(now.toJSON()).eq((time as any).toJSON()); + flat.destroy(); }, { unsafeCleanup: true }); }); @@ -90,6 +96,7 @@ describe('#Caching', function () { expect(cachedProg).to.not.be.undefined; expect(cachedProg instanceof ListenProgressTS).is.true; expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON()); + flat.destroy(); }, { unsafeCleanup: true }); }); @@ -140,32 +147,47 @@ describe('#Caching', function () { }); }); + describe('#ScrobbleCache', function () { + afterEach(function () { + const root = getRoot(); + root.upsert({ cache: () => transientCache }); + root.items.cache().init(); + }); + it('Preserves scrobbles', async function () { - // it('File backend serializes and deserializes non-primitives', async function() { - // withLocalTmpDir(async () => { + await withLocalTmpDir(async () => { - // const cache = new MSCache(loggerTest, {scrobble: {provider: 'file', connection: process.cwd()}}); - // await cache.init(); + const root = getRoot(); + root.upsert({ cache: () => () => new MSCache(loggerTest, { scrobble: { provider: 'file', connection: process.cwd(), persistInterval: 100 } }) }); + root.items.cache().init(); - // const now = dayjs(); + const test = new TestScrobbler(); + await test.initialize(); + const play = generatePlay(); + test.queueScrobble(play, 'testSource'); + await sleep(101); + const dirContents = await promises.readdir('.'); + const hasCache = dirContents.some(x => x === 'ms-scrobble.cache'); + expect(hasCache).is.true; - // await cache.cacheScrobble.set('foo', now); - // await cache.cacheScrobble.secondary.store.save(); - // await cache.cacheScrobble.disconnect() + // reinit cache + root.upsert({ cache: () => () => new MSCache(loggerTest, { scrobble: { provider: 'file', connection: process.cwd(), persistInterval: 100 } }) }); + const newCache = root.items.cache(); + expect(newCache.cacheScrobble).to.be.undefined; + newCache.init(); - // const cleanCache = new MSCache(loggerTest, {scrobble: {provider: 'file', connection: process.cwd()}}); - // await cleanCache.init(); + const newTest = new TestScrobbler(); + await newTest.initialize(); + expect(newTest.queuedScrobbles.length).to.eq(1); + expect(newTest.queuedScrobbles[0].play.data.track).to.eq(play.data.track); - // const time = await cleanCache.cacheScrobble.get('foo'); + }, { unsafeCleanup: true }); - // expect(time).to.not.be.undefined; - // expect(time instanceof dayjs).is.true; - // expect(now.toJSON).eq((time as any).toJSON()); + }); - // }, {unsafeCleanup: true}); - // }); + }); }); -- 2.51.2 From 10dcb7f30dae3aa8f53043caebe94b123e3ea521 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 9 Sep 2025 19:55:31 +0000 Subject: [PATCH 17/22] feat: Implement cache file streaming with progress logging If cache files get large enough to cause noticeable delay it will be helpful to log progress --- src/backend/common/Cache.ts | 53 ++++++++++++++++++--------- src/backend/tests/cache/cache.test.ts | 16 +++----- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index c488b230..72245f8b 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -114,7 +114,7 @@ export class MSCache { logger.debug(`Building file cache from ${path.join(config.connection, `${ns}.cache`)}`); try { - const [keyvFile] = initFileCache({ ...config, cacheDir: config.connection, cacheId: `${ns}.cache` }, logger); + const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection, cacheId: `${ns}.cache` }, logger); secondaryCache = keyvFile; } catch (e) { logger.warn(e); @@ -177,38 +177,57 @@ export const flatCacheCreate = (opts: FlatCacheOptions) => { }); } -export const flatCacheLoad = (flatCache: FlatCache, logger: MaybeLogger): void => { +export const flatCacheLoad = async (flatCache: FlatCache, logger: MaybeLogger): Promise => { const cachePath = path.join(flatCache.cacheDir, flatCache.cacheId); try { fileOrDirectoryIsWriteable(cachePath); } catch (e) { - new Error(`Unable to use path for file cache at ${cachePath}`, { cause: e }) + throw new Error(`Unable to use path for file cache at ${cachePath}`, { cause: e }) } - let loadError: Error; + const streamPromise = new Promise((resolve, reject) => { + flatCache.loadFileStream(cachePath, (progress: number, total: number) => { + logger.debug(`Loading ${progress}/${total} chunks...`); + }, () => { + resolve(true); + }, (err: Error) => { + reject(err); + }); + }); - const onlySaveError = (e: Error) => { - loadError = e; - } - flatCache.on('error', onlySaveError); try { - logger.debug('Loading cache from file...'); - flatCache.load(); - if (loadError !== undefined) { - throw loadError; - } + await streamPromise; logger.debug(`File cache loaded`); - flatCache.off('error', onlySaveError); + return; } catch (e) { - throw new Error(`Unable to use file cache at ${cachePath}`, { cause: e }); + if (null !== e.message.match(/Cache file .+ does not exist/)) { + let loadError: Error; + try { + const onlySaveError = (e: Error) => { + loadError = e; + }; + flatCache.on('error', onlySaveError); + flatCache.load(); + if (loadError !== undefined) { + throw loadError; + } + flatCache.off('error', onlySaveError); + logger.debug(`File cache loaded`); + return; + } catch (e) { + throw new Error(`Unable to use file cache at ${cachePath}`, { cause: e }); + } + } else { + throw new Error(`Unable to use file cache at ${cachePath}`, { cause: e }); + } } } -export const initFileCache = (opts: FlatCacheOptions = {}, logger: MaybeLogger = new MaybeLogger()): [Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined] => { +export const initFileCache = async (opts: FlatCacheOptions = {}, logger: MaybeLogger = new MaybeLogger()): Promise<[Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined]> => { const flatCache = flatCacheCreate(opts); try { - flatCacheLoad(flatCache, logger); + await flatCacheLoad(flatCache, logger); flatCache.on('error', (e) => { logger.warn(e); }); diff --git a/src/backend/tests/cache/cache.test.ts b/src/backend/tests/cache/cache.test.ts index 2474896b..2acecdde 100644 --- a/src/backend/tests/cache/cache.test.ts +++ b/src/backend/tests/cache/cache.test.ts @@ -162,27 +162,21 @@ describe('#Caching', function () { const root = getRoot(); root.upsert({ cache: () => () => new MSCache(loggerTest, { scrobble: { provider: 'file', connection: process.cwd(), persistInterval: 100 } }) }); - root.items.cache().init(); const test = new TestScrobbler(); await test.initialize(); - const play = generatePlay(); - test.queueScrobble(play, 'testSource'); + const plays = generatePlays(100); + test.queueScrobble(plays, 'testSource'); + const queued = test.queuedScrobbles.map(x => x.play); await sleep(101); const dirContents = await promises.readdir('.'); const hasCache = dirContents.some(x => x === 'ms-scrobble.cache'); expect(hasCache).is.true; - // reinit cache - root.upsert({ cache: () => () => new MSCache(loggerTest, { scrobble: { provider: 'file', connection: process.cwd(), persistInterval: 100 } }) }); - const newCache = root.items.cache(); - expect(newCache.cacheScrobble).to.be.undefined; - newCache.init(); - const newTest = new TestScrobbler(); await newTest.initialize(); - expect(newTest.queuedScrobbles.length).to.eq(1); - expect(newTest.queuedScrobbles[0].play.data.track).to.eq(play.data.track); + expect(newTest.queuedScrobbles.length).to.eq(plays.length); + expect(newTest.queuedScrobbles[0].play.data.track).to.eq(queued[0].data.track); }, { unsafeCleanup: true }); -- 2.51.2 From 351edd8102d2fe8666723fefd0c71b99132f9ada Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 9 Sep 2025 19:58:01 +0000 Subject: [PATCH 18/22] fix: Fix cache test missing awaits --- src/backend/tests/cache/cache.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/tests/cache/cache.test.ts b/src/backend/tests/cache/cache.test.ts index 2acecdde..d2580efb 100644 --- a/src/backend/tests/cache/cache.test.ts +++ b/src/backend/tests/cache/cache.test.ts @@ -59,14 +59,14 @@ describe('#Caching', function () { withLocalTmpDir(async () => { - const [keyv, flat] = initFileCache({ cacheDir: process.cwd() }); + const [keyv, flat] = await initFileCache({ cacheDir: process.cwd() }); const now = dayjs(); await keyv.set('foo', now); flat.save(); - const [cleanKeyv, cleanFlat] = initFileCache({ cacheDir: process.cwd() }); + const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd() }); const time = await cleanKeyv.get('foo'); @@ -82,14 +82,14 @@ describe('#Caching', function () { withLocalTmpDir(async () => { - const [keyv, flat] = initFileCache({ cacheDir: process.cwd() }); + const [keyv, flat] = await initFileCache({ cacheDir: process.cwd() }); const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 }); await keyv.set('foo', prog); await flat.save(); - const [cleanKeyv, cleanFlat] = initFileCache({ cacheDir: process.cwd() }); + const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd() }); const cachedProg = await cleanKeyv.get('foo'); -- 2.51.2 From 9bea31e1b2abddafe71d19cd2757da099a3f5541 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 9 Sep 2025 20:35:11 +0000 Subject: [PATCH 19/22] docs: Add caching docs --- docsite/docs/configuration/configuration.mdx | 87 ++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/docsite/docs/configuration/configuration.mdx b/docsite/docs/configuration/configuration.mdx index fd29b3e6..3fb54a0c 100644 --- a/docsite/docs/configuration/configuration.mdx +++ b/docsite/docs/configuration/configuration.mdx @@ -186,6 +186,93 @@ Useful when running with [docker](../installation/installation.mdx#docker) so th +#### Caching + +Multi-scrobbler can cache some of its data for the purpose of **persisting queued and dead scrobbles** after restarting. + +It supports caching using either: + +* **File** cache stored in the `CONFIG_DIR` directory (next to your File/[AIO](./?configType=aio#configuration-types) file) +* [**Valkey**](https://valkey.io/), an open-source fork of Redis. + +**File** caching is **enabled by default** when no other configuration is present. + +The type of cache used, and it's connection properties, can be configured through ENV or **AIO** config. + +##### Caching Configuration + + + + + +**File** cache is stored in the `CONFIG_DIR` directory using the pre-defined file name `ms-scrobble.cache`. + +It is **enabled by default** if `CACHE_SCROBBLE`, or the respective AIO configuration, is not defined. + + + + + +| Environmental Variable | Required? | Default | Description | +| :--------------------- | --------- | -------------------- | ------------------------------------------------------------ | +| `CACHE_SCROBBLE` | No | `file` | The cache type to use | +| `CACHE_SCROBBLE_CONN` | No | The config directory | The directory, within the container, to store the cache file | + + + + + +```json5 title="config.json" +{ + "cache": { + "scrobble": { + "provider": "file", + "connection": "/config" + } + }, + // ... +} +``` + + + + + + + + + + + + +| Environmental Variable | Required? | Default | Description | +| :--------------------- | --------- | -------------------- | ------------------------------------------------------------ | +| `CACHE_SCROBBLE` | Yes | `valkey` | The cache type to use | +| `CACHE_SCROBBLE_CONN` | Yes | | The host/IP and port to connect to, prefixed with `redis://` -- EX: `redis://192.168.0.120:6379` | + + + + + +```json5 title="config.json" +{ + "cache": { + "scrobble": { + "provider": "valkey", + "connection": "redis://192.168.0.120:6379" + } + }, + // ... +} +``` + + + + + + + + #### Debug Mode Turning on Debug Mode will -- 2.51.2 From c1e1de2255372a077c7c35c4cef26ba51da40ee6 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 9 Sep 2025 20:40:55 +0000 Subject: [PATCH 20/22] Increase file cache size to 2000 Should be ~700kb, full up. This should be a generous upper limit for number of scrobbles queued/dead at the same time. --- src/backend/common/Cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index 72245f8b..70dddc81 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -168,7 +168,7 @@ export const initMemoryCache = (opts: Parameters[0] = {}): Ke export const flatCacheCreate = (opts: FlatCacheOptions) => { return new FlatCache({ ttl: 0, - lruSize: 500, + lruSize: 2000, cacheDir: opts.cacheDir ?? configDir, cacheId: opts.cacheId ?? 'scrobble.cache', persistInterval: 1 * 1000 * 60, -- 2.51.2 From fd316a8e84ba343624f1dc07cfcf30f3559e3e78 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 9 Sep 2025 21:13:41 +0000 Subject: [PATCH 21/22] fix: Add full coverage for cached queue updating during client lifecycle --- .../scrobblers/AbstractScrobbleClient.ts | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 0abfea83..3fc0c3b1 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -804,12 +804,14 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); this.logger.warn(new Error(`Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${currQueuedPlay.source}' but error was not show stopping. Adding scrobble to Dead Letter Queue and will retry on next heartbeat.`, {cause: e})); } else { this.queuedScrobbles.unshift(currQueuedPlay); + this.updateQueuedScrobblesCache(); throw new Error('Error occurred while trying to scrobble', {cause: e}); } } } else if (!timeFrameValid) { this.logger.debug(`Will not scrobble ${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' because it ${timeFrameValidLog}`); } + this.updateQueuedScrobblesCache(); this.emitEvent('scrobbleDequeued', {queuedScrobble: currQueuedPlay}) } await sleep(this.scrobbleSleep); @@ -908,12 +910,12 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); } this.logger.info(`Removed scrobble ${buildTrackString(this.deadLetterScrobbles[index].play)} from queue`, {leaf: 'Dead Letter'}); this.deadLetterScrobbles.splice(index, 1); - this.cache.cacheScrobble.set(`${this.getMachineId()}-dead`, this.deadLetterScrobbles); + this.updateDeadLetterCache(); } removeDeadLetterScrobbles = () => { this.deadLetterScrobbles = []; - this.cache.cacheScrobble.set(`${this.getMachineId()}-dead`, []); + this.updateDeadLetterCache(); this.logger.info('Removed all scrobbles from queue', {leaf: 'Dead Letter'}); } @@ -933,7 +935,7 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); this.queuedScrobbles.push(queuedPlay); } this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); - this.cache.cacheScrobble.set(`${this.getMachineId()}-queue`, this.queuedScrobbles); + this.updateQueuedScrobblesCache(); } protected addDeadLetterScrobble = (data: QueuedScrobble, error: (Error | string) = 'Unspecified error') => { @@ -947,7 +949,7 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); this.deadLetterScrobbles.push(deadData); this.deadLetterScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); this.emitEvent('deadLetter', {dead: deadData}); - this.cache.cacheScrobble.set(`${this.getMachineId()}-dead`, this.deadLetterScrobbles); + this.updateDeadLetterCache(); } queuePlayingNow = (data: PlayObject, source: SourceIdentifier) => { @@ -1027,4 +1029,16 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); from: 'client' }); } + + protected updateDeadLetterCache = () => { + this.cache.cacheScrobble.set(`${this.getMachineId()}-dead`, this.deadLetterScrobbles) + .then(() => isDebugMode() ? this.logger.debug('Updated dead letter cache') : null) + .catch((e) => this.logger.warn(new Error('Error while updating dead letter cache', {cause: e}))); + } + + protected updateQueuedScrobblesCache = () => { + this.cache.cacheScrobble.set(`${this.getMachineId()}-queue`, this.queuedScrobbles) + .then(() => isDebugMode() ? this.logger.debug('Updated queued scrobble cache') : null) + .catch((e) => this.logger.warn(new Error('Error while updating queued scrobble cache', {cause: e}))); + } } -- 2.51.2 From 0a673fd33380c3f910dc24c10f3d16ddffa3dce9 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 10 Sep 2025 13:15:23 +0000 Subject: [PATCH 22/22] refactor: Decrease persist interval to 10 seconds Faster interval means less likely user will update queue and exit MS before cache is saved --- src/backend/common/Cache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index 70dddc81..33617e67 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -171,8 +171,8 @@ export const flatCacheCreate = (opts: FlatCacheOptions) => { lruSize: 2000, cacheDir: opts.cacheDir ?? configDir, cacheId: opts.cacheId ?? 'scrobble.cache', - persistInterval: 1 * 1000 * 60, - expirationInterval: 1 * 1000 * 60, // 1 minute + persistInterval: 1 * 1000 * 10, + expirationInterval: 1 * 1000 * 10, // 10 seconds ...opts }); } -- 2.51.2