diff --git a/package-lock.json b/package-lock.json index a2a4fd33..2edba090 100644 --- a/package-lock.json +++ b/package-lock.json @@ -88,6 +88,8 @@ "tough-cookie": "^5.1.2", "ts-json-schema-generator": "^2.3.0", "tsx": "^4.7.0", + "typeson": "^9.0.4", + "typeson-registry": "^11.1.1", "vite-express": "^0.16.0", "vlc-client": "^1.1.1", "xml2js": "0.6.1", @@ -3642,6 +3644,15 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-arraybuffer-es6": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/base64-arraybuffer-es6/-/base64-arraybuffer-es6-3.1.0.tgz", + "integrity": "sha512-QKKtftiSrKjilihGNLXxnrb9LJj7rnEdB1cYAqVpekFy0tisDklAf1RAgvpm0HsGYx9sv7FUbgpsrfwTyCPVLg==", + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -9392,7 +9403,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, "engines": { "node": ">=6" } @@ -11232,6 +11242,63 @@ "resolved": "https://registry.npmjs.org/typescript-event-target/-/typescript-event-target-1.1.1.tgz", "integrity": "sha512-dFSOFBKV6uwaloBCCUhxlD3Pr/P1a/tJdcmPrTXCHlEFD3faj0mztjcGn6VBAhQ0/Bdy8K3VWrrqwbt/ffsYsg==" }, + "node_modules/typeson": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/typeson/-/typeson-9.0.4.tgz", + "integrity": "sha512-umRYLe37m4fTu0AlgFaqu1C+N+i3LqAehCzbvNx9w7alyN1wpPyS6FbBEsvWD2FppS2yJ9gk0FkFBkZ/wJE13Q==", + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/typeson-registry": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/typeson-registry/-/typeson-registry-11.1.1.tgz", + "integrity": "sha512-WF8meNtURfGXXY2RitEMgZrMEclHU8IdxxN4YTcnXTQUHvisPYsKrxxPFhc4DKTCAc6sVvWVtKelOXFDtw/OQQ==", + "license": "MIT", + "dependencies": { + "base64-arraybuffer-es6": "^3.1.0", + "typeson": "^9.0.3", + "whatwg-url": "^14.0.0" + }, + "engines": { + "node": ">=18.14.0" + } + }, + "node_modules/typeson-registry/node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typeson-registry/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/typeson-registry/node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/uid-safe": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", diff --git a/package.json b/package.json index af071362..19f2778e 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,8 @@ "tough-cookie": "^5.1.2", "ts-json-schema-generator": "^2.3.0", "tsx": "^4.7.0", + "typeson": "^9.0.4", + "typeson-registry": "^11.1.1", "vite-express": "^0.16.0", "vlc-client": "^1.1.1", "xml2js": "0.6.1", diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index 4c8d4f1b..e82ee946 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -1,20 +1,23 @@ -import { Cacheable, CacheableMemory, Keyv, KeyvStoreAdapter } from 'cacheable'; -import { FlatCache } from 'flat-cache'; -import {parse} from 'flatted'; +import { Cacheable, createKeyv, Keyv, KeyvStoreAdapter, KeyvOptions, CacheableOptions } from 'cacheable'; +import { FlatCache, FlatCacheOptions } from 'flat-cache'; import KeyvValkey from '@keyv/valkey'; -import dayjs from 'dayjs'; +import dayjs, { Dayjs } from 'dayjs'; import duration from 'dayjs/plugin/duration.js'; import isBetween from 'dayjs/plugin/isBetween.js'; import relativeTime from 'dayjs/plugin/relativeTime.js'; import isToday from 'dayjs/plugin/isToday.js'; import timezone from 'dayjs/plugin/timezone.js'; import utc from 'dayjs/plugin/utc.js'; +import clone from 'clone'; import { childLogger, Logger } from '@foxxmd/logging'; import { projectDir } from './index.js'; import path from 'path'; import { fileOrDirectoryIsWriteable } from '../utils.js'; -import { asCacheMetadataProvider, asCacheScrobbleProvider, CacheConfig, CacheConfigOptions, CacheMetadaProvider, CacheProvider } from './infrastructure/Atomic.js'; - +import { asCacheMetadataProvider, asCacheScrobbleProvider, CacheConfig, CacheConfigOptions, CacheMetadaProvider, CacheProvider, CacheScrobbleProvider } from './infrastructure/Atomic.js'; +import { Typeson } from 'typeson'; +import { builtin } from 'typeson-registry'; +import { MaybeLogger } from './logging.js'; +import { ListenProgressPositional, ListenProgressTS } from '../sources/PlayerState/ListenProgress.js'; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); dayjs.extend(utc) @@ -24,6 +27,19 @@ dayjs.extend(duration); dayjs.extend(timezone); dayjs.extend(isToday); +const typeson = new Typeson().register([ + builtin, +]); +typeson.register({ + Dayjs: [ + (x) => dayjs.isDayjs(x), + (d: Dayjs) => d.toJSON(), + (date) => dayjs(date) + ], + ListenProgressTS, + ListenProgressPositional +}); + export class MSCache { config: Required @@ -43,7 +59,7 @@ export class MSCache { ...restMetadata } = {}, scrobble: { - provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheProvider | undefined) ?? 'file'), + provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'), connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir), ...restScrobble } = {}, @@ -64,82 +80,51 @@ export class MSCache { } init = async () => { - await this.initMetadataCache(); + // disabled for now + //await this.initMetadataCache(); await this.initScrobbleCache(); } protected initCacheable = async (config: CacheConfig, cacheFor: string) => { let logger = childLogger(this.logger, cacheFor); + const providerHints = ['In-Memory (Primary)']; + if(config.provider !== false) { + providerHints.push(`${config.provider} (Secondary)`) + } + logger.verbose(`Cache Providers: ${providerHints.join(' | ')}`) const ns = `ms-${cacheFor.toLocaleLowerCase()}`; - - const memoryCache = new CacheableMemory({ - ttl: '1h', - useClone: true, - lruSize: 200 - }); - const primaryCache = new Keyv({ store: memoryCache, namespace: ns }); - - let secondaryCache: Keyv | KeyvStoreAdapter | undefined; - - if (config.provider === 'valkey') { - logger.debug('Building valkey cache...'); - const valkey = new KeyvValkey(config.connection, {maxRetriesPerRequest: 5, connectTimeout: 1100}); - const metadataKv = new Keyv({ store: valkey, throwOnErrors: true, namespace: ns}); - try { - await metadataKv.get('test'); - secondaryCache = metadataKv; - logger.debug('valkey cache connected'); - } catch (e) { - this.logger.warn(new Error(`Unable to connect to cache ${config.connection}`, { cause: e })); - } - } else if(config.provider === 'file') { - logger.debug('Building file cache...'); - try { - fileOrDirectoryIsWriteable(config.connection); - } catch (e) { - logger.warn(new Error(`Unable to use path for file cache at ${config.connection}`, {cause: e})); - } - const flatCache = new FlatCache({ - ttl: 0, - lruSize: 500, - cacheDir: config.connection, - cacheId: 'scrobble.cache', - persistInterval: 1 * 1000 * 60, - expirationInterval: 1 * 1000 * 60, // 1 minute - // deserialize: (str) => { - // const data = parse(str) - // return data; - // } - }); - - let loadError: Error; - - const onlySaveError = (e: Error) => { - loadError = e; - } - flatCache.on('error', onlySaveError); - try { - logger.debug('Loading cache from file...'); - flatCache.load('scrobble.cache'); - if(loadError !== undefined) { - throw loadError; - } - logger.debug('File cache loaded.'); - flatCache.off('error', onlySaveError); - flatCache.on('error', (e) => { - logger.warn(e); - }); - flatCache.on('save', () => { - logger.debug('Saved cache to file'); - }); - } catch (e) { - logger.warn(new Error(`Unable to use file cache at ${path.join(config.connection, 'scrobble.cache')}`, {cause: e})); - } - secondaryCache = new Keyv({store: flatCache, throwOnErrors: true}); + + const cacheOpts: CacheableOptions = { + primary: initMemoryCache({ namespace: ns }) + } + + let secondaryCache: Keyv | KeyvStoreAdapter | undefined; + + if (config.provider === 'valkey') { + logger.debug(`Building valkey cache from ${config.connection}`); + try { + secondaryCache = await initValkeyCache(ns, config.connection); + logger.debug('valkey cache connected'); + } catch (e) { + this.logger.warn(e); + } + } else if (config.provider === 'file') { + logger.debug(`Building file cache from ${path.join(config.connection, `${ns}.cache`)}`); + + try { + const [keyvFile] = initFileCache({ cacheDir: config.connection, cacheId: `${ns}.cache` }, logger); + secondaryCache = keyvFile; + } catch (e) { + logger.warn(e); } - return new Cacheable({ primary: primaryCache, secondary: secondaryCache }); + } + + if(secondaryCache !== undefined) { + cacheOpts.secondary = secondaryCache; + } + return new Cacheable(cacheOpts); } @@ -162,4 +147,115 @@ export class MSCache { this.cacheMetadata = await this.initCacheable(this.config.metadata, 'Metadata'); } } +} + + +export const initMemoryCache = (opts: Parameters[0] = {}): Keyv | KeyvStoreAdapter => { + const memory = createKeyv({ + ttl: '1h', + lruSize: 200, + ...opts, + useClone: false, + }); + // structuredClone does not work well with dayjs https://github.com/iamkun/dayjs/issues/2236 + // but deep cloning is fine so disable useClone and provide our own cloning function + memory.serialize = (data) => { + return clone(data) as string; + } + return memory; +} + +export const flatCacheCreate = (opts: FlatCacheOptions) => { + return new FlatCache({ + ttl: 0, + lruSize: 500, + cacheDir: opts.cacheDir ?? configDir, + cacheId: opts.cacheId ?? 'scrobble.cache', + persistInterval: 1 * 1000 * 60, + expirationInterval: 1 * 1000 * 60, // 1 minute + ...opts + }); +} + +export const flatCacheLoad = (flatCache: FlatCache, logger: MaybeLogger): void => { + + const cachePath = path.join(flatCache.cacheDir, flatCache.cacheId); + try { + fileOrDirectoryIsWriteable(cachePath); + } catch (e) { + new Error(`Unable to use path for file cache at ${cachePath}`, { cause: e }) + } + + let loadError: Error; + + const onlySaveError = (e: Error) => { + loadError = e; + } + flatCache.on('error', onlySaveError); + try { + logger.debug('Loading cache from file...'); + flatCache.load(); + if (loadError !== undefined) { + throw loadError; + } + logger.debug(`File cache loaded`); + flatCache.off('error', onlySaveError); + } catch (e) { + throw new Error(`Unable to use file cache at ${cachePath}`, { cause: e }); + } +} + +export const initFileCache = (opts: FlatCacheOptions = {}, logger: MaybeLogger = new MaybeLogger()): [Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined] => { + const flatCache = flatCacheCreate(opts); + try { + flatCacheLoad(flatCache, logger); + flatCache.on('error', (e) => { + logger.warn(e); + }); + flatCache.on('save', () => { + logger.debug('Saved cache to file'); + }); + + const cache = new Keyv({ + store: flatCache, + throwOnErrors: true, + ...typesonMarshalling + }); + return [cache, flatCache]; + } catch (e) { + throw e; + } +} + +export const valkeyCacheCreate = (ns: string, ...args: ConstructorParameters): Keyv => { + const [connection, valkeyOpts = {}] = args; + const valkey = new KeyvValkey(connection, { maxRetriesPerRequest: 5, connectTimeout: 1100, ...valkeyOpts }); + const kv = new Keyv({ + store: valkey, + throwOnErrors: true, + namespace: ns, + ...typesonMarshalling + }); + return kv; +} + +export const initValkeyCache = async (ns: string, ...args: ConstructorParameters): Promise => { + const kv = valkeyCacheCreate(ns, ...args); + try { + await kv.get('test'); + return kv; + } catch (e) { + throw new Error(`Unable to connect to cache ${args[0]}`, { cause: e }) + } +} + +const typesonMarshalling: Pick = { + serialize: (data) => { + const str = typeson.stringifySync(data); + return str; + }, + deserialize: (str) => { + const data = typeson.parseSync(str); + return data; + } } \ No newline at end of file diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 2c7f3dc4..e73818fa 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -349,21 +349,23 @@ export type WhenConditionsConfig = WhenConditions; export type WithRequiredProperty = Type & { [Property in Key]-?: Type[Property]; }; -export type CacheProvider = 'memory' | 'valkey' | 'file'; +export type CacheProvider = 'memory' | 'valkey' | 'file' | false; export interface CacheConfig { provider: T; connection?: string; } -export type CacheMetadaProvider = Exclude; +export type CacheMetadaProvider = CacheProvider;//Exclude; export type CacheMetadataConfig = CacheConfig; -export const asCacheMetadataProvider = (val: string): val is CacheScrobbleProvider => { - return ['memory', 'valkey'].includes(val); -}; +export const asCacheProvider = (val: boolean | string): val is CacheProvider => { + if(typeof val === 'string') { + return ['memory', 'valkey', 'file'].includes(val); + } + return val === false; +} +export const asCacheMetadataProvider = (val: any): val is CacheScrobbleProvider => asCacheProvider(val); export type CacheScrobbleProvider = CacheProvider; export type CacheScrobbleConfig = CacheConfig; -export const asCacheScrobbleProvider = (val: string): val is CacheScrobbleProvider => { - return ['memory', 'valkey', 'file'].includes(val); -}; +export const asCacheScrobbleProvider = (val: any): val is CacheScrobbleProvider => asCacheProvider(val); export interface CacheConfigOptions { metadata?: CacheMetadataConfig; scrobble?: CacheScrobbleConfig; diff --git a/src/backend/tests/cache/cache.test.ts b/src/backend/tests/cache/cache.test.ts new file mode 100644 index 00000000..580be240 --- /dev/null +++ b/src/backend/tests/cache/cache.test.ts @@ -0,0 +1,171 @@ +import { loggerTest, loggerDebug } from "@foxxmd/logging"; +import chai, { assert, expect } from 'chai'; +import asPromised from 'chai-as-promised'; +import { after, before, describe, it } from 'mocha'; +import dayjs from "dayjs"; +import withLocalTmpDir from 'with-local-tmp-dir'; +import path from 'path'; +import { initFileCache, initMemoryCache, initValkeyCache, MSCache } from "../../common/Cache.js"; +import { generatePlay, generatePlayerStateData, generatePlays, normalizePlays } from "../utils/PlayTestUtils.js"; +import { ListenProgressPositional, ListenProgressTS } from "../../sources/PlayerState/ListenProgress.js"; +import { isPortReachableConnect } from "../../utils/NetworkUtils.js"; + +chai.use(asPromised); + +describe('#Caching', function () { + + describe('#MemoryCaching', function () { + + it('Memory cache preserves dayjs', async function () { + + const cache = initMemoryCache(); + + const now = dayjs(); + + await cache.set('foo', now); + + const time = await cache.get('foo'); + + expect(time).to.not.be.undefined; + expect(dayjs.isDayjs(time)).is.true; + expect(now.toJSON()).eq((time as any).toJSON()); + }); + + it('Memory cache preserves ListenProgress', async function () { + + const cache = initMemoryCache(); + + const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 }); + + await cache.set('foo', prog); + + const cachedProg = await cache.get('foo') as ListenProgressTS; + + expect(cachedProg).to.not.be.undefined; + expect(cachedProg instanceof ListenProgressTS).is.true; + expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON()); + }); + + }); + + describe('#FileCaching', function () { + + it('File cache serializes and deserializes dayjs', async function () { + + withLocalTmpDir(async () => { + + const [keyv, flat] = initFileCache({ cacheDir: process.cwd() }); + + const now = dayjs(); + + await keyv.set('foo', now); + flat.save(); + + const [cleanKeyv, cleanFlat] = initFileCache({ cacheDir: process.cwd() }); + + const time = await cleanKeyv.get('foo'); + + expect(time).to.not.be.undefined; + expect(time instanceof dayjs).is.true; + expect(now.toJSON()).eq((time as any).toJSON()); + + }, { unsafeCleanup: true }); + }); + + it('File cache serializes and deserializes ListenProgress', async function () { + + withLocalTmpDir(async () => { + + const [keyv, flat] = initFileCache({ cacheDir: process.cwd() }); + + const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 }); + + await keyv.set('foo', prog); + await flat.save(); + + const [cleanKeyv, cleanFlat] = initFileCache({ cacheDir: process.cwd() }); + + const cachedProg = await cleanKeyv.get('foo'); + + expect(cachedProg).to.not.be.undefined; + expect(cachedProg instanceof ListenProgressTS).is.true; + expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON()); + + }, { unsafeCleanup: true }); + }); + }); + + describe('#ValkeyCaching', function () { + before(async function () { + try { + await isPortReachableConnect(6379, { host: 'valkey' }); + } catch (e) { + // don't run valkey tests if valkey isn't present + this.skip(); + } + }); + + it('Valkey cache serializes and deserializes dayjs', async function () { + + const keyv = await initValkeyCache('test', 'redis://valkey:6379'); + await keyv.clear(); + + const now = dayjs(); + + await keyv.set('foo', now); + + const time = await keyv.get('foo'); + + expect(time).to.not.be.undefined; + expect(time instanceof dayjs).is.true; + expect(now.toJSON()).eq((time as any).toJSON()); + + }); + + it('Valkey cache serializes and deserializes ListenProgress', async function () { + + const keyv = await initValkeyCache('test', 'redis://valkey:6379'); + await keyv.clear(); + + const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 }); + + await keyv.set('foo', prog); + + const cachedProg = await keyv.get('foo'); + + expect(cachedProg).to.not.be.undefined; + expect(cachedProg instanceof ListenProgressTS).is.true; + expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON()); + + }); + }); + + + + + // it('File backend serializes and deserializes non-primitives', async function() { + + // withLocalTmpDir(async () => { + + // const cache = new MSCache(loggerTest, {scrobble: {provider: 'file', connection: process.cwd()}}); + // await cache.init(); + + // const now = dayjs(); + + // await cache.cacheScrobble.set('foo', now); + // await cache.cacheScrobble.secondary.store.save(); + // await cache.cacheScrobble.disconnect() + + // const cleanCache = new MSCache(loggerTest, {scrobble: {provider: 'file', connection: process.cwd()}}); + // await cleanCache.init(); + + // const time = await cleanCache.cacheScrobble.get('foo'); + + // expect(time).to.not.be.undefined; + // expect(time instanceof dayjs).is.true; + // expect(now.toJSON).eq((time as any).toJSON()); + + // }, {unsafeCleanup: true}); + // }); + +});