diff --git a/docsite/docs/configuration/configuration.mdx b/docsite/docs/configuration/configuration.mdx index 13ce858a..9ffb4941 100644 --- a/docsite/docs/configuration/configuration.mdx +++ b/docsite/docs/configuration/configuration.mdx @@ -245,11 +245,8 @@ WARN : [App] [Sources] [spotify Secrets] Matched: None | Unmatched: SPOTIFY_SECR -## Application Options -These options affect multi-scrobbler's behavior and are not specific to any source/client. - -### Base URL +## Base URL Defines the URL that is used to generate default redirect URLs for authentication on [spotify](/configuration/sources/spotify) and [lastfm](/configuration/clients/lastfm) -- as well as some logging hints. @@ -280,185 +277,82 @@ Useful when running with [docker](../installation/installation.mdx#docker) so th -### Caching - -Multi-scrobbler caches some activities to persist important data across restarts, reduce external API calls, and make some actions faster. - -All of the activities below are **always** cached **in-memory** with an optional, configurable [**secondary** store](#secondary-caching-configuration) for persistence. - - - - -**Queued** and **Failed** Scrobbles are cached so that any un-scrobbled data you have is persisted across restarts of multi-scrobbler. - -:::tip - -By default, this data use a [Secondary](#secondary-caching-configuration) [File](./?cacheType=file#secondary-caching-configuration) store, configured for you automatically. - -If you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)) then you are already done. If you are not persisting this directory then you should consider setting up [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration) for this. - -::: - -##### Configuration - -Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values: - - - - -| Environmental Variable | Required? | Default | Description | -| :--------------------- | --------- | --------- | --------------------- | -| `CACHE_SCROBBLE` | No | `file` | The cache type to use | -| `CACHE_SCROBBLE_CONN` | No | `/config` | | +## Caching - +Multi-scrobbler implements caching to persist important data across restarts, reduce external API calls, and make some actions faster. - +A default **in-memory** cache store is used so that you always benefit from some caching. An optional, [**secondary** store](#secondary-caching) can be configured for greater caching capabilities. -```json5 title="config.json" -{ - "cache": { - "scrobble": { - "provider": "file", - "connection": "/config" - } - }, - // ... -} -``` - - + - + - Authentication sessions/tokens/etc... are cached for quicker requests and for persistence across restarts. - -:::tip - -By default, this data use a [Secondary](#secondary-caching-configuration) [File](./?cacheType=file#secondary-caching-configuration) store, configured for you automatically. - -If you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)) then you are already done. If you are not persisting this directory then you should consider setting up [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration) for this. - -::: - -##### Configuration - -Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values: - - - - -| Environmental Variable | Required? | Default | Description | -| :--------------------- | --------- | --------- | --------------------- | -| `CACHE_AUTH` | No | `file` | The cache type to use | -| `CACHE_AUTH_CONN` | No | `/config` | | - - - - -```json5 title="config.json" -{ - "cache": { - "auth": { - "provider": "file", - "connection": "/config" - } - }, - // ... -} -``` - - - + +The results of [transform rules](/configuration/transforms) are cached so that if a scrobble with identical data (track/artists/album) is identified and it has the same set of transform rules then the cached transform results can be applied. - API Calls to external (metadata) services used to [Enhance Scrobbles](/configuration/transforms), like calls to [Musicbrainz](/configuration/transforms/musicbrainz), can be cached to avoid duplicate calls and speed up scrobble transformations. - -By default, these calls are only cached in memory. If you wish for cached calls to be persisted across restarts then setup [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration). - -##### Configuration - -Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values: - - - - -| Environmental Variable | Required? | Default | Description | -| :--------------------- | --------- | ------- | --------------------- | -| `CACHE_METADATA` | No | | The cache type to use | -| `CACHE_METADATA_CONN` | No | | | - - - - - -```json5 title="config.json" -{ - "cache": { - "metadata": { - "provider": "valkey", - "connection": "yourConnectionStringHere" - } - }, - // ... -} -``` - - + -#### Secondary Caching Configuration + -The type of cache used, and its connection properties, can be configured through **ENV** or **AIO** config. +Auth caching defaults to a **file** that is stored in the `CONFIG_DIR` directory using the pre-defined file name `ms-auth.cache`. - +This provides automatic persistence across restarts for long-lived auth data/credentials if you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)). - +If you wish to use the [secondary store](#secondary-caching) for caching Auth you must explicitly configure it. This is because valkey can potentially be *ephemeral* if you do not provide a volume for its data directory. -**File** cache is stored in the `CONFIG_DIR` directory using the pre-defined file name `ms-[cacheName].cache`. +To explicitly configure auth to use the secondary store: - -Example - -| Environmental Variable | Required? | Default | Description | -| :--------------------- | --------- | --------- | ------------------------------------------------------------ | -| `CACHE_SCROBBLE` | No | `file` | The cache type to use | -| `CACHE_SCROBBLE_CONN` | No | `/config` | The directory, within the container, to store the cache file | +```yaml title="compose.yaml" +services: + multi-scrobbler: + # ... + environment: + // highlight-start + - CACHE_AUTH=valkey + // highlight-end + # ... +``` -Example - ```json5 title="config.json" { "cache": { - "scrobble": { - "provider": "file", - "connection": "/config" + "auth": { + "provider": "valkey" } }, // ... } ``` - - - - + + +### Secondary Caching Store {#secondary-caching} + +Using a secondary store enables: -[**Valkey**](https://valkey.io/) is an open-source fork of Redis. +* persistence of cached data across restarts +* a larger store (more data is saved) +* a longer time-to-live in the store (cached data is fetchable for a longer period) + +These benefits are particularly beneficial when using [transforms](/configuration/transforms) like Musicbrainz and it is **strongly recommended** for these scenarios. + +Currently, Multi-scrobbler only supports [**Valkey**](https://valkey.io/), an open-source fork of Redis, as a secondary store.
@@ -466,11 +360,13 @@ Example A valkey container can be added to the [multi-scrobbler docker compose stack](/installation?runType=docker-compose#docker): + + + ```yaml title="docker-compose.yml" services: multi-scrobbler: # ... - # adding everything below // highlight-start valkey: image: valkey/valkey @@ -483,6 +379,25 @@ volumes: // highlight-end ``` + + + +```yaml title="docker-compose.yml" +services: + multi-scrobbler: + # ... + // highlight-start + valkey: + image: valkey/valkey + volumes: + - ./valkeyData:/data + // highlight-end +``` + + + + + Use `redis://valkey:6379` as the connection string in the configurations below.
@@ -497,12 +412,16 @@ redis://HOST_IP:HOST_PORT -Example - -| Environmental Variable | Required? | Default | Description | -| :--------------------- | --------- | -------- | ------------------------------------------------------------------- | -| `CACHE_METADATA` | Yes | `valkey` | The cache type to use | -| `CACHE_METADATA_CONN` | Yes | | The host/IP and port to connect to EX: `redis://192.168.0.120:6379` | +```yaml title="compose.yaml" +services: + multi-scrobbler: + # ... + environment: + // highlight-start + - CACHE_VALKEY=redis://192.168.0.120:6379 + // highlight-end + # ... +``` @@ -513,10 +432,7 @@ Example ```json5 title="config.json" { "cache": { - "metadata": { - "provider": "valkey", - "connection": "redis://192.168.0.120:6379" - } + "valkey": "redis://192.168.0.120:6379" }, // ... } @@ -524,12 +440,9 @@ Example - - - -### Debug Mode +## Debug Mode Turning on Debug Mode will @@ -550,7 +463,7 @@ To set debug mode either add it to [AIO `config.json`](./?configType=aio#configu or set the [ENV](./?configType=env#configuration-types) `DEBUG_MODE=true` -### Disable Web +## Disable Web If you do not need the dashboard and/or ingress sources, or have security concerns about ingress and cannot control your hosting environment, the web server and API can be disabled. diff --git a/docsite/docs/configuration/transforms/transforms.mdx b/docsite/docs/configuration/transforms/transforms.mdx index be1a3bca..b5780c69 100644 --- a/docsite/docs/configuration/transforms/transforms.mdx +++ b/docsite/docs/configuration/transforms/transforms.mdx @@ -591,7 +591,7 @@ The output shows the diff between the previous stage (or original Play) and the MS uses [caching](/configuration/#caching) to reduce the number of API calls needed for stages like [Musicbrainz](/configuration/transforms/musicbrainz) and to speed up all transforms by caching steps and results. However, the default caching strategy uses a small cache size and very short [TTLs](https://en.wikipedia.org/wiki/Time_to_live) because it is *in-memory*. -**If you are using any Transform stages you should configure [secondary caching with Valkey for Metadata](/configuration/?cacheType=valkey&cachedThings=metadata#secondary-caching-configuration)** to increase the cache size and lifetime of cached items. This will also reduce memory usage in MS. +**If you are using any Transform stages you should configure [secondary caching](/configuration#secondary-caching)** to increase the cache size and lifetime of cached items. This will also reduce memory usage in MS.
diff --git a/docsite/docs/installation/installation.mdx b/docsite/docs/installation/installation.mdx index 83f6f9bb..93a29296 100644 --- a/docsite/docs/installation/installation.mdx +++ b/docsite/docs/installation/installation.mdx @@ -132,7 +132,7 @@ services: -**Optionally**, add a [Valkey](https://valkey.io/) service to your stack for [secondary caching](/configuration/?cacheType=valkey#secondary-caching-configuration) to take advantage of faster performance and reduced memory usage. +**Optionally**, add a [Valkey](https://valkey.io/) service to your stack for [secondary caching](/configuration#secondary-caching) to take advantage of faster performance and reduced memory usage. ```yaml title="docker-compose.yml" services: @@ -142,8 +142,7 @@ services: environment: # ... // highlight-start - CACHE_METADATA=valkey - CACHE_METADATA_CONN=redis://valkey:6379 + CACHE_VALKEY=redis://valkey:6379 // highlight-end # ... diff --git a/docsite/docs/quickstart.mdx b/docsite/docs/quickstart.mdx index 140d106e..f38abac6 100644 --- a/docsite/docs/quickstart.mdx +++ b/docsite/docs/quickstart.mdx @@ -336,7 +336,7 @@ Visit `http://192.168.0.100:9078` to see the dashboard where ## Next Steps * See more advanced docker options as well as other install methods in the [**Installation**](/installation#docker) docs - * Setup [secondary caching](/configuration/?cacheType=valkey#secondary-caching-configuration) with [valkey](/installation?dockerSetting=caching#recommended-settings) for increased performance and reduced memory usage + * Setup [secondary caching](/configuration#secondary-caching) with [valkey](/installation?dockerSetting=caching#recommended-settings) for increased performance and reduced memory usage * Review the [**Configuration**](/configuration) docs * Learn about how to configure multi-scrobbler using files for more complicated Source/Client scenarios * See all available [**Sources**](/configuration/sources) and [**Clients**](/configuration/clients) alongside configuration examples diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index a1f19189..b0ed73a6 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -15,13 +15,14 @@ import path from 'path'; import { cacheFunctions } from "@foxxmd/regex-buddy-core"; import { fileExists, fileOrDirectoryIsWriteable } from '../utils/FSUtils.js'; import { copyFile } from 'fs/promises'; -import { asCacheAuthProvider, asCacheConfig, asCacheMetadataProvider, asCacheScrobbleProvider, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheMetadataProvider, CacheProvider, CacheScrobbleProvider } from './infrastructure/Atomic.js'; +import { asCacheConfig, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheConfigUser, CacheScrobbleProvider } from './infrastructure/Atomic.js'; import { Typeson } from 'typeson'; import { builtin } from 'typeson-registry'; import { loggerNoop } from './MaybeLogger.js'; import { ListenProgressPositional, ListenProgressTS } from '../sources/PlayerState/ListenProgress.js'; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); import prom, { Gauge } from 'prom-client'; +import { nonEmptyStringOrDefault } from '../../core/StringUtils.js'; dayjs.extend(utc) dayjs.extend(isBetween); @@ -43,6 +44,14 @@ typeson.register({ ListenProgressPositional }); +const unsupportedEnvKeys = [ + 'CACHE_AUTH_CONN', + 'CACHE_METADATA', + 'CACHE_SCROBBLE', + 'CACHE_SCROBBLE_CONN', + 'CACHE_AUTH_CONN' +]; + export class MSCache { config: Required @@ -70,19 +79,19 @@ export class MSCache { const { metadata: { - provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadataProvider | undefined) ?? false), - connection: mConn = process.env.CACHE_METADATA_CONN, - ...restMetadata + provider: mProvider = false, + connection: mConn, + //...restMetadata } = {}, scrobble: { - provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'), - connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir), - ...restScrobble + provider: sProvider = false, + connection: sConnection, + //...restScrobble } = {}, auth: { - provider: aProvider = (process.env.CACHE_AUTH as (CacheAuthProvider | undefined) ?? 'file'), - connection: aConn = (process.env.CACHE_AUTH_CONN ?? configDir), - ...restAuth + provider: aProvider = false, + connection: aConn, + //...restAuth } = {}, regex = 200, } = config; @@ -91,17 +100,14 @@ export class MSCache { metadata: { provider: mProvider, connection: mConn, - ...restMetadata, }, scrobble: { provider: sProvider, - connection, - ...restScrobble + connection: sConnection, }, auth: { provider: aProvider, connection: aConn, - ...restAuth }, regex }; @@ -275,10 +281,10 @@ export class MSCache { } } if (config.provider === 'file') { - logger.debug(`Building file cache from ${path.join(config.connection, `${namespace}.cache`)}`); + logger.debug(`Building file cache from ${path.join(config.connection ?? configDir, `${namespace}.cache`)}`); try { - const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger); + const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection ?? configDir, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger); return keyvFile; } catch (e) { throw e; @@ -368,8 +374,8 @@ export const flatCacheCreate = (opts: FlatCacheOptions) => { return new FlatCache({ ttl: 0, lruSize: 2000, - cacheDir: opts.cacheDir ?? configDir, - cacheId: opts.cacheId ?? 'scrobble.cache', + cacheDir: opts.cacheDir, + cacheId: opts.cacheId ?? 'ms.cache', persistInterval: 1 * 1000 * 10, expirationInterval: 1 * 1000 * 10, // 10 seconds ...opts @@ -507,4 +513,86 @@ const noopKeyv: KeyvStoreAdapter = { delete: (_) => undefined, clear: () => Promise.resolve(), on: (_, __) => undefined +} + +export const parseUserConfig = (config: CacheConfigUser = {}, parentLogger: Logger = loggerNoop): CacheConfigOptions => { + const logger = childLogger(parentLogger, 'Cache'); + + let valkeyEnvVal: string | undefined = nonEmptyStringOrDefault(process.env.CACHE_VALKEY); + if(valkeyEnvVal === undefined) { + valkeyEnvVal = nonEmptyStringOrDefault(process.env.CACHE_METADATA_CONN); + if(valkeyEnvVal !== undefined) { + logger.warn('ENV CACHE_METADATA_CONN is deprecated! Replace it with CACHE_VALKEY'); + } + } + + for(const key of unsupportedEnvKeys) { + if(nonEmptyStringOrDefault(process.env[key]) !== undefined) { + logger.warn(`ENV ${key} is no longer supported. Refer to the Caching docs.`); + } + } + + const { + valkey = valkeyEnvVal, + // metadata: { + // provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadataProvider | undefined) ?? false), + // connection: mConn = process.env.CACHE_METADATA_CONN, + // //...restMetadata + // } = {}, + scrobble: { + provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'), + connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir), + ...restScrobble + } = {}, + auth: { + provider: aProvider = (process.env.CACHE_AUTH as (CacheAuthProvider | undefined) ?? 'file'), + //...restAuth + } = {}, + regex = 200, + } = config; + + if(config.metadata !== undefined) { + logger.warn('Configuring cache.metadata is no longer supported. Refer to the Caching docs.'); + } + if(config.scrobble !== undefined) { + logger.warn('Configuring cache.scrobble is no longer supported. Refer to the Caching docs.'); + } + if(config.auth?.connection !== undefined) { + logger.warn('Configuring cache.auth.connection is no longer supported. Refer to the Caching docs.'); + } + + let authConn: string, + authProvider = aProvider; + if(authProvider === 'valkey') { + if(valkey === undefined) { + logger.warn(`Auth Provider set to 'valkey' but not valkey connection string was not provided, falling back to file.`); + authConn = configDir; + authProvider = 'file'; + } else { + authConn = valkey; + } + } else { + if(authProvider !== 'file') { + logger.warn(`Unsupported provider given for auth: ${authProvider}`); + } + authConn = configDir; + authProvider = 'file'; + } + + return { + metadata: { + provider: valkey !== undefined ? 'valkey' : false, + connection: valkey, + }, + scrobble: { + provider: sProvider, + connection, + ...restScrobble + }, + auth: { + provider: authProvider, + connection: authConn, + }, + regex + }; } \ No newline at end of file diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index b5f869f3..eb607941 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -296,6 +296,21 @@ export interface CacheConfigOptions { regex?: number } +export interface CacheConfigUser { + auth?: { + provider: 'valkey' | 'file', + [key: string]: any + }; + valkey?: string + /** Number of regex functions to cache (LRU) + * + * @default 200 + */ + regex?: number + // to allow deprecated scrobble config without having it show up in schema docs + [key: string]: any +} + export interface MusicbrainzApiConfigData { url?: string contact: string, diff --git a/src/backend/common/infrastructure/config/aioConfig.ts b/src/backend/common/infrastructure/config/aioConfig.ts index 51c11778..6b7f9dfd 100644 --- a/src/backend/common/infrastructure/config/aioConfig.ts +++ b/src/backend/common/infrastructure/config/aioConfig.ts @@ -5,7 +5,7 @@ import { RequestRetryOptions } from "./common.js"; import { WebhookConfig } from "./health/webhooks.js"; import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js"; import { SourceAIOConfig } from "./source/sources.js"; -import { CacheConfigOptions, DurationValue } from "../Atomic.js"; +import { CacheConfigOptions, CacheConfigUser, DurationValue } from "../Atomic.js"; import { TransformerCommonConfig } from "../../../../core/Atomic.js"; import { RetentionConfig } from "./database.js"; @@ -67,7 +67,7 @@ export interface AIOConfig { * */ debugMode?: boolean - cache?: CacheConfigOptions + cache?: CacheConfigUser transformers?: TransformerCommonConfig[] diff --git a/src/backend/index.ts b/src/backend/index.ts index d5e9615a..0f1e657c 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -24,6 +24,7 @@ import { Notifiers } from './notifier/Notifiers.js'; import { getDb, performDbMigrationWithBackup } from './common/database/drizzle/drizzleUtils.js'; import { getDbPath } from './common/database/Database.js'; import { createRetentionCleanupTask } from './tasks/retentionCleanup.js'; +import { parseUserConfig } from './common/Cache.js'; dayjs.extend(utc) dayjs.extend(isBetween); @@ -75,6 +76,7 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) webhooks = [], logging = {}, debugMode, + cache, } = (config || {}) as AIOConfig; if (process.env.DEBUG_MODE === undefined && debugMode !== undefined) { @@ -100,6 +102,7 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) const root = getRoot({ ...config, + cache: parseUserConfig(cache, logger), logger, loggingConfig: logging, loggerStream: appLoggerStream, diff --git a/src/backend/tests/utils/TransientTestUtils.ts b/src/backend/tests/utils/TransientTestUtils.ts index 3f8f51da..713f8dd4 100644 --- a/src/backend/tests/utils/TransientTestUtils.ts +++ b/src/backend/tests/utils/TransientTestUtils.ts @@ -2,7 +2,7 @@ import { loggerTest } from "@foxxmd/logging"; import { MSCache } from "../../common/Cache.js"; import { getDb, migrateDbSync } from "../../common/database/drizzle/drizzleUtils.js"; -export const transientCache = () => new MSCache(loggerTest, { scrobble: { provider: 'memory' }, auth: { provider: 'memory' }, metadata: { provider: 'memory' } }); +export const transientCache = () => new MSCache(loggerTest); export const transientDb = () => { const db = getDb(':memory:');