diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index f3197853..45c19fc0 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -5,187 +5,43 @@ import { import deepEqual from 'fast-deep-equal'; import { Simulate } from "react-dom/test-utils"; import { PlayObject } from "../../core/Atomic.js"; -import { buildTrackString, truncateStringToLength } from "../../core/StringUtils.js"; +import { buildTrackString } from "../../core/StringUtils.js"; import { - configPartsToStrongParts, countRegexes, - isUserStage, - transformPlayUsingParts + configPartsToStrongParts, countRegexes, transformPlayUsingParts } from "../utils/PlayTransformUtils.js"; -import { hasNodeNetworkException } from "./errors/NodeErrors.js"; -import { hasUpstreamError } from "./errors/UpstreamError.js"; import { CommonClientConfig } from "./infrastructure/config/client/index.js"; import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; -import play = Simulate.play; -import { WebhookPayload } from "./infrastructure/config/health/webhooks.js"; -import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, TransformRulesError } from "./errors/MSErrors.js"; -import { messageWithCauses, messageWithCausesTruncatedDefault } from "../utils/ErrorUtils.js"; +import { TransformRulesError } from "./errors/MSErrors.js"; import { - ConditionalSearchAndReplaceRegExp, - PlayTransformParts, - PlayTransformPartsArray, + ConditionalSearchAndReplaceRegExp, PlayTransformPartsArray, PlayTransformRules, TRANSFORM_HOOK, TransformHook } from "./infrastructure/Transform.js"; +import AbstractInitializable from "./AbstractInitializable.js"; +import play = Simulate.play; -export default abstract class AbstractComponent { - requiresAuth: boolean = false; - requiresAuthInteraction: boolean = false; - authed: boolean = false; - authFailure?: boolean; - - buildOK?: boolean | null; - connectionOK?: boolean | null; - cacheOK?: boolean | null; - - initializing: boolean = false; +export default abstract class AbstractComponent extends AbstractInitializable { - config: CommonClientConfig | CommonSourceConfig; + declare config: CommonClientConfig | CommonSourceConfig; transformRules: PlayTransformRules = {}; regexCache!: ReturnType; - logger: Logger; - componentLogger?: Logger; - protected constructor(config: CommonClientConfig | CommonSourceConfig) { - this.config = config; + super(config); } - public abstract notify(payload: WebhookPayload): Promise; - - protected abstract getIdentifier(): string; - - initialize = async (options: {force?: boolean, notify?: boolean, notifyTitle?: string} = {}) => { - - const {force = false, notify = false, notifyTitle = 'Init Error'} = options; - - this.logger.debug('Attempting to initialize...'); + protected postCache(): Promise { try { - this.initializing = true; - if(this.componentLogger === undefined) { - await this.buildComponentLogger(); - } - await this.buildInitData(force); - await this.parseCache(force); this.buildTransformRules(); - await this.checkConnection(force); - await this.testAuth(force); - this.logger.info('Fully Initialized!'); - try { - await this.postInitialize(); - } catch (e) { - throw new PostInitError('Error occurred during post-initialization hook', {cause: e}); - } - return true; - } catch(e) { - if(notify) { - await this.notify({title: `${this.getIdentifier()} - ${notifyTitle}`, message: truncateStringToLength(500)(messageWithCausesTruncatedDefault(e)), priority: 'error'}); - } - throw new Error('Initialization failed', {cause: e}); - } finally { - this.initializing = false; - } - } - - private async buildComponentLogger() { - await this.doBuildComponentLogger(); - return; - } - - protected async doBuildComponentLogger() { - return; - } - - tryInitialize = async (options: {force?: boolean, notify?: boolean, notifyTitle?: string} = {}) => { - if(this.initializing) { - throw new Error(`Already trying to initialize, cannot attempt while an existing initialization attempt is running.`) - } - try { - return await this.initialize(options); + return; } catch (e) { throw e; } } - public async parseCache(force: boolean = false) { - if(this.cacheOK) { - if(!force) { - return; - } - this.logger.debug('Cache OK but step was forced'); - } - try { - const res = await this.doParseCache(); - if(res === undefined) { - this.cacheOK = null; - this.logger.debug('No cache to parse.'); - return; - } - if (res === true) { - this.logger.verbose('Parsing caching succeeded'); - } else if (typeof res === 'string') { - this.logger.verbose(`Parsing caching succeeded => ${res}`); - } - this.cacheOK = true; - } catch (e) { - this.cacheOK = false; - throw new ParseCacheError('Parsing cache for initialization failed', {cause: e}); - } - } - - /** - * Build or parse any cache required for this Component - * - * * Return undefined if not possible or not required - * * Return TRUE if build succeeded - * * Return string if build succeeded and should log result - * * Throw error on failure - * */ - protected async doParseCache(): Promise { - return; - } - - - public async buildInitData(force: boolean = false) { - if(this.buildOK) { - if(!force) { - return; - } - this.logger.debug('Build OK but step was forced'); - } - try { - const res = await this.doBuildInitData(); - if(res === undefined) { - this.buildOK = null; - this.logger.debug('No required data to build.'); - return; - } - if (res === true) { - this.logger.verbose('Building required data init succeeded'); - } else if (typeof res === 'string') { - this.logger.verbose(`Building required data init succeeded => ${res}`); - } - this.buildOK = true; - } catch (e) { - this.buildOK = false; - throw new BuildDataError('Building required data for initialization failed', {cause: e}); - } - } - - /** - * Build any data/config/objects required for this Source to communicate with upstream service - * - * * Return undefined if not possible or not required - * * Return TRUE if build succeeded - * * Return string if build succeeded and should log result - * * Throw error on failure - * */ - protected async doBuildInitData(): Promise { - return; - } - public buildTransformRules() { try { this.doBuildTransformRules(); @@ -260,104 +116,6 @@ export default abstract class AbstractComponent { } } - public async checkConnection(force: boolean = false) { - if(this.connectionOK) { - if(!force) { - return; - } - this.logger.debug('Connection OK but step was forced') - } - try { - const res = await this.doCheckConnection(); - if (res === undefined) { - this.logger.debug('Connection check was not required.'); - this.connectionOK = null; - return; - } else if (res === true) { - this.logger.verbose('Connection check succeeded'); - } else { - this.logger.verbose(`Connection check succeeded => ${res}`); - } - this.connectionOK = true; - } catch (e) { - this.connectionOK = false; - throw new ConnectionCheckError('Communicating with upstream service failed', {cause: e}); - } - } - - /** - * Check Scrobbler upstream API/connection to ensure we can communicate - * - * * Return undefined if not possible or not required to check - * * Return TRUE if communication succeeded - * * Return string if communication succeeded and should log result - * * Throw error if communication failed - * */ - protected async doCheckConnection(): Promise { - return; - } - - authGated = () => this.requiresAuth && !this.authed - - canTryAuth = () => this.isUsable() && this.authGated() && this.authFailure !== true - - canAuthUnattended = () => !this.authGated || !this.requiresAuthInteraction || (this.requiresAuthInteraction && !this.authFailure); - - protected doAuthentication = async (): Promise => this.authed - - // default init function, should be overridden if auth stage is required - testAuth = async (force: boolean = false) => { - if(!this.requiresAuth) { - return; - } - if(this.authed) { - if(!force) { - return; - } - this.logger.debug('Auth OK but step was forced'); - } - - if(this.authFailure) { - if(!force) { - if(this.requiresAuthInteraction) { - throw new AuthCheckError('Authentication failure: Will not retry auth because user interaction is required for authentication'); - } - throw new AuthCheckError('Authentication failure: Will not retry auth because authentication previously failed and must be reauthenticated'); - } - this.logger.debug('Auth previously failed for non upstream/network reasons but retry is being forced'); - } - - try { - this.authed = await this.doAuthentication(); - this.authFailure = !this.authed; - } catch (e) { - // only signal as auth failure if error was NOT either a node network error or a non-showstopping upstream error - this.authFailure = !(hasNodeNetworkException(e) || hasUpstreamError(e, false)); - this.authed = false; - throw new AuthCheckError(`Authentication test failed!${this.authFailure === false ? ' Due to a network issue. Will retry authentication on next heartbeat.' : ''}`, {cause: e}); - } - } - - public isReady() { - return (this.buildOK === null || this.buildOK === true) && - (this.connectionOK === null || this.connectionOK === true) - && !this.authGated(); - } - - public isUsable() { - return (this.buildOK === null || this.buildOK === true) && - (this.connectionOK === null || this.connectionOK === true); - } - - /** - * Override to perform some action after successfully initializing - * - * Results will be try-catched and swallowed/logged if an error is thrown. This will not affect initialized state. - * */ - protected async postInitialize(): Promise { - return; - } - public transformPlay = (play: PlayObject, hookType: TransformHook, log?: boolean) => { let logger: Logger; @@ -422,8 +180,4 @@ export default abstract class AbstractComponent { return play; } } - - public additionalApiData(): Record { - return {}; - } } diff --git a/src/backend/common/AbstractInitializable.ts b/src/backend/common/AbstractInitializable.ts new file mode 100644 index 00000000..4e5bd8db --- /dev/null +++ b/src/backend/common/AbstractInitializable.ts @@ -0,0 +1,291 @@ +import { childLogger, Logger } from "@foxxmd/logging"; +import { + cacheFunctions, +} from "@foxxmd/regex-buddy-core"; +import deepEqual from 'fast-deep-equal'; +import { Simulate } from "react-dom/test-utils"; +import { PlayObject } from "../../core/Atomic.js"; +import { buildTrackString, truncateStringToLength } from "../../core/StringUtils.js"; + +import { + configPartsToStrongParts, countRegexes, + isUserStage, + transformPlayUsingParts +} from "../utils/PlayTransformUtils.js"; +import { hasNodeNetworkException } from "./errors/NodeErrors.js"; +import { hasUpstreamError } from "./errors/UpstreamError.js"; +import { CommonClientConfig } from "./infrastructure/config/client/index.js"; +import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; +import play = Simulate.play; +import { WebhookPayload } from "./infrastructure/config/health/webhooks.js"; +import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError, TransformRulesError } from "./errors/MSErrors.js"; +import { messageWithCauses, messageWithCausesTruncatedDefault } from "../utils/ErrorUtils.js"; + +export default abstract class AbstractInitializable { + requiresAuth: boolean = false; + requiresAuthInteraction: boolean = false; + authed: boolean = false; + authFailure?: boolean; + + buildOK?: boolean | null; + connectionOK?: boolean | null; + cacheOK?: boolean | null; + + initializing: boolean = false; + + config: CommonClientConfig | CommonSourceConfig; + + logger: Logger; + componentLogger?: Logger; + + protected constructor(config: Record) { + this.config = config; + } + + public abstract notify(payload: WebhookPayload): Promise; + + protected abstract getIdentifier(): string; + + initialize = async (options: {force?: boolean, notify?: boolean, notifyTitle?: string} = {}) => { + + const {force = false, notify = false, notifyTitle = 'Init Error'} = options; + + this.logger.debug('Attempting to initialize...'); + try { + this.initializing = true; + if(this.componentLogger === undefined) { + await this.buildComponentLogger(); + } + await this.buildInitData(force); + await this.parseCache(force); + try { + await this.postCache(); + } catch (e) { + if(e instanceof StageError) { + throw e; + } else { + throw new Error('Error occurred during post-cache hook', {cause: e}); + } + } + await this.checkConnection(force); + await this.testAuth(force); + this.logger.info('Fully Initialized!'); + try { + await this.postInitialize(); + } catch (e) { + throw new PostInitError('Error occurred during post-initialization hook', {cause: e}); + } + return true; + } catch(e) { + if(notify) { + await this.notify({title: `${this.getIdentifier()} - ${notifyTitle}`, message: truncateStringToLength(500)(messageWithCausesTruncatedDefault(e)), priority: 'error'}); + } + throw new Error('Initialization failed', {cause: e}); + } finally { + this.initializing = false; + } + } + + protected async buildComponentLogger() { + await this.doBuildComponentLogger(); + return; + } + + protected async doBuildComponentLogger() { + return; + } + + tryInitialize = async (options: {force?: boolean, notify?: boolean, notifyTitle?: string} = {}) => { + if(this.initializing) { + throw new Error(`Already trying to initialize, cannot attempt while an existing initialization attempt is running.`) + } + try { + return await this.initialize(options); + } catch (e) { + throw e; + } + } + + public async parseCache(force: boolean = false) { + if(this.cacheOK) { + if(!force) { + return; + } + this.logger.debug('Cache OK but step was forced'); + } + try { + const res = await this.doParseCache(); + if(res === undefined) { + this.cacheOK = null; + this.logger.debug('No cache to parse.'); + return; + } + if (res === true) { + this.logger.verbose('Parsing caching succeeded'); + } else if (typeof res === 'string') { + this.logger.verbose(`Parsing caching succeeded => ${res}`); + } + this.cacheOK = true; + } catch (e) { + this.cacheOK = false; + throw new ParseCacheError('Parsing cache for initialization failed', {cause: e}); + } + } + + /** + * Build or parse any cache required for this Component + * + * * Return undefined if not possible or not required + * * Return TRUE if build succeeded + * * Return string if build succeeded and should log result + * * Throw error on failure + * */ + protected async doParseCache(): Promise { + return; + } + + + protected async postCache(): Promise { + return; + } + + public async buildInitData(force: boolean = false) { + if(this.buildOK) { + if(!force) { + return; + } + this.logger.debug('Build OK but step was forced'); + } + try { + const res = await this.doBuildInitData(); + if(res === undefined) { + this.buildOK = null; + this.logger.debug('No required data to build.'); + return; + } + if (res === true) { + this.logger.verbose('Building required data init succeeded'); + } else if (typeof res === 'string') { + this.logger.verbose(`Building required data init succeeded => ${res}`); + } + this.buildOK = true; + } catch (e) { + this.buildOK = false; + throw new BuildDataError('Building required data for initialization failed', {cause: e}); + } + } + + /** + * Build any data/config/objects required for this Source to communicate with upstream service + * + * * Return undefined if not possible or not required + * * Return TRUE if build succeeded + * * Return string if build succeeded and should log result + * * Throw error on failure + * */ + protected async doBuildInitData(): Promise { + return; + } + + public async checkConnection(force: boolean = false) { + if(this.connectionOK) { + if(!force) { + return; + } + this.logger.debug('Connection OK but step was forced') + } + try { + const res = await this.doCheckConnection(); + if (res === undefined) { + this.logger.debug('Connection check was not required.'); + this.connectionOK = null; + return; + } else if (res === true) { + this.logger.verbose('Connection check succeeded'); + } else { + this.logger.verbose(`Connection check succeeded => ${res}`); + } + this.connectionOK = true; + } catch (e) { + this.connectionOK = false; + throw new ConnectionCheckError('Communicating with upstream service failed', {cause: e}); + } + } + + /** + * Check Scrobbler upstream API/connection to ensure we can communicate + * + * * Return undefined if not possible or not required to check + * * Return TRUE if communication succeeded + * * Return string if communication succeeded and should log result + * * Throw error if communication failed + * */ + protected async doCheckConnection(): Promise { + return; + } + + authGated = () => this.requiresAuth && !this.authed + + canTryAuth = () => this.isUsable() && this.authGated() && this.authFailure !== true + + canAuthUnattended = () => !this.authGated || !this.requiresAuthInteraction || (this.requiresAuthInteraction && !this.authFailure); + + protected doAuthentication = async (): Promise => this.authed + + // default init function, should be overridden if auth stage is required + testAuth = async (force: boolean = false) => { + if(!this.requiresAuth) { + return; + } + if(this.authed) { + if(!force) { + return; + } + this.logger.debug('Auth OK but step was forced'); + } + + if(this.authFailure) { + if(!force) { + if(this.requiresAuthInteraction) { + throw new AuthCheckError('Authentication failure: Will not retry auth because user interaction is required for authentication'); + } + throw new AuthCheckError('Authentication failure: Will not retry auth because authentication previously failed and must be reauthenticated'); + } + this.logger.debug('Auth previously failed for non upstream/network reasons but retry is being forced'); + } + + try { + this.authed = await this.doAuthentication(); + this.authFailure = !this.authed; + } catch (e) { + // only signal as auth failure if error was NOT either a node network error or a non-showstopping upstream error + this.authFailure = !(hasNodeNetworkException(e) || hasUpstreamError(e, false)); + this.authed = false; + throw new AuthCheckError(`Authentication test failed!${this.authFailure === false ? ' Due to a network issue. Will retry authentication on next heartbeat.' : ''}`, {cause: e}); + } + } + + public isReady() { + return (this.buildOK === null || this.buildOK === true) && + (this.connectionOK === null || this.connectionOK === true) + && !this.authGated(); + } + + public isUsable() { + return (this.buildOK === null || this.buildOK === true) && + (this.connectionOK === null || this.connectionOK === true); + } + + /** + * Override to perform some action after successfully initializing + * + * Results will be try-catched and swallowed/logged if an error is thrown. This will not affect initialized state. + * */ + protected async postInitialize(): Promise { + return; + } + + public additionalApiData(): Record { + return {}; + } +} diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts new file mode 100644 index 00000000..cf62499c --- /dev/null +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -0,0 +1,6 @@ +import AbstractInitializable from "../AbstractInitializable.js"; + + +export default abstract class AbstractTransformer extends AbstractInitializable { + +} \ No newline at end of file -- 2.51.2 From 1af01936369378425535d06b648986d17fa08d9a Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 21 Nov 2025 18:31:35 +0000 Subject: [PATCH 02/12] feat(transform): Implement abstract transformer #365 #363 #364 --- src/backend/common/AbstractInitializable.ts | 2 +- src/backend/common/Cache.ts | 8 +- src/backend/common/infrastructure/Atomic.ts | 5 + .../common/transforms/AbstractTransformer.ts | 129 +++++++++++++++++- src/core/Atomic.ts | 10 +- 5 files changed, 147 insertions(+), 7 deletions(-) diff --git a/src/backend/common/AbstractInitializable.ts b/src/backend/common/AbstractInitializable.ts index 4e5bd8db..39fefce7 100644 --- a/src/backend/common/AbstractInitializable.ts +++ b/src/backend/common/AbstractInitializable.ts @@ -33,7 +33,7 @@ export default abstract class AbstractInitializable { initializing: boolean = false; - config: CommonClientConfig | CommonSourceConfig; + config: Record; logger: Logger; componentLogger?: Logger; diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index 719f8de3..04210193 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -12,6 +12,7 @@ import clone from 'clone'; import { childLogger, Logger } from '@foxxmd/logging'; import { projectDir } from './index.js'; import path from 'path'; +import { cacheFunctions } from "@foxxmd/regex-buddy-core"; import { fileOrDirectoryIsWriteable } from '../utils.js'; import { asCacheAuthProvider, asCacheMetadataProvider, asCacheScrobbleProvider, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheMetadataProvider, CacheProvider, CacheScrobbleProvider } from './infrastructure/Atomic.js'; import { Typeson } from 'typeson'; @@ -47,6 +48,7 @@ export class MSCache { cacheMetadata: Cacheable; cacheScrobble: Cacheable; cacheAuth: Cacheable; + regexCache: ReturnType; logger: Logger; @@ -69,6 +71,7 @@ export class MSCache { connection: aConn = (process.env.CACHE_AUTH_CONN ?? configDir), ...restAuth } = {}, + regex = 200, } = config; this.config = { @@ -86,8 +89,11 @@ export class MSCache { provider: aProvider, connection: aConn, ...restAuth - } + }, + regex }; + + this.regexCache = cacheFunctions(this.config.regex); } init = async () => { diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 01f675f0..d43700a0 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -327,5 +327,10 @@ export interface CacheConfigOptions { metadata?: CacheMetadataConfig; scrobble?: CacheScrobbleConfig; auth?: CacheAuthConfig; + /** Number of regex functions to cache (LRU) + * + * @default 200 + */ + regex?: number } diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index cf62499c..f7deb5d6 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -1,6 +1,133 @@ +import { ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js"; +import { getRoot } from "../../ioc.js"; +import { testWhenConditions } from "../../utils/PlayTransformUtils.js"; import AbstractInitializable from "../AbstractInitializable.js"; +import { ConditionalSearchAndReplaceRegExp, PlayTransformParts } from "../infrastructure/Transform.js"; +import { cacheFunctions } from "@foxxmd/regex-buddy-core"; +export interface TransformerCommonConfig { + data?: Record + options?: { + failOnFetch?: boolean + throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album')[] + } +} +export default abstract class AbstractTransformer extends AbstractInitializable { -export default abstract class AbstractTransformer extends AbstractInitializable { + declare config: TransformerCommonConfig; + regexCache: ReturnType + + protected constructor(config: TransformerCommonConfig) { + super(config); + this.regexCache = getRoot().items.cache().regexCache; + } + + public async handle(parts: PlayTransformParts, play: PlayObject): Promise { + + if (parts.when !== undefined) { + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex })) { + this.logger.debug('When condition not met, returning original Play'); + return play; + } + } + + const { + failOnFetch = false, + throwOnFailure = false, + } = this.config.options || {}; + + let transformData: T; + try { + transformData = await this.getTransformerData(play); + } catch (e) { + if (failOnFetch) { + throw new Error(`Could not fetch transformer data`, { cause: e }); + } + this.logger.warn(new Error(`Could not fetch transformer data, returning original Play`, { cause: e })); + return play; + } + + try { + await this.checkShouldTransform(play, transformData); + } catch (e) { + this.logger.debug(new Error('checkShouldTransform did not pass, returning original Play', { cause: e })); + return play; + } + + const transformedPlayData: Partial = {}; + + try { + const title = await this.handleTitle(play, transformData); + transformedPlayData.track = title; + } catch (e) { + const err = new Error(`Failed to transform title: ${play.data.track}`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('title'))) { + throw err; + } else { + this.logger.warn(err); + } + } + + try { + const artists = await this.handleArtists(play, transformData); + transformedPlayData.artists = artists; + } catch (e) { + const err = new Error(`Failed to transform artists`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('artists'))) { + throw err; + } else { + this.logger.warn(err); + } + } + + try { + const albumArtists = await this.handleArtists(play, transformData); + transformedPlayData.albumArtists = albumArtists; + } catch (e) { + const err = new Error(`Failed to transform album artists`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('albumArtists'))) { + throw err; + } else { + this.logger.warn(err); + } + } + + try { + const album = await this.handleTitle(play, transformData); + transformedPlayData.album = album; + } catch (e) { + const err = new Error(`Failed to transform album: ${play.data.album}`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('album'))) { + throw err; + } else { + this.logger.warn(err); + } + } + + const transformedPlay = { + ...play, + data: { + ...play.data, + ...transformedPlayData + } + } + + return transformedPlay; + } + + public abstract getTransformerData(play: PlayObject): Promise; + + public async checkShouldTransform(play: PlayObject, transformData: T): Promise { + return; + } + + protected abstract handleTitle(play: PlayObject, transformData: T): Promise; + protected abstract handleArtists(play: PlayObject, transformData: T): Promise; + protected abstract handleAlbumArtists(play: PlayObject, transformData: T): Promise; + protected abstract handleAlbum(play: PlayObject, transformData: T): Promise; + + protected async handleMeta(play: PlayObject, transformData: T): Promise { + return play.data.meta; + } } \ No newline at end of file diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 6e0b7674..f309c738 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -136,6 +136,11 @@ export interface SpotifyMeta { track?: string } +export interface TrackMeta { + brainz?: BrainzMeta + spotify?: SpotifyMeta +} + export interface TrackData { artists?: string[] albumArtists?: string[] @@ -146,10 +151,7 @@ export interface TrackData { * */ duration?: number - meta?: { - brainz?: BrainzMeta - spotify?: SpotifyMeta - } + meta?: TrackMeta } export interface PlayData extends TrackData { -- 2.51.2 From 56409dd1d71dd08f2d1bf83955d8a228914ddd26 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 21 Nov 2025 20:00:49 +0000 Subject: [PATCH 03/12] feat(transformer): Implement User Transformer class --- src/backend/common/AbstractComponent.ts | 12 +- src/backend/common/AbstractInitializable.ts | 2 +- .../common/infrastructure/Transform.ts | 11 +- .../common/transforms/AbstractTransformer.ts | 107 ++++++++++-------- .../common/transforms/UserTransformer.ts | 68 +++++++++++ src/backend/utils/PlayTransformUtils.ts | 56 ++++----- 6 files changed, 169 insertions(+), 87 deletions(-) create mode 100644 src/backend/common/transforms/UserTransformer.ts diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 45c19fc0..78e53c7a 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -8,13 +8,13 @@ import { PlayObject } from "../../core/Atomic.js"; import { buildTrackString } from "../../core/StringUtils.js"; import { - configPartsToStrongParts, countRegexes, transformPlayUsingParts + configPartsToStrongParts, isUserStage, transformPlayUsingParts } from "../utils/PlayTransformUtils.js"; import { CommonClientConfig } from "./infrastructure/config/client/index.js"; import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; import { TransformRulesError } from "./errors/MSErrors.js"; import { - ConditionalSearchAndReplaceRegExp, PlayTransformPartsArray, + ConditionalSearchAndReplaceRegExp, ExternalMetadataTerm, PlayTransformPartsArray, PlayTransformRules, TRANSFORM_HOOK, TransformHook @@ -50,8 +50,8 @@ export default abstract class AbstractComponent extends AbstractInitializable { throw new TransformRulesError('Could not build playTransform rules. Check your configuration is valid.', {cause: e}); } try { - const ruleCount = countRegexes(this.transformRules); - this.regexCache = cacheFunctions(ruleCount); + //const ruleCount = countRegexes(this.transformRules); + this.regexCache = cacheFunctions(200); } catch (e) { this.logger.warn(new TransformRulesError('Failed to count number of rule regexes for caching but will continue will fallback to 100', {cause: e})); } @@ -123,7 +123,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { const getLogger = () => logger !== undefined ? logger : childLogger(this.logger, labels); try { - let hook: PlayTransformPartsArray | undefined; + let hook: PlayTransformPartsArray | undefined; switch (hookType) { case TRANSFORM_HOOK.preCompare: @@ -147,7 +147,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { let transformedPlay: PlayObject = play; const transformDetails: string[] = []; for(const hookItem of hook) { - if(hookItem.type === 'user') { + if(isUserStage(hookItem)) { const newTransformedPlay = transformPlayUsingParts(transformedPlay, hookItem, { logger: getLogger, regex: { diff --git a/src/backend/common/AbstractInitializable.ts b/src/backend/common/AbstractInitializable.ts index 39fefce7..7441c438 100644 --- a/src/backend/common/AbstractInitializable.ts +++ b/src/backend/common/AbstractInitializable.ts @@ -8,7 +8,7 @@ import { PlayObject } from "../../core/Atomic.js"; import { buildTrackString, truncateStringToLength } from "../../core/StringUtils.js"; import { - configPartsToStrongParts, countRegexes, + configPartsToStrongParts, isUserStage, transformPlayUsingParts } from "../utils/PlayTransformUtils.js"; diff --git a/src/backend/common/infrastructure/Transform.ts b/src/backend/common/infrastructure/Transform.ts index 677e1df0..b9b13dff 100644 --- a/src/backend/common/infrastructure/Transform.ts +++ b/src/backend/common/infrastructure/Transform.ts @@ -8,8 +8,9 @@ export type ConditionalSearchAndReplaceTerm = Omit = Extract, Y> & { when?: WhenConditionsConfig }; +export type PlayTransformParts = Extract, Y> & { when?: WhenConditionsConfig }; export type PlayTransformUserParts = PlayTransformUserStage & { when?: WhenConditionsConfig }; +export type PlayTransformMetaParts = PlayTransformMetadataStage & { when?: WhenConditionsConfig }; export type PlayTransformPartsArray = PlayTransformParts[]; /** Represents the weakly-defined user config. May be an array of parts or one parts object */ @@ -42,7 +43,7 @@ export interface PlayTransformStageTyped extends PlayTransformPartsAtomic type: StageType } -export interface PlayTransformMetadataStage extends PlayTransformStageTyped { +export interface PlayTransformMetadataStage extends PlayTransformStageTyped { score?: number // all?: ExternalMetadataTerm type: StageTypeMetadata @@ -53,7 +54,7 @@ export interface PlayTransformUserStage extends PlayTransformStageTyped { } export type UntypedPlayTransformUserStage = Omit, 'type'> & {type?: never}; -export type PlayTransformStage = PlayTransformMetadataStage | PlayTransformUserStage | UntypedPlayTransformUserStage; +export type PlayTransformStage = PlayTransformMetadataStage | PlayTransformUserStage | UntypedPlayTransformUserStage; /** Represents the plain json user-configured structure (input) */ export interface PlayTransformHooksConfig { @@ -75,7 +76,7 @@ export interface PlayTransformHooks extends PlayTransformHooksConfig { postCompare?: PlayTransformPartsArray } -export type PlayTransformRules = PlayTransformHooks +export type PlayTransformRules = PlayTransformHooks export type TransformHook = 'preCompare' | 'compare' | 'candidate' | 'existing' | 'postCompare'; export const TRANSFORM_HOOK = { preCompare: 'preCompare' as TransformHook, @@ -83,7 +84,7 @@ export const TRANSFORM_HOOK = { existing: 'existing' as TransformHook, postCompare: 'postCompare' as TransformHook, } -export type PlayTransformConfig = PlayTransformHooksConfig; +export type PlayTransformConfig = PlayTransformHooksConfig; export type PlayTransformOptions = PlayTransformConfig & { log?: boolean | 'all' } export type WhenParts = PlayTransformPartsAtomic; export type WhenConditions = WhenParts[]; diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index f7deb5d6..44d51dea 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -2,7 +2,7 @@ import { ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js"; import { getRoot } from "../../ioc.js"; import { testWhenConditions } from "../../utils/PlayTransformUtils.js"; import AbstractInitializable from "../AbstractInitializable.js"; -import { ConditionalSearchAndReplaceRegExp, PlayTransformParts } from "../infrastructure/Transform.js"; +import { ConditionalSearchAndReplaceRegExp, PlayTransformMetaParts, PlayTransformParts, PlayTransformUserParts } from "../infrastructure/Transform.js"; import { cacheFunctions } from "@foxxmd/regex-buddy-core"; export interface TransformerCommonConfig { @@ -12,18 +12,23 @@ export interface TransformerCommonConfig { throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album')[] } } -export default abstract class AbstractTransformer extends AbstractInitializable { + +export interface TransformerCommon extends TransformerCommonConfig { + regexCache?: ReturnType +} + +export default abstract class AbstractTransformer extends AbstractInitializable { declare config: TransformerCommonConfig; regexCache: ReturnType - protected constructor(config: TransformerCommonConfig) { + protected constructor(config: TransformerCommon) { super(config); - this.regexCache = getRoot().items.cache().regexCache; + this.regexCache = config.regexCache ?? getRoot().items.cache().regexCache; } - public async handle(parts: PlayTransformParts, play: PlayObject): Promise { + public async handle(parts: PlayTransformParts, play: PlayObject): Promise { if (parts.when !== undefined) { if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex })) { @@ -57,51 +62,57 @@ export default abstract class AbstractTransformer extends AbstractIniti const transformedPlayData: Partial = {}; - try { - const title = await this.handleTitle(play, transformData); - transformedPlayData.track = title; - } catch (e) { - const err = new Error(`Failed to transform title: ${play.data.track}`, { cause: e }); - if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('title'))) { - throw err; - } else { - this.logger.warn(err); + if (parts.title !== undefined) { + try { + const title = await this.handleTitle(play, parts.title, transformData); + transformedPlayData.track = title; + } catch (e) { + const err = new Error(`Failed to transform title: ${play.data.track}`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('title'))) { + throw err; + } else { + this.logger.warn(err); + } } } - try { - const artists = await this.handleArtists(play, transformData); - transformedPlayData.artists = artists; - } catch (e) { - const err = new Error(`Failed to transform artists`, { cause: e }); - if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('artists'))) { - throw err; - } else { - this.logger.warn(err); + if (parts.artists !== undefined) { + try { + const artists = await this.handleArtists(play, parts.artists, transformData); + transformedPlayData.artists = artists; + } catch (e) { + const err = new Error(`Failed to transform artists`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('artists'))) { + throw err; + } else { + this.logger.warn(err); + } } - } - try { - const albumArtists = await this.handleArtists(play, transformData); - transformedPlayData.albumArtists = albumArtists; - } catch (e) { - const err = new Error(`Failed to transform album artists`, { cause: e }); - if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('albumArtists'))) { - throw err; - } else { - this.logger.warn(err); + try { + const albumArtists = await this.handleAlbumArtists(play, parts.artists, transformData); + transformedPlayData.albumArtists = albumArtists; + } catch (e) { + const err = new Error(`Failed to transform album artists`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('albumArtists'))) { + throw err; + } else { + this.logger.warn(err); + } } } - try { - const album = await this.handleTitle(play, transformData); - transformedPlayData.album = album; - } catch (e) { - const err = new Error(`Failed to transform album: ${play.data.album}`, { cause: e }); - if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('album'))) { - throw err; - } else { - this.logger.warn(err); + if (parts.album !== undefined) { + try { + const album = await this.handleTitle(play, parts.album, transformData); + transformedPlayData.album = album; + } catch (e) { + const err = new Error(`Failed to transform album: ${play.data.album}`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('album'))) { + throw err; + } else { + this.logger.warn(err); + } } } @@ -116,16 +127,18 @@ export default abstract class AbstractTransformer extends AbstractIniti return transformedPlay; } - public abstract getTransformerData(play: PlayObject): Promise; + public async getTransformerData(play: PlayObject): Promise { + return undefined; + } public async checkShouldTransform(play: PlayObject, transformData: T): Promise { return; } - protected abstract handleTitle(play: PlayObject, transformData: T): Promise; - protected abstract handleArtists(play: PlayObject, transformData: T): Promise; - protected abstract handleAlbumArtists(play: PlayObject, transformData: T): Promise; - protected abstract handleAlbum(play: PlayObject, transformData: T): Promise; + protected abstract handleTitle(play: PlayObject, parts: Y, transformData: T): Promise; + protected abstract handleArtists(play: PlayObject, parts: Y, transformData: T): Promise; + protected abstract handleAlbumArtists(play: PlayObject, parts: Y, transformData: T): Promise; + protected abstract handleAlbum(play: PlayObject, parts: Y, transformData: T): Promise; protected async handleMeta(play: PlayObject, transformData: T): Promise { return play.data.meta; diff --git a/src/backend/common/transforms/UserTransformer.ts b/src/backend/common/transforms/UserTransformer.ts new file mode 100644 index 00000000..1b315870 --- /dev/null +++ b/src/backend/common/transforms/UserTransformer.ts @@ -0,0 +1,68 @@ +import { searchAndReplace } from "@foxxmd/regex-buddy-core"; +import { PlayObject } from "../../../core/Atomic.js"; +import { testWhenConditions } from "../../utils/PlayTransformUtils.js"; +import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; +import { ConditionalSearchAndReplaceRegExp, PlayTransformUserParts } from "../infrastructure/Transform.js"; +import AbstractTransformer, { TransformerCommon } from "./AbstractTransformer.js" + +export default class UserTransformer extends AbstractTransformer { + + protected constructor(config: TransformerCommon) { + super(config); + } + + protected generateMapper(play: PlayObject) { + return (x: ConditionalSearchAndReplaceRegExp): ConditionalSearchAndReplaceRegExp => ({ ...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex }) : undefined) }); + } + + protected async handleTitle(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise { + if (play.data.track === undefined) { + return undefined; + } + const mapper = this.generateMapper(play); + return searchAndReplace(play.data.track, parts.map(mapper)); + } + protected async handleArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise { + if(play.data.artists === undefined || play.data.artists.length === 0) { + return play.data.artists; + } + const mapper = this.generateMapper(play); + const transformedArtists = []; + for(const artist of play.data.artists) { + const a = searchAndReplace(artist, parts.map(mapper)); + if(a.trim() !== '') { + transformedArtists.push(a); + } + } + return transformedArtists; + } + protected async handleAlbumArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise { + if(play.data.albumArtists === undefined || play.data.albumArtists.length === 0) { + return play.data.albumArtists; + } + const mapper = this.generateMapper(play); + const transformedArtists = []; + for(const artist of play.data.albumArtists) { + const a = searchAndReplace(artist, parts.map(mapper)); + if(a.trim() !== '') { + transformedArtists.push(a); + } + } + return transformedArtists; + } + protected async handleAlbum(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise { + if (play.data.track === undefined) { + return undefined; + } + const mapper = this.generateMapper(play); + return searchAndReplace(play.data.album, parts.map(mapper)); + } + + public notify(payload: WebhookPayload): Promise { + throw new Error("Method not implemented."); + } + protected getIdentifier(): string { + return 'User Transformer'; + } + +} \ No newline at end of file diff --git a/src/backend/utils/PlayTransformUtils.ts b/src/backend/utils/PlayTransformUtils.ts index a61a3055..b813b1d5 100644 --- a/src/backend/utils/PlayTransformUtils.ts +++ b/src/backend/utils/PlayTransformUtils.ts @@ -129,7 +129,7 @@ export const isUserStage = (val: PlayTransformStage): val is PlayTransform return val.type === 'user'; } -export const configPartsToStrongParts = (val: PlayTransformPartsConfig | undefined): PlayTransformPartsArray => { +export const configPartsToStrongParts = (val: PlayTransformPartsConfig | undefined): PlayTransformPartsArray => { if (val === undefined) { return [] } @@ -343,33 +343,33 @@ export const transformPlayUsingParts = (play: PlayObject, parts: PlayTransformUs return play; } -export const countRegexes = (rules: PlayTransformRules): number => { - let rulesCount = 0; - if(rules.preCompare !== undefined) { - for(const hookItem of rules.preCompare) { - rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); - } - - } - if(rules.postCompare !== undefined) { - for(const hookItem of rules.postCompare) { - rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); - } - } - if(rules.compare !== undefined) { - if(rules.compare.existing !== undefined) { - for(const hookItem of rules.compare.existing) { - rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); - } - } - if(rules.compare.candidate !== undefined) { - for(const hookItem of rules.compare.candidate) { - rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); - } - } - } - return rulesCount; -} +// export const countRegexes = (rules: PlayTransformRules): number => { +// let rulesCount = 0; +// if(rules.preCompare !== undefined) { +// for(const hookItem of rules.preCompare) { +// rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); +// } + +// } +// if(rules.postCompare !== undefined) { +// for(const hookItem of rules.postCompare) { +// rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); +// } +// } +// if(rules.compare !== undefined) { +// if(rules.compare.existing !== undefined) { +// for(const hookItem of rules.compare.existing) { +// rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); +// } +// } +// if(rules.compare.candidate !== undefined) { +// for(const hookItem of rules.compare.candidate) { +// rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); +// } +// } +// } +// return rulesCount; +// } const countWhens = (when: WhenConditionsConfig | undefined): number => { if(when === undefined) { -- 2.51.2 From affb16fc0ec78e98b15b2cda1c76e6787a17818f Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 25 Nov 2025 18:33:22 +0000 Subject: [PATCH 04/12] feat: Implement user transformer and refactor components to use async --- src/backend/common/AbstractComponent.ts | 50 ++++---- src/backend/common/AbstractInitializable.ts | 15 +-- .../common/infrastructure/Transform.ts | 16 ++- .../common/infrastructure/config/aioConfig.ts | 3 + .../common/transforms/AbstractTransformer.ts | 121 +++++------------- .../transforms/AtomicPartsTransformer.ts | 108 ++++++++++++++++ .../common/transforms/TransformerManager.ts | 103 +++++++++++++++ .../common/transforms/UserTransformer.ts | 52 ++++++-- src/backend/index.ts | 2 + src/backend/ioc.ts | 17 ++- .../scrobblers/AbstractScrobbleClient.ts | 26 ++-- src/backend/sources/AbstractSource.ts | 30 +++-- src/backend/sources/AzuracastSource.ts | 2 +- src/backend/sources/ChromecastSource.ts | 2 +- src/backend/sources/DeezerInternalSource.ts | 13 +- src/backend/sources/EndpointLastfmSource.ts | 6 +- .../sources/EndpointListenbrainzSource.ts | 6 +- src/backend/sources/IcecastSource.ts | 4 +- src/backend/sources/JRiverSource.ts | 2 +- src/backend/sources/JellyfinApiSource.ts | 2 +- src/backend/sources/JellyfinSource.ts | 6 +- src/backend/sources/KodiSource.ts | 2 +- src/backend/sources/KoitoSource.ts | 2 +- src/backend/sources/LastfmSource.ts | 2 +- src/backend/sources/ListenbrainzSource.ts | 2 +- src/backend/sources/MPDSource.ts | 2 +- src/backend/sources/MPRISSource.ts | 2 +- src/backend/sources/MalojaSource.ts | 2 +- src/backend/sources/MemorySource.ts | 30 +++-- src/backend/sources/MopidySource.ts | 2 +- src/backend/sources/MusicCastSource.ts | 6 +- src/backend/sources/MusikcubeSource.ts | 2 +- src/backend/sources/PlexApiSource.ts | 2 +- src/backend/sources/PlexSource.ts | 4 +- src/backend/sources/SpotifySource.ts | 2 +- src/backend/sources/SubsonicSource.ts | 2 +- src/backend/sources/TealfmSource.ts | 2 +- src/backend/sources/VLCSource.ts | 2 +- src/backend/sources/WebScrobblerSource.ts | 6 +- src/backend/tests/component/component.test.ts | 103 ++++++++------- src/backend/tests/plays/playParsing.test.ts | 33 ++++- .../tests/scrobbler/scrobblers.test.ts | 6 +- src/backend/tests/source/source.test.ts | 106 +++++++-------- src/backend/tests/utils/strings.test.ts | 35 ----- src/backend/tests/ytm/ytm.test.ts | 4 +- src/backend/utils/AsyncUtils.ts | 45 +++++++ src/backend/utils/PlayTransformUtils.ts | 71 ++-------- src/core/Atomic.ts | 11 +- 48 files changed, 645 insertions(+), 429 deletions(-) create mode 100644 src/backend/common/transforms/AtomicPartsTransformer.ts create mode 100644 src/backend/common/transforms/TransformerManager.ts create mode 100644 src/backend/utils/AsyncUtils.ts diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 78e53c7a..a88ce616 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -6,21 +6,19 @@ import deepEqual from 'fast-deep-equal'; import { Simulate } from "react-dom/test-utils"; import { PlayObject } from "../../core/Atomic.js"; import { buildTrackString } from "../../core/StringUtils.js"; - -import { - configPartsToStrongParts, isUserStage, transformPlayUsingParts -} from "../utils/PlayTransformUtils.js"; import { CommonClientConfig } from "./infrastructure/config/client/index.js"; import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; import { TransformRulesError } from "./errors/MSErrors.js"; import { - ConditionalSearchAndReplaceRegExp, ExternalMetadataTerm, PlayTransformPartsArray, PlayTransformRules, + StageConfig, TRANSFORM_HOOK, TransformHook } from "./infrastructure/Transform.js"; import AbstractInitializable from "./AbstractInitializable.js"; import play = Simulate.play; +import TransformerManager from "./transforms/TransformerManager.js"; +import { getRoot } from "../ioc.js"; export default abstract class AbstractComponent extends AbstractInitializable { @@ -28,9 +26,11 @@ export default abstract class AbstractComponent extends AbstractInitializable { transformRules: PlayTransformRules = {}; regexCache!: ReturnType; + protected transformManager: TransformerManager; protected constructor(config: CommonClientConfig | CommonSourceConfig) { super(config); + this.transformManager = getRoot().items.transformerManager; } protected postCache(): Promise { @@ -83,25 +83,25 @@ export default abstract class AbstractComponent extends AbstractInitializable { postCompare; try { - preCompare = configPartsToStrongParts(preConfig) + preCompare = this.transformPartToStrong(preConfig); } catch (e) { throw new Error('preCompare was not valid', {cause: e}); } try { - candidate = configPartsToStrongParts(candidateConfig) + candidate = this.transformPartToStrong(candidateConfig); } catch (e) { throw new Error('candidate was not valid', {cause: e}); } try { - existing = configPartsToStrongParts(existingConfig) + existing = this.transformPartToStrong(existingConfig); } catch (e) { throw new Error('existing was not valid', {cause: e}); } try { - postCompare = configPartsToStrongParts(postConfig) + postCompare = this.transformPartToStrong(postConfig); } catch (e) { throw new Error('postCompare was not valid', {cause: e}); } @@ -116,14 +116,24 @@ export default abstract class AbstractComponent extends AbstractInitializable { } } - public transformPlay = (play: PlayObject, hookType: TransformHook, log?: boolean) => { + protected transformPartToStrong(data: any) { + if(data === undefined) { + return undefined; + } + // default to user transform type for backward compatibility + const partArr = (Array.isArray(data) ? data : [data]).map(x => ({type: 'user', ...x})); + + return partArr.map(x => this.transformManager.parseTransformerConfig(x)); + } + + public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean) => { let logger: Logger; const labels = ['Play Transform', hookType]; const getLogger = () => logger !== undefined ? logger : childLogger(this.logger, labels); try { - let hook: PlayTransformPartsArray | undefined; + let hook: StageConfig[]; switch (hookType) { case TRANSFORM_HOOK.preCompare: @@ -147,19 +157,13 @@ export default abstract class AbstractComponent extends AbstractInitializable { let transformedPlay: PlayObject = play; const transformDetails: string[] = []; for(const hookItem of hook) { - if(isUserStage(hookItem)) { - const newTransformedPlay = transformPlayUsingParts(transformedPlay, hookItem, { - logger: getLogger, - regex: { - searchAndReplace: this.regexCache.searchAndReplace, - testMaybeRegex: this.regexCache.testMaybeRegex, - } - }); - if(!deepEqual(newTransformedPlay, transformedPlay)) { - transformDetails.push(buildTrackString(transformedPlay, {include: ['artist', 'track', 'album']})); - } - transformedPlay = newTransformedPlay; + + const newTransformedPlay = await this.transformManager.handleStage(hookItem, transformedPlay); + + if(!deepEqual(newTransformedPlay, transformedPlay)) { + transformDetails.push(`${hookItem.type} - ${buildTrackString(transformedPlay, {include: ['artist', 'track', 'album']})}`); } + transformedPlay = newTransformedPlay; } if(transformDetails.length > 0) { diff --git a/src/backend/common/AbstractInitializable.ts b/src/backend/common/AbstractInitializable.ts index 7441c438..32d91301 100644 --- a/src/backend/common/AbstractInitializable.ts +++ b/src/backend/common/AbstractInitializable.ts @@ -1,21 +1,8 @@ import { childLogger, Logger } from "@foxxmd/logging"; -import { - cacheFunctions, -} from "@foxxmd/regex-buddy-core"; -import deepEqual from 'fast-deep-equal'; import { Simulate } from "react-dom/test-utils"; -import { PlayObject } from "../../core/Atomic.js"; -import { buildTrackString, truncateStringToLength } from "../../core/StringUtils.js"; - -import { - configPartsToStrongParts, - isUserStage, - transformPlayUsingParts -} from "../utils/PlayTransformUtils.js"; +import {truncateStringToLength } from "../../core/StringUtils.js"; import { hasNodeNetworkException } from "./errors/NodeErrors.js"; import { hasUpstreamError } from "./errors/UpstreamError.js"; -import { CommonClientConfig } from "./infrastructure/config/client/index.js"; -import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; import play = Simulate.play; import { WebhookPayload } from "./infrastructure/config/health/webhooks.js"; import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError, TransformRulesError } from "./errors/MSErrors.js"; diff --git a/src/backend/common/infrastructure/Transform.ts b/src/backend/common/infrastructure/Transform.ts index b9b13dff..497c3a8d 100644 --- a/src/backend/common/infrastructure/Transform.ts +++ b/src/backend/common/infrastructure/Transform.ts @@ -24,7 +24,7 @@ export interface PlayTransformPartsAtomic { export type StageTypeMetadata = 'spotify' | 'listenbrainz' | 'native'; export type StageTypeUser = 'user'; -export type StageType = StageTypeMetadata | StageTypeUser; +export type StageType = StageTypeMetadata | StageTypeUser | string; export const STAGE_TYPES_USER: StageTypeUser[] = ['user']; export const STAGE_TYPES_METADATA: StageTypeMetadata[] = ['spotify','listenbrainz','native']; export const STAGE_TYPES: StageType[] = [...STAGE_TYPES_METADATA, ...STAGE_TYPES_USER]; @@ -39,6 +39,18 @@ export interface NotStageTyped { export type MaybeStageTyped = StageTyped | NotStageTyped; +export interface StageTypedConfig { + type: StageType +} + +export interface Whennable { + when?: WhenConditionsConfig +} + +export interface StageConfig extends StageTypedConfig, Whennable {} + +export interface AtomicStageConfig extends StageConfig, PlayTransformPartsAtomic {} + export interface PlayTransformStageTyped extends PlayTransformPartsAtomic { type: StageType } @@ -49,7 +61,7 @@ export interface PlayTransformMetadataStage extends Pl type: StageTypeMetadata } -export interface PlayTransformUserStage extends PlayTransformStageTyped { +export interface PlayTransformUserStage extends StageTypedConfig, PlayTransformPartsAtomic { type: StageTypeUser } export type UntypedPlayTransformUserStage = Omit, 'type'> & {type?: never}; diff --git a/src/backend/common/infrastructure/config/aioConfig.ts b/src/backend/common/infrastructure/config/aioConfig.ts index 95d5fdce..6e8d032d 100644 --- a/src/backend/common/infrastructure/config/aioConfig.ts +++ b/src/backend/common/infrastructure/config/aioConfig.ts @@ -6,6 +6,7 @@ import { WebhookConfig } from "./health/webhooks.js"; import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js"; import { SourceAIOConfig } from "./source/sources.js"; import { CacheConfigOptions, ClientType, SourceType } from "../Atomic.js"; +import { TransformerCommonConfig } from "../../../../core/Atomic.js"; export interface SourceDefaults extends CommonSourceOptions { @@ -66,6 +67,8 @@ export interface AIOConfig { debugMode?: boolean cache?: CacheConfigOptions + + transformers?: TransformerCommonConfig[] } export interface AIOClientConfig { diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index 44d51dea..bba5c576 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -1,37 +1,45 @@ -import { ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js"; +import { childLogger, Logger } from "@foxxmd/logging"; +import { PlayObject, TransformerCommonConfig } from "../../../core/Atomic.js"; import { getRoot } from "../../ioc.js"; -import { testWhenConditions } from "../../utils/PlayTransformUtils.js"; +import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import AbstractInitializable from "../AbstractInitializable.js"; -import { ConditionalSearchAndReplaceRegExp, PlayTransformMetaParts, PlayTransformParts, PlayTransformUserParts } from "../infrastructure/Transform.js"; +import { StageConfig } from "../infrastructure/Transform.js"; import { cacheFunctions } from "@foxxmd/regex-buddy-core"; -export interface TransformerCommonConfig { - data?: Record - options?: { - failOnFetch?: boolean - throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album')[] - } -} - export interface TransformerCommon extends TransformerCommonConfig { - regexCache?: ReturnType + regexCache: ReturnType + name: string + logger: Logger } -export default abstract class AbstractTransformer extends AbstractInitializable { +export default abstract class AbstractTransformer extends AbstractInitializable { declare config: TransformerCommonConfig; + transformType: string + regexCache: ReturnType - protected constructor(config: TransformerCommon) { + public constructor(config: TransformerCommon) { super(config); - this.regexCache = config.regexCache ?? getRoot().items.cache().regexCache; + this.logger = childLogger(config.logger, ['Transformer', this.config.type, this.config.name]); + this.transformType = config.type; + this.regexCache = config.regexCache; + } + + public parseConfig(data: any) { + if (!isStageTyped(data)) { + throw new Error(`Must be an object with a 'type' property.`); + } + return this.doParseConfig(data); } - public async handle(parts: PlayTransformParts, play: PlayObject): Promise { + protected abstract doParseConfig(data: StageConfig): StageConfig; + + public async handle(data: StageConfig, play: PlayObject): Promise { - if (parts.when !== undefined) { - if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex })) { + if (data.when !== undefined) { + if (!testWhenConditions(data.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex })) { this.logger.debug('When condition not met, returning original Play'); return play; } @@ -60,73 +68,11 @@ export default abstract class AbstractTransformer extends AbstractIn return play; } - const transformedPlayData: Partial = {}; - - if (parts.title !== undefined) { - try { - const title = await this.handleTitle(play, parts.title, transformData); - transformedPlayData.track = title; - } catch (e) { - const err = new Error(`Failed to transform title: ${play.data.track}`, { cause: e }); - if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('title'))) { - throw err; - } else { - this.logger.warn(err); - } - } - } - - if (parts.artists !== undefined) { - try { - const artists = await this.handleArtists(play, parts.artists, transformData); - transformedPlayData.artists = artists; - } catch (e) { - const err = new Error(`Failed to transform artists`, { cause: e }); - if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('artists'))) { - throw err; - } else { - this.logger.warn(err); - } - } - - try { - const albumArtists = await this.handleAlbumArtists(play, parts.artists, transformData); - transformedPlayData.albumArtists = albumArtists; - } catch (e) { - const err = new Error(`Failed to transform album artists`, { cause: e }); - if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('albumArtists'))) { - throw err; - } else { - this.logger.warn(err); - } - } - } - - if (parts.album !== undefined) { - try { - const album = await this.handleTitle(play, parts.album, transformData); - transformedPlayData.album = album; - } catch (e) { - const err = new Error(`Failed to transform album: ${play.data.album}`, { cause: e }); - if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('album'))) { - throw err; - } else { - this.logger.warn(err); - } - } - } - - const transformedPlay = { - ...play, - data: { - ...play.data, - ...transformedPlayData - } - } - - return transformedPlay; + return await this.doHandle(data, play, transformData); } + protected abstract doHandle(data: StageConfig, play: PlayObject, transformData: T): Promise; + public async getTransformerData(play: PlayObject): Promise { return undefined; } @@ -134,13 +80,4 @@ export default abstract class AbstractTransformer extends AbstractIn public async checkShouldTransform(play: PlayObject, transformData: T): Promise { return; } - - protected abstract handleTitle(play: PlayObject, parts: Y, transformData: T): Promise; - protected abstract handleArtists(play: PlayObject, parts: Y, transformData: T): Promise; - protected abstract handleAlbumArtists(play: PlayObject, parts: Y, transformData: T): Promise; - protected abstract handleAlbum(play: PlayObject, parts: Y, transformData: T): Promise; - - protected async handleMeta(play: PlayObject, transformData: T): Promise { - return play.data.meta; - } } \ No newline at end of file diff --git a/src/backend/common/transforms/AtomicPartsTransformer.ts b/src/backend/common/transforms/AtomicPartsTransformer.ts new file mode 100644 index 00000000..3786f796 --- /dev/null +++ b/src/backend/common/transforms/AtomicPartsTransformer.ts @@ -0,0 +1,108 @@ +import { ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js"; +import { getRoot } from "../../ioc.js"; +import { testWhenConditions } from "../../utils/PlayTransformUtils.js"; +import AbstractInitializable from "../AbstractInitializable.js"; +import { AtomicStageConfig, ConditionalSearchAndReplaceRegExp, PlayTransformMetaParts, PlayTransformParts, PlayTransformUserParts } from "../infrastructure/Transform.js"; +import { cacheFunctions } from "@foxxmd/regex-buddy-core"; +import AbstractTransformer from "./AbstractTransformer.js"; + +export default abstract class AtomicPartsTransformer extends AbstractTransformer { + + protected async doHandle(parts: AtomicStageConfig, play: PlayObject, transformData: T): Promise { + + const { + throwOnFailure = false, + } = this.config.options || {}; + + try { + await this.checkShouldTransform(play, transformData); + } catch (e) { + this.logger.debug(new Error('checkShouldTransform did not pass, returning original Play', { cause: e })); + return play; + } + + const transformedPlayData: Partial = {}; + + if (parts.title !== undefined) { + try { + const title = await this.handleTitle(play, parts.title, transformData); + transformedPlayData.track = title; + } catch (e) { + const err = new Error(`Failed to transform title: ${play.data.track}`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('title'))) { + throw err; + } else { + this.logger.warn(err); + } + } + } + + if (parts.artists !== undefined) { + try { + const artists = await this.handleArtists(play, parts.artists, transformData); + transformedPlayData.artists = artists; + } catch (e) { + const err = new Error(`Failed to transform artists`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('artists'))) { + throw err; + } else { + this.logger.warn(err); + } + } + + try { + const albumArtists = await this.handleAlbumArtists(play, parts.artists, transformData); + transformedPlayData.albumArtists = albumArtists; + } catch (e) { + const err = new Error(`Failed to transform album artists`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('albumArtists'))) { + throw err; + } else { + this.logger.warn(err); + } + } + } + + if (parts.album !== undefined) { + try { + const album = await this.handleAlbum(play, parts.album, transformData); + transformedPlayData.album = album; + } catch (e) { + const err = new Error(`Failed to transform album: ${play.data.album}`, { cause: e }); + if (throwOnFailure === true || (throwOnFailure !== false && throwOnFailure.includes('album'))) { + throw err; + } else { + this.logger.warn(err); + } + } + } + + const transformedPlay = { + ...play, + data: { + ...play.data, + ...transformedPlayData + } + } + + return transformedPlay; + } + + public async getTransformerData(play: PlayObject): Promise { + return undefined; + } + + public async checkShouldTransform(play: PlayObject, transformData: T): Promise { + return; + } + + protected abstract handleTitle(play: PlayObject, parts: Y, transformData: T): Promise; + protected abstract handleArtists(play: PlayObject, parts: Y, transformData: T): Promise; + protected abstract handleAlbumArtists(play: PlayObject, parts: Y, transformData: T): Promise; + protected abstract handleAlbum(play: PlayObject, parts: Y, transformData: T): Promise; + + protected async handleMeta(play: PlayObject, transformData: T): Promise { + return play.data.meta; + } + +} \ No newline at end of file diff --git a/src/backend/common/transforms/TransformerManager.ts b/src/backend/common/transforms/TransformerManager.ts new file mode 100644 index 00000000..2a8ba543 --- /dev/null +++ b/src/backend/common/transforms/TransformerManager.ts @@ -0,0 +1,103 @@ +import { childLogger, Logger } from "@foxxmd/logging"; +import AbstractTransformer from "./AbstractTransformer.js"; +import { TransformerCommonConfig } from "../../../core/Atomic.js"; +import UserTransformer from "./UserTransformer.js"; +import { StageConfig } from "../infrastructure/Transform.js"; +import { PlayObject } from "../../../core/Atomic.js"; +import { isStageTyped } from "../../utils/PlayTransformUtils.js"; +import { MSCache } from "../Cache.js"; + +export default class TransformerManager { + + logger: Logger; + parentLogger: Logger; + transformers: Map = new Map(); + cache: MSCache; + + public constructor(logger: Logger, cache: MSCache) { + this.logger = childLogger(logger, 'Transformer Manager'); + this.parentLogger = logger; + this.cache = cache; + } + + public register(config: TransformerCommonConfig): void { + let transformers: AbstractTransformer[] = []; + if (!this.transformers.has(config.type)) { + this.transformers.set(config.type, []); + } else { + transformers = this.transformers.get(config.type); + } + + if (config.name !== undefined && transformers.some(x => x.config.name === config.name)) { + throw new Error(`Cannot register ${config.type} with name '${config.name}' because an existing transformer already has that name`); + } + const tName = config.name ?? `unnamed-${transformers.length + 1}`; + + let t: AbstractTransformer; + switch (config.type) { + case 'user': + t = new UserTransformer({ name: tName, logger: this.parentLogger, regexCache: this.cache.regexCache, ...config }); + break; + default: + throw new Error(`No transformer of type '${config.type}' exists.`); + } + + this.transformers.set(config.type, [...transformers, t]); + } + + public async initTransformers() { + for (const list of this.transformers.values()) { + for (const transformer of list) { + if (!transformer.isReady()) { + if (!transformer.canAuthUnattended()) { + transformer.logger.warn({ label: 'Heartbeat' }, 'Transformer is not ready but will not try to initialize because auth state is not good and cannot be correct unattended.'); + } + try { + await transformer.tryInitialize({ force: false, notify: true, notifyTitle: 'Could not initialize automatically' }); + } catch (e) { + transformer.logger.error(new Error('Could not initialize source automatically', { cause: e })); + } + } + } + } + } + + protected getTransformerByStage(data: StageConfig): AbstractTransformer { + const list = this.transformers.get(data.type); + if (list === undefined || list.length === 0) { + throw new Error(`No transformer of type '${data.type}' is registered.`); + } + + if (list.length > 0 && (data as any).name === undefined) { + this.logger.warn(`More than one '${data.type}' transformer but name was not specified, using first registered`); + return list[0]; + } else { + return list[0] + } + } + + public parseTransformerConfig(data: any) { + if (!isStageTyped(data)) { + throw new Error(`Must be an object with a 'type' property.`); + } + const t = this.getTransformerByStage(data); + return t.parseConfig(data); + } + + public async handleStage(data: StageConfig, play: PlayObject): Promise { + const list = this.transformers.get(data.type); + if (list === undefined || list.length === 0) { + throw new Error(`No transformer of type '${data.type}' is registered.`); + } + + let t: AbstractTransformer; + if (list.length > 0 && (data as any).name === undefined) { + this.logger.warn(`More than one '${data.type}' transformer but name was not specified, using first registered`); + t = list[0]; + } else { + t = list[0]; + } + + return await t.handle(data, play); + } +} \ No newline at end of file diff --git a/src/backend/common/transforms/UserTransformer.ts b/src/backend/common/transforms/UserTransformer.ts index 1b315870..77713a76 100644 --- a/src/backend/common/transforms/UserTransformer.ts +++ b/src/backend/common/transforms/UserTransformer.ts @@ -1,14 +1,42 @@ import { searchAndReplace } from "@foxxmd/regex-buddy-core"; import { PlayObject } from "../../../core/Atomic.js"; -import { testWhenConditions } from "../../utils/PlayTransformUtils.js"; +import { configValToSearchReplace, isSearchAndReplaceTerm, isStageTyped, isUserStage, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; -import { ConditionalSearchAndReplaceRegExp, PlayTransformUserParts } from "../infrastructure/Transform.js"; +import { ConditionalSearchAndReplaceRegExp, PlayTransformUserParts, PlayTransformUserStage, StageConfig } from "../infrastructure/Transform.js"; import AbstractTransformer, { TransformerCommon } from "./AbstractTransformer.js" +import AtomicPartsTransformer from "./AtomicPartsTransformer.js"; -export default class UserTransformer extends AbstractTransformer { +export default class UserTransformer extends AtomicPartsTransformer { - protected constructor(config: TransformerCommon) { - super(config); + // protected constructor(config: TransformerCommon) { + // super(name, config); + // } + + protected doParseConfig(data: StageConfig) { + if (!isUserStage(data)) { + throw new Error(`UserTransformer is only usable with 'user' type stages`); + } + + const stage: PlayTransformUserStage = { + ...data, + type: 'user' + } + + for (const k of ['artists', 'title', 'album']) { + if (!(k in data)) { + continue; + } + if (!Array.isArray(data[k])) { + throw new Error(`${k} must be an array`); + } + try { + isSearchAndReplaceTerm(data[k]); + stage[k] = data[k].map(configValToSearchReplace); + } catch (e) { + throw new Error(`Property '${k}' was not a valid type`, { cause: e }); + } + } + return stage; } protected generateMapper(play: PlayObject) { @@ -20,7 +48,11 @@ export default class UserTransformer extends AbstractTransformer { if(play.data.artists === undefined || play.data.artists.length === 0) { @@ -51,11 +83,15 @@ export default class UserTransformer extends AbstractTransformer { - if (play.data.track === undefined) { + if (play.data.album === undefined) { return undefined; } const mapper = this.generateMapper(play); - return searchAndReplace(play.data.album, parts.map(mapper)); + const result = searchAndReplace(play.data.album, parts.map(mapper)); + if(result.trim() === '') { + return undefined; + } + return result.trim(); } public notify(payload: WebhookPayload): Promise { diff --git a/src/backend/index.ts b/src/backend/index.ts index aa85c734..4c94afb2 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -119,6 +119,8 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) const notifiers = new Notifiers(root.get('notifierEmitter'), root.get('clientEmitter'), root.get('sourceEmitter'), root.get('logger')); //root.get('notifiers'); await notifiers.buildWebhooks(webhooks); + await root.items.transformerManager.initTransformers(); + /* * setup clients * */ diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index f1a14c45..8dea51d4 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -10,6 +10,8 @@ import { generateBaseURL } from "./utils/NetworkUtils.js"; import { PassThrough } from "stream"; import { CacheConfigOptions } from "./common/infrastructure/Atomic.js"; import { MSCache } from "./common/Cache.js"; +import TransformerManager from "./common/transforms/TransformerManager.js"; +import { TransformerCommonConfig } from "../core/Atomic.js"; export let version: string = 'unknown'; @@ -27,6 +29,7 @@ export interface RootOptions { loggerStream?: PassThrough loggingConfig?: LogOptions cache?: CacheConfigOptions | MSCache | (() => MSCache) + transformerConfigs?: TransformerCommonConfig[] } const createRoot = (options: RootOptions = {logger: loggerDebug}) => { @@ -37,7 +40,8 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { loggerStream, loggingConfig, logger, - cache + cache, + transformerConfigs = [], } = options || {}; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); let disableWeb = dw; @@ -65,6 +69,16 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { const f = e; }); + const transformerManager = new TransformerManager(logger, maybeSingletonCache !== undefined ? maybeSingletonCache : cacheFunc()); + transformerManager.register({type: 'user'}); + for(const c of transformerConfigs) { + try { + transformerManager.register(c); + } catch (e) { + logger.warn(new Error('Could not register a transformer', {cause: e})); + } + } + const portVal: number | string = process.env.PORT ?? port; return createContainer().add({ @@ -80,6 +94,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { loggerStream, loggingConfig, logger: logger, + transformerManager, cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc }).add((items) => { const localUrl = generateBaseURL(baseUrl, items.port) diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index ec28c2e9..396bb5bc 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -56,6 +56,7 @@ import { AsyncTask, SimpleIntervalJob, Task, ToadScheduler } from "toad-schedule import { MSCache } from "../common/Cache.js"; import { getRoot } from "../ioc.js"; import { rehydratePlay } from "../utils/CacheUtils.js"; +import { findAsyncSequential } from "../utils/AsyncUtils.js"; type PlatformMappedPlays = Map; type NowPlayingQueue = Map; @@ -469,12 +470,12 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i getScrobbledPlays = () => this.scrobbledPlayObjs.data.map(x => x.scrobble) - findExistingSubmittedPlayObj = (playObjPre: PlayObject): ([undefined, undefined] | [ScrobbledPlayObject, ScrobbledPlayObject[]]) => { + findExistingSubmittedPlayObj = async (playObjPre: PlayObject): Promise<([undefined, undefined] | [ScrobbledPlayObject, ScrobbledPlayObject[]])> => { - const playObj = this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate); + const playObj = await this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate); - const dtInvariantMatches = this.scrobbledPlayObjs.data - .map(x => ({...x, play: this.transformPlay(x.play, TRANSFORM_HOOK.existing)})) + const dtInvariantMatches = (await Promise.all(this.scrobbledPlayObjs.data + .map(async x => ({...x, play: await this.transformPlay(x.play, TRANSFORM_HOOK.existing)})))) .filter(x => playObjDataMatch(playObj, x.play)); if (dtInvariantMatches.length === 0) { @@ -514,7 +515,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i existingScrobble = async (playObjPre: PlayObject) => { - const playObj = this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate); + const playObj = await this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate); const tr = truncateStringToLength(27); const scoreTrackOpts: TrackStringOptions = {include: ['track', 'artist', 'time'], transformers: {track: (t: any, data, existing) => `${existing ? '- ': ''}${tr(t)}`}}; @@ -531,7 +532,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i let closestMatch: {score: number, breakdowns: string[], confidence: string, scrobble?: PlayObject} = {score: 0, breakdowns: [], confidence: 'No existing scrobble matched with a score higher than 0'}; // then check if we have already recorded this - const [existingExactSubmitted, existingDataSubmitted = []] = this.findExistingSubmittedPlayObj(playObjPre); + const [existingExactSubmitted, existingDataSubmitted = []] = await this.findExistingSubmittedPlayObj(playObjPre); // if we have an submitted play with matching data and play date then we can just return the response from the original scrobble if (existingExactSubmitted !== undefined) { @@ -567,9 +568,10 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i // -- this is info we only know if play was generated from MS player so we can be reasonably sure const looseTimeAccuracy = playObj.data.repeat ? [TA_DURING] : [TA_FUZZY, TA_DURING]; - existingScrobble = this.recentScrobbles.find((xPre) => { + + existingScrobble = findAsyncSequential(this.recentScrobbles, async (xPre) => { - const x = this.transformPlay(xPre, TRANSFORM_HOOK.existing); + const x = await this.transformPlay(xPre, TRANSFORM_HOOK.existing); //const referenceMatch = referenceApiScrobbleResponse !== undefined && playObjDataMatch(x, referenceApiScrobbleResponse); @@ -795,8 +797,8 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); const currQueuedPlay = this.queuedScrobbles.shift(); const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(currQueuedPlay.play); - if (timeFrameValid && !(await this.alreadyScrobbled(this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.preCompare)))) { - const transformedScrobble = this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.postCompare); + if (timeFrameValid && !(await this.alreadyScrobbled((await this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.preCompare))))) { + const transformedScrobble = await this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.postCompare); try { const scrobbledPlay = await this.scrobble(transformedScrobble); this.emitEvent('scrobble', {play: transformedScrobble}); @@ -880,8 +882,8 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); await this.refreshScrobbles(); } const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(deadScrobble.play); - if (timeFrameValid && !(await this.alreadyScrobbled(this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.preCompare)))) { - const transformedScrobble = this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.postCompare); + if (timeFrameValid && !(await this.alreadyScrobbled((await this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.preCompare))))) { + const transformedScrobble = await this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.postCompare); try { const scrobbledPlay = await this.scrobble(transformedScrobble); this.emitEvent('scrobble', {play: transformedScrobble}); diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index da8ebaeb..7fc3cc28 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -40,6 +40,7 @@ import { componentFileLogger } from '../common/logging.js'; import { WebhookPayload } from '../common/infrastructure/config/health/webhooks.js'; import { messageWithCauses, messageWithCausesTruncatedDefault } from '../utils/ErrorUtils.js'; import { genericSourcePlayMatch } from '../utils/PlayComparisonUtils.js'; +import { findAsync } from '../utils/AsyncUtils.js'; export interface RecentlyPlayedOptions { limit?: number @@ -173,12 +174,13 @@ export default abstract class AbstractSource extends AbstractComponent implement return lists; } - existingDiscovered = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject | undefined => { + existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise => { const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts); - const candidate = this.transformPlay(play, TRANSFORM_HOOK.candidate); + const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate); for(const list of lists) { - const existing = list.find(x => { - const e = this.transformPlay(x, TRANSFORM_HOOK.existing); + + const existing = await findAsync(list,async x => { + const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); return genericSourcePlayMatch(e, candidate); }); if(existing) { @@ -188,18 +190,18 @@ export default abstract class AbstractSource extends AbstractComponent implement return undefined; } - alreadyDiscovered = (play: PlayObject, opts: {checkAll?: boolean} = {}): boolean => { - const existing = this.existingDiscovered(play, opts); + alreadyDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise => { + const existing = await this.existingDiscovered(play, opts); return existing !== undefined; } - discover = (plays: PlayObject[], options: { checkAll?: boolean, [key: string]: any } = {}): PlayObject[] => { + discover = async (plays: PlayObject[], options: { checkAll?: boolean, [key: string]: any } = {}): Promise => { const newDiscoveredPlays: PlayObject[] = []; - const transformedPlayed = plays.map(x => this.transformPlay(x, TRANSFORM_HOOK.preCompare)); + const transformedPlayed = await Promise.all(plays.map(x => this.transformPlay(x, TRANSFORM_HOOK.preCompare))); for(const play of transformedPlayed) { - if(!this.alreadyDiscovered(play, options)) { + if(!(await this.alreadyDiscovered(play, options))) { this.addPlayToDiscovered(play); newDiscoveredPlays.push(play); } @@ -222,7 +224,7 @@ export default abstract class AbstractSource extends AbstractComponent implement } - protected scrobble = (newDiscoveredPlays: PlayObject[], options: { forceRefresh?: boolean, [key: string]: any, discoverLocation?: 'backlog' | [key: string] } = {}) => { + protected scrobble = async (newDiscoveredPlays: PlayObject[], options: { forceRefresh?: boolean, [key: string]: any, discoverLocation?: 'backlog' | [key: string] } = {}) => { if(newDiscoveredPlays.length > 0) { if(!this.shouldScrobble(options.discoverLocation)) { @@ -230,7 +232,7 @@ export default abstract class AbstractSource extends AbstractComponent implement } newDiscoveredPlays.sort(sortByOldestPlayDate); this.emitter.emit('discoveredToScrobble', { - data: newDiscoveredPlays.map(x => this.transformPlay(x, TRANSFORM_HOOK.postCompare)), + data: await Promise.all(newDiscoveredPlays.map(x => this.transformPlay(x, TRANSFORM_HOOK.postCompare))), options: { ...options, checkTime: newDiscoveredPlays[newDiscoveredPlays.length-1].data.playDate.add(2, 'second'), @@ -259,7 +261,7 @@ export default abstract class AbstractSource extends AbstractComponent implement } catch (e) { throw new Error('Error occurred while fetching backlogged plays', {cause: e}); } - const discovered = this.discover(backlogPlays, {discoverLocation: 'backlog'}); + const discovered = await this.discover(backlogPlays, {discoverLocation: 'backlog'}); const { options: { @@ -270,7 +272,7 @@ export default abstract class AbstractSource extends AbstractComponent implement if (scrobbleBacklog) { if (discovered.length > 0) { this.logger.info('Scrobbling backlogged tracks...'); - this.scrobble(discovered); + await this.scrobble(discovered); this.logger.info('Backlog scrobbling complete.'); } else { this.logger.info('All tracks already discovered!'); @@ -451,7 +453,7 @@ export default abstract class AbstractSource extends AbstractComponent implement this.logger.info(`Potential plays were discovered close to polling interval! Delaying scrobble clients refresh by ${maxDelay} seconds so other clients have time to scrobble first`); await sleep(maxDelay * 1000); } - newDiscovered = this.discover(playObjs); + newDiscovered = await this.discover(playObjs); this.scrobble(newDiscovered, { forceRefresh: closeToInterval diff --git a/src/backend/sources/AzuracastSource.ts b/src/backend/sources/AzuracastSource.ts index b9c8b9bf..7b058079 100644 --- a/src/backend/sources/AzuracastSource.ts +++ b/src/backend/sources/AzuracastSource.ts @@ -224,7 +224,7 @@ export class AzuracastSource extends MemorySource { position: online && play !== undefined ? play.meta.trackProgressPosition : undefined } - return this.processRecentPlays([playerState]); + return await this.processRecentPlays([playerState]); } } diff --git a/src/backend/sources/ChromecastSource.ts b/src/backend/sources/ChromecastSource.ts index 0c11f617..359d0124 100644 --- a/src/backend/sources/ChromecastSource.ts +++ b/src/backend/sources/ChromecastSource.ts @@ -641,7 +641,7 @@ export class ChromecastSource extends MemoryPositionalSource { } } - const playsToReturn = this.processRecentPlays(plays); + const playsToReturn = await this.processRecentPlays(plays); this.pruneApplications(); diff --git a/src/backend/sources/DeezerInternalSource.ts b/src/backend/sources/DeezerInternalSource.ts index 00d879c2..4451b50b 100644 --- a/src/backend/sources/DeezerInternalSource.ts +++ b/src/backend/sources/DeezerInternalSource.ts @@ -12,6 +12,7 @@ import { MixedCookieAgent } from 'http-cookie-agent/http'; import MemorySource from "./MemorySource.js"; import { genericSourcePlayMatch } from "../utils/PlayComparisonUtils.js"; import { TemporalPlayComparisonOptions } from "../utils/TimeUtils.js"; +import { findAsync, findIndexAsync } from "../utils/AsyncUtils.js"; interface DeezerHistoryResponse { errors: [] @@ -202,20 +203,20 @@ export default class DeezerInternalSource extends MemorySource { protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getRecentlyPlayed({formatted: true, ...options}) - existingDiscovered = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject | undefined => { + existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise => { const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts); - const candidate = this.transformPlay(play, TRANSFORM_HOOK.candidate); + const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate); for(const list of lists) { - const existing = list.find(x => { - const e = this.transformPlay(x, TRANSFORM_HOOK.existing); + const existing = await findAsync(list, async x => { + const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); return genericSourcePlayMatch(e, candidate); }); if(existing) { return existing; } if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { - const fuzzyIndex = list.findIndex(x => { - const e = this.transformPlay(x, TRANSFORM_HOOK.existing); + const fuzzyIndex = await findIndexAsync(list, async x => { + const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); let temporalOptions: TemporalPlayComparisonOptions = {}; const temporalAccuracy: TemporalAccuracy[] = [TA_EXACT, TA_CLOSE, TA_FUZZY]; if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { diff --git a/src/backend/sources/EndpointLastfmSource.ts b/src/backend/sources/EndpointLastfmSource.ts index 340ba711..0dd71af4 100644 --- a/src/backend/sources/EndpointLastfmSource.ts +++ b/src/backend/sources/EndpointLastfmSource.ts @@ -72,12 +72,12 @@ export class EndpointLastfmSource extends MemorySource { handle = async (stateData: PlayerStateData) => { - this.processRecentPlays([stateData]); + await this.processRecentPlays([stateData]); if (stateData.play.meta.nowPlaying === false && this.isValidScrobble(stateData.play)) { - const discovered = this.discover([stateData.play]); + const discovered = await this.discover([stateData.play]); if (discovered.length > 0) { - this.scrobble(discovered); + await this.scrobble(discovered); } } } diff --git a/src/backend/sources/EndpointListenbrainzSource.ts b/src/backend/sources/EndpointListenbrainzSource.ts index 42249ff1..2c508ab9 100644 --- a/src/backend/sources/EndpointListenbrainzSource.ts +++ b/src/backend/sources/EndpointListenbrainzSource.ts @@ -89,12 +89,12 @@ export class EndpointListenbrainzSource extends MemorySource { handle = async (stateData: PlayerStateData) => { - this.processRecentPlays([stateData]); + await this.processRecentPlays([stateData]); if (stateData.play.meta.nowPlaying === false && this.isValidScrobble(stateData.play)) { - const discovered = this.discover([stateData.play]); + const discovered = await this.discover([stateData.play]); if (discovered.length > 0) { - this.scrobble(discovered); + await this.scrobble(discovered); } } } diff --git a/src/backend/sources/IcecastSource.ts b/src/backend/sources/IcecastSource.ts index 033b3d0b..7855ec84 100644 --- a/src/backend/sources/IcecastSource.ts +++ b/src/backend/sources/IcecastSource.ts @@ -122,7 +122,7 @@ export class IcecastSource extends MemorySource { } if (this.currentMetadata === undefined) { - return this.processRecentPlays([]); + return await this.processRecentPlays([]); } // if (this.manualListening === false || (this.config.options.scrobbleOnStart === false && this.manualListening === undefined)) { @@ -146,7 +146,7 @@ export class IcecastSource extends MemorySource { play } - return this.processRecentPlays([playerState]); + return await this.processRecentPlays([playerState]); } } diff --git a/src/backend/sources/JRiverSource.ts b/src/backend/sources/JRiverSource.ts index 35866244..fb09ce8d 100644 --- a/src/backend/sources/JRiverSource.ts +++ b/src/backend/sources/JRiverSource.ts @@ -142,7 +142,7 @@ export class JRiverSource extends MemoryPositionalSource { } } - return this.processRecentPlays(play); + return await this.processRecentPlays(play); } } diff --git a/src/backend/sources/JellyfinApiSource.ts b/src/backend/sources/JellyfinApiSource.ts index 97f63853..6f865dd9 100644 --- a/src/backend/sources/JellyfinApiSource.ts +++ b/src/backend/sources/JellyfinApiSource.ts @@ -497,7 +497,7 @@ export default class JellyfinApiSource extends MemoryPositionalSource { this.logger[this.logFilterFailure](dropReason); } } - return this.processRecentPlays(validSessions); + return await this.processRecentPlays(validSessions); } sessionToPlayerState = (obj: SessionInfo): PlayerStateDataMaybePlay => { diff --git a/src/backend/sources/JellyfinSource.ts b/src/backend/sources/JellyfinSource.ts index 6e9aa002..c50c4c9b 100644 --- a/src/backend/sources/JellyfinSource.ts +++ b/src/backend/sources/JellyfinSource.ts @@ -354,13 +354,13 @@ export default class JellyfinSource extends MemorySource { scrobbleOpts.checkAll = true; } else { - newPlays = this.processRecentPlays([playObj]); + newPlays = await this.processRecentPlays([playObj]); } if(newPlays.length > 0) { try { - const discovered = this.discover(newPlays, scrobbleOpts); - this.scrobble(discovered); + const discovered = await this.discover(newPlays, scrobbleOpts); + await this.scrobble(discovered); } catch (e) { this.logger.error('Encountered error while scrobbling') this.logger.error(e) diff --git a/src/backend/sources/KodiSource.ts b/src/backend/sources/KodiSource.ts index d670bff8..5ccb6155 100644 --- a/src/backend/sources/KodiSource.ts +++ b/src/backend/sources/KodiSource.ts @@ -59,7 +59,7 @@ export class KodiSource extends MemoryPositionalSource { const play = await this.client.getRecentlyPlayed(options); - return this.processRecentPlays(play); + return await this.processRecentPlays(play); } } diff --git a/src/backend/sources/KoitoSource.ts b/src/backend/sources/KoitoSource.ts index 0692f923..3c1940ec 100644 --- a/src/backend/sources/KoitoSource.ts +++ b/src/backend/sources/KoitoSource.ts @@ -61,7 +61,7 @@ export default class KoitoSource extends MemorySource { getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => { const {limit = 20} = options; - this.processRecentPlays([]); + await this.processRecentPlays([]); return await this.api.getRecentlyPlayed(limit); } diff --git a/src/backend/sources/LastfmSource.ts b/src/backend/sources/LastfmSource.ts index 4bc7aadf..86ea52da 100644 --- a/src/backend/sources/LastfmSource.ts +++ b/src/backend/sources/LastfmSource.ts @@ -131,7 +131,7 @@ export default class LastfmSource extends MemorySource { getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}): Promise => { try { const [history, now] = await this.getLastfmRecentTrack(options); - this.processRecentPlays(now); + await this.processRecentPlays(now); return history; } catch (e) { throw e; diff --git a/src/backend/sources/ListenbrainzSource.ts b/src/backend/sources/ListenbrainzSource.ts index 68def67f..b1d35a53 100644 --- a/src/backend/sources/ListenbrainzSource.ts +++ b/src/backend/sources/ListenbrainzSource.ts @@ -79,7 +79,7 @@ export default class ListenbrainzSource extends MemorySource { return await this.api.getRecentlyPlayedKoito(limit); } const now = await this.api.getPlayingNow(); - this.processRecentPlays(now.listens.map(x => ListenbrainzSource.formatPlayObj(x))); + await this.processRecentPlays(now.listens.map(x => ListenbrainzSource.formatPlayObj(x))); return await this.api.getRecentlyPlayed(limit); } diff --git a/src/backend/sources/MPDSource.ts b/src/backend/sources/MPDSource.ts index bf00efc4..de17de8b 100644 --- a/src/backend/sources/MPDSource.ts +++ b/src/backend/sources/MPDSource.ts @@ -228,7 +228,7 @@ export class MPDSource extends MemoryPositionalSource { position: state.elapsed } - return this.processRecentPlays([playerState]); + return await this.processRecentPlays([playerState]); } } diff --git a/src/backend/sources/MPRISSource.ts b/src/backend/sources/MPRISSource.ts index 3ee7de58..c0c140ff 100644 --- a/src/backend/sources/MPRISSource.ts +++ b/src/backend/sources/MPRISSource.ts @@ -243,7 +243,7 @@ export class MPRISSource extends MemorySource { if(options.display === true) { return deduped; } - return this.processRecentPlays(deduped); + return await this.processRecentPlays(deduped); } } diff --git a/src/backend/sources/MalojaSource.ts b/src/backend/sources/MalojaSource.ts index bab09c93..d456d172 100644 --- a/src/backend/sources/MalojaSource.ts +++ b/src/backend/sources/MalojaSource.ts @@ -55,7 +55,7 @@ export default class MalojaSource extends MemorySource { getRecentlyPlayed = async (options: RecentlyPlayedOptions = {}) => { const { limit = 20 } = options; - this.processRecentPlays([]); + await this.processRecentPlays([]); return await this.api.getRecentScrobbles(limit); } diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts index 2ff594fe..f7b1d1ef 100644 --- a/src/backend/sources/MemorySource.ts +++ b/src/backend/sources/MemorySource.ts @@ -1,7 +1,7 @@ import { Logger } from "@foxxmd/logging"; import dayjs, { Dayjs } from "dayjs"; import { EventEmitter } from "events"; -import { SimpleIntervalJob, Task, ToadScheduler } from "toad-scheduler"; +import { AsyncTask, SimpleIntervalJob, Task, ToadScheduler } from "toad-scheduler"; import { PlayObject, SOURCE_SOT, SOURCE_SOT_TYPES, SourcePlayerObj } from "../../core/Atomic.js"; import { buildTrackString } from "../../core/StringUtils.js"; import { @@ -24,6 +24,7 @@ import { thresholdResultSummary, } from "../utils.js"; import { timePassesScrobbleThreshold, timeToHumanTimestamp } from "../utils/TimeUtils.js"; +import { PromisePool } from "@supercharge/promise-pool"; import AbstractSource from "./AbstractSource.js"; import { AbstractPlayerState, createPlayerOptions, PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js"; import { GenericPlayerState } from "./PlayerState/GenericPlayerState.js"; @@ -56,10 +57,17 @@ export default class MemorySource extends AbstractSource { // player cleanup on *schedule* is needed when the Source is non-polling (ingress) // because if the source stops sending updates then processRecentPlays() was never called so we never remove old players - this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({seconds: 15}, new Task('Player Cleanup', () => { - if(!this.canPoll) { - this.cleanupPlayers(); + this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({ seconds: 15 }, new AsyncTask('Player Cleanup', (): Promise => { + if (!this.canPoll) { + return Promise.resolve(); } + return PromisePool + .withConcurrency(1) + .for(this.players.keys()) + .process(async (key) => { + + await this.cleanupPlayer(key); + }); }))); } @@ -69,7 +77,7 @@ export default class MemorySource extends AbstractSource { } } - cleanupPlayer = (key: string): PlayObject | undefined => { + cleanupPlayer = async (key: string): Promise => { const player = this.players.get(key); if(player === undefined) { this.logger.warn({labels: 'Player Cleanup'},`No Player with ID ${key} exists! Cannot cleanup.`); @@ -113,7 +121,7 @@ export default class MemorySource extends AbstractSource { const cleanupPlay = player.getPlayedObject(true); let discoverablePlay: boolean; if(cleanupPlay !== undefined) { - const [discoverable, discoverableReason] = this.isListenedPlayDiscoverable(cleanupPlay); + const [discoverable, discoverableReason] = await this.isListenedPlayDiscoverable(cleanupPlay); discoverablePlay = discoverable; if(this.playerSourceOfTruth === SOURCE_SOT.PLAYER) { player.logger.verbose({labels: label}, discoverableReason); @@ -188,7 +196,7 @@ export default class MemorySource extends AbstractSource { return sessions[0]; } - processRecentPlays = (datas: (PlayObject | PlayerStateDataMaybePlay)[], reportedTS?: Dayjs) => { + processRecentPlays = async (datas: (PlayObject | PlayerStateDataMaybePlay)[], reportedTS?: Dayjs) => { const { options: { @@ -250,7 +258,7 @@ export default class MemorySource extends AbstractSource { // wait to discover play until it is stale or current play has changed // so that our discovered track has an accurate "listenedFor" count if (candidate !== undefined && (playChanged || player.isUpdateStale())) { - const [discoverable, discoverableReason] = this.isListenedPlayDiscoverable(candidate); + const [discoverable, discoverableReason] = await this.isListenedPlayDiscoverable(candidate); if(discoverable) { if(this.playerSourceOfTruth === SOURCE_SOT.PLAYER) { player.logger.verbose(discoverableReason); @@ -273,7 +281,7 @@ export default class MemorySource extends AbstractSource { } }); } else { - const playFromCleanup = this.cleanupPlayer(key); + const playFromCleanup = await this.cleanupPlayer(key); if(playFromCleanup !== undefined) { newStatefulPlays.push(playFromCleanup); } @@ -283,7 +291,7 @@ export default class MemorySource extends AbstractSource { return newStatefulPlays; } - protected isListenedPlayDiscoverable = (candidate: PlayObject): [boolean, string] => { + protected isListenedPlayDiscoverable = async (candidate: PlayObject): Promise<[boolean, string]> => { const { options: { @@ -295,7 +303,7 @@ export default class MemorySource extends AbstractSource { const thresholdResults = timePassesScrobbleThreshold(scrobbleThresholds, candidate.data.listenedFor, candidate.data.duration); if (thresholdResults.passes) { - const matchingRecent = this.existingDiscovered(candidate); //sRecentlyPlayed.find(x => playObjDataMatch(x, candidate)); + const matchingRecent = await this.existingDiscovered(candidate); //sRecentlyPlayed.find(x => playObjDataMatch(x, candidate)); if (matchingRecent === undefined) { return [true,`${stPrefix} added after ${thresholdResultSummary(thresholdResults)} and not matching any prior plays`]; } else { diff --git a/src/backend/sources/MopidySource.ts b/src/backend/sources/MopidySource.ts index 7f5357f5..48fd6b0a 100644 --- a/src/backend/sources/MopidySource.ts +++ b/src/backend/sources/MopidySource.ts @@ -213,7 +213,7 @@ export class MopidySource extends MemoryPositionalSource { play } - return this.processRecentPlays([playerState]); + return await this.processRecentPlays([playerState]); } } diff --git a/src/backend/sources/MusicCastSource.ts b/src/backend/sources/MusicCastSource.ts index b02b1ec6..04b7756a 100644 --- a/src/backend/sources/MusicCastSource.ts +++ b/src/backend/sources/MusicCastSource.ts @@ -112,12 +112,12 @@ export class MusicCastSource extends MemoryPositionalSource { } if ((statusResp.body as DeviceStatusResponse).power !== 'on') { this.logger.debug('MusicCast device is offline'); - return this.processRecentPlays([]); + return await this.processRecentPlays([]); } const playInfo = await this.getAnyPlayInfo(); if(playInfo === undefined) { - return this.processRecentPlays([]); + return await this.processRecentPlays([]); } const play = formatPlayObj(playInfo); @@ -130,7 +130,7 @@ export class MusicCastSource extends MemoryPositionalSource { position: play.meta.trackProgressPosition } - return this.processRecentPlays([playerState]); + return await this.processRecentPlays([playerState]); } } diff --git a/src/backend/sources/MusikcubeSource.ts b/src/backend/sources/MusikcubeSource.ts index d00bea90..0c19c74b 100644 --- a/src/backend/sources/MusikcubeSource.ts +++ b/src/backend/sources/MusikcubeSource.ts @@ -240,7 +240,7 @@ export class MusikcubeSource extends MemoryPositionalSource { position: playbackOverview.options.playing_current_time } - return this.processRecentPlays([playerState]); + return await this.processRecentPlays([playerState]); } } diff --git a/src/backend/sources/PlexApiSource.ts b/src/backend/sources/PlexApiSource.ts index 775198cc..8e538e77 100644 --- a/src/backend/sources/PlexApiSource.ts +++ b/src/backend/sources/PlexApiSource.ts @@ -406,7 +406,7 @@ export default class PlexApiSource extends MemoryPositionalSource { } } } - return this.processRecentPlays(validSessions); + return await this.processRecentPlays(validSessions); } getSourceArt = async (data: string): Promise<[Readable, string]> => { diff --git a/src/backend/sources/PlexSource.ts b/src/backend/sources/PlexSource.ts index 6f7daf69..1d754d1f 100644 --- a/src/backend/sources/PlexSource.ts +++ b/src/backend/sources/PlexSource.ts @@ -223,8 +223,8 @@ export default class PlexSource extends AbstractSource { } try { - const discovered = this.discover([playObj]); - this.scrobble(discovered); + const discovered = await this.discover([playObj]); + await this.scrobble(discovered); } catch (e) { this.logger.error('Encountered error while scrobbling') this.logger.error(e) diff --git a/src/backend/sources/SpotifySource.ts b/src/backend/sources/SpotifySource.ts index 1ef35205..87f6f04f 100644 --- a/src/backend/sources/SpotifySource.ts +++ b/src/backend/sources/SpotifySource.ts @@ -356,7 +356,7 @@ export default class SpotifySource extends MemoryPositionalSource { plays.push(currPlay); } } - const newPlays = this.processRecentPlays(plays); + const newPlays = await this.processRecentPlays(plays); // hint that scrobble timestamp source of truth should be when the track ended (player changed tracks) // rather than when we first saw the track // diff --git a/src/backend/sources/SubsonicSource.ts b/src/backend/sources/SubsonicSource.ts index 1e83b661..e18308ee 100644 --- a/src/backend/sources/SubsonicSource.ts +++ b/src/backend/sources/SubsonicSource.ts @@ -292,7 +292,7 @@ export class SubsonicSource extends MemorySource { // sometimes subsonic sources will return the same track as being played twice on the same player, need to remove this so we don't duplicate plays const deduped = removeDuplicates(entry.map(x => SubsonicSource.formatPlayObj(x, {sourceData: this.sourceData}))); const userFiltered = this.usersAllow.length == 0 ? deduped : deduped.filter(x => x.meta.user === undefined || this.usersAllow.map(x => x.toLocaleLowerCase()).includes(x.meta.user.toLocaleLowerCase())); - return this.processRecentPlays(userFiltered); + return await this.processRecentPlays(userFiltered); } getNewPlayer = (logger: Logger, id: PlayPlatformId, opts: PlayerStateOptions) => new SubsonicPlayerState(logger, id, opts); diff --git a/src/backend/sources/TealfmSource.ts b/src/backend/sources/TealfmSource.ts index 3d6cd742..830c6130 100644 --- a/src/backend/sources/TealfmSource.ts +++ b/src/backend/sources/TealfmSource.ts @@ -95,7 +95,7 @@ export default class TealfmSource extends MemorySource { } catch (e) { throw new Error('Error occurred while trying to fetch records', {cause: e}); } - this.processRecentPlays([]); + await this.processRecentPlays([]); const plays = list.map(x => listRecordToPlay(x)); return plays; } diff --git a/src/backend/sources/VLCSource.ts b/src/backend/sources/VLCSource.ts index 485586f0..84c206b1 100644 --- a/src/backend/sources/VLCSource.ts +++ b/src/backend/sources/VLCSource.ts @@ -259,7 +259,7 @@ export class VLCSource extends MemoryPositionalSource { position: state.time } - return this.processRecentPlays([playerState]); + return await this.processRecentPlays([playerState]); } } diff --git a/src/backend/sources/WebScrobblerSource.ts b/src/backend/sources/WebScrobblerSource.ts index efff92ca..b42ce426 100644 --- a/src/backend/sources/WebScrobblerSource.ts +++ b/src/backend/sources/WebScrobblerSource.ts @@ -180,12 +180,12 @@ export class WebScrobblerSource extends MemorySource { handle = async (stateData: PlayerStateData) => { - this.processRecentPlays([stateData]); + await this.processRecentPlays([stateData]); if (stateData.play.meta.nowPlaying === false && this.isValidScrobble(stateData.play)) { - const discovered = this.discover([stateData.play]); + const discovered = await this.discover([stateData.play]); if (discovered.length > 0) { - this.scrobble(discovered); + await this.scrobble(discovered); } } } diff --git a/src/backend/tests/component/component.test.ts b/src/backend/tests/component/component.test.ts index 752e3a25..488d4ad1 100644 --- a/src/backend/tests/component/component.test.ts +++ b/src/backend/tests/component/component.test.ts @@ -4,7 +4,7 @@ import asPromised from 'chai-as-promised'; import { after, before, describe, it } from 'mocha'; import AbstractComponent from "../../common/AbstractComponent.js"; -import { ConditionalSearchAndReplaceRegExp, STAGE_TYPES, STAGE_TYPES_METADATA, TRANSFORM_HOOK } from "../../common/infrastructure/Transform.js"; +import { ConditionalSearchAndReplaceRegExp, STAGE_TYPES, STAGE_TYPES_METADATA, STAGE_TYPES_USER, TRANSFORM_HOOK } from "../../common/infrastructure/Transform.js"; import { isConditionalSearchAndReplace } from "../../utils/PlayTransformUtils.js"; import { asPlays, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js"; @@ -71,7 +71,6 @@ describe('Play Transforms', function () { preCompare: { // @ts-expect-error type: "test", - // @ts-expect-error title: ['something'] } } @@ -80,7 +79,7 @@ describe('Play Transforms', function () { // https://github.com/chaijs/chai/issues/655#issuecomment-204386414 expect(() => component.buildTransformRules()).to.throw(Error).that.satisfies((e) => { - return findCauseByMessage(e, `Stage has invalid 'type'`); + return findCauseByMessage(e, `No transformer of type 'test'`); }); }); @@ -221,49 +220,49 @@ describe('Play Transforms', function () { }); }); - describe('Non-User Stage Parsing', function () { + // describe('Non-User Stage Parsing', function () { - describe('Non-User Stage Types', function () { + // describe('Non-User Stage Types', function () { - for(const t of STAGE_TYPES_METADATA) { + // for(const t of STAGE_TYPES_USER) { - it(`Allows non-user Stage Type ${t}`, function () { - component.config = { - options: { - playTransform: { - preCompare: { - type: t, - title: true - } - } - } - } + // it(`Allows non-user Stage Type ${t}`, function () { + // component.config = { + // options: { + // playTransform: { + // preCompare: { + // type: t, + // title: true + // } + // } + // } + // } - expect(() => component.buildTransformRules()).to.not.throw(); - expect(component.transformRules.preCompare).to.be.an('array'); - expect(component.transformRules.preCompare).to.be.length(1); - expect(component.transformRules.preCompare).to.have.nested.property('0.type'); - expect(component.transformRules.preCompare[0].type).eq(t); - }); + // expect(() => component.buildTransformRules()).to.not.throw(); + // expect(component.transformRules.preCompare).to.be.an('array'); + // expect(component.transformRules.preCompare).to.be.length(1); + // expect(component.transformRules.preCompare).to.have.nested.property('0.type'); + // expect(component.transformRules.preCompare[0].type).eq(t); + // }); - } + // } - }); + // }); - }); + // }); describe('Play Transforming', function () { - it('Returns original play if no hooks are defined', function () { + it('Returns original play if no hooks are defined', async function () { component.buildTransformRules(); const play = generatePlay(); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(JSON.stringify(play)).equal(JSON.stringify(transformed)); }); describe('User Play Transforming', function () { - it('Transforms when hook is present', function () { + it('Transforms when hook is present', async function () { component.config = { options: { playTransform: { @@ -276,11 +275,11 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ track: 'My coolsomething track' }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.track).equal('My cool track'); }); - it('Transforms consecutively when hook is present with multiple values', function () { + it('Transforms consecutively when hook is present with multiple values', async function () { component.config = { options: { playTransform: { @@ -293,11 +292,11 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ track: 'My coolsomething track' }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.track).equal('My track'); }); - it('Transforms using parsed regex', function () { + it('Transforms using parsed regex', async function () { component.config = { options: { playTransform: { @@ -315,12 +314,12 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ track: 'My cool something track' }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.track).equal('My cool thing track'); }); - it('Transforms using parsed regex to get primary artist from delimited artist string', function () { + it('Transforms using parsed regex to get primary artist from delimited artist string', async function () { component.config = { options: { playTransform: { @@ -338,12 +337,12 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ artists: ['My Artist One / My Artist Two / Another Guy'] }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists).length(1) expect(transformed.data.artists[0]).equal('My Artist One'); }); - it('Removes title when transform replaces with empty string', function () { + it('Removes title when transform replaces with empty string', async function () { component.config = { options: { playTransform: { @@ -356,11 +355,11 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ track: 'something' }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.track).is.undefined; }); - it('Removes album when transform replaces with empty string', function () { + it('Removes album when transform replaces with empty string', async function () { component.config = { options: { playTransform: { @@ -373,11 +372,11 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ album: 'something' }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.album).is.undefined; }); - it('Removes an artist when transform replaces with empty string', function () { + it('Removes an artist when transform replaces with empty string', async function () { component.config = { options: { playTransform: { @@ -390,7 +389,7 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ artists: ['something', 'big'] }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists!.length).is.eq(1) expect(transformed.data.artists![0]).is.eq('big') }); @@ -403,7 +402,7 @@ describe('Play Transforms', function () { describe('Conditional Transforming', function () { describe('On Hook', function () { - it('Does not run hook if when conditions do not match', function () { + it('Does not run hook if when conditions do not match', async function () { component.config = { options: { playTransform: { @@ -421,12 +420,12 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ artists: ['something', 'big'], album: 'It Has No Match' }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists!.length).is.eq(2) expect(transformed.data.artists![0]).is.eq('something') }); - it('Does run hook if when conditions matches', function () { + it('Does run hook if when conditions matches', async function () { component.config = { options: { playTransform: { @@ -444,14 +443,14 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ artists: ['something', 'big'], album: 'It Has This Match' }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists!.length).is.eq(1) expect(transformed.data.artists![0]).is.eq('big') }); }); describe('On Search-And-Replace', function () { - it('Does not run hook if when conditions do not match', function () { + it('Does not run hook if when conditions do not match', async function () { component.config = { options: { playTransform: { @@ -474,12 +473,12 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ artists: ['something', 'big'], album: 'It Has No Match' }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists!.length).is.eq(2) expect(transformed.data.artists![0]).is.eq('something') }); - it('Does run hook if when conditions matches', function () { + it('Does run hook if when conditions matches', async function () { component.config = { options: { playTransform: { @@ -502,7 +501,7 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ artists: ['something', 'big'], album: 'It Has This Match' }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists!.length).is.eq(1) expect(transformed.data.artists![0]).is.eq('big') }); @@ -512,7 +511,7 @@ describe('Play Transforms', function () { describe('Multiple hook transforms', function () { - it('Accumulates transforms', function () { + it('Accumulates transforms', async function () { component.config = { options: { playTransform: { @@ -540,7 +539,7 @@ describe('Play Transforms', function () { component.buildTransformRules(); const play = generatePlay({ track: 'My cool something track' }); - const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.track).equal('My cool final thing track'); }); diff --git a/src/backend/tests/plays/playParsing.test.ts b/src/backend/tests/plays/playParsing.test.ts index ae12ce2e..201a9344 100644 --- a/src/backend/tests/plays/playParsing.test.ts +++ b/src/backend/tests/plays/playParsing.test.ts @@ -4,7 +4,20 @@ import asPromised from 'chai-as-promised'; import { after, before, describe, it } from 'mocha'; import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js"; -import { parseArtistCredits, parseContextAwareStringList, parseCredits } from "../../utils/StringUtils.js"; +import { parseArtistCredits, parseContextAwareStringList, parseCredits, parseTrackCredits, uniqueNormalizedStrArr } from "../../utils/StringUtils.js"; +import testData from '../utils/playTestData.json' with { type: "json" }; +import { intersect } from "../../utils.js"; +import { ExpectedResults } from "../utils/interfaces.js"; + +interface PlayTestFixture { + caseHints: string[] + data: { + track: string + artists: string[] + album?: string + } + expected: ExpectedResults +} describe('#PlayParse Parsing Artists from String', function() { @@ -137,4 +150,22 @@ Found => ${parsed.join(' || ')}`) }); +}); + +describe('Play Track Strings',function () { + + const testFixtures = testData as unknown as PlayTestFixture[]; + const joinerData = testFixtures.filter(x => intersect(['joiner','track'], x.caseHints).length === 2); + + it('should parse joiners from track title', function() { + for(const test of joinerData) { + const res = parseTrackCredits(test.data.track); + let artists: string[] = [...test.data.artists]; + if(res.secondary !== undefined) { + artists = uniqueNormalizedStrArr([...artists, ...res.secondary]); + } + assert.equal(res.primaryComposite, test.expected.track); + assert.sameDeepMembers(artists, test.expected.artists); + } + }); }); \ No newline at end of file diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index 1beb3e35..f860aa52 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -453,7 +453,7 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds') }); - const [matchedPlay, matchedData] = testScrobbler.findExistingSubmittedPlayObj(newScrobble); + const [matchedPlay, matchedData] = await testScrobbler.findExistingSubmittedPlayObj(newScrobble); assert.isUndefined(matchedPlay); assert.isEmpty(matchedData); @@ -465,7 +465,7 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble }); testScrobbler.addScrobbledTrack(newScrobble, newScrobble); - const [matchedPlay, matchedData] = testScrobbler.findExistingSubmittedPlayObj(newScrobble); + const [matchedPlay, matchedData] = await testScrobbler.findExistingSubmittedPlayObj(newScrobble); assert.isDefined(matchedPlay); assert.isNotEmpty(matchedData); @@ -480,7 +480,7 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble const dupScrobble = clone(newScrobble); dupScrobble.data.playDate = newScrobble.data.playDate.add(2, 'seconds'); - const [matchedPlay, matchedData] = testScrobbler.findExistingSubmittedPlayObj(dupScrobble); + const [matchedPlay, matchedData] = await testScrobbler.findExistingSubmittedPlayObj(dupScrobble); assert.isDefined(matchedPlay); assert.isNotEmpty(matchedData); diff --git a/src/backend/tests/source/source.test.ts b/src/backend/tests/source/source.test.ts index 860591b6..1056aeb8 100644 --- a/src/backend/tests/source/source.test.ts +++ b/src/backend/tests/source/source.test.ts @@ -51,7 +51,7 @@ describe('Sources use transform plays correctly', function () { source = generateSource(); }); - it('Transforms play on preCompare', function() { + it('Transforms play on preCompare', async function() { source.config.options = { playTransform: { preCompare: { @@ -68,7 +68,7 @@ describe('Sources use transform plays correctly', function () { const newScrobble = generatePlay({ track: 'my cool track' }); - const discovered = source.discover([newScrobble]) + const discovered = await source.discover([newScrobble]) expect(discovered.length).eq(1); expect(discovered[0].data.track).is.eq('my fun track'); }); @@ -90,7 +90,7 @@ describe('Sources use transform plays correctly', function () { const newScrobble = generatePlay({ track: 'my cool track' }); - const discovered = source.discover([newScrobble]) + const discovered = await source.discover([newScrobble]) expect(discovered.length).eq(1); expect(discovered[0].data.track).is.eq('my cool track'); @@ -101,7 +101,7 @@ describe('Sources use transform plays correctly', function () { expect(e.data[0].data.track).is.eq('my fun track'); }); - it('Transforms play existing comparison', function() { + it('Transforms play existing comparison', async function() { source.config.options = { playTransform: { compare: { @@ -120,14 +120,14 @@ describe('Sources use transform plays correctly', function () { const newScrobble = generatePlay({ track: 'my hugely cool and very different track title', }); - const discovered = source.discover([newScrobble]) + const discovered = await source.discover([newScrobble]) expect(discovered.length).eq(1); expect(discovered[0].data.track).is.eq('my hugely cool and very different track title'); - expect(source.discover([newScrobble]).length).is.eq(1); + expect((await source.discover([newScrobble])).length).is.eq(1); }); - it('Transforms play candidate comparison', function() { + it('Transforms play candidate comparison', async function() { source.config.options = { playTransform: { compare: { @@ -146,11 +146,11 @@ describe('Sources use transform plays correctly', function () { const newScrobble = generatePlay({ track: 'my hugely cool and very different track title', }); - const discovered = source.discover([newScrobble]) + const discovered = await source.discover([newScrobble]) expect(discovered.length).eq(1); expect(discovered[0].data.track).is.eq('my hugely cool and very different track title'); - expect(source.discover([newScrobble]).length).is.eq(1); + expect((await source.discover([newScrobble])).length).is.eq(1); }); }) @@ -200,7 +200,7 @@ describe('Player Cleanup', function () { const source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}}); const initialDate = dayjs(); const initialState = generatePlayerStateData({position: 0, playData: {duration: 50}, timestamp: initialDate, status: REPORTED_PLAYER_STATUSES.playing}); - expect(source.processRecentPlays([initialState]).length).to.be.eq(0); + expect((await source.processRecentPlays([initialState])).length).to.be.eq(0); let position = 0; let timeSince = 0; @@ -212,7 +212,7 @@ describe('Player Cleanup', function () { MockDate.set(initialDate.add(position, 'seconds').toDate()); await sleep(1); const advancedState = generatePlayerStateData({play: initialState.play, timestamp: dayjs(), position, status: REPORTED_PLAYER_STATUSES.playing}); - expect(source.processRecentPlays([advancedState]).length).to.be.eq(0); + expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0); } // simulate polling another 20 seconds without any updates from the Source @@ -220,12 +220,12 @@ describe('Player Cleanup', function () { timeSince += 10; MockDate.set(initialDate.add(timeSince, 'seconds').toDate()); await sleep(1); - expect(source.processRecentPlays([]).length).to.be.eq(0); + expect((await source.processRecentPlays([])).length).to.be.eq(0); } MockDate.set(initialDate.add(timeSince + 2, 'seconds').toDate()); await sleep(1); - const discoveredPlays = source.processRecentPlays([]); + const discoveredPlays = await source.processRecentPlays([]); // cleanup should discover stale play expect(discoveredPlays.length).to.be.eq(1); expect(discoveredPlays[0].data.listenedFor).closeTo(30, 2); @@ -244,7 +244,7 @@ describe('Player Cleanup', function () { const source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}}); const initialDate = dayjs(); const initialState = generatePlayerStateData({position: 0, playData: {duration: 50}, timestamp: initialDate, status: REPORTED_PLAYER_STATUSES.playing}); - expect(source.processRecentPlays([initialState]).length).to.be.eq(0); + expect((await source.processRecentPlays([initialState])).length).to.be.eq(0); let position = 0; let timeSince = 0; @@ -256,7 +256,7 @@ describe('Player Cleanup', function () { MockDate.set(initialDate.add(position, 'seconds').toDate()); await sleep(1); const advancedState = generatePlayerStateData({play: initialState.play, timestamp: dayjs(), position, status: REPORTED_PLAYER_STATUSES.playing}); - expect(source.processRecentPlays([advancedState]).length).to.be.eq(0); + expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0); } // simulate polling another 20 seconds without any updates from the Source @@ -264,14 +264,14 @@ describe('Player Cleanup', function () { timeSince += 10; MockDate.set(initialDate.add(timeSince, 'seconds').toDate()); await sleep(1); - expect(source.processRecentPlays([]).length).to.be.eq(0); + expect((await source.processRecentPlays([])).length).to.be.eq(0); } timeSince += 2; MockDate.set(initialDate.add(timeSince, 'seconds').toDate()); await sleep(1); - const discoveredPlays = source.processRecentPlays([]); + const discoveredPlays = await source.processRecentPlays([]); // cleanup should discover stale play expect(discoveredPlays.length).to.be.eq(1); expect(discoveredPlays[0].data.listenedFor).closeTo(30, 2); @@ -285,7 +285,7 @@ describe('Player Cleanup', function () { MockDate.set(initialDate.add(timeSince, 'seconds').toDate()); await sleep(1); const advancedState = generatePlayerStateData({play: initialState.play, timestamp: dayjs(), position, status: REPORTED_PLAYER_STATUSES.playing}); - expect(source.processRecentPlays([advancedState]).length).to.be.eq(0); + expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0); } timeSince += 10; @@ -294,7 +294,7 @@ describe('Player Cleanup', function () { // new Play const advancedState = generatePlayerStateData({timestamp: dayjs(), position: 0, status: REPORTED_PLAYER_STATUSES.playing}); // should not return play because it has only been played for ~20 seconds, less than 50% of duration - const plays = source.processRecentPlays([advancedState]) + const plays = await source.processRecentPlays([advancedState]) expect(plays.length).to.be.eq(0); } @@ -314,7 +314,7 @@ describe('Player Cleanup', function () { // if player incorrectly counted stale time then 30s of actual play + 20s of stale time > scrobble threshold of 50% of 90s const initialState = generatePlayerStateData({position: 0, playData: {duration: 90}, timestamp: initialDate, status: REPORTED_PLAYER_STATUSES.playing}); - expect(source.processRecentPlays([initialState]).length).to.be.eq(0); + expect((await source.processRecentPlays([initialState])).length).to.be.eq(0); let position = 0; let timeSince = 0; @@ -326,7 +326,7 @@ describe('Player Cleanup', function () { MockDate.set(initialDate.add(position, 'seconds').toDate()); await sleep(1); const advancedState = generatePlayerStateData({play: initialState.play, timestamp: initialDate, position, status: REPORTED_PLAYER_STATUSES.playing}); - expect(source.processRecentPlays([advancedState]).length).to.be.eq(0); + expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0); } // simulate polling another 20 seconds without any updates from the Source @@ -334,11 +334,11 @@ describe('Player Cleanup', function () { timeSince += 10; MockDate.set(initialDate.add(timeSince, 'seconds').toDate()); await sleep(1); - expect(source.processRecentPlays([]).length).to.be.eq(0); + expect((await source.processRecentPlays([])).length).to.be.eq(0); } MockDate.set(initialDate.add(timeSince + 2, 'seconds').toDate()); - const discoveredPlays = source.processRecentPlays([]); + const discoveredPlays = await source.processRecentPlays([]); // cleanup should not discover stale play expect(discoveredPlays.length).to.be.eq(0); @@ -359,7 +359,7 @@ describe('Player Cleanup', function () { // if player incorrectly counted stale time then 30s of actual play + 20s of stale time > scrobble threshold of 50% of 90s const initialState = generatePlayerStateData({position: 0, playData: {duration: 90}, timestamp: initialDate, status: REPORTED_PLAYER_STATUSES.playing}); - expect(source.processRecentPlays([initialState]).length).to.be.eq(0); + expect((await source.processRecentPlays([initialState])).length).to.be.eq(0); let position = 0; let timeSince = 0; @@ -371,7 +371,7 @@ describe('Player Cleanup', function () { MockDate.set(initialDate.add(position, 'seconds').toDate()); await sleep(1); const advancedState = generatePlayerStateData({play: initialState.play, timestamp: dayjs(), position, status: REPORTED_PLAYER_STATUSES.playing}); - expect(source.processRecentPlays([advancedState]).length).to.be.eq(0); + expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0); } // simulate polling another 20 seconds without any updates from the Source @@ -379,14 +379,14 @@ describe('Player Cleanup', function () { timeSince += 10; MockDate.set(initialDate.add(timeSince, 'seconds').toDate()); await sleep(1); - expect(source.processRecentPlays([]).length).to.be.eq(0); + expect((await source.processRecentPlays([])).length).to.be.eq(0); } timeSince += 2; MockDate.set(initialDate.add(timeSince, 'seconds').toDate()); await sleep(1); - const discoveredPlays = source.processRecentPlays([]); + const discoveredPlays = await source.processRecentPlays([]); // cleanup should not discover stale play expect(discoveredPlays.length).to.be.eq(0); @@ -401,7 +401,7 @@ describe('Player Cleanup', function () { MockDate.set(initialDate.add(position, 'seconds').toDate()); await sleep(1); const advancedState = generatePlayerStateData({play: initialState.play, timestamp: dayjs(), position, status: REPORTED_PLAYER_STATUSES.playing}); - expect(source.processRecentPlays([advancedState]).length).to.be.eq(0); + expect((await source.processRecentPlays([advancedState])).length).to.be.eq(0); } timeSince += 10; @@ -410,7 +410,7 @@ describe('Player Cleanup', function () { // new Play const advancedState = generatePlayerStateData({timestamp: dayjs(), position: 0, status: REPORTED_PLAYER_STATUSES.playing}); // should return discovered play with ~90 seconds of duration - const plays = source.processRecentPlays([advancedState]) + const plays = await source.processRecentPlays([advancedState]) expect(plays.length).to.be.eq(1); expect(plays[0].data.duration).to.be.closeTo(90, 2); @@ -436,7 +436,7 @@ describe('Deezer Internal Source', function() { describe('When fuzzyDiscoveryIgnore is not defined or false', function () { - it('discovers fuzzy play', function() { + it('discovers fuzzy play', async function() { const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80}); const targetPlay = normalizedPlays[normalizedPlays.length - 2] const fuzzyPlay = clone(targetPlay); @@ -445,7 +445,7 @@ describe('Deezer Internal Source', function() { const source = generateDeezerSource(); source.discover([...normalizedPlays, interimPlay]); - const discovered = source.discover([fuzzyPlay]); + const discovered = await source.discover([fuzzyPlay]); expect(discovered.length).to.eq(1); }); @@ -453,41 +453,41 @@ describe('Deezer Internal Source', function() { describe('When fuzzyDiscoveryIgnore is true', function () { - it('does not discover fuzzy play with interim plays', function() { + it('does not discover fuzzy play with interim plays', async function() { const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80}); const targetPlay = normalizedPlays[normalizedPlays.length - 2] const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); const source = generateDeezerSource({fuzzyDiscoveryIgnore: true}); - source.discover([...normalizedPlays, interimPlay]); + await source.discover([...normalizedPlays, interimPlay]); - const discovered = source.discover([fuzzyPlay]); + const discovered = await source.discover([fuzzyPlay]); expect(discovered.length).to.eq(0); }); - it('discovers fuzzy play when it is the last play ', function() { + it('discovers fuzzy play when it is the last play ', async function() { const targetPlay = normalizedPlays[normalizedPlays.length - 1] const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); const source = generateDeezerSource({fuzzyDiscoveryIgnore: true}); - source.discover(normalizedPlays); + await source.discover(normalizedPlays); - const discovered = source.discover([fuzzyPlay]); + const discovered = await source.discover([fuzzyPlay]); expect(discovered.length).to.eq(1); }); - it('discovers fuzzy play when it is played consecutively', function() { + it('discovers fuzzy play when it is played consecutively', async function() { const targetPlay = normalizedPlays[normalizedPlays.length - 1] const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate}); const source = generateDeezerSource({fuzzyDiscoveryIgnore: true}); - const discovered = source.discover(morePlays); + const discovered = await source.discover(morePlays); expect(discovered.length).to.eq(morePlays.length); }); @@ -495,69 +495,69 @@ describe('Deezer Internal Source', function() { describe('When fuzzyDiscoveryIgnore is aggressive', function () { - it('does not discover fuzzy play with interim plays', function() { + it('does not discover fuzzy play with interim plays', async function() { const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80}); const targetPlay = normalizedPlays[normalizedPlays.length - 2] const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); - source.discover([...normalizedPlays, interimPlay]); + await source.discover([...normalizedPlays, interimPlay]); - const discovered = source.discover([fuzzyPlay]); + const discovered = await source.discover([fuzzyPlay]); expect(discovered.length).to.eq(0); }); - it('does not discover play found during duration of previous', function() { + it('does not discover play found during duration of previous', async function() { const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80}); const targetPlay = normalizedPlays[normalizedPlays.length - 2] const duringPlay = clone(targetPlay); duringPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration * 0.5, 's'); const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); - source.discover([...normalizedPlays, interimPlay]); + await source.discover([...normalizedPlays, interimPlay]); - const discovered = source.discover([duringPlay]); + const discovered = await source.discover([duringPlay]); expect(discovered.length).to.eq(0); }); - it('does not discover fuzzy play with delay of up to 40 seconds', function() { + it('does not discover fuzzy play with delay of up to 40 seconds', async function() { const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80}); const targetPlay = normalizedPlays[normalizedPlays.length - 2] const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration + 39, 's'); const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); - source.discover([...normalizedPlays, interimPlay]); + await source.discover([...normalizedPlays, interimPlay]); - const discovered = source.discover([fuzzyPlay]); + const discovered = await source.discover([fuzzyPlay]); expect(discovered.length).to.eq(0); }); - it('it does not discover fuzzy play when it is the last play ', function() { + it('it does not discover fuzzy play when it is the last play ', async function() { const targetPlay = normalizedPlays[normalizedPlays.length - 1] const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); - source.discover(normalizedPlays); + await source.discover(normalizedPlays); - const discovered = source.discover([fuzzyPlay]); + const discovered = await source.discover([fuzzyPlay]); expect(discovered.length).to.eq(0); }); - it('does not discover fuzzy play when it is played consecutively', function() { + it('does not discover fuzzy play when it is played consecutively', async function() { const targetPlay = normalizedPlays[normalizedPlays.length - 1] const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate}); const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); - const discovered = source.discover(morePlays); + const discovered = await source.discover(morePlays); expect(discovered.length).to.eq(morePlays.length - 1); }); diff --git a/src/backend/tests/utils/strings.test.ts b/src/backend/tests/utils/strings.test.ts index 09310d8c..73f5c358 100644 --- a/src/backend/tests/utils/strings.test.ts +++ b/src/backend/tests/utils/strings.test.ts @@ -1,26 +1,11 @@ import { assert, expect } from 'chai'; import { describe, it } from 'mocha'; -import { intersect } from "../../utils.js"; import { compareNormalizedStrings, normalizeStr, - parseTrackCredits, - uniqueNormalizedStrArr } from "../../utils/StringUtils.js"; -import { ExpectedResults } from "./interfaces.js"; -import testData from './playTestData.json' with { type: "json" }; import { splitByFirstFound } from '../../../core/StringUtils.js'; -interface PlayTestFixture { - caseHints: string[] - data: { - track: string - artists: string[] - album?: string - } - expected: ExpectedResults -} - describe('String Comparisons', function () { it('should ignore symbols', async function () { @@ -127,26 +112,6 @@ describe('String Comparisons', function () { }); }); -describe('Play Strings',function () { - - const testFixtures = testData as unknown as PlayTestFixture[]; - const joinerData = testFixtures.filter(x => intersect(['joiner','track'], x.caseHints).length === 2); - - it('should parse joiners from track title', function() { - for(const test of joinerData) { - const res = parseTrackCredits(test.data.track); - let artists: string[] = [...test.data.artists]; - if(res.secondary !== undefined) { - artists = uniqueNormalizedStrArr([...artists, ...res.secondary]); - } - assert.equal(res.primaryComposite, test.expected.track); - assert.sameDeepMembers(artists, test.expected.artists); - } - }); -}); - - - describe('String Splitting', function() { it('should not split string with no delimiter', function() { diff --git a/src/backend/tests/ytm/ytm.test.ts b/src/backend/tests/ytm/ytm.test.ts index 9830136d..79946207 100644 --- a/src/backend/tests/ytm/ytm.test.ts +++ b/src/backend/tests/ytm/ytm.test.ts @@ -174,7 +174,7 @@ describe('Handles interim tracks', function () { expect(source.parseRecentAgainstResponse(plays).plays).length(20); source.polling = true; - source.discover(plays); + await source.discover(plays); // first true poll emulating no new tracks played (should not add new tracks from base truth) expect(source.parseRecentAgainstResponse(plays).plays).length(0); @@ -201,7 +201,7 @@ describe('Handles interim tracks', function () { expect(source.parseRecentAgainstResponse(plays).plays).length(20); source.polling = true; - source.discover(plays); + await source.discover(plays); // first true poll emulating no new tracks played (should not add new tracks from base truth) expect(source.parseRecentAgainstResponse(plays).plays).length(0); diff --git a/src/backend/utils/AsyncUtils.ts b/src/backend/utils/AsyncUtils.ts new file mode 100644 index 00000000..af3cf930 --- /dev/null +++ b/src/backend/utils/AsyncUtils.ts @@ -0,0 +1,45 @@ +/** https://stackoverflow.com/a/63795192/1469797 */ +export async function findAsyncSequential( + array: T[], + predicate: (t: T) => Promise, +): Promise { + const i = await findIndexAsyncSequential(array, predicate); + if(i === undefined) { + return undefined; + } + return array[i]; +} + +export async function findIndexAsyncSequential( + array: T[], + predicate: (t: T) => Promise, +): Promise { + let index = 0; + for (const t of array) { + if (await predicate(t)) { + return index; + } + index++; + } + return undefined; +} + +/** https://stackoverflow.com/a/55601090/1469797 */ +export async function findAsync( + array: T[], + predicate: (t: T) => Promise): Promise { + const i = await findIndexAsync(array, predicate); + if(i === undefined) { + return undefined; + } + return array[i]; +} + +export async function findIndexAsync( + array: T[], + predicate: (t: T) => Promise): Promise { + const promises = array.map(predicate); + const results = await Promise.all(promises); + const index = results.findIndex(result => result); + return index; +} \ No newline at end of file diff --git a/src/backend/utils/PlayTransformUtils.ts b/src/backend/utils/PlayTransformUtils.ts index b813b1d5..62550a21 100644 --- a/src/backend/utils/PlayTransformUtils.ts +++ b/src/backend/utils/PlayTransformUtils.ts @@ -13,6 +13,7 @@ import { PlayTransformRules, PlayTransformStage, PlayTransformUserParts, PlayTransformUserStage, SearchAndReplaceTerm, STAGE_TYPES, StageType, + StageTypedConfig, WhenConditionsConfig, WhenParts } from "../common/infrastructure/Transform.js"; @@ -91,6 +92,13 @@ export const isExternalMetadataTerm = (val: unknown): val is ExternalMetadataTer throw new Error(`Value is type of ${tf} but must be one of: boolean, undefined, or object with 'when'`); } +export const isStageTyped = (val: unknown): val is StageTypedConfig => { + if(typeof val !== 'object' || val === null) { + return false; + } + return 'type' in val; +} + export const isPlayTransformStage = (val: object | Partial>): val is PlayTransformStage => { if (!('type' in val)) { throw new Error(`Stage is missing 'type'. Must be one of: ${STAGE_TYPES.join(', ')}`); @@ -125,71 +133,10 @@ export const isPlayTransformStage = (val: object | Partial(val: PlayTransformStage): val is PlayTransformUserStage => { +export const isUserStage = (val: StageTypedConfig): val is StageTypedConfig => { return val.type === 'user'; } -export const configPartsToStrongParts = (val: PlayTransformPartsConfig | undefined): PlayTransformPartsArray => { - if (val === undefined) { - return [] - } - const arr = Array.isArray(val) ? val : [val]; - - return arr.map((x) => { - const { - title: titleConfig, - artists: artistConfig, - album: albumConfig, - when: whenConfig, - type = 'user', - ...rest - } = x; - - let stage: PlayTransformStage; - try { - const candidateStage = {...x, type}; - if(isPlayTransformStage(candidateStage)) { - stage = candidateStage; - } - } catch (e) { - throw e; - } - - if (whenConfig !== undefined) { - if (!isWhenConditionConfig(whenConfig)) { - throw new Error(`'when' must be an array of artist/title/album objects and each object's property must be a string`); - } - } - - let title, - artists, - album, - when; - - if(isUserStage(stage)) { - title = stage.title?.map(configValToSearchReplace); - artists = stage.artists?.map(configValToSearchReplace); - album = stage.album?.map(configValToSearchReplace); - } else { - title = stage.title; - artists = stage.artists; - album = stage.album; - } - - when = whenConfig; - - return { - title, - artists, - album, - when, - type, - ...rest - } - }); - -} - export const testWhen = (parts: WhenParts, play: PlayObject, options?: SuppliedRegex): boolean => { const { testMaybeRegex = testMaybeRegexFunc, diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index f309c738..7080f333 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -389,4 +389,13 @@ export type FinalJoiners = '&'; export const JOINERS_FINAL: FinalJoiners[] = ['&']; export type Feat = 'ft' | 'feat' | 'vs' | 'ft.' | 'feat.' | 'vs.' | 'featuring' -export const FEAT: Feat[] = ['ft','feat','vs','ft.','feat.','vs.','featuring']; \ No newline at end of file +export const FEAT: Feat[] = ['ft','feat','vs','ft.','feat.','vs.','featuring']; +export interface TransformerCommonConfig { + data?: Record; + type: string; + name?: string; + options?: { + failOnFetch?: boolean; + throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album')[]; + }; +} -- 2.51.2 From b6417ed780ed686f4061cf3171872ede0e09341c Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 26 Nov 2025 02:17:26 +0000 Subject: [PATCH 05/12] Implement initial native transformer #364 --- .../common/infrastructure/Transform.ts | 8 +- .../common/transforms/NativeTransformer.ts | 97 +++++++++++++++++++ .../common/transforms/TransformerManager.ts | 4 + src/backend/ioc.ts | 1 + src/backend/tests/component/component.test.ts | 47 ++++----- 5 files changed, 133 insertions(+), 24 deletions(-) create mode 100644 src/backend/common/transforms/NativeTransformer.ts diff --git a/src/backend/common/infrastructure/Transform.ts b/src/backend/common/infrastructure/Transform.ts index 497c3a8d..a353cb73 100644 --- a/src/backend/common/infrastructure/Transform.ts +++ b/src/backend/common/infrastructure/Transform.ts @@ -6,7 +6,7 @@ export interface ConditionalSearchAndReplaceRegExp extends SearchAndReplaceRegEx export type ConditionalSearchAndReplaceTerm = Omit export type SearchAndReplaceTerm = string | ConditionalSearchAndReplaceTerm; -export type ExternalMetadataTerm = boolean | undefined | { when: WhenConditionsConfig }; +export type ExternalMetadataTerm = boolean | undefined | Whennable; export type PlayTransformParts = Extract, Y> & { when?: WhenConditionsConfig }; export type PlayTransformUserParts = PlayTransformUserStage & { when?: WhenConditionsConfig }; @@ -64,6 +64,12 @@ export interface PlayTransformMetadataStage extends Pl export interface PlayTransformUserStage extends StageTypedConfig, PlayTransformPartsAtomic { type: StageTypeUser } + +export interface PlayTransformNativeStage extends StageTypedConfig, PlayTransformPartsAtomic { + type: 'native' +} + + export type UntypedPlayTransformUserStage = Omit, 'type'> & {type?: never}; export type PlayTransformStage = PlayTransformMetadataStage | PlayTransformUserStage | UntypedPlayTransformUserStage; diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts new file mode 100644 index 00000000..b98e0267 --- /dev/null +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -0,0 +1,97 @@ +import { PlayObject } from "../../../core/Atomic.js"; +import { isUserStage, isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js"; +import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; +import { ExternalMetadataTerm, PlayTransformNativeStage, StageConfig } from "../infrastructure/Transform.js"; +import AtomicPartsTransformer from "./AtomicPartsTransformer.js"; +import { parseArtistCredits, parseTrackCredits, uniqueNormalizedStrArr } from "../../utils/StringUtils.js"; + +export default class NativeTransformer extends AtomicPartsTransformer { + + protected doParseConfig(data: StageConfig) { + if (data.type !== 'native') { + throw new Error(`NativeTransformer is only usable with 'native' type stages`); + } + + const stage: PlayTransformNativeStage = { + ...data, + type: 'native' + } + + for (const k of ['artists', 'title', 'album']) { + if (!(k in data)) { + continue; + } + if (Array.isArray(data[k])) { + throw new Error(`${k} must be a boolean or when object`); + } + if (typeof data[k] === 'boolean') { + continue; + } + if (typeof data[k] === 'object' && !isWhenCondition(data[k])) { + throw new Error(`${k} is not a valid when object`); + } + } + return stage; + } + + public async getTransformerData(play: PlayObject): Promise { + let artists = []; + if (play.data.artists.length === 1) { + const artistCredits = parseArtistCredits(play.data.artists[0]); + if (artistCredits !== undefined) { + if (artistCredits.primary !== undefined) { + artists.push(artistCredits.primary); + } + if (artistCredits.secondary !== undefined) { + artists = artists.concat(artistCredits.secondary); + } + } + } + const trackArtists = parseTrackCredits(play.data.track); + if (trackArtists !== undefined && trackArtists.secondary !== undefined) { + artists = artists.concat(trackArtists.secondary); + } + artists = uniqueNormalizedStrArr([...artists]); + + return { + ...play, + data: { + ...play.data, + artists + } + }; + } + + protected async handleTitle(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise { + return play.data.track; + } + protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { + if (parts === false || parts === undefined) { + return play.data.artists; + } + if (typeof parts === 'object') { + if (parts.when !== undefined) { + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex })) { + this.logger.debug('When condition for artists not met, returning original artists'); + return play.data.artists; + } + } + } + + return transformData.data.artists; + } + protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise { + return play.data.albumArtists; + } + protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise { + return play.data.album; + } + + public notify(payload: WebhookPayload): Promise { + throw new Error("Method not implemented."); + } + protected getIdentifier(): string { + return 'Native Transformer'; + } + +} \ No newline at end of file diff --git a/src/backend/common/transforms/TransformerManager.ts b/src/backend/common/transforms/TransformerManager.ts index 2a8ba543..77bbae47 100644 --- a/src/backend/common/transforms/TransformerManager.ts +++ b/src/backend/common/transforms/TransformerManager.ts @@ -6,6 +6,7 @@ import { StageConfig } from "../infrastructure/Transform.js"; import { PlayObject } from "../../../core/Atomic.js"; import { isStageTyped } from "../../utils/PlayTransformUtils.js"; import { MSCache } from "../Cache.js"; +import NativeTransformer from "./NativeTransformer.js"; export default class TransformerManager { @@ -38,6 +39,9 @@ export default class TransformerManager { case 'user': t = new UserTransformer({ name: tName, logger: this.parentLogger, regexCache: this.cache.regexCache, ...config }); break; + case 'native': + t = new NativeTransformer({ name: tName, logger: this.parentLogger, regexCache: this.cache.regexCache, ...config }); + break; default: throw new Error(`No transformer of type '${config.type}' exists.`); } diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index 8dea51d4..d9230ab1 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -71,6 +71,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { const transformerManager = new TransformerManager(logger, maybeSingletonCache !== undefined ? maybeSingletonCache : cacheFunc()); transformerManager.register({type: 'user'}); + transformerManager.register({type: 'native'}); for(const c of transformerConfigs) { try { transformerManager.register(c); diff --git a/src/backend/tests/component/component.test.ts b/src/backend/tests/component/component.test.ts index 488d4ad1..573ca7fe 100644 --- a/src/backend/tests/component/component.test.ts +++ b/src/backend/tests/component/component.test.ts @@ -220,36 +220,37 @@ describe('Play Transforms', function () { }); }); - // describe('Non-User Stage Parsing', function () { + describe('Non-User Stage Parsing', function () { - // describe('Non-User Stage Types', function () { + describe('Non-User Stage Types', function () { - // for(const t of STAGE_TYPES_USER) { + for(const t of ['native']) { - // it(`Allows non-user Stage Type ${t}`, function () { - // component.config = { - // options: { - // playTransform: { - // preCompare: { - // type: t, - // title: true - // } - // } - // } - // } + it(`Allows non-user Stage Type ${t}`, function () { + component.config = { + options: { + playTransform: { + preCompare: { + // @ts-expect-error + type: t, + title: true + } + } + } + } - // expect(() => component.buildTransformRules()).to.not.throw(); - // expect(component.transformRules.preCompare).to.be.an('array'); - // expect(component.transformRules.preCompare).to.be.length(1); - // expect(component.transformRules.preCompare).to.have.nested.property('0.type'); - // expect(component.transformRules.preCompare[0].type).eq(t); - // }); + expect(() => component.buildTransformRules()).to.not.throw(); + expect(component.transformRules.preCompare).to.be.an('array'); + expect(component.transformRules.preCompare).to.be.length(1); + expect(component.transformRules.preCompare).to.have.nested.property('0.type'); + expect(component.transformRules.preCompare[0].type).eq(t); + }); - // } + } - // }); + }); - // }); + }); describe('Play Transforming', function () { -- 2.51.2 From 883050e0b4cf383dfb5f38f4da2c199cffa57a29 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 26 Nov 2025 16:11:51 +0000 Subject: [PATCH 06/12] feat: Add native transformer options for delimiters and artist exceptions #364 --- src/backend/common/infrastructure/Atomic.ts | 1 + .../common/transforms/AbstractTransformer.ts | 15 ++-- .../transforms/AtomicPartsTransformer.ts | 6 +- .../common/transforms/NativeTransformer.ts | 86 ++++++++++++++++--- .../common/transforms/UserTransformer.ts | 5 +- src/backend/utils/DataUtils.ts | 6 ++ src/backend/utils/StringUtils.ts | 4 +- src/core/Atomic.ts | 8 +- 8 files changed, 101 insertions(+), 30 deletions(-) create mode 100644 src/backend/utils/DataUtils.ts diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index d43700a0..5fccaa28 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -267,6 +267,7 @@ export interface numberFormatOptions { } export const DELIMITERS = [',','&','/','\\']; +export const DELIMITERS_NO_AMP = [',','/','\\']; export const ARTIST_WEIGHT = 0.3; export const TITLE_WEIGHT = 0.4; diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index bba5c576..61991d7d 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -1,15 +1,14 @@ import { childLogger, Logger } from "@foxxmd/logging"; -import { PlayObject, TransformerCommonConfig } from "../../../core/Atomic.js"; +import { PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.js"; import { getRoot } from "../../ioc.js"; import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import AbstractInitializable from "../AbstractInitializable.js"; import { StageConfig } from "../infrastructure/Transform.js"; import { cacheFunctions } from "@foxxmd/regex-buddy-core"; -export interface TransformerCommon extends TransformerCommonConfig { - regexCache: ReturnType - name: string - logger: Logger +export interface TransformerOptions { + logger: Logger + regexCache: ReturnType } export default abstract class AbstractTransformer extends AbstractInitializable { @@ -20,11 +19,11 @@ export default abstract class AbstractTransformer extends AbstractIniti regexCache: ReturnType - public constructor(config: TransformerCommon) { + public constructor(config: TransformerCommon, options: TransformerOptions) { super(config); - this.logger = childLogger(config.logger, ['Transformer', this.config.type, this.config.name]); + this.logger = childLogger(options.logger, ['Transformer', this.config.type, this.config.name]); this.transformType = config.type; - this.regexCache = config.regexCache; + this.regexCache = options.regexCache; } public parseConfig(data: any) { diff --git a/src/backend/common/transforms/AtomicPartsTransformer.ts b/src/backend/common/transforms/AtomicPartsTransformer.ts index 3786f796..46a749c9 100644 --- a/src/backend/common/transforms/AtomicPartsTransformer.ts +++ b/src/backend/common/transforms/AtomicPartsTransformer.ts @@ -1,9 +1,5 @@ import { ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js"; -import { getRoot } from "../../ioc.js"; -import { testWhenConditions } from "../../utils/PlayTransformUtils.js"; -import AbstractInitializable from "../AbstractInitializable.js"; -import { AtomicStageConfig, ConditionalSearchAndReplaceRegExp, PlayTransformMetaParts, PlayTransformParts, PlayTransformUserParts } from "../infrastructure/Transform.js"; -import { cacheFunctions } from "@foxxmd/regex-buddy-core"; +import { AtomicStageConfig } from "../infrastructure/Transform.js"; import AbstractTransformer from "./AbstractTransformer.js"; export default abstract class AtomicPartsTransformer extends AbstractTransformer { diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts index b98e0267..c4d63b4b 100644 --- a/src/backend/common/transforms/NativeTransformer.ts +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -1,12 +1,69 @@ -import { PlayObject } from "../../../core/Atomic.js"; -import { isUserStage, isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js"; +import { PlayObject, TransformerCommon } from "../../../core/Atomic.js"; +import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; import { ExternalMetadataTerm, PlayTransformNativeStage, StageConfig } from "../infrastructure/Transform.js"; import AtomicPartsTransformer from "./AtomicPartsTransformer.js"; import { parseArtistCredits, parseTrackCredits, uniqueNormalizedStrArr } from "../../utils/StringUtils.js"; +import { parseRegexSingle, parseToRegexOrLiteralSearch } from "@foxxmd/regex-buddy-core"; +import { TransformerOptions } from "./AbstractTransformer.js"; +import { DELIMITERS_NO_AMP } from "../infrastructure/Atomic.js"; +import { asArray } from "../../utils/DataUtils.js"; + +export interface NativeTransformerData { + delimiters?: string[] + extraDelimiters?: string[] + ignoreArtists?: string[] +} + +export type NativeTransformerConfig = TransformerCommon; export default class NativeTransformer extends AtomicPartsTransformer { + declare config: NativeTransformerConfig; + + ignoreArtistsRegex: RegExp[] = []; + delimiters?: string[] + + // public constructor(config: NativeTransformerConfig, options: TransformerOptions) { + // super(config, options); + // } + + protected async doBuildInitData(): Promise { + if(this.config.data === undefined) { + return true; + } + if(this.config.data === null || typeof this.config.data !== 'object') { + throw new Error('Native Transformer data should be an object or not defined.'); + } + if(this.config.data.ignoreArtists !== undefined) { + this.config.data.ignoreArtists = asArray(this.config.data.ignoreArtists); + const nonStr = this.config.data.ignoreArtists.filter(x => typeof x !== 'string'); + if(nonStr.length > 0) { + throw new Error(`ignoreArtists must be an array of strings but non-strings found: ${nonStr.map(x => (x as unknown).toString()).join(' | ')}`) + } + for(const i of this.config.data.ignoreArtists) { + try { + this.ignoreArtistsRegex.push(parseToRegexOrLiteralSearch(i)); + } catch (e) { + throw new Error(`Could not convert ignoreArtist string to regex (or literal): ${i}`); + } + } + this.logger.debug(`Ignoring artists using ${this.config.data.ignoreArtists.length} rules`); + } + + if(this.config.data.delimiters !== undefined) { + this.delimiters = asArray(this.config.data.delimiters); + this.logger.debug(`Using user-defined delimiters '${this.config.data.extraDelimiters.join(' ')}' instead of built-ins`); + } else if(this.config.data.extraDelimiters !== undefined) { + this.delimiters = [...DELIMITERS_NO_AMP, ...(asArray(this.config.data.extraDelimiters))]; + this.logger.debug(`Using extra delimiters '${this.config.data.extraDelimiters.join(' ')}' with built-in delimiters '${DELIMITERS_NO_AMP.join(' ')}'`); + } + + if(this.delimiters !== undefined) { + this.delimiters.map(x => x.trim()); + } + } + protected doParseConfig(data: StageConfig) { if (data.type !== 'native') { throw new Error(`NativeTransformer is only usable with 'native' type stages`); @@ -37,17 +94,26 @@ export default class NativeTransformer extends AtomicPartsTransformer { let artists = []; if (play.data.artists.length === 1) { - const artistCredits = parseArtistCredits(play.data.artists[0]); - if (artistCredits !== undefined) { - if (artistCredits.primary !== undefined) { - artists.push(artistCredits.primary); - } - if (artistCredits.secondary !== undefined) { - artists = artists.concat(artistCredits.secondary); + const matchedIgnoreArtists = this.ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, play.data.artists[0])})).filter(x => x !== undefined); + if(matchedIgnoreArtists.length > 0) { + this.logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`) + } else { + const artistCredits = parseArtistCredits(play.data.artists[0], this.delimiters); + if (artistCredits !== undefined) { + if (artistCredits.primary !== undefined) { + artists.push(artistCredits.primary); + } + if (artistCredits.secondary !== undefined) { + artists = artists.concat(artistCredits.secondary); + } } } + if(artists.length === 0) { + artists.push(play.data.artists[0]); + } } - const trackArtists = parseTrackCredits(play.data.track); + + const trackArtists = parseTrackCredits(play.data.track, this.delimiters); if (trackArtists !== undefined && trackArtists.secondary !== undefined) { artists = artists.concat(trackArtists.secondary); } diff --git a/src/backend/common/transforms/UserTransformer.ts b/src/backend/common/transforms/UserTransformer.ts index 77713a76..e0a5ea8b 100644 --- a/src/backend/common/transforms/UserTransformer.ts +++ b/src/backend/common/transforms/UserTransformer.ts @@ -1,9 +1,8 @@ import { searchAndReplace } from "@foxxmd/regex-buddy-core"; import { PlayObject } from "../../../core/Atomic.js"; -import { configValToSearchReplace, isSearchAndReplaceTerm, isStageTyped, isUserStage, testWhenConditions } from "../../utils/PlayTransformUtils.js"; +import { configValToSearchReplace, isSearchAndReplaceTerm, isUserStage, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; -import { ConditionalSearchAndReplaceRegExp, PlayTransformUserParts, PlayTransformUserStage, StageConfig } from "../infrastructure/Transform.js"; -import AbstractTransformer, { TransformerCommon } from "./AbstractTransformer.js" +import { ConditionalSearchAndReplaceRegExp, PlayTransformUserStage, StageConfig } from "../infrastructure/Transform.js"; import AtomicPartsTransformer from "./AtomicPartsTransformer.js"; export default class UserTransformer extends AtomicPartsTransformer { diff --git a/src/backend/utils/DataUtils.ts b/src/backend/utils/DataUtils.ts new file mode 100644 index 00000000..7722466c --- /dev/null +++ b/src/backend/utils/DataUtils.ts @@ -0,0 +1,6 @@ +export const asArray = (data: T | T[]): T[] => { + if(Array.isArray(data)) { + return data; + } + return [data]; +} \ No newline at end of file diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts index 95002ab6..92c7f7e8 100644 --- a/src/backend/utils/StringUtils.ts +++ b/src/backend/utils/StringUtils.ts @@ -1,7 +1,7 @@ import { strategies, stringSameness, StringSamenessResult } from "@foxxmd/string-sameness"; import { hasher } from 'node-object-hash'; import { PlayObject } from "../../core/Atomic.js"; -import { asPlayerStateData, DELIMITERS, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; +import { asPlayerStateData, DELIMITERS, DELIMITERS_NO_AMP, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; import { genGroupIdStr, getPlatformIdFromData, intersect, parseRegexSingleOrFail } from "../utils.js"; import { buildTrackString } from "../../core/StringUtils.js"; @@ -183,7 +183,7 @@ export const parseStringList = (str: string, delimiters: string[] = DELIMITERS): return explodedStrings.flat(1); }, [str]).map(x => x.trim()); } -export const parseContextAwareStringList = (str: string, delimiters: string[] = [',', '/', '\\'], opts: {ignoreGlobalAmpersand?: boolean} = {}): string[] => { +export const parseContextAwareStringList = (str: string, delimiters: string[] = DELIMITERS_NO_AMP, opts: {ignoreGlobalAmpersand?: boolean} = {}): string[] => { if (delimiters.length === 0) { return [str]; } diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 7080f333..987a1431 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -390,8 +390,8 @@ export const JOINERS_FINAL: FinalJoiners[] = ['&']; export type Feat = 'ft' | 'feat' | 'vs' | 'ft.' | 'feat.' | 'vs.' | 'featuring' export const FEAT: Feat[] = ['ft','feat','vs','ft.','feat.','vs.','featuring']; -export interface TransformerCommonConfig { - data?: Record; +export interface TransformerCommonConfig> { + data?: T; type: string; name?: string; options?: { @@ -399,3 +399,7 @@ export interface TransformerCommonConfig { throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album')[]; }; } + +export interface TransformerCommon> extends TransformerCommonConfig { + name: string +} \ No newline at end of file -- 2.51.2 From a7df5a8d30eedd68a70de2f00de6d391aebfef32 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 26 Nov 2025 16:59:35 +0000 Subject: [PATCH 07/12] feat: Add flow control to transforms and fix bugs --- src/backend/common/AbstractComponent.ts | 61 +++++- src/backend/common/Cache.ts | 16 +- .../common/infrastructure/Transform.ts | 30 ++- .../common/transforms/AbstractTransformer.ts | 47 +++-- .../common/transforms/NativeTransformer.ts | 21 ++- .../common/transforms/TransformerManager.ts | 26 ++- .../common/transforms/UserTransformer.ts | 2 +- src/backend/ioc.ts | 17 +- ...component.test.ts => transformers.test.ts} | 93 ++++++++- src/backend/utils/PlayComparisonUtils.ts | 17 ++ src/backend/utils/PlayTransformUtils.ts | 176 +----------------- 11 files changed, 271 insertions(+), 235 deletions(-) rename src/backend/tests/component/{component.test.ts => transformers.test.ts} (85%) diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index a88ce616..f950b841 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -43,6 +43,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { } public buildTransformRules() { + this.logger.debug('Building transformer rules...'); try { this.doBuildTransformRules(); } catch (e) { @@ -65,6 +66,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { } = this.config; if (playTransform === undefined) { + this.logger.debug(`No rules found under property 'playTransform'`); return; } @@ -82,30 +84,54 @@ export default abstract class AbstractComponent extends AbstractInitializable { existing, postCompare; + const builtHooks: string[] = []; + const emptyHooks: string[] = []; try { preCompare = this.transformPartToStrong(preConfig); + if(preCompare === undefined) { + emptyHooks.push('preCompare') + } else { + builtHooks.push('preCompare'); + } } catch (e) { throw new Error('preCompare was not valid', {cause: e}); } try { candidate = this.transformPartToStrong(candidateConfig); + if(candidate === undefined) { + emptyHooks.push('candidate') + } else { + builtHooks.push('candidate'); + } } catch (e) { throw new Error('candidate was not valid', {cause: e}); } try { existing = this.transformPartToStrong(existingConfig); + if(existing === undefined) { + emptyHooks.push('existing') + } else { + builtHooks.push('existing'); + } } catch (e) { throw new Error('existing was not valid', {cause: e}); } try { postCompare = this.transformPartToStrong(postConfig); + if(postCompare === undefined) { + emptyHooks.push('postCompare') + } else { + builtHooks.push('postCompare'); + } } catch (e) { throw new Error('postCompare was not valid', {cause: e}); } + this.logger.debug(`Hooks built. Configured: ${builtHooks.join(', ')} | Empty: ${emptyHooks.join(', ')}`); + this.transformRules = { preCompare, compare: { @@ -155,15 +181,46 @@ export default abstract class AbstractComponent extends AbstractInitializable { } let transformedPlay: PlayObject = play; - const transformDetails: string[] = []; + let transformDetails: string[] = []; for(const hookItem of hook) { - const newTransformedPlay = await this.transformManager.handleStage(hookItem, transformedPlay); + const { + onSuccess = 'continue', + onFailure = 'stop', + failureReturnPartial = false + } = hookItem; + + let newTransformedPlay: PlayObject; + let err: Error; + try { + newTransformedPlay = await this.transformManager.handleStage(hookItem, transformedPlay); + } catch (e) { + err = e; + } + + if(err !== undefined) { + if(onFailure === 'continue') { + this.logger.warn(new Error('A transform encountered an error but continuing due to onFailure: continue', {cause: err})); + } else { + this.logger.error(new Error('Transform encountered an error', {cause: err})); + if(!failureReturnPartial) { + // rewind to original play so we don't return partial transform + transformedPlay = play; + transformDetails = []; + } + break; + } + } if(!deepEqual(newTransformedPlay, transformedPlay)) { transformDetails.push(`${hookItem.type} - ${buildTrackString(transformedPlay, {include: ['artist', 'track', 'album']})}`); } transformedPlay = newTransformedPlay; + + if(err === undefined && onSuccess === 'stop') { + this.logger.debug('Stopping transform due to onSuccess: stop'); + break; + } } if(transformDetails.length > 0) { diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index 04210193..b8713a81 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -49,6 +49,7 @@ export class MSCache { cacheScrobble: Cacheable; cacheAuth: Cacheable; regexCache: ReturnType; + cacheTransform: Cacheable; logger: Logger; @@ -94,6 +95,7 @@ export class MSCache { }; this.regexCache = cacheFunctions(this.config.regex); + this.cacheTransform = new Cacheable({primary: initMemoryCache({lruSize: 500})}); } init = async () => { @@ -101,6 +103,7 @@ export class MSCache { //await this.initMetadataCache(); await this.initScrobbleCache(); await this.initAuthCache(); + //this.cacheTransform = await this.initCacheable({provider: false, memory: {lruSize: 500}}, 'transform'); } protected initCacheable = async (config: CacheConfig, cacheFor: string) => { @@ -115,7 +118,7 @@ export class MSCache { const ns = `ms-${cacheFor.toLocaleLowerCase()}`; const cacheOpts: CacheableOptions = { - primary: initMemoryCache({ namespace: ns }) + primary: initMemoryCache({ namespace: ns, lruSize: config.memory?.lruSize, ttl: config.memory?.ttl }) } let secondaryCache: Keyv | KeyvStoreAdapter | undefined; @@ -180,10 +183,15 @@ export class MSCache { export const initMemoryCache = (opts: Parameters[0] = {}): Keyv | KeyvStoreAdapter => { + const { + ttl = '1h', + lruSize = 200, + ...restOpts + } = opts; const memory = createKeyv({ - ttl: '1h', - lruSize: 200, - ...opts, + ttl, + lruSize, + ...restOpts, useClone: false, }); // structuredClone does not work well with dayjs https://github.com/iamkun/dayjs/issues/2236 diff --git a/src/backend/common/infrastructure/Transform.ts b/src/backend/common/infrastructure/Transform.ts index a353cb73..824da292 100644 --- a/src/backend/common/infrastructure/Transform.ts +++ b/src/backend/common/infrastructure/Transform.ts @@ -1,16 +1,15 @@ import { SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core"; -export interface ConditionalSearchAndReplaceRegExp extends SearchAndReplaceRegExp { - when?: WhenConditionsConfig +export interface ConditionalSearchAndReplaceRegExp extends SearchAndReplaceRegExp, Whennable { } export type ConditionalSearchAndReplaceTerm = Omit export type SearchAndReplaceTerm = string | ConditionalSearchAndReplaceTerm; export type ExternalMetadataTerm = boolean | undefined | Whennable; -export type PlayTransformParts = Extract, Y> & { when?: WhenConditionsConfig }; -export type PlayTransformUserParts = PlayTransformUserStage & { when?: WhenConditionsConfig }; -export type PlayTransformMetaParts = PlayTransformMetadataStage & { when?: WhenConditionsConfig }; +export type PlayTransformParts = Extract, Y> & Whennable; +//export type PlayTransformUserParts = PlayTransformUserStage & { when?: WhenConditionsConfig }; +//export type PlayTransformMetaParts = PlayTransformMetadataStage & { when?: WhenConditionsConfig }; export type PlayTransformPartsArray = PlayTransformParts[]; /** Represents the weakly-defined user config. May be an array of parts or one parts object */ @@ -47,7 +46,15 @@ export interface Whennable { when?: WhenConditionsConfig } -export interface StageConfig extends StageTypedConfig, Whennable {} +export type FlowControlTerm = 'continue' | 'stop' + +export interface FlowControl { + onSuccess: FlowControlTerm + onFailure: FlowControlTerm + failureReturnPartial: boolean +} + +export interface StageConfig extends StageTypedConfig, Whennable, Partial {} export interface AtomicStageConfig extends StageConfig, PlayTransformPartsAtomic {} @@ -55,24 +62,27 @@ export interface PlayTransformStageTyped extends PlayTransformPartsAtomic type: StageType } -export interface PlayTransformMetadataStage extends PlayTransformStageTyped { +export interface PlayTransformMetadataStage extends StageConfig, PlayTransformPartsAtomic { score?: number // all?: ExternalMetadataTerm type: StageTypeMetadata } -export interface PlayTransformUserStage extends StageTypedConfig, PlayTransformPartsAtomic { +export interface PlayTransformUserStage extends StageConfig, PlayTransformPartsAtomic { type: StageTypeUser } -export interface PlayTransformNativeStage extends StageTypedConfig, PlayTransformPartsAtomic { +export interface PlayTransformNativeStage extends StageConfig, PlayTransformPartsAtomic { type: 'native' } +export interface PlayTransformGenericStage extends StageConfig, PlayTransformPartsAtomic { + type: string +} export type UntypedPlayTransformUserStage = Omit, 'type'> & {type?: never}; -export type PlayTransformStage = PlayTransformMetadataStage | PlayTransformUserStage | UntypedPlayTransformUserStage; +export type PlayTransformStage = PlayTransformMetadataStage | PlayTransformUserStage | PlayTransformNativeStage | UntypedPlayTransformUserStage | PlayTransformGenericStage; /** Represents the plain json user-configured structure (input) */ export interface PlayTransformHooksConfig { diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index 61991d7d..9f2f953b 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -4,26 +4,40 @@ import { getRoot } from "../../ioc.js"; import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import AbstractInitializable from "../AbstractInitializable.js"; import { StageConfig } from "../infrastructure/Transform.js"; -import { cacheFunctions } from "@foxxmd/regex-buddy-core"; +import { cacheFunctions, parseToRegexOrLiteralSearch, testMaybeRegex, searchAndReplace} from "@foxxmd/regex-buddy-core"; +import { Cacheable } from "cacheable"; +import { hashObject } from "../../utils/StringUtils.js"; +import { playContentInvariantTransform } from "../../utils/PlayComparisonUtils.js"; export interface TransformerOptions { logger: Logger - regexCache: ReturnType + regexCache?: ReturnType + cache: Cacheable +} + +export interface RegexObject { + parseToRegexOrLiteralSearch: typeof parseToRegexOrLiteralSearch + testMaybeRegex: typeof testMaybeRegex, + searchAndReplace: typeof searchAndReplace } export default abstract class AbstractTransformer extends AbstractInitializable { declare config: TransformerCommonConfig; + configHash: string; transformType: string - regexCache: ReturnType + regex: RegexObject + cache: Cacheable; public constructor(config: TransformerCommon, options: TransformerOptions) { super(config); this.logger = childLogger(options.logger, ['Transformer', this.config.type, this.config.name]); this.transformType = config.type; - this.regexCache = options.regexCache; + this.regex = options.regexCache ?? { searchAndReplace, testMaybeRegex, parseToRegexOrLiteralSearch }; + this.cache = options.cache; + this.configHash = hashObject(this.config); } public parseConfig(data: any) { @@ -37,27 +51,26 @@ export default abstract class AbstractTransformer extends AbstractIniti public async handle(data: StageConfig, play: PlayObject): Promise { + const cacheKey = `${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}` + const cachedTransform = await this.cache.get(cacheKey); + if(cachedTransform !== undefined) { + this.logger.debug('Cache hit'); + return cachedTransform; + } + if (data.when !== undefined) { - if (!testWhenConditions(data.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex })) { + if (!testWhenConditions(data.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { this.logger.debug('When condition not met, returning original Play'); + await this.cache.set(cacheKey, play, '15s'); return play; } } - const { - failOnFetch = false, - throwOnFailure = false, - } = this.config.options || {}; - let transformData: T; try { transformData = await this.getTransformerData(play); } catch (e) { - if (failOnFetch) { - throw new Error(`Could not fetch transformer data`, { cause: e }); - } - this.logger.warn(new Error(`Could not fetch transformer data, returning original Play`, { cause: e })); - return play; + throw new Error(`Could not fetch transformer data`, { cause: e }); } try { @@ -67,7 +80,9 @@ export default abstract class AbstractTransformer extends AbstractIniti return play; } - return await this.doHandle(data, play, transformData); + const transformed = await this.doHandle(data, play, transformData); + await this.cache.set(cacheKey, transformed, '15s'); + return transformed; } protected abstract doHandle(data: StageConfig, play: PlayObject, transformData: T): Promise; diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts index c4d63b4b..135066c5 100644 --- a/src/backend/common/transforms/NativeTransformer.ts +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -24,9 +24,9 @@ export default class NativeTransformer extends AtomicPartsTransformer { if(this.config.data === undefined) { @@ -69,22 +69,23 @@ export default class NativeTransformer extends AtomicPartsTransformer = { + const stage: PlayTransformNativeStage = { ...data, type: 'native' } for (const k of ['artists', 'title', 'album']) { - if (!(k in data)) { + if (!(k in stage)) { + stage[k] = true; continue; } - if (Array.isArray(data[k])) { + if (Array.isArray(stage[k])) { throw new Error(`${k} must be a boolean or when object`); } - if (typeof data[k] === 'boolean') { + if (typeof stage[k] === 'boolean') { continue; } - if (typeof data[k] === 'object' && !isWhenCondition(data[k])) { + if (typeof stage[k] === 'object' && !isWhenCondition(stage[k])) { throw new Error(`${k} is not a valid when object`); } } @@ -132,12 +133,12 @@ export default class NativeTransformer extends AtomicPartsTransformer { - if (parts === false || parts === undefined) { + if (parts === false) { return play.data.artists; } if (typeof parts === 'object') { if (parts.when !== undefined) { - if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex })) { + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { this.logger.debug('When condition for artists not met, returning original artists'); return play.data.artists; } diff --git a/src/backend/common/transforms/TransformerManager.ts b/src/backend/common/transforms/TransformerManager.ts index 77bbae47..4623d602 100644 --- a/src/backend/common/transforms/TransformerManager.ts +++ b/src/backend/common/transforms/TransformerManager.ts @@ -10,10 +10,10 @@ import NativeTransformer from "./NativeTransformer.js"; export default class TransformerManager { - logger: Logger; - parentLogger: Logger; - transformers: Map = new Map(); - cache: MSCache; + protected logger: Logger; + protected parentLogger: Logger; + protected transformers: Map = new Map(); + protected cache: MSCache; public constructor(logger: Logger, cache: MSCache) { this.logger = childLogger(logger, 'Transformer Manager'); @@ -30,26 +30,29 @@ export default class TransformerManager { } if (config.name !== undefined && transformers.some(x => x.config.name === config.name)) { - throw new Error(`Cannot register ${config.type} with name '${config.name}' because an existing transformer already has that name`); + throw new Error(`Cannot register ${config.type} transformer with name '${config.name}' because an existing transformer already has that name`); } const tName = config.name ?? `unnamed-${transformers.length + 1}`; + this.logger.verbose(`Registering ${config.type} transformer with name '${tName}'`); + let t: AbstractTransformer; switch (config.type) { case 'user': - t = new UserTransformer({ name: tName, logger: this.parentLogger, regexCache: this.cache.regexCache, ...config }); + t = new UserTransformer({ name: tName, ...config }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); break; case 'native': - t = new NativeTransformer({ name: tName, logger: this.parentLogger, regexCache: this.cache.regexCache, ...config }); + t = new NativeTransformer({ name: tName, ...config }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); break; default: throw new Error(`No transformer of type '${config.type}' exists.`); } - this.transformers.set(config.type, [...transformers, t]); + this.logger.verbose(`${config.type} transformer with name '${tName}' registered`); } public async initTransformers() { + this.logger.verbose('Initializing transformers...'); for (const list of this.transformers.values()) { for (const transformer of list) { if (!transformer.isReady()) { @@ -64,6 +67,11 @@ export default class TransformerManager { } } } + this.logger.verbose('Done initializing transformers'); + } + + public hasTransformerType(type: string): boolean { + return this.transformers.has(type); } protected getTransformerByStage(data: StageConfig): AbstractTransformer { @@ -72,7 +80,7 @@ export default class TransformerManager { throw new Error(`No transformer of type '${data.type}' is registered.`); } - if (list.length > 0 && (data as any).name === undefined) { + if (list.length > 1 && (data as any).name === undefined) { this.logger.warn(`More than one '${data.type}' transformer but name was not specified, using first registered`); return list[0]; } else { diff --git a/src/backend/common/transforms/UserTransformer.ts b/src/backend/common/transforms/UserTransformer.ts index e0a5ea8b..9b1565a8 100644 --- a/src/backend/common/transforms/UserTransformer.ts +++ b/src/backend/common/transforms/UserTransformer.ts @@ -39,7 +39,7 @@ export default class UserTransformer extends AtomicPartsTransformer ({ ...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex }) : undefined) }); + return (x: ConditionalSearchAndReplaceRegExp): ConditionalSearchAndReplaceRegExp => ({ ...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, { testMaybeRegex: this.regex.testMaybeRegex }) : undefined) }); } protected async handleTitle(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise { diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index d9230ab1..dc24d04a 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -29,7 +29,7 @@ export interface RootOptions { loggerStream?: PassThrough loggingConfig?: LogOptions cache?: CacheConfigOptions | MSCache | (() => MSCache) - transformerConfigs?: TransformerCommonConfig[] + transformers?: TransformerCommonConfig[] } const createRoot = (options: RootOptions = {logger: loggerDebug}) => { @@ -41,7 +41,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { loggingConfig, logger, cache, - transformerConfigs = [], + transformers = [], } = options || {}; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); let disableWeb = dw; @@ -70,15 +70,22 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { }); const transformerManager = new TransformerManager(logger, maybeSingletonCache !== undefined ? maybeSingletonCache : cacheFunc()); - transformerManager.register({type: 'user'}); - transformerManager.register({type: 'native'}); - for(const c of transformerConfigs) { + for(const c of transformers) { try { transformerManager.register(c); } catch (e) { logger.warn(new Error('Could not register a transformer', {cause: e})); } } + if(transformers.length === 0) { + logger.debug('No user-supplied transformer configs were found.'); + } + if(!transformerManager.hasTransformerType('user')) { + transformerManager.register({type: 'user', name: 'MSDefault'}); + } + if(!transformerManager.hasTransformerType('native')) { + transformerManager.register({type: 'native', name: 'MSDefault'}); + } const portVal: number | string = process.env.PORT ?? port; diff --git a/src/backend/tests/component/component.test.ts b/src/backend/tests/component/transformers.test.ts similarity index 85% rename from src/backend/tests/component/component.test.ts rename to src/backend/tests/component/transformers.test.ts index 573ca7fe..a9f94696 100644 --- a/src/backend/tests/component/component.test.ts +++ b/src/backend/tests/component/transformers.test.ts @@ -7,10 +7,12 @@ import AbstractComponent from "../../common/AbstractComponent.js"; import { ConditionalSearchAndReplaceRegExp, STAGE_TYPES, STAGE_TYPES_METADATA, STAGE_TYPES_USER, TRANSFORM_HOOK } from "../../common/infrastructure/Transform.js"; import { isConditionalSearchAndReplace } from "../../utils/PlayTransformUtils.js"; -import { asPlays, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js"; +import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js"; import { WebhookPayload } from "../../common/infrastructure/config/health/webhooks.js"; import { findCauseByMessage } from "../../utils/ErrorUtils.js"; -import { ComAtprotoServerDescribeServer } from "@atproto/api"; +import NativeTransformer from "../../common/transforms/NativeTransformer.js"; +import { initMemoryCache } from "../../common/Cache.js"; +import { Cacheable } from "cacheable"; chai.use(asPromised); @@ -29,6 +31,8 @@ class TestComponent extends AbstractComponent { const component = new TestComponent(); component.logger = childLogger(loggerTest, 'App'); +const memorycache = () => new Cacheable({primary: initMemoryCache()}) + describe('Play Transforms', function () { beforeEach(function () { @@ -69,7 +73,6 @@ describe('Play Transforms', function () { options: { playTransform: { preCompare: { - // @ts-expect-error type: "test", title: ['something'] } @@ -231,7 +234,6 @@ describe('Play Transforms', function () { options: { playTransform: { preCompare: { - // @ts-expect-error type: t, title: true } @@ -400,6 +402,57 @@ describe('Play Transforms', function () { }); }); + describe('Native Transformer', function () { + + it('Uses artist parsing functions', async function() { + + const t = new NativeTransformer({name: 'test', type: 'native'}, {logger: loggerTest, cache: memorycache()}); + await t.tryInitialize(); + + const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); + const play = generatePlay({artists: [str]}); + + const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play); + expect(transformedPlay.data.artists).eql(primaries.concat(secondaries)); + }); + + it('Ignores artists', async function() { + + const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); + + const t = new NativeTransformer({name: 'test', type: 'native', data: {ignoreArtists: [str]}}, {logger: loggerTest, cache: memorycache()}); + + await t.tryInitialize(); + + const play = generatePlay({artists: [str], track: 'My Test'}); + + const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play); + expect(transformedPlay.data.artists).eql([str]); + }); + + it('Uses custom delimiters artists', async function() { + + const [str, primaries, secondaries] = generateArtistsStr({primary: { + max: 3, + joiner: '•', + spacedJoiners: true, + ambiguousJoinedNames: false, + trailingAmpersand: false, + finalJoiner: false + }}); + + const t = new NativeTransformer({name: 'test', type: 'native', data: {extraDelimiters: ['•']}}, {logger: loggerTest, cache: memorycache()}); + + await t.tryInitialize(); + + const play = generatePlay({artists: [str], track: 'My Test'}); + + const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play); + expect(transformedPlay.data.artists).eql(primaries.concat(secondaries)); + }); + + }); + describe('Conditional Transforming', function () { describe('On Hook', function () { @@ -512,7 +565,7 @@ describe('Play Transforms', function () { describe('Multiple hook transforms', function () { - it('Accumulates transforms', async function () { + it('Accumulates transforms within a single stage', async function () { component.config = { options: { playTransform: { @@ -544,5 +597,35 @@ describe('Play Transforms', function () { expect(transformed.data.track).equal('My cool final thing track'); }); + it('Accumulates transforms across multiple stages', async function () { + component.config = { + options: { + playTransform: { + preCompare: [ + { + title: [ + { + search: "something", + replace: "bar" + } + ] + }, + { + type: 'native' + } + ] + } + } + } + + const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); + + component.buildTransformRules(); + const play = generatePlay({ track: 'My cool something track', artists: [str] }); + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); + expect(transformed.data.track).equal('My cool bar track'); + expect(transformed.data.artists).eql(primaries.concat(secondaries)); + }); + }); }) diff --git a/src/backend/utils/PlayComparisonUtils.ts b/src/backend/utils/PlayComparisonUtils.ts index 81333201..7f0211bd 100644 --- a/src/backend/utils/PlayComparisonUtils.ts +++ b/src/backend/utils/PlayComparisonUtils.ts @@ -3,6 +3,7 @@ import { PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy } import { buildTrackString } from "../../core/StringUtils.js"; import { playObjDataMatch } from "../utils.js"; import { comparePlayTemporally, hasAcceptableTemporalAccuracy, TemporalPlayComparisonOptions } from "./TimeUtils.js"; +import { RestType } from "ts-json-schema-generator"; export const metaInvariantTransform = (play: PlayObject): PlayObject => { @@ -34,6 +35,22 @@ export const playDateInvariantTransform = (play: PlayObject): PlayObject => { } } +export const playContentInvariantTransform = (play: PlayObject): PlayObject => { + const { + data: { + playDate, + playDateCompleted, + ...rest + } + } = play; + return { + data: { + ...rest + }, + meta: {} + } +} + export type PlayTransformer = (play: PlayObject) => PlayObject; export type ListTransformers = PlayTransformer[]; diff --git a/src/backend/utils/PlayTransformUtils.ts b/src/backend/utils/PlayTransformUtils.ts index 62550a21..1d01c01b 100644 --- a/src/backend/utils/PlayTransformUtils.ts +++ b/src/backend/utils/PlayTransformUtils.ts @@ -8,9 +8,8 @@ import { ConditionalSearchAndReplaceTerm, ExternalMetadataTerm, PlayTransformParts, - PlayTransformPartsArray, - PlayTransformPartsConfig, - PlayTransformRules, PlayTransformStage, PlayTransformUserParts, PlayTransformUserStage, SearchAndReplaceTerm, + PlayTransformStage, + SearchAndReplaceTerm, STAGE_TYPES, StageType, StageTypedConfig, @@ -172,173 +171,4 @@ export interface SuppliedRegex { export interface TransformPlayPartsOptions { logger?: () => Logger, regex?: SuppliedRegex -} - -export const transformPlayUsingParts = (play: PlayObject, parts: PlayTransformUserParts, options?: TransformPlayPartsOptions): PlayObject => { - const { - data: { - track, - artists, - albumArtists, - album - } = {} - } = play; - - const { - logger = () => loggerTest, - regex: { - searchAndReplace = searchAndReplaceFunc, - testMaybeRegex = testMaybeRegexFunc, - } = {}, - } = options || {}; - - const transformedPlayData: Partial = {}; - - let isTransformed = false; - - if(parts.when !== undefined) { - if(!testWhenConditions(parts.when, play, {testMaybeRegex})) { - return play; - } - } - - const searchAndReplaceMapper = (x: ConditionalSearchAndReplaceRegExp): ConditionalSearchAndReplaceRegExp => ({...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, {testMaybeRegex}) : undefined)}) - - if (parts.title !== undefined && track !== undefined) { - try { - const t = searchAndReplace(track, parts.title.map(x => ({...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, {testMaybeRegex}) : undefined)}))); - if (t !== track) { - transformedPlayData.track = t.trim() === '' ? undefined : t; - isTransformed = true; - } - } catch (e) { - logger().warn(new Error(`Failed to transform title: ${track}`, {cause: e})); - } - } - - if (parts.artists !== undefined && artists !== undefined && artists.length > 0) { - const transformedArtists: string[] = []; - let anyArtistTransformed = false; - for (const artist of artists) { - try { - const t = searchAndReplace(artist, parts.artists.map(searchAndReplaceMapper)); - if (t !== artist) { - anyArtistTransformed = true; - isTransformed = true; - } - if (t.trim() !== '') { - transformedArtists.push(t); - } - } catch (e) { - logger().warn(new Error(`Failed to transform artist: ${artist}`, {cause: e})); - transformedArtists.push(artist); - } - } - if (anyArtistTransformed) { - transformedPlayData.artists = transformedArtists; - } - } - - if (parts.artists !== undefined && albumArtists !== undefined && albumArtists.length > 0) { - const transformedArtists: string[] = []; - let anyArtistTransformed = false; - for (const artist of albumArtists) { - try { - const t = searchAndReplace(artist, parts.artists.map(searchAndReplaceMapper)); - if (t !== artist) { - anyArtistTransformed = true; - isTransformed = true; - } - if (t.trim() !== '') { - transformedArtists.push(t); - } - } catch (e) { - logger().warn(new Error(`Failed to transform albumArtist: ${artist}`, {cause: e})); - transformedArtists.push(artist); - } - } - if (anyArtistTransformed) { - transformedPlayData.albumArtists = transformedArtists; - } - } - - if (parts.album !== undefined && album !== undefined) { - try { - const t = searchAndReplace(album, parts.album.map(searchAndReplaceMapper)); - if (t !== album) { - isTransformed = true; - transformedPlayData.album = t.trim() === '' ? undefined : t; - } - } catch (e) { - logger().warn(new Error(`Failed to transform album: ${album}`, {cause: e})); - } - } - - if (isTransformed) { - - const transformedPlay = { - ...play, - data: { - ...play.data, - ...transformedPlayData - } - } - - return transformedPlay; - } - - return play; -} - -// export const countRegexes = (rules: PlayTransformRules): number => { -// let rulesCount = 0; -// if(rules.preCompare !== undefined) { -// for(const hookItem of rules.preCompare) { -// rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); -// } - -// } -// if(rules.postCompare !== undefined) { -// for(const hookItem of rules.postCompare) { -// rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); -// } -// } -// if(rules.compare !== undefined) { -// if(rules.compare.existing !== undefined) { -// for(const hookItem of rules.compare.existing) { -// rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); -// } -// } -// if(rules.compare.candidate !== undefined) { -// for(const hookItem of rules.compare.candidate) { -// rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); -// } -// } -// } -// return rulesCount; -// } - -const countWhens = (when: WhenConditionsConfig | undefined): number => { - if(when === undefined) { - return 0; - } - return when.reduce((acc, curr) => { - return acc + Object.keys(curr).length; - },0) -} - -/** - * Counts all rules within title/artist/album + whens WITHIN those rules - * */ -const countRulesInParts = (parts: PlayTransformParts): number => { - return Object.entries(parts).reduce((acc: number, entries: [string, ConditionalSearchAndReplaceRegExp[]]) => { - let curr = acc; - for(const rule of (entries[1] ?? [])) { - curr++; - if(typeof rule !== 'string' && rule.when !== undefined) { - curr += countWhens(rule.when); - } - } - return curr; - }, 0) -} +} \ No newline at end of file -- 2.51.2 From b5e835a0c6c7c0458f2875b990b43e6066b29342 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Sat, 29 Nov 2025 04:36:17 +0000 Subject: [PATCH 08/12] fix: Don't throw on nonimplemented notify for transformers --- src/backend/common/transforms/NativeTransformer.ts | 2 +- src/backend/common/transforms/UserTransformer.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts index 135066c5..59bdbc7a 100644 --- a/src/backend/common/transforms/NativeTransformer.ts +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -155,7 +155,7 @@ export default class NativeTransformer extends AtomicPartsTransformer { - throw new Error("Method not implemented."); + return; } protected getIdentifier(): string { return 'Native Transformer'; diff --git a/src/backend/common/transforms/UserTransformer.ts b/src/backend/common/transforms/UserTransformer.ts index 9b1565a8..1b177554 100644 --- a/src/backend/common/transforms/UserTransformer.ts +++ b/src/backend/common/transforms/UserTransformer.ts @@ -94,7 +94,7 @@ export default class UserTransformer extends AtomicPartsTransformer { - throw new Error("Method not implemented."); + return; } protected getIdentifier(): string { return 'User Transformer'; -- 2.51.2 From b7271ea4c28da46c1634052de1ab1ad0dfc428e5 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Sat, 29 Nov 2025 04:51:06 +0000 Subject: [PATCH 09/12] feat(transformer): Allow configuring artist parsing string sources --- .../common/transforms/NativeTransformer.ts | 60 ++++++++++++++----- .../tests/component/transformers.test.ts | 4 +- 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts index 59bdbc7a..275f9f1f 100644 --- a/src/backend/common/transforms/NativeTransformer.ts +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -9,10 +9,26 @@ import { TransformerOptions } from "./AbstractTransformer.js"; import { DELIMITERS_NO_AMP } from "../infrastructure/Atomic.js"; import { asArray } from "../../utils/DataUtils.js"; +export type ArtistParseSource = 'artists' | 'title' + +export const asArtistParseSource = (str: string): ArtistParseSource => { + const clean = str.trim().toLocaleLowerCase(); + switch(clean) { + case 'track': + case 'title': + return 'title'; + case 'artist': + case 'artists': + return 'artists'; + } + throw new Error(`ArtistParseSource must be one of 'artist' or 'title', given: ${clean}`); +} + export interface NativeTransformerData { delimiters?: string[] - extraDelimiters?: string[] - ignoreArtists?: string[] + delimitersExtra?: string[] + artistsIgnore?: string[] + artistsParseFrom?: ArtistParseSource[] } export type NativeTransformerConfig = TransformerCommon; @@ -23,6 +39,7 @@ export default class NativeTransformer extends AtomicPartsTransformer typeof x !== 'string'); + if(this.config.data.artistsIgnore !== undefined) { + this.config.data.artistsIgnore = asArray(this.config.data.artistsIgnore); + const nonStr = this.config.data.artistsIgnore.filter(x => typeof x !== 'string'); if(nonStr.length > 0) { throw new Error(`ignoreArtists must be an array of strings but non-strings found: ${nonStr.map(x => (x as unknown).toString()).join(' | ')}`) } - for(const i of this.config.data.ignoreArtists) { + for(const i of this.config.data.artistsIgnore) { try { this.ignoreArtistsRegex.push(parseToRegexOrLiteralSearch(i)); } catch (e) { throw new Error(`Could not convert ignoreArtist string to regex (or literal): ${i}`); } } - this.logger.debug(`Ignoring artists using ${this.config.data.ignoreArtists.length} rules`); + this.logger.debug(`Ignoring artists using ${this.config.data.artistsIgnore.length} rules`); } if(this.config.data.delimiters !== undefined) { this.delimiters = asArray(this.config.data.delimiters); - this.logger.debug(`Using user-defined delimiters '${this.config.data.extraDelimiters.join(' ')}' instead of built-ins`); - } else if(this.config.data.extraDelimiters !== undefined) { - this.delimiters = [...DELIMITERS_NO_AMP, ...(asArray(this.config.data.extraDelimiters))]; - this.logger.debug(`Using extra delimiters '${this.config.data.extraDelimiters.join(' ')}' with built-in delimiters '${DELIMITERS_NO_AMP.join(' ')}'`); + this.logger.debug(`Using user-defined delimiters '${this.config.data.delimitersExtra.join(' ')}' instead of built-ins`); + } else if(this.config.data.delimitersExtra !== undefined) { + this.delimiters = [...DELIMITERS_NO_AMP, ...(asArray(this.config.data.delimitersExtra))]; + this.logger.debug(`Using extra delimiters '${this.config.data.delimitersExtra.join(' ')}' with built-in delimiters '${DELIMITERS_NO_AMP.join(' ')}'`); } if(this.delimiters !== undefined) { this.delimiters.map(x => x.trim()); } + + if(this.config.data.artistsParseFrom !== undefined) { + const arr = asArray(this.config.data.artistsParseFrom); + this.parseArtistsFrom = arr.map(asArtistParseSource); + } + this.logger.debug(`Will try to parse artists from ${this.parseArtistsFrom.join(' and ')} string`); } protected doParseConfig(data: StageConfig) { @@ -94,7 +117,11 @@ export default class NativeTransformer extends AtomicPartsTransformer { let artists = []; - if (play.data.artists.length === 1) { + const { + artistsParseFrom: parseArtistsFrom = ['artists', 'title'] + } = this.config.data; + + if (play.data.artists.length === 1 && parseArtistsFrom.includes('artists')) { const matchedIgnoreArtists = this.ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, play.data.artists[0])})).filter(x => x !== undefined); if(matchedIgnoreArtists.length > 0) { this.logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`) @@ -114,10 +141,13 @@ export default class NativeTransformer extends AtomicPartsTransformer Date: Sat, 29 Nov 2025 05:06:26 +0000 Subject: [PATCH 10/12] refactor(transformer): Add monolithic artist option and make artist string parsing optional --- .../common/transforms/NativeTransformer.ts | 55 ++++++++++++------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts index 275f9f1f..60695e5c 100644 --- a/src/backend/common/transforms/NativeTransformer.ts +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -29,6 +29,7 @@ export interface NativeTransformerData { delimitersExtra?: string[] artistsIgnore?: string[] artistsParseFrom?: ArtistParseSource[] + artistsParseMonolithicOnly?: boolean } export type NativeTransformerConfig = TransformerCommon; @@ -83,8 +84,8 @@ export default class NativeTransformer extends AtomicPartsTransformer { let artists = []; const { - artistsParseFrom: parseArtistsFrom = ['artists', 'title'] - } = this.config.data; - - if (play.data.artists.length === 1 && parseArtistsFrom.includes('artists')) { - const matchedIgnoreArtists = this.ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, play.data.artists[0])})).filter(x => x !== undefined); - if(matchedIgnoreArtists.length > 0) { - this.logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`) - } else { - const artistCredits = parseArtistCredits(play.data.artists[0], this.delimiters); - if (artistCredits !== undefined) { - if (artistCredits.primary !== undefined) { - artists.push(artistCredits.primary); - } - if (artistCredits.secondary !== undefined) { - artists = artists.concat(artistCredits.secondary); + artistsParseFrom: parseArtistsFrom = ['artists', 'title'], + artistsParseMonolithicOnly = true + } = this.config.data || {}; + + if(parseArtistsFrom.includes('artists')) { + + if(play.data.artists.length === 1 || (play.data.artists.length > 1 && artistsParseMonolithicOnly === false)) { + + for(const artist of play.data.artists) { + + const matchedIgnoreArtists = this.ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, artist)})).filter(x => x !== undefined); + if(matchedIgnoreArtists.length > 0) { + this.logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`); + artists.push(artist); + } else { + const artistCredits = parseArtistCredits(artist, this.delimiters); + if (artistCredits !== undefined) { + if (artistCredits.primary !== undefined) { + artists.push(artistCredits.primary); + } + if (artistCredits.secondary !== undefined) { + artists = artists.concat(artistCredits.secondary); + } + } else { + // couldn't parse anything from artist string, use as-is + artists.push(artist); + } } + } + + } else { + // user does not want to try to parse artists when we already have more than one artist string + // -- likely this is because the user knows the artist data is already good and shouldn't be modified + artists = play.data.artists; } - if(artists.length === 0) { - artists.push(play.data.artists[0]); - } + } if(parseArtistsFrom.includes('title')) { -- 2.51.2 From d6c4d7cf25aae14290a1bd88210fbbb2d1d9b74a Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 1 Dec 2025 15:16:24 +0000 Subject: [PATCH 11/12] refactor(transforms): Implement per-stage defaults with override --- .../common/transforms/AbstractTransformer.ts | 16 +-- .../transforms/AtomicPartsTransformer.ts | 14 +- .../common/transforms/NativeTransformer.ts | 128 +++++++++++------- .../tests/component/transformers.test.ts | 6 +- src/core/Atomic.ts | 5 +- 5 files changed, 101 insertions(+), 68 deletions(-) diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index 9f2f953b..d7a65d0c 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -21,7 +21,7 @@ export interface RegexObject { searchAndReplace: typeof searchAndReplace } -export default abstract class AbstractTransformer extends AbstractInitializable { +export default abstract class AbstractTransformer extends AbstractInitializable { declare config: TransformerCommonConfig; configHash: string; @@ -40,16 +40,16 @@ export default abstract class AbstractTransformer extends AbstractIniti this.configHash = hashObject(this.config); } - public parseConfig(data: any) { + public parseConfig(data: any): Y { if (!isStageTyped(data)) { throw new Error(`Must be an object with a 'type' property.`); } return this.doParseConfig(data); } - protected abstract doParseConfig(data: StageConfig): StageConfig; + protected abstract doParseConfig(data: StageConfig): Y; - public async handle(data: StageConfig, play: PlayObject): Promise { + public async handle(data: Y, play: PlayObject): Promise { const cacheKey = `${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}` const cachedTransform = await this.cache.get(cacheKey); @@ -68,13 +68,13 @@ export default abstract class AbstractTransformer extends AbstractIniti let transformData: T; try { - transformData = await this.getTransformerData(play); + transformData = await this.getTransformerData(play, data); } catch (e) { throw new Error(`Could not fetch transformer data`, { cause: e }); } try { - await this.checkShouldTransform(play, transformData); + await this.checkShouldTransform(play, transformData, data); } catch (e) { this.logger.debug(new Error('checkShouldTransform did not pass, returning original Play', { cause: e })); return play; @@ -87,11 +87,11 @@ export default abstract class AbstractTransformer extends AbstractIniti protected abstract doHandle(data: StageConfig, play: PlayObject, transformData: T): Promise; - public async getTransformerData(play: PlayObject): Promise { + public async getTransformerData(play: PlayObject, stageConfig: Y): Promise { return undefined; } - public async checkShouldTransform(play: PlayObject, transformData: T): Promise { + public async checkShouldTransform(play: PlayObject, transformData: T, stageConfig: Y): Promise { return; } } \ No newline at end of file diff --git a/src/backend/common/transforms/AtomicPartsTransformer.ts b/src/backend/common/transforms/AtomicPartsTransformer.ts index 46a749c9..a7280af5 100644 --- a/src/backend/common/transforms/AtomicPartsTransformer.ts +++ b/src/backend/common/transforms/AtomicPartsTransformer.ts @@ -1,17 +1,19 @@ import { ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js"; -import { AtomicStageConfig } from "../infrastructure/Transform.js"; +import { AtomicStageConfig, StageConfig } from "../infrastructure/Transform.js"; import AbstractTransformer from "./AbstractTransformer.js"; -export default abstract class AtomicPartsTransformer extends AbstractTransformer { +//export type GenericAtomicStageConfig = - protected async doHandle(parts: AtomicStageConfig, play: PlayObject, transformData: T): Promise { +export default abstract class AtomicPartsTransformer = StageConfig> extends AbstractTransformer { + + protected async doHandle(parts: Z, play: PlayObject, transformData: T): Promise { const { throwOnFailure = false, } = this.config.options || {}; try { - await this.checkShouldTransform(play, transformData); + await this.checkShouldTransform(play, transformData, parts); } catch (e) { this.logger.debug(new Error('checkShouldTransform did not pass, returning original Play', { cause: e })); return play; @@ -84,11 +86,11 @@ export default abstract class AtomicPartsTransformer extends Abstrac return transformedPlay; } - public async getTransformerData(play: PlayObject): Promise { + public async getTransformerData(play: PlayObject, stageConfig: Z): Promise { return undefined; } - public async checkShouldTransform(play: PlayObject, transformData: T): Promise { + public async checkShouldTransform(play: PlayObject, transformData: T, stageConfig: Z): Promise { return; } diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts index 60695e5c..7b86fafd 100644 --- a/src/backend/common/transforms/NativeTransformer.ts +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -8,6 +8,8 @@ import { parseRegexSingle, parseToRegexOrLiteralSearch } from "@foxxmd/regex-bud import { TransformerOptions } from "./AbstractTransformer.js"; import { DELIMITERS_NO_AMP } from "../infrastructure/Atomic.js"; import { asArray } from "../../utils/DataUtils.js"; +import { MaybeLogger } from "../logging.js"; +import { childLogger } from "@foxxmd/logging"; export type ArtistParseSource = 'artists' | 'title' @@ -32,12 +34,74 @@ export interface NativeTransformerData { artistsParseMonolithicOnly?: boolean } +export interface NativeTransformerDataStrong { + artistsParseFrom?: ArtistParseSource[] + artistsParseMonolithicOnly?: boolean + ignoreArtistsRegex?: RegExp[] + delimiters?: string[] +} + +export interface NativeTransformerDataStage extends NativeTransformerDataStrong,PlayTransformNativeStage { +} + export type NativeTransformerConfig = TransformerCommon; -export default class NativeTransformer extends AtomicPartsTransformer { +export const parseStageConfig = (data: NativeTransformerData | undefined, logger: MaybeLogger = new MaybeLogger()): NativeTransformerDataStrong => { + + if (data === undefined) { + return {}; + } + + const config: NativeTransformerDataStrong = { + }; + + if (data === null || typeof data !== 'object') { + throw new Error('Native Transformer data should be an object or not defined.'); + } + if (data.artistsIgnore !== undefined) { + data.artistsIgnore = asArray(data.artistsIgnore); + const nonStr = data.artistsIgnore.filter(x => typeof x !== 'string'); + if (nonStr.length > 0) { + throw new Error(`ignoreArtists must be an array of strings but non-strings found: ${nonStr.map(x => (x as unknown).toString()).join(' | ')}`) + } + config.ignoreArtistsRegex = []; + for (const i of data.artistsIgnore) { + try { + config.ignoreArtistsRegex.push(parseToRegexOrLiteralSearch(i)); + } catch (e) { + throw new Error(`Could not convert ignoreArtist string to regex (or literal): ${i}`); + } + } + logger.debug(`Defaults - Ignoring artists using ${data.artistsIgnore.length} rules`); + } + + if (data.delimiters !== undefined) { + config.delimiters = asArray(data.delimiters); + logger.debug(`Defaults - Using user-defined delimiters '${data.delimitersExtra.join(' ')}' instead of built-ins`); + } else if (data.delimitersExtra !== undefined) { + config.delimiters = [...DELIMITERS_NO_AMP, ...(asArray(data.delimitersExtra))]; + logger.debug(`Defaults - Using extra delimiters '${data.delimitersExtra.join(' ')}' with built-in delimiters '${DELIMITERS_NO_AMP.join(' ')}'`); + } + + if (config.delimiters !== undefined) { + config.delimiters.map(x => x.trim()); + } + + if (data.artistsParseFrom !== undefined) { + const arr = asArray(data.artistsParseFrom); + config.artistsParseFrom = arr.map(asArtistParseSource); + logger.debug(`Defaults - Will try to parse artists from ${config.artistsParseFrom.join(' and ')} string`); + } + + return config; +} + +export default class NativeTransformer extends AtomicPartsTransformer { declare config: NativeTransformerConfig; + protected defaults: NativeTransformerDataStrong = {}; + ignoreArtistsRegex: RegExp[] = []; delimiters?: string[] parseArtistsFrom: ArtistParseSource[] @@ -47,54 +111,18 @@ export default class NativeTransformer extends AtomicPartsTransformer { - if(this.config.data === undefined) { - return true; - } - if(this.config.data === null || typeof this.config.data !== 'object') { - throw new Error('Native Transformer data should be an object or not defined.'); - } - if(this.config.data.artistsIgnore !== undefined) { - this.config.data.artistsIgnore = asArray(this.config.data.artistsIgnore); - const nonStr = this.config.data.artistsIgnore.filter(x => typeof x !== 'string'); - if(nonStr.length > 0) { - throw new Error(`ignoreArtists must be an array of strings but non-strings found: ${nonStr.map(x => (x as unknown).toString()).join(' | ')}`) - } - for(const i of this.config.data.artistsIgnore) { - try { - this.ignoreArtistsRegex.push(parseToRegexOrLiteralSearch(i)); - } catch (e) { - throw new Error(`Could not convert ignoreArtist string to regex (or literal): ${i}`); - } - } - this.logger.debug(`Ignoring artists using ${this.config.data.artistsIgnore.length} rules`); - } - - if(this.config.data.delimiters !== undefined) { - this.delimiters = asArray(this.config.data.delimiters); - this.logger.debug(`Using user-defined delimiters '${this.config.data.delimitersExtra.join(' ')}' instead of built-ins`); - } else if(this.config.data.delimitersExtra !== undefined) { - this.delimiters = [...DELIMITERS_NO_AMP, ...(asArray(this.config.data.delimitersExtra))]; - this.logger.debug(`Using extra delimiters '${this.config.data.delimitersExtra.join(' ')}' with built-in delimiters '${DELIMITERS_NO_AMP.join(' ')}'`); - } - - if(this.delimiters !== undefined) { - this.delimiters.map(x => x.trim()); - } - - if(this.config.data.artistsParseFrom !== undefined) { - const arr = asArray(this.config.data.artistsParseFrom); - this.parseArtistsFrom = arr.map(asArtistParseSource); - this.logger.debug(`Will try to parse artists from ${this.parseArtistsFrom.join(' and ')} string`); - } + this.defaults = parseStageConfig(this.config.defaults, childLogger(this.logger, 'Defaults')); + return true; } - protected doParseConfig(data: StageConfig) { + protected doParseConfig(data: NativeTransformerDataStage) { if (data.type !== 'native') { throw new Error(`NativeTransformer is only usable with 'native' type stages`); } - const stage: PlayTransformNativeStage = { + const stage: NativeTransformerDataStage = { ...data, + ...parseStageConfig(data), type: 'native' } @@ -116,12 +144,14 @@ export default class NativeTransformer extends AtomicPartsTransformer { + public async getTransformerData(play: PlayObject, stageConfig: NativeTransformerDataStage): Promise { let artists = []; const { - artistsParseFrom: parseArtistsFrom = ['artists', 'title'], - artistsParseMonolithicOnly = true - } = this.config.data || {}; + artistsParseFrom: parseArtistsFrom = this.defaults.artistsParseFrom ?? ['artists', 'title'], + artistsParseMonolithicOnly = this.defaults.artistsParseMonolithicOnly ?? true, + ignoreArtistsRegex = this.defaults.ignoreArtistsRegex ?? [], + delimiters = this.defaults.delimiters + } = stageConfig || {}; if(parseArtistsFrom.includes('artists')) { @@ -129,12 +159,12 @@ export default class NativeTransformer extends AtomicPartsTransformer ({reg: x.toString(), res: parseRegexSingle(x, artist)})).filter(x => x !== undefined); + const matchedIgnoreArtists = ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, artist)})).filter(x => x !== undefined); if(matchedIgnoreArtists.length > 0) { this.logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`); artists.push(artist); } else { - const artistCredits = parseArtistCredits(artist, this.delimiters); + const artistCredits = parseArtistCredits(artist, delimiters); if (artistCredits !== undefined) { if (artistCredits.primary !== undefined) { artists.push(artistCredits.primary); @@ -159,7 +189,7 @@ export default class NativeTransformer extends AtomicPartsTransformer> { - data?: T; +export interface TransformerCommonConfig, Y = Record> { + defaults?: T; + data?: Y type: string; name?: string; options?: { -- 2.51.2 From 6ac44397d0f4b881c09a1390fef24c0b9aa23aa8 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 1 Dec 2025 18:04:42 +0000 Subject: [PATCH 12/12] docs: Refactor transformer docs with stages --- docsite/docs/configuration/transforms.mdx | 523 ------------------ .../configuration/transforms/_category_.json | 7 + .../docs/configuration/transforms/native.mdx | 174 ++++++ .../configuration/transforms/transforms.mdx | 475 ++++++++++++++++ .../docs/configuration/transforms/user.mdx | 275 +++++++++ 5 files changed, 931 insertions(+), 523 deletions(-) delete mode 100644 docsite/docs/configuration/transforms.mdx create mode 100644 docsite/docs/configuration/transforms/_category_.json create mode 100644 docsite/docs/configuration/transforms/native.mdx create mode 100644 docsite/docs/configuration/transforms/transforms.mdx create mode 100644 docsite/docs/configuration/transforms/user.mdx diff --git a/docsite/docs/configuration/transforms.mdx b/docsite/docs/configuration/transforms.mdx deleted file mode 100644 index 67336461..00000000 --- a/docsite/docs/configuration/transforms.mdx +++ /dev/null @@ -1,523 +0,0 @@ ---- -sidebar_position: 4 -title: Scrobble Modification -toc_max_heading_level: 4 ---- - -Multi-scrobbler configs support the ability to modify scrobble data in an automated fashion by matching and replacing strings in **title, artists, and album** at many different times in multi-scrobbler's lifecycle. - -### Why? - -You may need to "clean up" data from a Source or before sending to a scrobble Client due to any number of reasons: - -* ID3 tags in your music collection are dirty or have repeating garbage IE `[YourMusicSource.com] My Artist - My Title` -* A Source's service often incorrectly adds data to some field IE `My Artist - My Title (Album Version)` when the title should just be `My Title` -* An Artist you listen to often is spelled different between a Source and a Client which causes duplicate scrobbles - -In any scenario where a repeating pattern can be found in the data it would be nice to be able to fix it before the data gets downstream or to help prevent duplicate scrobbling. Multi-scrobbler can help you do this. - -## Overview - -### Journey of a Scrobble - -First, let's recap the lifecycle of a scrobble in multi-scrobbler: - -**Sources** are the beginning of the journey for a **Play** (song you've listened to long enough to be scrobblable) - -* A Source finds a new valid **Play** -* The Source **compares** this new Play to all the other Plays it has already seen, if the Play is unique (title/artist/album/listened datetime) then... -* The Source **discovers** the Play, adds it to Plays it has seen already, and broadcasts the Play should be scrobbled to all Clients - -Scrobble **Clients** listen for discovered Plays from Sources, then... - -* A Client receives a **Play** from a Source -* The Client **compares** this Play to all the other scrobbles it has already seen, if the Play is unique (title/artist/album/listened datetime) then... -* The Client **scrobbles** the Play downstream to the scrobble service and adds it as a Scrobble it has seen already - -### Lifecyle Hooks - -You'll notice there is a pattern above that looks like this: - -* **Before** data is compared -* Data is **compared** -* **After** data is compared - -These points, during both Source and Client processes, are when you can hook into the scrobble lifecycle and modify it. - -#### TLDR - -In more concrete terms this is the structure of hooks within a configuration (can be used in any **Source** or **Client**): - -```json5 title="lastfm.json" {10-14} -[ - { - "name": "myLastFm", - "enable": true, - "configureAs": "source", - "data": { - // ... - }, - "options": { - "playTransform": { - "preCompare": {/* ... */}, - "compare": {/* ... */}, - "postCompare": {/* ... */} - } - } - } -] -``` - -##### Hook - -For **Sources**: - -* `preCompare` - modify Play data immediately when received -* `compare` - temporarily modify Play data when it is being compared to see if Play was already discovered -* `postCompare` - modify Play data before sending to scrobble **Clients** - -For **Clients**: - -* `preCompare` - modify Play data immediately when received -* `compare` - temporarily modify Play data when it is being compared to see if it was already scrobbled -* `postCompare` - modify Play data before scrobbling it to downstream service and adding to already seen scrobbles - -:::tip - -Keep in mind that modifying Scrobble/Play data earlier in the lifecycle will affect that data at all times later in the lifecycle. - -For example, to modify the track so it's the same anywhere it is processed in multi-scrobbler you only need to modify it in the **Source's** `preCompare` hook because all later processes will receive the data with the modified track. - -::: - -### Modification Parts - - -Each [**hook**](#hook) (`preCompare` etc...) is an object that specifies what part of the **Play** to modify: - -```json5 -{ - "title": [/* ... */], - "artists": [/* ... */], - "album": [/* ... */] -} -``` - -##### Expression - -and then a **list** what pattern/replacements (expressions) to use for the modification by using either simple strings or `search-replace` objects: - -```json5 -[ - "badTerm", // remove all instances of 'badTerm' - { - "search": "anotherBadTerm", // and also match all instances of 'anotherBadTerm' - "replace": "goodTerm" // replace with the string 'goodTerm' - } -] -``` - -Putting it all together: - -```json5 title="lastfm.json" -[ - { - "name": "myLastFm", - "enable": true, - "configureAs": "source", - "data": { - // ... - }, - "options": { - "playTransform": { - "preCompare": { - "title": [ - "badTerm", - { - "search": "badTerm", - "replace": "goodTerm" - } - ] - }, - } - } - } -] -``` - -:::note - -If the value of the field (title, an artist, album) is an empty string after transforming then the field is **removed.** - -::: - -:::tip - -Modifications can also be applied to **all Sources** or **all Clients** when using the [AIO Config](./configuration.mdx?configType=aio#configuration-types) `config.json` by setting `playTransform` in `sourceDefaults` or `clientDefaults`: - -
- - Example -```json5 title="config.json" -{ - "sourceDefaults": { // will apply playTransform to all sources - "playTransform": { - "preCompare": { - "title": [ - "(Album Version)" - ] - } - } - }, - "sources": [/* ... */], - "clients": [/* ... */] -} -``` -
- -::: - -#### Compare Hook - -The `compare` [hook](#hook) is slightly different than `preCompare` and `postCompare`. It consists of an object where you define which side(s) of the comparison should be modified. It also **does not modify downstream data!** Instead, the modifications are made only for use in the comparison. - -```json5 title="lastfm.json" -[ - { - "name": "myLastFm", - // ... - "options": { - "playTransform": { - "compare": { - "candidate": {/* ... */}, // modify the "new" Play being compared - "existing": {/* ... */}, // modify all "existing" Play/Scrobbles the new Play is being compared against - }, - } - } - } -] -``` - -#### Regular Expressions - -In addition to plain strings [expressions](#expression) that are matched and removed you can also use Regular Expressions. Write your regex like you normally would, but as a string, and it'll automatically be parsed: - -```json5 -[ - "/^\(\w+.com)/i", // matches any string that starts with '(YourMusic.com)' and removes it - { - "search": "/^\(\w+.com)/i", // matches any string that starts with '(YourMusic.com)' - "replace": "[MySite.com]" // replace with the string '[MySite.com]' - } -] -``` - -The `replace` property uses javascript's [`replace()` function and so can use any special string characters.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_the_replacement) - -### Conditional Modification - -#### "When" Condition - -Top-level hooks **and** individual rules also support a `when` key for testing **if they should be run.** - -The `when` key is similar to a normal [modification](#modification-parts) except: - -* the keys accept a single string instead of an array -* the `when` key data is an array instead of a single object - -All parts of an individual `when` clause must test true to "pass" but if **any** `when` clauses pass the hook/rule is processed. Example `when` data: - -```json5 -{ - "when": [ - { - "artist": "Elephant Gym", // both of these must match the Play object (AND) - "album": "Dreams" // both of these must match the Play object (AND) - }, - // OR - { - "title": "/(Remastered)$/", // both of these must match the Play object (AND) - "album": "Various Artists" // both of these must match the Play object (AND) - } - ] -} -``` - -More succinctly: - -* All parts (`artist` `album` `title`) of a `when` are `AND` conditions -* All part-objects in the `when` array are `OR` conditions - -
- -Example of top-level hook with when condition - -```json5 -{ - // IF the artist is Elephant Gym - // THEN Run preCompare hook ELSE skip this hook - // - // Run search-replace on album - // Run regex title remove - "sourceDefaults": { - "playTransform": { - "preCompare": { - "when": [ - { - "artist": "/Elephant Gym/" - } - ], - "album": [ - { - "search": "Dreams", - "replace": "夢境" - } - ], - "title": ["/\s\-\s滾石40\s滾石撞樂隊\s40團拚經典(.+)$/i"] - }, - } - } -} -``` - -
- -
- -Example of individual rule with when condition - -```json5 -{ - // Always run preCompare - // - // On search-replace in title... - // IF artist matches "Elephant Gym" - // THEN Run regex search-replace ELSE skip this rule - // - // Run live|remastered regex remove - "sourceDefaults": { - "playTransform": { - "preCompare": { - "title": [ - { - "search": "/\\s\\-\\s滾石40\\s滾石撞樂隊\\s40團拚經典(.+)$/i", - "replace": "", - "when": [ - { - "artist": "/Elephant Gym/" - } - ] - }, - "/(\\s\\-\\s|\\s)(feat\\.(.+)|live|remastered(.+))$/i" - ], - } - } - } -} -``` - -
- -#### Top-level Hook array - -Top-level hooks can also be an array of hooks. This makes creating multiple scenarios for top-level `when`-gated hooks easier. All hooks in the array will be run (assuming their `when`'s pass, if they exist) and their **input will be the Play object output of the previous hook in the array.** - -
- -Example - -```json5 -{ - "sourceDefaults": { - "playTransform": { - "preCompare": [ - // first lifecycle hook of preCompare to run - { - "title": [ - { - "search": "something", - "replace": "else unique" - } - ] - }, - // second lifecycle hook of preCompare to run - { - "title": [ - { - "search": "else unique", - "replace": "very demure" - } - ] - }, - ] - } - } -} -``` - -
- -### Logging - -MS can log the output of hook transformations if/when they occur. In the `playTransform` object of a Source/Client config use `log`: - -* `"log": true` => Output original play + final transformed output of last hook in the array -* `"log": "all"` => Output original play + final transformed output of **each** hook in the array - -```json5 -{ - "name": "myThing", - "data": {/*...*/}, - "options": { - "playTransform": { - "preCompare": {/*...*/}, - "log": true - } - } -} -``` - -## Examples - -### Remove phrase from Title in all new Plays - -Removes the phrase `(Album Version)` from the Title of a Play - - -
- - Example -```json5 title="config.json" -{ - "sourceDefaults": { - "playTransform": { - "preCompare": { - "title": [ - "(Album Version)" - ] - } - } - } -} - -``` -
- -### Remove all parenthesized content from the end of a title - -
- - Example -```json5 title="lastfm.json" -[ - { - "name": "myLastFm", - // ... - "options": { - "playTransform": { - "compare": { - "candidate": { - "title": [ - "/(\(.+\))\s*$/" - ] - }, - "existing": { - "title": [ - "/(\(.+\))\s*$/" - ] - }, - }, - } - } - } -] -``` -
- -### Rename misspelled artist in all new Plays - -
- - Example -```json5 title="config.json" -{ - "sourceDefaults": { - "playTransform": { - "preCompare": { - "artists": [ - { - "search": "Boz Skaggs", - "replace": "Boz Scaggs" - } - ] - } - } - } -} -``` -
- -### Remove "Various Artists" albums in all new Plays - -
- - Example -```json5 title="config.json" -{ - "sourceDefaults": { - "playTransform": { - "preCompare": { - "album": [ - { - "search": "Various Artists", - "replace": "" - } - ] - } - } - } -} -``` -
- -### Extract primary Artist from delimited, multi-Artist string - -
- - When the Artist string is actually a multi-artist, delimited string, this search-and-replace will replace the string with just the first artist found. - - Ex - - ``` - My Artist One / My Artist Two / Another Guy - My Artist One - ``` - - Artists are delimited with a spaced forward slash (`/`) in the regex below. Replace the contents of the `delim` capture group with the delimiter for your use case. Some more common scenarios: - - * `(?\\/)` No spaces between slash IE `My Artist One/My Artist Two/Another Guy` - * `(?\\s*\\\\\s*)` Backslash instead of forward slash IE `My Artist One \ My Artist Two \ Another Guy` - * `(?,)` Comma IE `My Artist One, My Artist Two, Another Guy` - -
- - Example - ```json5 title="config.json" - { - "sourceDefaults": { - "playTransform": { - "preCompare": { - "artists": [ - { - "search": "(.*?)(?\\s*\\/\\s*)(.*$)", - "replace": "$1" - } - ] - } - } - } - } - ``` -
- -
\ No newline at end of file diff --git a/docsite/docs/configuration/transforms/_category_.json b/docsite/docs/configuration/transforms/_category_.json new file mode 100644 index 00000000..f84f8c7f --- /dev/null +++ b/docsite/docs/configuration/transforms/_category_.json @@ -0,0 +1,7 @@ +{ + "label": "Enhance Scrobbles", + "link": { + "type": "doc", + "id": "configuration/transforms/transforms" + } +} diff --git a/docsite/docs/configuration/transforms/native.mdx b/docsite/docs/configuration/transforms/native.mdx new file mode 100644 index 00000000..650c7fa0 --- /dev/null +++ b/docsite/docs/configuration/transforms/native.mdx @@ -0,0 +1,174 @@ +--- +title: Native Stage +toc_min_heading_level: 2 +toc_max_heading_level: 5 +--- + +The **Native** [Stage](/configuration/transforms#stage) uses [built-in heuristics](https://github.com/FoxxMD/multi-scrobbler/blob/master/src/backend/tests/plays/playParsing.test.ts) to try to extract Artists from Play artist/track data. + +This Stage is most useful for Sources that report limited data such as: + +* [Subsonic](/configuration/sources/subsonic) - Reports Artists as a single string + +A non-exhaustive list of heuristics: + +* Splits artists in artist string using common delimiters EX `Foo Artist, Bar Guy, Baz Band - My Cool Song` + * Does not split artists with `&` in name, if other delimiters are present + * Does not split artist name when only one delimiter is present +* Splits artists on common joiner phrases (ft. feat. vs. etc...) + * Extracts artists from Play title using joiner phrases EX `My Cool Song (feat. SomeGuy)` + + +## Configuration + +Available properties for [Stage Configuration](/configuration/transforms#configuring-stages): + + +* `delimitersExtra` - A list of string characters that should be considered delimiters for artists **in addition to** multi-scrobbler's default list (`, / \ `) +* `delimiters` - A list of string characters that should be considered delimiters for artists. + * **Replaces** all delimiters (MS not use any defaults, only what you give it) +* `artistsIgnore` - a list of strings and/or regular expressions. Any monolothic artist string that matches from the list _will not be modified._ +* `artistsParseFrom` a list of the properties that should be used to try to extract artists. Can be `artists` `title` or both. Defaults to both when not provided in options ( `["artists", "title"]` ) + * When `artists` is present it tries to extract additional artists from artist strings + * When `title` is present it tries to extract artists from ft. feat. vs. etc... found in the track title + * Importantly, if `artists` is _not_ present in `artistsParseFrom` then _no artists_ are used at all (only those from `title`, if present) +* `artistsParseMonolithicOnly` - boolean value, defaults to `true`. When `true` native tranformer will only attempt to extract artists if the scrobble data has _only one string for artist_ (pre-transform) + * This means that, for Sources like Spotify/Jellyfin/etc. that provide proper lists of artists in their data _and the list has more than one string_, it will not try to extract artists from their strings + +
+ +Example of the behavior for `artistsParseFrom` + +``` +The Foos, The Bars - My Cool Track (ft. Frank) +``` + +Config => extracted artists + +``` +"artistsParseFrom": ["artists", "title"] => The Foos, The Bars, Frank +"artistsParseFrom": ["artists"] => The Foos, The Bars +"artistsParseFrom": ["title"] => Frank +"artistsParseFrom": [] => +``` + +
+ +### Rules + +Each [Rule](/configuration/transforms#stage-rules) should be either a boolean, specifying if the transformed data should be used for this field, or a [`when` condition.](/configuration/transforms#conditional-moditication): + +```json5 +{ + "type": "native", + // ... + "title": false, // will not apply any changes to Play title + "artists": { + "when": {/* ... */}, // will only apply changes to Play artists if "when" is satisfied + /* ... */ + }, + "album": true // will always apply changes to Play album +} +``` + +If a rule is not present then multi-scrobbler defaults it to `true`. + +## Examples + +### Parse only artists string using a custom delimiter + +
+ +Example + +Your [AIO Config](/configuration?configType=aio#configuration-types): + +```json5 title="config.json" +{ + // ... + "transformers": [ + { + "type": "native", + "name": "MyNativeTransformer", + "defaults": { + // default delimiters when this Stage is used in a hook + "delimiters": [ + "•" + ], + // default delimiters when this Stage is used in a hook + "artistsParseFrom": ["artists"] + } + } + ] +} +``` + +In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types): + +```json5 title="subsonic.json" +[ + { + "name": "MySubsonic", + "data": { /* ... */}, + "options": { + "playTransform": { + "preCompare": [ + { + "type": "native", + "name": "MyNativeTransformer" + } + ] + } + } + } +] +``` +
+ +### Don't parse specific artists + +
+ +Example + +Your [AIO Config](/configuration?configType=aio#configuration-types): + +```json5 title="config.json" +{ + // ... + "transformers": [ + { + "type": "native", + "name": "MyNativeTransformer", + "defaults": { + "artistsIgnore": [ + "Crosby, Stills, Nash & Young", + "Polo & Pan" + ] + } + } + ] +} +``` + +In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types): + +```json5 title="subsonic.json" +[ + { + "name": "MySubsonic", + "data": { /* ... */}, + "options": { + "playTransform": { + "preCompare": [ + { + "type": "native", + "name": "MyNativeTransformer" + } + ] + } + } + } +] +``` +
\ No newline at end of file diff --git a/docsite/docs/configuration/transforms/transforms.mdx b/docsite/docs/configuration/transforms/transforms.mdx new file mode 100644 index 00000000..f7cbae79 --- /dev/null +++ b/docsite/docs/configuration/transforms/transforms.mdx @@ -0,0 +1,475 @@ +--- +sidebar_position: 4 +title: Enhancing Scrobbles +toc_max_heading_level: 5 +--- + +Multi-scrobbler configs support the ability to enhance scrobble data in an automated fashion by matching and replacing strings in **title, artists, and album** at many different times in multi-scrobbler's lifecycle. + +
+ +Why Would I Do This? + +You may need to "clean up" data from a Source or before sending to a scrobble Client due to any number of reasons: + +* ID3 tags in your music collection are dirty or have repeating garbage IE `[YourMusicSource.com] My Artist - My Title` +* A Source's service often incorrectly adds data to some field IE `My Artist - My Title (Album Version)` when the title should just be `My Title` +* An Artist you listen to often is spelled different between a Source and a Client which causes duplicate scrobbles + +In any scenario where a repeating pattern can be found in the data it would be nice to be able to fix it before the data gets downstream or to help prevent duplicate scrobbling. Multi-scrobbler can help you do this. + +
+ +## Journey of a Scrobble + +First, let's recap the lifecycle of a scrobble in multi-scrobbler: + +**Sources** are the beginning of the journey for a **Play** (song you've listened to long enough to be scrobblable) + +* A Source finds a new valid **Play** +* The Source **compares** this new Play to all the other Plays it has already seen, if the Play is unique (title/artist/album/listened datetime) then... +* The Source **discovers** the Play, adds it to Plays it has seen already, and broadcasts the Play should be scrobbled to all Clients + +Scrobble **Clients** listen for discovered Plays from Sources, then... + +* A Client receives a **Play** from a Source +* The Client **compares** this Play to all the other scrobbles it has already seen, if the Play is unique (title/artist/album/listened datetime) then... +* The Client **scrobbles** the Play downstream to the scrobble service and adds it as a Scrobble it has seen already + +## Lifecyle Hooks + +You'll notice there is a pattern above that looks like this: + +* **Before** data is compared +* Data is **compared** +* **After** data is compared + +These points, during both Source and Client processes, are when you can hook into the scrobble lifecycle and modify it. + +##### TLDR + +In more concrete terms this is the structure of hooks within a configuration (can be used in any **Source** or **Client**): + +```json5 title="lastfm.json" {10-14} +[ + { + "name": "myLastFm", + "enable": true, + "configureAs": "source", + "data": { + // ... + }, + "options": { + "playTransform": { + "preCompare": [/* ... */], + "compare": [/* ... */], + "postCompare": [/* ... */] + } + } + } +] +``` + +### Hook + +For **Sources**: + +* `preCompare` - modify Play data immediately when received +* `compare` - temporarily modify Play data when it is being compared to see if Play was already discovered +* `postCompare` - modify Play data before sending to scrobble **Clients** + +For **Clients**: + +* `preCompare` - modify Play data immediately when received +* `compare` - temporarily modify Play data when it is being compared to see if it was already scrobbled +* `postCompare` - modify Play data before scrobbling it to downstream service and adding to already seen scrobbles + +:::tip + +Keep in mind that modifying Scrobble/Play data earlier in the lifecycle will affect that data at all times later in the lifecycle (except when using the **compare** hook). + +For example, to modify the track so it's the same anywhere it is processed in multi-scrobbler you only need to modify it in the **Source's** `preCompare` hook because all later processes will receive the data with the modified track. + +::: + +
+ +Using `compare` hook + +The `compare` [hook](#hook) is slightly different than `preCompare` and `postCompare`. It consists of an object where you define which side(s) of the comparison should be modified. It also **does not modify downstream data!** Instead, the modifications are made only for use in the comparison. + +```json5 title="lastfm.json" +[ + { + "name": "myLastFm", + // ... + "options": { + "playTransform": { + "compare": [ + { + "candidate": {/* ... */}, // modify the "new" Play being compared + "existing": {/* ... */}, // modify all "existing" Play/Scrobbles the new Play is being compared against + } + ], + } + } + } +] +``` + +
+ +## Modification Stage {#stage} + +Each [**hook**](#hook) is made up of one or more **Stages**. A Stage is a self-contained, unique way of enhancing or modifying the Play data. Some examples of a Stage: + +* The [User](/configuration/transforms/user) Stage allows a user to define search-and-replace terms for Artist/Title/Album +* The [Native](/configuration/transforms/native) Stage uses MS's built-in heuristics to extract Artists from a single Artist string +* The Musicbrainz Stage tries to match Play data with the Musicbrainz database and to standardize the Artist/Title/Album data + +Each Stage in a Hook receives Play data from the previous Stage. + +Within a hook, each Stage minimally consists of a `type` to identify what Stage it is along with any other data specific to that stage: + +```json5 +{ + "type": "native" + // optional, stage specific data here... +} +``` + +### Configuring Stages {#configuring-stages} + +Stages may be globally configured using [AIO Config](/configuration?configType=aio#configuration-types) `config.json` file in the top-level `transformers` block. + +Each Stage consists of: + +* `type` the type of Stage +* `name` a unique name for the Stage, to be (potentially) used with hooks +* `defaults` - An object defining default configuration for this stage, when used in a Hook. +* `data` - An object containing any data required to initially configure the stage itself (Example: API URL, username, password, etc...) + +
+ +Example + +Your [AIO Config](/configuration?configType=aio#configuration-types): + +```json5 title="config.json" +{ + // ... + "transformers": [ + { + "type": "native", + "name": "MyNativeTransformer", + "defaults": { + // default delimiters when this Stage is used in a hook + "delimiters": [ + "•" + ], + // default delimiters when this Stage is used in a hook + "artistsParseFrom": ["artists"] + } + } + ] +} +``` + +In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types): + +```json5 title="subsonic.json" +[ + { + "name": "MySubsonic", + "data": { /* ... */}, + "options": { + "playTransform": { + "preCompare": [ + { + "type": "native" + // when "name" is not defined, uses first found "native" transformer + } + ] + } + } + } +] +``` + + +
+ +Multiple stages of the same type may be configured, allowing you to define several sets of default behavior. + +
+ +Example + +Your [AIO Config](/configuration?configType=aio#configuration-types): + +```json5 title="config.json" +{ + // ... + "transformers": [ + { + "type": "native", + "name": "DotTransformer", + "defaults": { + "delimiters": [ + "•" + ], + "artistsParseFrom": ["artists"] + } + }, + { + "type": "native", + "name": "TitleOnly", + "defaults": { + // extracts and uses *only* artists found in title string + "artistsParseFrom": ["title"] + } + } + ] +} +``` + +In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types): + +```json5 title="subsonic.json" +[ + { + "name": "MySubsonic", + "data": { /* ... */}, + "options": { + "playTransform": { + "preCompare": [ + { + "type": "native" + "name": "DotTransformer" + } + ] + } + } + } +] +``` + +In a [VLC](/configuration/sources/vlc) [File Config](/configuration?configType=file#configuration-types): + +```json5 title="vlc.json" +[ + { + "name": "MyVLC", + "data": { /* ... */}, + "options": { + "playTransform": { + "preCompare": [ + { + "type": "native" + "name": "TitleOnly" + } + ] + } + } + } +] +``` + +
+ +#### Overriding Configuration + +The default configuration you set for your Stage may be overridden in any usage of the Stage within a Hook. + +
+ +Example + +Your [AIO Config](/configuration?configType=aio#configuration-types): + +```json5 title="config.json" +{ + // ... + "transformers": [ + { + "type": "native", + "name": "MyNativeTransformer", + "defaults": { + // default delimiters when this Stage is used in a hook + "delimiters": [ + "•" + ], + // default delimiters when this Stage is used in a hook + "artistsParseFrom": ["artists"] + } + } + ] +} +``` + +In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types): + +```json5 title="subsonic.json" +[ + { + "name": "MySubsonic", + "data": { /* ... */}, + "options": { + "playTransform": { + "preCompare": [ + { + "type": "native" + "name": "MyNativeTransformer", + // overrides property from "defaults" + "artistsParseFrom": ["artists", "title"] + } + ] + } + } + } +] +``` + +
+ + +### Rules for Play Data {#stage-rules} + +Each [Stage](#stage) may specify whether it should apply the resulting transformation to different parts of the Play data by specifying `title`, `artists` and/or `album` in the Stage object. + +```json5 +{ + "type": "native", + // ... + "title": false, // will not apply any changes to Play title + "artists": { + "when": {/* ... */}, // will only apply changes to Play artists if "when" is satisfied + /* ... */ + }, + "album": true // will always apply changes to Play album +} +``` + +The actual value of each property may be different for each Stage. Check the docs for the Stage you want to use to see its usage of `title`, `artists`, and `album`. + +Generically, though, each property may be some value **or** an object combining a [`when` condition](#conditional-modification) and that value. + +If none of the properties are specified in the stage then it's assumed all transformed data should be used. + +:::note + +Specifying these Rules is **not** the same as [configuring the Stage](#configuring-stages). Rules only determine if the *result* of the transformation should be used (replace) the existing Play Data. + +::: + +## Conditional Modification + +[Stages](#stage) within a [Hook](#hook), and [Rules](#stage-rules) within each Stage, support a `when` object for testing **if they should be run.** + +The `when` object may have propertes for `artist`, `title` and/or `album`. Each property may be a string or regular expression. The value of the property is used to match the **pre-transformation** values from Play data. + +All parts of an individual `when` clause must test true to "pass" but if **any** `when` clauses pass the Stage/Rule is processed. + +```json5 +{ + "when": + { + "artist": "Elephant Gym", // Play must have an artist matching "Elephant Gym" (AND) + "album": "Dreams" // Play object must have an album matching "Dreams" (AND) + } +} +``` + +```json5 +{ + "when": [ + { + "artist": "Elephant Gym", // Play must have an artist matching "Elephant Gym" (AND) + "album": "Dreams" // Play object must have an album matching "Dreams" (AND) + }, + // OR + { + "title": "/(Remastered)$/", // Play title must match regular expression (AND) + "album": "Various Artists" // Play album must match "Various Artists" (AND) + } + ] +} +``` + +More succinctly: + +* All parts (`artist` `album` `title`) of a `when` are `AND` conditions +* All part-objects in the `when` array are `OR` conditions + +
+ +Example of Stage with `when` condition + +```json5 +{ + // IF the artist is Elephant Gym + // THEN Run native stage + "playTransform": { + "preCompare": [ + { + "type": "native", + "when": [ + { + "artist": "/Elephant Gym/" + } + ] + } + ], + } +} +``` + +
+ +
+ +Example of individual rule with when condition + +```json5 +{ + // Always run native Stage + // + // IF artist matches "Elephant Gym" + // THEN use result of native stage for "artists" of Play data + "playTransform": { + "preCompare": { + "type": "native", + "artists": + { + "when": [ + { + "artist": "/Elephant Gym/" + } + ] + }, + } + } +} +``` + +
+ +## Logging + +MS can log the output of Stage transformations if/when they occur. In the `playTransform` object of a Source/Client config use `log`: + +* `"log": true` => Output original play + final transformed output of last Stage in the array +* `"log": "all"` => Output original play + final transformed output of **each** Stage in the array + +```json5 +{ + "name": "myThing", + "data": {/*...*/}, + "options": { + "playTransform": { + "preCompare": {/*...*/}, + "log": true + } + } +} +``` \ No newline at end of file diff --git a/docsite/docs/configuration/transforms/user.mdx b/docsite/docs/configuration/transforms/user.mdx new file mode 100644 index 00000000..7492f8a7 --- /dev/null +++ b/docsite/docs/configuration/transforms/user.mdx @@ -0,0 +1,275 @@ +--- +title: User Stage +toc_min_heading_level: 2 +toc_max_heading_level: 5 +--- + +The **User** [Stage](/configuration/transforms#stage) uses [**search-and-replace** expressions](#search-and-replace-expression), provided by you, to modify/replace parts of Play data. + +This Stage is most useful for correcting individual instances of bad data, or patterns, in your known data. Example scenarios: + +* Removing `(Album Version)` from all Titles +* Removing `Various Artists` from Artist data +* Correcting spelling mistakes for individual Artist names + +The user [Stage `type`](/configuration/transforms#stage) is `user`. + +## Configuration + +All [Stage Configuration](/configuration/transforms#configuring-stage) is done using [**search-and-replace** expressions](#search-and-replace-expression) inside individual [rules](#rules). Default configuration for all Rules can still be done using Stage `defaults`. + +
+ +Example + +```json5 title="config.json" +{ + // ... + "transformers": [ + { + "type": "user", + "name": "Normalizer", + "defaults": { + "title": [ + "(Album Version)" + ], + "artists": [ + "Various Artists" + ] + } + } + ] +} +``` + +
+ +### Search-And-Replace Expression + +A Search-And-Replace Expression can be a plain string that matches a literal, then removes it: + +``` +Expression: "badTerm" + +"this is badTerm cool string" => "this is a cool string" +``` + +or a regular expression that matches and removes the match: + +``` +Expression: "/bad\w+/i" + +"this is badSomething cool string" => "this is a cool string" +``` + +Or it may be an object that specifies what to match (using either plain string or regular expression) and what to replace it with: + +```json5 +{ + "search": "anotherBadTerm", // match all instances of 'anotherBadTerm' + "replace": "goodTerm" // replace with the string 'goodTerm' +} +``` + +``` +"this is anotherBadTerm cool string" => "this is goodTerm cool string" +``` + +```json5 +{ + "search": "/^\(\w+.com)/i", // matches any string that starts with EX '(YourMusic.com)' + "replace": "[MySite.com]" // replace with the string '[MySite.com]' +} +``` + +``` +"(Foo.com) this is a cool string" => "[MySite.com] this is a cool string" +``` + +The `replace` property uses javascript's [`replace()` function and so can use any special string characters.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_the_replacement) + +### Rules + +Each [Rule](/configuration/transforms#stage-rules) must be an array of [search-and-replace expressions](#search-and-replace-expressions): + +```json5 title="lastfm.json" +[ + { + "name": "myLastFm", + "configureAs": "source", + "data": { + // ... + }, + "options": { + "playTransform": { + "type": "user", + "preCompare": [ + { + "title": [ + // removes "badTerm" from title + "badTerm", + { + // removes "fooTerm" from title and replaces it with "barTerm" + "search": "fooTerm", + "replace": "barTerm" + } + ] + } + ], + } + } + } +] +``` + +:::note + +If the value of the field (title, an artist, album) is an empty string after transforming then the field is **removed.** + +::: + + +## Examples + +### Usage with `when` condition + +Using `when` for [Conditional Modification](/configuration/transforms#conditional-modification) +
+ +Example + +```json5 +// On search-replace in title... +// IF artist matches "Elephant Gym" +// THEN Run regex search-replace ELSE skip this rule +// +// Run live|remastered regex remove on title +{ + "title": [ + { + "search": "/\\s\\-\\s滾石40\\s滾石撞樂隊\\s40團拚經典(.+)$/i", + "replace": "", + "when": [ + { + "artist": "/Elephant Gym/" + } + ] + }, + "/(\\s\\-\\s|\\s)(feat\\.(.+)|live|remastered(.+))$/i" + ], +} +``` + +
+ +### Remove phrase from Title in all new Plays + +Removes the phrase `(Album Version)` from the Title of a Play + + +
+ +Example + +```json5 +{ + "title": [ + "(Album Version)" + ] +} + +``` +
+ +### Remove all parenthesized content from the end of a title + +
+ + Example +```json5 +{ +"compare": { + "candidate": { + "title": [ + "/(\(.+\))\s*$/" + ] + }, + "existing": { + "title": [ + "/(\(.+\))\s*$/" + ] + }, + }, +} +``` +
+ +### Rename misspelled artist in all new Plays + +
+ + Example +```json5 +{ + "artists": [ + { + "search": "Boz Skaggs", + "replace": "Boz Scaggs" + } + ] +} +``` +
+ +### Remove "Various Artists" albums in all new Plays + +
+ + Example +```json5 +{ + "album": [ + { + "search": "Various Artists", + "replace": "" + } + ] +} +``` +
+ +### Extract primary Artist from delimited, multi-Artist string + +
+ + When the Artist string is actually a multi-artist, delimited string, this search-and-replace will replace the string with just the first artist found. + + Ex + + ``` + My Artist One / My Artist Two / Another Guy + My Artist One + ``` + + Artists are delimited with a spaced forward slash (`/`) in the regex below. Replace the contents of the `delim` capture group with the delimiter for your use case. Some more common scenarios: + + * `(?\\/)` No spaces between slash IE `My Artist One/My Artist Two/Another Guy` + * `(?\\s*\\\\\s*)` Backslash instead of forward slash IE `My Artist One \ My Artist Two \ Another Guy` + * `(?,)` Comma IE `My Artist One, My Artist Two, Another Guy` + +
+ + Example +```json +{ + "artists": [ + { + "search": "(.*?)(?\\s*\\/\\s*)(.*$)", + "replace": "$1" + } + ] +} +``` +
+ +
\ No newline at end of file