From 883050e0b4cf383dfb5f38f4da2c199cffa57a29 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 26 Nov 2025 16:11:51 +0000 Subject: [PATCH] feat: Add native transformer options for delimiters and artist exceptions #364 --- src/backend/common/infrastructure/Atomic.ts | 1 + .../common/transforms/AbstractTransformer.ts | 15 ++-- .../transforms/AtomicPartsTransformer.ts | 6 +- .../common/transforms/NativeTransformer.ts | 86 ++++++++++++++++--- .../common/transforms/UserTransformer.ts | 5 +- src/backend/utils/DataUtils.ts | 6 ++ src/backend/utils/StringUtils.ts | 4 +- src/core/Atomic.ts | 8 +- 8 files changed, 101 insertions(+), 30 deletions(-) create mode 100644 src/backend/utils/DataUtils.ts diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index d43700a0..5fccaa28 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -267,6 +267,7 @@ export interface numberFormatOptions { } export const DELIMITERS = [',','&','/','\\']; +export const DELIMITERS_NO_AMP = [',','/','\\']; export const ARTIST_WEIGHT = 0.3; export const TITLE_WEIGHT = 0.4; diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index bba5c576..61991d7d 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -1,15 +1,14 @@ import { childLogger, Logger } from "@foxxmd/logging"; -import { PlayObject, TransformerCommonConfig } from "../../../core/Atomic.js"; +import { PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.js"; import { getRoot } from "../../ioc.js"; import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import AbstractInitializable from "../AbstractInitializable.js"; import { StageConfig } from "../infrastructure/Transform.js"; import { cacheFunctions } from "@foxxmd/regex-buddy-core"; -export interface TransformerCommon extends TransformerCommonConfig { - regexCache: ReturnType - name: string - logger: Logger +export interface TransformerOptions { + logger: Logger + regexCache: ReturnType } export default abstract class AbstractTransformer extends AbstractInitializable { @@ -20,11 +19,11 @@ export default abstract class AbstractTransformer extends AbstractIniti regexCache: ReturnType - public constructor(config: TransformerCommon) { + public constructor(config: TransformerCommon, options: TransformerOptions) { super(config); - this.logger = childLogger(config.logger, ['Transformer', this.config.type, this.config.name]); + this.logger = childLogger(options.logger, ['Transformer', this.config.type, this.config.name]); this.transformType = config.type; - this.regexCache = config.regexCache; + this.regexCache = options.regexCache; } public parseConfig(data: any) { diff --git a/src/backend/common/transforms/AtomicPartsTransformer.ts b/src/backend/common/transforms/AtomicPartsTransformer.ts index 3786f796..46a749c9 100644 --- a/src/backend/common/transforms/AtomicPartsTransformer.ts +++ b/src/backend/common/transforms/AtomicPartsTransformer.ts @@ -1,9 +1,5 @@ import { ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js"; -import { getRoot } from "../../ioc.js"; -import { testWhenConditions } from "../../utils/PlayTransformUtils.js"; -import AbstractInitializable from "../AbstractInitializable.js"; -import { AtomicStageConfig, ConditionalSearchAndReplaceRegExp, PlayTransformMetaParts, PlayTransformParts, PlayTransformUserParts } from "../infrastructure/Transform.js"; -import { cacheFunctions } from "@foxxmd/regex-buddy-core"; +import { AtomicStageConfig } from "../infrastructure/Transform.js"; import AbstractTransformer from "./AbstractTransformer.js"; export default abstract class AtomicPartsTransformer extends AbstractTransformer { diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts index b98e0267..c4d63b4b 100644 --- a/src/backend/common/transforms/NativeTransformer.ts +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -1,12 +1,69 @@ -import { PlayObject } from "../../../core/Atomic.js"; -import { isUserStage, isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js"; +import { PlayObject, TransformerCommon } from "../../../core/Atomic.js"; +import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; import { ExternalMetadataTerm, PlayTransformNativeStage, StageConfig } from "../infrastructure/Transform.js"; import AtomicPartsTransformer from "./AtomicPartsTransformer.js"; import { parseArtistCredits, parseTrackCredits, uniqueNormalizedStrArr } from "../../utils/StringUtils.js"; +import { parseRegexSingle, parseToRegexOrLiteralSearch } from "@foxxmd/regex-buddy-core"; +import { TransformerOptions } from "./AbstractTransformer.js"; +import { DELIMITERS_NO_AMP } from "../infrastructure/Atomic.js"; +import { asArray } from "../../utils/DataUtils.js"; + +export interface NativeTransformerData { + delimiters?: string[] + extraDelimiters?: string[] + ignoreArtists?: string[] +} + +export type NativeTransformerConfig = TransformerCommon; export default class NativeTransformer extends AtomicPartsTransformer { + declare config: NativeTransformerConfig; + + ignoreArtistsRegex: RegExp[] = []; + delimiters?: string[] + + // public constructor(config: NativeTransformerConfig, options: TransformerOptions) { + // super(config, options); + // } + + protected async doBuildInitData(): Promise { + if(this.config.data === undefined) { + return true; + } + if(this.config.data === null || typeof this.config.data !== 'object') { + throw new Error('Native Transformer data should be an object or not defined.'); + } + if(this.config.data.ignoreArtists !== undefined) { + this.config.data.ignoreArtists = asArray(this.config.data.ignoreArtists); + const nonStr = this.config.data.ignoreArtists.filter(x => typeof x !== 'string'); + if(nonStr.length > 0) { + throw new Error(`ignoreArtists must be an array of strings but non-strings found: ${nonStr.map(x => (x as unknown).toString()).join(' | ')}`) + } + for(const i of this.config.data.ignoreArtists) { + try { + this.ignoreArtistsRegex.push(parseToRegexOrLiteralSearch(i)); + } catch (e) { + throw new Error(`Could not convert ignoreArtist string to regex (or literal): ${i}`); + } + } + this.logger.debug(`Ignoring artists using ${this.config.data.ignoreArtists.length} rules`); + } + + if(this.config.data.delimiters !== undefined) { + this.delimiters = asArray(this.config.data.delimiters); + this.logger.debug(`Using user-defined delimiters '${this.config.data.extraDelimiters.join(' ')}' instead of built-ins`); + } else if(this.config.data.extraDelimiters !== undefined) { + this.delimiters = [...DELIMITERS_NO_AMP, ...(asArray(this.config.data.extraDelimiters))]; + this.logger.debug(`Using extra delimiters '${this.config.data.extraDelimiters.join(' ')}' with built-in delimiters '${DELIMITERS_NO_AMP.join(' ')}'`); + } + + if(this.delimiters !== undefined) { + this.delimiters.map(x => x.trim()); + } + } + protected doParseConfig(data: StageConfig) { if (data.type !== 'native') { throw new Error(`NativeTransformer is only usable with 'native' type stages`); @@ -37,17 +94,26 @@ export default class NativeTransformer extends AtomicPartsTransformer { let artists = []; if (play.data.artists.length === 1) { - const artistCredits = parseArtistCredits(play.data.artists[0]); - if (artistCredits !== undefined) { - if (artistCredits.primary !== undefined) { - artists.push(artistCredits.primary); - } - if (artistCredits.secondary !== undefined) { - artists = artists.concat(artistCredits.secondary); + const matchedIgnoreArtists = this.ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, play.data.artists[0])})).filter(x => x !== undefined); + if(matchedIgnoreArtists.length > 0) { + this.logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`) + } else { + const artistCredits = parseArtistCredits(play.data.artists[0], this.delimiters); + if (artistCredits !== undefined) { + if (artistCredits.primary !== undefined) { + artists.push(artistCredits.primary); + } + if (artistCredits.secondary !== undefined) { + artists = artists.concat(artistCredits.secondary); + } } } + if(artists.length === 0) { + artists.push(play.data.artists[0]); + } } - const trackArtists = parseTrackCredits(play.data.track); + + const trackArtists = parseTrackCredits(play.data.track, this.delimiters); if (trackArtists !== undefined && trackArtists.secondary !== undefined) { artists = artists.concat(trackArtists.secondary); } diff --git a/src/backend/common/transforms/UserTransformer.ts b/src/backend/common/transforms/UserTransformer.ts index 77713a76..e0a5ea8b 100644 --- a/src/backend/common/transforms/UserTransformer.ts +++ b/src/backend/common/transforms/UserTransformer.ts @@ -1,9 +1,8 @@ import { searchAndReplace } from "@foxxmd/regex-buddy-core"; import { PlayObject } from "../../../core/Atomic.js"; -import { configValToSearchReplace, isSearchAndReplaceTerm, isStageTyped, isUserStage, testWhenConditions } from "../../utils/PlayTransformUtils.js"; +import { configValToSearchReplace, isSearchAndReplaceTerm, isUserStage, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; -import { ConditionalSearchAndReplaceRegExp, PlayTransformUserParts, PlayTransformUserStage, StageConfig } from "../infrastructure/Transform.js"; -import AbstractTransformer, { TransformerCommon } from "./AbstractTransformer.js" +import { ConditionalSearchAndReplaceRegExp, PlayTransformUserStage, StageConfig } from "../infrastructure/Transform.js"; import AtomicPartsTransformer from "./AtomicPartsTransformer.js"; export default class UserTransformer extends AtomicPartsTransformer { diff --git a/src/backend/utils/DataUtils.ts b/src/backend/utils/DataUtils.ts new file mode 100644 index 00000000..7722466c --- /dev/null +++ b/src/backend/utils/DataUtils.ts @@ -0,0 +1,6 @@ +export const asArray = (data: T | T[]): T[] => { + if(Array.isArray(data)) { + return data; + } + return [data]; +} \ No newline at end of file diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts index 95002ab6..92c7f7e8 100644 --- a/src/backend/utils/StringUtils.ts +++ b/src/backend/utils/StringUtils.ts @@ -1,7 +1,7 @@ import { strategies, stringSameness, StringSamenessResult } from "@foxxmd/string-sameness"; import { hasher } from 'node-object-hash'; import { PlayObject } from "../../core/Atomic.js"; -import { asPlayerStateData, DELIMITERS, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; +import { asPlayerStateData, DELIMITERS, DELIMITERS_NO_AMP, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; import { genGroupIdStr, getPlatformIdFromData, intersect, parseRegexSingleOrFail } from "../utils.js"; import { buildTrackString } from "../../core/StringUtils.js"; @@ -183,7 +183,7 @@ export const parseStringList = (str: string, delimiters: string[] = DELIMITERS): return explodedStrings.flat(1); }, [str]).map(x => x.trim()); } -export const parseContextAwareStringList = (str: string, delimiters: string[] = [',', '/', '\\'], opts: {ignoreGlobalAmpersand?: boolean} = {}): string[] => { +export const parseContextAwareStringList = (str: string, delimiters: string[] = DELIMITERS_NO_AMP, opts: {ignoreGlobalAmpersand?: boolean} = {}): string[] => { if (delimiters.length === 0) { return [str]; } diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 7080f333..987a1431 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -390,8 +390,8 @@ export const JOINERS_FINAL: FinalJoiners[] = ['&']; export type Feat = 'ft' | 'feat' | 'vs' | 'ft.' | 'feat.' | 'vs.' | 'featuring' export const FEAT: Feat[] = ['ft','feat','vs','ft.','feat.','vs.','featuring']; -export interface TransformerCommonConfig { - data?: Record; +export interface TransformerCommonConfig> { + data?: T; type: string; name?: string; options?: { @@ -399,3 +399,7 @@ export interface TransformerCommonConfig { throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album')[]; }; } + +export interface TransformerCommon> extends TransformerCommonConfig { + name: string +} \ No newline at end of file -- 2.51.2