diff --git a/package-lock.json b/package-lock.json index e46f67bc..5ee06c9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -71,6 +71,7 @@ "lastfm-node-client": "^2.2.0", "mopidy": "^1.3.0", "mpd-api": "^1.1.2", + "musicbrainz-api": "^0.26.0", "nanoid": "^3.3.1", "node-object-hash": "^3.1.1", "normalize-url": "^8.0.1", @@ -787,6 +788,161 @@ "node": ">= 18" } }, + "node_modules/@biomejs/biome": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.9.4.tgz", + "integrity": "sha512-1rkd7G70+o9KkTn5KLmDYXihGoTaIGO9PIIN2ZB7UJxFrWw04CZHPYiMRjYsaDvVV7hP1dYNRLxSANLaBFGpog==", + "hasInstallScript": true, + "license": "MIT OR Apache-2.0", + "bin": { + "biome": "bin/biome" + }, + "engines": { + "node": ">=14.21.3" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/biome" + }, + "optionalDependencies": { + "@biomejs/cli-darwin-arm64": "1.9.4", + "@biomejs/cli-darwin-x64": "1.9.4", + "@biomejs/cli-linux-arm64": "1.9.4", + "@biomejs/cli-linux-arm64-musl": "1.9.4", + "@biomejs/cli-linux-x64": "1.9.4", + "@biomejs/cli-linux-x64-musl": "1.9.4", + "@biomejs/cli-win32-arm64": "1.9.4", + "@biomejs/cli-win32-x64": "1.9.4" + } + }, + "node_modules/@biomejs/cli-darwin-arm64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.9.4.tgz", + "integrity": "sha512-bFBsPWrNvkdKrNCYeAp+xo2HecOGPAy9WyNyB/jKnnedgzl4W4Hb9ZMzYNbf8dMCGmUdSavlYHiR01QaYR58cw==", + "cpu": [ + "arm64" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-darwin-x64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.9.4.tgz", + "integrity": "sha512-ngYBh/+bEedqkSevPVhLP4QfVPCpb+4BBe2p7Xs32dBgs7rh9nY2AIYUL6BgLw1JVXV8GlpKmb/hNiuIxfPfZg==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-arm64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.9.4.tgz", + "integrity": "sha512-fJIW0+LYujdjUgJJuwesP4EjIBl/N/TcOX3IvIHJQNsAqvV2CHIogsmA94BPG6jZATS4Hi+xv4SkBBQSt1N4/g==", + "cpu": [ + "arm64" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-arm64-musl": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.9.4.tgz", + "integrity": "sha512-v665Ct9WCRjGa8+kTr0CzApU0+XXtRgwmzIf1SeKSGAv+2scAlW6JR5PMFo6FzqqZ64Po79cKODKf3/AAmECqA==", + "cpu": [ + "arm64" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-x64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.9.4.tgz", + "integrity": "sha512-lRCJv/Vi3Vlwmbd6K+oQ0KhLHMAysN8lXoCI7XeHlxaajk06u7G+UsFSO01NAs5iYuWKmVZjmiOzJ0OJmGsMwg==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-x64-musl": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.9.4.tgz", + "integrity": "sha512-gEhi/jSBhZ2m6wjV530Yy8+fNqG8PAinM3oV7CyO+6c3CEh16Eizm21uHVsyVBEB6RIM8JHIl6AGYCv6Q6Q9Tg==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-win32-arm64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.9.4.tgz", + "integrity": "sha512-tlbhLk+WXZmgwoIKwHIHEBZUwxml7bRJgk0X2sPyNR3S93cdRq6XulAZRQJ17FYGGzWne0fgrXBKpl7l4M87Hg==", + "cpu": [ + "arm64" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-win32-x64": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.9.4.tgz", + "integrity": "sha512-8Y5wMhVIPaWe6jw2H+KlEm4wP/f7EW3810ZLmDlrEEy5KvBsb9ECEfu/kMWD484ijfQ8+nIi0giMgu9g1UAuuA==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.21.3" + } + }, "node_modules/@bufbuild/protobuf": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.2.1.tgz", @@ -6904,6 +7060,12 @@ "node": ">= 0.8" } }, + "node_modules/http-status-codes": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/http-status-codes/-/http-status-codes-2.3.0.tgz", + "integrity": "sha512-RJ8XvFvpPM/Dmc5SV+dC4y5PCeOhT3x1Hq0NU3rjGeg5a/CqlhZ7uudknPwZFz4aeAXDcbAyaeP7GAo9lvngtA==", + "license": "MIT" + }, "node_modules/https-proxy-agent": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", @@ -7922,6 +8084,15 @@ "node": "*" } }, + "node_modules/jsontoxml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/jsontoxml/-/jsontoxml-1.0.1.tgz", + "integrity": "sha512-dtKGq0K8EWQBRqcAaePSgKR4Hyjfsz/LkurHSV3Cxk4H+h2fWDeaN2jzABz+ZmOJylgXS7FGeWmbZ6jgYUMdJQ==", + "license": "MIT", + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/jwa": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", @@ -8830,6 +9001,38 @@ "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==", "license": "(Apache-2.0 AND MIT)" }, + "node_modules/musicbrainz-api": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/musicbrainz-api/-/musicbrainz-api-0.26.0.tgz", + "integrity": "sha512-HBHchcJlBnTNzgrj9ZvlIzp4rMs31WvMg3VdXQ7nKdP3acxC5yD6+Mis98ubzUFappUZFzTbjhKgLYWR9JI47g==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.4", + "http-status-codes": "^2.1.4", + "json-stringify-safe": "^5.0.1", + "jsontoxml": "^1.0.1", + "rate-limit-threshold": "^0.2.0", + "spark-md5": "^3.0.2", + "tough-cookie": "^5.0.0", + "uuid": "^13.0.0" + }, + "engines": { + "node": "^14.13.1 || >=16.0.0" + } + }, + "node_modules/musicbrainz-api/node_modules/uuid": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", + "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, "node_modules/mute-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", @@ -10257,6 +10460,18 @@ "node": ">= 0.6" } }, + "node_modules/rate-limit-threshold": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/rate-limit-threshold/-/rate-limit-threshold-0.2.0.tgz", + "integrity": "sha512-4vRdh7MTX3wJaZG2Xj4IF6Z5Epr9KzZsq/FpLYVuIcgdmXht4jJADDyuyUOiSwB/ZXdZKdtEeCaY+keXVsBWQQ==", + "license": "MIT", + "dependencies": { + "@biomejs/biome": "^1.8.3" + }, + "engines": { + "node": "^14.13.1 || >=16.0.0" + } + }, "node_modules/raw-body": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", @@ -11070,6 +11285,12 @@ "node": ">=0.10.0" } }, + "node_modules/spark-md5": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/spark-md5/-/spark-md5-3.0.2.tgz", + "integrity": "sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw==", + "license": "(WTFPL OR MIT)" + }, "node_modules/spawn-wrap": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/spawn-wrap/-/spawn-wrap-2.0.0.tgz", diff --git a/package.json b/package.json index a933596c..4bda7bd8 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ "lastfm-node-client": "^2.2.0", "mopidy": "^1.3.0", "mpd-api": "^1.1.2", + "musicbrainz-api": "^0.26.0", "nanoid": "^3.3.1", "node-object-hash": "^3.1.1", "normalize-url": "^8.0.1", diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 5fccaa28..dd7c9030 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -335,3 +335,8 @@ export interface CacheConfigOptions { regex?: number } +export interface MusicbrainzConfigData { + url?: string + rateLimit?: number + contact: string +} \ No newline at end of file diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts new file mode 100644 index 00000000..26a038eb --- /dev/null +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -0,0 +1,91 @@ +import dayjs from "dayjs"; +import request, { Request, Response } from 'superagent'; +import { PlayObject, URLData } from "../../../../core/Atomic.js"; +import { nonEmptyStringOrDefault } from "../../../../core/StringUtils.js"; +import { UpstreamError } from "../../errors/UpstreamError.js"; +import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, MusicbrainzConfigData } from "../../infrastructure/Atomic.js"; +import AbstractApiClient from "../AbstractApiClient.js"; +import { isPortReachableConnect, joinedUrl, normalizeWebAddress } from '../../../utils/NetworkUtils.js'; +import { MusicBrainzApi, IRecording, ISearchResult, IRecordingList, ISearchQuery } from 'musicbrainz-api'; +import { sleep } from "../../../utils.js"; + +export interface SubmitResponse { + payload?: { + ignored_listens: number + submitted_listens: number + }, + status: string +} + +export class MusicbrainzApiClient extends AbstractApiClient { + + declare config: MusicbrainzConfigData; + protected api: MusicBrainzApi; + protected url: URLData; + + constructor(name: any, config: MusicbrainzConfigData & { version: string }, options: AbstractApiOptions) { + super('Musicbrainz', name, config, options); + const { + url = 'https://musicbrainz.org', + } = config; + + this.url = normalizeWebAddress(url); + + this.api = new MusicBrainzApi({ + appName: 'multi-scrobbler', + appVersion: config.version, + appContactInfo: config.contact, + baseUrl: url + }); + } + + callApi = async (func: (mb: MusicBrainzApi) => Promise, options?: { timeout?: number }): Promise => { + const { + timeout = 15000 + } = options || {}; + + try { + const res = Promise.race([ + func(this.api), + sleep(timeout) + ]); + await res; + if (res === undefined) { + throw new Error('Timeout occurred while waiting for Musicbrainz API rate limit'); + } + return res as T; + } catch (e) { + throw new UpstreamError('Error occurred in Musicbrainz API', { cause: e }); + } + } + + searchByRecording = async(play: PlayObject): Promise => { + return await this.callApi((mb) => { + const query: Record = { + recording: play.data.track + }; + if(play.data.artists !== undefined && play.data.artists.length > 0) { + query.artist = play.data.artists[0]; + } + if(play.data.album !== undefined) { + query.release = play.data.album; + } + return mb.search('recording', { + query + }); + }); + } + + testConnection = async () => { + try { + await isPortReachableConnect(this.url.port, { host: this.url.url.hostname }); + } catch (e) { + throw new Error('Could not reach API URL endpoint', { cause: e }); + } + return true; + } + + static formatPlayObj(obj: any, options: FormatPlayObjectOptions): PlayObject { + return obj; + } +} \ No newline at end of file -- 2.51.2 From 2024454f5b65e66d0e97fb6127deb7d8159d3d2c Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 2 Dec 2025 18:32:50 +0000 Subject: [PATCH 02/41] feat(transformers): More improvement to musicbrainz * Implement multiple endpoints for API client with round robin load balancing * Implement caching responses * Add transformer options for search triggers * Implement transformer checks based on search triggers and score --- package-lock.json | 32 +++ package.json | 1 + src/backend/common/Cache.ts | 3 +- src/backend/common/errors/MSErrors.ts | 23 ++ src/backend/common/infrastructure/Atomic.ts | 9 +- .../common/infrastructure/Transform.ts | 4 +- .../common/transforms/AbstractTransformer.ts | 35 ++- .../transforms/AtomicPartsTransformer.ts | 20 +- .../transforms/MusicbrainzTransformer.ts | 256 ++++++++++++++++++ .../common/transforms/TransformerManager.ts | 14 +- .../musicbrainz/MusicbrainzApiClient.ts | 122 +++++++-- src/backend/ioc.ts | 18 +- src/backend/utils.ts | 28 +- src/core/Atomic.ts | 7 +- 14 files changed, 525 insertions(+), 47 deletions(-) create mode 100644 src/backend/common/transforms/MusicbrainzTransformer.ts diff --git a/package-lock.json b/package-lock.json index 5ee06c9d..e0524800 100644 --- a/package-lock.json +++ b/package-lock.json @@ -84,6 +84,7 @@ "react-dom": "^18.2.0", "react-redux": "^8.1.2", "react-router-dom": "^6.15.0", + "round-robin-js": "^3.0.10", "sass": "^1.93.3", "spotify-web-api-node": "^5.0.2", "superagent": "^8.0.9", @@ -1027,6 +1028,27 @@ "react-dom": "^16.3.2 || ^17.0.0 || ^18.0.0" } }, + "node_modules/@datastructures-js/heap": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/@datastructures-js/heap/-/heap-4.3.7.tgz", + "integrity": "sha512-Dx4un7Uj0dVxkfoq4RkpzsY2OrvNJgQYZ3n3UlGdl88RxxdHd7oTi21/l3zoxUUe0sXFuNUrfmWqlHzqnoN6Ug==", + "license": "MIT" + }, + "node_modules/@datastructures-js/linked-list": { + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/@datastructures-js/linked-list/-/linked-list-5.2.5.tgz", + "integrity": "sha512-YhlkDh7yMOaJ2bGgmrwvyF3vCrMELo2ERTiRxFMMkQF6+5n6expCBoseHtMIwe8fHJu1bzT7IaEHRU58hBoMUQ==", + "license": "MIT" + }, + "node_modules/@datastructures-js/priority-queue": { + "version": "6.3.5", + "resolved": "https://registry.npmjs.org/@datastructures-js/priority-queue/-/priority-queue-6.3.5.tgz", + "integrity": "sha512-NmdmvLnkUyG67W6Iy8tP+865ILq5VDyZh90c8lHMOkYYydUNzOV0QNPCRAuj4AoHSm/V2DeuYvDKnvALaCwLvA==", + "license": "MIT", + "dependencies": { + "@datastructures-js/heap": "^4.3.7" + } + }, "node_modules/@dbus-types/dbus": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/@dbus-types/dbus/-/dbus-0.0.4.tgz", @@ -10901,6 +10923,16 @@ "fsevents": "~2.3.2" } }, + "node_modules/round-robin-js": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/round-robin-js/-/round-robin-js-3.0.10.tgz", + "integrity": "sha512-CTUQfhMboLGtwNk9oLPFxX1GJZryXIJ8eyKfpGeYOUQmOxBOD4nie/jUKlKHfVMyi3v3gRtauaoORQKi8NJ+yA==", + "license": "MIT", + "dependencies": { + "@datastructures-js/linked-list": "^5.2.2", + "@datastructures-js/priority-queue": "^6.1.0" + } + }, "node_modules/rpc-websocket-client": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/rpc-websocket-client/-/rpc-websocket-client-1.1.4.tgz", diff --git a/package.json b/package.json index 4bda7bd8..ea1809c9 100644 --- a/package.json +++ b/package.json @@ -116,6 +116,7 @@ "react-dom": "^18.2.0", "react-redux": "^8.1.2", "react-router-dom": "^6.15.0", + "round-robin-js": "^3.0.10", "sass": "^1.93.3", "spotify-web-api-node": "^5.0.2", "superagent": "^8.0.9", diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index b8713a81..7440189d 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -99,8 +99,7 @@ export class MSCache { } init = async () => { - // disabled for now - //await this.initMetadataCache(); + await this.initMetadataCache(); await this.initScrobbleCache(); await this.initAuthCache(); //this.cacheTransform = await this.initCacheable({provider: false, memory: {lruSize: 500}}, 'transform'); diff --git a/src/backend/common/errors/MSErrors.ts b/src/backend/common/errors/MSErrors.ts index 5060f497..69e2a8c0 100644 --- a/src/backend/common/errors/MSErrors.ts +++ b/src/backend/common/errors/MSErrors.ts @@ -26,4 +26,27 @@ export class AuthCheckError extends StageError { export class PostInitError extends StageError { name = 'Post Initialization'; +} + +export class SimpleError extends Error implements HasSimpleError { + simple: boolean; + + public constructor(msg: string, options?: ErrorOptions & { simple?: boolean }) { + super(msg, options); + const { + simple = true + } = options || {}; + this.simple = simple; + } +} + +export interface HasSimpleError extends Error { + simple: boolean +} + +export const isSimpleError = (e: unknown): e is HasSimpleError => { + if(!(e instanceof Error)) { + return false; + } + return 'simple' in e; } \ No newline at end of file diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index dd7c9030..61780e9b 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -5,6 +5,7 @@ import { NextFunction, ParamsDictionary, Query } from "express-serve-static-core import { FixedSizeList } from 'fixed-size-list'; import { isPlayObject, PlayMeta, PlayObject } from "../../../core/Atomic.js"; import TupleMap from "../TupleMap.js"; +import { MusicBrainzApi } from 'musicbrainz-api'; export type SourceType = 'spotify' @@ -335,8 +336,12 @@ export interface CacheConfigOptions { regex?: number } -export interface MusicbrainzConfigData { +export interface MusicbrainzApiConfigData { url?: string rateLimit?: number contact: string -} \ No newline at end of file +} + +export const MUSICBRAINZ_URL = 'https://musicbrainz.org'; + +export type MusicBrainzSingletonMap = Map; \ No newline at end of file diff --git a/src/backend/common/infrastructure/Transform.ts b/src/backend/common/infrastructure/Transform.ts index 824da292..032d6ea6 100644 --- a/src/backend/common/infrastructure/Transform.ts +++ b/src/backend/common/infrastructure/Transform.ts @@ -21,11 +21,11 @@ export interface PlayTransformPartsAtomic { album?: T } -export type StageTypeMetadata = 'spotify' | 'listenbrainz' | 'native'; +export type StageTypeMetadata = 'spotify' | 'musicbrainz' | 'native'; export type StageTypeUser = 'user'; export type StageType = StageTypeMetadata | StageTypeUser | string; export const STAGE_TYPES_USER: StageTypeUser[] = ['user']; -export const STAGE_TYPES_METADATA: StageTypeMetadata[] = ['spotify','listenbrainz','native']; +export const STAGE_TYPES_METADATA: StageTypeMetadata[] = ['spotify','musicbrainz','native']; export const STAGE_TYPES: StageType[] = [...STAGE_TYPES_METADATA, ...STAGE_TYPES_USER]; export interface StageTyped { diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index d7a65d0c..f8cbf644 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -8,6 +8,8 @@ import { cacheFunctions, parseToRegexOrLiteralSearch, testMaybeRegex, searchAnd import { Cacheable } from "cacheable"; import { hashObject } from "../../utils/StringUtils.js"; import { playContentInvariantTransform } from "../../utils/PlayComparisonUtils.js"; +import e from "express"; +import { isSimpleError } from "../errors/MSErrors.js"; export interface TransformerOptions { logger: Logger @@ -52,10 +54,14 @@ export default abstract class AbstractTransformer { const cacheKey = `${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}` - const cachedTransform = await this.cache.get(cacheKey); - if(cachedTransform !== undefined) { - this.logger.debug('Cache hit'); - return cachedTransform; + try { + const cachedTransform = await this.cache.get(cacheKey); + if(cachedTransform !== undefined) { + this.logger.debug('Cache hit'); + return cachedTransform; + } + } catch (e) { + this.logger.warn(new Error('Could not fetch cache key', {cause: e})); } if (data.when !== undefined) { @@ -66,6 +72,17 @@ export default abstract class AbstractTransformer { return; } + + public async checkShouldTransformPreData(play: PlayObject, stageConfig: Y): Promise { + return + } } \ No newline at end of file diff --git a/src/backend/common/transforms/AtomicPartsTransformer.ts b/src/backend/common/transforms/AtomicPartsTransformer.ts index a7280af5..e9745391 100644 --- a/src/backend/common/transforms/AtomicPartsTransformer.ts +++ b/src/backend/common/transforms/AtomicPartsTransformer.ts @@ -74,12 +74,30 @@ export default abstract class AtomicPartsTransformer { + const clean = str.trim().toLocaleLowerCase(); + switch(clean) { + case 'track': + case 'title': + return 'title'; + case 'artist': + case 'artists': + return 'artists'; + case 'album': + return 'album'; + } + throw new Error(`MissingMbidType must be one of 'artist' or 'title' or 'album', given: ${clean}`); +} + +export interface MusicbrainzTransformerData { + searchWhenMissing?: MissingMbidType[] + forceSearch?: boolean + score?: number +} + +export interface MusicbrainzTransformerDataStrong extends MusicbrainzTransformerData { + searchWhenMissing: MissingMbidType[] + score: number +} + +export interface MusicbrainzTransformerDataStage extends MusicbrainzTransformerDataStrong,PlayTransformMetadataStage { +} + +export interface MusicbrainzTransformerDataConfig { + apis: MusicbrainzApiConfigData[] +} + +export type MusicbrainzBestMatch = {play: PlayObject, score: number}; + +export type MusicbrainzTransformerConfig = TransformerCommon; + +export const parseStageConfig = (data: MusicbrainzTransformerData | undefined = {}, logger: MaybeLogger = new MaybeLogger()): MusicbrainzTransformerDataStrong => { + + const config: MusicbrainzTransformerDataStrong = { + searchWhenMissing: DEFAULT_MISSING_TYPES, + score: 90 + }; + + if (data === null || typeof data !== 'object') { + throw new Error('Musicbrainz Transformer data should be an object or not defined.'); + } + + if(data.searchWhenMissing !== undefined) { + config.searchWhenMissing = data.searchWhenMissing.map(asMissingMbid); + } + + if(data.score !== undefined) { + config.score = data.score; + } + + logger.debug(`Will search if missing: ${config.searchWhenMissing.join(', ')} | Match if (default) score is >= ${config.score}`); + + return config; +} + +export default class MusicbrainzTransformer extends AtomicPartsTransformer { + + declare config: MusicbrainzTransformerConfig; + + protected defaults: MusicbrainzTransformerDataStrong; + + protected api: MusicbrainzApiClient; + + public constructor(config: MusicbrainzTransformerConfig, options: TransformerOptions) { + super(config, options); + } + + protected async doBuildInitData(): Promise { + this.defaults = parseStageConfig(this.config.defaults, childLogger(this.logger, 'Defaults')); + + const mbMap = getRoot().items.mbMap(); + const mbApis: Record = {}; + for(const mbConfig of this.config.data.apis) { + const u = normalizeWebAddress(mbConfig.url ?? MUSICBRAINZ_URL); + let mb = mbMap.get(u.url.hostname); + if(mb === undefined) { + const api = new MusicBrainzApi({ + appName: 'multi-scrobbler', + appVersion: version, + appContactInfo: mbConfig.contact, + baseUrl: u.url.toString() + }); + mbApis[u.url.hostname] = {api, ...mbConfig}; + mbMap.set(u.url.hostname, api); + mb = api; + } else if(mbApis[u.url.hostname] === undefined) { + mbApis[u.url.hostname] = {api: mb, ...mbConfig}; + } + } + + this.api = new MusicbrainzApiClient(this.config.name, {apis: Object.values(mbApis)}, { + logger: this.logger + }); + + return true; + } + + protected doParseConfig(data: MusicbrainzTransformerDataStage) { + if (data.type !== 'musicbrainz') { + throw new Error(`Musicbrainz Transformer is only usable with 'musicbrinz' type stages`); + } + + const stage: MusicbrainzTransformerDataStage = { + ...data, + ...parseStageConfig(data), + type: 'musicbrainz' + } + + for (const k of ['artists', 'title', 'album']) { + if (!(k in stage)) { + stage[k] = true; + continue; + } + if (Array.isArray(stage[k])) { + throw new Error(`${k} must be a boolean or when object`); + } + if (typeof stage[k] === 'boolean') { + continue; + } + if (typeof stage[k] === 'object' && !isWhenCondition(stage[k])) { + throw new Error(`${k} is not a valid when object`); + } + } + return stage; + } + + public async checkShouldTransformPreData(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise { + const { + searchWhenMissing = this.defaults.searchWhenMissing, + forceSearch = false + } = stageConfig; + + const missing = missingMbidTypes(play); + if(intersect(searchWhenMissing, missing).length > 0) { + this.logger.debug(`Missing desired MBIDs for ${missing.join(', ')}`); + } else if(forceSearch) { + this.logger.debug('No desired MBIDs are missing but forceSearch = true'); + } else { + throw new SimpleError('No desired MBIDs are missing'); + } + } + + public async getTransformerData(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise { + + // TODO maybe search more broadly if first query doesn't hit? + const results = await this.api.searchByRecording(play); + + if(results === undefined || results.recordings.length === 0) { + return undefined; + } + + return { play: recordingToPlay(results.recordings[0]), score: results.recordings[0].score }; + } + + public async checkShouldTransform(play: PlayObject, transformData: MusicbrainzBestMatch | undefined, stageConfig: MusicbrainzTransformerDataStage): Promise { + if(transformData === undefined) { + throw new SimpleError('No match returned from Musicbrainz API'); + } + + const { + score = this.defaults.score + } = stageConfig; + + if(transformData.score < score) { + this.logger.debug({bestMatch: transformData.play}, 'Best Match'); + throw new SimpleError(`Musicbrainz best match score of ${transformData.score} was less than minimum score of ${stageConfig.score}`); + } + } + + protected async handleTitle(play: PlayObject, parts: ExternalMetadataTerm, transformData: MusicbrainzBestMatch): Promise { + if (parts === false) { + return play.data.track; + } + if (typeof parts === 'object') { + if (parts.when !== undefined) { + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { + this.logger.debug('When condition for track not met, returning original track'); + return play.data.track; + } + } + } + + return transformData.play.data.track; + } + protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: MusicbrainzBestMatch): Promise { + if (parts === false) { + return play.data.artists; + } + if (typeof parts === 'object') { + if (parts.when !== undefined) { + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { + this.logger.debug('When condition for artists not met, returning original artists'); + return play.data.artists; + } + } + } + + return transformData.play.data.artists; + } + protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: MusicbrainzBestMatch): Promise { + // TODO + return play.data.albumArtists; + } + protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, transformData: MusicbrainzBestMatch): Promise { + if (parts === false) { + return play.data.album; + } + if (typeof parts === 'object') { + if (parts.when !== undefined) { + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { + this.logger.debug('When condition for album not met, returning original album'); + return play.data.album; + } + } + } + + return transformData.play.data.album; + } + + protected async handleMeta(play: PlayObject, transformData: MusicbrainzBestMatch): Promise { + return transformData.play.data.meta; + } + + public notify(payload: WebhookPayload): Promise { + return; + } + protected getIdentifier(): string { + return 'Musicbrainz Transformer'; + } + +} \ No newline at end of file diff --git a/src/backend/common/transforms/TransformerManager.ts b/src/backend/common/transforms/TransformerManager.ts index 4623d602..6055f24b 100644 --- a/src/backend/common/transforms/TransformerManager.ts +++ b/src/backend/common/transforms/TransformerManager.ts @@ -7,6 +7,11 @@ import { PlayObject } from "../../../core/Atomic.js"; import { isStageTyped } from "../../utils/PlayTransformUtils.js"; import { MSCache } from "../Cache.js"; import NativeTransformer from "./NativeTransformer.js"; +import { MusicbrainzApiClient } from "../vendor/musicbrainz/MusicbrainzApiClient.js"; +import { MUSICBRAINZ_URL, MusicbrainzApiConfigData } from "../infrastructure/Atomic.js"; +import { normalizeWebAddress } from "../../utils/NetworkUtils.js"; +import { MusicBrainzApi } from "musicbrainz-api"; +import MusicbrainzTransformer, { MusicbrainzTransformerConfig } from "./MusicbrainzTransformer.js"; export default class TransformerManager { @@ -44,6 +49,9 @@ export default class TransformerManager { case 'native': t = new NativeTransformer({ name: tName, ...config }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); break; + case 'musicbrainz': + t = new MusicbrainzTransformer({ name: tName, ...config as MusicbrainzTransformerConfig }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); + break; default: throw new Error(`No transformer of type '${config.type}' exists.`); } @@ -110,6 +118,10 @@ export default class TransformerManager { t = list[0]; } - return await t.handle(data, play); + try { + return await t.handle(data, play); + } catch (e) { + throw new Error('Stage processing failed', {cause: e}); + } } } \ No newline at end of file diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index 26a038eb..6459f022 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -1,13 +1,18 @@ import dayjs from "dayjs"; import request, { Request, Response } from 'superagent'; import { PlayObject, URLData } from "../../../../core/Atomic.js"; -import { nonEmptyStringOrDefault } from "../../../../core/StringUtils.js"; +import { buildTrackString, nonEmptyStringOrDefault } from "../../../../core/StringUtils.js"; import { UpstreamError } from "../../errors/UpstreamError.js"; -import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, MusicbrainzConfigData } from "../../infrastructure/Atomic.js"; +import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, MUSICBRAINZ_URL, MusicbrainzApiConfigData } from "../../infrastructure/Atomic.js"; import AbstractApiClient from "../AbstractApiClient.js"; import { isPortReachableConnect, joinedUrl, normalizeWebAddress } from '../../../utils/NetworkUtils.js'; -import { MusicBrainzApi, IRecording, ISearchResult, IRecordingList, ISearchQuery } from 'musicbrainz-api'; +import { MusicBrainzApi, IRecording, ISearchResult, IRecordingList, ISearchQuery, IRecordingMatch, IRelease } from 'musicbrainz-api'; import { sleep } from "../../../utils.js"; +import {SequentialRoundRobin} from 'round-robin-js'; +import { Cacheable } from "cacheable"; +import { getRoot } from "../../../ioc.js"; +import { hashObject } from "../../../utils/StringUtils.js"; +import { playContentInvariantTransform } from "../../../utils/PlayComparisonUtils.js"; export interface SubmitResponse { payload?: { @@ -17,50 +22,70 @@ export interface SubmitResponse { status: string } +export interface MusicbrainzApiConfig extends MusicbrainzApiConfigData { + api: MusicBrainzApi +} + +export interface MusicbrainzApiClientConfig { + apis: MusicbrainzApiConfig[] +} + export class MusicbrainzApiClient extends AbstractApiClient { - declare config: MusicbrainzConfigData; - protected api: MusicBrainzApi; + declare config: MusicbrainzApiClientConfig; + protected apis: MusicBrainzApi[]; + protected rrApis: SequentialRoundRobin; protected url: URLData; + cache: Cacheable; - constructor(name: any, config: MusicbrainzConfigData & { version: string }, options: AbstractApiOptions) { + constructor(name: any, config: MusicbrainzApiClientConfig, options: AbstractApiOptions & {cache?: Cacheable}) { super('Musicbrainz', name, config, options); - const { - url = 'https://musicbrainz.org', - } = config; - - this.url = normalizeWebAddress(url); + this.apis = config.apis.map(x => x.api); + this.rrApis = new SequentialRoundRobin(this.apis); + this.cache = options.cache ?? getRoot().items.cache().cacheMetadata; - this.api = new MusicBrainzApi({ - appName: 'multi-scrobbler', - appVersion: config.version, - appContactInfo: config.contact, - baseUrl: url - }); + this.logger.debug(`Round Robin API calls using hosts: ${config.apis.map(x => x.url ?? MUSICBRAINZ_URL).join(' | ')}`); } callApi = async (func: (mb: MusicBrainzApi) => Promise, options?: { timeout?: number }): Promise => { const { - timeout = 15000 + timeout = 30000 } = options || {}; try { - const res = Promise.race([ - func(this.api), + const res = await Promise.race([ + func(this.rrApis.next().value), sleep(timeout) ]); - await res; if (res === undefined) { throw new Error('Timeout occurred while waiting for Musicbrainz API rate limit'); } return res as T; } catch (e) { + if(e.message.includes('Timeout occurred')) { + throw e; + } throw new UpstreamError('Error occurred in Musicbrainz API', { cause: e }); } } - searchByRecording = async(play: PlayObject): Promise => { - return await this.callApi((mb) => { + searchByRecording = async(play: PlayObject): Promise => { + + const cacheKey = `mb-recSearch-${hashObject(playContentInvariantTransform(play))}`; + + try { + const cachedTransform = await this.cache.get(cacheKey); + if(cachedTransform !== undefined) { + this.logger.debug('Cache hit'); + return cachedTransform; + } + } catch (e) { + this.logger.warn(new Error('Could not fetch cache key', {cause: e})); + } + + + this.logger.debug(`Starting search for ${buildTrackString(play)}`); + const res = await this.callApi((mb) => { const query: Record = { recording: play.data.track }; @@ -74,18 +99,57 @@ export class MusicbrainzApiClient extends AbstractApiClient { query }); }); + + await this.cache.set(cacheKey, res, '1hr'); + return res; } testConnection = async () => { - try { - await isPortReachableConnect(this.url.port, { host: this.url.url.hostname }); - } catch (e) { - throw new Error('Could not reach API URL endpoint', { cause: e }); + for(const a of this.config.apis) { + try { + const u = normalizeWebAddress(a.url); + await isPortReachableConnect(u.port, { host: u.url.hostname }); + } catch (e) { + throw new Error('Could not reach API URL endpoint', { cause: e }); + } + return true; } - return true; } static formatPlayObj(obj: any, options: FormatPlayObjectOptions): PlayObject { - return obj; + return recordingToPlay(obj); + } +} + +export const recordingToPlay = (data: IRecording): PlayObject => { + + let album: IRelease; + // try to find official album first + album = data.releases.find(x => x.status === 'Official'); + if(album === undefined) { + // find the first album that isn't "bad" + album = data.releases.find(x => !['Expunged','Withdrawn'].includes(x.status)); } + + const play: PlayObject = { + data: { + // TODO figure out album artists + track: data.title, + artists: (data["artist-credit"] ?? []).map(x => x.name), + album: album !== undefined ? album.title : undefined, + meta: { + brainz: { + track: data.id, + artist: data["artist-credit"].map(x => x.artist.id), + album: album !== undefined ? album.id : undefined, + } + } + }, + meta: { + source: 'musicbrainz', + trackId: data.id + } + } + + return play; } \ No newline at end of file diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index dc24d04a..b91e6369 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -8,7 +8,7 @@ import { WildcardEmitter } from "./common/WildcardEmitter.js"; import { generateBaseURL } from "./utils/NetworkUtils.js"; import { PassThrough } from "stream"; -import { CacheConfigOptions } from "./common/infrastructure/Atomic.js"; +import { CacheConfigOptions, MusicBrainzSingletonMap } from "./common/infrastructure/Atomic.js"; import { MSCache } from "./common/Cache.js"; import TransformerManager from "./common/transforms/TransformerManager.js"; import { TransformerCommonConfig } from "../core/Atomic.js"; @@ -17,6 +17,7 @@ export let version: string = 'unknown'; export const parseVersion = async () => { version = await getVersion({priority: ['env', 'git', 'file']}); + return version; } let root: ReturnType; @@ -29,6 +30,7 @@ export interface RootOptions { loggerStream?: PassThrough loggingConfig?: LogOptions cache?: CacheConfigOptions | MSCache | (() => MSCache) + mbMap?: MusicBrainzSingletonMap | (() => MusicBrainzSingletonMap) transformers?: TransformerCommonConfig[] } @@ -41,6 +43,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { loggingConfig, logger, cache, + mbMap, transformers = [], } = options || {}; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); @@ -60,6 +63,16 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { maybeSingletonCache = new MSCache(logger, cache); } + let mbFunc: () => MusicBrainzSingletonMap; + let maybeSingletonMb: MusicBrainzSingletonMap; + if(typeof mbMap === 'function') { + mbFunc = mbMap; + } else if(maybeSingletonMb !== undefined) { + maybeSingletonMb = mbMap; + } else { + maybeSingletonMb = new Map(); + } + const cEmitter = new WildcardEmitter(); // do nothing, just catch @@ -103,7 +116,8 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { loggingConfig, logger: logger, transformerManager, - cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc + cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc, + mbMap: () => maybeSingletonMb !== undefined ? () => maybeSingletonMb : mbFunc }).add((items) => { const localUrl = generateBaseURL(baseUrl, items.port) return { diff --git a/src/backend/utils.ts b/src/backend/utils.ts index 646ea1bf..a7b884a9 100644 --- a/src/backend/utils.ts +++ b/src/backend/utils.ts @@ -9,7 +9,7 @@ import JSON5 from 'json5'; // https://github.com/jfromaniello/url-join#in-nodejs import pathUtil from "path"; import { TimeoutError, WebapiError } from "spotify-web-api-node/src/response-error.js"; -import { PlayObject } from "../core/Atomic.js"; +import { DEFAULT_MISSING_TYPES, MissingMbidType, PlayObject } from "../core/Atomic.js"; import { asPlayerStateDataMaybePlay, NO_DEVICE, @@ -611,6 +611,32 @@ export const comparingMultipleArtists = (existing: PlayObject, candidate: PlayOb return eArtists.length > 1 || cArtists.length > 1; } +export const missingMbidTypes = (play: PlayObject): MissingMbidType[] => { + let missing: MissingMbidType[] = []; + + if(play.data.meta?.brainz === undefined) { + return DEFAULT_MISSING_TYPES; + } + + const { + track, + album, + artist + } = play.data.meta.brainz; + + if(track === undefined) { + missing.push('title'); + } + if(album === undefined) { + missing.push('album'); + } + if((artist ?? []).length !== (play.data.artists ?? []).length) { + missing.push('artists') + } + return missing; +} + + export interface NonEmptyOptions { ofType?: string, test?: (val: T) => boolean diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 936da1e3..7d0c54ca 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -401,6 +401,9 @@ export interface TransformerCommonConfig, Y = Record> extends TransformerCommonConfig { +export interface TransformerCommon, Y = Record> extends TransformerCommonConfig { name: string -} \ No newline at end of file +} + +export type MissingMbidType = 'artists' | 'title' | 'album'; +export const DEFAULT_MISSING_TYPES: MissingMbidType[] = ['artists','title','album']; \ No newline at end of file -- 2.51.2 From db34c33c42cbf0395007f08c9b3c9f8880a51203 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 2 Dec 2025 18:59:44 +0000 Subject: [PATCH 03/41] Fix name usage --- .../common/transforms/AbstractTransformer.ts | 3 +++ .../common/transforms/TransformerManager.ts | 14 +++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index f8cbf644..82f422f2 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -33,8 +33,11 @@ export default abstract class AbstractTransformer 0 && (data as any).name === undefined) { - this.logger.warn(`More than one '${data.type}' transformer but name was not specified, using first registered`); - t = list[0]; + if (list.length > 1) { + if((data as any).name === undefined) { + this.logger.warn(`More than one '${data.type}' transformer but name was not specified, using first registered`); + t = list[0]; + } else { + const named = list.find(x => x.name === (data as any).name); + if(named === undefined) { + throw new Error(`No ${data.type} transformer with name '${(data as any).name}'`) + } + t = named; + } } else { t = list[0]; } -- 2.51.2 From d6d0ce515f4d3f2e0f4a43670baa423db0d194e6 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 2 Dec 2025 18:59:50 +0000 Subject: [PATCH 04/41] fix score usage --- src/backend/common/transforms/MusicbrainzTransformer.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index e9b733bb..f041e98c 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -40,7 +40,6 @@ export interface MusicbrainzTransformerData { export interface MusicbrainzTransformerDataStrong extends MusicbrainzTransformerData { searchWhenMissing: MissingMbidType[] - score: number } export interface MusicbrainzTransformerDataStage extends MusicbrainzTransformerDataStrong,PlayTransformMetadataStage { @@ -183,7 +182,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer Date: Tue, 2 Dec 2025 21:20:34 +0000 Subject: [PATCH 05/41] feat(transformer): Implement staggered api calls to respect rate limit --- package-lock.json | 48 +++++++++++++++---- package.json | 1 + .../transforms/MusicbrainzTransformer.ts | 8 ++-- .../musicbrainz/MusicbrainzApiClient.ts | 3 ++ .../scrobblers/AbstractScrobbleClient.ts | 2 +- src/backend/sources/AbstractSource.ts | 12 +++-- src/backend/utils/AsyncUtils.ts | 24 ++++++++++ 7 files changed, 80 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index e0524800..c352121b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -77,6 +77,7 @@ "normalize-url": "^8.0.1", "ntfy": "^1.7.6", "p-event": "^4.2.0", + "p-map": "^7.0.4", "passport": "^0.6.0", "passport-deezer": "^0.2.0", "patch-package": "^8.0.0", @@ -4145,6 +4146,7 @@ "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "clean-stack": "^2.0.0", @@ -4928,6 +4930,7 @@ "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", "dev": true, + "license": "MIT", "peer": true, "engines": { "node": ">=6" @@ -7751,6 +7754,20 @@ "node": ">=8" } }, + "node_modules/istanbul-lib-processinfo/node_modules/p-map": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", + "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/istanbul-lib-report": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", @@ -9403,6 +9420,20 @@ "node": ">=8" } }, + "node_modules/nyc/node_modules/p-map": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", + "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/nyc/node_modules/resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", @@ -9702,16 +9733,15 @@ } }, "node_modules/p-map": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", - "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", - "dev": true, - "peer": true, - "dependencies": { - "aggregate-error": "^3.0.0" - }, + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.4.tgz", + "integrity": "sha512-tkAQEw8ysMzmkhgw8k+1U/iPhWNhykKnSk4Rd5zLoPJCuJaGRPo6YposrZgaxHKzDHdDWWZvE/Sk7hsL2X/CpQ==", + "license": "MIT", "engines": { - "node": ">=8" + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-retry": { diff --git a/package.json b/package.json index ea1809c9..28d47425 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,7 @@ "normalize-url": "^8.0.1", "ntfy": "^1.7.6", "p-event": "^4.2.0", + "p-map": "^7.0.4", "passport": "^0.6.0", "passport-deezer": "^0.2.0", "patch-package": "^8.0.0", diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index f041e98c..d885f58f 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -16,6 +16,7 @@ import { getRoot, version } from "../../ioc.js"; import { normalizeWebAddress } from "../../utils/NetworkUtils.js"; import { intersect, missingMbidTypes } from "../../utils.js"; import { isSimpleError, SimpleError } from "../errors/MSErrors.js"; +import { buildTrackString } from "../../../core/StringUtils.js"; export const asMissingMbid = (str: string): MissingMbidType => { const clean = str.trim().toLocaleLowerCase(); @@ -156,9 +157,9 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer 0) { - this.logger.debug(`Missing desired MBIDs for ${missing.join(', ')}`); + this.logger.debug(`${buildTrackString(play)} - Missing desired MBIDs for ${missing.join(', ')}`); } else if(forceSearch) { - this.logger.debug('No desired MBIDs are missing but forceSearch = true'); + this.logger.debug(`${buildTrackString(play)} - No desired MBIDs are missing but forceSearch = true`); } else { throw new SimpleError('No desired MBIDs are missing'); } @@ -168,8 +169,9 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer; type NowPlayingQueue = Map; diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index 7fc3cc28..2b16730e 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -40,7 +40,9 @@ import { componentFileLogger } from '../common/logging.js'; import { WebhookPayload } from '../common/infrastructure/config/health/webhooks.js'; import { messageWithCauses, messageWithCausesTruncatedDefault } from '../utils/ErrorUtils.js'; import { genericSourcePlayMatch } from '../utils/PlayComparisonUtils.js'; -import { findAsync } from '../utils/AsyncUtils.js'; +import { findAsync, staggerMapper } from '../utils/AsyncUtils.js'; +import pMap, {pMapIterable} from 'p-map'; +import { randomInt } from 'crypto'; export interface RecentlyPlayedOptions { limit?: number @@ -198,9 +200,8 @@ export default abstract class AbstractSource extends AbstractComponent implement discover = async (plays: PlayObject[], options: { checkAll?: boolean, [key: string]: any } = {}): Promise => { const newDiscoveredPlays: PlayObject[] = []; - const transformedPlayed = await Promise.all(plays.map(x => this.transformPlay(x, TRANSFORM_HOOK.preCompare))); - - for(const play of transformedPlayed) { + const sm = staggerMapper({concurrency: 2}); + for await(const play of pMapIterable(plays, sm(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 2})) { if(!(await this.alreadyDiscovered(play, options))) { this.addPlayToDiscovered(play); newDiscoveredPlays.push(play); @@ -231,8 +232,9 @@ export default abstract class AbstractSource extends AbstractComponent implement return; } newDiscoveredPlays.sort(sortByOldestPlayDate); + const sm = staggerMapper({concurrency: 2}); this.emitter.emit('discoveredToScrobble', { - data: await Promise.all(newDiscoveredPlays.map(x => this.transformPlay(x, TRANSFORM_HOOK.postCompare))), + data: await pMap(newDiscoveredPlays, sm(async (x) => await this.transformPlay(x, TRANSFORM_HOOK.postCompare)), {concurrency: 2}), // Promise.all(newDiscoveredPlays.map(x => this.transformPlay(x, TRANSFORM_HOOK.postCompare))), options: { ...options, checkTime: newDiscoveredPlays[newDiscoveredPlays.length-1].data.playDate.add(2, 'second'), diff --git a/src/backend/utils/AsyncUtils.ts b/src/backend/utils/AsyncUtils.ts index af3cf930..1ac79c6b 100644 --- a/src/backend/utils/AsyncUtils.ts +++ b/src/backend/utils/AsyncUtils.ts @@ -1,3 +1,7 @@ +import { randomInt } from "crypto"; +import pMap, { Mapper, Options, pMapIterable } from "p-map"; +import { sleep } from "../utils.js"; + /** https://stackoverflow.com/a/63795192/1469797 */ export async function findAsyncSequential( array: T[], @@ -42,4 +46,24 @@ export async function findIndexAsync( const results = await Promise.all(promises); const index = results.findIndex(result => result); return index; +} + +export function staggerMapper(options: { maxRandomStagger?: number, initialInterval?: number, concurrency: number }) { + const { + initialInterval = 300, + maxRandomStagger = 300, + concurrency + } = options; + let initialStagger = 0; + + return (mapper: Mapper) => async (x: Element, index: number) => { + if (index < concurrency) { + sleep(initialStagger); + initialStagger += initialInterval; + } else { + const s = randomInt(maxRandomStagger) + await sleep(s); + } + return await mapper(x, index); + } } \ No newline at end of file -- 2.51.2 From d217c91a306fdc5f84bd65a828e1e609b4adb88c Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 3 Dec 2025 14:49:49 +0000 Subject: [PATCH 06/41] Add musicbrainz-api patch for rate limit --- patches/musicbrainz-api+0.26.0.patch | 26 +++++++++++++++++++ .../transforms/MusicbrainzTransformer.ts | 3 ++- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 patches/musicbrainz-api+0.26.0.patch diff --git a/patches/musicbrainz-api+0.26.0.patch b/patches/musicbrainz-api+0.26.0.patch new file mode 100644 index 00000000..07255284 --- /dev/null +++ b/patches/musicbrainz-api+0.26.0.patch @@ -0,0 +1,26 @@ +diff --git a/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts b/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts +index dfd2937..09ed531 100644 +--- a/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts ++++ b/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts +@@ -64,6 +64,7 @@ export interface IMusicBrainzConfig { + */ + appContactInfo?: string; + disableRateLimiting?: boolean; ++ rateLimit?: [number, number] + } + interface IInternalConfig extends IMusicBrainzConfig { + baseUrl: string; +diff --git a/node_modules/musicbrainz-api/lib/musicbrainz-api.js b/node_modules/musicbrainz-api/lib/musicbrainz-api.js +index 299be1c..7ee41b0 100644 +--- a/node_modules/musicbrainz-api/lib/musicbrainz-api.js ++++ b/node_modules/musicbrainz-api/lib/musicbrainz-api.js +@@ -36,7 +36,8 @@ export class MusicBrainzApi { + ..._config + }; + this.httpClient = this.initHttpClient(); +- this.rateLimiter = new RateLimitThreshold(15, 18); ++ const limits = this.config.rateLimit ?? [15,18]; ++ this.rateLimiter = new RateLimitThreshold(limits[0], limits[1]); + } + initHttpClient() { + return new HttpClient({ diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index d885f58f..341b74cf 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -103,7 +103,8 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer Date: Wed, 3 Dec 2025 17:04:42 +0000 Subject: [PATCH 07/41] recfactor(transform): Use async hooks for logging to identify transform Reduces logging noise --- package-lock.json | 8 +++--- package.json | 2 +- src/backend/common/AbstractComponent.ts | 20 +++++++------- .../common/transforms/AbstractTransformer.ts | 26 +++++++++--------- .../transforms/MusicbrainzTransformer.ts | 27 +++++++++---------- .../common/transforms/NativeTransformer.ts | 3 --- .../common/transforms/TransformerManager.ts | 22 ++++++++------- .../common/transforms/UserTransformer.ts | 3 --- .../musicbrainz/MusicbrainzApiClient.ts | 2 +- src/backend/tests/ytm/ytm.test.ts | 2 +- 10 files changed, 57 insertions(+), 58 deletions(-) diff --git a/package-lock.json b/package-lock.json index c352121b..36262411 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "@fortawesome/react-fontawesome": "^0.2.0", "@foxxmd/chromecast-client": "^1.0.4", "@foxxmd/get-version": "^0.0.3", - "@foxxmd/logging": "^0.2.3", + "@foxxmd/logging": "^0.2.4", "@foxxmd/redact-string": "^0.1.2", "@foxxmd/regex-buddy-core": "^0.1.2", "@foxxmd/string-sameness": "^0.4.0", @@ -1634,9 +1634,9 @@ } }, "node_modules/@foxxmd/logging": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@foxxmd/logging/-/logging-0.2.3.tgz", - "integrity": "sha512-KMqWy42niMLMyz8YmQ09XVay5szhU2Y8lWzM3Fbm+zyfFRTiaYCSal5tQSeWmVdjps6LqtQNGDH4KG5a3Wgnrw==", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@foxxmd/logging/-/logging-0.2.4.tgz", + "integrity": "sha512-pBqfs7OVtRtdmqyfF7eIRMGTpv8aOQwss7ROBt5HdF+0dnmmyz9HMcc9FHKSV2WzhV5K55+DTzPfrwJlFedFbg==", "license": "MIT", "dependencies": { "pino": "^9.2.0", diff --git a/package.json b/package.json index 28d47425..613c56aa 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "@fortawesome/react-fontawesome": "^0.2.0", "@foxxmd/chromecast-client": "^1.0.4", "@foxxmd/get-version": "^0.0.3", - "@foxxmd/logging": "^0.2.3", + "@foxxmd/logging": "^0.2.4", "@foxxmd/redact-string": "^0.1.2", "@foxxmd/regex-buddy-core": "^0.1.2", "@foxxmd/string-sameness": "^0.4.0", diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index f950b841..f9733e05 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -19,6 +19,7 @@ import AbstractInitializable from "./AbstractInitializable.js"; import play = Simulate.play; import TransformerManager from "./transforms/TransformerManager.js"; import { getRoot } from "../ioc.js"; +import { nanoid } from "nanoid"; export default abstract class AbstractComponent extends AbstractInitializable { @@ -154,9 +155,9 @@ export default abstract class AbstractComponent extends AbstractInitializable { public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean) => { - let logger: Logger; - const labels = ['Play Transform', hookType]; - const getLogger = () => logger !== undefined ? logger : childLogger(this.logger, labels); + + const asyncId = nanoid(6); + let logger = childLogger(this.logger, ['Play Transform', hookType, asyncId]); try { let hook: StageConfig[]; @@ -180,6 +181,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { return play; } + logger.debug(`Transform start for => ${buildTrackString(play)}`); let transformedPlay: PlayObject = play; let transformDetails: string[] = []; for(const hookItem of hook) { @@ -193,16 +195,16 @@ export default abstract class AbstractComponent extends AbstractInitializable { let newTransformedPlay: PlayObject; let err: Error; try { - newTransformedPlay = await this.transformManager.handleStage(hookItem, transformedPlay); + newTransformedPlay = await this.transformManager.handleStage(hookItem, transformedPlay, asyncId); } catch (e) { err = e; } if(err !== undefined) { if(onFailure === 'continue') { - this.logger.warn(new Error('A transform encountered an error but continuing due to onFailure: continue', {cause: err})); + logger.warn(new Error(`A transform encountered an error but continuing due to onFailure: continue`, {cause: err})); } else { - this.logger.error(new Error('Transform encountered an error', {cause: err})); + logger.error(new Error(`Transform encountered an error`, {cause: err})); if(!failureReturnPartial) { // rewind to original play so we don't return partial transform transformedPlay = play; @@ -218,7 +220,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { transformedPlay = newTransformedPlay; if(err === undefined && onSuccess === 'stop') { - this.logger.debug('Stopping transform due to onSuccess: stop'); + logger.debug(`${nanoid} Stopping transform due to onSuccess: stop`); break; } } @@ -232,12 +234,12 @@ export default abstract class AbstractComponent extends AbstractInitializable { } else { transformStatements.push(`=> ${transformDetails[transformDetails.length - 1]}`); } - this.logger.debug({labels: [...labels, hookType]}, `Transform Pipeline:\n${transformStatements.join('\n')}`); + logger.debug({labels: [hookType]}, `Transform Pipeline:\n${transformStatements.join('\n')}`); } } return transformedPlay; } catch (e) { - getLogger().warn(new Error(`Unexpected error occurred, returning original play.`, {cause: e})); + logger.warn(new Error(`Unexpected error occurred, returning original play.`, {cause: e})); return play; } } diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index 82f422f2..fff470e4 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -1,6 +1,5 @@ import { childLogger, Logger } from "@foxxmd/logging"; import { PlayObject, TransformerCommon, TransformerCommonConfig } from "../../../core/Atomic.js"; -import { getRoot } from "../../ioc.js"; import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import AbstractInitializable from "../AbstractInitializable.js"; import { StageConfig } from "../infrastructure/Transform.js"; @@ -8,8 +7,8 @@ import { cacheFunctions, parseToRegexOrLiteralSearch, testMaybeRegex, searchAnd import { Cacheable } from "cacheable"; import { hashObject } from "../../utils/StringUtils.js"; import { playContentInvariantTransform } from "../../utils/PlayComparisonUtils.js"; -import e from "express"; import { isSimpleError } from "../errors/MSErrors.js"; +import { capitalize } from "../../../core/StringUtils.js"; export interface TransformerOptions { logger: Logger @@ -38,11 +37,15 @@ export default abstract class AbstractTransformer { - const cacheKey = `${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}` try { const cachedTransform = await this.cache.get(cacheKey); if(cachedTransform !== undefined) { - this.logger.debug('Cache hit'); + this.logger.debug('Transform cache hit'); return cachedTransform; } } catch (e) { - this.logger.warn(new Error('Could not fetch cache key', {cause: e})); + this.logger.warn(new Error(`Could not fetch cache key ${cacheKey}`, {cause: e})); } if (data.when !== undefined) { if (!testWhenConditions(data.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { - this.logger.debug('When condition not met, returning original Play'); + this.logger.debug('Returning original Play because because when condition not met'); await this.cache.set(cacheKey, play, '15s'); return play; } @@ -79,9 +81,9 @@ export default abstract class AbstractTransformer { const clean = str.trim().toLocaleLowerCase(); @@ -115,7 +111,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer 0) { - this.logger.debug(`${buildTrackString(play)} - Missing desired MBIDs for ${missing.join(', ')}`); + this.logger.debug(`Missing desired MBIDs for: ${missing.join(', ')}`); } else if(forceSearch) { - this.logger.debug(`${buildTrackString(play)} - No desired MBIDs are missing but forceSearch = true`); + this.logger.debug(`No desired MBIDs are missing but forceSearch = true`); } else { throw new SimpleError('No desired MBIDs are missing'); } @@ -170,9 +166,11 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { if(transformData === undefined) { - throw new SimpleError('No match returned from Musicbrainz API'); + throw new SimpleError('No matches returned from Musicbrainz API'); } const { @@ -192,6 +190,8 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { @@ -251,8 +251,5 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { return; } - protected getIdentifier(): string { - return 'Musicbrainz Transformer'; - } } \ No newline at end of file diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts index 7b86fafd..e99e6c0c 100644 --- a/src/backend/common/transforms/NativeTransformer.ts +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -234,8 +234,5 @@ export default class NativeTransformer extends AtomicPartsTransformer { return; } - protected getIdentifier(): string { - return 'Native Transformer'; - } } \ No newline at end of file diff --git a/src/backend/common/transforms/TransformerManager.ts b/src/backend/common/transforms/TransformerManager.ts index 0de6410e..bfe01515 100644 --- a/src/backend/common/transforms/TransformerManager.ts +++ b/src/backend/common/transforms/TransformerManager.ts @@ -7,11 +7,9 @@ import { PlayObject } from "../../../core/Atomic.js"; import { isStageTyped } from "../../utils/PlayTransformUtils.js"; import { MSCache } from "../Cache.js"; import NativeTransformer from "./NativeTransformer.js"; -import { MusicbrainzApiClient } from "../vendor/musicbrainz/MusicbrainzApiClient.js"; -import { MUSICBRAINZ_URL, MusicbrainzApiConfigData } from "../infrastructure/Atomic.js"; -import { normalizeWebAddress } from "../../utils/NetworkUtils.js"; -import { MusicBrainzApi } from "musicbrainz-api"; import MusicbrainzTransformer, { MusicbrainzTransformerConfig } from "./MusicbrainzTransformer.js"; +import { AsyncLocalStorage } from 'node:async_hooks'; +import { nanoid } from "nanoid"; export default class TransformerManager { @@ -19,11 +17,13 @@ export default class TransformerManager { protected parentLogger: Logger; protected transformers: Map = new Map(); protected cache: MSCache; + protected asyncStore: AsyncLocalStorage; public constructor(logger: Logger, cache: MSCache) { this.logger = childLogger(logger, 'Transformer Manager'); this.parentLogger = logger; this.cache = cache; + this.asyncStore = new AsyncLocalStorage(); } public register(config: TransformerCommonConfig): void { @@ -41,16 +41,18 @@ export default class TransformerManager { this.logger.verbose(`Registering ${config.type} transformer with name '${tName}'`); + const tLogger = childLogger(this.parentLogger, ['Transformer', () => this.asyncStore.getStore() ?? undefined]); + let t: AbstractTransformer; switch (config.type) { case 'user': - t = new UserTransformer({ name: tName, ...config }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); + t = new UserTransformer({ name: tName, ...config }, {logger: tLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); break; case 'native': - t = new NativeTransformer({ name: tName, ...config }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); + t = new NativeTransformer({ name: tName, ...config }, {logger: tLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); break; case 'musicbrainz': - t = new MusicbrainzTransformer({ name: tName, ...config as MusicbrainzTransformerConfig }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); + t = new MusicbrainzTransformer({ name: tName, ...config as MusicbrainzTransformerConfig }, {logger: tLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); break; default: throw new Error(`No transformer of type '${config.type}' exists.`); @@ -104,7 +106,7 @@ export default class TransformerManager { return t.parseConfig(data); } - public async handleStage(data: StageConfig, play: PlayObject): Promise { + public async handleStage(data: StageConfig, play: PlayObject, asyncId: string = nanoid(6)): Promise { const list = this.transformers.get(data.type); if (list === undefined || list.length === 0) { throw new Error(`No transformer of type '${data.type}' is registered.`); @@ -127,7 +129,9 @@ export default class TransformerManager { } try { - return await t.handle(data, play); + return this.asyncStore.run(asyncId, async () => { + return await t.handle(data, play); + }); } catch (e) { throw new Error('Stage processing failed', {cause: e}); } diff --git a/src/backend/common/transforms/UserTransformer.ts b/src/backend/common/transforms/UserTransformer.ts index 1b177554..9fe161c0 100644 --- a/src/backend/common/transforms/UserTransformer.ts +++ b/src/backend/common/transforms/UserTransformer.ts @@ -96,8 +96,5 @@ export default class UserTransformer extends AtomicPartsTransformer { return; } - protected getIdentifier(): string { - return 'User Transformer'; - } } \ No newline at end of file diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index 54f5ae4c..54621637 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -87,7 +87,7 @@ export class MusicbrainzApiClient extends AbstractApiClient { } - this.logger.debug(`Starting search for ${buildTrackString(play)}`); + this.logger.debug(`Starting search`); const res = await this.callApi((mb) => { const query: Record = { recording: play.data.track diff --git a/src/backend/tests/ytm/ytm.test.ts b/src/backend/tests/ytm/ytm.test.ts index 79946207..f11ac815 100644 --- a/src/backend/tests/ytm/ytm.test.ts +++ b/src/backend/tests/ytm/ytm.test.ts @@ -211,7 +211,7 @@ describe('Handles interim tracks', function () { const interimPlays = [generatePlay({duration: 40}, { comment: 'Today' }), generatePlay({duration: 200}, { comment: 'Today' })] const prependedPlays = [firstPlay, ...interimPlays, ...plays]; const prependResult = source.parseRecentAgainstResponse(prependedPlays); - expect(prependResult.plays).length(2); + expect(prependResult.plays).length(3); expect(prependResult.plays[prependResult.plays.length - 1].data.track).eq(firstPlay.data.track) }); }); \ No newline at end of file -- 2.51.2 From 03bb1e05521b8c1f2e2c0b43660a41b80a97ccff Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 3 Dec 2025 17:04:58 +0000 Subject: [PATCH 08/41] fix: Use pmap for all bulk transformer --- src/backend/scrobblers/AbstractScrobbleClient.ts | 5 +++-- src/backend/sources/AbstractSource.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 00085e78..4c772885 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -57,6 +57,7 @@ import { MSCache } from "../common/Cache.js"; import { getRoot } from "../ioc.js"; import { rehydratePlay } from "../utils/CacheUtils.js"; import { findAsyncSequential, staggerMapper } from "../utils/AsyncUtils.js"; +import pMap from "p-map"; type PlatformMappedPlays = Map; type NowPlayingQueue = Map; @@ -474,8 +475,8 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i const playObj = await this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate); - const dtInvariantMatches = (await Promise.all(this.scrobbledPlayObjs.data - .map(async x => ({...x, play: await this.transformPlay(x.play, TRANSFORM_HOOK.existing)})))) + const sm = staggerMapper({concurrency: 2}); + const dtInvariantMatches = (await pMap(this.scrobbledPlayObjs.data, sm(async x => ({...x, play: await this.transformPlay(x.play, TRANSFORM_HOOK.existing)})), {concurrency: 2})) .filter(x => playObjDataMatch(playObj, x.play)); if (dtInvariantMatches.length === 0) { diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index 2b16730e..eea72c51 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -234,7 +234,7 @@ export default abstract class AbstractSource extends AbstractComponent implement newDiscoveredPlays.sort(sortByOldestPlayDate); const sm = staggerMapper({concurrency: 2}); this.emitter.emit('discoveredToScrobble', { - data: await pMap(newDiscoveredPlays, sm(async (x) => await this.transformPlay(x, TRANSFORM_HOOK.postCompare)), {concurrency: 2}), // Promise.all(newDiscoveredPlays.map(x => this.transformPlay(x, TRANSFORM_HOOK.postCompare))), + data: await pMap(newDiscoveredPlays, sm(async (x) => await this.transformPlay(x, TRANSFORM_HOOK.postCompare)), {concurrency: 2}), options: { ...options, checkTime: newDiscoveredPlays[newDiscoveredPlays.length-1].data.playDate.add(2, 'second'), -- 2.51.2 From dab2414a92b4318ebc6dd52a69c398512231c92d Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 3 Dec 2025 19:12:07 +0000 Subject: [PATCH 09/41] chore: Update musicbrainz-api to use ratelimit Borewit/musicbrainz-api#1115 --- package-lock.json | 8 ++++---- package.json | 2 +- patches/musicbrainz-api+0.26.0.patch | 26 -------------------------- 3 files changed, 5 insertions(+), 31 deletions(-) delete mode 100644 patches/musicbrainz-api+0.26.0.patch diff --git a/package-lock.json b/package-lock.json index 36262411..d963c684 100644 --- a/package-lock.json +++ b/package-lock.json @@ -71,7 +71,7 @@ "lastfm-node-client": "^2.2.0", "mopidy": "^1.3.0", "mpd-api": "^1.1.2", - "musicbrainz-api": "^0.26.0", + "musicbrainz-api": "^0.27.0", "nanoid": "^3.3.1", "node-object-hash": "^3.1.1", "normalize-url": "^8.0.1", @@ -9041,9 +9041,9 @@ "license": "(Apache-2.0 AND MIT)" }, "node_modules/musicbrainz-api": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/musicbrainz-api/-/musicbrainz-api-0.26.0.tgz", - "integrity": "sha512-HBHchcJlBnTNzgrj9ZvlIzp4rMs31WvMg3VdXQ7nKdP3acxC5yD6+Mis98ubzUFappUZFzTbjhKgLYWR9JI47g==", + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/musicbrainz-api/-/musicbrainz-api-0.27.0.tgz", + "integrity": "sha512-0U+rwgtxE7Q2AjIA6eRKzDPfYa6ailktpL6UfpEK2lFDQxWw4pfG50ZwL0irbjHPyEtjB5umbtvHTH/6tFKkLw==", "license": "MIT", "dependencies": { "debug": "^4.3.4", diff --git a/package.json b/package.json index 613c56aa..edcafda1 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,7 @@ "lastfm-node-client": "^2.2.0", "mopidy": "^1.3.0", "mpd-api": "^1.1.2", - "musicbrainz-api": "^0.26.0", + "musicbrainz-api": "^0.27.0", "nanoid": "^3.3.1", "node-object-hash": "^3.1.1", "normalize-url": "^8.0.1", diff --git a/patches/musicbrainz-api+0.26.0.patch b/patches/musicbrainz-api+0.26.0.patch deleted file mode 100644 index 07255284..00000000 --- a/patches/musicbrainz-api+0.26.0.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts b/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts -index dfd2937..09ed531 100644 ---- a/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts -+++ b/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts -@@ -64,6 +64,7 @@ export interface IMusicBrainzConfig { - */ - appContactInfo?: string; - disableRateLimiting?: boolean; -+ rateLimit?: [number, number] - } - interface IInternalConfig extends IMusicBrainzConfig { - baseUrl: string; -diff --git a/node_modules/musicbrainz-api/lib/musicbrainz-api.js b/node_modules/musicbrainz-api/lib/musicbrainz-api.js -index 299be1c..7ee41b0 100644 ---- a/node_modules/musicbrainz-api/lib/musicbrainz-api.js -+++ b/node_modules/musicbrainz-api/lib/musicbrainz-api.js -@@ -36,7 +36,8 @@ export class MusicBrainzApi { - ..._config - }; - this.httpClient = this.initHttpClient(); -- this.rateLimiter = new RateLimitThreshold(15, 18); -+ const limits = this.config.rateLimit ?? [15,18]; -+ this.rateLimiter = new RateLimitThreshold(limits[0], limits[1]); - } - initHttpClient() { - return new HttpClient({ -- 2.51.2 From 1e558de5e1e07b6f9f1cd0cd1aab164335bd0b82 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 3 Dec 2025 20:49:57 +0000 Subject: [PATCH 10/41] feat(transformer): Implement filtering and sorting of fetched musicbrainz recordings --- .../common/transforms/AbstractTransformer.ts | 23 +- .../transforms/AtomicPartsTransformer.ts | 10 +- .../transforms/MusicbrainzTransformer.ts | 275 ++++++++++++++++-- src/core/Atomic.ts | 41 ++- 4 files changed, 299 insertions(+), 50 deletions(-) diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index fff470e4..ab33f1bb 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -78,30 +78,31 @@ export default abstract class AbstractTransformer; - public async getTransformerData(play: PlayObject, stageConfig: Y): Promise { + public async getTransformerData(play: PlayObject, stageConfig: Y): Promise { return undefined; } - public async checkShouldTransform(play: PlayObject, transformData: T, stageConfig: Y): Promise { - return; + public async handlePostFetch(play: PlayObject, transformData: any, stageConfig: Y): Promise { + return transformData; } - public async checkShouldTransformPreData(play: PlayObject, stageConfig: Y): Promise { + public async handlePreFetch(play: PlayObject, stageConfig: Y): Promise { return } } \ No newline at end of file diff --git a/src/backend/common/transforms/AtomicPartsTransformer.ts b/src/backend/common/transforms/AtomicPartsTransformer.ts index e9745391..3d3369de 100644 --- a/src/backend/common/transforms/AtomicPartsTransformer.ts +++ b/src/backend/common/transforms/AtomicPartsTransformer.ts @@ -13,7 +13,7 @@ export default abstract class AtomicPartsTransformer { - return undefined; - } - - public async checkShouldTransform(play: PlayObject, transformData: T, stageConfig: Z): Promise { - return; - } - protected abstract handleTitle(play: PlayObject, parts: Y, transformData: T): Promise; protected abstract handleArtists(play: PlayObject, parts: Y, transformData: T): Promise; protected abstract handleAlbumArtists(play: PlayObject, parts: Y, transformData: T): Promise; diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 53fe46a3..a4e0e350 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -1,4 +1,4 @@ -import { DEFAULT_MISSING_TYPES, MissingMbidType, PlayObject, TrackMeta, TransformerCommon } from "../../../core/Atomic.js"; +import { asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, isMBReleasePrimaryGroupType, MBReleaseGroupPrimaryType, MBReleaseGroupSecondaryType, MBReleaseStatus, MissingMbidType, PlayObject, TrackMeta, TransformerCommon } from "../../../core/Atomic.js"; import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; import { ExternalMetadataTerm, PlayTransformMetadataStage } from "../infrastructure/Transform.js"; @@ -6,13 +6,15 @@ import AtomicPartsTransformer from "./AtomicPartsTransformer.js"; import { TransformerOptions } from "./AbstractTransformer.js"; import { MUSICBRAINZ_URL, MusicbrainzApiConfigData } from "../infrastructure/Atomic.js"; import { MaybeLogger } from "../logging.js"; -import { childLogger } from "@foxxmd/logging"; +import { childLogger, Logger } from "@foxxmd/logging"; import { MusicbrainzApiClient, MusicbrainzApiConfig, recordingToPlay } from "../vendor/musicbrainz/MusicbrainzApiClient.js"; -import { MusicBrainzApi } from "musicbrainz-api"; +import { IRecordingList, IRecordingMatch, MusicBrainzApi } from "musicbrainz-api"; import { getRoot, version } from "../../ioc.js"; import { normalizeWebAddress } from "../../utils/NetworkUtils.js"; import { intersect, missingMbidTypes } from "../../utils.js"; import { SimpleError } from "../errors/MSErrors.js"; +import { parseArrayFromMaybeString } from "../../utils/StringUtils.js"; +import clone from "clone"; export const asMissingMbid = (str: string): MissingMbidType => { const clean = str.trim().toLocaleLowerCase(); @@ -33,10 +35,78 @@ export interface MusicbrainzTransformerData { searchWhenMissing?: MissingMbidType[] forceSearch?: boolean score?: number + + /** Allow only releases with release groups with these primary types + * + * @see https://wiki.musicbrainz.org/Release_Group/Type#Primary_types + */ + releaseGroupPrimaryTypeAllow?: string[] + /** Filter out any releases with release groups with these primary types + * + * @see https://wiki.musicbrainz.org/Release_Group/Type#Primary_types + */ + releaseGroupPrimaryTypeDeny?: string[] + /** Prioritise releases to use based on the order of these release group types + * + * @see https://wiki.musicbrainz.org/Release_Group/Type#Primary_types + */ + releaseGroupPrimaryTypePriority?: string[] + + /** Allow only releases with release groups with these secondary types + * + * @see https://wiki.musicbrainz.org/Release_Group/Type#Secondary_types + */ + releaseGroupSecondaryTypeAllow?: string[] + /** Filter out any releases with release groups with these secondary types + * + * @see https://wiki.musicbrainz.org/Release_Group/Type#Secondary_types + */ + releaseGroupSecondaryTypeDeny?: string[] + /** Prioritise releases to use based on the order of these release group secondary types + * + * @see https://wiki.musicbrainz.org/Release_Group/Type#Secondary_types + */ + releaseGroupSecondaryTypePriority?: string[] + + /** Allow only releases with these statuses + * + * @see https://wiki.musicbrainz.org/Release#Status + */ + releaseStatusAllow?: string[] + /** Filter out any releases with these statuses + * + * @see https://wiki.musicbrainz.org/Release#Status + */ + releaseStatusDeny?: string[] + /** Prioritise releases to used based on the order of these statuses + * + * @see https://wiki.musicbrainz.org/Release#Status + */ + releaseStatusPriority?: string[] + + /** Do not filter out a recording if it initially has no releases + * + * Use in conjunction with release filters by setting to `true` + * to prevent recordings from being filtered out solely becauase they don't have any releases to begin with + * + */ + releaseAllowEmpty?: boolean } export interface MusicbrainzTransformerDataStrong extends MusicbrainzTransformerData { searchWhenMissing: MissingMbidType[] + + releaseGroupPrimaryTypeAllow?: MBReleaseGroupPrimaryType[] + releaseGroupPrimaryTypeDeny?: MBReleaseGroupPrimaryType[] + releaseGroupPrimaryTypePriority?: MBReleaseGroupPrimaryType[] + + releaseGroupSecondaryTypeAllow?: MBReleaseGroupSecondaryType[] + releaseGroupSecondaryTypeDeny?: MBReleaseGroupSecondaryType[] + releaseGroupSecondaryTypePriority?: MBReleaseGroupSecondaryType[] + + releaseStatusAllow?: MBReleaseStatus[] + releaseStatusDeny?: MBReleaseStatus[] + releaseStatusPriority?: MBReleaseStatus[] } export interface MusicbrainzTransformerDataStage extends MusicbrainzTransformerDataStrong,PlayTransformMetadataStage { @@ -50,11 +120,27 @@ export type MusicbrainzBestMatch = {play: PlayObject, score: number}; export type MusicbrainzTransformerConfig = TransformerCommon; +export type RecordingRankedMatched = IRecordingMatch & {rankScore: number} + export const parseStageConfig = (data: MusicbrainzTransformerData | undefined = {}, logger: MaybeLogger = new MaybeLogger()): MusicbrainzTransformerDataStrong => { const config: MusicbrainzTransformerDataStrong = { searchWhenMissing: DEFAULT_MISSING_TYPES, - score: 90 + score: 90, + + releaseGroupPrimaryTypeAllow: data.releaseGroupPrimaryTypeAllow !== undefined ? parseArrayFromMaybeString(data.releaseGroupPrimaryTypeAllow, {lower: true}).map(asMBReleasePrimaryGroupType) : undefined, + releaseGroupPrimaryTypeDeny: data.releaseGroupPrimaryTypeDeny !== undefined ?parseArrayFromMaybeString(data.releaseGroupPrimaryTypeDeny, {lower: true}).map(asMBReleasePrimaryGroupType) : undefined, + releaseGroupPrimaryTypePriority: data.releaseGroupPrimaryTypePriority !== undefined ? parseArrayFromMaybeString(data.releaseGroupPrimaryTypePriority, {lower: true}).map(asMBReleasePrimaryGroupType) : undefined, + + releaseGroupSecondaryTypeAllow: data.releaseGroupSecondaryTypeAllow !== undefined ? parseArrayFromMaybeString(data.releaseGroupSecondaryTypeAllow, {lower: true}).map(asMBReleaseSecondaryGroupType) : undefined, + releaseGroupSecondaryTypeDeny: data.releaseGroupSecondaryTypeDeny !== undefined ? parseArrayFromMaybeString(data.releaseGroupSecondaryTypeDeny, {lower: true}).map(asMBReleaseSecondaryGroupType) : undefined, + releaseGroupSecondaryTypePriority: data.releaseGroupSecondaryTypePriority !== undefined ? parseArrayFromMaybeString(data.releaseGroupSecondaryTypePriority, {lower: true}).map(asMBReleaseSecondaryGroupType) : undefined, + + releaseStatusAllow: data.releaseStatusAllow !== undefined ? parseArrayFromMaybeString(data.releaseStatusAllow, {lower: true}).map(asMBReleaseStatus) : undefined, + releaseStatusDeny: data.releaseStatusAllow !== undefined ? parseArrayFromMaybeString(data.releaseStatusDeny, {lower: true}).map(asMBReleaseStatus) : undefined, + releaseStatusPriority: data.releaseStatusAllow !== undefined ? parseArrayFromMaybeString(data.releaseStatusPriority, {lower: true}).map(asMBReleaseStatus) : undefined, + + releaseAllowEmpty: data.releaseAllowEmpty }; if (data === null || typeof data !== 'object') { @@ -71,10 +157,16 @@ export const parseStageConfig = (data: MusicbrainzTransformerData | undefined = logger.debug(`Will search if missing: ${config.searchWhenMissing.join(', ')} | Match if (default) score is >= ${config.score}`); + for(const [k,v] of Object.entries(config)) { + if(k.includes('release') && v !== undefined) { + logger.debug(`${k}: ${v.join(' | ')}`); + } + } + return config; } -export default class MusicbrainzTransformer extends AtomicPartsTransformer { +export default class MusicbrainzTransformer extends AtomicPartsTransformer { declare config: MusicbrainzTransformerConfig; @@ -146,7 +238,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + public async handlePreFetch(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise { const { searchWhenMissing = this.defaults.searchWhenMissing, forceSearch = false @@ -162,23 +254,24 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + public async getTransformerData(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise { // TODO maybe search more broadly if first query doesn't hit? const results = await this.api.searchByRecording(play); - if(results === undefined || results.recordings?.length === 0) { - if(results === undefined) { - this.logger.warn('results were unexpectedly undefined! API should have thrown...'); - } - return undefined; + if(results === undefined) { + throw new Error('results were unexpectedly undefined! API should have thrown...'); + } + if(results.recordings === undefined) { + this.logger.debug(results); + throw new Error('results returned by no recordings list in response data, something handled incorrectly?'); } - return { play: recordingToPlay(results.recordings[0]), score: results.recordings[0].score }; + return results; } - public async checkShouldTransform(play: PlayObject, transformData: MusicbrainzBestMatch | undefined, stageConfig: MusicbrainzTransformerDataStage): Promise { - if(transformData === undefined) { + public async handlePostFetch(play: PlayObject, transformData: IRecordingList, stageConfig: MusicbrainzTransformerDataStage): Promise { + if(transformData.recordings.length === 0) { throw new SimpleError('No matches returned from Musicbrainz API'); } @@ -186,15 +279,27 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer x.score >= score); + if(filteredList.length === 0) { + throw new SimpleError(`All ${transformData.count} fetched matches had a score < ${score}, best match was ${transformData.recordings[0].score}`); + } + const mergedConfig = Object.assign({}, this.defaults, stageConfig); + filteredList = filterByValidReleaseStatus(filteredList, mergedConfig); + filteredList = filterByValidReleaseGroupPrimary(filteredList, mergedConfig); + filteredList = filterByValidReleaseGroupSecondary(filteredList, mergedConfig); + + if(filteredList.length === 0) { + throw new SimpleError(`All ${transformData.count} recordings were filtered out by allow/deny release config`); } - this.logger.debug(`Got valid match`); + filteredList = rankReleasesByPriority(filteredList, mergedConfig); + + this.logger.debug(`${filteredList.length} of ${transformData.count} were valid, filtered matches. Using match with best score of ${filteredList[0].score}`); + + return recordingToPlay(filteredList[0]); } - protected async handleTitle(play: PlayObject, parts: ExternalMetadataTerm, transformData: MusicbrainzBestMatch): Promise { + protected async handleTitle(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { if (parts === false) { return play.data.track; } @@ -207,9 +312,9 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { if (parts === false) { return play.data.artists; } @@ -222,13 +327,13 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { // TODO return play.data.albumArtists; } - protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, transformData: MusicbrainzBestMatch): Promise { + protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { if (parts === false) { return play.data.album; } @@ -241,15 +346,127 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { - return transformData.play.data.meta; + protected async handleMeta(play: PlayObject, transformData: PlayObject): Promise { + return transformData.data.meta; } public notify(payload: WebhookPayload): Promise { return; } -} \ No newline at end of file +} + +export const filterByValidReleaseStatus = (list: IRecordingMatch[], stageConfig: MusicbrainzTransformerDataStage, logger: MaybeLogger = new MaybeLogger()) => { + const { + releaseStatusAllow = [], + releaseStatusDeny = [], + releaseAllowEmpty + } = stageConfig; + if(releaseStatusAllow.length === 0 && releaseStatusDeny.length === 0) { + return list; + } + const releaseFiltered = list.map(x => { + return { + ...x, + releases: x.releases.filter(y => { + if(releaseStatusAllow.length > 0) { + return releaseStatusAllow.includes(y.status.toLocaleLowerCase() as MBReleaseStatus) + } + return !releaseStatusAllow.includes(y.status.toLocaleLowerCase() as MBReleaseStatus) + }) + } + }); + return releaseFiltered.filter(x => ( + list.find(y => y.id === x.id).releases.length === 0 + && releaseAllowEmpty + ) + || x.releases.length > 0); +} + +export const filterByValidReleaseGroupPrimary = (list: IRecordingMatch[], stageConfig: MusicbrainzTransformerDataStage, logger: MaybeLogger = new MaybeLogger()) => { + const { + releaseGroupPrimaryTypeAllow = [], + releaseGroupPrimaryTypeDeny = [], + releaseAllowEmpty + } = stageConfig; + if(releaseGroupPrimaryTypeAllow.length === 0 && releaseGroupPrimaryTypeAllow.length === 0) { + return list; + } + const releaseFiltered = list.map(x => { + return { + ...x, + releases: x.releases.filter(y => { + if(releaseGroupPrimaryTypeAllow.length > 0) { + return releaseGroupPrimaryTypeAllow.includes(y["release-group"]["primary-type"].toLocaleLowerCase() as MBReleaseGroupPrimaryType) + } + return !releaseGroupPrimaryTypeDeny.includes(y["release-group"]["primary-type"].toLocaleLowerCase() as MBReleaseGroupPrimaryType) + }) + } + }); + return releaseFiltered.filter(x => ( + list.find(y => y.id === x.id).releases.length === 0 + && releaseAllowEmpty + ) + || x.releases.length > 0); +} + +export const filterByValidReleaseGroupSecondary = (list: IRecordingMatch[], stageConfig: MusicbrainzTransformerDataStage, logger: MaybeLogger = new MaybeLogger()) => { + const { + releaseGroupSecondaryTypeAllow = [], + releaseGroupSecondaryTypeDeny = [], + releaseAllowEmpty + } = stageConfig; + if(releaseGroupSecondaryTypeAllow.length === 0 && releaseGroupSecondaryTypeDeny.length === 0) { + return list; + } + const releaseFiltered = list.map(x => { + return { + ...x, + releases: x.releases.filter(y => { + if(releaseGroupSecondaryTypeAllow.length > 0) { + return intersect(releaseGroupSecondaryTypeAllow, (y["release-group"]["secondary-types"] ?? []).map(x => x.toLocaleLowerCase()) as MBReleaseGroupSecondaryType[]).length > 0; + } + return intersect(releaseGroupSecondaryTypeDeny, (y["release-group"]["secondary-types"] ?? []).map(x => x.toLocaleLowerCase()) as MBReleaseGroupSecondaryType[]).length === 0; + }) + } + }); + return releaseFiltered.filter(x => ( + list.find(y => y.id === x.id).releases.length === 0 + && releaseAllowEmpty + ) + || x.releases.length > 0); +} + +export const rankReleasesByPriority = (list: IRecordingMatch[], stageConfig: MusicbrainzTransformerDataStage, logger: MaybeLogger = new MaybeLogger()) => { + const { + releaseStatusPriority = [], + releaseGroupPrimaryTypePriority = [], + releaseGroupSecondaryTypePriority = [] + } = stageConfig; + if(releaseStatusPriority.length === 0 && releaseGroupPrimaryTypePriority.length === 0 && releaseGroupSecondaryTypePriority.length === 0) { + return list; + } + + const cList = clone(list); + const rankedList = cList.map((x) => { + return { + ...x, + releases: (x.releases ?? []).map((a) => { + const statAScore = releaseStatusPriority.findIndex(x => x === a.status.toLocaleLowerCase()) + 1; + const grpPAScore = releaseGroupPrimaryTypePriority.findIndex(x => x === a["release-group"]["primary-type"].toLocaleLowerCase()) + 1; + const grpSAScore = (a["release-group"]["secondary-types"] ?? []).reduce((acc: number, curr: MBReleaseGroupSecondaryType) => acc + releaseGroupSecondaryTypePriority.findIndex(x => x === (curr as MBReleaseGroupSecondaryType).toLocaleLowerCase()) + 1,0); + + return { + ...a, + rankedScore: statAScore + grpPAScore + grpSAScore + }; + }) + } + }); + for(const rec of rankedList) { + rec.releases.sort((a, b) => b.rankedScore - a.rankedScore); + } +}; \ No newline at end of file diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 7d0c54ca..e2e570e6 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -406,4 +406,43 @@ export interface TransformerCommon, Y = Record { + return MB_RELEASE_STATUSES.includes(str as MBReleaseStatus); +} +export const asMBReleaseStatus = (str: string): MBReleaseStatus => { + if(isMBReleaseStatus(str)) { + return str; + } else { + throw new Error(`Release Status is not valid: ${str}`); + } +} + +export type MBReleaseGroupPrimaryType = 'album' | 'single' | 'ep' | 'broadcast' | 'other'; +export const MB_RELEASE_GROUP_PRIMARY_TYPES: MBReleaseGroupPrimaryType[] = ['album','single','ep','broadcast','other']; +export const isMBReleasePrimaryGroupType = (str: string): str is MBReleaseGroupPrimaryType => { + return MB_RELEASE_GROUP_PRIMARY_TYPES.includes(str as MBReleaseGroupPrimaryType); +} +export const asMBReleasePrimaryGroupType = (str: string): MBReleaseGroupPrimaryType => { + if(isMBReleasePrimaryGroupType(str)) { + return str; + } else { + throw new Error(`Primary Release Group is not valid: ${str}`); + } +} + +export type MBReleaseGroupSecondaryType = 'compilation' | 'soundtrack' | 'live' | 'remix'; +export const MB_RELEASE_GROUP_SECONDARY_TYPES: MBReleaseGroupSecondaryType[] = ['compilation','soundtrack','live','remix']; +export const isMBReleaseSecondaryGroupType = (str: string): str is MBReleaseGroupSecondaryType => { + return MB_RELEASE_GROUP_SECONDARY_TYPES.includes(str as MBReleaseGroupSecondaryType); +} +export const asMBReleaseSecondaryGroupType = (str: string): MBReleaseGroupSecondaryType => { + if(isMBReleaseSecondaryGroupType(str)) { + return str; + } else { + throw new Error(`Secondary Release Group is not valid: ${str}`); + } +} -- 2.51.2 From 1410e3a952b9c9957dd6711c709b32828892c1eb Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 3 Dec 2025 21:47:34 +0000 Subject: [PATCH 11/41] fix leftover method from refactor --- src/backend/common/transforms/AtomicPartsTransformer.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/backend/common/transforms/AtomicPartsTransformer.ts b/src/backend/common/transforms/AtomicPartsTransformer.ts index 3d3369de..5a2637fb 100644 --- a/src/backend/common/transforms/AtomicPartsTransformer.ts +++ b/src/backend/common/transforms/AtomicPartsTransformer.ts @@ -12,13 +12,6 @@ export default abstract class AtomicPartsTransformer = {}; if (parts.title !== undefined) { -- 2.51.2 From 14c87304355a0bf718642ad7f68d39db2bd42a33 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 4 Dec 2025 02:54:34 +0000 Subject: [PATCH 12/41] Handle album artists --- .../transforms/MusicbrainzTransformer.ts | 12 ++++++++++- .../musicbrainz/MusicbrainzApiClient.ts | 21 ++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index a4e0e350..37c1d7f4 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -330,7 +330,17 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { - // TODO + if (parts === false) { + return play.data.artists; + } + if (typeof parts === 'object') { + if (parts.when !== undefined) { + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { + this.logger.debug('When condition for albumArtists not met, returning original artists'); + return play.data.artists; + } + } + } return play.data.albumArtists; } protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index 54621637..50394503 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -127,24 +127,31 @@ export class MusicbrainzApiClient extends AbstractApiClient { export const recordingToPlay = (data: IRecording): PlayObject => { let album: IRelease; - // try to find official album first - album = data.releases.find(x => x.status === 'Official'); - if(album === undefined) { - // find the first album that isn't "bad" - album = data.releases.find(x => !['Expunged','Withdrawn'].includes(x.status)); + + let albumArtists: string[]; + const artists = (data["artist-credit"] ?? []).map(x => x.name); + if(data.releases !== undefined && data.releases.length > 0) { + album = data.releases[0]; + const aa = album["artist-credit"].map(x => x.name); + // if not every artist of the recording is also on the album + // then use release album artists + if(!artists.every(x => aa.includes(x))) { + albumArtists = aa; + } } const play: PlayObject = { data: { - // TODO figure out album artists track: data.title, - artists: (data["artist-credit"] ?? []).map(x => x.name), + artists, album: album !== undefined ? album.title : undefined, + albumArtists, meta: { brainz: { track: data.id, artist: data["artist-credit"].map(x => x.artist.id), album: album !== undefined ? album.id : undefined, + releaseGroup: album !== undefined ? album["release-group"]?.id : undefined } } }, -- 2.51.2 From e055e0bd53ca295940503bc9d6f50c04f908f911 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 4 Dec 2025 17:11:15 +0000 Subject: [PATCH 13/41] feat(transform): Implement more exhaustive, human-readable transform diff --- package-lock.json | 104 ++++++++++++++++-- package.json | 1 + src/backend/common/AbstractComponent.ts | 72 +++++++++--- .../common/infrastructure/Transform.ts | 4 +- src/core/StringUtils.ts | 43 ++++++++ 5 files changed, 198 insertions(+), 26 deletions(-) diff --git a/package-lock.json b/package-lock.json index d963c684..c5bbdb8b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -65,6 +65,7 @@ "icecast-metadata-stats": "^0.1.12", "iso-websocket": "^0.3.0", "iti": "^0.7.0", + "jest-diff": "^30.2.0", "json5": "^2.2.3", "keyv": "^5.5.0", "kodi-api": "^0.2.1", @@ -2020,6 +2021,36 @@ "axios": "^1.3.4" } }, + "node_modules/@jest/diff-sequences": { + "version": "30.0.1", + "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz", + "integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==", + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/get-type": { + "version": "30.1.0", + "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", + "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/schemas": { + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", + "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.34.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", @@ -2758,6 +2789,12 @@ "win32" ] }, + "node_modules/@sinclair/typebox": { + "version": "0.34.41", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.41.tgz", + "integrity": "sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g==", + "license": "MIT" + }, "node_modules/@sindresorhus/is": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz", @@ -5518,6 +5555,7 @@ "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-27.5.1.tgz", "integrity": "sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ==", "dev": true, + "license": "MIT", "engines": { "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } @@ -7919,20 +7957,52 @@ } }, "node_modules/jest-diff": { - "version": "27.5.1", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-27.5.1.tgz", - "integrity": "sha512-m0NvkX55LDt9T4mctTEgnZk3fmEg3NRYutvMPWM/0iPnkFj2wIeF45O1718cMSOFO1vINkqmxqD8vE37uTEbqw==", - "dev": true, + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz", + "integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==", + "license": "MIT", "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "^27.5.1", - "jest-get-type": "^27.5.1", - "pretty-format": "^27.5.1" + "@jest/diff-sequences": "30.0.1", + "@jest/get-type": "30.1.0", + "chalk": "^4.1.2", + "pretty-format": "30.2.0" }, "engines": { - "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-diff/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-diff/node_modules/pretty-format": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", + "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.5", + "ansi-styles": "^5.2.0", + "react-is": "^18.3.1" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, + "node_modules/jest-diff/node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, "node_modules/jest-get-type": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-27.5.1.tgz", @@ -7957,6 +8027,22 @@ "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } }, + "node_modules/jest-matcher-utils/node_modules/jest-diff": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-27.5.1.tgz", + "integrity": "sha512-m0NvkX55LDt9T4mctTEgnZk3fmEg3NRYutvMPWM/0iPnkFj2wIeF45O1718cMSOFO1vINkqmxqD8vE37uTEbqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "^27.5.1", + "jest-get-type": "^27.5.1", + "pretty-format": "^27.5.1" + }, + "engines": { + "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" + } + }, "node_modules/jintr": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/jintr/-/jintr-3.3.1.tgz", diff --git a/package.json b/package.json index edcafda1..37c26ab6 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "icecast-metadata-stats": "^0.1.12", "iso-websocket": "^0.3.0", "iti": "^0.7.0", + "jest-diff": "^30.2.0", "json5": "^2.2.3", "keyv": "^5.5.0", "kodi-api": "^0.2.1", diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index f9733e05..859fd2f3 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -4,8 +4,8 @@ import { } from "@foxxmd/regex-buddy-core"; import deepEqual from 'fast-deep-equal'; import { Simulate } from "react-dom/test-utils"; -import { PlayObject } from "../../core/Atomic.js"; -import { buildTrackString } from "../../core/StringUtils.js"; +import { PlayData, PlayObject, TransformResult } from "../../core/Atomic.js"; +import { buildPlayHumanDiffable, buildTrackString } from "../../core/StringUtils.js"; import { CommonClientConfig } from "./infrastructure/config/client/index.js"; import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; import { TransformRulesError } from "./errors/MSErrors.js"; @@ -20,6 +20,8 @@ import play = Simulate.play; import TransformerManager from "./transforms/TransformerManager.js"; import { getRoot } from "../ioc.js"; import { nanoid } from "nanoid"; +import {diffStringsUnified, DiffOptionsColor} from 'jest-diff'; +import chalk from 'chalk'; export default abstract class AbstractComponent extends AbstractInitializable { @@ -183,7 +185,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { logger.debug(`Transform start for => ${buildTrackString(play)}`); let transformedPlay: PlayObject = play; - let transformDetails: string[] = []; + let transformHistory: TransformResult[] = []; for(const hookItem of hook) { const { @@ -208,15 +210,18 @@ export default abstract class AbstractComponent extends AbstractInitializable { if(!failureReturnPartial) { // rewind to original play so we don't return partial transform transformedPlay = play; - transformDetails = []; + transformHistory = []; } break; } } - if(!deepEqual(newTransformedPlay, transformedPlay)) { - transformDetails.push(`${hookItem.type} - ${buildTrackString(transformedPlay, {include: ['artist', 'track', 'album']})}`); - } + transformHistory.push({ + type: hookItem.type, + name: hookItem.name, + play: newTransformedPlay.data + }); + transformedPlay = newTransformedPlay; if(err === undefined && onSuccess === 'stop') { @@ -225,18 +230,53 @@ export default abstract class AbstractComponent extends AbstractInitializable { } } - if(transformDetails.length > 0) { - let transformStatements = [`Original: ${buildTrackString(play, {include: ['artist', 'track', 'album']})}`]; - const shouldLog = log ?? this.config.options?.playTransform?.log ?? false; - if (shouldLog === true || shouldLog === 'all') { - if(shouldLog === 'all') { - transformStatements = transformStatements.concat(transformDetails.map(x => `=> ${x}`)); - } else { - transformStatements.push(`=> ${transformDetails[transformDetails.length - 1]}`); + const shouldLog = log ?? this.config.options?.playTransform?.log ?? false; + + if(transformedPlay.meta.transforms === undefined) { + transformedPlay.meta.transforms = { + original: play.data + }; + } + if(shouldLog !== false) { + if(transformHistory.length === 0) { + logger.debug('Transform Diff: No Change'); + } + const historyToDiff: {name: string, data: PlayData}[] = [ + {name: 'Original', data: transformedPlay.meta.transforms.original} + ]; + if(shouldLog === true) { + const last = transformHistory[transformHistory.length - 1]; + historyToDiff.push({name: `${last.type}-${last.name}`, data: last.play}); + } else { + for(const t of transformHistory) { + historyToDiff.push({name: `${t.type}-${t.name}`, data: t.play}) } - logger.debug({labels: [hookType]}, `Transform Pipeline:\n${transformStatements.join('\n')}`); } + const diffs: string[] = []; + historyToDiff.forEach((curr, index) => { + if(index === 0) { + return; + } + const last = historyToDiff[index - 1]; + if(deepEqual(last.data, curr.data)) { + diffs.push(`${last.name} => ${curr.name} -- No Change`); + } else { + diffs.push(diffStringsUnified( + buildPlayHumanDiffable(last.data, {expandMeta: true}), + buildPlayHumanDiffable(curr.data, {expandMeta: true}), + { + aAnnotation: last.name, + aColor: chalk.red, + bAnnotation: curr.name, + bColor: chalk.green + } + )) + } + }); + logger.debug(`Transform Diff\n${diffs}`) } + const previousHistory = transformedPlay.meta.transforms[hookType] ?? []; + transformedPlay.meta.transforms[hookType] = [...previousHistory, ...transformHistory]; return transformedPlay; } catch (e) { logger.warn(new Error(`Unexpected error occurred, returning original play.`, {cause: e})); diff --git a/src/backend/common/infrastructure/Transform.ts b/src/backend/common/infrastructure/Transform.ts index 032d6ea6..058b2926 100644 --- a/src/backend/common/infrastructure/Transform.ts +++ b/src/backend/common/infrastructure/Transform.ts @@ -54,7 +54,9 @@ export interface FlowControl { failureReturnPartial: boolean } -export interface StageConfig extends StageTypedConfig, Whennable, Partial {} +export interface StageConfig extends StageTypedConfig, Whennable, Partial { + name?: string +} export interface AtomicStageConfig extends StageConfig, PlayTransformPartsAtomic {} diff --git a/src/core/StringUtils.ts b/src/core/StringUtils.ts index 3b60a501..2d561c29 100644 --- a/src/core/StringUtils.ts +++ b/src/core/StringUtils.ts @@ -6,6 +6,8 @@ import timezone from "dayjs/plugin/timezone.js"; import utc from "dayjs/plugin/utc.js"; import { AmbPlayObject, + PlayData, + PlayObject, SCROBBLE_TS_SOC_END, SCROBBLE_TS_SOC_START, ScrobbleTsSOC, @@ -132,6 +134,47 @@ export const buildTrackString = (playObj: AmbPlayObject, options: Tr return reducer(strParts); //strParts.join(' '); } +export const buildPlayHumanDiffable = (play: PlayData, options?: {expandMeta?: boolean}): string => { + const { + expandMeta = false + } = options || {}; + + let meta: string[] = []; + if(play.meta !== undefined) { + for(const [metaType,metaObject] of Object.entries(play.meta)) { + for(const [metaKey, metaValue] of Object.entries(metaObject)) { + if(metaValue === undefined) { + continue; + } + const id = `${metaType}-${metaKey}`; + if(expandMeta) { + meta.push(`${id}: ${metaValue}`); + } else { + meta.push(id); + } + } + } + } + let metaStr = '(None)'; + if(meta.length > 0) { + if(expandMeta) { + metaStr = `\n${meta.map(x => `* ${x}`).join('\n')}`; + } else { + metaStr = meta.join(', '); + } + } + const parts: string[] = [ + `${'Title'.padEnd(13)}: ${play.track ?? '(None)'}`, + `${'Artists'.padEnd(13)}: ${play.artists === undefined || play.artists.length === 0 ? '(None)' : play.artists.join(', ')}`, + `${'Album Artists'.padEnd(13)}: ${play.albumArtists === undefined || play.albumArtists.length === 0 ? '(None)' : play.albumArtists.join(', ')}`, + `${'Album'.padEnd(13)}: ${play.album ?? '(None)'}`, + `${'Meta'.padEnd(13)}: ${metaStr}` + ]; + + const final = parts.join('\n'); + return final; +} + export const slice = (str: string, index: number, count: number, add?: string): string => { // We cannot pass negative indexes directly to the 2nd slicing operation. if (index < 0) { -- 2.51.2 From 9c89142c47043c2d78d4aaaa9dfc438cac6e9fb3 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 4 Dec 2025 17:12:22 +0000 Subject: [PATCH 14/41] feat: Improve album artist parsing for musicbrainz data * Change meta to string array * Compare artist ids * Add transformer option to ignore various artists --- .../transforms/MusicbrainzTransformer.ts | 11 ++++++-- .../common/vendor/ListenbrainzApiClient.ts | 12 ++++----- .../musicbrainz/MusicbrainzApiClient.ts | 26 +++++++++++++------ src/backend/sources/JellyfinApiSource.ts | 2 +- src/backend/sources/JellyfinSource.ts | 2 +- src/backend/sources/MPDSource.ts | 2 +- src/core/Atomic.ts | 4 +-- 7 files changed, 38 insertions(+), 21 deletions(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 37c1d7f4..1076fb3a 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -91,6 +91,12 @@ export interface MusicbrainzTransformerData { * */ releaseAllowEmpty?: boolean + + /** Ignore album artist if it is "Various Artists" + * + * @default true + */ + ignoreVA?: boolean } export interface MusicbrainzTransformerDataStrong extends MusicbrainzTransformerData { @@ -140,7 +146,8 @@ export const parseStageConfig = (data: MusicbrainzTransformerData | undefined = releaseStatusDeny: data.releaseStatusAllow !== undefined ? parseArrayFromMaybeString(data.releaseStatusDeny, {lower: true}).map(asMBReleaseStatus) : undefined, releaseStatusPriority: data.releaseStatusAllow !== undefined ? parseArrayFromMaybeString(data.releaseStatusPriority, {lower: true}).map(asMBReleaseStatus) : undefined, - releaseAllowEmpty: data.releaseAllowEmpty + releaseAllowEmpty: data.releaseAllowEmpty, + ignoreVA: data.ignoreVA }; if (data === null || typeof data !== 'object') { @@ -296,7 +303,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { diff --git a/src/backend/common/vendor/ListenbrainzApiClient.ts b/src/backend/common/vendor/ListenbrainzApiClient.ts index 14e13fa9..8ee40ad2 100644 --- a/src/backend/common/vendor/ListenbrainzApiClient.ts +++ b/src/backend/common/vendor/ListenbrainzApiClient.ts @@ -534,7 +534,7 @@ export const listenResponseToPlay = (listen: ListenResponse): PlayObject => { if(primaryArtistMBMapping !== undefined) { // only include as primary if musicbrainz does not disagree with us if(release_artist_names.length === 0 || (release_artist_names.length > 0 && release_artist_names.includes(primaryArtist))) { - brainzMetaRaw.albumArtist = primaryArtistMBMapping.artist_mbid; + brainzMetaRaw.albumArtist = [primaryArtistMBMapping.artist_mbid]; } } @@ -711,11 +711,11 @@ export const playToListenPayload = (play: PlayObject): ListenPayload => { const msAdditionalInfo = brainz.additionalInfo ?? {}; let addInfo: SubmitListenAdditionalTrackInfo = { - // all artists - artist_names: Array.from(new Set([...artists, ...albumArtists])), + // primary artists + artist_names: Array.from(new Set([...artists])), // primary artist - release_artist_name: artists[0], - release_artist_names: [artists[0]], + release_artist_name: albumArtists.length === 1 ? albumArtists[0] : undefined, + release_artist_names: albumArtists.length > 0 ? albumArtists : undefined, // use data from LZ response, if this Play was originally from LZ Source media_player: mediaPlayerName ?? msAdditionalInfo.media_player, media_player_version: mediaPlayerVersion ?? msAdditionalInfo.media_player_version, @@ -737,7 +737,7 @@ export const playToListenPayload = (play: PlayObject): ListenPayload => { } const minTrackData: MinimumTrack = removeUndefinedKeys({ - artist_name: Array.from(new Set([...artists, ...albumArtists])).join(', '), + artist_name: Array.from(new Set([...artists])).join(', '), track_name: track, release_name: al, }); diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index 50394503..31d27868 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -7,7 +7,7 @@ import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, import AbstractApiClient from "../AbstractApiClient.js"; import { isPortReachableConnect, joinedUrl, normalizeWebAddress } from '../../../utils/NetworkUtils.js'; import { MusicBrainzApi, IRecording, ISearchResult, IRecordingList, ISearchQuery, IRecordingMatch, IRelease } from 'musicbrainz-api'; -import { sleep } from "../../../utils.js"; +import { difference, sleep } from "../../../utils.js"; import {SequentialRoundRobin} from 'round-robin-js'; import { Cacheable } from "cacheable"; import { getRoot } from "../../../ioc.js"; @@ -124,19 +124,28 @@ export class MusicbrainzApiClient extends AbstractApiClient { } } -export const recordingToPlay = (data: IRecording): PlayObject => { +export const recordingToPlay = (data: IRecording, options?: {ignoreVA?: boolean}): PlayObject => { + + const { + ignoreVA = true, + } = options || {}; let album: IRelease; let albumArtists: string[]; + let albumArtistIds: string[]; const artists = (data["artist-credit"] ?? []).map(x => x.name); if(data.releases !== undefined && data.releases.length > 0) { album = data.releases[0]; - const aa = album["artist-credit"].map(x => x.name); - // if not every artist of the recording is also on the album - // then use release album artists - if(!artists.every(x => aa.includes(x))) { - albumArtists = aa; + if(album["artist-credit"] !== undefined) { + if(difference(album["artist-credit"].map(x => x.artist.id), (data["artist-credit"] ?? []).map(x => x.artist.id)).length > 0) { + albumArtists = album["artist-credit"].map(x => x.artist.name); + albumArtistIds = album["artist-credit"].map(x => x.artist.id); + } + if(albumArtists !== undefined && ignoreVA && albumArtists.includes('Various Artists')) { + albumArtists = undefined; + albumArtistIds = undefined; + } } } @@ -149,7 +158,8 @@ export const recordingToPlay = (data: IRecording): PlayObject => { meta: { brainz: { track: data.id, - artist: data["artist-credit"].map(x => x.artist.id), + artist: data["artist-credit"] !== undefined ? data["artist-credit"].map(x => x.artist.id) : undefined, + albumArtist: albumArtists, album: album !== undefined ? album.id : undefined, releaseGroup: album !== undefined ? album["release-group"]?.id : undefined } diff --git a/src/backend/sources/JellyfinApiSource.ts b/src/backend/sources/JellyfinApiSource.ts index 6f865dd9..03594be4 100644 --- a/src/backend/sources/JellyfinApiSource.ts +++ b/src/backend/sources/JellyfinApiSource.ts @@ -438,7 +438,7 @@ export default class JellyfinApiSource extends MemoryPositionalSource { meta.artist = [ProviderIds.MusicBrainzArtist]; } if(ProviderIds.MusicBrainzAlbumArtist !== undefined) { - meta.albumArtist = ProviderIds.MusicBrainzAlbumArtist; + meta.albumArtist = [ProviderIds.MusicBrainzAlbumArtist]; } const play: PlayObject = { diff --git a/src/backend/sources/JellyfinSource.ts b/src/backend/sources/JellyfinSource.ts index c50c4c9b..21e5b03a 100644 --- a/src/backend/sources/JellyfinSource.ts +++ b/src/backend/sources/JellyfinSource.ts @@ -168,7 +168,7 @@ export default class JellyfinSource extends MemorySource { brainz: { artist: splitByFirstFound(Provider_musicbrainzartist, [';'], undefined), album: Provider_musicbrainzalbum, - albumArtist: Provider_musicbrainzalbumartist, + albumArtist: [Provider_musicbrainzalbumartist], track: Provider_musicbrainztrack, releaseGroup: Provider_musicbrainzreleasegroup } diff --git a/src/backend/sources/MPDSource.ts b/src/backend/sources/MPDSource.ts index de17de8b..9ddbfa5b 100644 --- a/src/backend/sources/MPDSource.ts +++ b/src/backend/sources/MPDSource.ts @@ -179,7 +179,7 @@ export class MPDSource extends MemoryPositionalSource { } const brainz: BrainzMeta = { - albumArtist: musicbrainz_albumartistid, + albumArtist: [musicbrainz_albumartistid], album: musicbrainz_albumid, track: musicbrainz_trackid, }; diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index e2e570e6..9c7b8b96 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -112,8 +112,8 @@ export interface BrainzMeta { /** * artists_mbid * - * If multiple artists for track this is the "original" artist who is releasing the single/album */ - albumArtist?: string + * If multiple artists for track this is the "original" artist(s) who is releasing the single/album */ + albumArtist?: string[] /** * release_mbid * -- 2.51.2 From dadea4b90c30a20896224aad8d802c205a7fbc68 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 4 Dec 2025 17:22:25 +0000 Subject: [PATCH 15/41] feat(transformer): Implement country filter/priority for musicbrainz --- .../transforms/MusicbrainzTransformer.ts | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 1076fb3a..285c8084 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -84,6 +84,22 @@ export interface MusicbrainzTransformerData { */ releaseStatusPriority?: string[] + /** Allow only releases from these ISO2 countries + * + * @see https://beta.musicbrainz.org/doc/Release/Country + */ + releaseCountryAllow?: string[] + /** Filter out any releases rom these ISO2 countries + * + * @see https://beta.musicbrainz.org/doc/Release/Country + */ + releaseCountryDeny?: string[] + /** Prioritise releases to used based on the order of these ISO2 countries + * + * @see https://beta.musicbrainz.org/doc/Release/Country + */ + releaseCountryPriority?: string[] + /** Do not filter out a recording if it initially has no releases * * Use in conjunction with release filters by setting to `true` @@ -146,6 +162,10 @@ export const parseStageConfig = (data: MusicbrainzTransformerData | undefined = releaseStatusDeny: data.releaseStatusAllow !== undefined ? parseArrayFromMaybeString(data.releaseStatusDeny, {lower: true}).map(asMBReleaseStatus) : undefined, releaseStatusPriority: data.releaseStatusAllow !== undefined ? parseArrayFromMaybeString(data.releaseStatusPriority, {lower: true}).map(asMBReleaseStatus) : undefined, + releaseCountryAllow: data.releaseCountryAllow !== undefined ? parseArrayFromMaybeString(data.releaseCountryAllow, {lower: true}) : undefined, + releaseCountryDeny: data.releaseCountryDeny !== undefined ? parseArrayFromMaybeString(data.releaseCountryDeny, {lower: true}) : undefined, + releaseCountryPriority: data.releaseCountryPriority !== undefined ? parseArrayFromMaybeString(data.releaseCountryPriority, {lower: true}) : undefined, + releaseAllowEmpty: data.releaseAllowEmpty, ignoreVA: data.ignoreVA }; @@ -294,6 +314,8 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer 0) { return releaseStatusAllow.includes(y.status.toLocaleLowerCase() as MBReleaseStatus) } - return !releaseStatusAllow.includes(y.status.toLocaleLowerCase() as MBReleaseStatus) + return !releaseStatusDeny.includes(y.status.toLocaleLowerCase() as MBReleaseStatus) }) } }); @@ -457,13 +479,41 @@ export const filterByValidReleaseGroupSecondary = (list: IRecordingMatch[], stag || x.releases.length > 0); } +export const filterByValidReleaseCountry = (list: IRecordingMatch[], stageConfig: MusicbrainzTransformerDataStage, logger: MaybeLogger = new MaybeLogger()) => { + const { + releaseCountryAllow = [], + releaseCountryDeny = [], + releaseAllowEmpty + } = stageConfig; + if(releaseCountryAllow.length === 0 && releaseCountryDeny.length === 0) { + return list; + } + const releaseFiltered = list.map(x => { + return { + ...x, + releases: x.releases.filter(y => { + if(releaseCountryAllow.length > 0) { + return releaseCountryAllow.includes(y.country?.toLocaleLowerCase()) + } + return !releaseCountryDeny.includes(y.country?.toLocaleLowerCase()) + }) + } + }); + return releaseFiltered.filter(x => ( + list.find(y => y.id === x.id).releases.length === 0 + && releaseAllowEmpty + ) + || x.releases.length > 0); +} + export const rankReleasesByPriority = (list: IRecordingMatch[], stageConfig: MusicbrainzTransformerDataStage, logger: MaybeLogger = new MaybeLogger()) => { const { releaseStatusPriority = [], releaseGroupPrimaryTypePriority = [], - releaseGroupSecondaryTypePriority = [] + releaseGroupSecondaryTypePriority = [], + releaseCountryPriority = [] } = stageConfig; - if(releaseStatusPriority.length === 0 && releaseGroupPrimaryTypePriority.length === 0 && releaseGroupSecondaryTypePriority.length === 0) { + if(releaseStatusPriority.length === 0 && releaseGroupPrimaryTypePriority.length === 0 && releaseGroupSecondaryTypePriority.length === 0 && releaseCountryPriority.length === 0) { return list; } @@ -475,10 +525,10 @@ export const rankReleasesByPriority = (list: IRecordingMatch[], stageConfig: Mus const statAScore = releaseStatusPriority.findIndex(x => x === a.status.toLocaleLowerCase()) + 1; const grpPAScore = releaseGroupPrimaryTypePriority.findIndex(x => x === a["release-group"]["primary-type"].toLocaleLowerCase()) + 1; const grpSAScore = (a["release-group"]["secondary-types"] ?? []).reduce((acc: number, curr: MBReleaseGroupSecondaryType) => acc + releaseGroupSecondaryTypePriority.findIndex(x => x === (curr as MBReleaseGroupSecondaryType).toLocaleLowerCase()) + 1,0); - + const countryAScore = releaseCountryPriority.findIndex(x => x === a.country.toLocaleLowerCase()) + 1; return { ...a, - rankedScore: statAScore + grpPAScore + grpSAScore + rankedScore: statAScore + grpPAScore + grpSAScore + countryAScore }; }) } @@ -486,4 +536,5 @@ export const rankReleasesByPriority = (list: IRecordingMatch[], stageConfig: Mus for(const rec of rankedList) { rec.releases.sort((a, b) => b.rankedScore - a.rankedScore); } + return rankedList; }; \ No newline at end of file -- 2.51.2 From 0d02215f71df0ae5fa1a756c7e2939db372d7e29 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 4 Dec 2025 17:42:58 +0000 Subject: [PATCH 16/41] More forgiving filter/comparison for maybe values --- .../transforms/MusicbrainzTransformer.ts | 22 +++++++++---------- src/core/Atomic.ts | 15 ++++++++----- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 285c8084..28d29e06 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -11,7 +11,7 @@ import { MusicbrainzApiClient, MusicbrainzApiConfig, recordingToPlay } from "../ import { IRecordingList, IRecordingMatch, MusicBrainzApi } from "musicbrainz-api"; import { getRoot, version } from "../../ioc.js"; import { normalizeWebAddress } from "../../utils/NetworkUtils.js"; -import { intersect, missingMbidTypes } from "../../utils.js"; +import { intersect, missingMbidTypes, removeUndefinedKeys } from "../../utils.js"; import { SimpleError } from "../errors/MSErrors.js"; import { parseArrayFromMaybeString } from "../../utils/StringUtils.js"; import clone from "clone"; @@ -310,7 +310,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { if(releaseStatusAllow.length > 0) { - return releaseStatusAllow.includes(y.status.toLocaleLowerCase() as MBReleaseStatus) + return releaseStatusAllow.includes(y.status?.toLocaleLowerCase() as MBReleaseStatus) } - return !releaseStatusDeny.includes(y.status.toLocaleLowerCase() as MBReleaseStatus) + return !releaseStatusDeny.includes(y.status?.toLocaleLowerCase() as MBReleaseStatus) }) } }); @@ -439,9 +439,9 @@ export const filterByValidReleaseGroupPrimary = (list: IRecordingMatch[], stageC ...x, releases: x.releases.filter(y => { if(releaseGroupPrimaryTypeAllow.length > 0) { - return releaseGroupPrimaryTypeAllow.includes(y["release-group"]["primary-type"].toLocaleLowerCase() as MBReleaseGroupPrimaryType) + return releaseGroupPrimaryTypeAllow.includes(y["release-group"]?.["primary-type"]?.toLocaleLowerCase() as MBReleaseGroupPrimaryType) } - return !releaseGroupPrimaryTypeDeny.includes(y["release-group"]["primary-type"].toLocaleLowerCase() as MBReleaseGroupPrimaryType) + return !releaseGroupPrimaryTypeDeny.includes(y["release-group"]?.["primary-type"]?.toLocaleLowerCase() as MBReleaseGroupPrimaryType) }) } }); @@ -466,9 +466,9 @@ export const filterByValidReleaseGroupSecondary = (list: IRecordingMatch[], stag ...x, releases: x.releases.filter(y => { if(releaseGroupSecondaryTypeAllow.length > 0) { - return intersect(releaseGroupSecondaryTypeAllow, (y["release-group"]["secondary-types"] ?? []).map(x => x.toLocaleLowerCase()) as MBReleaseGroupSecondaryType[]).length > 0; + return intersect(releaseGroupSecondaryTypeAllow, (y["release-group"]?.["secondary-types"] ?? []).map(x => x.toLocaleLowerCase()) as MBReleaseGroupSecondaryType[]).length > 0; } - return intersect(releaseGroupSecondaryTypeDeny, (y["release-group"]["secondary-types"] ?? []).map(x => x.toLocaleLowerCase()) as MBReleaseGroupSecondaryType[]).length === 0; + return intersect(releaseGroupSecondaryTypeDeny, (y["release-group"]?.["secondary-types"] ?? []).map(x => x.toLocaleLowerCase()) as MBReleaseGroupSecondaryType[]).length === 0; }) } }); @@ -523,9 +523,9 @@ export const rankReleasesByPriority = (list: IRecordingMatch[], stageConfig: Mus ...x, releases: (x.releases ?? []).map((a) => { const statAScore = releaseStatusPriority.findIndex(x => x === a.status.toLocaleLowerCase()) + 1; - const grpPAScore = releaseGroupPrimaryTypePriority.findIndex(x => x === a["release-group"]["primary-type"].toLocaleLowerCase()) + 1; - const grpSAScore = (a["release-group"]["secondary-types"] ?? []).reduce((acc: number, curr: MBReleaseGroupSecondaryType) => acc + releaseGroupSecondaryTypePriority.findIndex(x => x === (curr as MBReleaseGroupSecondaryType).toLocaleLowerCase()) + 1,0); - const countryAScore = releaseCountryPriority.findIndex(x => x === a.country.toLocaleLowerCase()) + 1; + const grpPAScore = releaseGroupPrimaryTypePriority.findIndex(x => x === a["release-group"]?.["primary-type"]?.toLocaleLowerCase()) + 1; + const grpSAScore = (a["release-group"]?.["secondary-types"] ?? []).reduce((acc: number, curr: MBReleaseGroupSecondaryType) => acc + releaseGroupSecondaryTypePriority.findIndex(x => x === (curr as MBReleaseGroupSecondaryType).toLocaleLowerCase()) + 1,0); + const countryAScore = releaseCountryPriority.findIndex(x => a.country === undefined ? false : x === a.country.toLocaleLowerCase()) + 1; return { ...a, rankedScore: statAScore + grpPAScore + grpSAScore + countryAScore diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 9c7b8b96..a5e2c805 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -414,8 +414,9 @@ export const isMBReleaseStatus = (str: string): str is MBReleaseStatus => { return MB_RELEASE_STATUSES.includes(str as MBReleaseStatus); } export const asMBReleaseStatus = (str: string): MBReleaseStatus => { - if(isMBReleaseStatus(str)) { - return str; + const clean = str.toLocaleLowerCase(); + if(isMBReleaseStatus(clean)) { + return clean; } else { throw new Error(`Release Status is not valid: ${str}`); } @@ -427,8 +428,9 @@ export const isMBReleasePrimaryGroupType = (str: string): str is MBReleaseGroupP return MB_RELEASE_GROUP_PRIMARY_TYPES.includes(str as MBReleaseGroupPrimaryType); } export const asMBReleasePrimaryGroupType = (str: string): MBReleaseGroupPrimaryType => { - if(isMBReleasePrimaryGroupType(str)) { - return str; + const clean = str.toLocaleLowerCase(); + if(isMBReleasePrimaryGroupType(clean)) { + return clean; } else { throw new Error(`Primary Release Group is not valid: ${str}`); } @@ -440,8 +442,9 @@ export const isMBReleaseSecondaryGroupType = (str: string): str is MBReleaseGrou return MB_RELEASE_GROUP_SECONDARY_TYPES.includes(str as MBReleaseGroupSecondaryType); } export const asMBReleaseSecondaryGroupType = (str: string): MBReleaseGroupSecondaryType => { - if(isMBReleaseSecondaryGroupType(str)) { - return str; + const clean = str.toLocaleLowerCase(); + if(isMBReleaseSecondaryGroupType(clean)) { + return clean; } else { throw new Error(`Secondary Release Group is not valid: ${str}`); } -- 2.51.2 From c36ffa5ffd8eca7c63915e721157bcb02a1b74f4 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 4 Dec 2025 17:52:48 +0000 Subject: [PATCH 17/41] more bug fixes --- src/backend/common/transforms/MusicbrainzTransformer.ts | 6 +++--- .../common/vendor/musicbrainz/MusicbrainzApiClient.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 28d29e06..f9d1cc14 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -360,17 +360,17 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { if (parts === false) { - return play.data.artists; + return play.data.albumArtists; } if (typeof parts === 'object') { if (parts.when !== undefined) { if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { this.logger.debug('When condition for albumArtists not met, returning original artists'); - return play.data.artists; + return play.data.albumArtists; } } } - return play.data.albumArtists; + return transformData.data.albumArtists; } protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { if (parts === false) { diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index 31d27868..6761e2f6 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -159,7 +159,7 @@ export const recordingToPlay = (data: IRecording, options?: {ignoreVA?: boolean} brainz: { track: data.id, artist: data["artist-credit"] !== undefined ? data["artist-credit"].map(x => x.artist.id) : undefined, - albumArtist: albumArtists, + albumArtist: albumArtistIds, album: album !== undefined ? album.id : undefined, releaseGroup: album !== undefined ? album["release-group"]?.id : undefined } -- 2.51.2 From 08301d866436dbb486c99116b1fb78a7c75a92be Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 4 Dec 2025 18:00:15 +0000 Subject: [PATCH 18/41] mysteriously deleted interface? --- src/core/Atomic.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index a5e2c805..c6406d93 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -2,6 +2,8 @@ import { LogDataPretty, LogLevel } from "@foxxmd/logging"; import { Dayjs } from "dayjs"; import { ListenProgress } from "../backend/sources/PlayerState/ListenProgress.js"; import { AdditionalTrackInfoResponse } from "../backend/common/vendor/listenbrainz/interfaces.js"; +import { Name } from "ajv"; +import { StringMap } from "ts-json-schema-generator"; export interface SourceStatusData { status: string; @@ -449,3 +451,9 @@ export const asMBReleaseSecondaryGroupType = (str: string): MBReleaseGroupSecond throw new Error(`Secondary Release Group is not valid: ${str}`); } } + +export interface TransformResult { + type: string, + name: string, + play: PlayData +} \ No newline at end of file -- 2.51.2 From 8ba8a054a61f00476bd0276892d2c341e9cc962b Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 4 Dec 2025 20:03:02 +0000 Subject: [PATCH 19/41] docs: Add musicbrainz stage --- .../configuration/transforms/musicbrainz.mdx | 343 ++++++++++++++++++ 1 file changed, 343 insertions(+) create mode 100644 docsite/docs/configuration/transforms/musicbrainz.mdx diff --git a/docsite/docs/configuration/transforms/musicbrainz.mdx b/docsite/docs/configuration/transforms/musicbrainz.mdx new file mode 100644 index 00000000..d5732503 --- /dev/null +++ b/docsite/docs/configuration/transforms/musicbrainz.mdx @@ -0,0 +1,343 @@ +--- +title: Musicbrainz Stage +toc_min_heading_level: 2 +toc_max_heading_level: 5 +--- + +The **Musicbrainz** [Stage](/configuration/transforms#stage) matches your Play data with the [MusicBrainz](https://musicbrainz.org/) database. If the match score is high enough then Multi-Scrobbler uses the match to correct and fill-in missing information in your Play data. + +**This Stage is useful for standardizing your Scrobble's Play data, regardless of the Source it is coming from.** + +Unless you have strong opinions about the way your existing music is already tagged/organized you should consider adding the Musicbrainz Stage to all of your Sources, in some capacity. [Even if it's only MBIDs.](#using-partial-match) + +
+ +Why Should I Prefer MusicBrainz? + +

More Than Mainstream Data

+ +The [MusicBrainz database contains](https://musicbrainz.org/statistics) + +* 37 million+ tracks +* 5 million+ albums +* 2.7 million+ artists + +in over 200 languages, contributed by 10,000+ editors each year with many more users voting on edits. + +The database covers a much wider range of music than most commercial services since it isn't restricted by licensing or incentivized to prioritize mainstream music due to business interests. You are much more likely to find data for obscure, indie, and international artists. + +

As The Artist Intended

+ +While some large services (Spotify) may be more consistent in data structure, the MusicBrainz equivalent data is likely to be more true to the way the Artist originally intended since individuals can devote effort to individual artists. A fan of an artist is much more likely to "get it right" for their favorite band than a corporate business trying to do the least amount of work for the largest result. + +
+ +## Configuration + +### API Setup + +To avoid rate limiting, MusicBrainz requires API users to identify themself using a [User-Agent that contains an email or URL.](https://wiki.musicbrainz.org/MusicBrainz_API/Rate_Limiting#How_can_I_be_a_good_citizen_and_be_smart_about_using_the_Web_Service?) You must set this in the `data` for your [stage configuration](http://localhost:3000/docs/configuration/transforms/#configuring-stages) in your [AIO Config](/configuration?configType=aio#configuration-types): + +```json5 title="config.json" +{ + // ... + "transformers": [ + { + "type": "musicbrainz", + "name": "MyMB", + "data": { + "apis": [ + { + "contact": "contact@mydomain.com" + } + ] + }, + } + ] +} +``` + +### Search + +Available properties for [Stage Configuration](/configuration/transforms#configuring-stages): + +:::note + +**All of the below parameters are optional.** + +If you just want a sensible default for configuraion refer to the [Best Practices](#best-practices) section. + +::: + +#### Score + +Each match returned by MusicBrainz contains a numeric score representing how close it was to the search parameters. Set `score` in configuration to set a minimum score that must be met by matches. Default is `90`. + +```json5 +{ + // ... + "score": 90 // matches must score 90 or higher to be considered +} +``` + +#### Filtering and Sorting + +There are several attributes associated with the [Release](https://wiki.musicbrainz.org/Release) a [Recording](https://wiki.musicbrainz.org/Recording) (individual Track/song) belongs to that you may be interested in controlling. MS can use these attributes to filter what the final Recording selected to match against your Play data is. + +An easy way to think about this: + +* Do you prefer Albums, Singles, or EPs? +* Do you always want to use an official album release, or are bootleg or cancelled albums okay to use? +* Should a track that belongs to a Compilation album be allowed to match? +* What country do you prefer an album release to be from? + +These attributes (`attribute_name` in config) are: + +* [Release **Status**](https://wiki.musicbrainz.org/Release#Status) (`releaseStatus`) - Is this release/album/ep official, promotional, bootleg, etc... +* [Release Group **Primary Type**](https://wiki.musicbrainz.org/Release_Group/Type#Primary_types) (`releaseGroupPrimaryType`) - Is this release an Album, Single, EP, etc... +* [Release Group **Secondary Type**](https://wiki.musicbrainz.org/Release_Group/Type#Secondary_types) (`releaseGroupSecondaryType`) - Is this release a compilation, soundtrack, remix, etc... +* [Release **Country**](https://beta.musicbrainz.org/doc/Release/Country) (`releaseCountry`) - What ISO2 country was this released in? US, GB, MX, etc... + * MusicBrainz uses the special code `XW` to represent a "Worldwide" release AKA release not made specifically for a country + +Each of the above attributes can be used to **filter** matches using `allow` (explicitly include) or `deny` (exclude) terms. The correct property name for each is like so: + +``` +[attribute_name]Allow = [...] +[attribute_name]Deny + +EX to explicitly allow only release status "official" + +"releaseStatusAllow": ["official"] + +EX to exclude compilations + +"releaseGroupSecondaryTypeDeny": ["compilation"] +``` + +##### Sorting + +Each of the above attributes has one additional property that can be used to **rank** releases based on the order of the values you give it. This is the `priority` property. + +*After* matches have been [filtered](#filtering-and-sorting), the remaining matches will have their releases sorted. Release with an attribute that does not match are **not** removed, but they are sorted lower than releases that do match. + +Sorting can be a good alternative to filtering: with filters there is a possibility your filters may eliminate all matches; if you want to ensure that **some** match will be used then sorting can ensure that the **best choice** out of those given will always be used, without accidentally ending up with **no choice**. + +Sorting can be used instead of filters, or in conjunction with filters, it's your choice. + +In configuration: + +``` +[attribute_name]Priority = [...] + +EX to prefer "official" releases over everything else + +"releaseStatusPriority": ["official"] + +EX to prefer albums, then singles, over everything else + +"releaseGroupPrimaryTypePriority": ["album", "single"] +``` + +#### Empty Releases + +Sometimes a Recording may not have any associated Releases. This may be because there is not enough information about the Recording yet, or it was never included on an actual Release. + +You may want to filter/sort the majority of your Play data by releases but allow matching a Recording if it does not have a release, in this instance. To allow this set `releaseAllowEmpty` to `true` in configuration: + +```json5 +{ + // ... + "releaseAllowEmpty": true // don't remove a match during filtering just because it has no releases +} +``` + +### Rules + +Each [Rule](/configuration/transforms#stage-rules) should be either a boolean, specifying if the transformed data should be used for this field, or a [`when` condition](/configuration/transforms#conditional-moditication): + +```json5 +{ + "type": "musicbrainz", + "name": "MyMB" + // ... + "title": false, // will not apply any changes to Play title + "artists": { + "when": {/* ... */}, // will only apply changes to Play artists if "when" is satisfied + /* ... */ + }, + "album": true // will always apply changes to Play album + "meta": true // adds MusicBrainz MBIDs to scrobble data +} +``` + +If a rule is not present then multi-scrobbler defaults it to `true`. + +## Best Practices + +### Sensible Default + +Generally, the Musicbrainz Stage can be used without any of the [optional configuration](#configuration) and you should still see good results from matches. The top [scored](#score) match is, anecdotally, good enough for correcting and filling in surface-level play data like Title and Artist names. + +For a more opinionated match that will mirror what you would expect from data from large music services: + +```json5 +{ + // use official release over anything else + "releaseStatusPriority": ["official"], + // prefer album, then single, then ep + "releaseGroupPrimaryTypePriority": ["album", "single", "ep"], + // prefer worldwide release + "releaseCountryPriority": ["XW"] +} +``` + +### Filter Considerations + +When using your own filters consider: + +* Prefer `deny` over `allow` + * Releases come in all kinds of formats. Since `allow` is explicit you may filter out your desired match without realizing it (correct data except for release type). Or the Musicbrainz data for a higher scored match may be appropriate but you did not include it, exhaustively. +* Prefer [Sorting](#sorting) over [Filtering](#filtering-and-sorting) + * Sorting does not eliminate any matches. It is, generally, better to get **some** match than it is to have your Play data completely uncorrected because filtering eliminated all matches + +### Using Partial Match + +Use [Rules](#rules) to apply MusicBrainz match data selectively. + +If you know that your music collection is well organized and you do not want to change the artists/title/album etc... sent to a [Client](/configuration/clients), you can still benefit from matches by only applying MBIDs using `meta` so that any Client that supports Musicbrainz data ([Koito](/configuration/clients/koito), [Tealfm](/configuration/clients/tealfm), [Listenbrainz](/configuration/clients/listenbrainz), [Rocksky](/configuration/clients/rocksky)) can still get that data. + +
+ +Example + +```json5 title="subsonic.json" +[ + { + "name": "MySubsonic", + "data": { /* ... */}, + "options": { + "playTransform": { + "preCompare": [ + { + "type": "musicbrainz", + "name": "MyMB", + "title": false, + "artists": false, + "album": false, + "meta": true + } + ] + } + } + } +] +``` + +
+ +## Examples + +### Minimal + +
+ +Example + +Your [AIO Config](/configuration?configType=aio#configuration-types): + +```json5 title="config.json" +{ + // ... + "transformers": [ + { + "type": "musicbrainz", + "name": "MyMB", + "data": { + "apis": [ + { + "contact": "contact@mydomain.com" + } + ] + }, + } + ] +} +``` + +In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types): + +```json5 title="subsonic.json" +[ + { + "name": "MySubsonic", + "data": { /* ... */}, + "options": { + "playTransform": { + "preCompare": [ + { + "type": "musicbrainz", + "name": "MyMB" + } + ] + } + } + } +] +``` +
+ +### Sensible Default + +Using the config shown in [Sensible Default](#sensible-default). + +
+ +Example + +Your [AIO Config](/configuration?configType=aio#configuration-types): + +```json5 title="config.json" +{ + // ... + "transformers": [ + { + "type": "musicbrainz", + "name": "MyMB", + "data": { + "apis": [ + { + "contact": "contact@mydomain.com" + } + ] + }, + "defaults": { + "releaseStatusPriority": ["official"], + "releaseGroupPrimaryTypePriority": ["album", "single", "ep"], + "releaseCountryPriority": ["XW"] + } + } + ] +} +``` + +In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types): + +```json5 title="subsonic.json" +[ + { + "name": "MySubsonic", + "data": { /* ... */}, + "options": { + "playTransform": { + "preCompare": [ + { + "type": "musicbrainz", + "name": "MyMB" + } + ] + } + } + } +] +``` +
\ No newline at end of file -- 2.51.2 From 2bcd0aa858445e7ef286527937f56742528a9a6a Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 4 Dec 2025 20:38:57 +0000 Subject: [PATCH 20/41] test commit for netlify --- docsite/docs/configuration/transforms/musicbrainz.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/docs/configuration/transforms/musicbrainz.mdx b/docsite/docs/configuration/transforms/musicbrainz.mdx index d5732503..3217823b 100644 --- a/docsite/docs/configuration/transforms/musicbrainz.mdx +++ b/docsite/docs/configuration/transforms/musicbrainz.mdx @@ -6,7 +6,7 @@ toc_max_heading_level: 5 The **Musicbrainz** [Stage](/configuration/transforms#stage) matches your Play data with the [MusicBrainz](https://musicbrainz.org/) database. If the match score is high enough then Multi-Scrobbler uses the match to correct and fill-in missing information in your Play data. -**This Stage is useful for standardizing your Scrobble's Play data, regardless of the Source it is coming from.** +**This Stage is useful for standardizing your Scrobble's Play data, regardless of the Source being used** Unless you have strong opinions about the way your existing music is already tagged/organized you should consider adding the Musicbrainz Stage to all of your Sources, in some capacity. [Even if it's only MBIDs.](#using-partial-match) -- 2.51.2 From 949c1f91768e6b4360d82367ccb0c88f9d2053d6 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 4 Dec 2025 20:42:58 +0000 Subject: [PATCH 21/41] revert for netlify test --- docsite/docs/configuration/transforms/musicbrainz.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/docs/configuration/transforms/musicbrainz.mdx b/docsite/docs/configuration/transforms/musicbrainz.mdx index 3217823b..d5732503 100644 --- a/docsite/docs/configuration/transforms/musicbrainz.mdx +++ b/docsite/docs/configuration/transforms/musicbrainz.mdx @@ -6,7 +6,7 @@ toc_max_heading_level: 5 The **Musicbrainz** [Stage](/configuration/transforms#stage) matches your Play data with the [MusicBrainz](https://musicbrainz.org/) database. If the match score is high enough then Multi-Scrobbler uses the match to correct and fill-in missing information in your Play data. -**This Stage is useful for standardizing your Scrobble's Play data, regardless of the Source being used** +**This Stage is useful for standardizing your Scrobble's Play data, regardless of the Source it is coming from.** Unless you have strong opinions about the way your existing music is already tagged/organized you should consider adding the Musicbrainz Stage to all of your Sources, in some capacity. [Even if it's only MBIDs.](#using-partial-match) -- 2.51.2 From 3e57b006d22492175b3560cea79dd56f677a5345 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Dec 2025 16:32:41 +0000 Subject: [PATCH 22/41] fix(transformer): Escape lucene specia characters when searching musicbrainz https://github.com/FoxxMD/multi-scrobbler/pull/406#issuecomment-3615254005 --- src/backend/common/infrastructure/Atomic.ts | 3 +- .../common/transforms/AbstractTransformer.ts | 4 +- .../transforms/MusicbrainzTransformer.ts | 6 +- .../musicbrainz/MusicbrainzApiClient.ts | 91 ++++++++++++++++--- src/backend/utils/StringUtils.ts | 7 +- src/core/Atomic.ts | 11 ++- 6 files changed, 100 insertions(+), 22 deletions(-) diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 61780e9b..5161838d 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -339,7 +339,8 @@ export interface CacheConfigOptions { export interface MusicbrainzApiConfigData { url?: string rateLimit?: number - contact: string + contact: string, + ttl?: string } export const MUSICBRAINZ_URL = 'https://musicbrainz.org'; diff --git a/src/backend/common/transforms/AbstractTransformer.ts b/src/backend/common/transforms/AbstractTransformer.ts index ab33f1bb..8061c0ef 100644 --- a/src/backend/common/transforms/AbstractTransformer.ts +++ b/src/backend/common/transforms/AbstractTransformer.ts @@ -72,7 +72,7 @@ export default abstract class AbstractTransformer { const clean = str.trim().toLocaleLowerCase(); @@ -200,9 +201,11 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { @@ -231,6 +234,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer; + protected rrApis: SequentialRoundRobin; protected url: URLData; cache: Cacheable; constructor(name: any, config: MusicbrainzApiClientConfig, options: AbstractApiOptions & {cache?: Cacheable}) { super('Musicbrainz', name, config, options); - this.apis = config.apis.map(x => x.api); - this.rrApis = new SequentialRoundRobin(this.apis); + this.rrApis = new SequentialRoundRobin(this.config.apis); this.cache = options.cache ?? getRoot().items.cache().cacheMetadata; this.logger.debug(`Round Robin API calls using hosts: ${config.apis.map(x => x.url ?? MUSICBRAINZ_URL).join(' | ')}`); } - callApi = async (func: (mb: MusicBrainzApi) => Promise, options?: { timeout?: number }): Promise => { + callApi = async (func: (mb: MusicBrainzApi) => Promise, options?: { timeout?: number, ttl?: string, cacheKey?: string }): Promise => { + + const apiConfig = this.rrApis.next().value; + const { - timeout = 30000 + timeout = 30000, + ttl = apiConfig.ttl ?? '1hr', + cacheKey } = options || {}; try { const res = await Promise.race([ - func(this.rrApis.next().value), + func(apiConfig.api), sleep(timeout) ]); if (res === undefined) { @@ -63,6 +73,9 @@ export class MusicbrainzApiClient extends AbstractApiClient { if(`error` in res) { throw Error(res.error); } + if(cacheKey !== undefined) { + await this.cache.set(cacheKey, res, ttl); + } return res as T; } catch (e) { if(e.message.includes('Timeout occurred')) { @@ -72,7 +85,7 @@ export class MusicbrainzApiClient extends AbstractApiClient { } } - searchByRecording = async(play: PlayObject): Promise => { + searchByRecording = async(play: PlayObject, options?: SearchOptions): Promise => { const cacheKey = `mb-recSearch-${hashObject(playContentInvariantTransform(play))}`; @@ -86,24 +99,49 @@ export class MusicbrainzApiClient extends AbstractApiClient { this.logger.warn(new Error('Could not fetch cache key', {cause: e})); } + const { + escapeCharacters = true, + removeCharacters = false, + using = ['album','artist','title'], + ttl + } = options || {}; this.logger.debug(`Starting search`); + // https://github.com/Borewit/musicbrainz-api?tab=readme-ov-file#search-function + // https://wiki.musicbrainz.org/MusicBrainz_API/Search#Recording + // https://beta.musicbrainz.org/doc/MusicBrainz_API/Search const res = await this.callApi((mb) => { const query: Record = { - recording: play.data.track - }; - if(play.data.artists !== undefined && play.data.artists.length > 0) { + }; + + + if(using.includes('title')) { + query.recording = play.data.track; + } + if(play.data.artists !== undefined && play.data.artists.length > 0 && using.includes('artist')) { query.artist = play.data.artists[0]; } - if(play.data.album !== undefined) { + if(play.data.album !== undefined && using.includes('album')) { query.release = play.data.album; } + if(escapeCharacters) { + for(const [k,v] of Object.entries(query)) { + query[k] = escapeLuceneSpecialChars(v); + } + } + if(removeCharacters) { + for(const [k,v] of Object.entries(query)) { + query[k] = removeNonWordCharacters(v); + } + } return mb.search('recording', { query }); + }, { + ttl, + cacheKey }); - await this.cache.set(cacheKey, res, '1hr'); return res; } @@ -172,4 +210,31 @@ export const recordingToPlay = (data: IRecording, options?: {ignoreVA?: boolean} } return play; +} + + +export const LUCENE_SPECIAL_CHARACTER_REGEX: string[] = ['\\','+','-','&&','||','!','(',')','{','}','[',']','^','"','~','*','?',':','/']; +/** + * https://lucene.apache.org/core/7_7_2/queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package.description + * https://beta.musicbrainz.org/doc/MusicBrainz_API/Search + * */ +export const escapeLuceneSpecialChars = (str: string): string => { + let cleaned = str; + for(const char of LUCENE_SPECIAL_CHARACTER_REGEX) { + cleaned = cleaned.replaceAll(char, `\\$&`); + } + return cleaned; +} + +const NON_WORD_ADJACENT_BOUNDARY_REGEX: RegExp = new RegExp(/\w([^a-zA-Z\d\s])\w/g); +const NON_WORDWHITESPACE_REGEX: RegExp = new RegExp(/[^a-zA-Z\d\s]/g); +export const removeNonWordCharacters = (str: string): string => { + // replace any non-alphanumeric, non-whitespace characters that are surrounded by non-whitespace characters + // with a whitespace EX "My Cool-Fun Title" => "My Cool Fun Title" + let cleaned = str.replaceAll(NON_WORD_ADJACENT_BOUNDARY_REGEX, ' '); + + // remove any non-alphanumeric, non-whitespace characters + // with a whitespace EX "My Cool (Title)" => "My Cool Title" + cleaned = str.replaceAll(NON_WORDWHITESPACE_REGEX, ''); + return cleaned; } \ No newline at end of file diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts index 92c7f7e8..0e3783f3 100644 --- a/src/backend/utils/StringUtils.ts +++ b/src/backend/utils/StringUtils.ts @@ -425,4 +425,9 @@ export const normalizeListenbrainzUrl = (urlVal: string): string | undefined => type HashFunction = (obj: object) => string; const defaultHasher = hasher(); const defaultHashFunc: HashFunction = (obj) => defaultHasher.hash(obj); -export const hashObject = (obj: object, h: HashFunction = defaultHashFunc): string => h(obj); \ No newline at end of file +export const hashObject = (obj: object, h: HashFunction = defaultHashFunc): string => h(obj); + +const NON_ALPHANUMWHITESPACE_CHARS: RegExp = new RegExp(/[^a-zA-Z\d\s]/); +export const hasNonAlphanumericChars = (str: string): boolean => { + return NON_ALPHANUMWHITESPACE_CHARS.test(str); +} \ No newline at end of file diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index c6406d93..055bc256 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -392,15 +392,18 @@ export const JOINERS_FINAL: FinalJoiners[] = ['&']; export type Feat = 'ft' | 'feat' | 'vs' | 'ft.' | 'feat.' | 'vs.' | 'featuring' export const FEAT: Feat[] = ['ft','feat','vs','ft.','feat.','vs.','featuring']; + +export interface TransformOptions { + failOnFetch?: boolean; + throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album')[]; + ttl?: string +} export interface TransformerCommonConfig, Y = Record> { defaults?: T; data?: Y type: string; name?: string; - options?: { - failOnFetch?: boolean; - throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album')[]; - }; + options?: TransformOptions } export interface TransformerCommon, Y = Record> extends TransformerCommonConfig { -- 2.51.2 From fa6cc13c864e4fe45e37be7ea84a8352fd61a475 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Dec 2025 17:09:24 +0000 Subject: [PATCH 23/41] tests: Add edge case musicbrainz tests Need to have MB_TEST=true in environment to run them --- .../transforms/MusicbrainzTransformer.ts | 12 +- .../musicbrainz/MusicbrainzApiClient.ts | 51 ++++++--- src/backend/tests/musicbrainz/badapple.json | 36 ++++++ .../tests/musicbrainz/musicbrainz.test.ts | 108 ++++++++++++++++++ 4 files changed, 191 insertions(+), 16 deletions(-) create mode 100644 src/backend/tests/musicbrainz/badapple.json create mode 100644 src/backend/tests/musicbrainz/musicbrainz.test.ts diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 4c0db20a..abf19b45 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -287,8 +287,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { - // TODO maybe search more broadly if first query doesn't hit? - const results = await this.api.searchByRecording(play); + let results = await this.api.searchByRecording(play); if(results === undefined) { throw new Error('results were unexpectedly undefined! API should have thrown...'); @@ -298,6 +297,15 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer 0) { + this.logger.debug('No matches found, trying search with only track+album'); + results = await this.api.searchByRecording(play, {using: ['title','album']}); + } + } + return results; } diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index 56841ad9..30505a0b 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -87,7 +87,14 @@ export class MusicbrainzApiClient extends AbstractApiClient { searchByRecording = async(play: PlayObject, options?: SearchOptions): Promise => { - const cacheKey = `mb-recSearch-${hashObject(playContentInvariantTransform(play))}`; + const { + escapeCharacters = true, + removeCharacters = false, + using = ['album','artist','title'], + ttl + } = options || {}; + + const cacheKey = `mb-recSearch-${hashObject({...playContentInvariantTransform(play), using})}`; try { const cachedTransform = await this.cache.get(cacheKey); @@ -99,43 +106,59 @@ export class MusicbrainzApiClient extends AbstractApiClient { this.logger.warn(new Error('Could not fetch cache key', {cause: e})); } - const { - escapeCharacters = true, - removeCharacters = false, - using = ['album','artist','title'], - ttl - } = options || {}; - this.logger.debug(`Starting search`); // https://github.com/Borewit/musicbrainz-api?tab=readme-ov-file#search-function // https://wiki.musicbrainz.org/MusicBrainz_API/Search#Recording // https://beta.musicbrainz.org/doc/MusicBrainz_API/Search const res = await this.callApi((mb) => { const query: Record = { - }; - + }; if(using.includes('title')) { query.recording = play.data.track; } if(play.data.artists !== undefined && play.data.artists.length > 0 && using.includes('artist')) { - query.artist = play.data.artists[0]; + query.artist = play.data.artists; } if(play.data.album !== undefined && using.includes('album')) { query.release = play.data.album; } if(escapeCharacters) { for(const [k,v] of Object.entries(query)) { - query[k] = escapeLuceneSpecialChars(v); + query[k] = Array.isArray(v) ? v.map(escapeLuceneSpecialChars) : escapeLuceneSpecialChars(v); } } if(removeCharacters) { for(const [k,v] of Object.entries(query)) { - query[k] = removeNonWordCharacters(v); + query[k] = Array.isArray(v) ? v.map(removeNonWordCharacters) : removeNonWordCharacters(v); + } + } + + let q = ''; + + if(query.recording !== undefined) { + q += `recording:"${query.recording}"`; + } + if(query.artist !== undefined) { + if(q !== '') { + q += ' AND '; + } + if(Array.isArray(query.artist)) { + q += `artist:(${query.artist.map(x => `"${x}"`).join(' OR ')})` + } else { + q += `artist:"${query.artist}"`; } } + if(query.release !== undefined) { + if(q !== '') { + q += ' AND '; + } + q += `release:"${query.release}"` + } + this.logger.debug(`Search Query => ${q}`); + return mb.search('recording', { - query + query: q }); }, { ttl, diff --git a/src/backend/tests/musicbrainz/badapple.json b/src/backend/tests/musicbrainz/badapple.json new file mode 100644 index 00000000..87b3baa1 --- /dev/null +++ b/src/backend/tests/musicbrainz/badapple.json @@ -0,0 +1,36 @@ +{ + "artist": { + "mbid": "", + "#text": "25時、ナイトコードで。" + }, + "streamable": "0", + "image": [ + { + "size": "small", + "#text": "https:\/\/lastfm.freetls.fastly.net\/i\/u\/34s\/545a925a5de1944f7a6ce0b703c94ffa.jpg" + }, + { + "size": "medium", + "#text": "https:\/\/lastfm.freetls.fastly.net\/i\/u\/64s\/545a925a5de1944f7a6ce0b703c94ffa.jpg" + }, + { + "size": "large", + "#text": "https:\/\/lastfm.freetls.fastly.net\/i\/u\/174s\/545a925a5de1944f7a6ce0b703c94ffa.jpg" + }, + { + "size": "extralarge", + "#text": "https:\/\/lastfm.freetls.fastly.net\/i\/u\/300x300\/545a925a5de1944f7a6ce0b703c94ffa.jpg" + } + ], + "mbid": "", + "album": { + "mbid": "088bddaf-f1c9-4bb8-b4e1-8fed9df14d64", + "#text": "25時、ナイトコードで。 SEKAI ALBUM Vol.3" + }, + "name": "Bad Apple!! feat.SEKAI", + "url": "https:\/\/www.last.fm\/music\/25%E6%99%82%E3%80%81%E3%83%8A%E3%82%A4%E3%83%88%E3%82%B3%E3%83%BC%E3%83%89%E3%81%A7%E3%80%82\/_\/Bad+Apple!!+feat.SEKAI", + "date": { + "uts": "1764906912", + "#text": "05 Dec 2025, 03:55" + } +} \ No newline at end of file diff --git a/src/backend/tests/musicbrainz/musicbrainz.test.ts b/src/backend/tests/musicbrainz/musicbrainz.test.ts new file mode 100644 index 00000000..0393e4e0 --- /dev/null +++ b/src/backend/tests/musicbrainz/musicbrainz.test.ts @@ -0,0 +1,108 @@ +import * as dotenv from 'dotenv'; +import { loggerTest, loggerDebug, childLogger } from "@foxxmd/logging"; +import chai, { assert, expect } from 'chai'; +import asPromised from 'chai-as-promised'; +import { after, before, describe, it } from 'mocha'; +import { initMemoryCache } from "../../common/Cache.js"; +import { Cacheable } from "cacheable"; +import LastfmApiClient from "../../common/vendor/LastfmApiClient.js"; +import badapple from './badapple.json' with { type: "json" }; +import { TrackObject } from "lastfm-node-client"; +import MusicbrainzTransformer from "../../common/transforms/MusicbrainzTransformer.js"; +import { PlayObject } from "../../../core/Atomic.js"; +import { projectDir } from '../../common/index.js'; +import path from 'path'; + +const envPath = path.join(projectDir, '.env'); +dotenv.config({path: envPath}); + +const memorycache = () => new Cacheable({ primary: initMemoryCache({ ttl: '1ms' }) }); + +const mbTransformer = new MusicbrainzTransformer({ + name: 'test', + type: 'musicbrainz', + data: { + apis: [ + { + contact: 'contact@foxxmd.dev', + ttl: '1ms' + } + ] + }, + options: { + ttl: '1ms' + } +}, { + logger: loggerTest, + clientCache: memorycache(), + cache: memorycache() +}) + +describe('Musicbrainz API', function () { + + before(function () { + if(process.env.MB_TEST !== 'true') { + this.skip(); + } + }); + + it('escapes lucene special characters', async function () { + + const play: PlayObject = { + data: { + track: 'Cyber Space (CrossWorlds Remix): Final Lap (No Chants)', + album: "Sonic Racing: CrossWorlds Original Soundtrack - Echoes of Dimensions", + artists: [ + "Kanon Oguni" + ] + }, + meta: { + source: "Lastfm", + url: { + web: "https://www.last.fm/music/Kanon+Oguni/_/Cyber+Space+(CrossWorlds+Remix):+Final+Lap+-+No+Chants", + }, + } + }; + await mbTransformer.tryInitialize(); + + const res = await mbTransformer.getTransformerData(play, { + type: "musicbrainz", + searchWhenMissing: ["artists", "album", "title"] + }); + expect(res.recordings).to.exist; + expect(res.recordings).to.not.be.empty; + }); + + it('tries second query using only track and album', async function () { + + const play: PlayObject = { + data: { + track: "Roulette Road (CrossWorlds Remix)", + artists: ["Takahiro Kai, SEGA GAME MUSIC & SEGA SOUND TEAM"], + album: "Sonic Racing: CrossWorlds Original Soundtrack - Echoes of Dimensions" + }, + meta: {} + } + await mbTransformer.tryInitialize(); + + const res = await mbTransformer.getTransformerData(play, { + type: "musicbrainz", + searchWhenMissing: ["artists", "album", "title"] + }); + expect(res.recordings).to.exist; + expect(res.recordings).to.not.be.empty; + }); + + // it('handles non-ascii', async function () { + // const play = LastfmApiClient.formatPlayObj(badapple as unknown as TrackObject); + // await mbTransformer.tryInitialize(); + + // const res = await mbTransformer.getTransformerData(play, { + // type: "musicbrainz", + // searchWhenMissing: ["artists", "album", "title"] + // }); + // expect(res.recordings).to.exist; + // expect(res.recordings).to.not.be.empty; + // }); + +}); \ No newline at end of file -- 2.51.2 From aeae449abf31b26a5d737918e91a0f7eb46a8575 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Dec 2025 17:26:04 +0000 Subject: [PATCH 24/41] test(transformer): Add working non-ascii example --- src/backend/tests/musicbrainz/badapple.json | 36 ------------------- .../tests/musicbrainz/musicbrainz.test.ts | 27 ++++++++------ 2 files changed, 17 insertions(+), 46 deletions(-) delete mode 100644 src/backend/tests/musicbrainz/badapple.json diff --git a/src/backend/tests/musicbrainz/badapple.json b/src/backend/tests/musicbrainz/badapple.json deleted file mode 100644 index 87b3baa1..00000000 --- a/src/backend/tests/musicbrainz/badapple.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "artist": { - "mbid": "", - "#text": "25時、ナイトコードで。" - }, - "streamable": "0", - "image": [ - { - "size": "small", - "#text": "https:\/\/lastfm.freetls.fastly.net\/i\/u\/34s\/545a925a5de1944f7a6ce0b703c94ffa.jpg" - }, - { - "size": "medium", - "#text": "https:\/\/lastfm.freetls.fastly.net\/i\/u\/64s\/545a925a5de1944f7a6ce0b703c94ffa.jpg" - }, - { - "size": "large", - "#text": "https:\/\/lastfm.freetls.fastly.net\/i\/u\/174s\/545a925a5de1944f7a6ce0b703c94ffa.jpg" - }, - { - "size": "extralarge", - "#text": "https:\/\/lastfm.freetls.fastly.net\/i\/u\/300x300\/545a925a5de1944f7a6ce0b703c94ffa.jpg" - } - ], - "mbid": "", - "album": { - "mbid": "088bddaf-f1c9-4bb8-b4e1-8fed9df14d64", - "#text": "25時、ナイトコードで。 SEKAI ALBUM Vol.3" - }, - "name": "Bad Apple!! feat.SEKAI", - "url": "https:\/\/www.last.fm\/music\/25%E6%99%82%E3%80%81%E3%83%8A%E3%82%A4%E3%83%88%E3%82%B3%E3%83%BC%E3%83%89%E3%81%A7%E3%80%82\/_\/Bad+Apple!!+feat.SEKAI", - "date": { - "uts": "1764906912", - "#text": "05 Dec 2025, 03:55" - } -} \ No newline at end of file diff --git a/src/backend/tests/musicbrainz/musicbrainz.test.ts b/src/backend/tests/musicbrainz/musicbrainz.test.ts index 0393e4e0..499edd27 100644 --- a/src/backend/tests/musicbrainz/musicbrainz.test.ts +++ b/src/backend/tests/musicbrainz/musicbrainz.test.ts @@ -93,16 +93,23 @@ describe('Musicbrainz API', function () { expect(res.recordings).to.not.be.empty; }); - // it('handles non-ascii', async function () { - // const play = LastfmApiClient.formatPlayObj(badapple as unknown as TrackObject); - // await mbTransformer.tryInitialize(); + it('handles non-ascii', async function () { + const play: PlayObject = { + data: { + track: "Bad Apple!! feat.SEKAI", + artists: ["、ナイトコードで。"], + album: "25時、ナイトコードで。 SEKAI ALBUM Vol.3" + }, + meta: {} + } + await mbTransformer.tryInitialize(); - // const res = await mbTransformer.getTransformerData(play, { - // type: "musicbrainz", - // searchWhenMissing: ["artists", "album", "title"] - // }); - // expect(res.recordings).to.exist; - // expect(res.recordings).to.not.be.empty; - // }); + const res = await mbTransformer.getTransformerData(play, { + type: "musicbrainz", + searchWhenMissing: ["artists", "album", "title"] + }); + expect(res.recordings).to.exist; + expect(res.recordings).to.not.be.empty; + }); }); \ No newline at end of file -- 2.51.2 From 0972a419623b9abb5fe5fee0618f166125d8e147 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Dec 2025 17:51:05 +0000 Subject: [PATCH 25/41] feat(transformer): Try naive split on artist for additional search --- src/backend/common/infrastructure/Atomic.ts | 1 + .../transforms/MusicbrainzTransformer.ts | 15 ++++++++++++ src/backend/sources/LastfmSource.ts | 12 +++++++++- .../tests/musicbrainz/musicbrainz.test.ts | 24 +++++++++++++++---- src/core/StringUtils.ts | 18 ++++++++++++++ 5 files changed, 65 insertions(+), 5 deletions(-) diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 5161838d..43d4d803 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -269,6 +269,7 @@ export interface numberFormatOptions { export const DELIMITERS = [',','&','/','\\']; export const DELIMITERS_NO_AMP = [',','/','\\']; +export const DELIMETERS_REGEX: RegExp = new RegExp(/[,&\/\\]/); export const ARTIST_WEIGHT = 0.3; export const TITLE_WEIGHT = 0.4; diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index abf19b45..815a4607 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -16,6 +16,7 @@ import { SimpleError } from "../errors/MSErrors.js"; import { parseArrayFromMaybeString } from "../../utils/StringUtils.js"; import clone from "clone"; import { Cacheable } from "cacheable"; +import { splitByFirstRegexFound } from "../../../core/StringUtils.js"; export const asMissingMbid = (str: string): MissingMbidType => { const clean = str.trim().toLocaleLowerCase(); @@ -304,6 +305,20 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer x.trim()); + if(naiveSplit.length > 1) { + this.logger.debug('No matches found, trying search with track + first value from artist string split'); + results = await this.api.searchByRecording({...play, data: { + ...play.data, + artists: [naiveSplit[0]] + }}, {using: ['title','artist']}); + } + } } return results; diff --git a/src/backend/sources/LastfmSource.ts b/src/backend/sources/LastfmSource.ts index 86ea52da..6df986e2 100644 --- a/src/backend/sources/LastfmSource.ts +++ b/src/backend/sources/LastfmSource.ts @@ -65,7 +65,17 @@ export default class LastfmSource extends MemorySource { } doAuthentication = async () => { try { - return await this.api.testAuth(); + await this.api.testAuth(); + + //5e6f41d4-9021-4d5d-b4a6-e39be60abac5 + const resp = await this.api.callApi((client: any) => client.trackGetInfo({ + //mbid: '5e6f41d4-9021-4d5d-b4a6-e39be60abac5', + artist: "Bowling For Soup & Punk Rock Factory", + track: "Endless Possibility (feat. Wheatus)", + sk: this.api.client.sessionKey + })); + const play = LastfmApiClient.formatPlayObj(resp.track); + return true; } catch (e) { throw e; } diff --git a/src/backend/tests/musicbrainz/musicbrainz.test.ts b/src/backend/tests/musicbrainz/musicbrainz.test.ts index 499edd27..88465acb 100644 --- a/src/backend/tests/musicbrainz/musicbrainz.test.ts +++ b/src/backend/tests/musicbrainz/musicbrainz.test.ts @@ -5,9 +5,6 @@ import asPromised from 'chai-as-promised'; import { after, before, describe, it } from 'mocha'; import { initMemoryCache } from "../../common/Cache.js"; import { Cacheable } from "cacheable"; -import LastfmApiClient from "../../common/vendor/LastfmApiClient.js"; -import badapple from './badapple.json' with { type: "json" }; -import { TrackObject } from "lastfm-node-client"; import MusicbrainzTransformer from "../../common/transforms/MusicbrainzTransformer.js"; import { PlayObject } from "../../../core/Atomic.js"; import { projectDir } from '../../common/index.js'; @@ -33,7 +30,7 @@ const mbTransformer = new MusicbrainzTransformer({ ttl: '1ms' } }, { - logger: loggerTest, + logger: loggerDebug, clientCache: memorycache(), cache: memorycache() }) @@ -93,6 +90,25 @@ describe('Musicbrainz API', function () { expect(res.recordings).to.not.be.empty; }); + it('tries additional query using only track and naively split artist', async function () { + + const play: PlayObject = { + data: { + track: "Endless Possibility (feat. Wheatus)", + artists: ["Bowling For Soup & Punk Rock Factory"], + }, + meta: {} + } + await mbTransformer.tryInitialize(); + + const res = await mbTransformer.getTransformerData(play, { + type: "musicbrainz", + searchWhenMissing: ["artists", "album", "title"] + }); + expect(res.recordings).to.exist; + expect(res.recordings).to.not.be.empty; + }); + it('handles non-ascii', async function () { const play: PlayObject = { data: { diff --git a/src/core/StringUtils.ts b/src/core/StringUtils.ts index 2d561c29..93486b89 100644 --- a/src/core/StringUtils.ts +++ b/src/core/StringUtils.ts @@ -13,6 +13,8 @@ import { ScrobbleTsSOC, TrackStringOptions } from "./Atomic.js"; +import { DELIMETERS_REGEX } from "../backend/common/infrastructure/Atomic.js"; +import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; dayjs.extend(utc) dayjs.extend(isBetween); @@ -208,6 +210,22 @@ export const splitByFirstFound = (str: any, delims = [','], onNotAStringVal: return [str]; } +/** + * Split a string-ish variable by a regex and return the first actually split array or default to returning the string as the first element. + * + * Returns empty array, or user defined value, if variable is undefined / null / not a string / or an empty string. + * */ +export const splitByFirstRegexFound = (str: any, onNotAStringVal: T, delimsReg: RegExp = DELIMETERS_REGEX): string[] | T => { + if (str === undefined || str === null || typeof str !== 'string' || str.trim() === '') { + return onNotAStringVal; + } + const res = parseRegexSingle(delimsReg, str); + if (res !== undefined) { + return [str.slice(0, res.index - 1), str.slice(res.index + 1)]; + } + return [str]; +} + /** * Returns value if it is a non-empty string or returns default value * */ -- 2.51.2 From c07b572abb00a1fa8993dde2cf8262db2d16db14 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Dec 2025 18:30:11 +0000 Subject: [PATCH 26/41] feat(transformer): Allow fallback musicbrainz search to use aggressive native parsing --- .../transforms/MusicbrainzTransformer.ts | 60 +++++++---- .../common/transforms/NativeTransformer.ts | 100 +++++++++++------- .../musicbrainz/MusicbrainzApiClient.ts | 6 +- .../tests/musicbrainz/musicbrainz.test.ts | 7 +- 4 files changed, 113 insertions(+), 60 deletions(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 815a4607..6c046720 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -4,7 +4,7 @@ import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; import { ExternalMetadataTerm, PlayTransformMetadataStage } from "../infrastructure/Transform.js"; import AtomicPartsTransformer from "./AtomicPartsTransformer.js"; import { TransformerOptions } from "./AbstractTransformer.js"; -import { MUSICBRAINZ_URL, MusicbrainzApiConfigData } from "../infrastructure/Atomic.js"; +import { DELIMITERS, MUSICBRAINZ_URL, MusicbrainzApiConfigData } from "../infrastructure/Atomic.js"; import { MaybeLogger } from "../logging.js"; import { childLogger, Logger } from "@foxxmd/logging"; import { MusicbrainzApiClient, MusicbrainzApiConfig, recordingToPlay } from "../vendor/musicbrainz/MusicbrainzApiClient.js"; @@ -17,6 +17,7 @@ import { parseArrayFromMaybeString } from "../../utils/StringUtils.js"; import clone from "clone"; import { Cacheable } from "cacheable"; import { splitByFirstRegexFound } from "../../../core/StringUtils.js"; +import { nativeParse } from "./NativeTransformer.js"; export const asMissingMbid = (str: string): MissingMbidType => { const clean = str.trim().toLocaleLowerCase(); @@ -37,6 +38,13 @@ export interface MusicbrainzTransformerData { searchWhenMissing?: MissingMbidType[] forceSearch?: boolean score?: number + fallbackArtistSearch?: ('naive' | 'native') + + /** Ignore album artist if it is "Various Artists" + * + * @default true + */ + ignoreVA?: boolean /** Allow only releases with release groups with these primary types * @@ -109,12 +117,6 @@ export interface MusicbrainzTransformerData { * */ releaseAllowEmpty?: boolean - - /** Ignore album artist if it is "Various Artists" - * - * @default true - */ - ignoreVA?: boolean } export interface MusicbrainzTransformerDataStrong extends MusicbrainzTransformerData { @@ -273,7 +275,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { const { searchWhenMissing = this.defaults.searchWhenMissing, - forceSearch = false + forceSearch = this.defaults.forceSearch ?? false } = stageConfig; const missing = missingMbidTypes(play); @@ -305,18 +307,38 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer x.trim()); - if(naiveSplit.length > 1) { - this.logger.debug('No matches found, trying search with track + first value from artist string split'); - results = await this.api.searchByRecording({...play, data: { - ...play.data, - artists: [naiveSplit[0]] - }}, {using: ['title','artist']}); + + const { + fallbackArtistSearch = this.defaults.fallbackArtistSearch + } = stageConfig; + + if(fallbackArtistSearch === 'naive') { + // try a naive split using any common delimiter found and use the first value as artist + // -- this will likely result in a less accurate match but at least it might find something + // -- usually the "primary artist" is listed first in a combined artist string so cross your fingers this works + const naiveSplit = splitByFirstRegexFound(play.data.artists[0], [play.data.artists[0]]).map(x => x.trim()); + if(naiveSplit.length > 1) { + this.logger.debug('No matches found, trying search with track + first value from artist string split'); + results = await this.api.searchByRecording({...play, data: { + ...play.data, + artists: [naiveSplit[0]] + }}, {using: ['title','artist']}); + } + } else if(fallbackArtistSearch === 'native') { + // use MS native parsing to extract artists from artist string and title + // -- cleaning title is aggressive but at this point MB has not found anything which means its likely not "proper" + // IE "My Track (Cool Remix)" is a proper recording name but "My Track (feat. Someone)" is not bc MB would have removed the joiner from the name + // so we do the same to hopefully get a match + // + // ...additionally, since MB hasn't found anything with single artist string its likely the artist name does not have a common delimiter as part of their proper name + // IE "Crosby, Stills, Nash & Young" is a proper artist name with delimiters included but "My Artist & My Feat Artists" is not + // so we split out all artists by all found delimiters + const nativePlay = nativeParse(play, {titleClean: true, delimiters: DELIMITERS}) + this.logger.debug('No matches found, trying search with aggressive native parsing'); + results = await this.api.searchByRecording(nativePlay, {using: ['title','artist']}); } } } diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts index e99e6c0c..7a26f27f 100644 --- a/src/backend/common/transforms/NativeTransformer.ts +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -32,9 +32,10 @@ export interface NativeTransformerData { artistsIgnore?: string[] artistsParseFrom?: ArtistParseSource[] artistsParseMonolithicOnly?: boolean + titleClean?: boolean } -export interface NativeTransformerDataStrong { +export interface NativeTransformerDataStrong extends Omit { artistsParseFrom?: ArtistParseSource[] artistsParseMonolithicOnly?: boolean ignoreArtistsRegex?: RegExp[] @@ -145,7 +146,6 @@ export default class NativeTransformer extends AtomicPartsTransformer { - let artists = []; const { artistsParseFrom: parseArtistsFrom = this.defaults.artistsParseFrom ?? ['artists', 'title'], artistsParseMonolithicOnly = this.defaults.artistsParseMonolithicOnly ?? true, @@ -153,7 +153,62 @@ export default class NativeTransformer extends AtomicPartsTransformer { + return play.data.track; + } + protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { + if (parts === false) { + return play.data.artists; + } + if (typeof parts === 'object') { + if (parts.when !== undefined) { + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { + this.logger.debug('When condition for artists not met, returning original artists'); + return play.data.artists; + } + } + } + + return transformData.data.artists; + } + protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise { + return play.data.albumArtists; + } + protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise { + return play.data.album; + } + + public notify(payload: WebhookPayload): Promise { + return; + } + +} + +export const nativeParse = (play: PlayObject, options?: NativeTransformerDataStrong & {logger?: MaybeLogger}): PlayObject => { + const { + artistsParseFrom = ['artists', 'title'], + titleClean = false, + artistsParseMonolithicOnly = true, + ignoreArtistsRegex = [], + delimiters, + logger = new MaybeLogger() + } = options || {}; + + let artists = []; + let track = play.data.track; + + if(artistsParseFrom.includes('artists')) { if(play.data.artists.length === 1 || (play.data.artists.length > 1 && artistsParseMonolithicOnly === false)) { @@ -161,7 +216,7 @@ export default class NativeTransformer extends AtomicPartsTransformer ({reg: x.toString(), res: parseRegexSingle(x, artist)})).filter(x => x !== undefined); if(matchedIgnoreArtists.length > 0) { - this.logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`); + logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`); artists.push(artist); } else { const artistCredits = parseArtistCredits(artist, delimiters); @@ -188,10 +243,13 @@ export default class NativeTransformer extends AtomicPartsTransformer { - return play.data.track; - } - protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { - if (parts === false) { - return play.data.artists; - } - if (typeof parts === 'object') { - if (parts.when !== undefined) { - if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { - this.logger.debug('When condition for artists not met, returning original artists'); - return play.data.artists; - } - } - } - - return transformData.data.artists; - } - protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise { - return play.data.albumArtists; - } - protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise { - return play.data.album; - } - - public notify(payload: WebhookPayload): Promise { - return; - } - } \ No newline at end of file diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index 30505a0b..d01b0c64 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -143,10 +143,10 @@ export class MusicbrainzApiClient extends AbstractApiClient { if(q !== '') { q += ' AND '; } - if(Array.isArray(query.artist)) { - q += `artist:(${query.artist.map(x => `"${x}"`).join(' OR ')})` + if(Array.isArray(query.artist) && query.artist.length > 1) { + q += `(artist:(${query.artist.map(x => `"${x}"`).join(' AND ')})) OR (artist:(${query.artist.map(x => `"${x}"`).join(' OR ')}))` } else { - q += `artist:"${query.artist}"`; + q += `artist:"${Array.isArray(query.artist) ? query.artist[0] : query.artist}"`; } } if(query.release !== undefined) { diff --git a/src/backend/tests/musicbrainz/musicbrainz.test.ts b/src/backend/tests/musicbrainz/musicbrainz.test.ts index 88465acb..7fceb6d6 100644 --- a/src/backend/tests/musicbrainz/musicbrainz.test.ts +++ b/src/backend/tests/musicbrainz/musicbrainz.test.ts @@ -9,6 +9,8 @@ import MusicbrainzTransformer from "../../common/transforms/MusicbrainzTransform import { PlayObject } from "../../../core/Atomic.js"; import { projectDir } from '../../common/index.js'; import path from 'path'; +import { nativeParse } from '../../common/transforms/NativeTransformer.js'; +import { DELIMITERS } from '../../common/infrastructure/Atomic.js'; const envPath = path.join(projectDir, '.env'); dotenv.config({path: envPath}); @@ -90,7 +92,7 @@ describe('Musicbrainz API', function () { expect(res.recordings).to.not.be.empty; }); - it('tries additional query using only track and naively split artist', async function () { + it('tries additional query using only track and native parsing', async function () { const play: PlayObject = { data: { @@ -103,7 +105,8 @@ describe('Musicbrainz API', function () { const res = await mbTransformer.getTransformerData(play, { type: "musicbrainz", - searchWhenMissing: ["artists", "album", "title"] + searchWhenMissing: ["artists", "album", "title"], + fallbackArtistSearch: 'native' }); expect(res.recordings).to.exist; expect(res.recordings).to.not.be.empty; -- 2.51.2 From 8083696469b93e063537997b444e9ad0b3bf59cb Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Dec 2025 18:47:57 +0000 Subject: [PATCH 27/41] feat(transform): Add proper support for meta and duration transform parts --- .../common/infrastructure/Transform.ts | 3 + .../transforms/AtomicPartsTransformer.ts | 57 ++++++++++++++----- .../transforms/MusicbrainzTransformer.ts | 28 ++++++++- .../musicbrainz/MusicbrainzApiClient.ts | 1 + .../tests/musicbrainz/musicbrainz.test.ts | 2 +- src/core/Atomic.ts | 6 +- 6 files changed, 79 insertions(+), 18 deletions(-) diff --git a/src/backend/common/infrastructure/Transform.ts b/src/backend/common/infrastructure/Transform.ts index 058b2926..3971785b 100644 --- a/src/backend/common/infrastructure/Transform.ts +++ b/src/backend/common/infrastructure/Transform.ts @@ -18,7 +18,10 @@ export type PlayTransformPartsConfig = PlayTransformPart export interface PlayTransformPartsAtomic { title?: T artists?: T + albumArtists?: T album?: T + duration?: T + meta?: T } export type StageTypeMetadata = 'spotify' | 'musicbrainz' | 'native'; diff --git a/src/backend/common/transforms/AtomicPartsTransformer.ts b/src/backend/common/transforms/AtomicPartsTransformer.ts index 5a2637fb..1615d987 100644 --- a/src/backend/common/transforms/AtomicPartsTransformer.ts +++ b/src/backend/common/transforms/AtomicPartsTransformer.ts @@ -40,9 +40,11 @@ export default abstract class AtomicPartsTransformer; protected abstract handleAlbumArtists(play: PlayObject, parts: Y, transformData: T): Promise; protected abstract handleAlbum(play: PlayObject, parts: Y, transformData: T): Promise; + protected async handleDuration(play: PlayObject, parts: Y, transformData: T): Promise { + return play.data.duration; + } - protected async handleMeta(play: PlayObject, transformData: T): Promise { + protected async handleMeta(play: PlayObject, parts: Y, transformData: T): Promise { return play.data.meta; } diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 6c046720..38eef675 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -436,8 +436,34 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + if (parts === false) { + return play.data.duration; + } + if (typeof parts === 'object') { + if (parts.when !== undefined) { + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { + this.logger.debug('When condition for duration not met, returning original duration'); + return play.data.duration; + } + } + } + + return transformData.data.duration; + } - protected async handleMeta(play: PlayObject, transformData: PlayObject): Promise { + protected async handleMeta(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { + if (parts === false) { + return play.data.meta; + } + if (typeof parts === 'object') { + if (parts.when !== undefined) { + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { + this.logger.debug('When condition for duration not met, returning original duration'); + return play.data.meta; + } + } + } return transformData.data.meta; } diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index d01b0c64..2155ac7b 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -216,6 +216,7 @@ export const recordingToPlay = (data: IRecording, options?: {ignoreVA?: boolean} artists, album: album !== undefined ? album.title : undefined, albumArtists, + duration: data.length, meta: { brainz: { track: data.id, diff --git a/src/backend/tests/musicbrainz/musicbrainz.test.ts b/src/backend/tests/musicbrainz/musicbrainz.test.ts index 7fceb6d6..76039db3 100644 --- a/src/backend/tests/musicbrainz/musicbrainz.test.ts +++ b/src/backend/tests/musicbrainz/musicbrainz.test.ts @@ -32,7 +32,7 @@ const mbTransformer = new MusicbrainzTransformer({ ttl: '1ms' } }, { - logger: loggerDebug, + logger: loggerTest, clientCache: memorycache(), cache: memorycache() }) diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 055bc256..997b6dc6 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -395,7 +395,7 @@ export const FEAT: Feat[] = ['ft','feat','vs','ft.','feat.','vs.','featuring']; export interface TransformOptions { failOnFetch?: boolean; - throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album')[]; + throwOnFailure?: boolean | ('artists' | 'title' | 'albumArtists' | 'album' | 'duration' | 'meta')[]; ttl?: string } export interface TransformerCommonConfig, Y = Record> { @@ -410,8 +410,8 @@ export interface TransformerCommon, Y = Record Date: Fri, 5 Dec 2025 19:10:08 +0000 Subject: [PATCH 28/41] feat(transformer): Add musicbrainz api url logging --- patches/musicbrainz-api+0.27.0.patch | 62 +++++++++++++++++++ .../transforms/MusicbrainzTransformer.ts | 9 +-- .../tests/musicbrainz/musicbrainz.test.ts | 2 +- 3 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 patches/musicbrainz-api+0.27.0.patch diff --git a/patches/musicbrainz-api+0.27.0.patch b/patches/musicbrainz-api+0.27.0.patch new file mode 100644 index 00000000..00f3ee4d --- /dev/null +++ b/patches/musicbrainz-api+0.27.0.patch @@ -0,0 +1,62 @@ +diff --git a/node_modules/musicbrainz-api/lib/http-client.js b/node_modules/musicbrainz-api/lib/http-client.js +index a5ca5f7..14227ca 100644 +--- a/node_modules/musicbrainz-api/lib/http-client.js ++++ b/node_modules/musicbrainz-api/lib/http-client.js +@@ -9,6 +9,7 @@ function isConnectionReset(err) { + export class HttpClient { + constructor(httpOptions) { + this.httpOptions = httpOptions; ++ this.logger = (httpOptions ?? {}).logger; + } + get(path, options) { + return this._fetch('get', path, options); +@@ -30,6 +31,9 @@ export class HttpClient { + let retryLimit = options.retryLimit && options.retryLimit > 1 ? options.retryLimit : 1; + const retryTimeout = this.httpOptions.timeout ? this.httpOptions.timeout : 500; + const url = this._buildUrl(path, options.query); ++ if(this.logger !== undefined) { ++ this.logger.debug(`${method} ${url}`); ++ } + const cookies = await this.getCookies(); + const headers = new Headers(options.headers); + headers.set('User-Agent', this.httpOptions.userAgent); +diff --git a/node_modules/musicbrainz-api/lib/musicbrainz-api-node.js b/node_modules/musicbrainz-api/lib/musicbrainz-api-node.js +index c9cad5a..7d772a9 100644 +--- a/node_modules/musicbrainz-api/lib/musicbrainz-api-node.js ++++ b/node_modules/musicbrainz-api/lib/musicbrainz-api-node.js +@@ -15,7 +15,8 @@ export class MusicBrainzApi extends MusicBrainzApiDefault { + return new HttpClientNode({ + baseUrl: this.config.baseUrl, + timeout: 500, +- userAgent: `${this.config.appName}/${this.config.appVersion} ( ${this.config.appContactInfo} )` ++ userAgent: `${this.config.appName}/${this.config.appVersion} ( ${this.config.appContactInfo} )`, ++ logger: this.config.logger + }); + } + async login() { +diff --git a/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts b/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts +index cd05017..487404f 100644 +--- a/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts ++++ b/node_modules/musicbrainz-api/lib/musicbrainz-api.d.ts +@@ -75,6 +75,7 @@ export interface IMusicBrainzConfig { + * Default is [15, 18], which allows up to 15 requests every 18 seconds + */ + rateLimit?: [number, number]; ++ logger?: any + } + interface IInternalConfig extends IMusicBrainzConfig { + baseUrl: string; +diff --git a/node_modules/musicbrainz-api/lib/musicbrainz-api.js b/node_modules/musicbrainz-api/lib/musicbrainz-api.js +index 72fb15d..0e9c32d 100644 +--- a/node_modules/musicbrainz-api/lib/musicbrainz-api.js ++++ b/node_modules/musicbrainz-api/lib/musicbrainz-api.js +@@ -43,7 +43,8 @@ export class MusicBrainzApi { + return new HttpClient({ + baseUrl: this.config.baseUrl, + timeout: 500, +- userAgent: `${this.config.appName}/${this.config.appVersion} ( ${this.config.appContactInfo} )` ++ userAgent: `${this.config.appName}/${this.config.appVersion} ( ${this.config.appContactInfo} )`, ++ logger: this.config.logger + }); + } + async restGet(relUrl, query = {}) { diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 38eef675..a6a36a37 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -1,4 +1,4 @@ -import { asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, isMBReleasePrimaryGroupType, MBReleaseGroupPrimaryType, MBReleaseGroupSecondaryType, MBReleaseStatus, MissingMbidType, PlayObject, TrackMeta, TransformerCommon } from "../../../core/Atomic.js"; +import { asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, isMBReleasePrimaryGroupType, MBReleaseGroupPrimaryType, MBReleaseGroupSecondaryType, MBReleaseStatus, MissingMbidType, PlayObject, TrackMeta, TransformerCommon, TransformOptions } from "../../../core/Atomic.js"; import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; import { ExternalMetadataTerm, PlayTransformMetadataStage } from "../infrastructure/Transform.js"; @@ -11,7 +11,7 @@ import { MusicbrainzApiClient, MusicbrainzApiConfig, recordingToPlay } from "../ import { IRecordingList, IRecordingMatch, MusicBrainzApi } from "musicbrainz-api"; import { getRoot, version } from "../../ioc.js"; import { normalizeWebAddress } from "../../utils/NetworkUtils.js"; -import { intersect, missingMbidTypes, removeUndefinedKeys } from "../../utils.js"; +import { intersect, isDebugMode, missingMbidTypes, removeUndefinedKeys } from "../../utils.js"; import { SimpleError } from "../errors/MSErrors.js"; import { parseArrayFromMaybeString } from "../../utils/StringUtils.js"; import clone from "clone"; @@ -144,7 +144,7 @@ export interface MusicbrainzTransformerDataConfig { export type MusicbrainzBestMatch = {play: PlayObject, score: number}; -export type MusicbrainzTransformerConfig = TransformerCommon; +export type MusicbrainzTransformerConfig = TransformerCommon & {options?: TransformOptions & {logUrl?: boolean}} export type RecordingRankedMatched = IRecordingMatch & {rankScore: number} @@ -225,7 +225,8 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer Date: Fri, 5 Dec 2025 19:18:39 +0000 Subject: [PATCH 29/41] fix: revert lastfm debug code --- src/backend/sources/LastfmSource.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/backend/sources/LastfmSource.ts b/src/backend/sources/LastfmSource.ts index 6df986e2..86ea52da 100644 --- a/src/backend/sources/LastfmSource.ts +++ b/src/backend/sources/LastfmSource.ts @@ -65,17 +65,7 @@ export default class LastfmSource extends MemorySource { } doAuthentication = async () => { try { - await this.api.testAuth(); - - //5e6f41d4-9021-4d5d-b4a6-e39be60abac5 - const resp = await this.api.callApi((client: any) => client.trackGetInfo({ - //mbid: '5e6f41d4-9021-4d5d-b4a6-e39be60abac5', - artist: "Bowling For Soup & Punk Rock Factory", - track: "Endless Possibility (feat. Wheatus)", - sk: this.api.client.sessionKey - })); - const play = LastfmApiClient.formatPlayObj(resp.track); - return true; + return await this.api.testAuth(); } catch (e) { throw e; } -- 2.51.2 From 8976bb99f5f20cc804560fd1dc7275ef79676031 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Dec 2025 19:19:50 +0000 Subject: [PATCH 30/41] fix(transformer): Return stage name for proper diff logging --- src/backend/common/AbstractComponent.ts | 9 +++++---- src/backend/common/transforms/TransformerManager.ts | 11 ++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 859fd2f3..276e11ef 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -194,10 +194,11 @@ export default abstract class AbstractComponent extends AbstractInitializable { failureReturnPartial = false } = hookItem; - let newTransformedPlay: PlayObject; - let err: Error; + let newTransformedPlay: PlayObject, + stageName: string, + err: Error; try { - newTransformedPlay = await this.transformManager.handleStage(hookItem, transformedPlay, asyncId); + [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, transformedPlay, asyncId); } catch (e) { err = e; } @@ -218,7 +219,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { transformHistory.push({ type: hookItem.type, - name: hookItem.name, + name: stageName, play: newTransformedPlay.data }); diff --git a/src/backend/common/transforms/TransformerManager.ts b/src/backend/common/transforms/TransformerManager.ts index bfe01515..c2f3b3fa 100644 --- a/src/backend/common/transforms/TransformerManager.ts +++ b/src/backend/common/transforms/TransformerManager.ts @@ -106,7 +106,7 @@ export default class TransformerManager { return t.parseConfig(data); } - public async handleStage(data: StageConfig, play: PlayObject, asyncId: string = nanoid(6)): Promise { + public async handleStage(data: StageConfig, play: PlayObject, asyncId: string = nanoid(6)): Promise<[PlayObject, string]> { const list = this.transformers.get(data.type); if (list === undefined || list.length === 0) { throw new Error(`No transformer of type '${data.type}' is registered.`); @@ -114,13 +114,13 @@ export default class TransformerManager { let t: AbstractTransformer; if (list.length > 1) { - if((data as any).name === undefined) { + if(data.name === undefined) { this.logger.warn(`More than one '${data.type}' transformer but name was not specified, using first registered`); t = list[0]; } else { - const named = list.find(x => x.name === (data as any).name); + const named = list.find(x => x.name === data.name); if(named === undefined) { - throw new Error(`No ${data.type} transformer with name '${(data as any).name}'`) + throw new Error(`No ${data.type} transformer with name '${data.name}'`) } t = named; } @@ -129,9 +129,10 @@ export default class TransformerManager { } try { - return this.asyncStore.run(asyncId, async () => { + const transformedPlay = await this.asyncStore.run(asyncId, async () => { return await t.handle(data, play); }); + return [transformedPlay, t.name]; } catch (e) { throw new Error('Stage processing failed', {cause: e}); } -- 2.51.2 From 83eb7a1b65a5a6962fb3d19a976a33c94d9567ec Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Dec 2025 19:20:05 +0000 Subject: [PATCH 31/41] fix(transformer): Fix optional option object --- src/backend/common/transforms/MusicbrainzTransformer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index a6a36a37..e37025ef 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -226,7 +226,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer Date: Fri, 5 Dec 2025 19:20:31 +0000 Subject: [PATCH 32/41] test(tranformer): Add longer timeout and fix logger used in musicbrainz suite --- src/backend/tests/musicbrainz/musicbrainz.test.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/backend/tests/musicbrainz/musicbrainz.test.ts b/src/backend/tests/musicbrainz/musicbrainz.test.ts index 7fceb6d6..d08565a7 100644 --- a/src/backend/tests/musicbrainz/musicbrainz.test.ts +++ b/src/backend/tests/musicbrainz/musicbrainz.test.ts @@ -32,7 +32,7 @@ const mbTransformer = new MusicbrainzTransformer({ ttl: '1ms' } }, { - logger: loggerDebug, + logger: loggerTest, clientCache: memorycache(), cache: memorycache() }) @@ -47,6 +47,8 @@ describe('Musicbrainz API', function () { it('escapes lucene special characters', async function () { + this.timeout(3500); + const play: PlayObject = { data: { track: 'Cyber Space (CrossWorlds Remix): Final Lap (No Chants)', @@ -74,6 +76,8 @@ describe('Musicbrainz API', function () { it('tries second query using only track and album', async function () { + this.timeout(3500); + const play: PlayObject = { data: { track: "Roulette Road (CrossWorlds Remix)", @@ -94,6 +98,8 @@ describe('Musicbrainz API', function () { it('tries additional query using only track and native parsing', async function () { + this.timeout(3500); + const play: PlayObject = { data: { track: "Endless Possibility (feat. Wheatus)", @@ -113,6 +119,9 @@ describe('Musicbrainz API', function () { }); it('handles non-ascii', async function () { + + this.timeout(3500); + const play: PlayObject = { data: { track: "Bad Apple!! feat.SEKAI", -- 2.51.2 From ea4d9b376489d2f7ae6c198df391106a552b02bf Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Dec 2025 19:25:38 +0000 Subject: [PATCH 33/41] fix(transformers): Add missing parts parsing --- src/backend/common/transforms/MusicbrainzTransformer.ts | 2 +- src/backend/common/transforms/NativeTransformer.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index e37025ef..f603fd78 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -255,7 +255,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer Date: Fri, 5 Dec 2025 20:48:37 +0000 Subject: [PATCH 34/41] feat(transform): Always log transform diff if in debug mode --- src/backend/common/AbstractComponent.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 276e11ef..f290a482 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -22,6 +22,7 @@ import { getRoot } from "../ioc.js"; import { nanoid } from "nanoid"; import {diffStringsUnified, DiffOptionsColor} from 'jest-diff'; import chalk from 'chalk'; +import { isDebugMode } from "../utils.js"; export default abstract class AbstractComponent extends AbstractInitializable { @@ -231,7 +232,7 @@ export default abstract class AbstractComponent extends AbstractInitializable { } } - const shouldLog = log ?? this.config.options?.playTransform?.log ?? false; + const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode(); if(transformedPlay.meta.transforms === undefined) { transformedPlay.meta.transforms = { -- 2.51.2 From 1289c53941a8396f86f073d6498f5d3c994881cf Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 5 Dec 2025 20:54:52 +0000 Subject: [PATCH 35/41] docs: Update musicbrainz and add additional parts to general transform docs --- .../configuration/transforms/musicbrainz.mdx | 317 +++++++++++++++--- .../configuration/transforms/transforms.mdx | 11 +- 2 files changed, 282 insertions(+), 46 deletions(-) diff --git a/docsite/docs/configuration/transforms/musicbrainz.mdx b/docsite/docs/configuration/transforms/musicbrainz.mdx index d5732503..d76afc2c 100644 --- a/docsite/docs/configuration/transforms/musicbrainz.mdx +++ b/docsite/docs/configuration/transforms/musicbrainz.mdx @@ -3,6 +3,8 @@ title: Musicbrainz Stage toc_min_heading_level: 2 toc_max_heading_level: 5 --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; The **Musicbrainz** [Stage](/configuration/transforms#stage) matches your Play data with the [MusicBrainz](https://musicbrainz.org/) database. If the match score is high enough then Multi-Scrobbler uses the match to correct and fill-in missing information in your Play data. @@ -57,19 +59,214 @@ To avoid rate limiting, MusicBrainz requires API users to identify themself usin } ``` -### Search +### Stage Configuration -Available properties for [Stage Configuration](/configuration/transforms#configuring-stages): +All of the properties found in [**Matching with Musicbrainz**](#matching-with-musicbrainz) section are configured in [Stage Configuration](/configuration/transforms#configuring-stages) as `defaults`. + +Example: + +```json5 title="config.json" +{ + // ... + "transformers": [ + { + "type": "musicbrainz", + "name": "MyMB", + "data": { + "apis": [ + { + "contact": "contact@mydomain.com" + } + ], + "defaults": { + "releaseStatusPriority": ["official"], + "fallbackArtistSearch": "native", + "releaseAllowEmpty": true + } + }, + } + ] +} +``` + +### Rules + +Each [Rule](/configuration/transforms#stage-rules) should be either a boolean, specifying if the transformed data should be used for this field, or a [`when` condition](/configuration/transforms#conditional-moditication): + +```json5 +{ + "type": "musicbrainz", + "name": "MyMB" + // ... + "title": false, // will not apply any changes to Play title + "artists": { + "when": {/* ... */}, // will only apply changes to Play artists if "when" is satisfied + /* ... */ + }, + "album": true // will always apply changes to Play album + "meta": true // adds MusicBrainz MBIDs to scrobble data +} +``` + +## Matching with Musicbrainz :::note -**All of the below parameters are optional.** +**All properties found in this section are optional.** If you just want a sensible default for configuraion refer to the [Best Practices](#best-practices) section. ::: -#### Score +Matching your Scrobble's Play data with a result from Musicbrainz is comprised of two steps: + +* [**Searching**](#searching) Musicbrainz using parts of your Scrobble as queries +* [**Refining**](#refining) matched results to select the desired/allowed match + +Both steps have separate configuration. + +### Searching + +The first step is making [search queries to the Musicbrainz database](https://musicbrainz.org/doc/MusicBrainz_API/Search) to try to get potential candidates. Multi-scrobbler uses the **title, artist(s), and album** from your Scrobble data to query for matches. + +Musicbrainz agressively normalizes these fields in its database which means that if your Scrobble data contains major innaccuracies or phrases fields in a non-normal way, it's possible Musicbrainz won't find any matches. This is a more common occurrence when trying to match Scrobble data from Sources that don't support multiple artist data, like [Subsonic](/configuration/sources/subsonic) and [Last.fm](/configuration/sources/lastfm). + +
+ +Examples of Bad Data + +Last.fm Scrobble returns an Artist in the title and combines two artists into one string: + +```json +{ + "title": "Endless Possibility (feat. Wheatus)", + "artists": ["Bowling For Soup & Punk Rock Factory"] +} +``` + +The corresponding [Musicbrainz Recording](https://musicbrainz.org/release/85bf284b-8b39-414e-8ad9-61f593072588) has all artists separated and no artist in the title: + +```json +{ + "title": "Endless Possibility", + "artists": ["Bowling For Soup", "Punk Rock Factory", "Wheatus"] +} +``` + +
+ +If a normal search using all the fields mentioned above does not return any matches Multi-scrobbler can try two additional searches with modified queries: + +#### Album Only + +If your Scrobble data contains a title, artist(s), and an album (all three fields) then Multi-scrobbler will **automatically** retry the search using only title and album. + +#### Artist Extraction + +If... + +* [Album Only](#album-only) returns no results or did not run due to missing an album +* Your scrobble data contains only **one** artist string +* `fallbackArtistSearch` is set in [Stage Configuration](/configuration/transforms#configuring-stages) + +Then Multi-scrobbler will attempt to extract multiple artists from your artist and track string. **This is not automatic. You must set `fallbackArtistSearch` to enable this additional search.** + +`fallbackArtistSearch` can be set to **Native** or **Naive** mode: + + + + + **Native** (`native`) mode uses an aggressive configuration of the [Native Stage](/configuration/transforms/native) to extract artists using common delimiters and common "joined" artist patterns from the artist and title string of your Scrobble data. + + + :::tip + + If you already have a Native Stage configured you should use that instead, running it before the Musicbrainz stage. + + ::: + +
+ + Native Mode Example + + ```json + { + "title": "Endless Possibility (feat. Wheatus)", + "artists": ["Bowling For Soup & Punk Rock Factory, My Cool Band"] + } + ``` + + * Extracts `Bowling For Soup` `Punk Rock Factory` `My Cool Band` from artist string + * Extracts `Wheatus` from title string + * Removes `(feat. Wheatus)` from title string because it found an artist there + + Resulting data used for Musicbrainz search: + + ```json + { + "title": "Endless Possibility", + "artists": ["Bowling For Soup", "Punk Rock Factory", "My Cool Band", "Wheatus"] + } + ``` + +
+ + [Stage Configuration](/configuration/transforms#configuring-stages) example: + + ```json5 + // ... + "defaults": { + "releaseStatusPriority": ["official"], + // ... + "fallbackArtistSearch": "native" + } + ``` + + + + + **Naive** (`naive`) mode looks for the first found common delimiter in the artist string. If it finds one then it uses the preceding value as the only artist in the Musicbrainz search. It does not try to extract additional artists from the artist string, or extract anything from the title string. + +
+ + Naive Mode Example + + ```json + { + "title": "Endless Possibility (feat. Wheatus)", + "artists": ["Bowling For Soup, Punk Rock Factory & Wheatus"] + } + ``` + + * Finds `&` as first common delimiter, extracts "Bowling For Soup" + + Resulting data used for Musicbrainz search: + + ```json + { + "title": "Endless Possibility (feat. Wheatus)", + "artists": ["Bowling For Soup"] + } + ``` +
+ + [Stage Configuration](/configuration/transforms#configuring-stages) example: + + ```json5 + // ... + "defaults": { + "releaseStatusPriority": ["official"], + // ... + "fallbackArtistSearch": "naive" + } + ``` + + + + +### Refining + +### Score Each match returned by MusicBrainz contains a numeric score representing how close it was to the search parameters. Set `score` in configuration to set a minimum score that must be met by matches. Default is `90`. @@ -80,7 +277,7 @@ Each match returned by MusicBrainz contains a numeric score representing how clo } ``` -#### Filtering and Sorting +### Filtering There are several attributes associated with the [Release](https://wiki.musicbrainz.org/Release) a [Recording](https://wiki.musicbrainz.org/Recording) (individual Track/song) belongs to that you may be interested in controlling. MS can use these attributes to filter what the final Recording selected to match against your Play data is. @@ -91,6 +288,8 @@ An easy way to think about this: * Should a track that belongs to a Compilation album be allowed to match? * What country do you prefer an album release to be from? +##### Release Attributes + These attributes (`attribute_name` in config) are: * [Release **Status**](https://wiki.musicbrainz.org/Release#Status) (`releaseStatus`) - Is this release/album/ep official, promotional, bootleg, etc... @@ -114,11 +313,24 @@ EX to exclude compilations "releaseGroupSecondaryTypeDeny": ["compilation"] ``` -##### Sorting +#### Empty Releases + +Sometimes a Recording may not have any associated Releases. This may be because there is not enough information about the Recording yet, or it was never included on an actual Release. + +You may want to filter the majority of your matches by releases but allow matching a Recording that has no releases to begin with. To allow this set `releaseAllowEmpty` to `true` in configuration: + +```json5 +{ + // ... + "releaseAllowEmpty": true // don't remove a match during filtering just because it has no releases +} +``` + +### Sorting -Each of the above attributes has one additional property that can be used to **rank** releases based on the order of the values you give it. This is the `priority` property. +Each [Release Attribute](#release-attribute) has one additional property that can be used to **rank** releases based on the order of the values you give it. This is the `priority` property. -*After* matches have been [filtered](#filtering-and-sorting), the remaining matches will have their releases sorted. Release with an attribute that does not match are **not** removed, but they are sorted lower than releases that do match. +*After* matches have been [filtered](#filtering), the remaining matches will have their releases sorted. Release with an attribute that does not match are **not** removed, but they are sorted lower than releases that do match. Sorting can be a good alternative to filtering: with filters there is a possibility your filters may eliminate all matches; if you want to ensure that **some** match will be used then sorting can ensure that the **best choice** out of those given will always be used, without accidentally ending up with **no choice**. @@ -138,38 +350,6 @@ EX to prefer albums, then singles, over everything else "releaseGroupPrimaryTypePriority": ["album", "single"] ``` -#### Empty Releases - -Sometimes a Recording may not have any associated Releases. This may be because there is not enough information about the Recording yet, or it was never included on an actual Release. - -You may want to filter/sort the majority of your Play data by releases but allow matching a Recording if it does not have a release, in this instance. To allow this set `releaseAllowEmpty` to `true` in configuration: - -```json5 -{ - // ... - "releaseAllowEmpty": true // don't remove a match during filtering just because it has no releases -} -``` - -### Rules - -Each [Rule](/configuration/transforms#stage-rules) should be either a boolean, specifying if the transformed data should be used for this field, or a [`when` condition](/configuration/transforms#conditional-moditication): - -```json5 -{ - "type": "musicbrainz", - "name": "MyMB" - // ... - "title": false, // will not apply any changes to Play title - "artists": { - "when": {/* ... */}, // will only apply changes to Play artists if "when" is satisfied - /* ... */ - }, - "album": true // will always apply changes to Play album - "meta": true // adds MusicBrainz MBIDs to scrobble data -} -``` - If a rule is not present then multi-scrobbler defaults it to `true`. ## Best Practices @@ -191,13 +371,23 @@ For a more opinionated match that will mirror what you would expect from data fr } ``` +Consider adding [Artist Extraction](./?fallbackArtist=native#artist-extraction) in **Native Mode** if any of your Sources do not support multiple artists or your music collection is not tagged well. + +```json5 +{ + // add the line below to the sensible defaults above + // (don't forget commas) + "fallbackArtistSearch": "native" +} +``` + ### Filter Considerations When using your own filters consider: * Prefer `deny` over `allow` * Releases come in all kinds of formats. Since `allow` is explicit you may filter out your desired match without realizing it (correct data except for release type). Or the Musicbrainz data for a higher scored match may be appropriate but you did not include it, exhaustively. -* Prefer [Sorting](#sorting) over [Filtering](#filtering-and-sorting) +* Prefer [Sorting](#sorting) over [Filtering](#filtering) * Sorting does not eliminate any matches. It is, generally, better to get **some** match than it is to have your Play data completely uncorrected because filtering eliminated all matches ### Using Partial Match @@ -235,6 +425,48 @@ If you know that your music collection is well organized and you do not want to +## Logging + +If + +* Musicbrainz is not returning matches +* Multi-scrobbler is using the wrong match +* or the resulting enhanced Scrobble is not what you expected + +Enable logging by turning on [**Debug Mode**](/configuration#debug-mode) to help diagnose any issues with the Musicbrainz API and Scrobble enhancement. **Before creating an issue** please enable logging and include any logs with your issue as this is needed to debug. + +If you have multiple Modification Stages and need to see the diff for your Play between each Stage, enable `"log": "all"` in the individual [Modification Stage](http://localhost:3000/docs/configuration/transforms/musicbrainz/?fallbackArtist=native#artist-extraction). + +
+ +Example + +In a [Subsonic](/configuration/sources/subsonic) [File Config](/configuration?configType=file#configuration-types): + +```json5 title="subsonic.json" +[ + { + "name": "MySubsonic", + "data": { /* ... */}, + "options": { + "playTransform": { + // highlight-start + "log": "all", + // highlight-end + "preCompare": [ + { + "type": "musicbrainz", + "name": "MyMB", + } + ] + } + } + } +] +``` + +
+ ## Examples ### Minimal @@ -313,7 +545,8 @@ Your [AIO Config](/configuration?configType=aio#configuration-types): "defaults": { "releaseStatusPriority": ["official"], "releaseGroupPrimaryTypePriority": ["album", "single", "ep"], - "releaseCountryPriority": ["XW"] + "releaseCountryPriority": ["XW"], + "fallbackArtistSearch": "native" } } ] diff --git a/docsite/docs/configuration/transforms/transforms.mdx b/docsite/docs/configuration/transforms/transforms.mdx index f7cbae79..f40e6a39 100644 --- a/docsite/docs/configuration/transforms/transforms.mdx +++ b/docsite/docs/configuration/transforms/transforms.mdx @@ -346,11 +346,14 @@ Each [Stage](#stage) may specify whether it should apply the resulting transform "when": {/* ... */}, // will only apply changes to Play artists if "when" is satisfied /* ... */ }, - "album": true // will always apply changes to Play album + "albumArtists": true, // will always apply changes to Play album artists + "album": true, // will always apply changes to Play album + "duration": true, // will always apply changes to Play duration (length of track) + "meta": true, // will always apply changes to Play meta (MBIDs, spotify links, etc...) } ``` -The actual value of each property may be different for each Stage. Check the docs for the Stage you want to use to see its usage of `title`, `artists`, and `album`. +The actual value of each property may be different for each Stage. Check the docs for the Stage you want to use to see its usage of each. Generically, though, each property may be some value **or** an object combining a [`when` condition](#conditional-modification) and that value. @@ -366,7 +369,7 @@ Specifying these Rules is **not** the same as [configuring the Stage](#configuri [Stages](#stage) within a [Hook](#hook), and [Rules](#stage-rules) within each Stage, support a `when` object for testing **if they should be run.** -The `when` object may have propertes for `artist`, `title` and/or `album`. Each property may be a string or regular expression. The value of the property is used to match the **pre-transformation** values from Play data. +The `when` object may have propertes for rule. Each property may be a string or regular expression. The value of the property is used to match the **pre-transformation** values from Play data. All parts of an individual `when` clause must test true to "pass" but if **any** `when` clauses pass the Stage/Rule is processed. @@ -398,7 +401,7 @@ All parts of an individual `when` clause must test true to "pass" but if **any** More succinctly: -* All parts (`artist` `album` `title`) of a `when` are `AND` conditions +* All parts (`artist` `album` `albumArtist` `duration` `meta` `title`) of a `when` are `AND` conditions * All part-objects in the `when` array are `OR` conditions
-- 2.51.2 From a7134e2dae250a7f77cf91a5187b83b3e6442891 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Sun, 7 Dec 2025 20:10:12 +0000 Subject: [PATCH 36/41] fix(transformer): Missing fallbacksearch assignment --- src/backend/common/transforms/MusicbrainzTransformer.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index f603fd78..2d1e542d 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -188,6 +188,15 @@ export const parseStageConfig = (data: MusicbrainzTransformerData | undefined = logger.debug(`Will search if missing: ${config.searchWhenMissing.join(', ')} | Match if (default) score is >= ${config.score}`); + if(data.fallbackArtistSearch !== undefined) { + const cleanFallback = data.fallbackArtistSearch; + if(['native','naive'].includes(cleanFallback)) { + throw new Error(`fallbackArtistSearch must be one of 'native' or 'naive', given: ${cleanFallback}`); + } + config.fallbackArtistSearch = cleanFallback; + logger.debug(`Will make an additional search using ${config.fallbackArtistSearch} method as fallback`); + } + for(const [k,v] of Object.entries(config)) { if(k.includes('release') && v !== undefined) { logger.debug(`${k}: ${v.join(' | ')}`); -- 2.51.2 From 5e63936dda2454cf4a16acdb86ef72c90b8402cb Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 8 Dec 2025 13:11:03 +0000 Subject: [PATCH 37/41] fix: if --- src/backend/common/transforms/MusicbrainzTransformer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 2d1e542d..5c2ed4a3 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -190,7 +190,7 @@ export const parseStageConfig = (data: MusicbrainzTransformerData | undefined = if(data.fallbackArtistSearch !== undefined) { const cleanFallback = data.fallbackArtistSearch; - if(['native','naive'].includes(cleanFallback)) { + if(!['native','naive'].includes(cleanFallback)) { throw new Error(`fallbackArtistSearch must be one of 'native' or 'naive', given: ${cleanFallback}`); } config.fallbackArtistSearch = cleanFallback; -- 2.51.2 From 761c0f79fb017a2df1b24f5c249246622a9464f8 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 8 Dec 2025 17:04:30 +0000 Subject: [PATCH 38/41] feat(transformer): Add freetext fallback search for musicbrainz api --- .../transforms/MusicbrainzTransformer.ts | 44 ++++++++++++----- .../musicbrainz/MusicbrainzApiClient.ts | 48 ++++++++++++------- .../tests/musicbrainz/musicbrainz.test.ts | 31 +++++++++++- 3 files changed, 92 insertions(+), 31 deletions(-) diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 5c2ed4a3..815c3246 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -39,6 +39,7 @@ export interface MusicbrainzTransformerData { forceSearch?: boolean score?: number fallbackArtistSearch?: ('naive' | 'native') + fallbackFreeText?: boolean /** Ignore album artist if it is "Various Artists" * @@ -146,7 +147,12 @@ export type MusicbrainzBestMatch = {play: PlayObject, score: number}; export type MusicbrainzTransformerConfig = TransformerCommon & {options?: TransformOptions & {logUrl?: boolean}} -export type RecordingRankedMatched = IRecordingMatch & {rankScore: number} +export type RecordingRankedMatched = IRecordingMatch & {rankScore?: number} + +export interface IRecordingMSList extends IRecordingList { + recordings: RecordingRankedMatched[] + freeText?: boolean +} export const parseStageConfig = (data: MusicbrainzTransformerData | undefined = {}, logger: MaybeLogger = new MaybeLogger()): MusicbrainzTransformerDataStrong => { @@ -197,6 +203,11 @@ export const parseStageConfig = (data: MusicbrainzTransformerData | undefined = logger.debug(`Will make an additional search using ${config.fallbackArtistSearch} method as fallback`); } + if(data.fallbackFreeText === true) { + config.fallbackFreeText = true; + logger.debug('Will make an additional search with free text'); + } + for(const [k,v] of Object.entries(config)) { if(k.includes('release') && v !== undefined) { logger.debug(`${k}: ${v.join(' | ')}`); @@ -298,9 +309,9 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + public async getTransformerData(play: PlayObject, stageConfig: MusicbrainzTransformerDataStage): Promise { - let results = await this.api.searchByRecording(play); + let results: IRecordingMSList = await this.api.searchByRecording(play); if(results === undefined) { throw new Error('results were unexpectedly undefined! API should have thrown...'); @@ -311,6 +322,12 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer 0) { @@ -321,10 +338,6 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + public async handlePostFetch(play: PlayObject, transformData: IRecordingMSList, stageConfig: MusicbrainzTransformerDataStage): Promise { if(transformData.recordings.length === 0) { throw new SimpleError('No matches returned from Musicbrainz API'); } @@ -365,7 +385,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer x.score >= score); + let filteredList: RecordingRankedMatched[] = transformData.recordings.filter(x => x.score >= score); if(filteredList.length === 0) { throw new SimpleError(`All ${transformData.count} fetched matches had a score < ${score}, best match was ${transformData.recordings[0].score}`); } @@ -483,7 +503,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { +export const filterByValidReleaseStatus = (list: T, stageConfig: MusicbrainzTransformerDataStage, logger: MaybeLogger = new MaybeLogger()) => { const { releaseStatusAllow = [], releaseStatusDeny = [], @@ -510,7 +530,7 @@ export const filterByValidReleaseStatus = (list: IRecordingMatch[], stageConfig: || x.releases.length > 0); } -export const filterByValidReleaseGroupPrimary = (list: IRecordingMatch[], stageConfig: MusicbrainzTransformerDataStage, logger: MaybeLogger = new MaybeLogger()) => { +export const filterByValidReleaseGroupPrimary = (list: T, stageConfig: MusicbrainzTransformerDataStage, logger: MaybeLogger = new MaybeLogger()) => { const { releaseGroupPrimaryTypeAllow = [], releaseGroupPrimaryTypeDeny = [], diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index 2155ac7b..07b77561 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -34,7 +34,8 @@ export interface SearchOptions { escapeCharacters?: boolean removeCharacters?: boolean, using?: ('artist' | 'album' | 'title')[] - ttl?: string + ttl?: string, + freetext?: boolean } export class MusicbrainzApiClient extends AbstractApiClient { @@ -91,7 +92,8 @@ export class MusicbrainzApiClient extends AbstractApiClient { escapeCharacters = true, removeCharacters = false, using = ['album','artist','title'], - ttl + ttl, + freetext } = options || {}; const cacheKey = `mb-recSearch-${hashObject({...playContentInvariantTransform(play), using})}`; @@ -136,25 +138,37 @@ export class MusicbrainzApiClient extends AbstractApiClient { let q = ''; - if(query.recording !== undefined) { - q += `recording:"${query.recording}"`; - } - if(query.artist !== undefined) { - if(q !== '') { - q += ' AND '; + if(freetext) { + + q += `${query.recording ?? ''} `; + if(query.artist !== undefined) { + q += `${(Array.isArray(query.artist) ? query.artist : [query.artist]).join(' ')} `; } - if(Array.isArray(query.artist) && query.artist.length > 1) { - q += `(artist:(${query.artist.map(x => `"${x}"`).join(' AND ')})) OR (artist:(${query.artist.map(x => `"${x}"`).join(' OR ')}))` - } else { - q += `artist:"${Array.isArray(query.artist) ? query.artist[0] : query.artist}"`; + q += `${query.release ?? ''}` + + } else { + if(query.recording !== undefined) { + q += `recording:"${query.recording}"`; } - } - if(query.release !== undefined) { - if(q !== '') { - q += ' AND '; + if(query.artist !== undefined) { + if(q !== '') { + q += ' AND '; + } + if(Array.isArray(query.artist) && query.artist.length > 1) { + q += `(artist:(${query.artist.map(x => `"${x}"`).join(' AND ')})) OR (artist:(${query.artist.map(x => `"${x}"`).join(' OR ')}))` + } else if(query.artist !== undefined) { + q += `artist:"${Array.isArray(query.artist) ? query.artist[0] : query.artist}"`; + } + } + if(query.release !== undefined) { + if(q !== '') { + q += ' AND '; + } + q += `release:"${query.release}"` } - q += `release:"${query.release}"` } + + this.logger.debug(`Search Query => ${q}`); return mb.search('recording', { diff --git a/src/backend/tests/musicbrainz/musicbrainz.test.ts b/src/backend/tests/musicbrainz/musicbrainz.test.ts index d08565a7..7ef5f265 100644 --- a/src/backend/tests/musicbrainz/musicbrainz.test.ts +++ b/src/backend/tests/musicbrainz/musicbrainz.test.ts @@ -5,7 +5,7 @@ import asPromised from 'chai-as-promised'; import { after, before, describe, it } from 'mocha'; import { initMemoryCache } from "../../common/Cache.js"; import { Cacheable } from "cacheable"; -import MusicbrainzTransformer from "../../common/transforms/MusicbrainzTransformer.js"; +import MusicbrainzTransformer, { MusicbrainzTransformerDataStage } from "../../common/transforms/MusicbrainzTransformer.js"; import { PlayObject } from "../../../core/Atomic.js"; import { projectDir } from '../../common/index.js'; import path from 'path'; @@ -32,7 +32,7 @@ const mbTransformer = new MusicbrainzTransformer({ ttl: '1ms' } }, { - logger: loggerTest, + logger: loggerDebug, clientCache: memorycache(), cache: memorycache() }) @@ -140,4 +140,31 @@ describe('Musicbrainz API', function () { expect(res.recordings).to.not.be.empty; }); + it('psuedo-releases', async function () { + + //this.timeout(500000); + + const play: PlayObject = { + data: { + track: "Lagtrain (feat. 宵崎奏, 暁山瑞希 & MEIKO)", + artists: ["Nightcord at 25:00"], + album: "Nightcord at 25:00 SEKAI ALBUM vol.3" + }, + meta: {} + } + await mbTransformer.tryInitialize(); + + const stageConfig: MusicbrainzTransformerDataStage = { + type: "musicbrainz", + searchWhenMissing: ["artists", "album", "title"], + fallbackArtistSearch: "native", + fallbackFreeText: true + }; + + const res = await mbTransformer.getTransformerData(play, stageConfig); + const postFetch = mbTransformer.handlePostFetch(play, res, stageConfig); + expect(res.recordings).to.exist; + expect(res.recordings).to.not.be.empty; + }); + }); \ No newline at end of file -- 2.51.2 From 021ce4d03d5cff76bf9f292b1368ebc1a3abf2a3 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 8 Dec 2025 17:08:56 +0000 Subject: [PATCH 39/41] test: freetext fallback --- src/backend/tests/musicbrainz/musicbrainz.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/tests/musicbrainz/musicbrainz.test.ts b/src/backend/tests/musicbrainz/musicbrainz.test.ts index 7ef5f265..92ac55d5 100644 --- a/src/backend/tests/musicbrainz/musicbrainz.test.ts +++ b/src/backend/tests/musicbrainz/musicbrainz.test.ts @@ -142,13 +142,13 @@ describe('Musicbrainz API', function () { it('psuedo-releases', async function () { - //this.timeout(500000); + this.timeout(3500); const play: PlayObject = { data: { - track: "Lagtrain (feat. 宵崎奏, 暁山瑞希 & MEIKO)", - artists: ["Nightcord at 25:00"], - album: "Nightcord at 25:00 SEKAI ALBUM vol.3" + track: "HIBANA - Reloaded - (feat. 星乃一歌 & Hatsune Miku)", + artists: ["Leo/need"], + album: "Leo / need SEKAI ALBUM Vol.1" }, meta: {} } -- 2.51.2 From 023bd74f122e3fc05fea33dca563ba69a9071fc2 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 8 Dec 2025 17:30:24 +0000 Subject: [PATCH 40/41] docs(transformer): Add freetext search --- .../configuration/transforms/musicbrainz.mdx | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docsite/docs/configuration/transforms/musicbrainz.mdx b/docsite/docs/configuration/transforms/musicbrainz.mdx index d76afc2c..42affa08 100644 --- a/docsite/docs/configuration/transforms/musicbrainz.mdx +++ b/docsite/docs/configuration/transforms/musicbrainz.mdx @@ -155,7 +155,7 @@ The corresponding [Musicbrainz Recording](https://musicbrainz.org/release/85bf28
-If a normal search using all the fields mentioned above does not return any matches Multi-scrobbler can try two additional searches with modified queries: +**If a normal search using all the fields mentioned above does not return any matches Multi-scrobbler can try additional searches with modified queries:** #### Album Only @@ -264,6 +264,35 @@ Then Multi-scrobbler will attempt to extract multiple artists from your artist a +#### Free Text + +If both [Album Only](#album-only) and [Artist Extraction](#artist-extraction) searches fail to return results you can additionally enable **Free Text** search. This will search Musicbrainz for **all** text of your Scrobble data artist/title/album, without constraint. **This is not automatic. You must set `fallbackFreeTextSeach: true` to enable this additional search.** + +:::warning + +Free text search is **unconstrained** which means that Musicbrainz will match text in **any** part of a Recording (artist, release, or recording field), regardless of where it is found. This may lead to results that are unexpected and undesired. + +If you know your Source's Play data is well organized you should not enable this. Free Text search should only be used if: + +* your music is not well organized or +* there may be many alternative titles/artists for the music you listen to (such as with [psuedo-releases](https://wiki.musicbrainz.org/Release#Status)) +* you can tolerate that matches may not be accurate for releases. + * Generally, if you keep [score](#score) high then matched releases should *at least* be the right artist or what you would expect within a reason, as a match. The release may not be the one you actually listened to but it would be "close enough". + +::: + +[Stage Configuration](/configuration/transforms#configuring-stages) example: + +```json5 +// ... +"defaults": { + "releaseStatusPriority": ["official"], + // ... + "fallbackArtistSearch": "native", + "fallbackFreeTextSearch": true + } +``` + ### Refining ### Score -- 2.51.2 From 17baebdbf2e3ad640a34392de055c3b09e67edba Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 8 Dec 2025 19:02:54 +0000 Subject: [PATCH 41/41] feat(transformers): Implement applyng limited transform configs via ENV --- .../configuration/transforms/musicbrainz.mdx | 60 +++++++++++++ .../docs/configuration/transforms/native.mdx | 44 +++++++++ .../transforms/MusicbrainzTransformer.ts | 71 ++++++++++++++- .../common/transforms/TransformerManager.ts | 25 +++++- src/backend/index.ts | 2 + src/backend/scrobblers/ScrobbleClients.ts | 49 ++++++++-- src/backend/sources/ScrobbleSources.ts | 90 ++++++++++++++----- 7 files changed, 312 insertions(+), 29 deletions(-) diff --git a/docsite/docs/configuration/transforms/musicbrainz.mdx b/docsite/docs/configuration/transforms/musicbrainz.mdx index 42affa08..9238bd1b 100644 --- a/docsite/docs/configuration/transforms/musicbrainz.mdx +++ b/docsite/docs/configuration/transforms/musicbrainz.mdx @@ -108,6 +108,66 @@ Each [Rule](/configuration/transforms#stage-rules) should be either a boolean, s } ``` +## ENV Configuration + +The general configuration shown above can also be configured from a selection of *presets* using [ENV Config](/configuration?configType=env#configuration-types) for individual Sources/Clients. + +You must **always** include the `MB_CONTACT` ENV to [identify your application](#api-setup): + +```ini +MB_CONTACT=contact@mydomain.com +``` + +To configure [stage defaults](http://localhost:3000/docs/configuration/transforms/musicbrainz/#stage-configuration) use `MB_PRESETS` with a comma-delimited list of presets you wish to apply. More than one preset can be applied, in which case they combine. You must choose at least one of the presets below: + +* `default` - Applies no defaults. This is the same as using no options from [Matching With Musicbrainz](#matching-with-musicbrainz) +* `sensible` - Applies the [Sensible Default](#sensible-default) configuration for sorting releases +* `native` - Applies `fallbackArtistSearch: native` for [Artist Extraction](#artist-extraction) if the first search fails to find matches +* `aggressive` - Applies [Free Text Search](#free-text) `fallbackFreeTextSearch: true` if all other searches fail + +Finally, use ENV `*_TRANSFORMS=musicbrainz` on each Source/Client you wish to apply this stage to. This applies the stage in the [`preTransform` Hook](#/configuration/transforms#lifecycle-hooks) with all [Rules](#rules) enabled. + +The `*` stands for the prefix used for each Source/Client's ENV keys. Refer to the individual Source/Client Configuration sections to find this. + +:::tip + +Enhancing a scrobble with the `preTransform` hook for a Source means that all Clients that recieve the scrobble get the enhanced version. You only need to apply the transform once, for a Source, to use it everywhere for downstream Clients. + +::: + +
+ +Example Full Docker Deploy with ENV Configuration + +Using [Jellyfin + Maloja](/quickstart#create-docker-compose-file) example from Quickstart: + +```yaml +services: + multi-scrobbler: + image: foxxmd/multi-scrobbler + container_name: multi-scrobbler + environment: + - MB_CONTACT=contact@mydomain.com + - MB_PRESETS=sensible,native # creates sensible defaults with native fallback searching + + - JELLYFIN_URL=192.168.0.110:8096 + - JELLYFIN_APIKEY=c9fae8756fbf481ebd9c5bb56b + - JELLYFIN_USER=MyUser + - JELLYFIN_TRANSFORMS=musicbrainz # applies musicbrainz Stage to preTransform of Jellyfin source + + - MALOJA_URL=http://192.168.0.100:42010 # maloja receives enhanced scrobble from Jellyfin + - MALOJA_API_KEY=myApiKey + + + volumes: + - "./config:/config" + ports: + - "9078:9078" + restart: unless-stopped +``` + +
+ ## Matching with Musicbrainz :::note diff --git a/docsite/docs/configuration/transforms/native.mdx b/docsite/docs/configuration/transforms/native.mdx index 650c7fa0..45f2e773 100644 --- a/docsite/docs/configuration/transforms/native.mdx +++ b/docsite/docs/configuration/transforms/native.mdx @@ -73,6 +73,50 @@ Each [Rule](/configuration/transforms#stage-rules) should be either a boolean, s If a rule is not present then multi-scrobbler defaults it to `true`. +## ENV Configuration + +The default configuration shown above can also be applied using [ENV Config](/configuration?configType=env#configuration-types) for individual Sources/Clients. + +Use ENV `*_TRANSFORMS=native` on each Source/Client you wish to apply this stage to. This applies the stage in the [`preTransform` Hook](#/configuration/transforms#lifecycle-hooks) with all [Rules](#rules) enabled. + +The `*` stands for the prefix used for each Source/Client's ENV keys. Refer to the individual Source/Client Configuration sections to find this. + +:::tip + +Enhancing a scrobble with the `preTransform` hook for a Source means that all Clients that recieve the scrobble get the enhanced version. You only need to apply the transform once, for a Source, to use it everywhere for downstream Clients. + +::: + +
+ +Example Full Docker Deploy with ENV Configuration + +Using [Jellyfin + Maloja](/quickstart#create-docker-compose-file) example from Quickstart: + +```yaml +services: + multi-scrobbler: + image: foxxmd/multi-scrobbler + container_name: multi-scrobbler + environment: + - JELLYFIN_URL=192.168.0.110:8096 + - JELLYFIN_APIKEY=c9fae8756fbf481ebd9c5bb56b + - JELLYFIN_USER=MyUser + - JELLYFIN_TRANSFORMS=native # applies native Stage to preTransform of Jellyfin source + + - MALOJA_URL=http://192.168.0.100:42010 # maloja receives enhanced scrobble from Jellyfin + - MALOJA_API_KEY=myApiKey + + + volumes: + - "./config:/config" + ports: + - "9078:9078" + restart: unless-stopped +``` + +
+ ## Examples ### Parse only artists string using a custom delimiter diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index 815c3246..66626405 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -642,4 +642,73 @@ export const rankReleasesByPriority = (list: IRecordingMatch[], stageConfig: Mus rec.releases.sort((a, b) => b.rankedScore - a.rankedScore); } return rankedList; -}; \ No newline at end of file +}; + +export const DEFAULTS_SENSIBLE = { + // use official release over anything else + "releaseStatusPriority": ["official"], + // prefer album, then single, then ep + "releaseGroupPrimaryTypePriority": ["album", "single", "ep"], + // prefer worldwide release + "releaseCountryPriority": ["XW"] +} +export const DEFAULTS_NATIVE: {fallbackArtistSearch: "native"} = { + "fallbackArtistSearch": "native" +} + +export const DEFAULTS_AGGRESSIVE = { + "fallbackFreeText": true +} + +export const configFromEnv = (logger: MaybeLogger = new MaybeLogger()) => { + const mbEnv = process.env.MB_PRESETS; + const mbContact = process.env.MB_CONTACT; + let mbConfig: MusicbrainzTransformerConfig; + if (mbEnv !== undefined && mbEnv.trim() !== '') { + if (mbContact === undefined || mbContact.trim() === '') { + throw new SimpleError('Must provide a contact url/email for musicbrainz ENV present!'); + } + mbConfig = { + type: 'musicbrainz', + name: 'MSDefault', + data: { + apis: [ + { + contact: mbEnv + } + ] + }, + defaults: { + + } + } + const presets = mbEnv.split(',').map(x => x.trim().toLocaleLowerCase()); + for (const p of presets) { + switch (p) { + case 'default': + break; + case 'sensible': + mbConfig.defaults = { + ...mbConfig.defaults, + ...DEFAULTS_SENSIBLE + } + break; + case 'native': + mbConfig.defaults = { + ...mbConfig.defaults, + ...DEFAULTS_NATIVE + } + break; + case 'aggressive': + mbConfig.defaults = { + ...mbConfig.defaults, + ...DEFAULTS_AGGRESSIVE + } + break; + } + } + logger.debug(`Using presets: ${presets.join(',')}`); + } + + return mbConfig; +} \ No newline at end of file diff --git a/src/backend/common/transforms/TransformerManager.ts b/src/backend/common/transforms/TransformerManager.ts index c2f3b3fa..a11f5911 100644 --- a/src/backend/common/transforms/TransformerManager.ts +++ b/src/backend/common/transforms/TransformerManager.ts @@ -7,9 +7,11 @@ import { PlayObject } from "../../../core/Atomic.js"; import { isStageTyped } from "../../utils/PlayTransformUtils.js"; import { MSCache } from "../Cache.js"; import NativeTransformer from "./NativeTransformer.js"; -import MusicbrainzTransformer, { MusicbrainzTransformerConfig } from "./MusicbrainzTransformer.js"; +import MusicbrainzTransformer, { configFromEnv, MusicbrainzTransformerConfig } from "./MusicbrainzTransformer.js"; import { AsyncLocalStorage } from 'node:async_hooks'; import { nanoid } from "nanoid"; +import { SimpleError } from "../errors/MSErrors.js"; +import { toHaveStyle } from "@testing-library/jest-dom/matchers.js"; export default class TransformerManager { @@ -61,6 +63,27 @@ export default class TransformerManager { this.logger.verbose(`${config.type} transformer with name '${tName}' registered`); } + public async registeryDefaults() { + if(!this.hasTransformerType('user')) { + this.register({type: 'user', name: 'MSDefault'}); + } + if(!this.hasTransformerType('native')) { + this.register({type: 'native', name: 'MSDefault'}); + } + } + + public async registerFromEnv() { + try { + const mbConfig = configFromEnv(this.logger); + this.register(mbConfig); + } catch (e) { + if(e instanceof SimpleError) { + this.logger.error(`Unable to build Musicbrainz Transformer from ENV: ${e.message}`); + } + this.logger.error(new Error('Unable to build Musicbrainz Transformer from ENV', {cause: e})); + } + } + public async initTransformers() { this.logger.verbose('Initializing transformers...'); for (const list of this.transformers.values()) { diff --git a/src/backend/index.ts b/src/backend/index.ts index 4c94afb2..765ae5b1 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -119,6 +119,8 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) const notifiers = new Notifiers(root.get('notifierEmitter'), root.get('clientEmitter'), root.get('sourceEmitter'), root.get('logger')); //root.get('notifiers'); await notifiers.buildWebhooks(webhooks); + await root.items.transformerManager.registerFromEnv(); + await root.items.transformerManager.registeryDefaults(); await root.items.transformerManager.initTransformers(); /* diff --git a/src/backend/scrobblers/ScrobbleClients.ts b/src/backend/scrobblers/ScrobbleClients.ts index 215d8375..b2fb598c 100644 --- a/src/backend/scrobblers/ScrobbleClients.ts +++ b/src/backend/scrobblers/ScrobbleClients.ts @@ -25,6 +25,8 @@ import TealScrobbler from './TealfmScrobbler.js'; import { TealClientConfig } from '../common/infrastructure/config/client/tealfm.js'; import RockskyScrobbler from './RockskyScrobbler.js'; import { RockSkyClientConfig } from '../common/infrastructure/config/client/rocksky.js'; +import { CommonClientOptions } from '../common/infrastructure/config/client/index.js'; +import { ExternalMetadataTerm, PlayTransformHooks } from '../common/infrastructure/Transform.js'; type groupedNamedConfigs = {[key: string]: ParsedConfig[]}; @@ -192,7 +194,8 @@ export default class ScrobbleClients { data: { url, apiKey - } + }, + options: transformPresetEnv('MALOJA') }) } break; @@ -210,7 +213,8 @@ export default class ScrobbleClients { source: 'ENV', mode: 'single', configureAs: 'client', - data: {...lfm, redirectUri: lfm.redirectUri ?? joinedUrl(this.localUrl, 'lastfm/callback').toString()} + data: {...lfm, redirectUri: lfm.redirectUri ?? joinedUrl(this.localUrl, 'lastfm/callback').toString()}, + options: transformPresetEnv('LASTFM') }) } break; @@ -227,7 +231,8 @@ export default class ScrobbleClients { source: 'ENV', mode: 'single', configureAs: 'client', - data: lz + data: lz, + options: transformPresetEnv('LZ') }) } break; @@ -244,7 +249,8 @@ export default class ScrobbleClients { source: 'ENV', mode: 'single', configureAs: 'client', - data: koit + data: koit, + options: transformPresetEnv('KOITO') }) } break; @@ -261,7 +267,8 @@ export default class ScrobbleClients { source: 'ENV', mode: 'single', configureAs: 'client', - data: teal + data: teal, + options: transformPresetEnv('TEALFM') }) } break; @@ -277,7 +284,8 @@ export default class ScrobbleClients { source: 'ENV', mode: 'single', configureAs: 'client', - data: rocksky + data: rocksky, + options: transformPresetEnv('ROCKSKY') }) } break; @@ -468,3 +476,32 @@ ${sources.join('\n')}`); } } } + +const transformPresetEnv = (prefix: string, existing: T = undefined): undefined | T => { + + const env = process.env[`${prefix}_TRANSFORMS`]; + if(env === undefined || env.trim() === '') { + return existing; + } + + const popts: PlayTransformHooks = { + preCompare: [ + ] + } + for(const p of env.split(',').map(x => x.trim().toLocaleLowerCase())) { + switch(p) { + case 'native': + popts.preCompare.push({type: 'native'}); + break; + case 'musicbrainz': + popts.preCompare.push({type: 'musicbrainz'}); + break; + } + } + + // @ts-ignore + return { + ...(existing || {}), + playTransform: popts + }; +} \ No newline at end of file diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index ac0344a9..02602eeb 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -20,7 +20,7 @@ import { LastfmSourceConfig } from "../common/infrastructure/config/source/lastf import { ListenBrainzSourceConfig } from "../common/infrastructure/config/source/listenbrainz.js"; import { MopidySourceConfig } from "../common/infrastructure/config/source/mopidy.js"; import { MusicCastData, MusicCastSourceConfig } from "../common/infrastructure/config/source/musiccast.js"; -import { IcecastData, IcecastSourceConfig } from "../common/infrastructure/config/source/icecast.js"; +import { IcecastData, IcecastSourceConfig, IcecastSourceOptions } from "../common/infrastructure/config/source/icecast.js"; import { MPDSourceConfig } from "../common/infrastructure/config/source/mpd.js"; import { MPRISData, MPRISSourceConfig } from "../common/infrastructure/config/source/mpris.js"; import { MusikcubeData, MusikcubeSourceConfig } from "../common/infrastructure/config/source/musikcube.js"; @@ -73,6 +73,8 @@ import TealfmSource from './TealfmSource.js'; import { TealSourceConfig } from '../common/infrastructure/config/source/tealfm.js'; import { RockskySourceConfig } from '../common/infrastructure/config/source/rocksky.js'; import RockskySource from './RockskySource.js'; +import { CommonSourceOptions } from '../common/infrastructure/config/source/index.js'; +import { ExternalMetadataTerm, PlayTransformHooks, PlayTransformOptions } from '../common/infrastructure/Transform.js'; type groupedNamedConfigs = {[key: string]: ParsedConfig[]}; @@ -339,7 +341,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: s as SpotifySourceData + data: s as SpotifySourceData, + options: transformPresetEnv('SPOTIFY') }) } break; @@ -358,7 +361,7 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: t + data: t, }) } break; @@ -381,7 +384,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: p + data: p, + options: transformPresetEnv('PLEX') }) } break; @@ -398,7 +402,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: sub as SubsonicData + data: sub as SubsonicData, + options: transformPresetEnv('SUBSONIC') }) } break; @@ -425,7 +430,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: j + data: j, + options: transformPresetEnv('JELLYFIN') }) } break; @@ -448,7 +454,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: d as DeezerData + data: d as DeezerData, + options: transformPresetEnv('DEEZER') }); } break; @@ -465,7 +472,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: mp as MPRISData + data: mp as MPRISData, + options: transformPresetEnv('MPRIS') }); } break; @@ -486,7 +494,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: lze as ListenbrainzEndpointData + data: lze as ListenbrainzEndpointData, + options: transformPresetEnv('LZE') }); } break; @@ -502,7 +511,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: lfme as LastFMEndpointData + data: lfme as LastFMEndpointData, + options: transformPresetEnv('LFM') }); } break; @@ -519,9 +529,9 @@ export default class ScrobbleSources { mode: 'single', configureAs: defaultConfigureAs, data: icecast as IcecastData, - options: { + options: transformPresetEnv('ICECAST', { systemScrobble: parseBool(icecast.scrobbleOnStart) - } + }) }); } break; @@ -538,7 +548,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: jr as JRiverData + data: jr as JRiverData, + options: transformPresetEnv('JRIVER') }); } break; @@ -555,7 +566,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: ko as KodiData + data: ko as KodiData, + options: transformPresetEnv('KODI') }); } break; @@ -575,7 +587,8 @@ export default class ScrobbleSources { data: { blacklist: ws.blacklist !== undefined ? ws.blacklist.split(',') : [], whitelist: ws.whitelist !== undefined ? ws.whitelist.split(',') : [], - } + }, + options: transformPresetEnv('WS') }); } break; @@ -599,7 +612,8 @@ export default class ScrobbleSources { whitelistDevices: cc.whitelistDevices !== undefined ? cc.whitelistDevices.split(',') : [], blacklistApps: cc.blacklistApps !== undefined ? cc.blacklistApps.split(',') : [], whitelistApps: cc.whitelistApps !== undefined ? cc.whitelistApps.split(',') : [], - } + }, + options: transformPresetEnv('CC') }); } break; @@ -614,7 +628,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: musecase as MusicCastData + data: musecase as MusicCastData, + options: transformPresetEnv('MCAST') }); } break; @@ -630,7 +645,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: mc as MusikcubeData + data: mc as MusikcubeData, + options: transformPresetEnv('MC') }); } break; @@ -646,7 +662,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: vlc as VLCData + data: vlc as VLCData, + options: transformPresetEnv('VLC') }); } break; @@ -664,7 +681,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: ytm as YTMusicData + data: ytm as YTMusicData, + options: transformPresetEnv('YTM') }); } break; @@ -683,7 +701,8 @@ export default class ScrobbleSources { source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: azura as unknown as AzuracastData + data: azura as unknown as AzuracastData, + options: transformPresetEnv('AZURA') }); } break; @@ -911,3 +930,32 @@ export default class ScrobbleSources { newSource.logger.info(`Source Added from ${source}`); } } + +const transformPresetEnv = (prefix: string, existing: T = undefined): undefined | T => { + + const env = process.env[`${prefix}_TRANSFORMS`]; + if(env === undefined || env.trim() === '') { + return existing; + } + + const popts: PlayTransformHooks = { + preCompare: [ + ] + } + for(const p of env.split(',').map(x => x.trim().toLocaleLowerCase())) { + switch(p) { + case 'native': + popts.preCompare.push({type: 'native'}); + break; + case 'musicbrainz': + popts.preCompare.push({type: 'musicbrainz'}); + break; + } + } + + // @ts-ignore + return { + ...(existing || {}), + playTransform: popts + }; +} \ No newline at end of file