From 4cd9ac92c2244c32262bfcd46cc4f07bf01e9407 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 7 Oct 2026 14:29:07 +0000 Subject: [PATCH] recator(musicbrainz): Extract lucene string building to own functions to reduce repeated code --- .../common/vendor/musicbrainz/LuceneUtils.ts | 96 ++++++++ .../musicbrainz/MusicbrainzApiClientPool.ts | 217 +++--------------- src/backend/tests/musicbrainz/lucene.test.ts | 30 +++ 3 files changed, 164 insertions(+), 179 deletions(-) create mode 100644 src/backend/common/vendor/musicbrainz/LuceneUtils.ts create mode 100644 src/backend/tests/musicbrainz/lucene.test.ts diff --git a/src/backend/common/vendor/musicbrainz/LuceneUtils.ts b/src/backend/common/vendor/musicbrainz/LuceneUtils.ts new file mode 100644 index 00000000..69931e97 --- /dev/null +++ b/src/backend/common/vendor/musicbrainz/LuceneUtils.ts @@ -0,0 +1,96 @@ +/** Field name => value(s) to search for. Keys are used verbatim as lucene field names, undefined values are ignored */ +export type LuceneQuery = Record; + +export interface LuceneQueryOptions { + /** Operator used between fields, defaults to AND */ + operator?: 'AND' | 'OR' + /** + * Fields that should prefer matching all of their values, with matching any value as fallback + * + * Multiple values for fields not listed here are matched with OR + */ + preferAll?: (keyof T)[] +} + +export interface LuceneCleanOptions { + escapeCharacters?: boolean + removeCharacters?: boolean +} + +export const LUCENE_SPECIAL_CHARACTER_REGEX: string[] = ['\\','+','-','&&','||','!','(',')','{','}','[',']','^','"','~','*','?',':','/']; +/** + * https://lucene.apache.org/core/7_7_2/queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package.description + * https://beta.musicbrainz.org/doc/MusicBrainz_API/Search + * */ +export const escapeLuceneSpecialChars = (str: string): string => { + let cleaned = str; + for(const char of LUCENE_SPECIAL_CHARACTER_REGEX) { + cleaned = cleaned.replaceAll(char, `\\$&`); + } + return cleaned; +} + +const NON_WORD_ADJACENT_BOUNDARY_REGEX: RegExp = new RegExp(/(?<=\w)[^a-zA-Z\d\s](?=\w)/g); +const NON_WORDWHITESPACE_REGEX: RegExp = new RegExp(/[^a-zA-Z\d\s]/g); +export const removeNonWordCharacters = (str: string): string => { + // replace any non-alphanumeric, non-whitespace characters that are surrounded by non-whitespace characters + // with a whitespace EX "My Cool-Fun Title" => "My Cool Fun Title" + let cleaned = str.replaceAll(NON_WORD_ADJACENT_BOUNDARY_REGEX, ' '); + + // remove any non-alphanumeric, non-whitespace characters + // with a whitespace EX "My Cool (Title)" => "My Cool Title" + cleaned = cleaned.replaceAll(NON_WORDWHITESPACE_REGEX, ''); + return cleaned; +} + +const fieldValues = (val: string | string[] | undefined): string[] => val === undefined ? [] : (Array.isArray(val) ? val : [val]); + +/** + * Returns a copy of the query with the values of the given fields escaped and/or stripped of non-word characters + * + * Only pass fields with free text values (names, titles). Identifiers like MBIDs should be left out so they are used verbatim + */ +export const cleanLuceneFields = (query: T, fields: (keyof T)[], options: LuceneCleanOptions = {}): T => { + const { + escapeCharacters = true, + removeCharacters = false + } = options; + const clean = (str: string): string => { + const cleaned = escapeCharacters ? escapeLuceneSpecialChars(str) : str; + return removeCharacters ? removeNonWordCharacters(cleaned) : cleaned; + } + const cleaned: LuceneQuery = {...query}; + for(const field of fields as string[]) { + const val = cleaned[field]; + if(val !== undefined) { + cleaned[field] = Array.isArray(val) ? val.map(clean) : clean(val); + } + } + return cleaned as T; +} + +/** Builds a fielded lucene query string. Fields are output in the order they were added to the query object */ +export const buildLuceneQuery = (query: T, options: LuceneQueryOptions = {}): string => { + const { + operator = 'AND', + preferAll = [] + } = options; + const clauses: string[] = []; + for(const [field, val] of Object.entries(query)) { + const quoted = fieldValues(val).map(x => `"${x}"`); + if(quoted.length === 0) { + continue; + } + if(quoted.length === 1) { + clauses.push(`${field}:${quoted[0]}`); + } else if(preferAll.includes(field)) { + clauses.push(`(${field}:(${quoted.join(' AND ')}) OR ${field}:(${quoted.join(' OR ')}))`); + } else { + clauses.push(`${field}:(${quoted.join(' OR ')})`); + } + } + return clauses.join(` ${operator} `); +} + +/** Builds a query string with no field names, using only the values of the given fields in the order given */ +export const buildFreetextQuery = (query: T, fields: (keyof T)[]): string => fields.flatMap(x => fieldValues(query[x])).join(' '); diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClientPool.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClientPool.ts index 019f158a..b406d9a4 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClientPool.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClientPool.ts @@ -23,6 +23,7 @@ import {ProxyWithCircuitBreaker, type CircuitBreakerProxy} from '@foxxmd/load-ba import {ConsecutiveBreaker} from 'cockatiel'; import { MusicbrainzApiWrapped } from './MusicbrainzApi.ts'; import { formatNumber } from '../../../../core/DataUtils.ts'; +import { buildFreetextQuery, buildLuceneQuery, cleanLuceneFields } from './LuceneUtils.ts'; import type { ArtistSearchQueryOpts, ReleaseSearchQueryOpts, TrackSearchQueryOpts } from './MusicbrainzTypes.ts'; export interface SubmitResponse { payload?: { @@ -211,21 +212,7 @@ export class MusicbrainzApiClientPool extends AbstractApiClient { trackMbid = creditMbid(play.data.track, 'track'), releaseMbid = creditMbid(play.data.album, 'release'), artistMbids = creditIds(play.data.artists, 'musicbrainz', 'artist'); - if(recordingMbid !== undefined && using.includes('mbidrecording')) { - query.rid = recordingMbid; - } - if(trackMbid !== undefined && using.includes('mbidtrack')) { - query.tid = trackMbid; - } - if(releaseMbid !== undefined && using.includes('mbidrelease')) { - query.reid = releaseMbid; - } - if(artistMbids.length > 0 && using.includes('mbidartist')) { - query.arid = artistMbids; - } - if(play.data.isrc !== undefined && using.includes('isrc')) { - query.isrc = isrcNoHyphens(play.data.isrc); - } + // output order of fields in the query string follows the order they are added here if(using.includes('title')) { query.recording = creditToName(play.data.track); } @@ -235,79 +222,26 @@ export class MusicbrainzApiClientPool extends AbstractApiClient { if(play.data.album !== undefined && using.includes('album')) { query.release = creditToName(play.data.album); } - // only names are cleaned, MBIDs and ISRC are used verbatim - const nameEntries = () => Object.entries({recording: query.recording, artist: query.artist, release: query.release}).filter(([,v]) => v !== undefined); - if(escapeCharacters) { - for(const [k,v] of nameEntries()) { - (query as any)[k] = Array.isArray(v) ? v.map(escapeLuceneSpecialChars) : escapeLuceneSpecialChars(v); - } + if(play.data.isrc !== undefined && using.includes('isrc')) { + query.isrc = isrcNoHyphens(play.data.isrc); } - if(removeCharacters) { - for(const [k,v] of nameEntries()) { - (query as any)[k] = Array.isArray(v) ? v.map(removeNonWordCharacters) : removeNonWordCharacters(v); - } + if(recordingMbid !== undefined && using.includes('mbidrecording')) { + query.rid = recordingMbid; } - - if(freetext) { - - q += `${query.recording ?? ''} `; - if(query.artist !== undefined) { - q += `${(Array.isArray(query.artist) ? query.artist : [query.artist]).join(' ')} `; - } - q += `${query.release ?? ''}` - - } else { - if(query.recording !== undefined) { - q += `recording:"${query.recording}"`; - } - if(query.artist !== undefined) { - if(q !== '') { - q += ' AND '; - } - if(Array.isArray(query.artist) && query.artist.length > 1) { - q += `(artist:(${query.artist.map(x => `"${x}"`).join(' AND ')}) OR artist:(${query.artist.map(x => `"${x}"`).join(' OR ')}))` - } else if(query.artist !== undefined) { - q += `artist:"${Array.isArray(query.artist) ? query.artist[0] : query.artist}"`; - } - } - if(query.release !== undefined) { - if(q !== '') { - q += ' AND '; - } - q += `release:"${query.release}"` - } - if(query.isrc !== undefined) { - if(q !== '') { - q += ' AND '; - } - q+= `isrc:${query.isrc}`; - } - if(query.rid !== undefined) { - if(q !== '') { - q += ' AND '; - } - q += `rid:"${query.rid}"` - } - if(query.tid !== undefined) { - if(q !== '') { - q += ' AND '; - } - q += `tid:"${query.tid}"` - } - if(query.arid !== undefined) { - if(q !== '') { - q += ' AND '; - } - q += `(arid:(${query.arid.map(x => `"${x}"`).join(' AND ')}) OR arid:(${query.arid.map(x => `"${x}"`).join(' OR ')}))` - } - if(query.reid !== undefined) { - if(q !== '') { - q += ' AND '; - } - q += `reid:"${query.reid}"` - } + if(trackMbid !== undefined && using.includes('mbidtrack')) { + query.tid = trackMbid; + } + if(artistMbids.length > 0 && using.includes('mbidartist')) { + query.arid = artistMbids; + } + if(releaseMbid !== undefined && using.includes('mbidrelease')) { + query.reid = releaseMbid; } + // only names are cleaned, MBIDs and ISRC are used verbatim + const nameFields: (keyof TrackSearchQueryOpts)[] = ['recording', 'artist', 'release']; + const cleaned = cleanLuceneFields(query, nameFields, {escapeCharacters, removeCharacters}); + q = freetext ? buildFreetextQuery(cleaned, nameFields) : buildLuceneQuery(cleaned, {preferAll: ['artist', 'arid']}); this.logger.debug(`Search Query => ${q}`); this.cache.set(`${cacheKey}-qs`, q); @@ -357,53 +291,26 @@ export class MusicbrainzApiClientPool extends AbstractApiClient { const query: ReleaseSearchQueryOpts = { }; - // only names are cleaned, MBIDs are used verbatim - const clean = (str: string): string => { - let cleaned = escapeCharacters ? escapeLuceneSpecialChars(str) : str; - if(removeCharacters) { - cleaned = removeNonWordCharacters(cleaned); - } - return cleaned; - } - const releaseMbid = creditMbid(data.album, 'release'), releaseGroupMbid = creditMbid(data.album, 'release-group'); + // output order of fields in the query string follows the order they are added here + if(data.album !== undefined && using.includes('album')) { + query.release = creditToName(data.album); + } + if(data.artists !== undefined && data.artists.length > 0 && using.includes('artist')) { + query.artistname = creditsToNames(data.artists); + } if(using.includes('mbidrelease')) { if(releaseMbid !== undefined) { query.reid = [releaseMbid]; } query.rgid = releaseGroupMbid; } - if(data.album !== undefined && using.includes('album')) { - query.release = clean(creditToName(data.album)); - } - if(data.artists !== undefined && data.artists.length > 0 && using.includes('artist')) { - query.artistname = creditsToNames(data.artists).map(clean); - } - if(freetext) { - q = [query.release, ...(query.artistname ?? [])].filter(x => x !== undefined).join(' '); - } else { - const quoted = (vals: string[]) => vals.map(x => `"${x}"`); - const parts: string[] = []; - if(query.release !== undefined) { - parts.push(`release:"${query.release}"`); - } - if(query.artistname !== undefined) { - if(query.artistname.length > 1) { - parts.push(`(artistname:(${quoted(query.artistname).join(' AND ')}) OR artistname:(${quoted(query.artistname).join(' OR ')}))`); - } else { - parts.push(`artistname:"${query.artistname[0]}"`); - } - } - if(query.reid !== undefined) { - parts.push(`reid:(${quoted(query.reid).join(' OR ')})`); - } - if(query.rgid !== undefined) { - parts.push(`rgid:"${query.rgid}"`); - } - q = parts.join(' AND '); - } + // only names are cleaned, MBIDs are used verbatim + const nameFields: (keyof ReleaseSearchQueryOpts)[] = ['release', 'artistname']; + const cleaned = cleanLuceneFields(query, nameFields, {escapeCharacters, removeCharacters}); + q = freetext ? buildFreetextQuery(cleaned, nameFields) : buildLuceneQuery(cleaned, {preferAll: ['artistname']}); this.logger.debug(`Search Query => ${q}`); this.cache.set(`${cacheKey}-qs`, q); @@ -449,42 +356,21 @@ export class MusicbrainzApiClientPool extends AbstractApiClient { const query: ArtistSearchQueryOpts = { }; - // only names are cleaned, MBIDs are used verbatim - const clean = (str: string): string => { - let cleaned = escapeCharacters ? escapeLuceneSpecialChars(str) : str; - if(removeCharacters) { - cleaned = removeNonWordCharacters(cleaned); - } - return cleaned; - } - const artistMbids = creditIds(data.artists, 'musicbrainz', 'artist'); - if(artistMbids.length > 0) { - query.arid = artistMbids; - } + // output order of fields in the query string follows the order they are added here if(data.artists !== undefined && data.artists.length > 0) { - query.artist = creditsToNames(data.artists).map(clean); + query.artist = creditsToNames(data.artists); query.primary_alias = query.artist; } - - if(freetext) { - q = (query.artist ?? []).join(' '); - } else { - // each result is a single artist so clauses are OR'd, a result only needs to match one of the given artists - const anyOf = (vals: string[]) => `(${vals.map(x => `"${x}"`).join(' OR ')})`; - const parts: string[] = []; - if(query.artist !== undefined) { - parts.push(`artist:${anyOf(query.artist)}`); - } - if(query.primary_alias !== undefined) { - parts.push(`primary_alias:${anyOf(query.primary_alias)}`); - } - if(query.arid !== undefined) { - parts.push(`arid:${anyOf(query.arid)}`); - } - q = parts.join(' OR '); + if(artistMbids.length > 0) { + query.arid = artistMbids; } + // only names are cleaned, MBIDs are used verbatim + const cleaned = cleanLuceneFields(query, ['artist', 'primary_alias'], {escapeCharacters, removeCharacters}); + // each result is a single artist so fields are OR'd, a result only needs to match one of the given artists + q = freetext ? buildFreetextQuery(cleaned, ['artist']) : buildLuceneQuery(cleaned, {operator: 'OR'}); + this.logger.debug(`Search Query => ${q}`); this.cache.set(`${cacheKey}-qs`, q); @@ -567,30 +453,3 @@ export const recordingToPlay = (data: IRecording, options?: {ignoreVA?: boolean} return baseFormatPlayObj(data, play); } - - -export const LUCENE_SPECIAL_CHARACTER_REGEX: string[] = ['\\','+','-','&&','||','!','(',')','{','}','[',']','^','"','~','*','?',':','/']; -/** - * https://lucene.apache.org/core/7_7_2/queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package.description - * https://beta.musicbrainz.org/doc/MusicBrainz_API/Search - * */ -export const escapeLuceneSpecialChars = (str: string): string => { - let cleaned = str; - for(const char of LUCENE_SPECIAL_CHARACTER_REGEX) { - cleaned = cleaned.replaceAll(char, `\\$&`); - } - return cleaned; -} - -const NON_WORD_ADJACENT_BOUNDARY_REGEX: RegExp = new RegExp(/\w([^a-zA-Z\d\s])\w/g); -const NON_WORDWHITESPACE_REGEX: RegExp = new RegExp(/[^a-zA-Z\d\s]/g); -export const removeNonWordCharacters = (str: string): string => { - // replace any non-alphanumeric, non-whitespace characters that are surrounded by non-whitespace characters - // with a whitespace EX "My Cool-Fun Title" => "My Cool Fun Title" - let cleaned = str.replaceAll(NON_WORD_ADJACENT_BOUNDARY_REGEX, ' '); - - // remove any non-alphanumeric, non-whitespace characters - // with a whitespace EX "My Cool (Title)" => "My Cool Title" - cleaned = cleaned.replaceAll(NON_WORDWHITESPACE_REGEX, ''); - return cleaned; -} diff --git a/src/backend/tests/musicbrainz/lucene.test.ts b/src/backend/tests/musicbrainz/lucene.test.ts new file mode 100644 index 00000000..3c64282f --- /dev/null +++ b/src/backend/tests/musicbrainz/lucene.test.ts @@ -0,0 +1,30 @@ +import { describe, it } from 'mocha'; +import { expect } from 'chai'; +import { buildFreetextQuery, buildLuceneQuery, cleanLuceneFields } from '../../common/vendor/musicbrainz/LuceneUtils.ts'; + +describe('#LuceneUtils', function () { + + it('builds fields in insertion order, skipping undefined and empty values', function () { + expect(buildLuceneQuery({recording: 'Song', artist: ['A'], release: undefined, arid: [], rid: '1-2'})) + .to.eq('recording:"Song" AND artist:"A" AND rid:"1-2"'); + }); + + it('matches any of multiple values unless field prefers all', function () { + const query = {artist: ['A', 'B'], arid: ['1', '2']}; + expect(buildLuceneQuery(query, {preferAll: ['artist']})) + .to.eq('(artist:("A" AND "B") OR artist:("A" OR "B")) AND arid:("1" OR "2")'); + expect(buildLuceneQuery(query, {operator: 'OR'})) + .to.eq('artist:("A" OR "B") OR arid:("1" OR "2")'); + }); + + it('cleans only the given fields', function () { + const query = {release: 'AC/DC (Live)', reid: ['1-2']}; + expect(cleanLuceneFields(query, ['release'])).to.eql({release: 'AC\\/DC \\(Live\\)', reid: ['1-2']}); + expect(cleanLuceneFields(query, ['release'], {escapeCharacters: false, removeCharacters: true})).to.eql({release: 'AC DC Live', reid: ['1-2']}); + }); + + it('builds freetext from only the given fields', function () { + expect(buildFreetextQuery({recording: 'Song', artist: ['A', 'B'], release: undefined, rid: '1'}, ['recording', 'artist', 'release'])) + .to.eq('Song A B'); + }); +}); -- 2.51.2