From a3d8372f4895348f2cc99d25521764dec87a3ad4 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 19 Mar 2025 19:54:25 +0000 Subject: [PATCH] feat: More artist parsing improvements * Use more context-aware list parsing with ampersands * Ignore artists with slashes when wrapped by word-boundary --- src/backend/tests/plays/playParsing.test.ts | 49 ++++++++++++++++++- src/backend/tests/utils/PlayTestUtils.ts | 38 ++++++++++++--- src/backend/utils/StringUtils.ts | 52 +++++++++++++++++++-- src/core/Atomic.ts | 4 +- 4 files changed, 129 insertions(+), 14 deletions(-) diff --git a/src/backend/tests/plays/playParsing.test.ts b/src/backend/tests/plays/playParsing.test.ts index 8cd70106..39d5dd19 100644 --- a/src/backend/tests/plays/playParsing.test.ts +++ b/src/backend/tests/plays/playParsing.test.ts @@ -4,7 +4,7 @@ import asPromised from 'chai-as-promised'; import { after, before, describe, it } from 'mocha'; import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js"; -import { parseArtistCredits, parseCredits } from "../../utils/StringUtils.js"; +import { parseArtistCredits, parseContextAwareStringList, parseCredits } from "../../utils/StringUtils.js"; describe('Parsing Artists from String', function() { @@ -18,10 +18,57 @@ describe('Parsing Artists from String', function() { '${str}' Expected => ${allArtists.join(' || ')} Found => ${parsed.join(' || ')}`) + .eql(parsed) } }); + it('Parses & as "local" joiner when other delimiters present', function () { + + const data = [{ + str: `Melendi \\ Ryan Lewis \\ The Righteous Brothers (featuring Joan Jett & The Blackhearts \\ Robin Schulz)`, + expected: ['Melendi', 'Ryan Lewis', 'The Righteous Brothers', 'Joan Jett & The Blackhearts', 'Robin Schulz'] + }, { + str: `Gigi D'Agostino \\ YOASOBI (vs Sam Hunt, Lisa Loeb & Booba)`, + expected: [`Gigi D'Agostino`, 'YOASOBI', 'Sam Hunt', 'Lisa Loeb', 'Booba'] + }]; + + for(const d of data) { + const credits = parseArtistCredits(d.str); + const parsed = [credits.primary].concat(credits.secondary ?? []) + expect(d.expected).eql(parsed) + } + + }); + + it('Only parses & as "global" joiner when no other delimiters present', function () { + + const data = [{ + str: `Melendi & Ryan Lewis & The Righteous Brothers (featuring The Blackhearts \\ Robin Schulz)`, + expected: ['Melendi', 'Ryan Lewis', 'The Righteous Brothers', 'The Blackhearts', 'Robin Schulz'] + }]; + + for(const d of data) { + const credits = parseArtistCredits(d.str); + const parsed = [credits.primary].concat(credits.secondary ?? []) + expect(d.expected).eql(parsed) + } + }); + + it('Parses secondary free regex', function () { + + const data = [{ + str: `Diddy & Grand Funk Railroad feat. Daya & (G)I-DLE`, + expected: ['Diddy', 'Grand Funk Railroad', 'Daya', '(G)I-DLE'] + }]; + + for(const d of data) { + const credits = parseArtistCredits(d.str); + const parsed = [credits.primary].concat(credits.secondary ?? []) + expect(d.expected).eql(parsed) + } + }); + it('Parses singlar Artist with wrapped vs multiple', function () { const [str, primaries, secondaries] = generateArtistsStr({primary: 1, secondary: {num: 2, ft: 'vs', joiner: '/', ftWrap: true}}); const credits = parseArtistCredits(str); diff --git a/src/backend/tests/utils/PlayTestUtils.ts b/src/backend/tests/utils/PlayTestUtils.ts index e9ec580e..ae1a3b1b 100644 --- a/src/backend/tests/utils/PlayTestUtils.ts +++ b/src/backend/tests/utils/PlayTestUtils.ts @@ -9,6 +9,7 @@ import { FEAT, JOINERS, JOINERS_FINAL, JsonPlayObject, ObjectPlayData, PlayMeta, import { sortByNewestPlayDate } from "../../utils.js"; import { NO_DEVICE, NO_USER, PlayerStateDataMaybePlay, PlayPlatformId, ReportedPlayerStatus } from '../../common/infrastructure/Atomic.js'; import { arrayListAnd } from '../../../core/StringUtils.js'; +import { findDelimiters } from '../../utils/StringUtils.js'; dayjs.extend(utc) dayjs.extend(isBetween); @@ -180,14 +181,37 @@ export const generatePlays = (numberOfPlays: number, data: ObjectPlayData = {}, export const generateArtist = () => faker.music.artist; -export const generateArtists = (num?: number, max: number = 3) => { - // if(num !== undefined) { - // return Array(num).map(x => faker.music.artist); - // } +export const generateArtists = (num?: number, max: number = 3, opts: {ambiguousJoinedNames?: boolean, trailingAmpersand?: boolean} = {}) => { if(num === 0 || max === 0) { return []; } - return faker.helpers.multiple(faker.music.artist, {count: {min: num ?? 1, max: num ?? max}}); + let artists = faker.helpers.multiple(faker.music.artist, {count: {min: num ?? 1, max: num ?? max}}); + + const { + trailingAmpersand = false, + ambiguousJoinedNames = false + } = opts; + + if(!trailingAmpersand) { + // its really hard to parse an artist name that contains an '&' when it comes at the end of a list + // because its ambigious if the list is joining the list with & or if & is part of the artist name + // so by default don't generate these (we test for specific scenarios in playParsing.test.ts) + while(artists[artists.length - 1].includes('&')) { + artists = artists.slice(0, artists.length - 1).concat(faker.music.artist()); + } + } + if(!ambiguousJoinedNames) { + artists = artists.map(x => { + let a = x; + let foundDelims = findDelimiters(a); + while(foundDelims !== undefined && foundDelims.length > 0 && !(foundDelims.length === 1 && foundDelims[0] === '&')) { + a = faker.music.artist(); + foundDelims = findDelimiters(a); + } + return a; + }); + } + return artists; } export interface ArtistGenerateOptions { @@ -223,7 +247,7 @@ export const generateArtistsStr = (options: CompoundArtistGenerateOptions = {}): let finalJoinerPrimary: string = joinerPrimary; if(primaryOpts.finalJoiner !== false) { if(primaryOpts.finalJoiner === undefined) { - if(joinerPrimary === ',') { + if(joinerPrimary === ',' && !primaryArt.some(x => x.includes('&'))) { finalJoinerPrimary = faker.helpers.arrayElement(JOINERS_FINAL); } @@ -242,7 +266,7 @@ export const generateArtistsStr = (options: CompoundArtistGenerateOptions = {}): let finalJoinerSecondary: string = joinerSecondary; if(secondaryOpts.finalJoiner !== false) { if(secondaryOpts.finalJoiner === undefined) { - if(joinerSecondary === ',') { + if(joinerSecondary === ',' && !secondaryArt.some(x => x.includes('&'))) { finalJoinerSecondary = faker.helpers.arrayElement(JOINERS_FINAL); } } else { diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts index 276bcbe7..07640756 100644 --- a/src/backend/utils/StringUtils.ts +++ b/src/backend/utils/StringUtils.ts @@ -1,7 +1,7 @@ import { strategies, stringSameness, StringSamenessResult } from "@foxxmd/string-sameness"; import { PlayObject } from "../../core/Atomic.js"; import { asPlayerStateData, DELIMITERS, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; -import { genGroupIdStr, getPlatformIdFromData, parseRegexSingleOrFail } from "../utils.js"; +import { genGroupIdStr, getPlatformIdFromData, intersect, parseRegexSingleOrFail } from "../utils.js"; import { buildTrackString } from "../../core/StringUtils.js"; const {levenStrategy, diceStrategy} = strategies; @@ -61,7 +61,7 @@ export const SECONDARY_CAPTURED_REGEX = new RegExp(/[([]\s*(?ft\.?\W|fea * !!!! ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ******* * * */ -export const SECONDARY_FREE_REGEX = new RegExp(/^\s*(?ft\.?\W|feat\.?\W|featuring|vs\.?\W)\s*(?(?:.+?(?= - |\s*[([]))|(?:.*))(?.*)/i); +export const SECONDARY_FREE_REGEX = new RegExp(/^\s*(?ft\.?\W|feat\.?\W|featuring|vs\.?\W)\s*(?(?:.+?(?= - |\s*[([].+[)\]]$))|(?:.*))(?.*)/i); const SECONDARY_REGEX_STRATS: RegExp[] = [SECONDARY_CAPTURED_REGEX, SECONDARY_FREE_REGEX]; @@ -116,7 +116,7 @@ export const parseCredits = (str: string, delimiters?: boolean | string[]): Play for(const strat of SECONDARY_REGEX_STRATS) { const secCredits = parseRegexSingleOrFail(strat, results.named.secondary); if(secCredits !== undefined) { - secondary = parseStringList(secCredits.named.credits as string, delims) + secondary = parseContextAwareStringList(secCredits.named.credits as string, delims) suffix = secCredits.named.creditsSuffix; break; } @@ -148,7 +148,7 @@ export const parseArtistCredits = (str: string, delimiters?: boolean | string[]) if (withJoiner !== undefined) { // all this does is make sure and "ft" or parenthesis/brackets are separated -- // it doesn't also separate primary artists so do that now - const primaries = parseStringList(withJoiner.primary, delims); + const primaries = parseContextAwareStringList(withJoiner.primary, delims); if (primaries.length > 1) { return { primary: primaries[0], @@ -182,6 +182,50 @@ export const parseStringList = (str: string, delimiters: string[] = [',', '&', ' return explodedStrings.flat(1); }, [str]).map(x => x.trim()); } +export const parseContextAwareStringList = (str: string, delimiters: string[] = [',', '/', '\\'], opts: {ignoreGlobalAmpersand?: boolean} = {}): string[] => { + if (delimiters.length === 0) { + return [str]; + } + // bypass tokens using slashes without spaces + const cleanStr = bypassJoiners(str); + const nonAmpersandDelims = delimiters.some(x => cleanStr.includes(x)); + const shouldIgnoreGlobalAmpersand = opts.ignoreGlobalAmpersand ?? nonAmpersandDelims; + + let awareList: string[] = []; + + const list = parseStringList(cleanStr, nonAmpersandDelims === false && shouldIgnoreGlobalAmpersand === false ? ['&'] : delimiters); + if(shouldIgnoreGlobalAmpersand && list.length > 1 && list[list.length - 1].includes('&') && nonAmpersandDelims) { //&& !list[list.length - 1].includes('& the') + awareList = list.slice(0, list.length - 1).concat(list[list.length - 1].split('&') ); + } else { + awareList = list; + } + return awareList.map(x =>rejoinBypassed(x.trim())); +} + +const bypassJoinerMap = [ + { + rejoin: str => str.replaceAll(/(.*?\S)(\^\^\^)(\S.*?)/g, '$1/$3'), + bypass: str => str.replaceAll(/(.*?\S)(\/)(\S.*?)/g, '$1^^^$3') + }, + { + rejoin: str => str.replaceAll(/(.*)(###)(.*)/g, '$1\\$3'), + bypass: str => str.replaceAll(/(.*\S)(\\)(.*\S)/g, '$1###$3') + } +]; +export const bypassJoiners = (str: string): string => { + let bypassed: string = str; + for(const b of bypassJoinerMap) { + bypassed = b.bypass(bypassed) + } + return bypassed; +} +export const rejoinBypassed = (str: string): string => { + let bypassed: string = str; + for(const b of bypassJoinerMap) { + bypassed = b.rejoin(bypassed) + } + return bypassed; +} export const containsDelimiters = (str: string) => null !== str.match(/[,&/\\]+/i) export const findDelimiters = (str: string) => { const found: string[] = []; diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 18c332c2..17e14d48 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -254,7 +254,7 @@ export interface SourcePlayerObj { play: PlayObject, playFirstSeenAt?: string, playLastUpdatedAt?: string, - playerLastUpdatedAt: strin + playerLastUpdatedAt: string position?: Second listenedDuration: Second status: { @@ -345,7 +345,7 @@ export interface URLData { } export type Joiner = ',' | '&' | '/' | '\\' | string; -export const JOINERS: Joiner[] = [',','&','/','\\']; +export const JOINERS: Joiner[] = [',','/','\\']; export type FinalJoiners = '&'; export const JOINERS_FINAL: FinalJoiners[] = ['&']; -- 2.51.2