diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 7619372d..478a127b 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -60,7 +60,8 @@ import { getRoot } from "../ioc.js"; import { rehydratePlay } from "../utils/CacheUtils.js"; import { findAsyncSequential, staggerMapper } from "../utils/AsyncUtils.js"; import pMap, { pMapIterable } from "p-map"; -import { comparePlayArtistsNormalized, comparePlayTracksNormalized, lifecyclelessInvariantTransform } from "../utils/PlayComparisonUtils.js"; +import { comparePlayArtistsNormalized, comparePlayTracksNormalized, existingScrobble, ExistingScrobbleOpts } from "../utils/PlayComparisonUtils.js"; +import { lifecyclelessInvariantTransform } from "../../core/PlayUtils.js"; import { normalizeStr } from "../utils/StringUtils.js"; import prom, { Counter, Gauge } from 'prom-client'; import { ScrobbleSubmitError, SimpleError } from "../common/errors/MSErrors.js"; @@ -111,6 +112,8 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i npLogger: Logger; dupeLogger: Logger; + existingScrobble: (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => Promise + declare config: CommonClientConfig; notifier: Notifiers; @@ -178,6 +181,13 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.queuedGauge = clientMetrics.queued; this.deadLetterGauge = clientMetrics.deadLetter; this.scrobbledCounter = clientMetrics.scrobbled; + const existingScrobbleOpts: ExistingScrobbleOpts = { + logger: this.dupeLogger, + transformRules: this.transformRules, + transformPlay: this.transformPlay, + existingSubmitted: this.findExistingSubmittedPlayObj + } + this.existingScrobble = (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => existingScrobble(playObjPre, existingScrobbles, existingScrobbleOpts, log) } protected getIdentifier() { @@ -452,162 +462,6 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i return [matchPlayDate, dtInvariantMatches]; } - existingScrobble = async (playObjPre: PlayObject, existingScrobbles: PlayObject[], log: boolean = true): Promise => { - - const result: PlayMatchResult = { - match: false, - score: 0, - breakdowns: [], - reason: 'No existing scrobble matched with a score higher than 0' - }; - - const playObj = await this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate); - if(this.transformRules.compare?.candidate !== undefined) { - result.transformedPlay = playObj; - } - - const tr = truncateStringToLength(27); - const scoreTrackOpts: TrackStringOptions = {include: ['track', 'artist', 'time'], transformers: {track: (t: any, data, existing) => `${existing ? '- ': ''}${tr(t)}`}}; - - // return early if we don't care about checking existing - if (false === this.checkExistingScrobbles) { - this.dupeLogger.trace(`${capitalize(playObj.meta.source ?? 'Source')}: ${buildTrackString(playObj, scoreTrackOpts)} => No Match because existing scrobble check is FALSE`); - result.reason = 'existing scrobble check is FALSE'; - return result; - } - - let existingScrobble; - - // then check if we have already recorded this - const [existingExactSubmitted, existingDataSubmitted = []] = await this.findExistingSubmittedPlayObj(playObjPre); - - // if we have an submitted play with matching data and play date then we can just return the response from the original scrobble - if (existingExactSubmitted !== undefined) { - result.closestMatchedPlay = lifecyclelessInvariantTransform(existingExactSubmitted.play); - result.score = 1; - result.match = true; - result.reason = 'Exact Match found in previously successfully scrobbled plays'; - - existingScrobble = existingExactSubmitted.scrobble; - } - // if not though then we need to check recent scrobbles from scrobble api. - // this will be less accurate than checking existing submitted (obv) but will happen if backlogging or on a fresh server start - - if (existingScrobble === undefined) { - - // if no recent scrobbles found then assume we haven't submitted it - // (either user doesnt want to check history or there is no history to check!) - if (existingScrobbles.length === 0) { - this.dupeLogger.trace(`${buildTrackString(playObj, scoreTrackOpts)} => No Match because no existing scrobbles returned from API`); - result.reason = 'no recent scrobbles returned from API'; - return result; - } - - // only check for fuzzy if we know this play is NOT a repeat - // otherwise we may get a false positive on the previously played track ending time == repeat start time - // -- this is info we only know if play was generated from MS player so we can be reasonably sure - // - // OR if play was generated from a source that uses History (endpoint sources, lfm or lz history sources) - // then we can be reasonably sure that our candidate play has an accurate timestamp and wouldn't fuzzy match a previous scrobble - const looseTimeAccuracy = playObj.data.repeat || playObj.meta.sourceSOT === SOURCE_SOT.HISTORY ? [TA_DURING] : [TA_FUZZY, TA_DURING]; - - - existingScrobble = await findAsyncSequential(existingScrobbles, async (xPre) => { - - const x = await this.transformPlay(xPre, TRANSFORM_HOOK.existing); - - //const referenceMatch = referenceApiScrobbleResponse !== undefined && playObjDataMatch(x, referenceApiScrobbleResponse); - - - const temporalComparison = comparePlayTemporally(x, playObj); - let timeMatch = 0; - if(hasAcceptableTemporalAccuracy(temporalComparison.match)) { - timeMatch = 1; - } else if(hasAcceptableTemporalAccuracy(temporalComparison.match, looseTimeAccuracy)) { - timeMatch = 0.6; - } - - const [titleMatch, titleResults] = comparePlayTracksNormalized(x, playObj); - - const [artistMatch, wholeMatches] = comparePlayArtistsNormalized(x, playObj); - - let artistScore = ARTIST_WEIGHT * artistMatch; - const titleScore = TITLE_WEIGHT * titleMatch; - const timeScore = TIME_WEIGHT * timeMatch; - //const referenceScore = REFERENCE_WEIGHT * (referenceMatch ? 1 : 0); - let score = artistScore + titleScore + timeScore; - - let artistWholeMatchBonus = 0; - let artistBreakdown = `Artist: ${artistMatch.toFixed(2)} * ${ARTIST_WEIGHT} = ${artistScore.toFixed(2)}`; - - if(score < 1 && timeMatch > 0 && titleMatch > 0.98 && artistMatch > 0.1 && wholeMatches > 0 && comparingMultipleArtists(x, playObj)) { - // address scenario where: - // * title is very close - // * time falls within plausible dup range - // * artist is not totally different - // * AND score is still not high enough for a dup - // - // if we detect the plays have multiple artists and we have at least one whole match (stricter comparison than regular score) - // then bump artist score a little to see if it gets it over the fence - // - // EX: Source: The Bongo Hop - Sonora @ 2023-09-28T10:54:06-04:00 => Closest Scrobble: Nidia Gongora / The Bongo Hop - Sonora @ 2023-09-28T10:59:34-04:00 => Score 0.83 => No Match - // one play is only returning primary artist, and timestamp is at beginning instead of end of play - - const scoreBonus = artistMatch * 0.5; - const scoreGapBonus = (1 - artistMatch) * 0.75; - // use the smallest bump or 0.1 - artistWholeMatchBonus = Math.max(scoreBonus, scoreGapBonus, 0.1); - artistScore = (ARTIST_WEIGHT + 0.05) * (artistMatch + artistWholeMatchBonus); - score = artistScore + titleScore + timeScore; - artistBreakdown = `Artist: (${artistMatch.toFixed(2)} + Whole Match Bonus ${artistWholeMatchBonus.toFixed(2)}) * (${ARTIST_WEIGHT} + Whole Match Bonus 0.05) = ${artistScore.toFixed(2)}`; - } - - const scoreBreakdowns = [ - //`Reference: ${(referenceMatch ? 1 : 0)} * ${REFERENCE_WEIGHT} = ${referenceScore.toFixed(2)}`, - artistBreakdown, - `Title: ${titleMatch.toFixed(2)} * ${TITLE_WEIGHT} = ${titleScore.toFixed(2)}`, - `Time: (${capitalize(temporalAccuracyToString(temporalComparison.match))}) ${timeMatch} * ${TIME_WEIGHT} = ${timeScore.toFixed(2)}`, - `Time Detail => ${temporalPlayComparisonSummary(temporalComparison, x, playObj)}`, - `Score ${score.toFixed(2)} => ${score >= DUP_SCORE_THRESHOLD ? 'Matched!' : 'No Match'}` - ]; - - const confidence = `Score ${score.toFixed(2)} => ${score >= DUP_SCORE_THRESHOLD ? 'Matched!' : 'No Match'}` - - if (result.score <= score && score > 0) { - result.reason = confidence; - result.closestMatchedPlay = x; - result.match = score >= DUP_SCORE_THRESHOLD; - result.breakdowns = scoreBreakdowns; - result.score = score; - - if(result.match === false && temporalComparison.match === TA_EXACT && score >= 0.90) { - // if we have a score >= 90 and time is an exact match - // it's likely the differences are due to source-scrobbler data presentation, or deficiencies, - // rather than actually being unique - // so force match in this instance - result.match = true; - result.reason = `Score ${score.toFixed(2)} is not greater than threshold (${DUP_SCORE_THRESHOLD}) but it is very close and timestamp is an exact match, vibe matching.`; - } - } - - return score >= DUP_SCORE_THRESHOLD; - }); - } - - const closestScrobbleParts: string[] = []; - if(result.closestMatchedPlay !== undefined) { - closestScrobbleParts.push(`Closest Scrobble: ${buildTrackString(result.closestMatchedPlay, scoreTrackOpts)}`); - } - closestScrobbleParts.push(result.reason); - let summaryStart = `${capitalize(playObj.meta.source ?? 'Source')}: ${buildTrackString(playObj, scoreTrackOpts)} => ${closestScrobbleParts.join(' => ')}`; - const summary = `${summaryStart}${result.breakdowns.length > 0 ? `\n${result.breakdowns.join('\n')}` : ''}` - result.summary = summary; - if(log) { - this.dupeLogger.trace(summary); - } - return result; - } - public scrobble = async (playObj: PlayObject, opts?: { delay?: number | false }): Promise => { const {delay} = opts || {}; const scrobbleDelay = delay === undefined ? this.scrobbleDelay : (delay === false ? 0 : delay); diff --git a/src/backend/tests/utilitiesTests/playFixtures.test.ts b/src/backend/tests/utilitiesTests/playFixtures.test.ts new file mode 100644 index 00000000..6da8283a --- /dev/null +++ b/src/backend/tests/utilitiesTests/playFixtures.test.ts @@ -0,0 +1,13 @@ +import { loggerTest } from "@foxxmd/logging"; +import { assert, expect } from 'chai'; +import clone from "clone"; +import { describe, it } from 'mocha'; +import { generatePlayWithLifecycle, playWithLifecycleScrobble } from "../../../core/tests/utils/fixtures.js"; + +describe('#PlayFixtures', function () { + + it('Generates a Play with lifecycle', async function () { + await assert.isFulfilled(playWithLifecycleScrobble(generatePlayWithLifecycle())) + }); + +}); \ No newline at end of file diff --git a/src/backend/utils/PlayComparisonUtils.ts b/src/backend/utils/PlayComparisonUtils.ts index e8502670..a2c283c7 100644 --- a/src/backend/utils/PlayComparisonUtils.ts +++ b/src/backend/utils/PlayComparisonUtils.ts @@ -1,12 +1,17 @@ import { getListDiff, ListDiff } from "@donedeal0/superdiff"; -import { PlayObject, PlayObjectLifecycleless, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy } from "../../core/Atomic.js"; -import { buildTrackString } from "../../core/StringUtils.js"; -import { playObjDataMatch, setIntersection } from "../utils.js"; -import { comparePlayTemporally, hasAcceptableTemporalAccuracy, TemporalPlayComparisonOptions } from "./TimeUtils.js"; +import { PlayMatchResult, PlayObject, PlayObjectLifecycleless, SOURCE_SOT, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_DURING, TA_EXACT, TA_FUZZY, TemporalAccuracy, TrackStringOptions } from "../../core/Atomic.js"; +import { buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.js"; +import { comparingMultipleArtists, playObjDataMatch, setIntersection } from "../utils.js"; +import { comparePlayTemporally, hasAcceptableTemporalAccuracy, temporalAccuracyToString, TemporalPlayComparisonOptions, temporalPlayComparisonSummary } from "./TimeUtils.js"; import { compareNormalizedStrings, compareScrobbleArtists, compareScrobbleTracks, compareTracks, normalizeStr, TrackSamenessResults } from "./StringUtils.js"; -import { ARTIST_WEIGHT, TITLE_WEIGHT } from "../common/infrastructure/Atomic.js"; +import { ARTIST_WEIGHT, DUP_SCORE_THRESHOLD, ScrobbledPlayObject, TIME_WEIGHT, TITLE_WEIGHT } from "../common/infrastructure/Atomic.js"; import { StringSamenessResult } from "@foxxmd/string-sameness"; import { Duration } from "dayjs/plugin/duration.js"; +import { PlayTransformRules, TRANSFORM_HOOK, TransformHook } from "../common/infrastructure/Transform.js"; +import { Logger } from "@foxxmd/logging"; +import { loggerNoop } from "../common/logging.js"; +import { lifecyclelessInvariantTransform } from "../../core/PlayUtils.js"; +import { findAsyncSequential } from "./AsyncUtils.js"; export const metaInvariantTransform = (play: PlayObject): PlayObjectLifecycleless => { @@ -56,22 +61,6 @@ export const playContentInvariantTransform = (play: PlayObject): PlayObjectLifec } } -export const lifecyclelessInvariantTransform = (play: PlayObject): PlayObjectLifecycleless => { - const { - meta: { - lifecycle, - ...rest - } = {}, - } = play; - return { - ...play, - meta: { - ...rest - } - } -} - - export type PlayTransformer = (play: PlayObject) => PlayObjectLifecycleless; export type ListTransformers = PlayTransformer[]; @@ -382,4 +371,176 @@ export const scorePlaySameness = (ref: PlayObject, candidate: PlayObject, option export const playDateWithinDurationOfAny = (play: PlayObject, plays: PlayObject[], dur: Duration): PlayObject | undefined => { return plays.find(x => Math.abs(x.data.playDate.diff(play.data.playDate, 's')) <= dur.asSeconds()); -} \ No newline at end of file +} + +export interface ExistingScrobbleOpts { + transformPlay?: (play: PlayObject, hookType: TransformHook) => Promise + existingSubmitted?: (play: PlayObject) => Promise<[ScrobbledPlayObject?, ScrobbledPlayObject[]?]> + transformRules?: PlayTransformRules + checkExistingScrobbles?: boolean + logger?: Logger +} + +export const existingScrobble = async (playObjPre: PlayObject, existingScrobbles: PlayObject[], opts: ExistingScrobbleOpts = {}, log?: boolean): Promise => { + + const { + transformPlay = (play, hook) => play, + existingSubmitted = (play) => [undefined, undefined], + transformRules, + checkExistingScrobbles = true, + logger = loggerNoop + } = opts; + + const result: PlayMatchResult = { + match: false, + score: 0, + breakdowns: [], + reason: 'No existing scrobble matched with a score higher than 0' + }; + + const playObj = await transformPlay(playObjPre, TRANSFORM_HOOK.candidate); + if(transformRules?.compare?.candidate !== undefined) { + result.transformedPlay = playObj; + } + + const tr = truncateStringToLength(27); + const scoreTrackOpts: TrackStringOptions = {include: ['track', 'artist', 'time'], transformers: {track: (t: any, data, existing) => `${existing ? '- ': ''}${tr(t)}`}}; + + // return early if we don't care about checking existing + if (false === checkExistingScrobbles) { + logger.trace(`${capitalize(playObj.meta.source ?? 'Source')}: ${buildTrackString(playObj, scoreTrackOpts)} => No Match because existing scrobble check is FALSE`); + result.reason = 'existing scrobble check is FALSE'; + return result; + } + + let existingScrobble; + + // then check if we have already recorded this + const [existingExactSubmitted, existingDataSubmitted = []] = await existingSubmitted(playObjPre); + + // if we have an submitted play with matching data and play date then we can just return the response from the original scrobble + if (existingExactSubmitted !== undefined) { + result.closestMatchedPlay = lifecyclelessInvariantTransform(existingExactSubmitted.play); + result.score = 1; + result.match = true; + result.reason = 'Exact Match found in previously successfully scrobbled plays'; + + existingScrobble = existingExactSubmitted.scrobble; + } + // if not though then we need to check recent scrobbles from scrobble api. + // this will be less accurate than checking existing submitted (obv) but will happen if backlogging or on a fresh server start + + if (existingScrobble === undefined) { + + // if no recent scrobbles found then assume we haven't submitted it + // (either user doesnt want to check history or there is no history to check!) + if (existingScrobbles.length === 0) { + logger.trace(`${buildTrackString(playObj, scoreTrackOpts)} => No Match because no existing scrobbles returned from API`); + result.reason = 'no recent scrobbles returned from API'; + return result; + } + + // only check for fuzzy if we know this play is NOT a repeat + // otherwise we may get a false positive on the previously played track ending time == repeat start time + // -- this is info we only know if play was generated from MS player so we can be reasonably sure + // + // OR if play was generated from a source that uses History (endpoint sources, lfm or lz history sources) + // then we can be reasonably sure that our candidate play has an accurate timestamp and wouldn't fuzzy match a previous scrobble + const looseTimeAccuracy = playObj.data.repeat || playObj.meta.sourceSOT === SOURCE_SOT.HISTORY ? [TA_DURING] : [TA_FUZZY, TA_DURING]; + + + existingScrobble = await findAsyncSequential(existingScrobbles, async (xPre) => { + + const x = await transformPlay(xPre, TRANSFORM_HOOK.existing); + + //const referenceMatch = referenceApiScrobbleResponse !== undefined && playObjDataMatch(x, referenceApiScrobbleResponse); + + + const temporalComparison = comparePlayTemporally(x, playObj); + let timeMatch = 0; + if(hasAcceptableTemporalAccuracy(temporalComparison.match)) { + timeMatch = 1; + } else if(hasAcceptableTemporalAccuracy(temporalComparison.match, looseTimeAccuracy)) { + timeMatch = 0.6; + } + + const [titleMatch, titleResults] = comparePlayTracksNormalized(x, playObj); + + const [artistMatch, wholeMatches] = comparePlayArtistsNormalized(x, playObj); + + let artistScore = ARTIST_WEIGHT * artistMatch; + const titleScore = TITLE_WEIGHT * titleMatch; + const timeScore = TIME_WEIGHT * timeMatch; + //const referenceScore = REFERENCE_WEIGHT * (referenceMatch ? 1 : 0); + let score = artistScore + titleScore + timeScore; + + let artistWholeMatchBonus = 0; + let artistBreakdown = `Artist: ${artistMatch.toFixed(2)} * ${ARTIST_WEIGHT} = ${artistScore.toFixed(2)}`; + + if(score < 1 && timeMatch > 0 && titleMatch > 0.98 && artistMatch > 0.1 && wholeMatches > 0 && comparingMultipleArtists(x, playObj)) { + // address scenario where: + // * title is very close + // * time falls within plausible dup range + // * artist is not totally different + // * AND score is still not high enough for a dup + // + // if we detect the plays have multiple artists and we have at least one whole match (stricter comparison than regular score) + // then bump artist score a little to see if it gets it over the fence + // + // EX: Source: The Bongo Hop - Sonora @ 2023-09-28T10:54:06-04:00 => Closest Scrobble: Nidia Gongora / The Bongo Hop - Sonora @ 2023-09-28T10:59:34-04:00 => Score 0.83 => No Match + // one play is only returning primary artist, and timestamp is at beginning instead of end of play + + const scoreBonus = artistMatch * 0.5; + const scoreGapBonus = (1 - artistMatch) * 0.75; + // use the smallest bump or 0.1 + artistWholeMatchBonus = Math.max(scoreBonus, scoreGapBonus, 0.1); + artistScore = (ARTIST_WEIGHT + 0.05) * (artistMatch + artistWholeMatchBonus); + score = artistScore + titleScore + timeScore; + artistBreakdown = `Artist: (${artistMatch.toFixed(2)} + Whole Match Bonus ${artistWholeMatchBonus.toFixed(2)}) * (${ARTIST_WEIGHT} + Whole Match Bonus 0.05) = ${artistScore.toFixed(2)}`; + } + + const scoreBreakdowns = [ + //`Reference: ${(referenceMatch ? 1 : 0)} * ${REFERENCE_WEIGHT} = ${referenceScore.toFixed(2)}`, + artistBreakdown, + `Title: ${titleMatch.toFixed(2)} * ${TITLE_WEIGHT} = ${titleScore.toFixed(2)}`, + `Time: (${capitalize(temporalAccuracyToString(temporalComparison.match))}) ${timeMatch} * ${TIME_WEIGHT} = ${timeScore.toFixed(2)}`, + `Time Detail => ${temporalPlayComparisonSummary(temporalComparison, x, playObj)}`, + `Score ${score.toFixed(2)} => ${score >= DUP_SCORE_THRESHOLD ? 'Matched!' : 'No Match'}` + ]; + + const confidence = `Score ${score.toFixed(2)} => ${score >= DUP_SCORE_THRESHOLD ? 'Matched!' : 'No Match'}` + + if (result.score <= score && score > 0) { + result.reason = confidence; + result.closestMatchedPlay = x; + result.match = score >= DUP_SCORE_THRESHOLD; + result.breakdowns = scoreBreakdowns; + result.score = score; + + if(result.match === false && temporalComparison.match === TA_EXACT && score >= 0.90) { + // if we have a score >= 90 and time is an exact match + // it's likely the differences are due to source-scrobbler data presentation, or deficiencies, + // rather than actually being unique + // so force match in this instance + result.match = true; + result.reason = `Score ${score.toFixed(2)} is not greater than threshold (${DUP_SCORE_THRESHOLD}) but it is very close and timestamp is an exact match, vibe matching.`; + } + } + + return score >= DUP_SCORE_THRESHOLD; + }); + } + + const closestScrobbleParts: string[] = []; + if(result.closestMatchedPlay !== undefined) { + closestScrobbleParts.push(`Closest Scrobble: ${buildTrackString(result.closestMatchedPlay, scoreTrackOpts)}`); + } + closestScrobbleParts.push(result.reason); + let summaryStart = `${capitalize(playObj.meta.source ?? 'Source')}: ${buildTrackString(playObj, scoreTrackOpts)} => ${closestScrobbleParts.join(' => ')}`; + const summary = `${summaryStart}${result.breakdowns.length > 0 ? `\n${result.breakdowns.join('\n')}` : ''}` + result.summary = summary; + if(log) { + logger.trace(summary); + } + return result; + } \ No newline at end of file diff --git a/src/core/PlayTestUtils.ts b/src/core/PlayTestUtils.ts index f2735fbd..b418a82d 100644 --- a/src/core/PlayTestUtils.ts +++ b/src/core/PlayTestUtils.ts @@ -5,7 +5,7 @@ import isBetween from "dayjs/plugin/isBetween.js"; import relativeTime from "dayjs/plugin/relativeTime.js"; import timezone from "dayjs/plugin/timezone.js"; import utc from "dayjs/plugin/utc.js"; -import { FEAT, JOINERS, JOINERS_FINAL, JsonPlayObject, MissingMbidType, ObjectPlayData, PlayMeta, PlayObject, SourcePlayerObj } from "./Atomic.js"; +import { BrainzMeta, FEAT, JOINERS, JOINERS_FINAL, JsonPlayObject, MissingMbidType, ObjectPlayData, PlayMeta, PlayObject, SourcePlayerObj } from "./Atomic.js"; import { genGroupIdStr } from './PlayUtils.js'; import { sortByNewestPlayDate } from './PlayUtils.js'; import { CALCULATED_PLAYER_STATUSES, NO_DEVICE, NO_USER, PlayerStateDataMaybePlay, PlayPlatformId, REPORTED_PLAYER_STATUSES, SINGLE_USER_PLATFORM_ID } from '../backend/common/infrastructure/Atomic.js'; @@ -146,8 +146,15 @@ export const generatePlayerStateData = (options: Omit = {}): PlayObject => { - return { +export interface GeneratePlayOpts { + playDateCompleted?: boolean +} +export const generatePlay = (data: ObjectPlayData = {}, meta: MarkOptional = {}, opts: GeneratePlayOpts = {}): PlayObject => { + const { + playDateCompleted = false + } = opts; + + const play: PlayObject = { data: { track: faker.music.songName(), artists: faker.helpers.multiple(faker.music.artist, {count: {min: 1, max: 3}}), @@ -168,6 +175,12 @@ export const generatePlay = (data: ObjectPlayData = {}, meta: MarkOptional): JsonPlayObject => { @@ -183,47 +196,42 @@ export const generateJsonPlays = (...args: Parameters): Js export interface WithBrainzOptions { include: ('track' | 'artist' | 'album')[] } -export const withBrainz = (play: PlayObject, opts: WithBrainzOptions): PlayObject => { +export const generateBrainz = (play: PlayObject, opts: WithBrainzOptions): BrainzMeta => { const {include} = opts; + const brainz: BrainzMeta = {}; for(const i of include) { switch(i) { case 'track': if(play.data.meta?.brainz?.recording === undefined) { - play.data.meta = { - ...(play.data.meta ?? {}), - brainz: { - ...(play.data.meta?.brainz ?? {}), - recording: generateMbid() - } - } + brainz.recording = generateMbid(); } break; case 'album': - if(play.data.meta?.brainz?.album === undefined) { - play.data.meta = { - ...(play.data.meta ?? {}), - brainz: { - ...(play.data.meta?.brainz ?? {}), - album: generateMbid() - } - } + if(play.data.meta?.brainz?.album === undefined && play.data.album !== undefined) { + brainz.album = generateMbid(); } break; case 'artist': - if(play.data.meta?.brainz?.artist === undefined) { + if(play.data.meta?.brainz?.artist === undefined && (play.data.artists ?? []).length > 0) { const artistMbids = play.data.artists.map(x => generateMbid()); - play.data.meta = { - ...(play.data.meta ?? {}), - brainz: { - ...(play.data.meta?.brainz ?? {}), - artist: artistMbids - } - } + brainz.artist = artistMbids; } break; } } + return brainz +} + +export const withBrainz = (play: PlayObject, opts: WithBrainzOptions): PlayObject => { + const brainz = generateBrainz(play, opts); + play.data.meta = { + ...(play.data.meta ?? {}), + brainz : { + ...(play.data.meta?.brainz ?? {}), + ...brainz + } + } return play; } diff --git a/src/core/PlayUtils.ts b/src/core/PlayUtils.ts index ddcb611d..3b8b5859 100644 --- a/src/core/PlayUtils.ts +++ b/src/core/PlayUtils.ts @@ -1,5 +1,5 @@ import { PlayPlatformId } from "../backend/common/infrastructure/Atomic.js"; -import { PlayObject } from "./Atomic.js"; +import { PlayObject, PlayObjectLifecycleless } from "./Atomic.js"; /** sorts playObj formatted objects by playDate in descending (newest first) order */ @@ -27,4 +27,17 @@ export const sortByNewestPlayDate = (a: PlayObject, b: PlayObject) => { };export const genGroupIdStr = (id: PlayPlatformId) => { return `${id[0]}-${id[1]}`; }; +export const lifecyclelessInvariantTransform = (play: PlayObject): PlayObjectLifecycleless => { + const { + meta: { + lifecycle, ...rest + } = {}, + } = play; + return { + ...play, + meta: { + ...rest + } + }; +}; diff --git a/src/core/tests/utils/fixtures.ts b/src/core/tests/utils/fixtures.ts index a764122f..e6b73409 100644 --- a/src/core/tests/utils/fixtures.ts +++ b/src/core/tests/utils/fixtures.ts @@ -1,11 +1,17 @@ import { Traverse, TraverseContext } from 'neotraverse/modern'; import { faker } from '@faker-js/faker'; import dayjs from 'dayjs'; -import { AmbPlayObject, JsonPlayObject, ObjectPlayData, PlayMeta, PlayObject, PlayProgressAmb, REGEX_ISO8601_LOOSE } from '../../Atomic.js'; +import { AmbPlayObject, JsonPlayObject, LifecycleInput, LifecycleStep, ObjectPlayData, PlayMeta, PlayObject, PlayProgressAmb, REGEX_ISO8601_LOOSE, ScrobbleResult } from '../../Atomic.js'; import { ListenRange } from '../../../backend/sources/PlayerState/ListenRange.js'; import { ListenProgressPositional, ListenProgressTS } from '../../../backend/sources/PlayerState/ListenProgress.js'; -import { clone } from 'jsondiffpatch'; import { MarkOptional } from 'ts-essentials'; +import { generateBrainz, generateMbid, generatePlay, GeneratePlayOpts, generatePlays } from '../../PlayTestUtils.js'; +import { lifecyclelessInvariantTransform } from '../../PlayUtils.js'; +import clone from 'clone'; +import { jdiff } from '../../DataUtils.js'; +import { existingScrobble } from '../../../backend/utils/PlayComparisonUtils.js'; +import { playToListenPayload } from '../../../backend/common/vendor/ListenbrainzApiClient.js'; +import { UpstreamError } from '../../../backend/common/errors/UpstreamError.js'; interface BlockPath { key: string, parent: string }; type BlockPaths = BlockPath[]; @@ -82,14 +88,173 @@ export const asPlay = (data: JsonPlayObject): PlayObject => { return cloned as unknown as PlayObject; } +export interface ScrobbleMatchOptions { + match?: boolean + warnings?: boolean + error?: boolean +} export interface GeneratePlayWithLifecycleOptions { - original: { + original?: { data?: ObjectPlayData, meta?: MarkOptional + opts?: GeneratePlayOpts + } +} + +export const generatePlayWithLifecycle = (opts: GeneratePlayWithLifecycleOptions = {}) => { + + const { + original: originalOpts = {}, + } = opts; + + const original = generatePlay(originalOpts.data, originalOpts.meta, originalOpts.opts); + + const lplay: PlayObject = { + data: {}, + meta: { + ...original.meta + } + }; + lplay.meta.lifecycle.original = lifecyclelessInvariantTransform(original); + lplay.meta.lifecycle.input = generateRandomObj(); + + let steps: LifecycleStep[] = []; + let transformedPlay = clone(original); + const firstStep = faker.datatype.boolean(0.7) ? generateLifecycleStep(transformedPlay, {name: 'preCompare'}) : undefined; + if(firstStep !== undefined) { + transformedPlay = firstStep[1]; + steps.push(firstStep[0]); + while(faker.datatype.boolean(0.2)) { + const [pc, modified] = generateLifecycleStep(transformedPlay, {name: 'preCompare'}); + steps.push(pc); + transformedPlay = modified; + } + } + + const lastStep = faker.datatype.boolean(0.5) ? generateLifecycleStep(transformedPlay, {name: 'postCompare'}) : undefined; + if(lastStep !== undefined) { + transformedPlay = lastStep[1]; + steps.push(lastStep[0]); + while(faker.datatype.boolean(0.1)) { + const [pc, modified] = generateLifecycleStep(transformedPlay, {name: 'postCompare'}); + steps.push(pc); + transformedPlay = modified; + } + } + + lplay.meta.lifecycle.steps = steps; + lplay.data = transformedPlay.data; + + return lplay; +} + +export const playWithLifecycleScrobble = async (play: PlayObject, opts: ScrobbleMatchOptions = {}): Promise => { + const { + match = false, + warnings = false, + error = false, + } = opts; + + const scrobbleRes: ScrobbleResult = {}; + + const existingPlays = generatePlays(2); + + if(match) { + existingPlays.push(play); + } + + const res = await existingScrobble(play, existingPlays); + scrobbleRes.match = res; + if(res.match) { + play.meta.lifecycle.scrobble = scrobbleRes; + return play; + } + + scrobbleRes.payload = playToListenPayload(play); + if(error) { + scrobbleRes.error = new Error('Failed to scrobble to client', {cause: new UpstreamError('Client returned a 400 or something')}); + play.meta.lifecycle.scrobble = scrobbleRes; + return play; } + + if(warnings) { + scrobbleRes.warnings = faker.helpers.multiple(faker.lorem.sentence, {count: {min: 1, max: 3}}); + } + + scrobbleRes.response = generateRandomObj(2); + scrobbleRes.mergedScrobble = lifecyclelessInvariantTransform(play); + play.meta.lifecycle.scrobble = scrobbleRes; + + return play; } -export const generatePlayWithLifecycle = (opts: GeneratePlayWithLifecycleOptions) => { +export const generateLifecycleInput = (typeName?: string): LifecycleInput => { + return { type: typeName ?? `${faker.string.alpha(2)}-${faker.hacker.noun()}`, input: generateRandomObj(2) }; +} + +export interface GenerateLifecycleOptions { + name?: string + source?: string + equal?: boolean + inputCount?: number +} + +const modifiableKeys: PropertyKey[] = ['track', 'album', 'albumArtists', 'artists', 'duration', 'meta','brainz']; +export const generateLifecycleStep = (play: PlayObject, opts: GenerateLifecycleOptions = {}): [LifecycleStep, PlayObject] => { + + const { + name = ['preCompare', 'postCompare'][faker.number.int({ min: 0, max: 1 })], + source = `${play.meta.source}-${faker.word.noun()}`, + equal = faker.datatype.boolean(0.1), + inputCount + } = opts; + + const inputs = faker.helpers.multiple(() => generateLifecycleInput(), { count: inputCount ?? { min: 0, max: 2 } }); + + const step: LifecycleStep = { + name, + source, + inputs + } + + if (equal) { + return [step, play]; + } + + play.data.meta = { + ...(play.data?.meta ?? {}), + brainz: { + ...(play.data?.meta?.brainz ?? {}) + } + } + + const modifiedPlay = clone(play); + const randomPlay = generatePlay(); + let somethingModified = false; + while (!somethingModified) { + new Traverse(modifiedPlay).forEach((ctx, x) => { + if (modifiableKeys.includes(ctx.key)) { + if (faker.datatype.boolean(0.3)) { + somethingModified = true; + if(ctx.key === 'brainz' && Object.keys(x).length === 0) { + ctx.update(generateBrainz(play, {include: ['album', 'artist', 'track']}), true); + } else if (ctx.parent !== undefined && ctx.parent.key === 'brainz') { + if (Array.isArray(x)) { + ctx.update(faker.helpers.multiple(generateMbid, { count: { min: 1, max: 3 } })); + } else { + ctx.update(generateMbid()); + } + } else { + ctx.update(randomPlay.data[ctx.key]); + } + } + } + }); + } + + step.patch = jdiff.diff(play, modifiedPlay); + + return [step, modifiedPlay]; } export interface RandomObjOptions { @@ -127,11 +292,11 @@ export const generateRandomObj = (depth: number = 0, opt: RandomObjOptions = {}) const keyCount = opt.keyCount ?? faker.number.int({ min: 1, max: 13 }); - for (let i = 0; i < keyCount; i++) { - const key = faker.lorem.slug({ min: 1, max: 3 }) - if (tgrt[key] === undefined) { - tgrt[key] = generateRandomVal(depth + 1, opt) - } + for (let i = 0; i < keyCount; i++) { + const key = faker.lorem.slug({ min: 1, max: 3 }) + if (tgrt[key] === undefined) { + tgrt[key] = generateRandomVal(depth + 1, opt) } + } return tgrt } \ No newline at end of file