diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index b5bd2ce9..b7ddb419 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -10,6 +10,8 @@ import { PlayObject, QueuedScrobble, TA_CLOSE, + TA_DEFAULT_ACCURACY, + TA_DURING, TA_FUZZY, TrackStringOptions, } from "../../core/Atomic.js"; @@ -42,7 +44,7 @@ import { messageWithCauses, messageWithCausesTruncatedDefault } from "../utils/E import { compareScrobbleArtists, compareScrobbleTracks, normalizeStr } from "../utils/StringUtils.js"; import { comparePlayTemporally, - temporalAccuracyIsAtLeast, + hasAcceptableTemporalAccuracy, temporalAccuracyToString, temporalPlayComparisonSummary, } from "../utils/TimeUtils.js"; @@ -315,7 +317,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i const matchPlayDate = dtInvariantMatches.find((x: ScrobbledPlayObject) => { const temporalComparison = comparePlayTemporally(x.play, playObj); - return temporalAccuracyIsAtLeast(TA_CLOSE, temporalComparison.match) + return hasAcceptableTemporalAccuracy(temporalComparison.match) }); return [matchPlayDate, dtInvariantMatches]; @@ -403,9 +405,9 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i const temporalComparison = comparePlayTemporally(x, playObj); let timeMatch = 0; - if(temporalAccuracyIsAtLeast(TA_CLOSE, temporalComparison.match)) { + if(hasAcceptableTemporalAccuracy(temporalComparison.match)) { timeMatch = 1; - } else if(temporalComparison.match === TA_FUZZY) { + } else if(hasAcceptableTemporalAccuracy(temporalComparison.match, [TA_FUZZY, TA_DURING])) { timeMatch = 0.6; } diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index 1a183818..867fe620 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -2,7 +2,7 @@ import { childLogger, LogDataPretty } from '@foxxmd/logging'; import dayjs, { Dayjs } from "dayjs"; import { EventEmitter } from "events"; import { FixedSizeList } from "fixed-size-list"; -import { PlayObject, TA_CLOSE } from "../../core/Atomic.js"; +import { PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY } from "../../core/Atomic.js"; import { buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.js"; import AbstractComponent from "../common/AbstractComponent.js"; import { @@ -33,7 +33,7 @@ import { sortByNewestPlayDate, sortByOldestPlayDate, } from "../utils.js"; -import { comparePlayTemporally, temporalAccuracyIsAtLeast, timeToHumanTimestamp, todayAwareFormat } from "../utils/TimeUtils.js"; +import { timeToHumanTimestamp, todayAwareFormat } from "../utils/TimeUtils.js"; import { getRoot } from '../ioc.js'; import { componentFileLogger } from '../common/logging.js'; import { WebhookPayload } from '../common/infrastructure/config/health/webhooks.js'; @@ -178,7 +178,7 @@ export default abstract class AbstractSource extends AbstractComponent implement for(const list of lists) { const existing = list.find(x => { const e = this.transformPlay(x, TRANSFORM_HOOK.existing); - return genericSourcePlayMatch(e, candidate, TA_CLOSE); + return genericSourcePlayMatch(e, candidate); }); if(existing) { return existing; diff --git a/src/backend/sources/DeezerInternalSource.ts b/src/backend/sources/DeezerInternalSource.ts index 5770dae7..e2ab1c6c 100644 --- a/src/backend/sources/DeezerInternalSource.ts +++ b/src/backend/sources/DeezerInternalSource.ts @@ -1,7 +1,7 @@ import dayjs from "dayjs"; import EventEmitter from "events"; import request, { Request, Response, SuperAgent } from 'superagent'; -import { PlayObject, SOURCE_SOT, TA_CLOSE, TA_FUZZY } from "../../core/Atomic.js"; +import { PlayObject, SOURCE_SOT, TA_CLOSE, TA_DURING, TA_EXACT, TA_FUZZY, TemporalAccuracy } from "../../core/Atomic.js"; import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig, TRANSFORM_HOOK } from "../common/infrastructure/Atomic.js"; import { DeezerInternalSourceConfig, DeezerInternalTrackData, DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js"; import { parseRetryAfterSecsFromObj, playObjDataMatch, readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js"; @@ -10,6 +10,7 @@ import { CookieJar, Cookie } from 'tough-cookie'; import { MixedCookieAgent } from 'http-cookie-agent/http'; import MemorySource from "./MemorySource.js"; import { genericSourcePlayMatch } from "../utils/PlayComparisonUtils.js"; +import { TemporalPlayComparisonOptions } from "../utils/TimeUtils.js"; interface DeezerHistoryResponse { errors: [] @@ -204,7 +205,7 @@ export default class DeezerInternalSource extends MemorySource { for(const list of lists) { const existing = list.find(x => { const e = this.transformPlay(x, TRANSFORM_HOOK.existing); - return genericSourcePlayMatch(e, candidate, TA_CLOSE); + return genericSourcePlayMatch(e, candidate); }); if(existing) { return existing; @@ -212,7 +213,16 @@ export default class DeezerInternalSource extends MemorySource { if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { const fuzzyIndex = list.findIndex(x => { const e = this.transformPlay(x, TRANSFORM_HOOK.existing); - return genericSourcePlayMatch(e, candidate, TA_FUZZY, {fuzzyDiffThreshold: this.config.options?.fuzzyDiscoveryIgnore === 'aggressive' ? 40 : undefined}); + let temporalOptions: TemporalPlayComparisonOptions = {}; + const temporalAccuracy: TemporalAccuracy[] = [TA_EXACT, TA_CLOSE, TA_FUZZY]; + if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { + temporalOptions = { + fuzzyDiffThreshold: Math.max(100, x.data.duration * 0.5), + duringReferences: ['duration', 'listenedFor', 'range'] + } + temporalAccuracy.push(TA_DURING); + } + return genericSourcePlayMatch(e, candidate, temporalAccuracy, temporalOptions); }); if(fuzzyIndex !== -1) { if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { diff --git a/src/backend/sources/JellyfinSource.ts b/src/backend/sources/JellyfinSource.ts index dc4d1a57..6e9aa002 100644 --- a/src/backend/sources/JellyfinSource.ts +++ b/src/backend/sources/JellyfinSource.ts @@ -1,7 +1,7 @@ import { Logger } from "@foxxmd/logging"; import dayjs from "dayjs"; import EventEmitter from "events"; -import { PlayObject, TA_CLOSE } from "../../core/Atomic.js"; +import { PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT } from "../../core/Atomic.js"; import { buildTrackString, combinePartsToString, @@ -19,7 +19,7 @@ import { import { parseDurationFromTimestamp } from '../utils/TimeUtils.js'; import { comparePlayTemporally, - temporalAccuracyIsAtLeast, + hasAcceptableTemporalAccuracy, temporalPlayComparisonSummary, } from "../utils/TimeUtils.js"; import MemorySource from "./MemorySource.js"; @@ -313,7 +313,7 @@ export default class JellyfinSource extends MemorySource { Temporal Comparison => ${temporalPlayComparisonSummary(temporalResult, currPlay, playObj)}`); } - if(temporalAccuracyIsAtLeast(TA_CLOSE,temporalResult.match)) { + if(hasAcceptableTemporalAccuracy(temporalResult.match)) { existingTracked = currPlay; } break; diff --git a/src/backend/tests/source/source.test.ts b/src/backend/tests/source/source.test.ts index e60906c8..2761cecd 100644 --- a/src/backend/tests/source/source.test.ts +++ b/src/backend/tests/source/source.test.ts @@ -545,6 +545,20 @@ describe('Deezer Internal Source', function() { expect(discovered.length).to.eq(0); }); + it('does not discover play found during duration of previous', function() { + const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80}); + const targetPlay = normalizedPlays[normalizedPlays.length - 2] + const duringPlay = clone(targetPlay); + duringPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration * 0.5, 's'); + + const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); + source.discover([...normalizedPlays, interimPlay]); + + const discovered = source.discover([duringPlay]); + + expect(discovered.length).to.eq(0); + }); + it('does not discover fuzzy play with delay of up to 40 seconds', function() { const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80}); const targetPlay = normalizedPlays[normalizedPlays.length - 2] diff --git a/src/backend/utils/PlayComparisonUtils.ts b/src/backend/utils/PlayComparisonUtils.ts index 18919422..81333201 100644 --- a/src/backend/utils/PlayComparisonUtils.ts +++ b/src/backend/utils/PlayComparisonUtils.ts @@ -1,8 +1,8 @@ import { getListDiff, ListDiff } from "@donedeal0/superdiff"; -import { PlayObject, TA_CLOSE, TemporalAccuracy } from "../../core/Atomic.js"; +import { PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy } from "../../core/Atomic.js"; import { buildTrackString } from "../../core/StringUtils.js"; import { playObjDataMatch } from "../utils.js"; -import { comparePlayTemporally, temporalAccuracyIsAtLeast, TemporalPlayComparisonOptions } from "./TimeUtils.js"; +import { comparePlayTemporally, hasAcceptableTemporalAccuracy, TemporalPlayComparisonOptions } from "./TimeUtils.js"; export const metaInvariantTransform = (play: PlayObject): PlayObject => { @@ -224,4 +224,6 @@ export const humanReadableDiff = (aPlay: PlayObject[], bPlay: PlayObject[], resu }).join('\n'); } -export const genericSourcePlayMatch = (a: PlayObject, b: PlayObject, t: TemporalAccuracy = TA_CLOSE, temporalOptions?: TemporalPlayComparisonOptions): boolean => playObjDataMatch(a, b) && temporalAccuracyIsAtLeast(t, comparePlayTemporally(a, b, temporalOptions).match); \ No newline at end of file +export const genericSourcePlayMatch = (a: PlayObject, b: PlayObject, t?: TemporalAccuracy[], temporalOptions?: TemporalPlayComparisonOptions): boolean => + playObjDataMatch(a, b) + && hasAcceptableTemporalAccuracy(comparePlayTemporally(a, b, temporalOptions).match, t); \ No newline at end of file diff --git a/src/backend/utils/TimeUtils.ts b/src/backend/utils/TimeUtils.ts index dca8dc5c..54b5ecc3 100644 --- a/src/backend/utils/TimeUtils.ts +++ b/src/backend/utils/TimeUtils.ts @@ -1,11 +1,14 @@ import dayjs, { Dayjs } from "dayjs"; import isToday from 'dayjs/plugin/isToday.js'; import { + AcceptableTemporalDuringReference, PlayObject, SCROBBLE_TS_SOC_END, SCROBBLE_TS_SOC_START, ScrobbleTsSOC, TA_CLOSE, + TA_DEFAULT_ACCURACY, + TA_DURING, TA_EXACT, TA_FUZZY, TA_NONE, @@ -38,19 +41,18 @@ export const temporalPlayComparisonSummary = (data: TemporalPlayComparison, exis parts.push(`Play Diff: ${formatNumber(data.date.diff, {toFixed: 0})}s (Needed <${data.date.threshold}s)`) } if (data.date.fuzzyDurationDiff !== undefined) { - parts.push(`Fuzzy Duration Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= 10s)`); + parts.push(`Fuzzy Duration Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= ${data.date.fuzzyDiffThreshold}s)`); } if (data.date.fuzzyListenedDiff !== undefined) { - parts.push(`Fuzzy Listened Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= 10s)`); + parts.push(`Fuzzy Listened Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= ${data.date.fuzzyDiffThreshold}s)`); } - if (data.range !== undefined) { - if (data.range === false) { - parts.push('Candidate not played during Existing tracked listening'); - } else { - parts.push(`Candidate played during tracked listening range from Existing ${data.range[0].timestamp.format('HH:mm:ssZ')} => ${data.range[1].timestamp.format('HH:mm:ssZ')}`); - } - } else { + + if(data.range === undefined) { parts.push('Range Comparison N/A'); + } else if(data.range.type === 'none') { + parts.push(`Candidate not played during Existing ${data.duringReferences.join(' or ')}`); + } else { + parts.push(`Candidate played during tracked listening range from Existing "${data.range.type}" ${data.range.timestamps[0].format('HH:mm:ssZ')} => ${data.range.timestamps[1].format('HH:mm:ssZ')}`); } return parts.join(' | '); } @@ -59,14 +61,11 @@ export interface TemporalPlayComparisonOptions { diffThreshold?: number, fuzzyDuration?: boolean, fuzzyDiffThreshold?: number - useListRanges?: boolean + duringReferences?: AcceptableTemporalDuringReference } export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: PlayObject, options: TemporalPlayComparisonOptions = {}): TemporalPlayComparison => { - const result: TemporalPlayComparison = { - match: TA_NONE - }; const { meta: { @@ -103,9 +102,14 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P diffThreshold = lowGranularitySources.some(x => x.toLocaleLowerCase() === source) ? 60 : 10, fuzzyDuration = false, fuzzyDiffThreshold = 10, - useListRanges = true, + duringReferences = ['range'] } = options; + const result: TemporalPlayComparison = { + match: TA_NONE, + duringReferences + }; + // cant compare! if (existingTsSOCDate === undefined || candidateTsSOCDate === undefined) { return result; @@ -120,7 +124,8 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P const scrobblePlayDiff = Math.abs(existingTsSOCDate.unix() - candidateTsSOCDate.unix()); result.date = { threshold: diffThreshold, - diff: scrobblePlayDiff + diff: scrobblePlayDiff, + fuzzyDiffThreshold }; if(scrobblePlayDiff <= 1) { @@ -129,22 +134,50 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P result.match = TA_CLOSE; } - if (useListRanges && existingRanges !== undefined) { - // since we know when the existing track was listened to - // we can check if the new track play date took place while the existing one was being listened to - // which would indicate (assuming same source) the new track is a duplicate - for (const range of existingRanges) { - if (candidateTsSOCDate.isBetween(range.start.timestamp, range.end.timestamp)) { - result.range = range; - if(!temporalAccuracyIsAtLeast(TA_CLOSE, result.match)) { - result.match = TA_CLOSE; + if(result.match !== TA_NONE) { + return result; + } + + if(duringReferences.length > 0) { + + if (duringReferences.includes('range') && existingRanges !== undefined) { + // since we know when the existing track was listened to + // we can check if the new track play date took place while the existing one was being listened to + // which would indicate (assuming same source) the new track is a duplicate + for (const range of existingRanges) { + if (candidateTsSOCDate.isBetween(range.start.timestamp, range.end.timestamp)) { + result.range = { + type: 'range', + timestamps: [range.start.timestamp, range.end.timestamp] + } + result.match = TA_DURING; + return result; + } + } + } + + if(duringReferences.includes('listenedFor') && existingPlay.data.listenedFor !== undefined) { + if (candidateTsSOCDate.isBetween(existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.listenedFor, 's'))) { + result.match = TA_DURING; + result.range = { + type: 'listenedFor', + timestamps: [existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.listenedFor, 's')] } - break; + return result; } } - if (result.range === undefined) { - result.range = false; + + if(duringReferences.includes('duration') && existingPlay.data.duration !== undefined) { + if (candidateTsSOCDate.isBetween(existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.duration, 's'))) { + result.match = TA_DURING; + result.range = { + type: 'duration', + timestamps: [existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.duration, 's')] + } + return result; + } } + } // if the source has a duration its possible one play was scrobbled at the beginning of the track and the other at the end @@ -199,15 +232,7 @@ export const timePassesScrobbleThreshold = (thresholds: ScrobbleThresholds, seco } } -export const temporalAccuracyIsAtLeast = (expected: TemporalAccuracy, found: TemporalAccuracy): boolean => { - if(typeof expected === 'number') { - if(typeof found === 'number') { - return found <= expected; - } - return false; - } - return found === false; -} +export const hasAcceptableTemporalAccuracy = (found: TemporalAccuracy, expected: TemporalAccuracy[] = TA_DEFAULT_ACCURACY): boolean => expected.includes(found); export const temporalAccuracyToString = (acc: TemporalAccuracy): string => { switch(acc) { @@ -217,7 +242,9 @@ export const temporalAccuracyToString = (acc: TemporalAccuracy): string => { return 'close'; case 3: return 'fuzzy'; - case false: + case 4: + return 'during'; + case 99: return 'no correlation'; } } diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index c54f00e7..d69ba893 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -244,7 +244,7 @@ export interface DeadLetterScrobble extends QueuedS export type Second = number; export type Millisecond = number; -export type TemporalAccuracy = 1 | 2 | 3 | false; +export type TemporalAccuracy = 1 | 2 | 3 | 4 | 99; /** Timestamp diffs are close to exact (less than or equal to 1 second difference) */ export const TA_EXACT: TemporalAccuracy = 1; @@ -253,10 +253,20 @@ export const TA_EXACT: TemporalAccuracy = 1; * low granularity (subsonic usually) is 60 seconds */ export const TA_CLOSE: TemporalAccuracy = 2; -/** Timestamp diffs are not EXACT/CLOSE but Scrobble A's timestamp +/- duration is within 10 seconds of Scrobble B's timestamp */ +/** Timestamp diffs are not CLOSE but Scrobble A's timestamp +/- duration is within fuzzyDiffThreshold seconds of Scrobble B's timestamp */ export const TA_FUZZY: TemporalAccuracy = 3; +/** Timestamp diffs are not FUZZY and Scrobble B's timestamp is within potential full play of Scrobble A (timestamp +/- duration) */ +export const TA_DURING: TemporalAccuracy = 4; /** No correlation between timestamps */ -export const TA_NONE: TemporalAccuracy = false; +export const TA_NONE: TemporalAccuracy = 99; + +export type AcceptableTemporaryAccuracy = TemporalAccuracy[] + +export const TA_DEFAULT_ACCURACY: AcceptableTemporaryAccuracy = [TA_EXACT, TA_CLOSE]; + +export type TemporalDuringReference = 'range' | 'duration' | 'listenedFor'; + +export type AcceptableTemporalDuringReference = TemporalDuringReference[]; export interface TemporalPlayComparison { match: TemporalAccuracy @@ -265,8 +275,13 @@ export interface TemporalPlayComparison { diff: number fuzzyDurationDiff?: number fuzzyListenedDiff?: number + fuzzyDiffThreshold?: number } - range?: false | ListenRangeData + duringReferences: AcceptableTemporalDuringReference + range?: { + timestamps: [Dayjs, Dayjs] + type: TemporalDuringReference + } | { type: 'none' } } export type SOURCE_SOT_TYPES = 'player' | 'history';