diff --git a/src/backend/common/database/drizzle/repositories/PlayRepository.ts b/src/backend/common/database/drizzle/repositories/PlayRepository.ts index 777a664f..6e4bbc4f 100644 --- a/src/backend/common/database/drizzle/repositories/PlayRepository.ts +++ b/src/backend/common/database/drizzle/repositories/PlayRepository.ts @@ -659,7 +659,7 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> { return undefined; } return res.map(x => ({...x, play: hydratePlaySelect(x)})).find(x => { - const temporalComparison = comparePlayTemporally(x.play, play); + const temporalComparison = comparePlayTemporally(x.play, play, {logger: this.logger}); return hasAcceptableTemporalAccuracy(temporalComparison.match, taAccuracy) }) } diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index a049c991..91126455 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -64,7 +64,6 @@ import { todayAwareFormat } from "../../core/TimeUtils.js"; import { WebhookPayload } from "../common/infrastructure/config/health/webhooks.js"; import { AsyncTask, SimpleIntervalJob, Task, ToadScheduler } from "toad-scheduler"; import { getRoot } from "../ioc.js"; -import { rehydratePlay } from "../utils/CacheUtils.js"; import { findAsyncSequential, staggerMapper, StaggerOptions } from "../utils/AsyncUtils.js"; import pMap, { pMapIterable } from "p-map"; import { comparePlayArtistsNormalized, comparePlayTracksNormalized, existingScrobble, ExistingScrobbleOpts } from "../utils/PlayComparisonUtils.js"; @@ -830,7 +829,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } const matchPlayDate = dtInvariantMatches.find((x: PlayObject) => { - const temporalComparison = comparePlayTemporally(x, playObj); + const temporalComparison = comparePlayTemporally(x, playObj, {logger: this.logger}); return hasAcceptableTemporalAccuracy(temporalComparison.match) }); diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index e7992905..288e5ee4 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -39,7 +39,7 @@ import { getRoot } from '../ioc.js'; import { componentFileLogger } from '../common/logging.js'; import { WebhookPayload } from '../common/infrastructure/config/health/webhooks.js';; import { messageWithCausesTruncatedDefault } from "../../core/ErrorUtils.js"; -import { existingScrobble, ExistingScrobbleOpts, genericSourcePlayMatch } from '../utils/PlayComparisonUtils.js'; +import { existingScrobble, ExistingScrobbleOpts } from '../utils/PlayComparisonUtils.js'; import { findAsync, staggerMapper, StaggerOptions } from '../utils/AsyncUtils.js'; import pMap, {pMapIterable} from 'p-map'; import prom, { Counter, Gauge } from 'prom-client'; diff --git a/src/backend/sources/DeezerInternalSource.ts b/src/backend/sources/DeezerInternalSource.ts index a8112c47..1914c7e8 100644 --- a/src/backend/sources/DeezerInternalSource.ts +++ b/src/backend/sources/DeezerInternalSource.ts @@ -348,7 +348,8 @@ export default class DeezerInternalSource extends MemorySource { if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { temporalOptions = { fuzzyDiffThreshold: Math.max(100, x.data.duration * 0.5), - duringReferences: ['duration', 'listenedFor', 'range'] + duringReferences: ['duration', 'listenedFor', 'range'], + logger: this.logger } temporalAccuracy.push(TA_DURING); } diff --git a/src/backend/utils/CacheUtils.ts b/src/backend/utils/CacheUtils.ts deleted file mode 100644 index 796b87fc..00000000 --- a/src/backend/utils/CacheUtils.ts +++ /dev/null @@ -1,48 +0,0 @@ -import dayjs from 'dayjs'; -import duration from 'dayjs/plugin/duration.js'; -import isBetween from 'dayjs/plugin/isBetween.js'; -import relativeTime from 'dayjs/plugin/relativeTime.js'; -import isToday from 'dayjs/plugin/isToday.js'; -import timezone from 'dayjs/plugin/timezone.js'; -import utc from 'dayjs/plugin/utc.js'; -import { AmbPlayObject, ListenRangeData, ListenRangeDataAmb, PlayObject, PlayProgress, PlayProgressAmb } from '../../core/Atomic.js'; - -dayjs.extend(utc) -dayjs.extend(isBetween); -dayjs.extend(relativeTime); -dayjs.extend(duration); -dayjs.extend(timezone); -dayjs.extend(isToday); - -export const rehydratePlay = (obj: AmbPlayObject): PlayObject => { - if(obj.data?.playDate !== undefined && typeof obj.data.playDate === 'string') { - obj.data.playDate = dayjs(obj.data.playDate); - if(obj.data.playDateCompleted !== undefined) { - obj.data.playDateCompleted = dayjs(obj.data.playDateCompleted); - } - } - if(obj.data.listenRanges !== undefined) { - obj.data.listenRanges = obj.data.listenRanges.map(rehydrateListenRangeData).filter(x => x !== undefined); - } - return obj as PlayObject; -} - -// this may become problematic since we aren't re-instantiating Progress class, just implementing interface -// but that may only be an issue if rehydrating source data which isn't in scope so far -export const rehydrateListenRangeData = (obj: ListenRangeDataAmb): ListenRangeData => { - // workaround for incompatible rehydration - if(obj.start === undefined || obj.end === undefined) { - return undefined; - } - return { - start: rehydratePlayProgress(obj.start), - end: rehydratePlayProgress(obj.end) - } as ListenRangeData; -} - -export const rehydratePlayProgress = (obj: PlayProgressAmb): PlayProgress => { - if(typeof obj.timestamp === 'string') { - obj.timestamp = dayjs(obj.timestamp); - } - return obj as PlayProgress; -} \ No newline at end of file diff --git a/src/backend/utils/PlayComparisonUtils.ts b/src/backend/utils/PlayComparisonUtils.ts index 4672cf3b..853ae118 100644 --- a/src/backend/utils/PlayComparisonUtils.ts +++ b/src/backend/utils/PlayComparisonUtils.ts @@ -503,7 +503,7 @@ export const existingScrobble = async (playObjPre: PlayObject, existingScrobbles //const referenceMatch = referenceApiScrobbleResponse !== undefined && playObjDataMatch(x, referenceApiScrobbleResponse); - const temporalComparison = comparePlayTemporally(x, playObj); + const temporalComparison = comparePlayTemporally(x, playObj, {logger}); let timeMatch = 0; if(hasAcceptableTemporalAccuracy(temporalComparison.match)) { timeMatch = 1; diff --git a/src/backend/utils/TimeUtils.ts b/src/backend/utils/TimeUtils.ts index 36e2aef4..6e8233ce 100644 --- a/src/backend/utils/TimeUtils.ts +++ b/src/backend/utils/TimeUtils.ts @@ -35,6 +35,8 @@ import { InvalidRegexError, SimpleError } from "../common/errors/MSErrors.js"; import { NamedGroup, parseRegex } from "@foxxmd/regex-buddy-core"; import { Duration } from "dayjs/plugin/duration.js"; import { SourceType } from "../common/infrastructure/config/source/sources.js"; +import { Logger } from "@foxxmd/logging"; +import { loggerNoop } from "../common/MaybeLogger.js"; //dayjs.extend(isToday); @@ -73,6 +75,7 @@ export interface TemporalPlayComparisonOptions { fuzzyDuration?: boolean, fuzzyDiffThreshold?: number duringReferences?: AcceptableTemporalDuringReference + logger?: Logger } export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: PlayObject, options: TemporalPlayComparisonOptions = {}): TemporalPlayComparison => { @@ -113,7 +116,8 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P diffThreshold = getTemporalAccuracyCloseVal(source as SourceType), fuzzyDuration = false, fuzzyDiffThreshold = 10, - duringReferences = ['range'] + duringReferences = ['range'], + logger = loggerNoop } = options; const result: TemporalPlayComparison = { @@ -151,20 +155,28 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P if(duringReferences.length > 0) { - if (duringReferences.includes('range') && existingRanges !== undefined) { - // since we know when the existing track was listened to - // we can check if the new track play date took place while the existing one was being listened to - // which would indicate (assuming same source) the new track is a duplicate - for (const range of existingRanges) { - if (candidateTsSOCDate.isBetween(range.start.timestamp, range.end.timestamp)) { - result.range = { - type: 'range', - timestamps: [range.start.timestamp, range.end.timestamp] + // guard against old, badly cached/marshalled range data: + // this should not be an issue in 0.14.0+ since listenprogress has been updated to be a plain object + // but for folks migrating with very old cached data it could end up being an issue + try { + if (duringReferences.includes('range') && existingRanges !== undefined) { + // since we know when the existing track was listened to + // we can check if the new track play date took place while the existing one was being listened to + // which would indicate (assuming same source) the new track is a duplicate + for (const range of existingRanges) { + + if (candidateTsSOCDate.isBetween(range.start.timestamp, range.end.timestamp)) { + result.range = { + type: 'range', + timestamps: [range.start.timestamp, range.end.timestamp] + } + result.match = TA_DURING; + return result; } - result.match = TA_DURING; - return result; } } + } catch (e) { + logger.warn(new Error('Failed to compare plays based on range but will continue', {cause: e})); } if(duringReferences.includes('listenedFor') && existingPlay.data.listenedFor !== undefined) {