import dayjs, { type Dayjs } from "dayjs"; //import isToday from 'dayjs/plugin/isToday.js'; import { type AcceptableTemporalDuringReference, type PlayObject, SCROBBLE_TS_SOC_END, SCROBBLE_TS_SOC_START, type ScrobbleTsSOC, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_DURING, TA_EXACT, TA_FUZZY, TA_NONE, type TemporalAccuracy, type TemporalPlayComparison, type UnixTimestamp, } from "../../core/Atomic.ts"; import { capitalize, stringIsOnlyNumbers } from "../../core/StringUtils.ts"; import { DEFAULT_CLOSE_POSITION_ABSOLUTE, DEFAULT_CLOSE_POSITION_PERCENT, DEFAULT_DURATION_REPEAT_ABSOLUTE, DEFAULT_DURATION_REPEAT_PERCENT, DEFAULT_SCROBBLE_DURATION_THRESHOLD, DEFAULT_SCROBBLE_PERCENT_THRESHOLD, type DurationValue, lowGranularitySources, type ScrobbleThresholdResult, } from "../common/infrastructure/Atomic.ts"; import type {ScrobbleThresholds} from "../common/infrastructure/config/source/index.ts"; import { formatNumber } from '../../core/DataUtils.ts'; import { InvalidRegexError, SimpleError } from "../common/errors/MSErrors.ts"; import { type NamedGroup, parseRegex } from "@foxxmd/regex-buddy-core"; import type {Duration} from "dayjs/plugin/duration.js"; import type {SourceType} from "../../core/Atomic.ts"; import type {Logger} from "@foxxmd/logging"; import { loggerNoop } from "../common/MaybeLogger.ts"; //dayjs.extend(isToday); export const temporalPlayComparisonSummary = (data: TemporalPlayComparison, existingPlay?: PlayObject, candidatePlay?: PlayObject) => { const parts: string[] = []; if (existingPlay !== undefined && candidatePlay !== undefined) { if (existingPlay.data.playDate.isSame(candidatePlay.data.playDate, 'day')) { parts.push(`Existing: ${existingPlay.data.playDate.format('HH:mm:ssZ')} - Candidate: ${candidatePlay.data.playDate.format('HH:mm:ssZ')}`); } else { parts.push(`Existing: ${existingPlay.data.playDate.toISOString()} - Candidate: ${candidatePlay.data.playDate.toISOString()}`); } } parts.push(`Temporal Sameness: ${capitalize(temporalAccuracyToString(data.match))}`); if (data.date !== undefined) { parts.push(`Play Diff: ${formatNumber(data.date.diff, {toFixed: 0})}s (Needed <${data.date.threshold}s)`) } if (data.date.fuzzyDurationDiff !== undefined) { parts.push(`Fuzzy Duration Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= ${data.date.fuzzyDiffThreshold}s)`); } if (data.date.fuzzyListenedDiff !== undefined) { parts.push(`Fuzzy Listened Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= ${data.date.fuzzyDiffThreshold}s)`); } if(data.range === undefined) { parts.push('Range Comparison N/A'); } else if(data.range.type === 'none') { parts.push(`Candidate not played during Existing ${data.duringReferences.join(' or ')}`); } else { parts.push(`Candidate played during tracked listening range from Existing "${data.range.type}" ${data.range.timestamps[0].format('HH:mm:ssZ')} => ${data.range.timestamps[1].format('HH:mm:ssZ')}`); } return parts.join(' | '); } export interface TemporalPlayComparisonOptions { diffThreshold?: number, fuzzyDuration?: boolean, fuzzyDiffThreshold?: number duringReferences?: AcceptableTemporalDuringReference logger?: Logger } export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: PlayObject, options: TemporalPlayComparisonOptions = {}): TemporalPlayComparison => { const { meta: { source, //scrobbleTsSOC: existingScrobbleTsSOC = SCROBBLE_TS_SOC_START, }, data: { // playDate: existingPlayDate, // playDateCompleted: existingPlayDateCompleted, duration: existingDuration, listenRanges: existingRanges, listenedFor: existingListenedFor, } } = existingPlay; const [existingTsSOCDate, existingTsSOC] = getScrobbleTsSOCDateWithContext(existingPlay); const { // meta: { // scrobbleTsSOC: candidateScrobbleTsSOC = SCROBBLE_TS_SOC_START, // }, data: { // playDate: newPlayDate, // playDateCompleted: candidatePlayDateCompleted, duration: newDuration, listenRanges: newRanges, listenedFor: newListenedFor, } } = candidatePlay; const [candidateTsSOCDate, candidateTsSOC] = getScrobbleTsSOCDateWithContext(candidatePlay); const { diffThreshold = getTemporalAccuracyCloseVal(source as SourceType), fuzzyDuration = false, fuzzyDiffThreshold = 10, duringReferences = ['range'], logger = loggerNoop } = options; const result: TemporalPlayComparison = { match: TA_NONE, duringReferences }; // cant compare! if (existingTsSOCDate === undefined || candidateTsSOCDate === undefined) { return result; } const referenceDuration = newDuration ?? existingDuration; const referenceListenedFor = newListenedFor ?? existingListenedFor; const playDiffThreshold = diffThreshold; // check if existing play time is same as new play date const scrobblePlayDiff = Math.abs(existingTsSOCDate.unix() - candidateTsSOCDate.unix()); result.date = { threshold: diffThreshold, diff: scrobblePlayDiff, fuzzyDiffThreshold }; if(scrobblePlayDiff <= 1) { result.match = TA_EXACT; } else if (scrobblePlayDiff <= playDiffThreshold) { result.match = TA_CLOSE; } if(result.match !== TA_NONE) { return result; } if(duringReferences.length > 0) { // guard against old, badly cached/marshalled range data: // this should not be an issue in 0.14.0+ since listenprogress has been updated to be a plain object // but for folks migrating with very old cached data it could end up being an issue try { if (duringReferences.includes('range') && existingRanges !== undefined) { // since we know when the existing track was listened to // we can check if the new track play date took place while the existing one was being listened to // which would indicate (assuming same source) the new track is a duplicate for (const range of existingRanges) { if (candidateTsSOCDate.isBetween(range.start.timestamp, range.end.timestamp)) { result.range = { type: 'range', timestamps: [range.start.timestamp, range.end.timestamp] } result.match = TA_DURING; return result; } } } } catch (e) { logger.warn(new Error('Failed to compare plays based on range but will continue', {cause: e})); } // NOTE: these two checks intentionally anchor on existingPlay.data.playDate directly, // NOT existingTsSOCDate -- existingTsSOCDate can resolve to playDateCompleted instead of // playDate depending on the play's scrobbleTsSOC, which would put the window in the wrong // place entirely (starting from roughly when the track ended, not when it started) if(duringReferences.includes('listenedFor') && existingPlay.data.listenedFor !== undefined) { const listenedForEnd = existingPlay.data.playDate.add(existingPlay.data.listenedFor, 's'); if (candidateTsSOCDate.isBetween(existingPlay.data.playDate, listenedForEnd)) { result.match = TA_DURING; result.range = { type: 'listenedFor', timestamps: [existingPlay.data.playDate, listenedForEnd] } return result; } } if(duringReferences.includes('duration') && existingPlay.data.duration !== undefined) { // prefer the play's actual observed completion time over the nominal duration -- // a real listen can take longer than the track's length if it was paused partway through, // and playDateCompleted reflects that real wall-clock span while playDate + duration doesn't const durationEnd = existingPlay.data.playDateCompleted ?? existingPlay.data.playDate.add(existingPlay.data.duration, 's'); if (candidateTsSOCDate.isBetween(existingPlay.data.playDate, durationEnd)) { result.match = TA_DURING; result.range = { type: 'duration', timestamps: [existingPlay.data.playDate, durationEnd] } return result; } } } // if the source has a duration its possible one play was scrobbled at the beginning of the track and the other at the end // so check if the duration matches the diff between the two play dates if (result.match === TA_NONE && referenceDuration !== undefined) { result.date.fuzzyDurationDiff = Math.abs(scrobblePlayDiff - referenceDuration); if (result.date.fuzzyDurationDiff <= fuzzyDiffThreshold) { // TODO use finer comparison for this? result.match = TA_FUZZY; } } // if the source has listened duration (maloja) it may differ from actual track duration // and its possible (spotify) the candidate play date is set at the end of this duration // so check if there is a close match between candidate play date and source + listened for if (result.match === TA_NONE && referenceListenedFor !== undefined && fuzzyDuration) { result.date.fuzzyListenedDiff = Math.abs(scrobblePlayDiff - referenceListenedFor); if (result.date.fuzzyListenedDiff <= fuzzyDiffThreshold) { // TODO use finer comparison for this? result.match = TA_FUZZY } } return result; } export const timePassesScrobbleThreshold = (thresholds: ScrobbleThresholds, secondsTracked: number, playDuration?: number): ScrobbleThresholdResult => { let durationPasses = undefined, percentPasses = undefined, percent: number | undefined; const durationThreshold: number | null = thresholds.duration ?? DEFAULT_SCROBBLE_DURATION_THRESHOLD, percentThreshold: number | null = thresholds.percent ?? DEFAULT_SCROBBLE_PERCENT_THRESHOLD; if (percentThreshold !== null && playDuration !== undefined && playDuration !== 0) { percent = Math.round(((secondsTracked / playDuration) * 100)); percentPasses = percent >= percentThreshold; } if (durationThreshold !== null || percentPasses === undefined) { durationPasses = secondsTracked >= durationThreshold; } return { passes: (durationPasses ?? false) || (percentPasses ?? false), duration: { passes: durationPasses, threshold: durationThreshold, value: secondsTracked }, percent: { passes: percentPasses, value: percent, threshold: percentThreshold } } } export const hasAcceptableTemporalAccuracy = (found: TemporalAccuracy, expected: TemporalAccuracy[] = TA_DEFAULT_ACCURACY): boolean => expected.includes(found); export const temporalAccuracyToString = (acc: TemporalAccuracy): string => { switch(acc) { case 1: return 'exact'; case 2: return 'close'; case 3: return 'fuzzy'; case 4: return 'during'; case 99: return 'no correlation'; } } export const getTemporalAccuracyCloseVal = (source: SourceType): number => { return lowGranularitySources.includes(source) ? 60 : 10; } export const getScrobbleTsSOCDateWithContext = (data: PlayObject): [Dayjs, ScrobbleTsSOC] => { const { meta: { scrobbleTsSOC = SCROBBLE_TS_SOC_START, }, data: { playDate = dayjs(), playDateCompleted } } = data; if(scrobbleTsSOC === SCROBBLE_TS_SOC_END && playDateCompleted !== undefined) { return [playDateCompleted, SCROBBLE_TS_SOC_END]; } return [playDate, SCROBBLE_TS_SOC_START]; } export const getScrobbleTsSOCDate = (data: PlayObject): Dayjs => { const [date, _] = getScrobbleTsSOCDateWithContext(data); return date; } export const parseDurationFromTimestamp = (timestamp: any) => { if (timestamp === null || timestamp === undefined) { return undefined; } if (!(typeof timestamp === 'string')) { throw new Error('Timestamp must be a string'); } if (timestamp.trim() === '') { return undefined; } const parsedRuntime = timestamp.split(':'); let hours = '0', minutes = '0', seconds = '0', milli = '0'; switch (parsedRuntime.length) { case 3: hours = parsedRuntime[0]; minutes = parsedRuntime[1]; seconds = parsedRuntime[2]; break; case 2: minutes = parsedRuntime[0]; seconds = parsedRuntime[1]; break; case 1: seconds = parsedRuntime[0]; } const splitSec = seconds.split('.'); if (splitSec.length > 1) { seconds = splitSec[0]; milli = splitSec[1]; } return dayjs.duration({ hours: Number.parseInt(hours), minutes: Number.parseInt(minutes), seconds: Number.parseInt(seconds), milliseconds: Number.parseInt(milli) }); }; /** Is Position earlier than X seconds or Y% percent of the start of a Play? */ export const closeToPlayStart = (play: PlayObject, position: number, thresholds: {absolute?: number, percent?: number, hintPrefix?: boolean} = {}): [boolean, string] => { const { absolute = DEFAULT_CLOSE_POSITION_ABSOLUTE, percent = DEFAULT_CLOSE_POSITION_PERCENT, hintPrefix = true } = thresholds; const hintStart = hintPrefix ? `Position (${position}) ` : ''; const trackDur = play.data.duration; const closeStartNum = position <= absolute; const hints: string[] = []; hints.push(`${closeStartNum ? 'is' : 'is not'} within ${absolute}s of track start`); let closeStartPer = false; if(trackDur !== undefined) { const positionPercent = (position / trackDur); closeStartPer = (positionPercent <= percent); if(!closeStartNum) { hints.push(`${closeStartPer ? 'is' : 'is not'} within ${formatNumber(percent * 100, {toFixed: 0})}% of track start (${formatNumber(positionPercent*100)}%)`); } } return [closeStartNum || closeStartPer, `${hintStart}${hints.join(' and ')}`]; } /** Is Position closer than X seconds or Y% percent of the end of a Play? */ export const closeToPlayEnd = (play: PlayObject, position: number, thresholds: {absolute?: number, percent?: number, hintPrefix?: boolean} = {}): [boolean, string] => { const { absolute = DEFAULT_CLOSE_POSITION_ABSOLUTE, percent = DEFAULT_CLOSE_POSITION_PERCENT, hintPrefix = true } = thresholds; const hintStart = hintPrefix ? `Position (${position}) ` : ''; const trackDur = play.data.duration; if(trackDur === undefined) { return [false, `Cannot determine how close Position ${position} is to end of track because no duration data is available.`]; } const nearEndNum = trackDur - position <= absolute; const hints: string[] = []; hints.push(`${nearEndNum ? 'is' : 'is not'} within ${absolute}s of track end`); const positionPercent = 1 - (position / trackDur); const nearEndPer = (positionPercent < percent); if(!nearEndNum) { hints.push(`${nearEndPer ? 'is' : 'is not'} within ${formatNumber(percent * 100, {toFixed: 0})}% of track end (${formatNumber(positionPercent*100)}%)`); } return [nearEndNum || nearEndPer, `${hintStart}${hints.join(' and ')}`]; } /** Has more than X seconds or Y% percent of Play duration been played? */ export const repeatDurationPlayed = (play: PlayObject, duration: number, thresholds: {absolute?: number, percent?: number, hintPrefix?: boolean} = {}): [boolean, string] => { const { absolute = DEFAULT_DURATION_REPEAT_ABSOLUTE, percent = DEFAULT_DURATION_REPEAT_PERCENT, hintPrefix = true } = thresholds; const hintStart = hintPrefix ? `Duration listened (${duration}s) ` : ''; const trackDur = play.data.duration; const absPlayed = duration >= absolute; const hints: string[] = []; hints.push(`${absPlayed ? 'is' : 'is not'} more than ${absolute}s`); let majorityDurationPercent = false; if(trackDur !== undefined) { const durationPercent = (duration / trackDur); majorityDurationPercent = (durationPercent >= percent); if(!absPlayed) { hints.push(`${majorityDurationPercent ? 'is' : 'is not'} more than ${formatNumber(percent * 100, {toFixed: 0})}% of track duration (${formatNumber(durationPercent*100)}%)`); } } return [absPlayed || majorityDurationPercent, `${hintStart}${hints.join(' and ')}`]; } /** Convert unix timestamp in microseconds to unix timestamp in seconds */ export const usecToUnix = (usec: number): UnixTimestamp => { return Math.floor(usec / 1000); } // string must only contain ISO8601 optionally wrapped by whitespace const ISO8601_REGEX: RegExp = /^\s*((-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?)\s*$/; // finds ISO8601 in any part of a string const ISO8601_SUBSTRING_REGEX: RegExp = /((-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?)/g; // string must only duration optionally wrapped by whitespace const DURATION_REGEX: RegExp = /^\s*(?