diff --git a/src/backend/common/database/drizzle/entityUtils.ts b/src/backend/common/database/drizzle/entityUtils.ts index aa2815ec..70a5d84a 100644 --- a/src/backend/common/database/drizzle/entityUtils.ts +++ b/src/backend/common/database/drizzle/entityUtils.ts @@ -19,7 +19,7 @@ export const generateComponentEntity = (data: MarkOptional) }; } -export type PlayEntityOpts = Partial> & { error?: ErrorLike }; +export type PlayEntityOpts = Partial> & { error?: ErrorLike }; export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}): PlayNew => { const { diff --git a/src/backend/common/database/drizzle/repositories/PlayRepository.ts b/src/backend/common/database/drizzle/repositories/PlayRepository.ts index e901d5f3..89282422 100644 --- a/src/backend/common/database/drizzle/repositories/PlayRepository.ts +++ b/src/backend/common/database/drizzle/repositories/PlayRepository.ts @@ -30,7 +30,6 @@ export interface PlayWhereOpts { state?: PlaySelect['state'][] stateNot?: PlaySelect['state'][] componentId?: number - platformId?: string seenAt?: CompareDateOp playedAt?: CompareDateOp queues?: QueueCriteria[] @@ -380,22 +379,6 @@ export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> { loggerCom.verbose(`Cleanup done! Summary:\n${summaryDelStates.join(' | ')}`); } - public selectRecentDistinctPlatforms = async (limitPlays: number = 500, opts: ComponentConstrainedRepoOpts = {}): Promise => { - const { - componentId = this.componentId - } = opts; - - const recentPlatformIds = await this.db.selectDistinct({platformId: plays.platformId}).from(plays) - .where( - and( - eq(plays.componentId, componentId), - ne(plays.state, 'queued')) - ) - .orderBy(desc(plays.playedAt)) - .limit(limitPlays); - return recentPlatformIds.map(x => x.platformId); - } - protected prepareGetQueueNext = () => this.db.query.plays.findFirst({ where: { componentId: sql.placeholder('componentId'), @@ -732,9 +715,6 @@ export const buildPlayWhere = (args: PlayWhereOpts): FindWhere<'plays'> => { if (args.playedAt !== undefined) { where.playedAt = buildDateCompare(args.playedAt); } - if(args.platformId !== undefined) { - where.platformId = args.platformId - } if(args.uid !== undefined) { where.uid = { in: args.uid @@ -785,7 +765,6 @@ export const playToRepositoryCreatePlayOpts = (data: MarkOptional, string>>; +export type RequestPlayQuery = Partial< Record, string>>; export const queryArgsFromRequest = (rec: RequestPlayQuery): QueryPlaysOpts => { diff --git a/src/backend/common/database/drizzle/schema/schema.ts b/src/backend/common/database/drizzle/schema/schema.ts index 1eb77def..25f3010d 100644 --- a/src/backend/common/database/drizzle/schema/schema.ts +++ b/src/backend/common/database/drizzle/schema/schema.ts @@ -61,7 +61,6 @@ export const plays = sqliteTable("plays", { state: text({enum: ['queued','discovered','discarded','scrobbled','failed','duped']}).notNull(), // https://orm.drizzle.team/docs/indexes-constraints#foreign-key parentId: integer().references((): AnySQLiteColumn => plays.id, {onDelete: 'set null', onUpdate: 'cascade'}), - platformId: text(), playHash: text(), mbidIdentifier: text(), compacted: text() @@ -70,8 +69,7 @@ export const plays = sqliteTable("plays", { index("play_component_id_idx").on(table.componentId), uniqueIndex("play_uid_idx").on(table.uid), index("play_playedAt_idx").on(table.playedAt), - index("play_seenAt_idx").on(table.seenAt), - index("play_platform_idx").on(table.platformId) + index("play_seenAt_idx").on(table.seenAt) ]); export const playInputs = sqliteTable("play_inputs", { diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index b080d0c2..17dc2409 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -214,19 +214,14 @@ export default abstract class AbstractSource extends AbstractComponent implement recentlyPlayedTrackIsValid = (playObj: PlayObject) => true protected addPlayToDiscovered = async (play: PlayObject): Promise => { - const platformId = this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID; - const playRow = await this.playRepo.createPlays([(playToRepositoryCreatePlayOpts({play, state: 'discovered', platformId: genGroupIdStr(platformId)}))]); - const recentPlays = await this.getRecentlyDiscoveredPlaysByPlatform(platformId, false); + const playRow = await this.playRepo.createPlays([(playToRepositoryCreatePlayOpts({play, state: 'discovered'}))]); + const recentPlays = await this.getRecentlyDiscoveredPlays(false); // only need to update if its already in memory, // and better to update in-memory than clear cache so we aren't refetching from db on every discover if(recentPlays !== undefined) { recentPlays.push(play); recentPlays.sort(sortByOldestPlayDate); - this.cache.cacheDb.set(this.recentDiscoveredCacheKey(platformId), recentPlays, '2m'); - } - const platformIds = await this.getRecentPlatformIds(false); - if(platformIds !== undefined && !platformIds.includes(genGroupIdStr(platformId))) { - this.cache.cacheDb.set(this.recentPlatformsCacheKey(), platformIds, '10m'); + this.cache.cacheDb.set(this.recentDiscoveredCacheKey(), recentPlays, '2m'); } this.tracksDiscovered++; this.logger.info(`Discovered => ${buildTrackString(play)}`); @@ -238,13 +233,8 @@ export default abstract class AbstractSource extends AbstractComponent implement } getFlatRecentlyDiscoveredPlays = async (): Promise => { - const platforms = await this.getRecentPlatformIds(); - const list: PlayObject[][] = []; - for(const platformId of platforms) { - list.push(await this.getRecentlyDiscoveredPlaysByPlatform(platformId)); - } - return list.flat().sort(sortByNewestPlayDate); - //Array.from(this.recentDiscoveredPlays.values()).map(x => x.data).flat(3).sort(sortByNewestPlayDate) + const list: PlayObject[] = await this.getRecentlyDiscoveredPlays(); + return list.sort(sortByNewestPlayDate); } getRecentPlaysApi = async (query: RequestPlayQuery) => { @@ -258,23 +248,15 @@ export default abstract class AbstractSource extends AbstractComponent implement }) } - protected recentDiscoveredCacheKey = (platformId: PlayPlatformId | string) => { - const platformStr = typeof platformId === 'string' ? platformId : genGroupIdStr(platformId); - return `recent-${this.dbComponent.id}-${platformStr}`; - } - protected recentPlatformsCacheKey = () => { - return `recentPlatformIds-${this.dbComponent.id}`; + protected recentDiscoveredCacheKey = () => { + return `recent-${this.dbComponent.id}`; } - getRecentlyDiscoveredPlaysByPlatform = async (platformId: PlayPlatformId | string, hydrate: boolean = true): Promise => { - - const platformStr = typeof platformId === 'string' ? platformId : genGroupIdStr(platformId); - const cacheKey = this.recentDiscoveredCacheKey(platformId); - + getRecentlyDiscoveredPlays = async (hydrate: boolean = true): Promise => { + const cacheKey = this.recentDiscoveredCacheKey(); let list = await this.cache.cacheDb.get(cacheKey); if(list === undefined && hydrate) { list = (await this.playRepo.findPlays({ - platformId: platformStr, stateNot: ['queued'], order: 'desc', sort: 'playedAt', @@ -286,49 +268,16 @@ export default abstract class AbstractSource extends AbstractComponent implement return list; } - protected getRecentPlatformIds = async (hydrate: boolean = true) => { - const cacheKey = this.recentPlatformsCacheKey(); - let list = await this.cache.cacheDb.get(cacheKey); - if(list === undefined && hydrate) { - list = await this.playRepo.selectRecentDistinctPlatforms(); - await this.cache.cacheDb.set(cacheKey, list, '10m'); - } - return list; - } - - protected getExistingDiscoveredLists = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise => { - const lists: PlayObject[][] = []; - if(opts.checkAll !== true) { - lists.push(await this.getRecentlyDiscoveredPlaysByPlatform(this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID)); - } else { - const platforms = await this.getRecentPlatformIds(); - // get as many as we can, optionally filtering by user - for(const platformId of platforms) { - if(play.meta.user !== undefined) { - if(platformId[1] === NO_USER || platformId[1] === play.meta.user) { - lists.push(await this.getRecentlyDiscoveredPlaysByPlatform(platformId)); - } - } else { - lists.push(await this.getRecentlyDiscoveredPlaysByPlatform(platformId)); - } - } - } - return lists; - } - existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise => { - const lists: PlayObject[][] = await this.getExistingDiscoveredLists(play, opts); + const list: PlayObject[] = await this.getRecentlyDiscoveredPlays(); const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate); - for(const list of lists) { - - const existing = await findAsync(list,async x => { + const existing = await findAsync(list,async x => { const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); return genericSourcePlayMatch(e, candidate); }); if(existing) { return existing; } - } return undefined; } diff --git a/src/backend/sources/DeezerInternalSource.ts b/src/backend/sources/DeezerInternalSource.ts index 17d82236..9359030d 100644 --- a/src/backend/sources/DeezerInternalSource.ts +++ b/src/backend/sources/DeezerInternalSource.ts @@ -330,45 +330,43 @@ export default class DeezerInternalSource extends MemorySource { existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise => { - const lists: PlayObject[][] = await this.getExistingDiscoveredLists(play, opts); + const list: PlayObject[] = await this.getRecentlyDiscoveredPlays(); const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate); - for(const list of lists) { - const existing = await findAsync(list, async x => { + const existing = await findAsync(list, async x => { + const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); + return genericSourcePlayMatch(e, candidate); + }); + if(existing) { + return existing; + } + if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { + const fuzzyIndex = await findIndexAsync(list, async x => { const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); - return genericSourcePlayMatch(e, candidate); - }); - if(existing) { - return existing; - } - if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { - const fuzzyIndex = await findIndexAsync(list, async x => { - const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); - let temporalOptions: TemporalPlayComparisonOptions = {}; - const temporalAccuracy: TemporalAccuracy[] = [TA_EXACT, TA_CLOSE, TA_FUZZY]; - if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { - temporalOptions = { - fuzzyDiffThreshold: Math.max(100, x.data.duration * 0.5), - duringReferences: ['duration', 'listenedFor', 'range'] - } - temporalAccuracy.push(TA_DURING); - } - return genericSourcePlayMatch(e, candidate, temporalAccuracy, temporalOptions); - }); - if(fuzzyIndex !== -1) { - if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { - // always return fuzzy match as existing - // likely will make MS miss scrobbles for repeated plays - return list[fuzzyIndex]; - } - if(fuzzyIndex + 1 === list.length || playObjDataMatch(list[fuzzyIndex], list[fuzzyIndex + 1])) { - // last discovered play was this one, or next played play was also this one - // so we'll assume this means the play is on repeat, don't count as existing - return undefined; + let temporalOptions: TemporalPlayComparisonOptions = {}; + const temporalAccuracy: TemporalAccuracy[] = [TA_EXACT, TA_CLOSE, TA_FUZZY]; + if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { + temporalOptions = { + fuzzyDiffThreshold: Math.max(100, x.data.duration * 0.5), + duringReferences: ['duration', 'listenedFor', 'range'] } - // next played play was *not* this one (Deezer reports play between candidate TS and fuzzy match) - // so this is likely a duplicate deezer should not have reported + temporalAccuracy.push(TA_DURING); + } + return genericSourcePlayMatch(e, candidate, temporalAccuracy, temporalOptions); + }); + if(fuzzyIndex !== -1) { + if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { + // always return fuzzy match as existing + // likely will make MS miss scrobbles for repeated plays return list[fuzzyIndex]; } + if(fuzzyIndex + 1 === list.length || playObjDataMatch(list[fuzzyIndex], list[fuzzyIndex + 1])) { + // last discovered play was this one, or next played play was also this one + // so we'll assume this means the play is on repeat, don't count as existing + return undefined; + } + // next played play was *not* this one (Deezer reports play between candidate TS and fuzzy match) + // so this is likely a duplicate deezer should not have reported + return list[fuzzyIndex]; } } return undefined; diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts index 6f2dfcc5..02b814a6 100644 --- a/src/backend/sources/MemorySource.ts +++ b/src/backend/sources/MemorySource.ts @@ -368,7 +368,7 @@ export default class MemorySource extends AbstractSource { } return [false, `${stPrefix} ${EXPECTED_NON_DISCOVERED_REASON}`] } else { - const discoveredPlays = await this.getRecentlyDiscoveredPlaysByPlatform(genGroupId(candidate)); + const discoveredPlays = await this.getRecentlyDiscoveredPlays(); if (discoveredPlays.length === 0 || !playObjDataMatch(discoveredPlays[0], candidate)) { // if most recent stateful play is not this track we'll add it return [true,`${stPrefix} added after ${thresholdResultSummary(thresholdResults)}. Matched other recent play but could not determine time frame due to missing duration. Allowed due to not being last played track.`];